def startDownload(driver):
driver.get(Config.Base_Url + Config.Target_Name + Config.Category)
Debug.log('fbPA:' + Config.Base_Url + Config.Target_Name + Config.Category)
Debug.sc_shot(driver, '2-Albums Index.png')
html_source = (driver.page_source).encode('utf-8')
getAllAlbumsInfo(driver, html_source)
startDownloadAlbumsPhoto(driver)
def calculate(self, cur_run_file):
'''determine the value of each run'''
# Get data to compare
self._cost = self._getData(cur_run_file)
# Apply the weight to the cost function
self._cost *= self.__weight
Debug.log('cost: {0}'.format(self._cost), False)
return self._cost
def getRunNameInDirForMutable(self, run_dir, mutable, slope=''):
gradient = str(mutable.name)
if(slope):
gradient += str(slope)
run_name = self.run_name_in_dir.format(
run_dir=run_dir, this=self, gradient=gradient)
Debug.log('run_name: ' + run_name)
return run_name
def startDownload(driver):
driver.get( Config.Base_Url + Config.Target_Name + Config.Category)
Debug.log('fbPA:'+ Config.Base_Url + Config.Target_Name + Config.Category)
Debug.sc_shot(driver,'2-Albums Index.png')
html_source = (driver.page_source).encode('utf-8')
getAllAlbumsInfo(driver, html_source)
startDownloadAlbumsPhoto(driver)
def calculateRelative(self, cur_run_file):
'''determine the value of each run'''
# Get data to compare
cur_info = self._getData(cur_run_file)
tgt_info = self._getData(self._tgt_run)
# Calculate difference
self._cost = tgt_info - cur_info # abs()?
# Apply the weight to the cost function
self._cost *= self.__weight
Debug.log('cost: {0} = {1} - {2}'.format(
self._cost,
tgt_info,
cur_info,
), False)
return self._cost
def run(self):
"""
retrieve geo location list and notify parent thread
"""
if is_android:
try:
addrs = get_geo_location(self.address, 10)
except Exception as err:
Debug.log("GEOTEST", error=err.__str__())
addrs = []
finally:
self.callback(addrs)
else:
addrs = Debug.get_geo_address(self.address, 10)
self.callback(addrs)
def buren(self, cel):
Debug.log_with_action(cel.position_string(), 'Cel + buren')
neighbours = []
c = self.buur_links(cel)
if c is not None:
neighbours.append(c)
c = self.buur_rechts(cel)
if c is not None:
neighbours.append(c)
c = self.buur_boven(cel)
if c is not None:
neighbours.append(c)
c = self.buur_onder(cel)
if c is not None:
neighbours.append(c)
for i in neighbours:
Debug.log(i.position_string())
return neighbours
def neighbours(self, cell):
Debug.log_with_action(cell.position_string(), 'Cell + neighbours')
neighbours = []
c = self.neighbour_left(cell)
if c is not None:
neighbours.append(c)
c = self.neighbour_right(cell)
if c is not None:
neighbours.append(c)
c = self.neighbour_up(cell)
if c is not None:
neighbours.append(c)
c = self.neighbour_down(cell)
if c is not None:
neighbours.append(c)
for i in neighbours:
Debug.log(i.position_string())
return neighbours
def download_image(filename, response, settings):
"""
download_image(str, str, object)
path should be the file path, filename should be the name of the file
os.path.join is used to append path to filename
response is the response returned from requests.get
"""
# read from socket
# store in memory
# images shouldnt be too large
byte_stream = BytesIO()
for buff in response.iter_content(1000):
byte_stream.write(buff)
# load image from buffer io
try:
image = Image.open(byte_stream)
except UnidentifiedImageError as err:
image = None
Debug.log("IMAGE_OPEN_ERROR",
err,
url=response.url,
error=err.__str__())
Debug.log_file("ImageOpenError", "download_image",
f"Error opening image from {response.url}")
if image:
width, height = image.size
# if image requirements met then save
if width > 200 and height > 200:
# check if directory exists
Threads.new_folder_lock.acquire()
if not os.path.exists(settings["save_path"]):
os.mkdir(settings["save_path"])
if settings["unique_pathname"]["enabled"]:
path = os.path.join(settings["save_path"],
settings["unique_pathname"]["name"])
if not os.path.exists(path):
os.mkdir(path)
else:
path = settings["save_path"]
Threads.new_folder_lock.release()
ImageFile.write_to_file(path, filename, byte_stream)
image.close()
byte_stream.close()
def _setupBFGSManager(self, costs):
# grab the first runset
init_run_batch = self._launcher.getLatestRunset()
Debug.log('init_run_batch')
for name in init_run_batch:
Debug.log(name)
# grab runs at k[0]
prime_run_dir = self._launcher.getDirNameFromRunName(init_run_batch[0])
prime_costs = \
dict((x, y) for x, y in costs.items() if prime_run_dir in x)
# grab runs at k[1]
first_run_dir = self._launcher.getDirNameFromRunName(
init_run_batch[-1])
first_costs = \
dict((x, y) for x, y in costs.items() if first_run_dir in x)
# init baseline using the data we've just gathered
self._bfgs_man.initBaselineData(prime_costs, first_costs)
self._previous_curve = self._bfgs_man.findSmallestCost(costs)
self._iterations = 0
def potential_next_moves(self, snake):
head = snake.snakeParts[0]
Debug.log_with_action(head.position_string(), 'Head + potential moves')
potential_cells = []
c = self.neighbour_left(head)
if c is not None:
potential_cells.append(PotentialSnakePart(c.x, c.y))
c = self.neighbour_right(head)
if c is not None:
potential_cells.append(PotentialSnakePart(c.x, c.y))
c = self.neighbour_up(head)
if c is not None:
potential_cells.append(PotentialSnakePart(c.x, c.y))
c = self.neighbour_down(head)
if c is not None:
potential_cells.append(PotentialSnakePart(c.x, c.y))
for i in potential_cells:
Debug.log(i.position_string())
return potential_cells
def is_valid_content_type(url, content_type, valid_types):
"""
is_valid_content_type(str, str, dict)
checks if mimetype is an image and matches valid images
url - the url of the content-type
content_type - is a string found in the headers['Content-Type'] dict
valid_types - a dict containing valid files for searching
returns an empty string if not valid or a file extension related to the file type
will always return a valid file extension if html document
"""
ext = ""
if 'text/html' in content_type:
ext = ".html"
elif content_type == 'image/gif' and valid_types["gif"]:
ext = ".gif"
elif content_type == 'image/png' and valid_types["png"]:
ext = ".png"
elif content_type == 'image/ico' and valid_types["ico"]:
ext = ".ico"
elif content_type == 'image/jpeg' and valid_types["jpg"]:
ext = ".jpg"
elif content_type == 'image/tiff' and valid_types["tiff"]:
ext = ".tiff"
elif content_type == 'image/tga' and valid_types["tga"]:
ext = ".tga"
elif content_type == 'image/bmp' and valid_types["bmp"]:
ext = ".bmp"
elif content_type == 'application/octet-stream':
# file attachemnt use the extension from the url
try:
ext = os.path.splitext(url)[1]
except IndexError as err:
Debug.log("is_valid_content_type web.py",
err,
error=err.__str__(),
url=url,
content_type=content_type)
return ext
def setTargetFile(self, tgt_run):
Debug.log(tgt_run)
self._tgt_run = tgt_run
def run(self, **kwargs):
# ==============================================================
# The basic strategy for this is to create a loop
# where we alternate calling super().run() and self._reInit().
# After each _reInit() we can update the start coordinates of
# our mutable group to achieve a new starting location before
# we call run() again.
# ==============================================================
# store our initial initial location
self._starting_locations = [self._default['mutables']]
# log time
start = time.time()
start_str = time.ctime()
Debug.log('random start : ' + str(start_str))
while (self._run_loops):
Debug.log('loops: ' + str(self._run_loops))
print('loops: ' + str(self._run_loops))
# Do a full run.
super(RandomWalker, self).run()
self._extrema_value.append(
(self._flattest_curve, self._bfgs_man.getMutables()))
# separate the runs in the debug log
Debug.log('\n')
Debug.log('/\n' * 10)
# reinit
self._reInitSuperOnly()
# update the starting locations for the next run
self._updateStartingLocation()
# decrement the loop counter
self._run_loops -= 1
# then back to the top
end = time.time()
Debug.log('random start : ' + str(start_str))
Debug.log('random end : ' + str(time.ctime()))
Debug.log('random elapsed: ' + str(end - start) + 's')
Debug.log('\nextrema found :\n')
for loc in sorted(self._extrema_value, key=lambda x: x[0][1]):
Debug.log(' Coord:\n' + str(loc[1]))
Debug.log(' Value: ' + str(loc[0]))
Debug.log('\n')
def run(self, epsilon=0.1):
# log start time
start = time.time()
start_str = time.ctime()
Debug.log('root start : ' + str(start_str))
# Set number of runs per step based on resolution percentage
# initialize the search slope and the bfgs matrix.
self._firstLaunchSet()
# prevent infinite loops durring testing.
test = 500
# search condition trigger
continue_searching = True
while (continue_searching):
self._iterations += 1
Debug.log('' + '*' * 50 + '')
# Launch info
self._launchSet()
# Calculate how far from target we are
costs = self._calculateCosts()
# find our next move
continue_searching = self.determineSteps(costs, epsilon)
if (test < 0):
Debug.log('Loop exit from test iteration limit.')
break
test -= 1
msg = 'Closest fit after {0._iterations}: {0._flattest_curve}'
Debug.log(msg.format(self))
Debug.log('Total run count: {0}'.format(self._total_run_count))
# log end/elapsed time
end = time.time()
Debug.log('root start : ' + str(start_str))
Debug.log('root end : ' + str(time.ctime()))
Debug.log('root elapsed: ' + str(end - start) + 's')
def determineSteps(self, costs, epsilon):
# our boolean return value:
# true : continue searching
# false: we've reached epsilon. stop looking.
continue_searching = True
self._flattest_curve = self._bfgs_man.findSmallestCost(costs)
msg = 'Flattest curve at {0._iterations}: {0._flattest_curve}'
Debug.log(msg.format(self))
# Detect if the first run was too steep. If run0 < run1 then
# we need to discard this run and rerun it from 0 -> 1 instead
# of from 0 -> 100.
# if(self._previous_curve[1] < self._flattest_curve[1]):
# msg = 'REFINING:: {0} < {1}\n'
# msg += 'REFINING:: Our gradient was too large.'
# Debug.log(msg.format(self._previous_curve[1],
# self._flattest_curve[1]))
# # self._bfgs_man.refineSearchToFirstStep(self._resolution)
# self.setRunResolution(self._resolution * 0.1)
# Debug.log('REFINING:: Increasing gradient by 10%')
# Debug.log('REFINING:: New Res {0}'.format(self._resolution))
# self._poor_debug_code += 1
# if(3 < self._poor_debug_code):
# exit()
# return continue_searching
# self.setRunResolution(0.01)
dir_k_next = self._launcher.getDirNameFromRunName(
self._flattest_curve[0])
# Archive the data and have a log of the path we took.
# self._launcher.archiveDir(dir_k_next, self._iterations)
# TODO: The following line was commented out for testing.
# Make sure to uncomment it.
step_k_next = self._launcher.getStepFromRunName(
self._flattest_curve[0])
# Trim info not at the step of closest fit.
for c in list(costs.keys())[:]:
if (dir_k_next not in c):
# Clean the expensive keys from our dictionary.
del costs[c]
# Update the states and gradients to use the latest calculations
self._bfgs_man.updateStatesAndGradients(costs, step_k_next,
self._resolution)
# determine if we're close enough to our result.
if (self._bfgs_man.determineExtrema(epsilon)):
Debug.log("ENDING RUN:: We've found a local minima.")
continue_searching = False
# if the same step is the flattest twice: then we're stuck
elif (self._flattest_curve[1] == self._previous_curve[1]):
Debug.log("ENDING RUN:: We're stuck...")
continue_searching = False
# TODO: https://github.com/DrewSimtech/optimizer/issues/5
else:
# Determine if we need to inject noise to escape a saddle
if (self._bfgs_man._gradient_norm == 0.0):
Debug.log("SADDLE POINT:: Beginging stocastic draws.")
self._bfgs_man.injectStocasticNoise()
pass
# Math functions are broken up and implemented below.
self._bfgs_man.updateBFGSandRHO()
self._bfgs_man.updateMutableValuesAndSteps()
self._previous_curve = self._flattest_curve
return continue_searching
