def on_line_received(self, json_payload):
Logger.debug('line received: %s', json_payload, 'server')
payload = json.loads(json_payload)
if payload and 'type' in payload and payload['type'] == 'user_input':
user_input = UserInputDataUnit(payload['content']).get_object()
self.game.apply_user_input_to_player(self.user_id, user_input)
self.update_game()
self.send_broadcast_payload_except_self({
'type':
'user_input',
'content':
payload['content'],
'user':
PlayerDataUnit(self.game.get_player(self.user_id)).get_pdu()
})
self.interval_executor.execute_if_interval_elapsed(
self, lambda self: self.send_broadcast_payload(
{
'type': 'game_state',
'content': GameDataUnit(self.game).get_pdu()
}))
else:
Logger.error('Unknown message type: (%s)', json_payload,
'server_protocol')
def on_line_received(self, payload):
if payload and 'type' in payload:
if payload['type'] == 'user_input':
Logger.trace(
"fetch input_queue from main, from network process to screen: (%s)",
payload, 'client_protocol')
user_input = UserInputDataUnit(payload['content']).get_object()
user_id = payload['user']['id']
self.game.apply_user_input_to_player(user_id, user_input)
elif payload['type'] == 'authentication':
Logger.info("Authentication (%s)", payload, 'client_protocol')
self.localPlayerId = payload['user']['id']
self.game.add_player(self.localPlayerId)
elif payload['type'] == 'game_state_initial':
Logger.info("Game state initial (%s)", payload,
'client_protocol')
for user_pdu in GameDataUnit(
payload['content']).get_players_pdu():
if not user_pdu.get_id() == self.localPlayerId:
player = self.game.add_player(user_pdu.get_id())
player.set_position(user_pdu.get_position())
elif payload['type'] == 'game_state':
Logger.info("Game state (%s)", payload, 'client_protocol')
for user_pdu in GameDataUnit(
payload['content']).get_players_pdu():
player = self.game.get_player(user_pdu.get_id())
player.set_position(user_pdu.get_position())
elif payload['type'] == 'new_connection':
Logger.info("User new connection (%s)", payload,
'client_protocol')
user_id = payload['user']['id']
self.game.add_player(user_id)
elif payload['type'] == 'lost_connection':
Logger.info("User lost connection (%s)", payload,
'client_protocol')
user_id = payload['id']
self.game.remove_player(user_id)
else:
Logger.error('Unknown payload type: (%s)', payload['type'],
'client_protocol')
else:
Logger.error('Payload not defined or "type" key not defined: %s',
payload, 'client_protocol')
def on_line_received(self, json_payload):
Logger.debug('line received: %s', json_payload, 'server')
payload = json.loads(json_payload)
if payload and 'type' in payload and payload['type'] == 'user_input':
user_input = UserInputDataUnit(payload['content']).get_object()
self.game.apply_user_input_to_player(self.user_id, user_input)
self.update_game()
self.send_broadcast_payload_except_self({
'type': 'user_input',
'content': payload['content'],
'user': PlayerDataUnit(self.game.get_player(self.user_id)).get_pdu()
})
self.interval_executor.execute_if_interval_elapsed(self, lambda self: self.send_broadcast_payload(
{'type': 'game_state', 'content': GameDataUnit(self.game).get_pdu()}))
else:
Logger.error('Unknown message type: (%s)', json_payload, 'server_protocol')
break
if start_x > end_x:
huidu -= step
else:
huidu += step
cv2.destroyAllWindows()
return ret, start_x, end_x
if __name__ == '__main__':
tt = Logger().get_log
#dpath = 'sshot/data/'
dpath = '../'
#files = os.listdir(dpath)
files = []
files.append('p20191113_173904.png')
for file_name in files:
print("xxxxxxxx:" + file_name)
ret, start_x, end_x = find_slot(dpath + file_name)
if (ret):
tt.error("%s final got start_x %u, end_x %u " %
(file_name, start_x, end_x))
else:
tt.error("%s not found", file_name)
break
class KivaRunner:
"""
This class can run kiva with a certain set of parameters
and different temperture inputs.
"""
def __init__(self, working_dir, log_dir, compare_file, parameter_format, logger=None):
"""
parameter_format specifies what parameters from which files are needed.
[file, name, min, max, step]
"""
if logger == None:
self.log = Logger()
else:
self.log = logger
self.error = False
self.working_dir = working_dir
self.log_dir = log_dir
self.kiva_path = '../ext'
self.kiva_name = 'kiva_0D'
self.kiva_parameter_files = ['itape17', 'itape5', 'itapeERC', 'itapeRs']
self.parameter_format = parameter_format
self.log.debug("Parameter Format: %s" % self.parameter_format)
# this will later hold all kiva_parameter_files that
# need to be loaded as ITapeFile objects
self.itapes = {}
self.compare_file = compare_file
# Check if working directory exists and that it is empty
self.working_dir_indicator = os.path.join(self.working_dir, 'kivagen.working.dir')
if not os.path.isdir(self.working_dir):
self.log.error("'%s' does not exist." % self.working_dir)
self.error = True
self.working_dir = None
elif os.listdir(self.working_dir): # not empty
if not os.path.isfile(self.working_dir_indicator):
self.log.error("'%s' does not seem to be a working directory." % self.working_dir)
self.error = True
self.working_dir = None
return
else:
open(self.working_dir_indicator, 'a').close()
# Check that log dir exists
if not os.path.isdir(self.log_dir):
self.log.error("'%s' does not exist." % self.log_dir)
self.error = True
self.log_dir = None
# Check if input files exist
input_files = [self.compare_file]
input_files.append(os.path.join(self.kiva_path, self.kiva_name))
for parameter_file in self.kiva_parameter_files:
input_files.append(os.path.join(self.kiva_path, parameter_file))
for input_file in input_files:
if not os.path.isfile(input_file):
self.log.error("'%s' not found." % input_file)
self.error = True
# Parse compare file
self.compare_values = self._readCompareFile(self.compare_file)
# self.log.debug("self.compare_values = %s" % self.compare_values)
# Load Parameters
self._loadParameters()
# self.log.debug("self.param_list = %s" % self.param_list)
# --------------- Parameters --------------------------------------------------
def _loadParameters(self):
"""
Loads the start parameters from the itape files.
Also creates a lookup table to match parameter list index to
itape file and value id
"""
self.param_list = []
self.param_lookup = {}
# relaod parameters from file
for itape in self.itapes.values():
itape._open(itape.name)
param_index = 0
for param in self.parameter_format:
# if file has not been loaded before
if param[0] not in self.itapes:
f = os.path.join(self.kiva_path, param[0])
if param[0] in self.kiva_parameter_files:
if param[0] in ['itapeERC', 'itapeRs']:
self.itapes[param[0]] = ITapeFile(f, self.log)
else:
self.log.error("Failed to read %s. Can only read itapeERC and itapeRs." % f)
else:
self.log.error("Failed to read %s. Not in self.kiva_parameter_files (%s)" % self.kiva_parameter_files)
# try to get parameter id
ii = self.itapes[param[0]].getId(param[1])
if ii < 0:
self.log.error("Could not find '%s' in '%s'." % (param[1], param[0]))
self.param_list.append(None) # append None to get index right
else:
value = self.itapes[param[0]].getValue(ii)
self.param_list.append(self._checkParameter(value, param))
# remember itap file and value id
self.param_lookup[param_index] = [self.itapes[param[0]], ii]
param_index = param_index + 1
# now we should have collected all parameters
if len(self.param_list) != len(self.parameter_format):
sel.log.error("Only found %s elements. Expected %s!"
% (self.param_list, len(self.parameter_format)))
def getParameters(self):
"""
Returns parameter list.
"""
return self.param_list
def setParameters(self, parameters):
"""
Hand a new set of parameters to this class.
They will be used for the next kiva runs.
"""
# 1.) check if parameter list length is correct
if len(parameters) != len(self.parameter_format):
sel.log.error("Parameter list needs to contain exactly %s elements. Not %s!"
% (len(self.parameter_format), len(parameters)))
return
# 2.) Check if parameters are correct and put them into the itape files
for ii in range(0, len(parameters)):
if self._checkParameter(parameters[ii], self.parameter_format[ii]) == None:
return
self.param_lookup[ii][0].setValue(self.param_lookup[ii][1], parameters[ii])
# 3.) Replace parameter list
self.param_list = parameters
def _checkParameter(self, value, format_line):
"""
Returns None if parameter is out of range.
Else retruns value.
"""
if value < format_line[2]:
self.log.error("%s < %s (min)." % (value, format_line[2]))
return None
if value > format_line[3]:
self.log.error("%s > %s (max)." % (value, format_line[3]))
return None
return value
# --------------- CompareFile -------------------------------------------------
def _readCompareFile(self, compare_file):
"""
Read the compare file and returns temperature and delay as lists.
"""
compare_values = []
with open(compare_file, 'rb') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
for row in csvreader:
try:
temperature = 1000.0 / float(row[0])
temperature = round(temperature, 0) # we do not need decimal places
time = float(row[1])
compare_values.append([temperature, time])
except ValueError:
pass
return compare_values
# --------------- Setup and run -----------------------------------------------
def _setupWorkingDir(self):
"""
Prepare working directory for a kiva run.
"""
if self.working_dir == None:
self.log.error("No valid working dir set.")
return False
# Delete contents
for node in os.listdir(self.working_dir):
if os.path.isdir(os.path.join(self.working_dir, node)):
shutil.rmtree(os.path.join(self.working_dir, node))
else:
os.unlink(os.path.join(self.working_dir, node))
# Recreate working dir indicator
open(self.working_dir_indicator, 'a').close()
# Create Directory containing all input files
path = os.path.join(self.working_dir, 'run0')
os.makedirs(path)
for f in self.kiva_parameter_files:
if f in self.itapes:
dst = os.path.join(path, os.path.basename(self.itapes[f].name))
self.itapes[f].save(dst)
else:
src = os.path.join(self.kiva_path, f)
dst = os.path.join(path, f)
shutil.copy(src, dst)
# Create as many copies of it as there are compare_values
for ii in range(1, len(self.compare_values)):
shutil.copytree(path, os.path.join(self.working_dir, 'run' + str(ii)))
# patch temperature in every directory's itape5 file line 230
for ii in range(0, len(self.compare_values)):
f = os.path.join(self.working_dir, 'run' + str(ii), 'itape5')
if not os.path.isfile(f):
self.log.error("'%s' does not exist." % f)
return False
# load file
with open(f, 'r') as file:
data = file.readlines()
# change line
line = "'tempi', "
line += '{0:.1f}'.format(self.compare_values[ii][0]) + '\n'
data[230 - 1] = line
# write data back
with open(f, 'w') as file:
file.writelines( data )
return True
def _collectIgnitionDelay(self):
"""
This needs to be called after kiva is run.
Data from the runXX/T_ign.dat files is collected.
"""
self.results = []
for ii in range(0, len(self.compare_values)):
f = os.path.join(self.working_dir, 'run' + str(ii), 'T_ign.dat')
if not os.path.isfile(f):
self.log.error("'%s' does not exist." % f)
delay = 0 # probably kiva was killed
# save log
log = os.path.join(self.log_dir, str(int(time.time())))
if not os.path.isdir(log):
os.makedirs(log)
shutil.move(os.path.join(self.working_dir, 'run' + str(ii)), log)
src = os.path.join(self.working_dir, 'run' + str(ii) + '.log')
des = os.path.join(log, 'run' + str(ii) + '.log')
os.rename(src, des)
src = os.path.join(self.working_dir, 'run' + str(ii) + '.error')
des = os.path.join(log, 'run' + str(ii) + '.error')
os.rename(src, des)
else:
with open(f, 'r') as file:
delay = float(file.readlines()[0].strip())
compare = self.compare_values[ii]
delta = delay - compare[1]
delta_abs = abs(delta)
delta_rel = round((delta_abs * 100 / compare[1]), 2)
self.log.debug("Delay for Temperatur %s: \texpected: %s \tcomputed: %s" % (compare[0], compare[1], delay))
self.log.debug("Delta: %s \tabs: %s \trelative: %s%%" % (delta, delta_abs, delta_rel))
self.results.append([delay, compare[1]])
def _restart(self, p, max_tries):
"""
Restarts process if number of max_tries has not been passed.
"""
if p[5] >= max_tries:
self.log.error("kiva in '%s' failed on the %s. try." % (p[1], p[5]))
p[2] = False
p[3].close() # close log file
p[4].close() # close error file
return False
else:
# clean up runXX dir
for node in os.listdir(p[1]):
if node not in self.kiva_parameter_files:
os.unlink(os.path.join(p[1], node))
# increment ties
p[5] = p[5] + 1
# print message to log
msg = "\n%s. Try -------------------------------------------\n" % p[5]
p[3].write(msg)
p[4].write(msg)
# restart p
kiva = os.path.abspath(os.path.join(self.kiva_path, self.kiva_name))
p[0] = subprocess.Popen(kiva, cwd=p[1], stdout=p[3], stderr=p[4])
self.log.warn("kiva in '%s' restarted for it's %s. try." % (p[1], p[5]))
return True
def run(self, time_out=120, max_tries=5):
"""
Executes kiva with current parameter set.
Once for every compare_value.
"""
if not self._setupWorkingDir():
return False
# Start one kiva process for every folder
processes = []
# devnull = open('/dev/null', 'w')
kiva = os.path.abspath(os.path.join(self.kiva_path, self.kiva_name))
for ii in range(0, len(self.compare_values)):
log = os.path.join(self.working_dir, 'run' + str(ii) + '.log')
log_file = open(log, 'w')
error = os.path.join(self.working_dir, 'run' + str(ii) + '.error')
error_file = open(error, 'w')
d = os.path.join(self.working_dir, 'run' + str(ii))
p = subprocess.Popen(kiva, cwd=d, stdout=log_file, stderr=error_file)
# process, path, running, log_file, error_file, tries
processes.append([p, d, True, log_file, error_file, 1])
self.log.debug("kiva in '%s' spawned." % d)
# Check for all kiva processes to terminate
all_finished = False
start_time = time.time()
while not all_finished:
all_finished = True
for p in processes:
if p[2]:
if p[0].poll() == 0:
self.log.debug("kiva in '%s' terminated." % p[1])
# Check if output file was generated
if os.path.isfile(os.path.join(p[1], 'T_ign.dat')):
p[2] = False
p[3].close() # close log file
p[4].close() # close error file
else: # if there is no T_ign.dat
self._restart(p, max_tries) # restart
start_time = time.time() # reset timeout
else:
all_finished = False
# is time up?
if time.time() - start_time > time_out:
self.log.warn("Timed out! (%s s)" % time_out)
for p in processes:
if p[2]:
p[0].kill()
p[3].close() # close log file
p[4].close() # close error file
self.log.warn("kiva in '%s' killed!" % p[1])
all_finished = True
# do not use up all CPU time
time.sleep(0.5)
# Collect Results
self._collectIgnitionDelay()
return True
def getFitness(self):
"""
Call this after calling run.
This will return a fitness value based on the temperatures computed.
"""
score = 0.0
for value in self.results:
score += abs(value[0] - value[1])
return score
class ImageHandler():
def __init__(self):
self._logger = Logger()
def save_image(self, image, save_dir_name, filename=None):
try:
if image is None or image.filename == '':
raise (FileNotSuppliedException(
'No file name found for the image. (File name was blank)'))
if self._allowed_file(image.filename):
image_name = self._append_date_to_filename(
secure_filename(image.filename))
if filename is not None:
image_name = secure_filename(filename + '.' +
image.filename.split('.')[-1])
if not os.path.exists(save_dir_name):
os.makedirs(save_dir_name, mode=0o770)
image.save(os.path.join(save_dir_name, image_name))
return image_name
except FileNotSuppliedException as e:
self._logger.error("save_image function encountered an error... " +
str(e.value))
except Exception:
self._logger.error("Invalid file, could not save.")
return None
def _append_date_to_filename(self, filename):
cur_time = datetime.datetime.utcnow()
extention = filename.split('.')[1]
temp_filename = filename[0:(len(filename) - len(extention) - 1)]
temp_filename += "_" + str(cur_time.day) + str(cur_time.month) + \
str(cur_time.year) + str(cur_time.hour) + str(cur_time.minute) + \
str(cur_time.second) + str(cur_time.microsecond) + '.' + extention
return temp_filename
def _allowed_file(self, filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in current_app.config['ALLOWED_EXTENSIONS']
def _resize_image_to_thumbnail(self,
path,
new_size=(64, 64),
save_path="",
new_file_name=""):
full_dir = os.path.split(path)
directory = full_dir[0]
existing_file_name = full_dir[1].split('.')[0]
if save_path == "":
save_path = directory
new_file = secure_filename(existing_file_name + "_thumbnail" + ".png")
if new_file_name != "":
new_file = secure_filename(new_file_name + "_thumbnail" + ".png")
try:
image = Image.open(path)
if new_size[0] == new_size[1]:
image = self._crop_to_square(image)
image.thumbnail(new_size)
new_path = os.path.join(save_path, new_file)
image.save(new_path, "png")
return new_file
except IOError as e:
self._logger.error(e)
self._logger.error('Unable to resize image ' +
str(existing_file_name) + ' ...')
return ""
def _get_image_size(self, pil_image):
return pil_image.size
def _crop_to_square(self, pil_image):
ls = self._largest_square(self._get_image_size(pil_image))
return pil_image.crop(ls)
def _largest_square_size(self, image_size):
s = image_size[0]
# Find the lesser of the two sides
if image_size[0] > image_size[1]:
s = image_size[1]
return s
def _largest_square(self, image_size):
sq_side_len = self._largest_square_size(image_size)
# Just need one point to set the square's origin
lp = (image_size[0] - sq_side_len) / 2
up = (image_size[1] - sq_side_len) / 2
# (left, upper, right, lowwer) as per pillow docs
return (lp, up, lp + sq_side_len, up + sq_side_len)
password = sys.argv[2]
request_cookies = get_logged_in_cookies(userid, password)
logger.info(
message="Consuming messages from sqs queue: {}".format(queue_url),
bucket=REAUTER_SCRAPER.recommendation_estimates,
stage='sqs_consumer')
signal_handler = SignalHandler()
while not signal_handler.received_signal:
res = sqs.consume_messages(queue_url)
if 'Messages' in res.keys():
for message in res['Messages']:
value = message['Body']
try:
fetch_and_save(request_cookies, ast.literal_eval(value))
except Exception as e:
error = {}
error['ExceptionMessage'] = str(e)
error['trace'] = traceback.format_exc().splitlines()
logger.error(
message="Exception occured for data {}".format(
value, str(value)),
bucket=REAUTER_SCRAPER.recommendation_estimates,
stage='dynamo_save',
extra_message=str(error))
raise
receipt_handle = message['ReceiptHandle']
# Delete received message from queue
sqs.delete_message(queue_url, receipt_handle)
if debug:
mod_strategy = __import__('strategy.' + strategy_name, globals(), locals(),
['run'], -1)
mod_dbase = __import__('storage.' + storage_type + '.crud', globals(),
locals(), ['Storage'], -1)
dbase = mod_dbase.Crud()
storage = mod_storage.Storage(dbase, session_id)
strategy = mod_strategy.Strategy(capi, logger, storage, conf)
else:
try:
mod_strategy = __import__('strategy.' + strategy_name, globals(),
locals(), ['run'], -1)
mod_dbase = __import__('storage.' + storage_type + '.crud', globals(),
locals(), ['Storage'], -1)
dbase = mod_dbase.Crud()
storage = mod_storage.Storage(dbase, session_id)
strategy = mod_strategy.Strategy(capi, logger, storage, conf)
except Exception, e:
print 'Startup Error: %s' % e
logger.error('Startup Error: %s' % e)
exit(1)
if debug:
strategy.run()
else:
try:
strategy.run()
except Exception, ex:
print 'Strategy.run: %s %s' % (ex, ex.message)
logger.error('Strategy.run: %s %s' % (ex, ex.message))
from logger.logger import Logger
import time
if __name__ == '__main__':
tt = Logger().get_log
tt.error("sdaaada3")
tt.debug("sdetsdebugtest1")
for i in range(1,20):
tt.debug("asda %u" %(i))
time.sleep(5.0)