def clean_buff(self):
for loop in range(1, 3):
exp_index = self.ssh_handle.expect(
[self.str_prompt, pexpect.TIMEOUT, pexpect.EOF], 0.01)
if 2 == exp_index:
log.log_info('expect EOF. what happened?', 'ERROR')
self.ssh_handle.close(force=True)
def get_input(submission_id):
'''从数据库获取input并写入data目录下对应的文件'''
sql= 'SELECT input FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
data = db.run_sql(sql)
input_data = data[0][0]
file_name = str(submission_id) + '.in'
try:
data_path = os.path.join(config.data_dir, str(submission_id))
secure.low_level()
os.mkdir(data_path)
except OSError as e:
if str(e).find('exist') > 0: # 文件夹已经存在
pass
else:
return False
try:
real_path = os.path.join(config.data_dir, str(submission_id), file_name)
except KeyError as e:
return False
try:
log.log_info('Importing input, %d' % submission_id)
secure.low_level()
f = open(real_path, 'w')
try:
f.write(input_data)
except:
f.close()
log.log_error('Importing input failed, %d' % submission_id)
return False
f.close()
except OSError as e:
return False
return True
def end_move(self, vector, invert_x_axis, invert_y_axis):
""" Given input parameters, moves the robot relative to it's current
position. It executes first the x component and then the y component.
Keyword Arguments:
self -- actuator object the function was called on
vector -- movement vector
invert_x_axis -- boolean of whether to invert on the x-axis
invert_y_axis -- boolean of whether to invert on the y-axis
"""
if (self.__in_x_movement and vector[0] != 0):
self.__in_x_movement = False
self.__x_direction = 0
log.log_info("stop x")
#X Axis
self.__issue_command(self.__x_device, 23, 0, 0, 0, 0)
if (self.__in_y_movement and vector[1] != 0):
self.__in_y_movement = False
self.__y_direction = 0
log.log_info("stop y")
#Y Axis
self.__issue_command(self.__y_device, 23, 0, 0, 0, 0)
def write(self, data):
global _output_to_log
interpreter = None
if "value" in dir(PythonScriptingInstance._interpreter):
interpreter = PythonScriptingInstance._interpreter.value
if interpreter is None:
if _output_to_log:
self.buffer += data
while True:
i = self.buffer.find('\n')
if i == -1:
break
line = self.buffer[:i]
self.buffer = self.buffer[i + 1:]
if self.is_error:
log.log_error(line)
else:
log.log_info(line)
else:
self.orig.write(data)
else:
PythonScriptingInstance._interpreter.value = None
try:
if self.is_error:
interpreter.instance.error(data)
else:
interpreter.instance.output(data)
finally:
PythonScriptingInstance._interpreter.value = interpreter
def setCurrentTopoID(new_topo_id):
log_info("Updating topology id: \t%s" % new_topo_id)
current = info.find_one()
if not current: current = {}
current['topoID'] = new_topo_id
current['last_updated'] = datetime.utcnow()
info.save(current)
def add_spreadsheet_list(user_id, language, language_translation, list_obj):
"""
uses the json object that was received by google spreadsheet and adds the words to the database
:param user:
:param language:
:param list_obj:
:return:
"""
# create the ID of this insert.
time_stamp = int(time.time())
log.log_info("add_spreadsheet_list adding words... ")
for row in list_obj:
context = row[0]
voc = row[1]
trans = row[2]
text = row[3]
if len(voc) > 0:
if len(trans) > 0:
add_one_word_txt(user_id, language, voc, language_translation,
trans, True, context, text, time_stamp)
add_one_word_txt(user_id, language_translation, trans,
language, voc, False, context, text,
time_stamp)
return ""
def _parse_payload_config(payload_name: str) -> dict:
result = {
"payload_name": payload_name,
"service": "0",
"params": config.AVAILABLE_PARAMS_FOR_PAYLOADS
}
try:
_uuid = utils.uuid.get_uuid()
exec(f"import payloads.{payload_name} as {_uuid}")
try:
service = eval(f"{_uuid}.SERVICE")
if type(service) != str:
log.log_warn(f"Service is not a string for payload {payload_name}, using default")
else:
result["service"] = service
except AttributeError:
log.log_info(f"Using default SERVICE value for payload {payload_name}")
try:
params = eval(f"{_uuid}.PARAMS")
if type(params) != list:
log.log_warn(f"Params is not a list for payload {payload_name}, using default")
else:
result["params"] = params
except AttributeError:
log.log_info(f"Using default SERVICE value for payload {payload_name}")
except ModuleNotFoundError:
log.log_error(f"Cannot import file {payload_name}")
return result
def send_mail(to_email, email_subject, email_message):
log.log_info("in send_mail")
FROM = cfg.parameters["email-user"]
TO = [to_email]
#print(email_message)
password = cfg.parameters["email-password"]
SUBJECT = "" + email_subject
TEXT = email_message
message = """From: %s\nTo: %s\nSubject: %s\n\n%s
""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
# SMTP_SSL Example
#server_ssl = smtplib.SMTP_SSL(host="smtp.alexandersickert.com", port=465)
server_ssl = smtplib.SMTP_SSL(host="smtp.gmail.com", port=465)
server_ssl.ehlo() # optional, called by login()
server_ssl.login(FROM, password)
# ssl server doesn't support or need tls, so don't call server_ssl.starttls()
server_ssl.sendmail(FROM, TO, message)
#server_ssl.quit()
server_ssl.close()
#print( 'successfully sent the mail')
log.log_info("end send_mail")
def serve_forever():
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#listen_socket = ssl.wrap_socket(listen_socket, certfile=cfg.parameters["certfile"], keyfile=cfg.parameters["keyfile"],server_side=True, ssl_version=ssl.PROTOCOL_TLSv1_2)
ssl_socket = ssl.wrap_socket(listen_socket,
certfile=cfg.parameters["certfile"],
keyfile=cfg.parameters["keyfile"],
server_side=True)
ssl_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ssl_socket.bind(SERVER_ADDRESS)
ssl_socket.listen(REQUEST_QUEUE_SIZE)
executor = ThreadPoolExecutor(max_workers=50)
log.log_info('MASTER Serving HTTPS on port {port} ...'.format(port=PORT))
while True:
#client_connection, client_address = listen_socket.accept()
try:
client_connection, client_address = ssl_socket.accept()
#log.log_info("client_user_address: " + str( client_address) )
#traffic.track(client_address[0], True)
a = executor.submit(handle_request, client_connection,
str(client_address[0]), client_address[1])
except:
print("Unexpected error:", sys.exc_info()[0])
log.log_error("Error in try-catch of main server loop: " +
str(sys.exc_info()[0]))
if "KeyboardInterrupt" in str(sys.exc_info()[0]):
exit()
def register_user_and_session_at_master(session, user):
# this is so that the master knows where to forward the request
# cfg.slave_id
# url = cfg.parameters["master-url"] + ":" + cfg.parameters["https"] + "/register-slave?security-key=YI64QZ8LMPJET0GV4CCV&"
# url += "register-session=" + session
# url += "&server=" + cfg.slave_id
# url += "®ister-user="******"calling MASTER via URL " + url)
# r = urllib.request.urlopen(url)
# log.log_info("MASTER responds " + str(r.read()))
conn = http.client.HTTPSConnection(
cfg.parameters["master-url"],
cfg.parameters["https"],
context=ssl._create_unverified_context())
query = "/register-slave?security-key=YI64QZ8LMPJET0GV4CCV&"
query += "register-session=" + session
query += "&slave-id=" + cfg.slave_id
query += "®ister-user="******"MASTER responds " + str(r.read()))
def write(self, data):
interpreter = None
if hasattr(PythonScriptingInstance._interpreter, "value"):
interpreter = PythonScriptingInstance._interpreter.value
if interpreter is None:
if log.is_output_redirected_to_log():
self.buffer += data
while True:
i = self.buffer.find('\n')
if i == -1:
break
line = self.buffer[:i]
self.buffer = self.buffer[i + 1:]
if self.is_error:
log.log_error(line)
else:
log.log_info(line)
else:
self.orig.write(data)
else:
PythonScriptingInstance._interpreter.value = None
try:
if self.is_error:
interpreter.instance.error(data)
else:
interpreter.instance.output(data)
finally:
PythonScriptingInstance._interpreter.value = interpreter
def send_move_request(self, direction):
try:
self._conn.request("ON", self._solenoid_number[direction])
response = self._conn.getresponse()
log.log_info(response.read())
except socket_error as serr:
log.log_error("Failed communication with HTTP server.")
def update():
# Download new NaPTAN data
download_naptan_data()
# Download new json data
bustrackerApi.scrapeStopsFromServer()
bustrackerApi.scrapeDestsFromServer()
bustrackerApi.scrapeServicesFromServer()
# Clear the database
bustrackerDb.clear_database()
# Update services
services = bustrackerApi.getServices()
bustrackerDb.populateServices(services)
# Update stops
apiStops = bustrackerApi.getStops()
bustrackerDb.populateStops(apiStops)
# Update destinations
dests = bustrackerApi.getDests()
bustrackerDb.populateDests(dests)
# Updating database topology id
bustrackerDb.setCurrentTopoID(api_topo_id)
log.log_info("Update complete: \tExiting")
log.email_log()
def setCurrentTopoID(new_topo_id):
log_info("Updating topology id: \t%s" % new_topo_id)
current = info.find_one()
if not current: current = {}
current['topoID'] = new_topo_id
current['last_updated'] = datetime.utcnow()
info.save(current)
def finished(d):
uploading_file = d["uploading_file"]
file_id = fm.generate_id()
result = fm.put_inference_zip(uploading_file,
file_id,
is_expanding=True)
log_info(result)
if result["status"] == "success":
image_path = result["image_path"]
model_id = d["model_id"]
recipe_id = d["recipe_id"]
model = CNN(recipe_id)
inference_type = d["inference_type"]
if inference_type == "classification":
inference_res = model.classify(model_id, image_path)
action = "finishClassfication"
elif inference_type == "vectorization":
inference_res = model.vectorize(model_id,
image_path,
is_similarity=True)
action = "finishVectorization"
res = fm.get_inferece_images(image_path)
image_list = res["list"]
res_list = []
for r, i in zip(inference_res, image_list):
r["body"] = i["body"]
res_list.append(r)
res = {"action": action, "id": file_id, "list": res_list}
return res
def process_json_master(json_string):
"""
This is used by the Google Spreadsheet upload
The master server needs to know to which slave to forward the message. We need to look up the server
:param json_string:
:return:
"""
jo = json.loads(json_string)
#user_id = sec.get_user_id(jo["user"]) # this is the email address
ip, port = sec.get_slave_ip_port(jo["user"])
if port > 0:
ret_obj = slave_request.get_from_slave(ip, port, jo)
log.log_info("Spreadsheet upload reponse from slave: " + str(ret_obj))
else:
# wrong login coordinates
ret_obj = {}
ret_obj["error"] = True
ret_obj["function"] = jo["function"]
ret_obj["error_description"] = "could not find user"
log.log_info("process_json_master(json_string): " + str(ret_obj))
return json.dumps(ret_obj)
def fill_cache_from_db(user_id):
"""
copies from db into the cache
:param user_id:
:return:
"""
log.log_info("in fill_cache_from_db(user_id)")
conn = dba.get_connection()
cur = conn.cursor()
sql = """
SELECT id, COALESCE(short_memory_position, 1)
FROM vocabulary
WHERE user_id = %s
and current = true
and COALESCE(last_studied, 0) > 1
order by COALESCE(short_memory_position, 1)
LIMIT 100
"""
cur.execute(sql, (user_id, ))
arr = cur.fetchall()
log.log_info("in fill_cache_from_db(user_id) - array has length " +
str(len(arr)))
for row in arr:
add_to_cache(user_id, row[0], row[1])
def update():
# Download new NaPTAN data
download_naptan_data()
# Download new json data
bustrackerApi.scrapeStopsFromServer()
bustrackerApi.scrapeDestsFromServer()
bustrackerApi.scrapeServicesFromServer()
# Clear the database
bustrackerDb.clear_database()
# Update services
services = bustrackerApi.getServices()
bustrackerDb.populateServices(services)
# Update stops
apiStops = bustrackerApi.getStops()
bustrackerDb.populateStops(apiStops)
# Update destinations
dests = bustrackerApi.getDests()
bustrackerDb.populateDests(dests)
# Updating database topology id
bustrackerDb.setCurrentTopoID(api_topo_id)
log.log_info("Update complete: \tExiting")
log.email_log()
def send_command(self, cmd, timeout=6, ignore_fail = 0, retry_on = 0):
buff = ''
result = 1
self.ssh_handle.sendline(cmd)
for count in range(1, 10):
if retry_on == 1 and count > 1:
self.ssh_handle.sendline(cmd)
exp_index = self.ssh_handle.expect([
self.str_prompt, #0
pexpect.EOF, #1
pexpect.TIMEOUT #2
], timeout)
buff += self.ssh_handle.before
self.clean_buff()
if exp_index == 0:
result = 0
break
elif exp_index == 1:
log.log_info('command: %s failed. (EOF)' %(cmd), 'ERROR')
break
elif exp_index == 2:
log.log_info('command: %s failed, wait: %s. (TIMEOUT)' %(cmd, count), 'WARNING')
time.sleep(timeout)
if result == 1:
log.log_info('Send %s failed, resend times %d(timeout=%d)' %(cmd, count, timeout), 'ERROR')
if ignore_fail == 0:
self.ssh_handle.close(force = True)
sys.exit()
else:
return False
if not buff:
log.log_info('command: %s [NO BUFF]' %(cmd), 'WARNING') #still return, handled by users
return False
log.log_info('Send [%s] command ok' %(cmd), 'DUBUG')
return buff
def create_hit_table(self):
for i in range(0, self.mysql_split):
table = "%s_%04d%02d%02d" % (self.mysql_table, self.today.year, self.today.month, self.today.day)
if not self.raw_handle.check_table_exists(table):
self.raw_cur.execute("CREATE TABLE %s(id INT PRIMARY KEY AUTO_INCREMENT, number INT, \
last_update TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL DEFAULT NOW())" % table)
self.raw_cur.execute("ALTER TABLE %s add index my_id(number)" % table)
log.log_info("create table %s" % table, "DEBUG")
def get_page(url):
res = requests.get(url, headers=headers)
if res:
log.log_info('fetch:访问' + url)
page = res.content.decode('utf-8')
return page, url
else:
log.log_error('fetch:访问' + url + '失败')
def move(self, vector, invert_x_axis, invert_y_axis):
""" Given input parameters, moves the robot relative to it's current
position. It executes first the x component and then the y component.
Keyword Arguments:
self -- actuator object the function was called on
vector -- movement vector
invert_x_axis -- boolean of whether to invert on the x-axis
invert_y_axis -- boolean of whether to invert on the y-axis
"""
vector = [x * self.__variable_step_size for x in vector]
if (vector[0] != 0 and self.__in_x_movement
and self.__x_direction != 0):
temp = -1 if vector[0] < 0 else 1
if temp != self.__x_direction:
self.end_move([temp, 0], invert_x_axis, invert_y_axis)
if (not self.__in_x_movement and vector[0] != 0):
self.__in_x_movement = True
#X Axis
if invert_x_axis:
vector[0] *= -1
log.log_info("start x " + str(vector[0]))
self.__x_direction = -1 if vector[0] < 0 else 1
byte_array = _convert_int_to_bytes(vector[0])
self.__issue_command(self.__x_device, 22, byte_array[0],
byte_array[1], byte_array[2], byte_array[3])
if (vector[1] != 0 and self.__in_y_movement
and self.__y_direction != 0):
temp = -1 if vector[1] < 0 else 1
if temp != self.__y_direction:
self.end_move([0, temp], invert_x_axis, invert_y_axis)
if (not self.__in_y_movement and vector[1] != 0):
self.__in_y_movement = True
#Y Axis
if invert_y_axis:
vector[1] *= -1
log.log_info("start y " + str(vector[1]))
self.__y_direction = -1 if vector[1] < 0 else 1
byte_array = _convert_int_to_bytes(vector[1])
self.__issue_command(self.__y_device, 22, byte_array[0],
byte_array[1], byte_array[2], byte_array[3])
def get_old_by_score(user_id):
"""
take one word from the pile of words we already learned
:param user_id:
:return:
"""
log.log_info(" in get_old_by_score(user_id) ")
conn = dba.get_connection()
cur = conn.cursor()
# age of old word needs to be at least from yesterday
delta = 24 * 60 * 60
threshold = int(time.time()) - delta
sql = """
SELECT
id, calc_rank
FROM
vocabulary
WHERE
user_id = %s
AND
current = false
AND
count_positive > 0
AND
last_studied < %s
ORDER BY
/* changed following line on 16th feb 2019 because yes = 1 and we need to know failures so asc and desc*/
/* calc_rank desc */
calc_rank asc
LIMIT 2;
"""
cur.execute(sql, (user_id, threshold))
#l = cur.fetchall()[0][0]
arr = cur.fetchall()
#print(arr)
if len(arr) > 0:
i = int(random.random() * float(len(arr)))
l = arr[i][0]
set_experiment_state(l, True)
else:
l = -1
log.log_info("get_old_by_score returns " + str(l) + " for user id " +
str(user_id))
return l
def __confirm_maze_frame(self, item):
""" Confirm the corners of the maze that have been set up """
log.log_info("Navigating maze...")
# Cleanup maze window
maze_window = self.__builder.get_object("maze_window")
maze_window.hide()
self.__navigate_maze(item)
def set_position_in_db(word_id, pos):
log.log_info("set_position " + str(word_id) + " to " + str(pos))
sql = "UPDATE vocabulary SET short_memory_position = %s WHERE id = %s "
conn = dba.get_connection()
cur = conn.cursor()
cur.execute(sql, (pos, word_id))
conn.commit()
def connect_to_redis(host,port):
log_info('Connecting to Redis at %s:%u' % (host, port))
try:
global redisdb
redisdb = redis.Redis(host=host,port=port)
return redisdb
except Exception, e:
log_error( 'Error initializing redis: %s' % str(e))
exit()
def connect_to_redis(host, port):
log_info('Connecting to Redis at %s:%u' % (host, port))
try:
global redisdb
redisdb = redis.Redis(host=host, port=port)
return redisdb
except Exception, e:
log_error('Error initializing redis: %s' % str(e))
exit()
def add_new_word(user_id):
"""
Decide if we want to add a word from the pile of new words or from the pile of old words that need repetition
The decision depends on the percentage fo successfully leanrned words. If the predictions says that more than XX%
of the words are not forgotten, then we add a new word.
if the ratio is 0.0 then we add new words because the ration could not be calculated. but if the ratio is between
0.05 and 0.8 then we repeat old words first. if the ratio is > 0.85 then we add new words
:return:
"""
parameter_name = "percentage_learned"
conn = dba.get_connection()
cur = conn.cursor()
sql = "SELECT value FROM parameters WHERE user_id = %s AND key = %s"
cur.execute(sql, (user_id, parameter_name))
arr = cur.fetchall()
ratio_learned = -1.0 # default value
#print(user_id)
#print(parameter_name)
#print(arr)
if arr is not None:
if len(arr) > 0:
ratio_learned = float(arr[0][0])
log.log_info("add_new_word(user_id) " + str(user_id) +
" ratio learned is " + str(ratio_learned))
r = random.random()
if ratio_learned < 0.0 or ratio_learned > 0.99: # this happens when not enough words were so far learned and parameter is not filled
# generally we would like to add only new words but we also need to start doing experiments
if r > 0.5:
return True
else:
return False
elif ratio_learned < 0.05: # ToDo this is probably an error and needs to be removed
return True
elif ratio_learned > 0.85: # we know most of the words and make rarely experiments
if r > 0.9: # only in 10 percent we make experiments
return False
else:
return True
else:
if r > (ratio_learned *
0.8): # if 70% learned, then we make 30% experiments
return False
else:
return True
def read_article(self, url):
print "reading url '%s'" % url
infile = self.opener.open(url)
page = infile.read()
log.log_info("read page length: " + str(page.__len__()))
log.log("page beings with: " + page[0:256])
# xmldoc = minidom.parseString(page)
#print ("xml length: " + str(xmldoc.__len__()))
infile.close()
return page
def __init__(self, config_file="config.ini"):
log.log_info("read config %s" % (config_file), "DEBUG")
self.config = ConfigParser.ConfigParser()
self.config.read(config_file)
self.mysql_user = self.config.get('mysql', 'MYSQL_USER')
self.mysql_pwd = self.config.get('mysql', 'MYSQL_PWD')
self.mysql_db = self.config.get('mysql', 'MYSQL_DB')
self.mysql_table = self.config.get('mysql', 'MYSQL_TABLE')
self.mysql_ip = self.config.get('mysql', 'MYSQL_IP')
self.mysql_port = int(self.config.get('mysql', 'MYSQL_PORT'))
def update_password(u, p):
conn = dba.get_connection()
cur = conn.cursor()
sql = "UPDATE users SET password = %s WHERE email = %s;"
hash_object = hashlib.md5(p.encode())
p_hash = hash_object.hexdigest()
log.log_info("p=" + p)
log.log_info("p_hash=" + p_hash)
cur.execute(sql, (p_hash, u))
conn.commit()
def register_user(u, p):
conn = dba.get_connection()
cur = conn.cursor()
sql = "INSERT INTO users (email, password) VALUES (%s, %s);"
hash_object = hashlib.md5(p.encode())
p_hash = hash_object.hexdigest()
log.log_info("p=" + p)
log.log_info("p_hash=" + p_hash)
cur.execute(sql, (u, p_hash))
conn.commit()
def toggle_adc(self):
""" Returns the new value of toggling adc
"""
try:
self._conn.request("ADC", self.__TOGGLE_ADC)
response = self._conn.getresponse()
log.log_info(response)
return response.read() == "true"
except socket_error as serr:
log.log_error("Failed communication with HTTP server.")
def getTipo(self):
try:
if self.getTp==-1:
return self.getTp
else:
getTp = int(self.instrument.read_register(0))
print "NUMSEM", getTp
self.getTp = getTp
return getTp
except IOError as e:
log_info("I/O error({0}): {1}".format(e.errno, e.strerror) + "id:" +str(self.address), self, "getTipo")
def get_random_slave_ip_port():
global slaves_arr
log.log_info("length of slaves_arr: " + str(len(slaves_arr)))
i = random.randint(0, len(slaves) - 1)
ip = slaves_arr[i][1]
port = slaves_arr[i][2]
return ip, int(port)
def populateStops(new_stops):
services_dict = {service['ref']: service for service in getServicesDB()}
log_info("Populating stops \t(%i)" % len(new_stops))
for stop in new_stops:
stop['heading'] = getHeadingForStopCap(stop['cap'])
stop = addNaptanDataToStop(stop)
stop = add_service_mnemos_to_stop(stop, services_dict)
if not 'street' in stop:
stop = add_street_name_to_stop(stop)
stops.insert(new_stops)
incomplete_count = len(list(find_incomplete_stops()))
log_info("Stops without complete information: \t%i" % incomplete_count)
def populateStops(new_stops):
services_dict = {service['ref']:service for service in getServicesDB()}
log_info("Populating stops \t(%i)" % len(new_stops))
for stop in new_stops:
stop['heading'] = getHeadingForStopCap(stop['cap'])
stop = addNaptanDataToStop(stop)
stop = add_service_mnemos_to_stop(stop, services_dict)
if not'street' in stop:
stop = add_street_name_to_stop(stop)
stops.insert(new_stops)
incomplete_count = len(list(find_incomplete_stops()))
log_info("Stops without complete information: \t%i" % incomplete_count)
def getNumSem(self):
try:
if self.numsem==-1:
numsem = int(self.instrument.read_register(6))
print "NUMSEM", numsem
self.numsem = numsem
return numsem
else:
return self.numsem
except IOError as e:
log_info("I/O error({0}): {1}".format(e.errno, e.strerror) + "id:" +str(self.address), self, "getNumSem")
def send_commands(self, cmds, timeout = 6, interval = 0.01, ignore_fail = 0, retry_on = 0):
buff = ''
for cmd in cmds.split('\n'):
if not cmd:
continue
cmd = cmd.strip('\t')
buff += self.send_command(cmd, timeout, ignore_fail, retry_on)
time.sleep(interval)
if not buff:
log.log_info('command: %s [NO BUFF]' %(cmds), 'WARNING') #still return, handled by users
return False
return buff
def step_change(self, new_value, increment):
if new_value != None and new_value >= 0:
log.log_info("Changing step size to " + str(new_value))
self.__variable_step_size = new_value
if increment != None:
temp = self.__variable_step_size + increment
if temp < 0:
temp = 0
log.log_info("Changing step size from " +
str(self.__variable_step_size) + " to " + str(temp))
self.__variable_step_size = temp
def get_next_word_id_array(user_id, last_word_id):
"""
NOTE: consider also the funciton get_next_word_id(user_id, last_word_id) because it is similar !!!
:param user_id:
:param last_word_id:
:return:
"""
# first we ensure there are enough current words
if count_current(user_id) < 7:
log.log_info("get_next_word_id - count_current less than 7")
# we need to add a new word question is if repeat old word or use new
while count_current(user_id) < 7:
if add_new_word(user_id):
# ToDo: if there are no new words left, then we need to process old words and send info to user,
# that there are no new words left
if count_not_learned(user_id) > 0:
word_id = get_new_random(user_id)
else:
# if there are no new words left then use old words
word_id = get_old_by_score(user_id)
else:
# we can add old only if such words exist
# if not, then we add from new category
num_learned = count_learned(user_id)
if num_learned > 0:
word_id = get_old_by_score(user_id)
else:
word_id = get_new_random(user_id)
set_word_current(word_id)
# now we load all the current words as an array
ret = []
conn = dba.get_connection()
cur = conn.cursor()
cur.execute(
"SELECT ID FROM vocabulary where user_id = %s AND current = true ORDER BY random() LIMIT 20",
(user_id, ))
l = cur.fetchall()
for id in l:
ret.append(id[0])
return ret
def download_naptan_data():
# Download the NaPTAN archive
log_info("Downloading NaPTAN data")
naptan_url = "http://www.dft.gov.uk/NaPTAN/snapshot/NaPTANcsv.zip"
naptan_archive_path = "%sNaPTANcsv.zip" % csv_path
storeUrlContentsAtPath(naptan_url, naptan_archive_path)
# Extract the Stops.csv file
log_info("Extracting NaPTAN Stops data")
stops_filename = 'Stops.csv'
command = "unzip %s %s -d %s" % (naptan_archive_path, stops_filename, csv_path)
call(command.split())
# Find rows corresponding to stops we're interested in
log_info("Extracting useful rows")
stops_path = "%s%s" % (csv_path,stops_filename)
stops_file = open(stops_path,'r')
stops_csv_reader = reader(stops_file)
useful_rows = []
for row in stops_csv_reader:
# add if the ACTOcode starts with one of our areas
for area in naptan_areas:
if row[0].startswith(str(area)):
useful_rows.append(row)
# Write useful rows to file
log_info("Storing useful rows")
naptan_path = '%sNaPTAN.csv' % csv_path
naptan_file = open(naptan_path,'w')
naptan_csv_writer = writer(naptan_file)
naptan_csv_writer.writerows(useful_rows)
# Remove the zip archive and Stops csv file
call(['rm',naptan_archive_path,stops_path])
def set_word_current(word_id):
"""
:param word_id:
:return:
"""
log.log_info("set_word_current " + str(word_id))
sql = "UPDATE vocabulary SET current = TRUE, short_memory_position = 1 WHERE id = %s "
conn = dba.get_connection()
cur = conn.cursor()
cur.execute(sql, (word_id, ))
conn.commit()
def get_simple_report(user_id):
log.log_info("get_simple_report(user_id): for user " + str(user_id))
conn = dba.get_connection()
cur = conn.cursor()
cur.execute(
"SELECT count(*) FROM vocabulary where user_id = %s AND current = false AND count_positive < 1 AND direction = TRUE",
(user_id, ))
new_words = cur.fetchall()[0][0]
cur.execute(
"SELECT count(*) FROM vocabulary where user_id = %s AND current = false AND count_positive > 0 AND direction = TRUE",
(user_id, ))
learned_words = cur.fetchall()[0][0]
parameter_name = "percentage_learned"
sql = "SELECT value FROM parameters WHERE user_id = %s AND key = %s"
cur.execute(sql, (user_id, parameter_name))
try:
arr = cur.fetchall()
p = arr[0][0]
log.log_info("value from rs is: " + str(p))
except:
log.log_info("value rs is not existing: ")
p = 0.0
ratio_learned = float(p) * 100
ratio_learned = round(ratio_learned, 2)
log.log_info("ratio_learned at end is is: " + str(ratio_learned))
return new_words, learned_words, ratio_learned
def check_login(u, p):
conn = dba.get_connection()
cur = conn.cursor()
hash_object = hashlib.md5(p.encode())
p_hash = hash_object.hexdigest()
log.log_info("p=" + p)
log.log_info("p_hash=" + p_hash)
cur.execute(
"SELECT count(*) FROM users WHERE email = %s AND password = %s",
(u, p_hash))
ret = cur.fetchall()[0][0]
print(ret)
return ret
def count_current(user_id):
"""
Counts how many words are in the carousel for a specific user.
:param user_id:
:return:
"""
conn = dba.get_connection()
cur = conn.cursor()
cur.execute(
"SELECT count(*) FROM vocabulary where user_id = %s AND current = true",
(user_id, ))
l = cur.fetchall()[0][0]
log.log_info("count_current is " + str(l))
return int(l)
def parse(self):
print "parsing XML"
log.log("parse_article(" + str(self) + "," + str(self.xmldoc) + ")")
for title in self.xmldoc.getElementsByTagName("title"):
log.log_info("title is: " + title.firstChild.nodeValue)
# TODO: should assert only one vcard or something, i guess...
vcard = self.get_vcards(self.xmldoc).next()
# find 'Born' values in vcard
print "Calling 'parse_class'"
items = {}
itemsOfIterest = ["Born", "Died"]
for tr in vcard.getElementsByTagName("tr"):
item = self.parse_class(tr)
if (item.name in itemsOfIterest):
items[item.name] = item
return items
def sendReadings(nextRun):
if datetime.now() >= nextRun:
#turn on the sensors
# the transisotr enable the sensors to be powered
sys_gpio.setTransistor(True)
log.current_state = "Reading Air Temperature"
air_temp = DHT11TemperatureSensor.read()
log.current_state = "Posting Air Temperature"
httpclient.post(api.__send_readings__, air_temp, auth.checkResponse)
scn.update(env_temp=str(air_temp['reading']))
log.current_state = "Reading Air Humidity"
air_hum = DHT11HumiditySensor.read()
log.current_state = "Posting Air Humidity"
httpclient.post(api.__send_readings__, air_hum, auth.checkResponse)
scn.update(env_hum=str(air_hum['reading']))
log.current_state = "Reading Luminosity"
air_lum = LDR.read()
log.current_state = "Posting Luminosity"
httpclient.post(api.__send_readings__, air_lum, auth.checkResponse)
scn.update(env_lum="{0:0.2f}".format(air_lum['reading']))
log.current_state = "Reading Soil Temperature"
soil_temp = DS18B20TemperatureSensor.read()
log.current_state = "Reading Soil Temperature"
httpclient.post(api.__send_readings__, soil_temp, auth.checkResponse)
scn.update(soil_temp="{0:0.2f}".format(soil_temp['reading']))
sys_gpio.setTransistor(False)
#print air_lum["date"]
#print "Env: Temp.: {0:0.2f}C Humidity: {1:0.2f}% & Luminosity:{2:0.2f}%".format(air_temp['reading'], air_hum['reading'], air_lum['reading'])
#print "Soil: Temp.: {0:0.2f}C & Moisture: 0%".format(soil_temp['reading'])
#print "\n"
log.log_info("System Ok")
nextRun = datetime.now() + timedelta(seconds=60)
return nextRun
def put_task_into_queue():
'''扫描数据库,将任务添加到队列'''
log.log_info('Scaning DB...')
while True:
init.queue.join() # 阻塞程序
sql = 'SELECT submission_id FROM ' + config.db_table_name + ' WHERE status = ' + status.QUEUING + ';'
data = db.run_sql(sql)
num = len(data)
if num > 0:
log.log_info('%d submission(s) coming...' % num)
for i in range(num):
time.sleep(1)
submission_id = data[i][0]
init.dblock.acquire()
ret = get_data.get_code(submission_id) and get_data.get_input(submission_id)
init.dblock.release()
if ret == False:
init.dblock.acquire()
update.update_status(submission_id, status.GET_DATA_ERROR)
init.dblock.release()
continue
inner_sql = 'SELECT language, time, memory FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
#print inner_sql
inner_data = db.run_sql(inner_sql)
language_id = inner_data[i][0]
Time = inner_data[i][1]
memory = inner_data[i][2]
task = {
'submission_id': submission_id,
'language_id': language_id,
'time': Time,
'memory': memory
}
init.queue.put(task)
time.sleep(1)
def pwm_change(self, new_value, increment):
""" Adjusts the PWM for the solenoids
Keyword Arguments:
increment -- whether the speed is increasing (1) or decreasing (-1)
"""
if increment != None:
if increment==1:
log.log_info("Incrementing PWM voltage.")
else:
log.log_info("Decrementing PWM voltage.")
try:
self._conn.request("PWM", self._DECREMENT if increment == -1 else self._INCREMENT)
response = self._conn.getresponse()
#log.log_info(response.read())
except socket_error as serr:
log.log_error("Failed communication with HTTP server.")
def end_move(self, vector, invert_x_axis, invert_y_axis):
""" Given input parameters, deactivates the specified solenoid
Keyword Arguments:
direciton -- direction of movement
invert_x_axis -- boolean of whether to invert on the x-axis
invert_y_axis -- boolean of whether to invert on the y-axis
"""
direction = ""
if invert_x_axis:
vector[0] *= -1
if invert_y_axis:
vector[1] *= -1
if (vector[0] != 0):
if (vector[0] > 0):
if (self.__x_movement == 2):
self.__x_movement = 0
direction = self._RIGHT
else:
if (self.__x_movement == 1):
self.__x_movement = 0
direction = self._LEFT
elif (vector[1] != 0):
self.__in_y_movement = False
if (vector[1] > 0):
if (self.__y_movement == 1):
self.__y_movement = 0
direction = self._UP
else:
if (self.__y_movement == 2):
self.__y_movement = 0
direction = self._DOWN
try:
self._conn.request("OFF", self._solenoid_number[direction])
response = self._conn.getresponse()
log.log_info(response.read())
except socket_error as serr:
log.log_error("Failed communication with HTTP server.")
def __init__(self, host, username, password, log_mode=2):
self.set_prompt('\# ')
self.ssh_handle = pexpect.spawn('ssh ' + username + '@' + host)
for loop in range(1,8):
exp_index = self.ssh_handle.expect ([
self.str_prompt, # 0 expect prompt
pexpect.TIMEOUT, # 1
pexpect.EOF, # 2
'yes/no', # 3 If added new line here, please notice
'password', # 4 the index is used in the if elif lines.
'password: '******'Permission denied, please try again.', # 6
'Connection closed by UNKNOWN' # 7
], timeout=6)
if 0 == exp_index:
if 1 == log_mode:
self.ssh_handle.logfile = sys.stdout
elif 2 == log_mode:
self.ssh_handle.logfile_read = sys.stdout
elif 3 == log_mode:
fout = file('mylog.txt','w')
self.ssh_handle.logfile = fout
elif 4 == log_mode:
fout = file('mylog.txt','w')
self.ssh_handle.logfile_read = fout
self.clean_buff()
log.log_info('ssh connect successfully.')
break
elif 1 == exp_index:
pass
elif 3 == exp_index:
self.ssh_handle.sendline('yes')
elif 4 == exp_index or 5 == exp_index:
self.ssh_handle.sendline(password)
elif 6 == exp_index:
pass
else: #EOF or others
log.log_info('ssh connect failed.', 'ERROR')
self.ssh_handle.close(force=True)
def get_code(submission_id):
'''从数据库获取代码并写入work目录下对应的文件'''
sql = 'SELECT code, language FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
data = db.run_sql(sql)
if data != False:
code = data[0][0]
language_id = data[0][1]
file_name = language.get_filename(language_id)
else:
return False
try:
work_path = os.path.join(config.work_dir, str(submission_id))
secure.low_level()
os.mkdir(work_path)
except OSError as e:
if str(e).find('exist') > 0: # 文件夹已经存在
pass
else:
return False
try:
real_path = os.path.join(config.work_dir, str(submission_id), file_name)
except KeyError as e:
# print e
return False
try:
log.log_info('Importing code, %d' % submission_id)
secure.low_level()
f = open(real_path, 'w')
try:
f.write(code)
except:
f.close()
log.log_error('Importing code failed, %d' % submission_id)
return False
f.close()
except OSError as e:
return False
return True
def SendMany(wallet_host,wallet_port,recipients,paymentid=None,mixin=None):
for address in recipients:
if not IsValidAddress(address):
log_error("Invalid address: %s" % address)
return
amount = recipients[address]
if amount <= 0:
log_error("Invalid amount: %s" % str(amount))
return
if paymentid != None:
if not IsValidPaymentID(paymentid):
log_error("Invalid payment ID")
return
if mixin < 0:
log_error("Invalid mixin: %d")
return
log_info("Pay: sending %s, payment id %s, mixin %d" % (str(recipients), str(paymentid),mixin))
try:
destinations=[]
for address in recipients:
destinations.append({'address':address,'amount':recipients[address]})
params = {
'destinations':destinations,
'payment_id': paymentid,
'fee': 0,
'mixin': mixin,
'unlock_time': 0,
}
j = SendJSONRPCCommand(wallet_host,wallet_port,"transfer",params)
except Exception,e:
log_error('Withdraw: Error in transfer: %s' % str(e))
return
def before_run(submission_id, time, memory, language_id):
secure.low_level()
'''获取程序执行时间和内存'''
if secure.check_dangerous_code(submission_id, language_id) is False:
update.update_status(submission_id, status.DANGEROUS_CODE)
log.log_error('Has DANGEROUS code, %d' % submission_id)
return False
update.update_status(submission_id, status.COMPILEING)
compile_result = compiler.compile(submission_id, language_id)
if compile_result is False: # 编译错误
update.update_status(submission_id, status.COMPILATION_ERROR)
log.log_error('Compile failed, %d' % submission_id)
return False
log.log_info('Compile completed, %d' % submission_id)
result = run(
solution_id,
time_limit,
mem_limit,
program_info,
language_id)
return result
def worker():
'''工作线程，循环扫描队列，获得评判任务并执行'''
while True:
if init.queue.empty() is True: # 队列为空，空闲
continue
task = init.queue.get() # 获取任务，如果队列为空则阻塞
submission_id = task['submission_id']
language_id = task['language_id']
time = task['time']
memory = task['memory']
log.log_info('Task input, %d' % submission_id)
log.log_info('Start judging, %d' % submission_id)
program_info = judge.judge(submission_id,
time,
memory,
language_id)
if program_info not False:
log.log_info('Judging complete, %d' % submission_id)
init.dblock.acquire()
update.write_result(submission_id, program_info) # 将结果写入数据库
update.write_output(submission_id)
init.dblock.release()
log.log_info('Update INFO complete, %d' % submission_id)
else:
log.log_error('Running Error, %d' % submission_id)
init.dblock.acquire()
update.write_result(submission_id, config.COMPILATION_ERROR) # 将结果写入数据库
update.write_output(submission_id)
init.dblock.release()
if config.auto_clean_work: # 清理work目录
clean.clean_work_dir(submission_id)
if config.auto_clean_data:
clean.clean_data_dir(submission_id)
init.queue.task_done() # 一个任务完成
def parse_subnetworklist():
inputf = file(input, 'r')
icnt = 0
ocnt = 0
subnetworklist = []
while True:
line = inputf.readline()
if len(line) == 0:
break
icnt += 1
range = parse_line(line)
if len(range) == 2:
if range[0] > range[1]:
log.log_info("%s err, %d=>%d" % (line, range[0], range[1]))
continue
subnetworklist.append(range)
ocnt += 1
log.log_info("%s=>%d:%d" % (line, range[0], range[1]))
inputf.close()
log.log_info("deal %d/%d records from %s" % (ocnt, icnt, input))
output(subnetworklist)
new_payments = []
n_confirming = 0
new_scan_block_height = scan_block_height
for p in payments:
payment_id=p["payment_id"]
tx_hash = p["tx_hash"]
bh = p["block_height"]
ut = p["block_height"]
amount=p["amount"]
if not full_history and redis_sismember("processed_txs",tx_hash):
continue
log_log('UpdateCoin: Looking at payment %s' % str(p))
confirmations = height-1-bh
confirmations_needed = max(config_confirmations,ut-height)
if confirmations >= confirmations_needed:
log_info('Payment %s is now confirmed' % str(p))
new_payments.append(p)
if new_scan_block_height and bh > new_scan_block_height:
new_scan_block_height = bh
else:
log_info('Payment %s has %d/%d confirmations' % (str(p),confirmations,confirmations_needed))
n_confirming += 1
new_scan_block_height = None
try:
recipient = GetUserIDFromPaymentID(payment_id)
if not recipient:
raise RuntimeError('Payment ID %s not found' % payment_id)
log_info('UpdateCoin: Found payment %s to %s for %s' % (tx_hash,recipient, AmountToString(amount)))
if not full_history:
cp.hincrby('confirming_payments',recipient,amount)
txs.append({'tx_hash':tx_hash,'amount':amount,'confirmed':False,'confirmations':confirmations,'payment_id':payment_id,'recipient':recipient})
def read_map(filename):
log.log_info(logger, "Automatic scaling of EM maps may not be reliable. Please make sure to check your map after using this functionality.")
# CCP4 map file format
# http://www.ccp4.ac.uk/html/maplib.html
with open(filename, "rb") as f:
# 1024 bytes header
header_buf = f.read(1024)
temp_int32 = numpy.frombuffer(header_buf, dtype="int32")
temp_float32 = numpy.frombuffer(header_buf, dtype="float32")
#1 NC # of Columns (fastest changing in map)
#2 NR # of Rows
#3 NS # of Sections (slowest changing in map)
NC = temp_int32[0]
NR = temp_int32[1]
NS = temp_int32[2]
if NC != NR or NR != NS:
log.log_and_raise_error(logger, "Cannot read a map with unequal dimensions")
N = NC
#4 MODE Data type
# 0 = envelope stored as signed bytes (from
# -128 lowest to 127 highest)
# 1 = Image stored as Integer*2
# 2 = Image stored as Reals
# 3 = Transform stored as Complex Integer*2
# 4 = Transform stored as Complex Reals
# 5 == 0
#
# Note: Mode 2 is the normal mode used in
# the CCP4 programs. Other modes than 2 and 0
# may NOT WORK
MODE = temp_int32[3]
dtype = ["int8", "int16", "float32", None, "complex64", "int8"][MODE]
if MODE == 3:
log.log_and_raise_error(logger, "Map file data type \"MODE=%i\" is not implemented yet." % MODE)
if MODE not in [0,1,2,5]:
log.log_warning(logger, "Map file data type \"MODE=%i\" not supported yet and may not work reliably." % MODE)
#11 X length Cell Dimensions (Angstroms)
#12 Y length "
#13 Z length "
dX = temp_float32[10]/float(N)*1E-10
dY = temp_float32[11]/float(N)*1E-10
dZ = temp_float32[12]/float(N)*1E-10
if dX != dY or dY != dZ:
log.log_and_raise_error(logger, "Cannot read a map with unequal voxel dimensions")
#17 MAPC Which axis corresponds to Cols. (1,2,3 for X,Y,Z)
#18 MAPR Which axis corresponds to Rows (1,2,3 for X,Y,Z)
#19 MAPS Which axis corresponds to Sects. (1,2,3 for X,Y,Z)
MAPC = temp_int32[16]
MAPR = temp_int32[17]
MAPS = temp_int32[18]
#24 NSYMBT Number of bytes used for storing symmetry operators
NSYMBT = temp_int32[23]
if NSYMBT > 0:
log.log_and_raise_error(logger, "Omitting symmetry operations in map file.")
f.read(NSYMBT)
# The remaining bytes are data
raw_data = f.read()
raw_data = numpy.frombuffer(raw_data, dtype=dtype)
# Now we need to project onto the right Z-Y-X array grid
S,R,C = numpy.meshgrid(numpy.arange(NS), numpy.arange(NR), numpy.arange(NC), indexing='ij')
S = S.flatten()
R = R.flatten()
C = C.flatten()
if MAPC == 1:
X = C
Xlen = NC
elif MAPC == 2:
Y = C
Ylen = NC
elif MAPC == 3:
Z = C
Zlen = NC
if MAPR == 1:
X = R
Xlen = NR
elif MAPR == 2:
Y = R
Ylen = NR
elif MAPR == 3:
Z = R
Zlen = NR
if MAPS == 1:
X = S
Xlen = NS
elif MAPS == 2:
Y = S
Ylen = NS
elif MAPS == 3:
Z = S
Zlen = NS
i = Z*(Ylen*Xlen) + Y*(Xlen) + X
i.sort()
data = numpy.zeros(Zlen*Ylen*Xlen, dtype=dtype)
data[:] = raw_data[i]
data = data.reshape((Zlen,Ylen,Xlen))
return data, dX
