def clean_buff(self): for loop in range(1, 3): exp_index = self.ssh_handle.expect( [self.str_prompt, pexpect.TIMEOUT, pexpect.EOF], 0.01) if 2 == exp_index: log.log_info('expect EOF. what happened?', 'ERROR') self.ssh_handle.close(force=True)
def get_input(submission_id): '''从数据库获取input并写入data目录下对应的文件''' sql= 'SELECT input FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';' data = db.run_sql(sql) input_data = data[0][0] file_name = str(submission_id) + '.in' try: data_path = os.path.join(config.data_dir, str(submission_id)) secure.low_level() os.mkdir(data_path) except OSError as e: if str(e).find('exist') > 0: # 文件夹已经存在 pass else: return False try: real_path = os.path.join(config.data_dir, str(submission_id), file_name) except KeyError as e: return False try: log.log_info('Importing input, %d' % submission_id) secure.low_level() f = open(real_path, 'w') try: f.write(input_data) except: f.close() log.log_error('Importing input failed, %d' % submission_id) return False f.close() except OSError as e: return False return True
def end_move(self, vector, invert_x_axis, invert_y_axis): """ Given input parameters, moves the robot relative to it's current position. It executes first the x component and then the y component. Keyword Arguments: self -- actuator object the function was called on vector -- movement vector invert_x_axis -- boolean of whether to invert on the x-axis invert_y_axis -- boolean of whether to invert on the y-axis """ if (self.__in_x_movement and vector[0] != 0): self.__in_x_movement = False self.__x_direction = 0 log.log_info("stop x") #X Axis self.__issue_command(self.__x_device, 23, 0, 0, 0, 0) if (self.__in_y_movement and vector[1] != 0): self.__in_y_movement = False self.__y_direction = 0 log.log_info("stop y") #Y Axis self.__issue_command(self.__y_device, 23, 0, 0, 0, 0)
def write(self, data): global _output_to_log interpreter = None if "value" in dir(PythonScriptingInstance._interpreter): interpreter = PythonScriptingInstance._interpreter.value if interpreter is None: if _output_to_log: self.buffer += data while True: i = self.buffer.find('\n') if i == -1: break line = self.buffer[:i] self.buffer = self.buffer[i + 1:] if self.is_error: log.log_error(line) else: log.log_info(line) else: self.orig.write(data) else: PythonScriptingInstance._interpreter.value = None try: if self.is_error: interpreter.instance.error(data) else: interpreter.instance.output(data) finally: PythonScriptingInstance._interpreter.value = interpreter
def setCurrentTopoID(new_topo_id): log_info("Updating topology id: \t%s" % new_topo_id) current = info.find_one() if not current: current = {} current['topoID'] = new_topo_id current['last_updated'] = datetime.utcnow() info.save(current)
def add_spreadsheet_list(user_id, language, language_translation, list_obj): """ uses the json object that was received by google spreadsheet and adds the words to the database :param user: :param language: :param list_obj: :return: """ # create the ID of this insert. time_stamp = int(time.time()) log.log_info("add_spreadsheet_list adding words... ") for row in list_obj: context = row[0] voc = row[1] trans = row[2] text = row[3] if len(voc) > 0: if len(trans) > 0: add_one_word_txt(user_id, language, voc, language_translation, trans, True, context, text, time_stamp) add_one_word_txt(user_id, language_translation, trans, language, voc, False, context, text, time_stamp) return ""
def _parse_payload_config(payload_name: str) -> dict: result = { "payload_name": payload_name, "service": "0", "params": config.AVAILABLE_PARAMS_FOR_PAYLOADS } try: _uuid = utils.uuid.get_uuid() exec(f"import payloads.{payload_name} as {_uuid}") try: service = eval(f"{_uuid}.SERVICE") if type(service) != str: log.log_warn(f"Service is not a string for payload {payload_name}, using default") else: result["service"] = service except AttributeError: log.log_info(f"Using default SERVICE value for payload {payload_name}") try: params = eval(f"{_uuid}.PARAMS") if type(params) != list: log.log_warn(f"Params is not a list for payload {payload_name}, using default") else: result["params"] = params except AttributeError: log.log_info(f"Using default SERVICE value for payload {payload_name}") except ModuleNotFoundError: log.log_error(f"Cannot import file {payload_name}") return result
def send_mail(to_email, email_subject, email_message): log.log_info("in send_mail") FROM = cfg.parameters["email-user"] TO = [to_email] #print(email_message) password = cfg.parameters["email-password"] SUBJECT = "" + email_subject TEXT = email_message message = """From: %s\nTo: %s\nSubject: %s\n\n%s """ % (FROM, ", ".join(TO), SUBJECT, TEXT) # SMTP_SSL Example #server_ssl = smtplib.SMTP_SSL(host="smtp.alexandersickert.com", port=465) server_ssl = smtplib.SMTP_SSL(host="smtp.gmail.com", port=465) server_ssl.ehlo() # optional, called by login() server_ssl.login(FROM, password) # ssl server doesn't support or need tls, so don't call server_ssl.starttls() server_ssl.sendmail(FROM, TO, message) #server_ssl.quit() server_ssl.close() #print( 'successfully sent the mail') log.log_info("end send_mail")
def serve_forever(): listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #listen_socket = ssl.wrap_socket(listen_socket, certfile=cfg.parameters["certfile"], keyfile=cfg.parameters["keyfile"],server_side=True, ssl_version=ssl.PROTOCOL_TLSv1_2) ssl_socket = ssl.wrap_socket(listen_socket, certfile=cfg.parameters["certfile"], keyfile=cfg.parameters["keyfile"], server_side=True) ssl_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ssl_socket.bind(SERVER_ADDRESS) ssl_socket.listen(REQUEST_QUEUE_SIZE) executor = ThreadPoolExecutor(max_workers=50) log.log_info('MASTER Serving HTTPS on port {port} ...'.format(port=PORT)) while True: #client_connection, client_address = listen_socket.accept() try: client_connection, client_address = ssl_socket.accept() #log.log_info("client_user_address: " + str( client_address) ) #traffic.track(client_address[0], True) a = executor.submit(handle_request, client_connection, str(client_address[0]), client_address[1]) except: print("Unexpected error:", sys.exc_info()[0]) log.log_error("Error in try-catch of main server loop: " + str(sys.exc_info()[0])) if "KeyboardInterrupt" in str(sys.exc_info()[0]): exit()
def register_user_and_session_at_master(session, user): # this is so that the master knows where to forward the request # cfg.slave_id # url = cfg.parameters["master-url"] + ":" + cfg.parameters["https"] + "/register-slave?security-key=YI64QZ8LMPJET0GV4CCV&" # url += "register-session=" + session # url += "&server=" + cfg.slave_id # url += "®ister-user="******"calling MASTER via URL " + url) # r = urllib.request.urlopen(url) # log.log_info("MASTER responds " + str(r.read())) conn = http.client.HTTPSConnection( cfg.parameters["master-url"], cfg.parameters["https"], context=ssl._create_unverified_context()) query = "/register-slave?security-key=YI64QZ8LMPJET0GV4CCV&" query += "register-session=" + session query += "&slave-id=" + cfg.slave_id query += "®ister-user="******"MASTER responds " + str(r.read()))
def write(self, data): interpreter = None if hasattr(PythonScriptingInstance._interpreter, "value"): interpreter = PythonScriptingInstance._interpreter.value if interpreter is None: if log.is_output_redirected_to_log(): self.buffer += data while True: i = self.buffer.find('\n') if i == -1: break line = self.buffer[:i] self.buffer = self.buffer[i + 1:] if self.is_error: log.log_error(line) else: log.log_info(line) else: self.orig.write(data) else: PythonScriptingInstance._interpreter.value = None try: if self.is_error: interpreter.instance.error(data) else: interpreter.instance.output(data) finally: PythonScriptingInstance._interpreter.value = interpreter
def send_move_request(self, direction): try: self._conn.request("ON", self._solenoid_number[direction]) response = self._conn.getresponse() log.log_info(response.read()) except socket_error as serr: log.log_error("Failed communication with HTTP server.")
def update(): # Download new NaPTAN data download_naptan_data() # Download new json data bustrackerApi.scrapeStopsFromServer() bustrackerApi.scrapeDestsFromServer() bustrackerApi.scrapeServicesFromServer() # Clear the database bustrackerDb.clear_database() # Update services services = bustrackerApi.getServices() bustrackerDb.populateServices(services) # Update stops apiStops = bustrackerApi.getStops() bustrackerDb.populateStops(apiStops) # Update destinations dests = bustrackerApi.getDests() bustrackerDb.populateDests(dests) # Updating database topology id bustrackerDb.setCurrentTopoID(api_topo_id) log.log_info("Update complete: \tExiting") log.email_log()
def finished(d): uploading_file = d["uploading_file"] file_id = fm.generate_id() result = fm.put_inference_zip(uploading_file, file_id, is_expanding=True) log_info(result) if result["status"] == "success": image_path = result["image_path"] model_id = d["model_id"] recipe_id = d["recipe_id"] model = CNN(recipe_id) inference_type = d["inference_type"] if inference_type == "classification": inference_res = model.classify(model_id, image_path) action = "finishClassfication" elif inference_type == "vectorization": inference_res = model.vectorize(model_id, image_path, is_similarity=True) action = "finishVectorization" res = fm.get_inferece_images(image_path) image_list = res["list"] res_list = [] for r, i in zip(inference_res, image_list): r["body"] = i["body"] res_list.append(r) res = {"action": action, "id": file_id, "list": res_list} return res
def process_json_master(json_string): """ This is used by the Google Spreadsheet upload The master server needs to know to which slave to forward the message. We need to look up the server :param json_string: :return: """ jo = json.loads(json_string) #user_id = sec.get_user_id(jo["user"]) # this is the email address ip, port = sec.get_slave_ip_port(jo["user"]) if port > 0: ret_obj = slave_request.get_from_slave(ip, port, jo) log.log_info("Spreadsheet upload reponse from slave: " + str(ret_obj)) else: # wrong login coordinates ret_obj = {} ret_obj["error"] = True ret_obj["function"] = jo["function"] ret_obj["error_description"] = "could not find user" log.log_info("process_json_master(json_string): " + str(ret_obj)) return json.dumps(ret_obj)
def fill_cache_from_db(user_id): """ copies from db into the cache :param user_id: :return: """ log.log_info("in fill_cache_from_db(user_id)") conn = dba.get_connection() cur = conn.cursor() sql = """ SELECT id, COALESCE(short_memory_position, 1) FROM vocabulary WHERE user_id = %s and current = true and COALESCE(last_studied, 0) > 1 order by COALESCE(short_memory_position, 1) LIMIT 100 """ cur.execute(sql, (user_id, )) arr = cur.fetchall() log.log_info("in fill_cache_from_db(user_id) - array has length " + str(len(arr))) for row in arr: add_to_cache(user_id, row[0], row[1])
def send_command(self, cmd, timeout=6, ignore_fail = 0, retry_on = 0): buff = '' result = 1 self.ssh_handle.sendline(cmd) for count in range(1, 10): if retry_on == 1 and count > 1: self.ssh_handle.sendline(cmd) exp_index = self.ssh_handle.expect([ self.str_prompt, #0 pexpect.EOF, #1 pexpect.TIMEOUT #2 ], timeout) buff += self.ssh_handle.before self.clean_buff() if exp_index == 0: result = 0 break elif exp_index == 1: log.log_info('command: %s failed. (EOF)' %(cmd), 'ERROR') break elif exp_index == 2: log.log_info('command: %s failed, wait: %s. (TIMEOUT)' %(cmd, count), 'WARNING') time.sleep(timeout) if result == 1: log.log_info('Send %s failed, resend times %d(timeout=%d)' %(cmd, count, timeout), 'ERROR') if ignore_fail == 0: self.ssh_handle.close(force = True) sys.exit() else: return False if not buff: log.log_info('command: %s [NO BUFF]' %(cmd), 'WARNING') #still return, handled by users return False log.log_info('Send [%s] command ok' %(cmd), 'DUBUG') return buff
def create_hit_table(self): for i in range(0, self.mysql_split): table = "%s_%04d%02d%02d" % (self.mysql_table, self.today.year, self.today.month, self.today.day) if not self.raw_handle.check_table_exists(table): self.raw_cur.execute("CREATE TABLE %s(id INT PRIMARY KEY AUTO_INCREMENT, number INT, \ last_update TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL DEFAULT NOW())" % table) self.raw_cur.execute("ALTER TABLE %s add index my_id(number)" % table) log.log_info("create table %s" % table, "DEBUG")
def get_page(url): res = requests.get(url, headers=headers) if res: log.log_info('fetch:访问' + url) page = res.content.decode('utf-8') return page, url else: log.log_error('fetch:访问' + url + '失败')
def move(self, vector, invert_x_axis, invert_y_axis): """ Given input parameters, moves the robot relative to it's current position. It executes first the x component and then the y component. Keyword Arguments: self -- actuator object the function was called on vector -- movement vector invert_x_axis -- boolean of whether to invert on the x-axis invert_y_axis -- boolean of whether to invert on the y-axis """ vector = [x * self.__variable_step_size for x in vector] if (vector[0] != 0 and self.__in_x_movement and self.__x_direction != 0): temp = -1 if vector[0] < 0 else 1 if temp != self.__x_direction: self.end_move([temp, 0], invert_x_axis, invert_y_axis) if (not self.__in_x_movement and vector[0] != 0): self.__in_x_movement = True #X Axis if invert_x_axis: vector[0] *= -1 log.log_info("start x " + str(vector[0])) self.__x_direction = -1 if vector[0] < 0 else 1 byte_array = _convert_int_to_bytes(vector[0]) self.__issue_command(self.__x_device, 22, byte_array[0], byte_array[1], byte_array[2], byte_array[3]) if (vector[1] != 0 and self.__in_y_movement and self.__y_direction != 0): temp = -1 if vector[1] < 0 else 1 if temp != self.__y_direction: self.end_move([0, temp], invert_x_axis, invert_y_axis) if (not self.__in_y_movement and vector[1] != 0): self.__in_y_movement = True #Y Axis if invert_y_axis: vector[1] *= -1 log.log_info("start y " + str(vector[1])) self.__y_direction = -1 if vector[1] < 0 else 1 byte_array = _convert_int_to_bytes(vector[1]) self.__issue_command(self.__y_device, 22, byte_array[0], byte_array[1], byte_array[2], byte_array[3])
def get_old_by_score(user_id): """ take one word from the pile of words we already learned :param user_id: :return: """ log.log_info(" in get_old_by_score(user_id) ") conn = dba.get_connection() cur = conn.cursor() # age of old word needs to be at least from yesterday delta = 24 * 60 * 60 threshold = int(time.time()) - delta sql = """ SELECT id, calc_rank FROM vocabulary WHERE user_id = %s AND current = false AND count_positive > 0 AND last_studied < %s ORDER BY /* changed following line on 16th feb 2019 because yes = 1 and we need to know failures so asc and desc*/ /* calc_rank desc */ calc_rank asc LIMIT 2; """ cur.execute(sql, (user_id, threshold)) #l = cur.fetchall()[0][0] arr = cur.fetchall() #print(arr) if len(arr) > 0: i = int(random.random() * float(len(arr))) l = arr[i][0] set_experiment_state(l, True) else: l = -1 log.log_info("get_old_by_score returns " + str(l) + " for user id " + str(user_id)) return l
def __confirm_maze_frame(self, item): """ Confirm the corners of the maze that have been set up """ log.log_info("Navigating maze...") # Cleanup maze window maze_window = self.__builder.get_object("maze_window") maze_window.hide() self.__navigate_maze(item)
def set_position_in_db(word_id, pos): log.log_info("set_position " + str(word_id) + " to " + str(pos)) sql = "UPDATE vocabulary SET short_memory_position = %s WHERE id = %s " conn = dba.get_connection() cur = conn.cursor() cur.execute(sql, (pos, word_id)) conn.commit()
def connect_to_redis(host,port): log_info('Connecting to Redis at %s:%u' % (host, port)) try: global redisdb redisdb = redis.Redis(host=host,port=port) return redisdb except Exception, e: log_error( 'Error initializing redis: %s' % str(e)) exit()
def connect_to_redis(host, port): log_info('Connecting to Redis at %s:%u' % (host, port)) try: global redisdb redisdb = redis.Redis(host=host, port=port) return redisdb except Exception, e: log_error('Error initializing redis: %s' % str(e)) exit()
def add_new_word(user_id): """ Decide if we want to add a word from the pile of new words or from the pile of old words that need repetition The decision depends on the percentage fo successfully leanrned words. If the predictions says that more than XX% of the words are not forgotten, then we add a new word. if the ratio is 0.0 then we add new words because the ration could not be calculated. but if the ratio is between 0.05 and 0.8 then we repeat old words first. if the ratio is > 0.85 then we add new words :return: """ parameter_name = "percentage_learned" conn = dba.get_connection() cur = conn.cursor() sql = "SELECT value FROM parameters WHERE user_id = %s AND key = %s" cur.execute(sql, (user_id, parameter_name)) arr = cur.fetchall() ratio_learned = -1.0 # default value #print(user_id) #print(parameter_name) #print(arr) if arr is not None: if len(arr) > 0: ratio_learned = float(arr[0][0]) log.log_info("add_new_word(user_id) " + str(user_id) + " ratio learned is " + str(ratio_learned)) r = random.random() if ratio_learned < 0.0 or ratio_learned > 0.99: # this happens when not enough words were so far learned and parameter is not filled # generally we would like to add only new words but we also need to start doing experiments if r > 0.5: return True else: return False elif ratio_learned < 0.05: # ToDo this is probably an error and needs to be removed return True elif ratio_learned > 0.85: # we know most of the words and make rarely experiments if r > 0.9: # only in 10 percent we make experiments return False else: return True else: if r > (ratio_learned * 0.8): # if 70% learned, then we make 30% experiments return False else: return True
def read_article(self, url): print "reading url '%s'" % url infile = self.opener.open(url) page = infile.read() log.log_info("read page length: " + str(page.__len__())) log.log("page beings with: " + page[0:256]) # xmldoc = minidom.parseString(page) #print ("xml length: " + str(xmldoc.__len__())) infile.close() return page
def __init__(self, config_file="config.ini"): log.log_info("read config %s" % (config_file), "DEBUG") self.config = ConfigParser.ConfigParser() self.config.read(config_file) self.mysql_user = self.config.get('mysql', 'MYSQL_USER') self.mysql_pwd = self.config.get('mysql', 'MYSQL_PWD') self.mysql_db = self.config.get('mysql', 'MYSQL_DB') self.mysql_table = self.config.get('mysql', 'MYSQL_TABLE') self.mysql_ip = self.config.get('mysql', 'MYSQL_IP') self.mysql_port = int(self.config.get('mysql', 'MYSQL_PORT'))
def update_password(u, p): conn = dba.get_connection() cur = conn.cursor() sql = "UPDATE users SET password = %s WHERE email = %s;" hash_object = hashlib.md5(p.encode()) p_hash = hash_object.hexdigest() log.log_info("p=" + p) log.log_info("p_hash=" + p_hash) cur.execute(sql, (p_hash, u)) conn.commit()
def register_user(u, p): conn = dba.get_connection() cur = conn.cursor() sql = "INSERT INTO users (email, password) VALUES (%s, %s);" hash_object = hashlib.md5(p.encode()) p_hash = hash_object.hexdigest() log.log_info("p=" + p) log.log_info("p_hash=" + p_hash) cur.execute(sql, (u, p_hash)) conn.commit()
def toggle_adc(self): """ Returns the new value of toggling adc """ try: self._conn.request("ADC", self.__TOGGLE_ADC) response = self._conn.getresponse() log.log_info(response) return response.read() == "true" except socket_error as serr: log.log_error("Failed communication with HTTP server.")
def getTipo(self): try: if self.getTp==-1: return self.getTp else: getTp = int(self.instrument.read_register(0)) print "NUMSEM", getTp self.getTp = getTp return getTp except IOError as e: log_info("I/O error({0}): {1}".format(e.errno, e.strerror) + "id:" +str(self.address), self, "getTipo")
def get_random_slave_ip_port(): global slaves_arr log.log_info("length of slaves_arr: " + str(len(slaves_arr))) i = random.randint(0, len(slaves) - 1) ip = slaves_arr[i][1] port = slaves_arr[i][2] return ip, int(port)
def populateStops(new_stops): services_dict = {service['ref']: service for service in getServicesDB()} log_info("Populating stops \t(%i)" % len(new_stops)) for stop in new_stops: stop['heading'] = getHeadingForStopCap(stop['cap']) stop = addNaptanDataToStop(stop) stop = add_service_mnemos_to_stop(stop, services_dict) if not 'street' in stop: stop = add_street_name_to_stop(stop) stops.insert(new_stops) incomplete_count = len(list(find_incomplete_stops())) log_info("Stops without complete information: \t%i" % incomplete_count)
def populateStops(new_stops): services_dict = {service['ref']:service for service in getServicesDB()} log_info("Populating stops \t(%i)" % len(new_stops)) for stop in new_stops: stop['heading'] = getHeadingForStopCap(stop['cap']) stop = addNaptanDataToStop(stop) stop = add_service_mnemos_to_stop(stop, services_dict) if not'street' in stop: stop = add_street_name_to_stop(stop) stops.insert(new_stops) incomplete_count = len(list(find_incomplete_stops())) log_info("Stops without complete information: \t%i" % incomplete_count)
def getNumSem(self): try: if self.numsem==-1: numsem = int(self.instrument.read_register(6)) print "NUMSEM", numsem self.numsem = numsem return numsem else: return self.numsem except IOError as e: log_info("I/O error({0}): {1}".format(e.errno, e.strerror) + "id:" +str(self.address), self, "getNumSem")
def send_commands(self, cmds, timeout = 6, interval = 0.01, ignore_fail = 0, retry_on = 0): buff = '' for cmd in cmds.split('\n'): if not cmd: continue cmd = cmd.strip('\t') buff += self.send_command(cmd, timeout, ignore_fail, retry_on) time.sleep(interval) if not buff: log.log_info('command: %s [NO BUFF]' %(cmds), 'WARNING') #still return, handled by users return False return buff
def step_change(self, new_value, increment): if new_value != None and new_value >= 0: log.log_info("Changing step size to " + str(new_value)) self.__variable_step_size = new_value if increment != None: temp = self.__variable_step_size + increment if temp < 0: temp = 0 log.log_info("Changing step size from " + str(self.__variable_step_size) + " to " + str(temp)) self.__variable_step_size = temp
def get_next_word_id_array(user_id, last_word_id): """ NOTE: consider also the funciton get_next_word_id(user_id, last_word_id) because it is similar !!! :param user_id: :param last_word_id: :return: """ # first we ensure there are enough current words if count_current(user_id) < 7: log.log_info("get_next_word_id - count_current less than 7") # we need to add a new word question is if repeat old word or use new while count_current(user_id) < 7: if add_new_word(user_id): # ToDo: if there are no new words left, then we need to process old words and send info to user, # that there are no new words left if count_not_learned(user_id) > 0: word_id = get_new_random(user_id) else: # if there are no new words left then use old words word_id = get_old_by_score(user_id) else: # we can add old only if such words exist # if not, then we add from new category num_learned = count_learned(user_id) if num_learned > 0: word_id = get_old_by_score(user_id) else: word_id = get_new_random(user_id) set_word_current(word_id) # now we load all the current words as an array ret = [] conn = dba.get_connection() cur = conn.cursor() cur.execute( "SELECT ID FROM vocabulary where user_id = %s AND current = true ORDER BY random() LIMIT 20", (user_id, )) l = cur.fetchall() for id in l: ret.append(id[0]) return ret
def download_naptan_data(): # Download the NaPTAN archive log_info("Downloading NaPTAN data") naptan_url = "http://www.dft.gov.uk/NaPTAN/snapshot/NaPTANcsv.zip" naptan_archive_path = "%sNaPTANcsv.zip" % csv_path storeUrlContentsAtPath(naptan_url, naptan_archive_path) # Extract the Stops.csv file log_info("Extracting NaPTAN Stops data") stops_filename = 'Stops.csv' command = "unzip %s %s -d %s" % (naptan_archive_path, stops_filename, csv_path) call(command.split()) # Find rows corresponding to stops we're interested in log_info("Extracting useful rows") stops_path = "%s%s" % (csv_path,stops_filename) stops_file = open(stops_path,'r') stops_csv_reader = reader(stops_file) useful_rows = [] for row in stops_csv_reader: # add if the ACTOcode starts with one of our areas for area in naptan_areas: if row[0].startswith(str(area)): useful_rows.append(row) # Write useful rows to file log_info("Storing useful rows") naptan_path = '%sNaPTAN.csv' % csv_path naptan_file = open(naptan_path,'w') naptan_csv_writer = writer(naptan_file) naptan_csv_writer.writerows(useful_rows) # Remove the zip archive and Stops csv file call(['rm',naptan_archive_path,stops_path])
def set_word_current(word_id): """ :param word_id: :return: """ log.log_info("set_word_current " + str(word_id)) sql = "UPDATE vocabulary SET current = TRUE, short_memory_position = 1 WHERE id = %s " conn = dba.get_connection() cur = conn.cursor() cur.execute(sql, (word_id, )) conn.commit()
def get_simple_report(user_id): log.log_info("get_simple_report(user_id): for user " + str(user_id)) conn = dba.get_connection() cur = conn.cursor() cur.execute( "SELECT count(*) FROM vocabulary where user_id = %s AND current = false AND count_positive < 1 AND direction = TRUE", (user_id, )) new_words = cur.fetchall()[0][0] cur.execute( "SELECT count(*) FROM vocabulary where user_id = %s AND current = false AND count_positive > 0 AND direction = TRUE", (user_id, )) learned_words = cur.fetchall()[0][0] parameter_name = "percentage_learned" sql = "SELECT value FROM parameters WHERE user_id = %s AND key = %s" cur.execute(sql, (user_id, parameter_name)) try: arr = cur.fetchall() p = arr[0][0] log.log_info("value from rs is: " + str(p)) except: log.log_info("value rs is not existing: ") p = 0.0 ratio_learned = float(p) * 100 ratio_learned = round(ratio_learned, 2) log.log_info("ratio_learned at end is is: " + str(ratio_learned)) return new_words, learned_words, ratio_learned
def check_login(u, p): conn = dba.get_connection() cur = conn.cursor() hash_object = hashlib.md5(p.encode()) p_hash = hash_object.hexdigest() log.log_info("p=" + p) log.log_info("p_hash=" + p_hash) cur.execute( "SELECT count(*) FROM users WHERE email = %s AND password = %s", (u, p_hash)) ret = cur.fetchall()[0][0] print(ret) return ret
def count_current(user_id): """ Counts how many words are in the carousel for a specific user. :param user_id: :return: """ conn = dba.get_connection() cur = conn.cursor() cur.execute( "SELECT count(*) FROM vocabulary where user_id = %s AND current = true", (user_id, )) l = cur.fetchall()[0][0] log.log_info("count_current is " + str(l)) return int(l)
def parse(self): print "parsing XML" log.log("parse_article(" + str(self) + "," + str(self.xmldoc) + ")") for title in self.xmldoc.getElementsByTagName("title"): log.log_info("title is: " + title.firstChild.nodeValue) # TODO: should assert only one vcard or something, i guess... vcard = self.get_vcards(self.xmldoc).next() # find 'Born' values in vcard print "Calling 'parse_class'" items = {} itemsOfIterest = ["Born", "Died"] for tr in vcard.getElementsByTagName("tr"): item = self.parse_class(tr) if (item.name in itemsOfIterest): items[item.name] = item return items
def sendReadings(nextRun): if datetime.now() >= nextRun: #turn on the sensors # the transisotr enable the sensors to be powered sys_gpio.setTransistor(True) log.current_state = "Reading Air Temperature" air_temp = DHT11TemperatureSensor.read() log.current_state = "Posting Air Temperature" httpclient.post(api.__send_readings__, air_temp, auth.checkResponse) scn.update(env_temp=str(air_temp['reading'])) log.current_state = "Reading Air Humidity" air_hum = DHT11HumiditySensor.read() log.current_state = "Posting Air Humidity" httpclient.post(api.__send_readings__, air_hum, auth.checkResponse) scn.update(env_hum=str(air_hum['reading'])) log.current_state = "Reading Luminosity" air_lum = LDR.read() log.current_state = "Posting Luminosity" httpclient.post(api.__send_readings__, air_lum, auth.checkResponse) scn.update(env_lum="{0:0.2f}".format(air_lum['reading'])) log.current_state = "Reading Soil Temperature" soil_temp = DS18B20TemperatureSensor.read() log.current_state = "Reading Soil Temperature" httpclient.post(api.__send_readings__, soil_temp, auth.checkResponse) scn.update(soil_temp="{0:0.2f}".format(soil_temp['reading'])) sys_gpio.setTransistor(False) #print air_lum["date"] #print "Env: Temp.: {0:0.2f}C Humidity: {1:0.2f}% & Luminosity:{2:0.2f}%".format(air_temp['reading'], air_hum['reading'], air_lum['reading']) #print "Soil: Temp.: {0:0.2f}C & Moisture: 0%".format(soil_temp['reading']) #print "\n" log.log_info("System Ok") nextRun = datetime.now() + timedelta(seconds=60) return nextRun
def put_task_into_queue(): '''扫描数据库,将任务添加到队列''' log.log_info('Scaning DB...') while True: init.queue.join() # 阻塞程序 sql = 'SELECT submission_id FROM ' + config.db_table_name + ' WHERE status = ' + status.QUEUING + ';' data = db.run_sql(sql) num = len(data) if num > 0: log.log_info('%d submission(s) coming...' % num) for i in range(num): time.sleep(1) submission_id = data[i][0] init.dblock.acquire() ret = get_data.get_code(submission_id) and get_data.get_input(submission_id) init.dblock.release() if ret == False: init.dblock.acquire() update.update_status(submission_id, status.GET_DATA_ERROR) init.dblock.release() continue inner_sql = 'SELECT language, time, memory FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';' #print inner_sql inner_data = db.run_sql(inner_sql) language_id = inner_data[i][0] Time = inner_data[i][1] memory = inner_data[i][2] task = { 'submission_id': submission_id, 'language_id': language_id, 'time': Time, 'memory': memory } init.queue.put(task) time.sleep(1)
def pwm_change(self, new_value, increment): """ Adjusts the PWM for the solenoids Keyword Arguments: increment -- whether the speed is increasing (1) or decreasing (-1) """ if increment != None: if increment==1: log.log_info("Incrementing PWM voltage.") else: log.log_info("Decrementing PWM voltage.") try: self._conn.request("PWM", self._DECREMENT if increment == -1 else self._INCREMENT) response = self._conn.getresponse() #log.log_info(response.read()) except socket_error as serr: log.log_error("Failed communication with HTTP server.")
def end_move(self, vector, invert_x_axis, invert_y_axis): """ Given input parameters, deactivates the specified solenoid Keyword Arguments: direciton -- direction of movement invert_x_axis -- boolean of whether to invert on the x-axis invert_y_axis -- boolean of whether to invert on the y-axis """ direction = "" if invert_x_axis: vector[0] *= -1 if invert_y_axis: vector[1] *= -1 if (vector[0] != 0): if (vector[0] > 0): if (self.__x_movement == 2): self.__x_movement = 0 direction = self._RIGHT else: if (self.__x_movement == 1): self.__x_movement = 0 direction = self._LEFT elif (vector[1] != 0): self.__in_y_movement = False if (vector[1] > 0): if (self.__y_movement == 1): self.__y_movement = 0 direction = self._UP else: if (self.__y_movement == 2): self.__y_movement = 0 direction = self._DOWN try: self._conn.request("OFF", self._solenoid_number[direction]) response = self._conn.getresponse() log.log_info(response.read()) except socket_error as serr: log.log_error("Failed communication with HTTP server.")
def __init__(self, host, username, password, log_mode=2): self.set_prompt('\# ') self.ssh_handle = pexpect.spawn('ssh ' + username + '@' + host) for loop in range(1,8): exp_index = self.ssh_handle.expect ([ self.str_prompt, # 0 expect prompt pexpect.TIMEOUT, # 1 pexpect.EOF, # 2 'yes/no', # 3 If added new line here, please notice 'password', # 4 the index is used in the if elif lines. 'password: '******'Permission denied, please try again.', # 6 'Connection closed by UNKNOWN' # 7 ], timeout=6) if 0 == exp_index: if 1 == log_mode: self.ssh_handle.logfile = sys.stdout elif 2 == log_mode: self.ssh_handle.logfile_read = sys.stdout elif 3 == log_mode: fout = file('mylog.txt','w') self.ssh_handle.logfile = fout elif 4 == log_mode: fout = file('mylog.txt','w') self.ssh_handle.logfile_read = fout self.clean_buff() log.log_info('ssh connect successfully.') break elif 1 == exp_index: pass elif 3 == exp_index: self.ssh_handle.sendline('yes') elif 4 == exp_index or 5 == exp_index: self.ssh_handle.sendline(password) elif 6 == exp_index: pass else: #EOF or others log.log_info('ssh connect failed.', 'ERROR') self.ssh_handle.close(force=True)
def get_code(submission_id): '''从数据库获取代码并写入work目录下对应的文件''' sql = 'SELECT code, language FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';' data = db.run_sql(sql) if data != False: code = data[0][0] language_id = data[0][1] file_name = language.get_filename(language_id) else: return False try: work_path = os.path.join(config.work_dir, str(submission_id)) secure.low_level() os.mkdir(work_path) except OSError as e: if str(e).find('exist') > 0: # 文件夹已经存在 pass else: return False try: real_path = os.path.join(config.work_dir, str(submission_id), file_name) except KeyError as e: # print e return False try: log.log_info('Importing code, %d' % submission_id) secure.low_level() f = open(real_path, 'w') try: f.write(code) except: f.close() log.log_error('Importing code failed, %d' % submission_id) return False f.close() except OSError as e: return False return True
def SendMany(wallet_host,wallet_port,recipients,paymentid=None,mixin=None): for address in recipients: if not IsValidAddress(address): log_error("Invalid address: %s" % address) return amount = recipients[address] if amount <= 0: log_error("Invalid amount: %s" % str(amount)) return if paymentid != None: if not IsValidPaymentID(paymentid): log_error("Invalid payment ID") return if mixin < 0: log_error("Invalid mixin: %d") return log_info("Pay: sending %s, payment id %s, mixin %d" % (str(recipients), str(paymentid),mixin)) try: destinations=[] for address in recipients: destinations.append({'address':address,'amount':recipients[address]}) params = { 'destinations':destinations, 'payment_id': paymentid, 'fee': 0, 'mixin': mixin, 'unlock_time': 0, } j = SendJSONRPCCommand(wallet_host,wallet_port,"transfer",params) except Exception,e: log_error('Withdraw: Error in transfer: %s' % str(e)) return
def before_run(submission_id, time, memory, language_id): secure.low_level() '''获取程序执行时间和内存''' if secure.check_dangerous_code(submission_id, language_id) is False: update.update_status(submission_id, status.DANGEROUS_CODE) log.log_error('Has DANGEROUS code, %d' % submission_id) return False update.update_status(submission_id, status.COMPILEING) compile_result = compiler.compile(submission_id, language_id) if compile_result is False: # 编译错误 update.update_status(submission_id, status.COMPILATION_ERROR) log.log_error('Compile failed, %d' % submission_id) return False log.log_info('Compile completed, %d' % submission_id) result = run( solution_id, time_limit, mem_limit, program_info, language_id) return result
def worker(): '''工作线程,循环扫描队列,获得评判任务并执行''' while True: if init.queue.empty() is True: # 队列为空,空闲 continue task = init.queue.get() # 获取任务,如果队列为空则阻塞 submission_id = task['submission_id'] language_id = task['language_id'] time = task['time'] memory = task['memory'] log.log_info('Task input, %d' % submission_id) log.log_info('Start judging, %d' % submission_id) program_info = judge.judge(submission_id, time, memory, language_id) if program_info not False: log.log_info('Judging complete, %d' % submission_id) init.dblock.acquire() update.write_result(submission_id, program_info) # 将结果写入数据库 update.write_output(submission_id) init.dblock.release() log.log_info('Update INFO complete, %d' % submission_id) else: log.log_error('Running Error, %d' % submission_id) init.dblock.acquire() update.write_result(submission_id, config.COMPILATION_ERROR) # 将结果写入数据库 update.write_output(submission_id) init.dblock.release() if config.auto_clean_work: # 清理work目录 clean.clean_work_dir(submission_id) if config.auto_clean_data: clean.clean_data_dir(submission_id) init.queue.task_done() # 一个任务完成
def parse_subnetworklist(): inputf = file(input, 'r') icnt = 0 ocnt = 0 subnetworklist = [] while True: line = inputf.readline() if len(line) == 0: break icnt += 1 range = parse_line(line) if len(range) == 2: if range[0] > range[1]: log.log_info("%s err, %d=>%d" % (line, range[0], range[1])) continue subnetworklist.append(range) ocnt += 1 log.log_info("%s=>%d:%d" % (line, range[0], range[1])) inputf.close() log.log_info("deal %d/%d records from %s" % (ocnt, icnt, input)) output(subnetworklist)
new_payments = [] n_confirming = 0 new_scan_block_height = scan_block_height for p in payments: payment_id=p["payment_id"] tx_hash = p["tx_hash"] bh = p["block_height"] ut = p["block_height"] amount=p["amount"] if not full_history and redis_sismember("processed_txs",tx_hash): continue log_log('UpdateCoin: Looking at payment %s' % str(p)) confirmations = height-1-bh confirmations_needed = max(config_confirmations,ut-height) if confirmations >= confirmations_needed: log_info('Payment %s is now confirmed' % str(p)) new_payments.append(p) if new_scan_block_height and bh > new_scan_block_height: new_scan_block_height = bh else: log_info('Payment %s has %d/%d confirmations' % (str(p),confirmations,confirmations_needed)) n_confirming += 1 new_scan_block_height = None try: recipient = GetUserIDFromPaymentID(payment_id) if not recipient: raise RuntimeError('Payment ID %s not found' % payment_id) log_info('UpdateCoin: Found payment %s to %s for %s' % (tx_hash,recipient, AmountToString(amount))) if not full_history: cp.hincrby('confirming_payments',recipient,amount) txs.append({'tx_hash':tx_hash,'amount':amount,'confirmed':False,'confirmations':confirmations,'payment_id':payment_id,'recipient':recipient})
def read_map(filename): log.log_info(logger, "Automatic scaling of EM maps may not be reliable. Please make sure to check your map after using this functionality.") # CCP4 map file format # http://www.ccp4.ac.uk/html/maplib.html with open(filename, "rb") as f: # 1024 bytes header header_buf = f.read(1024) temp_int32 = numpy.frombuffer(header_buf, dtype="int32") temp_float32 = numpy.frombuffer(header_buf, dtype="float32") #1 NC # of Columns (fastest changing in map) #2 NR # of Rows #3 NS # of Sections (slowest changing in map) NC = temp_int32[0] NR = temp_int32[1] NS = temp_int32[2] if NC != NR or NR != NS: log.log_and_raise_error(logger, "Cannot read a map with unequal dimensions") N = NC #4 MODE Data type # 0 = envelope stored as signed bytes (from # -128 lowest to 127 highest) # 1 = Image stored as Integer*2 # 2 = Image stored as Reals # 3 = Transform stored as Complex Integer*2 # 4 = Transform stored as Complex Reals # 5 == 0 # # Note: Mode 2 is the normal mode used in # the CCP4 programs. Other modes than 2 and 0 # may NOT WORK MODE = temp_int32[3] dtype = ["int8", "int16", "float32", None, "complex64", "int8"][MODE] if MODE == 3: log.log_and_raise_error(logger, "Map file data type \"MODE=%i\" is not implemented yet." % MODE) if MODE not in [0,1,2,5]: log.log_warning(logger, "Map file data type \"MODE=%i\" not supported yet and may not work reliably." % MODE) #11 X length Cell Dimensions (Angstroms) #12 Y length " #13 Z length " dX = temp_float32[10]/float(N)*1E-10 dY = temp_float32[11]/float(N)*1E-10 dZ = temp_float32[12]/float(N)*1E-10 if dX != dY or dY != dZ: log.log_and_raise_error(logger, "Cannot read a map with unequal voxel dimensions") #17 MAPC Which axis corresponds to Cols. (1,2,3 for X,Y,Z) #18 MAPR Which axis corresponds to Rows (1,2,3 for X,Y,Z) #19 MAPS Which axis corresponds to Sects. (1,2,3 for X,Y,Z) MAPC = temp_int32[16] MAPR = temp_int32[17] MAPS = temp_int32[18] #24 NSYMBT Number of bytes used for storing symmetry operators NSYMBT = temp_int32[23] if NSYMBT > 0: log.log_and_raise_error(logger, "Omitting symmetry operations in map file.") f.read(NSYMBT) # The remaining bytes are data raw_data = f.read() raw_data = numpy.frombuffer(raw_data, dtype=dtype) # Now we need to project onto the right Z-Y-X array grid S,R,C = numpy.meshgrid(numpy.arange(NS), numpy.arange(NR), numpy.arange(NC), indexing='ij') S = S.flatten() R = R.flatten() C = C.flatten() if MAPC == 1: X = C Xlen = NC elif MAPC == 2: Y = C Ylen = NC elif MAPC == 3: Z = C Zlen = NC if MAPR == 1: X = R Xlen = NR elif MAPR == 2: Y = R Ylen = NR elif MAPR == 3: Z = R Zlen = NR if MAPS == 1: X = S Xlen = NS elif MAPS == 2: Y = S Ylen = NS elif MAPS == 3: Z = S Zlen = NS i = Z*(Ylen*Xlen) + Y*(Xlen) + X i.sort() data = numpy.zeros(Zlen*Ylen*Xlen, dtype=dtype) data[:] = raw_data[i] data = data.reshape((Zlen,Ylen,Xlen)) return data, dX