def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ keys = message.keys() if 'info' in keys: Logger.info(self.__class__.__name__, message['info']) elif 'subscribe' in keys: Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \ (message['request']['args'], \ 'successful' if message['success'] else 'failed')) elif 'table' in keys: if message['table'] == 'trade': for trade_raw in message['data']: if trade_raw["symbol"] == instmt.get_instmt_code(): # Filter out the initial subscriptions trade = self.api_socket.parse_trade(instmt, trade_raw) if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) elif message['table'] == 'orderBookL2': l2_depth = self.api_socket.parse_l2_depth(instmt, message) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) else: Logger.info(self.__class__.__name__, json.dumps(message,indent=2)) else: Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ ExchGwQuoine.num_of_connections_lock.acquire() ExchGwQuoine.num_of_connections += 1 Logger.info( self.__class__.__name__, "Current number of connections = %d" % ExchGwQuoine.num_of_connections) ExchGwQuoine.num_of_connections_lock.release() instmt.set_order_book_id(self.get_order_book_init(instmt)) while True: try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff( instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) ExchGwQuoine.num_of_connections_lock.acquire() time.sleep(ExchGwQuoine.num_of_connections + ExchGwQuoine.extra_waiting_sec) ExchGwQuoine.num_of_connections_lock.release()
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ trade_id, last_exch_trade_id = self.get_trades_init(instmt) instmt.set_trade_id(trade_id) instmt.set_exch_trade_id(last_exch_trade_id) while True: try: ret = self.api_socket.get_trades(instmt) for trade in ret: instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s\nReturn: %s" % (e, ret)) time.sleep(0.5)
def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ if 'ping' in message: #handle ping response ts = message['ping'] self.api_socket.send(json.dumps({'pong': ts})) elif 'ch' in message: if 'trade.detail' in message['ch']: trades = self.api_socket.parse_trade(instmt, message['tick']['data']) for trade in trades: if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) elif 'depth.step' in message['ch']: instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) self.api_socket.parse_l2_depth(instmt, message['tick']) if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) else: Logger.error(self.__class__.__name__, 'Not Trade or Market')
def create_output_dir(): """Cria a pasta e os arquivos de saídas '.csv' (datasets).""" try: # se o diretório de saída existir, apaga seu conteúdo if os.path.exists(CSVParser.__output_dir): shutil.rmtree(CSVParser.__output_dir) # cria o diretório de saída os.mkdir(CSVParser.__output_dir) # cria todos os arquivos de saída '.csv' (datasets) CSVParser.__create_csv_file(CSVParser.__periodos_csv, Periodo.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__turmas_csv, Turma.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__atividades_csv, Atividade.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__estudantes_csv, Estudante.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__execucoes_csv, Execucao.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__solucoes_csv, Solucao.get_csv_header()) CSVParser.__create_csv_file(CSVParser.__erros_csv, Erro.get_csv_header()) except OSError: Logger.error('Erro ao criar diretório de saída!')
def start(dog_id, level): if level < 1 or level > 4: Logger.error("You have entered an invalid level for Phase 2.") return Logger.info("Phase Two: Starting") if level == 1: if level_one(dog_id): Logger.info("Phase Two: Complete") else: Logger.warning("Phase Two: Failed") elif level == 2: if level_two(dog_id): Logger.info("Phase Two: Complete") else: Logger.warning("Phase Two: Failed") elif level == 3: if level_three(dog_id): Logger.info("Phase Two: Complete") else: Logger.warning("Phase Two: Failed") elif level == 4: if level_four(dog_id): Logger.info("Phase Two: Complete") else: Logger.warning("Phase Two: Failed")
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ trade_id, exch_trade_id = self.get_trades_init(instmt) instmt.set_trade_id(trade_id) instmt.set_exch_trade_id(exch_trade_id) while True: try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: time.sleep(1) continue except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) for trade in ret: assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int(instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) time.sleep(1)
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta( seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff( instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def create(self, table, columns, types, primary_key_index=[], is_ifnotexists=True): """ Create table in the database. Caveat - Assign the first few column as the keys!!! :param table: Table name :param columns: Column array :param types: Type array :param is_ifnotexists: Create table if not exists keyword """ if len(columns) != len(types): raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \ (columns, types)) if is_ifnotexists: ret = self.conn("\\v") if ret is not None: for t in ret: if table == self.decode_qtypes(t): Logger.info(self.__class__.__name__, "Table %s has been created." % table) return True Logger.info(self.__class__.__name__, "Table %s is going to be created." % table) c = columns[:] for i in range(0, len(types)): t = self.convert_type(types[i]) if t is str: if columns[i].find('date_time') > -1: c[i] += ":`timestamp$()" else: c[i] += ":`symbol$()" elif t is float: c[i] += ":`float$()" elif t is int: c[i] += ":`int$()" keys = [] for i in primary_key_index: keys.append(c[i]) for i in sorted(primary_key_index, reverse=True): del c[i] if len(keys) > 0: command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c)) else: command = '%s:(%s)' % (table, '; '.join(c)) self.lock.acquire() try: self.conn.sync(command) except Exception as e: Logger.error(self.__class__.__name__, "Error in creat statement(%s).\n%s" % (command, e)) finally: self.lock.release() return True
def start(dog_id, level): if level != 1: Logger.error("You have entered an invalid level for Phase One.") return Logger.info("Phase One: Starting") if level_one(dog_id): Logger.info("Phase One: Complete") else: Logger.warning("Phase One: Failed")
def writeAllPaths(pathModel, outputFileName): Logger.info("-----Write File Started-----") Logger.info("Writing file :" + str(outputFileName)) try: with open(outputFileName, 'wb') as fp: pickle.dump(pathModel, fp) Logger.info("-----Write File Complete-----") except OSError as err: Logger.error("OS error: %s" % err.message) except Exception as exp: Logger.error("exception: %s" % exp.message)
def get_order_book(cls, instmt): """ Get order book :param instmt: Instrument :return: Object L2Depth """ res = cls.request(cls.get_order_book_link(instmt)) if len(res) > 0 and 'error' in res and len(res['error']) == 0: res = list(res['result'].values())[0] return cls.parse_l2_depth(instmt=instmt, raw=res) else: Logger.error(cls.__name__, "Cannot parse the order book. Return:\n%s" % res) return None
def phase_three(dog_id, level, quiet_length, fail_max, treat_frequency_min, treat_frequency_max=-1): if treat_frequency_max == -1: treat_frequency_max = treat_frequency_min quiet_timer = 0 treat_timer = 0 treat_dispense_time = 0 fail_count = 0 # Reset quiet status NoiseUtil.reset_bark_status() while quiet_timer < quiet_length and fail_count < fail_max: # Wait 1 second time.sleep(1) if not Config.RUN_FLAG: Logger.data(dog_id, 3, level, "cancelled") return False if treat_timer == 0: treat_dispense_time = random.randint(treat_frequency_min, treat_frequency_max) if NoiseUtil.has_dog_barked: Logger.info("Dog just barked, incrementing fail count.") Logger.data(dog_id, 3, level, "dog_bark") fail_count += 1 quiet_timer = 0 NoiseUtil.reset_bark_status() continue else: quiet_timer += 1 treat_timer += 1 # Check if a treat should be dispensed if treat_timer == treat_dispense_time: PetTutorUtil.dispense_treat() treat_timer = 0 fail_count = 0 if quiet_timer >= quiet_length: # Dog passed the Challenge. return True if fail_count >= fail_max: # Dog has failed the challenge return False Logger.error("[Quiet Challenge] CODE SHOULD NEVER GET HERE!")
def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ if not message: return keys = message.keys() if "bids" in keys: self.order_book = self.api_socket.parse_l2_depth(instmt, message) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif "create_update" in keys: if message['create_update']: message['create_update'].update({"timestamp": message['timestamp']}) self.api_socket.parse_l2_depth(instmt, message['create_update']) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif message['delete_update']: message['delete_update'].update({"timestamp": message['timestamp']}) self.api_socket.parse_l2_depth(instmt, message['delete_update']) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif message['trade_updates']: for new_trade in message['trade_updates']: new_trade.update({"timestamp": message['timestamp']}) trade = self.api_socket.parse_trade(instmt, new_trade) self.api_socket.parse_l2_depth(instmt, new_trade) if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) else: Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) time.sleep(1)
def start(dog_id, level): if str(level) not in ('1', '2', '1D', '2D', '3D', '4D', '5D', '6D', '7D', '8D', '9D'): Logger.error("You have entered an invalid level for Phase Five.") return Logger.info("Phase Five: Starting.") # Start mic recording NoiseUtil.record_bark_status() MoveUtil.record_down_status() if str(level) in ('1', '2'): if str(level) == '1': dog_standing(dog_id, 1) elif str(level) == '2': dog_standing(dog_id, 2) else: dog_down(dog_id, level)
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ ExchGwQuoine.num_of_connections_lock.acquire() ExchGwQuoine.num_of_connections += 1 Logger.info( self.__class__.__name__, "Current number of connections = %d" % ExchGwQuoine.num_of_connections) ExchGwQuoine.num_of_connections_lock.release() trade_id, exch_trade_id = self.get_trades_init(instmt) instmt.set_trade_id(trade_id) instmt.set_exch_trade_id(exch_trade_id) while True: try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: time.sleep(1) continue except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) for trade in ret: assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type( trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int(instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) ExchGwQuoine.num_of_connections_lock.acquire() time.sleep(ExchGwQuoine.num_of_connections + ExchGwQuoine.extra_waiting_sec) ExchGwQuoine.num_of_connections_lock.release()
def start(dog_id, level): if level < 2 or level > 6: Logger.error("You have entered an invalid level for Phase Three.") return Logger.info("Phase Three: Starting.") # Start mic recording NoiseUtil.record_bark_status() # if level == 1: # if level_one(dog_id): # Logger.info("Phase Three: Complete.") # return # else: # Logger.warning("Phase Three: Failed") if level == 2: if level_two(dog_id): Logger.info("Phase Three: Complete.") return else: Logger.warning("Phase Three: Failed") elif level == 3: if level_three(dog_id): Logger.info("Phase Three: Complete.") return else: Logger.warning("Phase Three: Failed") elif level == 4: if level_four(dog_id): Logger.info("Phase Three: Complete.") return else: Logger.warning("Phase Three: Failed") elif level == 5: if level_five(dog_id): Logger.info("Phase Three: Complete.") return else: Logger.warning("Phase Three: Failed") elif level == 6: if level_six(dog_id): Logger.info("Phase Three: Complete.") return else: Logger.warning("Phase Three: Failed")
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta( seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: ExchGwQuoine.last_query_time_lock.release() continue for trade in ret: assert isinstance( trade.trade_id, str), "trade.trade_id(%s) = %s" % (type( trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int( instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def tracePath(fileName): Logger.info("-----Read File Started-----") Logger.info("Reading File :" + str(fileName)) try: with open(fileName, 'rb') as fp: pathModel = pickle.load(fp) outputImage = numpy.full((int(pathModel.height), int(pathModel.width), 3), dtype='uint8', fill_value=[255, 255, 255]) for currentList in pathModel.tracks: i = (random.random() * 1000) % 255 j = (random.random() * 1000) % 255 k = (random.random() * 1000) % 255 for points in currentList: cv2.circle(outputImage, points[0], 3, color=(i, j, k)) cv2.imshow("output", outputImage) k = cv2.waitKey(5) & 0xff if k == 27: break cv2.destroyAllWindows() Logger.info("-----Read File Complete-----") except OSError as err: Logger.error("OS error: %s" % err.message) except Exception as exp: Logger.error("exception: %s" % exp.message)
if __name__ == "__main__": log.info("Starting contact management process") Management().create_db_table() parser = argparse.ArgumentParser( prog='contact_book', description= 'contact book management for storing, deleting and searching address book', usage='%(prog)s path') parser.add_argument('action', type=str, choices=['save', 'delete', 'edit', 'search'], help='Select one of the actions ' 'you want to perform') args = parser.parse_args() if args.action == 'save': save_info() elif args.action == 'delete': delete_info() elif args.action == 'edit': edit_info() elif args.action == 'search': search_info() else: log.error( "Action Value can only be one of this: ['save', 'delete', 'edit', 'search']" ) raise SystemExit(2) args = parser.parse_args()
def main(): Logger.open_log_files() # This must be run before any logging is done Logger.info("P4E Training Protocol is Starting...") signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGTERM, handler) # Read in Arguments parser = argparse.ArgumentParser() parser.add_argument( '--debug', default='False', dest='debug', help="Boolean whether debug is on or off. Default False", type=argbool) parser.add_argument('--com', default=Config.PETTUTOR_COMPORT, dest='comport', help="COM port to use. Default is %s" % Config.PETTUTOR_COMPORT, type=int) parser.add_argument( '--wait', default=Config.PETTUTOR_ERROR_WAIT_INTERVAL, dest='error_wait', help= "Number of seconds to wait if connection fails before retrying. -1 to not retry. Default is %s" % Config.PETTUTOR_ERROR_WAIT_INTERVAL, type=int) parser.add_argument( '--mic', default=Config.MIC_SENSITIVITY, dest='mic', type=float, help="Number in decibels for dog barking sensitivity. Default is %s" % Config.MIC_SENSITIVITY) parser.add_argument('--camera', default=Config.CAMERA_ID, dest='camera', type=int, help="Camera ID, generally 0 or 1. Default is %s" % Config.CAMERA_ID) parser.add_argument( '--disable-pettutor', default='False', dest='disable_pettutor', help="True to disable connection to PetTutor (testing purposes only)", type=argbool) # Set values argvalues = parser.parse_args() Config.PRINT_DEBUG = argvalues.debug Config.PETTUTOR_COMPORT = argvalues.comport Config.PETTUTOR_ERROR_WAIT_INTERVAL = argvalues.error_wait Config.MIC_SENSITIVITY = argvalues.mic Config.CAMERA_ID = argvalues.camera Config.DISABLE_PETTUTOR = argvalues.disable_pettutor # Log Configuration Logger.debug("Debug is %s" % Config.PRINT_DEBUG) Logger.debug("PetTutor Comport: %s" % Config.PETTUTOR_COMPORT) Logger.debug("PetTutor Wait Time: %s" % Config.PETTUTOR_ERROR_WAIT_INTERVAL) Logger.debug("Mic Sensitivity: %s" % Config.MIC_SENSITIVITY) # Connect to PetTutor PetTutorUtil.connect(Config.PETTUTOR_COMPORT) while True: Logger.prompt("Dog ID (-1 to Quit): ") dog_id = raw_input() Logger.debug("Dog ID is %s" % dog_id) if dog_id == '-1': break if re.match('^[\w-]+$', dog_id) is None: Logger.error( "'%s' is not a valid dog ID. Please only use alphanumeric characters." % dog_id) continue Config.RUN_FLAG = True data = FileIOUtil.load(dog_id) if data.get('dog_size') is None: Logger.prompt("Please enter dog's size (small/medium/large): ") dog_size = raw_input() Logger.debug("Dog Size is %s " % dog_size) if dog_size.lower() in ("small", "s", "0"): dog_size = 0 FileIOUtil.save_size(dog_id, 0) elif dog_size.lower() in ("medium", "m", "1"): dog_size = 1 FileIOUtil.save_size(dog_id, 1) elif dog_size.lower() in ("large", "l", "2"): dog_size = 2 FileIOUtil.save_size(dog_id, 2) else: Logger.error( "'%s' did not match expected input (small/medium/large)" % dog_size) continue else: dog_size = data["dog_size"] (phase, level) = find_next_level(data) input_required = False if phase != -1 and level != -1: Logger.info( "Dog %s's first incomplete level is Phase %s Level %s." % (dog_id, phase, level)) else: Logger.info("Dog %s has completed all Phases/Levels." % dog_id) input_required = True if not input_required: Logger.prompt("Continue? (Y/n): ") input_required = raw_input().lower() == 'n' Logger.debug("Continue is %s" % (not input_required)) if input_required: Logger.info("Please specify which Phase you want to run.") Logger.prompt("Enter a Phase Number (-1 to go back): ") try: phase = int(raw_input()) except ValueError: Logger.error("Phases can only be numbers.") continue Logger.debug("Selected phase is %d" % phase) if phase == -1: continue Logger.prompt("Enter a Level Number (-1 to go back): ") level = raw_input() Logger.debug("Selected level is %s" % level) if level == '-1': continue keyboard_thread = Thread(target=keyboard_listener) keyboard_thread.start() # Start a phase. if phase == 1: PhaseOne.start(dog_id, int(level)) Config.RUN_FLAG = False continue elif phase == 2: PhaseTwo.start(dog_id, int(level)) Config.RUN_FLAG = False continue elif phase == 3: PhaseThree.start(dog_id, int(level)) Config.RUN_FLAG = False continue elif phase == 4: PhaseFour.start(dog_id, dog_size, int(level)) Config.RUN_FLAG = False continue elif phase == 5: PhaseFive.start(dog_id, level.upper()) Config.RUN_FLAG = False continue Logger.error("You entered an invalid Phase.") Config.RUN_FLAG = False time.sleep(1) Logger.close_log_files()
class LarsRegression: def __init__(self, cv_num_vec, repeat_num, coef_precision): """ :param cv_num_vec: :param repeat_num: """ self._cv_num_vec = cv_num_vec self._repeat_num = repeat_num self._x_train_matrix = None self._x_train_matrix_scaled = None self._x_train_norm_vec = None self._x_train_mean_vec = None self._y_train_vec = None self._y_train_vec_scaled = None self._y_train_mean = None self._y_train_std = None self._x_train_remained_instance = None self._y_train_remained_instance = None self._predict_coef = None self._model_vec = [] self._coef_precision = coef_precision self._logger = Logger() @property def predict_coef(self): return self._predict_coef @property def predict_coef_nonzero_idx(self): nonzero_list = np.nonzero(self._predict_coef)[0].tolist() return nonzero_list @property def predict_triparams_list(self): return list( zip(self._predict_coef, self._x_train_mean_vec, self._x_train_norm_vec)) @property def pred_train_mean(self): return self._y_train_mean @property def pred_train_std(self): return self._y_train_std def filling(self, x_train, y_train): assert (len(x_train) == len(y_train) and len(x_train) > 1) # remain the last instance left as a valid instance, # so as to avoid extremely situation in model fitting. self._x_train_remained_instance = x_train[-1] self._y_train_remained_instance = y_train[-1] try: self._x_train_matrix = np.array(x_train[0:-1]) self._x_train_matrix_scaled, self._x_train_norm_vec, \ self._x_train_mean_vec = scale_train_x(self._x_train_matrix) self._y_train_vec = np.array(y_train[0:-1]) self._y_train_vec_scaled, self._y_train_mean = zeromean_y( self._y_train_vec) self._y_train_std = np.std(self._y_train_vec) except: self._x_train_matrix = None self._y_train_vec = None self._logger.error(module='factormodel', file='lasso.py', content=traceback.format_exc()) def fitting_with_cross_validation(self): # init predict coefficients to be zero vector self._predict_coef = np.array([0.0] * len(self._x_train_mean_vec)) if len(self._x_train_matrix) < 10: self._logger.warning( module='factormodel', file='lasso.py', content= "the num of training instances is less than 10. fitting exit.") return x_valid_instance_matrix = \ scale_test_x(np.array([self._x_train_remained_instance]), self._x_train_norm_vec, self._x_train_mean_vec) y_valid_instance_value = self._y_train_remained_instance x_train_matrix_shuffled = self._x_train_matrix_scaled y_train_vec_shuffled = self._y_train_vec_scaled model_num = 0 repeat_num = self._repeat_num for cv_num in self._cv_num_vec: while True: if repeat_num == 0 or model_num > 0: break repeat_num -= 1 x_train_matrix_shuffled, y_train_vec_shuffled = \ shuffle(x_train_matrix_shuffled, y_train_vec_shuffled) try: model = LarsRegression._model_fitting_cv( x_train_matrix_shuffled, y_train_vec_shuffled, cv_num) pred = model.predict(x_valid_instance_matrix)[0] dist_d = abs( (pred + self._y_train_mean - y_valid_instance_value) / self._y_train_std) if dist_d > 5: self._logger.warning(module='factormodel', file='lasso.py', content="fitting restart.") continue for i, v in enumerate(model.coef_): self._predict_coef[i] += v model_num += 1 except: self._logger.error(module='factormodel', file='lasso.py', content=traceback.format_exc()) continue if model_num == 0: self._logger.warning( module='factormodel', file='lasso.py', content= "fitting with cross validation failed. model num is zero.") precision_base = float(np.power(10, self._coef_precision)) # let coef vec keep 2 decimal precision self._predict_coef = [ int(v * precision_base) / precision_base for v in self._predict_coef ] def fitting_without_cross_validation(self): self._predict_coef = np.array([0.0] * len(self._x_train_mean_vec)) if len(self._x_train_matrix) < 10: self._logger.warning( module='factormodel', file='lasso.py', content= "the num of training instances is less than 10. fitting exit.") return try: model = LarsRegression._model_fitting( self._x_train_matrix_scaled + [self._x_train_remained_instance], self._y_train_vec_scaled + [self._y_train_remained_instance]) for i, v in enumerate(model.coef_): self._predict_coef[i] += v except: # traceback.print_exc() self._logger.error( module='factormodel', file='lasso.py', content="fitting without cross validation failed.") def predict(self, x_test_array): x_test_matrix = scale_test_x(np.array([x_test_array]), self._x_train_norm_vec, self._x_train_mean_vec) pred = 0 for v, p in list(zip(x_test_matrix[0], self._predict_coef)): pred += v * p pred += self._y_train_mean return pred def nonzero_list(self): # nonzero_idx_arr = sorted(np.nonzero(coef)[0].tolist(), # cmp=lambda x1, x2: 1 if abs(coef[x1]) < abs(coef[x2]) else -1) return sorted(self.predict_coef_nonzero_idx) # LassoLarsCV: least angle regression @classmethod def _model_fitting_cv(cls, x, y, num_cv, plotting=False): # Compute paths # print("Computing regularization path using the Lars lasso...") model = linear_model.LassoLarsCV(cv=num_cv).fit(x, y) # Display results if plotting: import matplotlib.pyplot as plt m_log_alphas = -np.log10(model.cv_alphas_) plt.figure(figsize=(20, 10)) plt.plot(m_log_alphas, model.cv_mse_path_, ':') plt.plot(m_log_alphas, model.cv_mse_path_.mean(axis=-1), 'k', label='Average across the folds', linewidth=2) plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k', label='alpha CV') plt.legend() plt.xlabel('-log(alpha)') plt.ylabel('Mean square error') plt.axis('tight') plt.savefig('cross_validation', dpi=None, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format=None, transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None) plt.plot() return model @classmethod def _model_fitting(cls, x, y, alpha=0, max_iter=500): model = linear_model.LassoLars(alpha=alpha, max_iter=max_iter).fit(x, y) return model
class BNPoissMF(Recommender): def __init__(self, trainMatrix, testMatrix, configHandler): Recommender.__init__(trainMatrix, testMatrix, configHandler) self.logger = Logger('BNPoissMF.log') def initModel(self): ''' Read the model parameters, and get some common values. ''' self.numUsers, self.numItems = self.trainMatrix.shape() self.prediction = dok_matrix((self.numUsers, self.numItems)) self.MAX_Iterations = int(self.configHandler.getParameter('BPoissMF', 'MAX_Iterations')) self.numFactors = int(self.configHandler.getParameter('BPoissMF', 'numFactors')) self.threshold = float(self.configHandler.getParameter('BPoissMF', 'threshold')) # Get the Parameters self.user_alpha = float(self.configHandler.getParameter('BPoissMF', 'user_alpha')) self.user_c = float(self.configHandler.getParameter('BPoissMF', 'user_c')) self.item_a = float(self.configHandler.getParameter('BPoissMF', 'item_a')) self.item_b = float(self.configHandler.getParameter('BPoissMF', 'item_b')) # The model parameters for users self.gamma0 = np.zeros(self.numUsers) self.gamma1 = np.zeros(self.numUsers) self.s = np.zeros(self.numUsers) self.nu = np.zeros((self.numUsers, self.numFactors)) self.theta = np.zeros((self.numUsers, self.numFactors)) # The model parameters for stick proportions self.tau = np.zeros((self.numUsers, self.numFactors)) # The model parameters for item weights self.lambda0 = np.zeros((self.numItems, self.numFactors)) self.lambda1 = np.zeros((self.numItems, self.numFactors)) self.beta = np.zeros((self.numItems, self.numFactors)) self.z = np.zeros((self.numUsers, self.numItems)) self.pi = np.zeros((self.numUsers, self.numItems)) self.logPi = np.zeros((self.numUsers, self.numItems)) def buildModels(self): pass def initUserScalingParameters(self): ''' initial equations for the user scaling parameters gamma_u0 and gamma_u1 ''' def initStickProportions(self): ''' The update equations for the stick proportions tau_uk can be obtained by taking the derivative of the objective function with respect to tau_uk ''' self.nu = 0.001 * np.random.random((self.numUsers, self.numFactors)) def computePi(self): ''' Equation (10) ''' def initItemWeights(self): pass def updateUserScalingParameters(self): pass def updateStickProportions(self): pass def updateItemWeights(self): pass def calculateConjugacy(self): pass def GammaPoisson(self): pass def solveQuadratic(self, a, b, c): ''' ''' s1 = (-b + sqrt(b*b - 4*a*c)) / (2*a) s2 = (-b - sqrt(b*b - 4*a*c)) / (2*a) if s1 > .0 and s1 <= 1.0 and s2 > .0 and s2 <= 1.0: self.logger.error('s1 %f and s2 %f are out of range in solve_quadratic()' % (s1, s2)) self.logger.error('a = %.5f, b = %.5f, c = %.5f\n' % (a, b, c)) if s1 < s2: return s1 + THRESHOLD else: return s2 + THRESHOLD if s1 > .0 and s1 <= 1.0: return s1 if s2 > .0 and s1 <= 1.0: return s2 if np.abs(s1 - .0) < THRESHOLD: return THRESHOLD if np.abs(1.0 - s1) < THRESHOLD: return 1.0 - THRESHOLD if np.abs(s2 - .0) < THRESHOLD: return THRESHOLD if np.abs(s2 - 1.0) < THRESHOLD: return 1.0 - THRESHOLD self.logger.error('WARNING: s1 %.10f and s2 %.10f are out of range in solve_quadratic()' % (s1, s2)) return s1
python = "python" Logger.info("===================Begin===================") # Step1. Initialize the database Logger.info("Step1. Initialize the database.") sql_str = "mysql -u%s -p%s -e 'drop database if exists %s; create database %s; use %s; source '%s';'" \ % (Configuration.configs['mysql_usr'], Configuration.configs['mysql_pwd'], Configuration.configs['mysql_db'], Configuration.configs['mysql_db'], Configuration.configs['mysql_db'], Configuration.configs['TraceSqlPath']) result = os.system(sql_str) if result != 0: Logger.error("Fail to create the database") exit() # Step2. Import trace data into the database Logger.info("Step2. Import trace data to the database.") result = os.system("%s %s/REP/recordToDatabase.py" % (python, Configuration.configs['ProjectPath'])) if result != 0: Logger.error("Fail to import trace data to the database") exit() # # Step2-2. Import log data into the database # Logger.info("Step2-2. Import log data to the database.") # result = os.system("%s %s/REP/process_log.py" % (python, Configuration.configs['ProjectPath'])) # if result != 0: # Logger.error("Fail to import log data to the database in Step2-2.")
def phase_four(dog_id, level, still_length, fail_max, treat_frequency_min, treat_frequency_max=-1): if treat_frequency_max == -1: treat_frequency_max = treat_frequency_min still_timer = 0 treat_timer = 0 treat_dispense_time = 0 fail_count = 0 # Reset quiet and move status NoiseUtil.reset_bark_status() MoveUtil.reset_move_status() while still_timer < still_length and fail_count < fail_max: # Wait 1 second time.sleep(1) if not Config.RUN_FLAG: Logger.data(dog_id, 4, level, "cancelled") return False if treat_timer == 0: treat_dispense_time = random.randint(treat_frequency_min, treat_frequency_max) if NoiseUtil.has_dog_barked or MoveUtil.has_dog_moved: if NoiseUtil.has_dog_barked: Logger.data(dog_id, 4, level, "dog_bark") else: Logger.data(dog_id, 4, level, "dog_move") fail_count += 1 still_timer = 0 NoiseUtil.reset_bark_status() MoveUtil.reset_move_status() # Check if the dog has failed too many times if fail_count >= fail_max: break """ Wait for 10 seconds to allow the dog to stop moving. """ sleep_time = 0 for i in range(1, Config.MOVE_PAUSE + 1): time.sleep(1) Logger.debug("Letting the dog stop moving... (%s seconds)" % i) if not MoveUtil.has_dog_moved: sleep_time += 1 else: sleep_time = 0 MoveUtil.reset_move_status() if sleep_time >= 2: break continue # Continue loop (timer has been reset, don't reward dog, etc) else: still_timer += 1 treat_timer += 1 # Check if a treat should be dispensed if treat_timer == treat_dispense_time: PetTutorUtil.dispense_treat() treat_timer = 0 fail_count = 0 """ Wait for 5 seconds to allow the dog to eat the treat. """ sleep_time = 0 MoveUtil.reset_move_status( ) # This is just a precaution, it should already be reset for i in range(1, Config.TREAT_PAUSE + 1): time.sleep(1) if not MoveUtil.has_dog_moved: sleep_time += 1 # Increment the amount of time the dog has not moved. else: sleep_time = 0 # The dog moved, reset sleep time MoveUtil.reset_move_status() if sleep_time >= 2: # Once dog has not moved for X seconds, continue training. break Logger.debug("Letting the dog eat... (%s seconds)" % i) if still_timer >= still_length: # Dog passed the Challenge. return True if fail_count >= fail_max: # Dog has failed the challenge return False Logger.error("[Still Challenge] CODE SHOULD NEVER GET HERE!")
class Controller: def __exit__(self): self.driver.closeDriver() def __init__(self, setting): try: self.logger = Logger(setting.saveDirPath) self.fm = FileManager(setting.saveDirPath, setting.downloadDirPath, self.logger) self.driver = IDriver(setting.chromeDriverPath) self.setting = setting except: raise ('[error] setting is not valid') def chrollPostList(self, postListUrl): self.driver.movePage(postListUrl) postList = PostList(postListUrl) soup = BeautifulSoup(self.driver.getPageSource(), 'html.parser') trList = soup.find_all('tr', attrs={'class': 'ub-content us-post'}) soup.get('') for trElement in trList: if len(trElement.find_all('b')) > 0: continue postList.addPost(self._initPost(trElement)) self.logger.print('[done] ' + postListUrl + ' 게시글 목록 chrolling 완료') return postList def chrollInRange(self, startPostNumber, endPostNumber): for pageIdx in range(1, 1000): postList = self.chrollPostList(self.setting.getPageUrl() % pageIdx) postsInRange = postList.getPostsInRange(startPostNumber, endPostNumber) for post in postsInRange: if self.setting.passExistFile and post.status == PostStatus.ALREADY_EXIST: continue self.chrollPost(post) for post in postsInRange: if post.status == PostStatus.END_CHROLLING: self.moveImages(post) if post.status == PostStatus.END_MOVING_IMAGES: self.makeHtml(post) self.logger.print('[END] ' + post.getTitle() + ' 상태 :' + str(post.status)) if not postList.needToContinue(startPostNumber): break def _initPost(self, trElement): aTagElement = trElement.find('td', attrs={ 'class': 'gall_tit ub-word' }).find('a') postNumber = trElement.find('td', attrs={'class': 'gall_num'}).text title = aTagElement.text.strip() url = aTagElement['href'] post = Post(url=url, number=postNumber, title=title) if self.fm.existFile(self.setting.saveDirPath + post.getTitle() + '.html'): post.status = PostStatus.ALREADY_EXIST else: post.status = PostStatus.END_INITIALIZING return post def chrollPost(self, post): self.driver.movePage(Url.postUrlPrefixs + post.url) soup = BeautifulSoup(self.driver.getPageSource(), features='html.parser') readInnerHTML_script = "return arguments[0].innerHTML;" contents = self.driver.excecuteScriptToElement( readInnerHTML_script, self.driver.findElement(Xpath.postBody)) post.writingTime = self.driver.findElement( Xpath.postWritingTime).getAttribute('title') post.bodySoup = BeautifulSoup(contents, 'html.parser') post.replySoup = soup.find('div', attrs={'class': 'comment_wrap show'}) emptyElement = self.driver.findElement(Xpath.emptyClickBox) emptyElement.click() downloadList = [] for imageIndex in range(1, 50): if (self.driver.notExistElement(Xpath.saveBtn % imageIndex)): break saveBtn = self.driver.findElement(Xpath.saveBtn % imageIndex) saveBtn.click() downloadFileName = saveBtn.getText().replace('~', '_').replace( '+', ' ') downloadList.append(downloadFileName) post.downloadList = downloadList post.status = PostStatus.END_CHROLLING self.logger.print('[done] ' + post.getTitle() + ' 게시글 chrolling') def moveImages(self, post): try: post.imgList = [] for file in post.downloadList: post.imgList.append( self.fm.moveFile2SaveDirectory(post.getTitle(), file)) post.status = PostStatus.END_MOVING_IMAGES except: self.logger.error('[error] ' + post.getTitle() + ' image 파일 옮기는 과정에서 에러남') def makeHtml(self, post): try: hb = HtmlBuilder(self.setting.saveDirPath, post.getTitle(), post.writingTime, self.logger) self._mappingAttributes2SaveImages(post) hb.writeBody(str(post.bodySoup)) hb.writeReply(str(post.replySoup)) hb.close() post.status = PostStatus.END_MAKING_HTML except: self.logger.error('[error] ' + post.getTitle() + ' html 빌드 과정에서 에러남') def _mappingAttributes2SaveImages(self, post): imgTagElementsList = post.bodySoup.find_all('img') tagListSize = len(imgTagElementsList) downloadImgSize = len(post.downloadList) # if tagListSize != downloadImgSize: # self.logger.error('[error] img tag의 갯수(%d)와 download img 갯수(%d)가 다름'%(tagListSize,downloadImgSize)) # dc 이모티콘으로 인해 img 갯수가 많을 수 있습니다 for i in range(downloadImgSize): filePath = './' + post.getTitle() + '/' + post.downloadList[i] element = imgTagElementsList[i] element['alt'] = '' element['onclick'] = '' element['src'] = filePath
if options.check is not None: for section in config.sections(): print("Section [%s]" % section) print(json.dumps(dict(config[section]), indent=4)) print("\n") elif options.start is not None: app = config.get('Base', 'APP') if app == 'PB': from PB import PBClient as Client elif app == "PKURunner": from PKURunner import PKURunnerClient as Client elif app == "Joyrun": from Joyrun import JoyrunClient as Client else: raise ValueError("unsupported running APP -- %s !" % app) try: client = Client() client.run() except Exception as err: logger.error("upload record failed !") raise err else: logger.info("upload record success !")
from util import LINE_MESSAGE_TEXTS # https://github.com/line/line-bot-sdk-python # https://developers.line.biz/en/reference/messaging-api/ # -- Setup # TODO: encrypt keys on AWS # TODO: mypy when matured # TODO: pytest with testing db # TODO: lambda layer to share enum and others # Logger logger = Logger(LAMBDA_NAME, (logging.StreamHandler(), logging.DEBUG)) # Get channel_secret from environment variable channel_secret = os.getenv('LINE_CHANNEL_SECRET', None) if channel_secret is None: logger.error('Specify LINE_CHANNEL_SECRET as environment variable.') sys.exit(1) # Line webhook line_handler = WebhookHandler(channel_secret) # -- Main handler def lambda_handler(requestEvent, context): # Get X-Line-Signature header value signature = requestEvent['headers']['X-Line-Signature'] # Get request body as text body = requestEvent['body'] # Give logger `lineUserId` info
""".format(file_path, THREAD_NUMS, TIMEOUT, RETRIES, RQ)) writer = open("../stat", "w", encoding='utf8') # 文件类型 可选 "tang" or "wu" if file_type == "fulltext": file_from = "wu" dir_path, file_name = get_file_name(file_path) # 输入文件 tag = time.strftime("%Y%m%d%H%M", time.localtime(time.time())) uni_format(file_path, file_from=file_from, id=RQ, logger=LOG, writer=writer) # 调度器 sch = Scheduler("./tmp/{}_referers.tsv".format(RQ), "utf8", "tsv") sch.scheduling() ret_file_path = "./Logs/{}_ret.txt".format(tag) LOG.info("开始合并文件") concat(ret_file_path, "./tmp/{}_{}.tsv".format(RQ, file_from), "./datas/{}_ret.csv".format(file_name), LOG, is_filter_by_word, is_filter_by_input, is_filter_by_country, writer=writer) writer.close() elif file_type == "urls": sch = Scheduler(file_path, "utf8", "tsv") sch.scheduling() writer.close() else: LOG.error("Wrong file type: {}, Please choose 'fulltext or 'urls'".format(file_type)) writer.close() import sys sys.exit(-1)
class Mail(object): """model for the Mail.""" id_is_valid = staticmethod(lambda num: 0 < int(num) <= 1L << 31) def __init__(self, env, id=None, db=None, messageid=None, row=None): self.env = env self.db = db self.log = Logger(env) if id is not None: self.resource = Resource('mailarchive', str(id), None) self._fetch_mail(id) elif messageid is not None: self._fetch_mail_by_messageid(messageid) self.resource = Resource('mailarchive', self.id, None) elif row is not None: self._fetch_mail_by_row(row) self.resource = Resource('mailarchive', self.id, None) else: self.messageid = '' self.subject = '' self.utcdate = 0 self.localdate = '' self.zoneoffset = 0 self.body = '' def __eq__(self, other): if isinstance(other, Mail): return self.messageid == other.messageid return super.__eq__(other) def _get_db(self): if self.db: return self.db else: return self.env.get_db_cnx() def _get_db_for_write(self): if self.db: return (self.db, False) else: return (self.env.get_db_cnx(), True) def get_sanitized_fromaddr(self): return self.fromaddr.replace('@', self.env.config.get('mailarchive', 'replaceat', '@')) def get_fromtext(self): return get_author(self.fromname, self.fromaddr) def get_category(self): yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate)) category = self.mlid + yearmonth return category.encode('utf-8') def get_plain_body(self): return self._sanitize(self.env, self.body) def get_html_body(self, req): # for HTML Mail if self.body.lstrip().startswith('<'): return Markup(self.body) contentlines = self.body.splitlines() htmllines = ['',] #customize! #http://d.hatena.ne.jp/ohgui/20090604/1244114483 wikimode = req.args.get('wikimode', 'on') for line in contentlines: if self.env.config.get('mailarchive', 'wikiview',' enabled') == 'enabled' and wikimode == 'on': htmllines.append(wiki_to_oneliner(line, self.env, self.db, False, False, req)) else: htmllines.append(Markup(Markup().escape(line).replace(' ',' '))) content = Markup('<br/>').join(htmllines) return content def _sanitize(self, env, text): return text.replace('@', env.config.get('mailarchive', 'replaceat','_at_') ) def _fetch_mail(self, id): row = None if self.id_is_valid(id): db = self._get_db() cursor = db.cursor() cursor.execute(SELECT_FROM_MAILARC + " WHERE id=%s", (id,)) row = cursor.fetchone() if not row: raise ResourceNotFound('Mail %s does not exist.' % id, 'Invalid Mail Number') self._fetch_mail_by_row(row) def _fetch_mail_by_messageid(self, messageid): row = None db = self._get_db() cursor = db.cursor() cursor.execute(SELECT_FROM_MAILARC + " WHERE messageid=%s", (messageid,)) row = cursor.fetchone() if not row: raise ResourceNotFound('Mail messageid %s does not exist.' % messageid, 'Invalid Mail messageid Number') self._fetch_mail_by_row(row) def _fetch_mail_by_row(self, row): self.id = row[0] self.messageid = row[1] self.utcdate = row[2] self.zoneoffset = row[3] self.subject = row[4] self.fromname = row[5] self.fromaddr = row[6] self.header =row[7] self.body = row[8] self.thread_root = row[9] self.thread_parent = row[10] self.zone = self._to_zone(self.zoneoffset) self.localdate = self._to_localdate(self.utcdate, self.zoneoffset) def _to_localdate(self, utcdate, zoneoffset): return time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate + zoneoffset)) def _to_zone(self, zoneoffset): #zone and date zone = '' if zoneoffset == '': zoneoffset = 0 if zoneoffset > 0: zone = ' +' + time.strftime('%H%M', time.gmtime(zoneoffset)) elif zoneoffset < 0: zone = ' -' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset)) return zone def get_href(self, req): return req.href.mailarchive(self.id) def get_subject(self): if is_empty(self.subject): return '(no subject)' else: return self.subject def get_senddate(self): return self.localdate + self.zone def get_thread_root(self): if self.thread_root == '': return self try: root_mail = Mail(self.env, messageid=self.thread_root) except ResourceNotFound: return self #self.thread_rootはオリジナル版だと親のメールになってしまっている。 #互換性維持のため、ルートではない場合は自力で探しにいくロジックを走らす if root_mail.thread_root == '': return root_mail else: if self.thread_parent != '': root_id = MailFinder.find_root_id(self.env, self.messageid) return Mail(self.env, messageid=root_id) def get_thread_parent_id(self): if self.thread_parent != '': return self.thread_parent.split(' ')[0] return None def get_thread_parent(self): if self.thread_parent != '': return Mail(self.env, db=self.db, messageid=self.get_thread_parent_id()) return self def get_children(self, desc=False, cached_mails=None): if cached_mails: self.log.debug("[%s] mail's threads is cached." % self.id) return [x for x in cached_mails if x.get_thread_parent_id() == self.messageid] db = self._get_db() cursor = db.cursor() sql = SELECT_FROM_MAILARC + " WHERE threadparent LIKE %s ORDER BY utcdate" if desc: sql += " DESC" cursor.execute(sql, ('%s %%' % self.messageid,)) children = [] for row in cursor: child_mail = Mail(self.env, row=row, db=self.db) children.append(child_mail) return children def get_thread_mails(self, desc=False): root = self.get_thread_root() db = self._get_db() cursor = db.cursor() sql = SELECT_FROM_MAILARC + " WHERE threadroot = %s ORDER BY utcdate" if desc: sql += " DESC" cursor.execute(sql, (root.messageid,)) mails = [] for row in cursor: mails.append(Mail(self.env, row=row, db=self.db)) return mails def has_children(self, cached_mails=None): rtn = len(self.get_children(cached_mails=cached_mails)) > 0 return rtn def get_related_tickets(self, req): db = self._get_db() return get_related_tickets(self.env, req, db, self.id) def has_attachments(self, req): attachment = MailArchiveAttachment(self.env, self.id) return attachment.has_attachments(req) def populate(self, author, msg, mlid): """Populate the mail with 'suitable' values from a message""" if 'message-id' not in msg: raise 'Illegal Format Mail!' self.is_new_mail = False self.mlid = mlid self._parse_messageid(msg) self._parse_date(msg) self._parse_subject(msg) if msg.is_multipart(): self._parse_multipart(author, msg) else: self._parse_body(msg) ref_messageid = self._parse_reference(msg) self._make_thread(ref_messageid) def update_or_save(self): if self.messageid is None or self.messageid == '': raise "Can't save mail to database." db, has_tran = self._get_db_for_write() cursor = db.cursor() yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate)) category = self.mlid + yearmonth cursor.execute("SELECT category, mlid, yearmonth, count FROM mailarc_category WHERE category=%s", (category.encode('utf-8'),)) row = cursor.fetchone() count = 0 if row: count = row[3] pass else: cursor.execute("INSERT INTO mailarc_category (category, mlid, yearmonth, count) VALUES(%s, %s, %s, %s)", (category.encode('utf-8'), self.mlid.encode('utf-8'), yearmonth, 0)) if self.is_new_mail: count = count + 1 cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s", (count, category.encode('utf-8'))) # insert or update mailarc #self.log.debug( # "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id), # category.encode('utf-8'), # messageid, # utcdate, # zoneoffset, # subject.encode('utf-8'), fromname.encode('utf-8'), # fromaddr.encode('utf-8'),'','', # thread_root,thread_parent)) cursor.execute("DELETE FROM mailarc where messageid=%s", (self.messageid,)) cursor.execute("INSERT INTO mailarc (" "id, category, messageid, utcdate, zoneoffset, subject," "fromname, fromaddr, header, text, threadroot, threadparent) " "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", (str(self.id), category.encode('utf-8'), self.messageid, self.utcdate, self.zoneoffset, self.subject.encode('utf-8'), self.fromname.encode('utf-8'), self.fromaddr.encode('utf-8'), '', self.body.encode('utf-8'), self.thread_root, self.thread_parent)) if has_tran: db.commit() def _parse_messageid(self, msg): self.messageid = msg['message-id'].strip('<>') #check messageid is unique self.log.debug("Creating new mailarc '%s'" % 'mailarc') db = self._get_db() cursor = db.cursor() cursor.execute("SELECT id from mailarc WHERE messageid=%s", (self.messageid,)) row = cursor.fetchone() id = None if row: id = row[0] if id == None or id == "": # why? get_last_id return 0 at first. #id = db.get_last_id(cursor, 'mailarc') self.is_new_mail = True cursor.execute("SELECT Max(id)+1 as id from mailarc") row = cursor.fetchone() if row and row[0] != None: id = row[0] else: id = 1 self.id = int(id) # Because id might be 'n.0', int() is called. def _parse_date(self, msg): if 'date' in msg: datetuple_tz = email.Utils.parsedate_tz(msg['date']) localdate = calendar.timegm(datetuple_tz[:9]) #toDB zoneoffset = datetuple_tz[9] # toDB utcdate = localdate - zoneoffset # toDB #make zone ( +HHMM or -HHMM zone = '' if zoneoffset > 0: zone = '+' + time.strftime('%H%M', time.gmtime(zoneoffset)) elif zoneoffset < 0: zone = '-' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset)) #self.log.debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9])) self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate))) self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(localdate))) self.log.debug(zone) fromname, fromaddr = email.Utils.parseaddr(msg['from']) self.fromname = self._decode_to_unicode(fromname) self.fromaddr = self._decode_to_unicode(fromaddr) self.zone = zone self.utcdate = utcdate self.zoneoffset = zoneoffset self.localdate = self._to_localdate(utcdate, zoneoffset) self.log.info(' ' + self.localdate + ' ' + zone +' '+ fromaddr) def _parse_subject(self, msg): if 'subject' in msg: self.subject = self._decode_to_unicode(msg['subject']) def _parse_reference(self, msg): # make thread infomations ref_messageid = '' if 'in-reply-to' in msg: ref_messageid = ref_messageid + msg['In-Reply-To'] + ' ' self.log.debug('In-Reply-To:%s' % ref_messageid ) if 'references' in msg: ref_messageid = ref_messageid + msg['References'] + ' ' m = re.findall(r'<(.+?)>', ref_messageid) ref_messageid = '' for text in m: ref_messageid = ref_messageid + "'%s'," % text ref_messageid = ref_messageid.strip(',') self.log.debug('RefMessage-ID:%s' % ref_messageid) return ref_messageid def _parse_multipart(self, author, msg): body = '' # delete all attachement at message-id Attachment.delete_all(self.env, 'mailarchive', self.id, self.db) for part in msg.walk(): content_type = part.get_content_type() self.log.debug('Content-Type:' + content_type) file_counter = 1 if content_type == 'multipart/mixed': pass elif content_type == 'text/html' and self._is_file(part) == False: if body != '': body += "\n------------------------------\n\n" body = part.get_payload(decode=True) charset = part.get_content_charset() self.log.debug('charset:' + str(charset)) # Todo:need try if charset != None: body = self._to_unicode(body, charset) elif content_type == 'text/plain' and self._is_file(part) == False: #body = part.get_payload(decode=True) if body != '': body += "\n------------------------------\n\n" current_body = part.get_payload(decode=True) charset = part.get_content_charset() self.log.debug('charset:' + str(charset)) # Todo:need try if charset != None: #body = self._to_unicode(body, charset) body += self._to_unicode(current_body, charset) else: body += current_body elif part.get_payload(decode=True) == None: pass # file attachment else: self.log.debug(part.get_content_type()) # get filename # Applications should really sanitize the given filename so that an # email message can't be used to overwrite important files filename = self._get_filename(part) if not filename: import mimetypes ext = mimetypes.guess_extension(part.get_content_type()) if not ext: # Use a generic bag-of-bits extension ext = '.bin' filename = 'part-%03d%s' % (file_counter, ext) file_counter += 1 self.log.debug("filename:" + filename.encode(OUTPUT_ENCODING)) # make attachment tmp = os.tmpfile() tempsize = len(part.get_payload(decode=True)) tmp.write(part.get_payload(decode=True)) tmp.flush() tmp.seek(0,0) attachment = Attachment(self.env, 'mailarchive', self.id) attachment.description = '' # req.args.get('description', '') attachment.author = author #req.args.get('author', '') attachment.ipnr = '127.0.0.1' try: attachment.insert(filename, tmp, tempsize, None, self.db) except Exception, e: try: ext = filename.split('.')[-1] if ext == filename: ext = '.bin' else: ext = '.' + ext filename = 'part-%03d%s' % (file_counter, ext) file_counter += 1 attachment.description += ', Original FileName: %s' % filename attachment.insert(filename, tmp, tempsize, None, self.db) self.log.warn('As name is too long, the attached file is renamed : ' + filename) except Exception, e: self.log.error('Exception at attach file of Message-ID:' + self.messageid) traceback.print_exc(e) tmp.close()