def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.payment = payment self.config = Config() self.log = Log() techcombank = self.get_techcombank_config() self.email_transport = EmailTransport() self.login_url = techcombank['login_url'] self.username = payment.get_username() self.password = payment.get_password() self.debug_mode = techcombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode()
def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params)
def set_as_used(id_ticket, id_user): t = Ticket.get_ticket(id_ticket) if t == None: raise ErrorCodeException(ErrorCode.TICKET_DOESNT_EXISTS) if t.id_user != id_user: raise ErrorCodeException(ErrorCode.TICKET_DOESNT_BELONG_TO_USER) if User.get_user(id_user) == None: raise ErrorCodeException(ErrorCode.USER_DOESNT_EXISTS) if t.used == True: raise ErrorCodeException(ErrorCode.TICKET_ALREADY_USED) t.set_used() History.add_entry(History(id_ticket=unhexlify(id_ticket), id_user=id_user))
def __init__(self, name=None, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.captcha = CaptchaResolver() section_config = self.get_section_config(name) self.email_transport = EmailTransport() self.home_url = section_config['home_url'] self.login_url = section_config['login_url'] self.username = section_config['username'] self.password = section_config['password'] self.debug_mode = section_config['debug_mode'] self.total_transactions = 0 self.max_attempt_login = 10 self.max_validate_captcha = 10 self.login_failed = 0 self.captcha_failed = 0 self.history = History() self.code = GenerateCode()
def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.username = payment.get_username() self.password = payment.get_password() sacombank = self.get_config() self.email_transport = EmailTransport() self.corp_url = sacombank['corp_url'] self.captcha = CaptchaResolver() self.debug_mode = sacombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 20 self.login_failed = 0 self.session = session self.proxy = proxy self.payment = payment
def register(wagon_id, camera_id): current_camera = WagonDB.get_by_id(wagon_id)[0].get_last_camera() if current_camera is not None and int(current_camera) is int( camera_id): return True else: w = Wagon(wagon_id, camera_id, datetime.datetime.now()) h = History(wagon_id, camera_id, datetime.datetime.now()) if WagonDB.update_last_camera_id(w) and HistoryDB.add_item(h): return True else: return False
def get(self, id_user): sender_id_user = get_jwt_identity() if sender_id_user != id_user: return unauthorized("Forbiden!"), 403 try: user_hist = History.get_all(id_user) return { "history": [he.to_json() for he in user_hist] if user_hist else [] }, 200 except ErrorCodeException as ec: return error_code(ec), 500
def __init__(self, payment, session=None, proxy={}): self.payment = payment self.session = session self.proxy = {} self.captcha = CaptchaResolver() self.config = Config() self.log = Log() self.history = History() self.code = GenerateCode() self.email_transport = EmailTransport() section_config = self.get_msb_config() self.username = payment.get_username() self.password = payment.get_password() self.home_url = section_config['home_url'] self.login_url = section_config['login_url'] self.account_list_url = section_config['account_list_url'] self.account_info_url = section_config['account_info_url'] self.debug_mode = section_config['debug_mode'] self.total_transactions = 0 self.max_attempt_login = 10 self.max_validate_captcha = 10 self.login_failed = 0 self.captcha_failed = 0
def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params)
def sync(cls): now = datetime.now().strftime('%Y-%m-%d_%H') log.info(f'begin sync {now}') file_name = f"./data/{now}.json" if not os.path.exists(file_name): CrawlerService.crawler() raw_data = json.load(open(file_name, 'r')) data = raw_data['nCoV2019'] date = datetime.now().strftime('%Y%m%d') world = World(date=date, name='中国', confirmed=data['totalConNum'], suspected=data['totalSusNum'], cured=data['totalCureNum'], dead=data['totalDeathNum'], add_confirmed=data['addCon'], add_cured=data['addCure'], add_dead=data['addDeath'], add_suspected=data['addSus']) history_list = data['historyList'] histories = [] for history in history_list: histories.append( History(date=history['date'], confirmed=history['conNum'], cured=history['cureNum'], dead=history['deathNum'], suspected=history['susNum'])) overseas_list = data['overseasList'] worlds = [] for overseas in overseas_list: worlds.append( World(date=date, confirmed=overseas['conNum'], cured=overseas['cureNum'], suspected=overseas['susNum'], dead=overseas['deathNum'], name=overseas['name'])) domestic_list = data['domesticList'] provinces = [] cities = [] for domestic in domestic_list: provinces.append( Province(date=date, province_name=domestic['name'], short_name=domestic['name'], confirmed=domestic['conNum'], dead=domestic['deathNum'], suspected=domestic['susNum'], cured=domestic['cureNum'])) city_list = domestic['cities'] for city in city_list: cities.append( City(date=date, province_name=domestic['name'], city_name=city['name'], confirmed=city['conNum'], suspected=city['susNum'], cured=city['cureNum'], dead=city['deathNum'])) History.select().delete() History.bulk_insert(histories) World.select().filter(World.date == date).delete() world.insert() World.bulk_insert(worlds) Province.select().filter(Province.date == date).delete() Province.bulk_insert(provinces) City.select().filter(City.date == date).delete() City.bulk_insert(cities) log.info(f'sync {now} done')
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. add `play` function to run tests in the simulated environment ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) @property def summary_writer(self): return self._summary_writer def train(self): start_step = self.sess.run(self.step_op) num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] trade_rem = self.env.new_random_episode(self.history, self.replay_memory) for self.step in tqdm(range(start_step, self.max_step), ncols=70, initial=start_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict((self.history.history, trade_rem)) # 2. act screen, reward, terminal, trade_rem = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal, trade_rem) if terminal: self.env.new_random_episode(self.history, self.replay_memory) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward if self.step >= self.learn_start: if self.step % self.test_step == self.test_step - 1: avg_reward = total_reward / self.test_step avg_loss = self.total_loss / self.update_count avg_q = self.total_q / self.update_count try: max_ep_reward = np.max(ep_rewards) min_ep_reward = np.min(ep_rewards) avg_ep_reward = np.mean(ep_rewards) except: max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0 message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_episodes) self.logger.info(message) if max_avg_ep_reward * 0.9 <= avg_ep_reward: self.sess.run( fetches=self.step_assign_op, feed_dict={self.step_input: self.step + 1} ) self.save_model(self.step + 1) max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward) if self.step > 180: self.inject_summary({ 'average.reward': avg_reward, 'average.loss': avg_loss, 'average.q': avg_q, 'episode.max reward': max_ep_reward, 'episode.min reward': min_ep_reward, 'episode.avg reward': avg_ep_reward, 'episode.num of episodes': num_episodes, 'episode.rewards': ep_rewards, 'episode.actions': actions, 'training.learning_rate': self.sess.run( fetches=self.learning_rate_op, feed_dict={self.learning_rate_step: self.step} ) }, self.step) num_episodes = 0 total_reward = 0. self.total_loss = 0. self.total_q = 0. self.update_count = 0 ep_reward = 0. ep_rewards = [] actions = [] def predict(self, state, test_ep=None): s_t = state[0] trade_rem_t = state[1] ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.config[NUM_ACTIONS]) else: action = self.sess.run( fetches=self.q.action, feed_dict={ self.q.phase: 0, self.s_t: [s_t], self.trade_rem_t: [trade_rem_t], self.q_conv_keep_prob: 1.0, self.q_dense_keep_prob: 1.0, self.q_gru_keep_prob: 1.0 } )[0] return action def observe(self, screen, reward, action, terminal, trade_rem): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal, trade_rem) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: state_t, action, reward, state_t_plus_1, terminal = self.replay_memory.sample s_t, trade_rem_t = state_t[0], state_t[1] s_t_plus_1, trade_rem_t_plus_1 = state_t_plus_1[0], state_t_plus_1[1] q_t_plus_1 = self.sess.run( fetches=self.t_q.values, feed_dict={ self.t_q.phase: 0, self.t_s_t: s_t_plus_1, self.t_trade_rem_t: trade_rem_t_plus_1 } ) max_q_t_plus_1 = np.max(q_t_plus_1, axis=1) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss, avg_q_summary = self.sess.run([self.optimizer, self.q.values, self.loss, self.q.avg_q_summary], { self.q.phase: 1, self.target_q: target_q, self.action: action, self.s_t: s_t, self.trade_rem_t: trade_rem_t, self.q_conv_keep_prob: self.config[CONV_KEEP_PROB], self.q_dense_keep_prob: self.config[DENSE_KEEP_PROB], self.q_gru_keep_prob: self.config[GRU_KEEP_PROB], self.learning_rate_step: self.step }) self.summary_writer.add_summary(avg_q_summary, self.step) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder( dtype=tf.float32, shape=[None, self.replay_memory.history_length, self.replay_memory.num_channels], name=HISTORICAL_PRICES ) self.trade_rem_t = tf.placeholder( dtype=tf.float32, shape=[None,], name=TRADE_REM ) with tf.variable_scope(DROPOUT_KEEP_PROBS): self.q_conv_keep_prob = tf.placeholder(tf.float32) self.q_dense_keep_prob = tf.placeholder(tf.float32) self.q_gru_keep_prob = tf.placeholder(tf.float32) params.dropoutkeepprobs = DropoutKeepProbs( self.q_conv_keep_prob, self.q_dense_keep_prob, self.q_gru_keep_prob ) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model((self.s_t, self.trade_rem_t)) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder( dtype=tf.float32, shape=[None, self.replay_memory.history_length, self.replay_memory.num_channels], name=HISTORICAL_PRICES ) self.t_trade_rem_t = tf.placeholder( dtype=tf.float32, shape=[None,], name=TRADE_REM ) params.dropoutkeepprobs = DropoutKeepProbs() self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model((self.t_s_t, self.t_trade_rem_t)) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list() ) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[name].assign( self.q_weights_placeholders[name] ) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.config[NUM_ACTIONS], 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder(tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum(self.learning_rate_minimum, tf.train.exponential_decay( self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) with tf.variable_scope(SUMMARY): scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \ 'episode.max reward', 'episode.min reward', 'episode.avg reward', \ 'episode.num of episodes', 'training.learning_rate'] self.summary_placeholders = {} self.summary_ops = {} for tag in scalar_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.scalar( name="{}-{}".format(self.env_name, tag.replace(' ', '_')), tensor=self.summary_placeholders[tag] ) histogram_summary_tags = ['episode.rewards', 'episode.actions'] for tag in histogram_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag) self.summary_ops[tag] = \ tf.summary.histogram( tag, self.summary_placeholders[tag] ) self.sess.run(tf.local_variables_initializer()) self.sess.run(tf.global_variables_initializer()) self._saver = tf.train.Saver(self.q.weights.values() + [self.step_op], max_to_keep=30) self.load_model() self.update_target_network() self._summary_writer = tf.summary.FileWriter(self.config[TENSORBOARD_LOG_DIR]) self._summary_writer.add_graph(self.sess.graph) def update_target_network(self): for name in self.q.weights.keys(): self.sess.run( fetches=self.t_weights_assign_ops[name], feed_dict= {self.q_weights_placeholders[name]: self.sess.run( fetches=self.q.weights[name] )} ) def inject_summary(self, tag_dict, step): summary_str_lists = self.sess.run([self.summary_ops[tag] for tag in tag_dict.keys()], { self.summary_placeholders[tag]: value for tag, value in tag_dict.items() }) for summary_str in summary_str_lists: self.summary_writer.add_summary(summary_str, self.step)
def history(cls): return History.select().order_by(History.date.asc()).all()
class AcbEnterprise: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.username = payment.get_username() self.password = payment.get_password() acb = self.get_config() self.email_transport = EmailTransport() self.corp_url = acb['corp_url'] self.captcha = CaptchaResolver() self.debug_mode = acb['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 20 self.login_failed = 0 self.session = session self.proxy = proxy self.payment = payment def get_config(self): config = self.config.get_section_config('AcbEnterprise') return config def perform_login(self): corp_url = self.corp_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver( ) is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: driver.get(corp_url) online_banking = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//a[contains(@href,'obk/login.jsp')]"))) online_banking.click() time.sleep(5) captcha = self.get_captcha(driver) # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="UserName"]'))) element.send_keys(username, Keys.ARROW_DOWN) # Get password input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="PassWord"]'))) driver.execute_script( 'arguments[0].style = ""; arguments[0].style.display = "block"; arguments[0].style.visibility = "visible";', element) element.send_keys(password) # Get captcha input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="SecurityCode"]'))) driver.execute_script( 'arguments[0].style = ""; arguments[0].style.display = "block"; arguments[0].style.visibility = "visible";', element) element.send_keys(captcha) time.sleep(2) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//a[@id="button-dangnhap"]'))) element.click() time.sleep(3) try: # Update Account Information WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//table[@id='table']//tr/td"))) account_info = driver.find_elements_by_xpath( "//table[@id='table']//tr/td") account_number = account_info[0].text.strip() account_name = self.payment.get_username() account_balance = float( account_info[2].text.strip().replace(' ', '').replace( '(VND)', '').replace('.', '')) account = self.update_account(account_name, account_number, account_balance, self.payment.get_id()) except: driver.close() return self.perform_login() # click to transaction menu element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//a[contains(@class,'acc_bold')]"))) element.click() time.sleep(3) view_link = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//input[contains(@onclick,'OutPrintByDate')]"))) view_link.click() try: window_after = driver.window_handles[1] driver.switch_to_window(window_after) WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//table[@id='table1']//tbody"))) transactions = driver.find_elements_by_xpath( "//table[@id='table1']//tbody/tr[position() >= 0 and position() <= 10]" ) for row in transactions: columns = row.find_elements_by_xpath("td") self.save_transaction(account, columns) self.log.update_log('Acb', self.username) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) driver.close() except: self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + "Cannot load transactions", 'error') exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ': ' + str(sys.exc_info()), 'debug') self.log.log( str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno), 'error') self.session.set_changing_proxy(1) except: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ': ' + str(sys.exc_info()), 'debug') self.log.log( str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno), 'error') self.session.set_changing_proxy(1) driver.quit() self.history.set_current_update('acb_enterprise', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): if len(detail) < 4: return False trading_date = self.convert_trading_date(detail[0].text) reference_number = detail[1].text account_id = account.get_account_id() if detail[3].text.strip() is not '': balance = -float(detail[3].text.strip().replace('.', '')) else: balance = float(detail[4].text.strip().replace('.', '')) description = detail[2].text transaction = Transaction(account_id, reference_number, trading_date, balance, description) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def get_captcha(self, driver): img_data = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//img[@src='Captcha.jpg']"))) img_file = img_data.screenshot_as_png self.captcha.save_from_source(img_file, 'png') captcha_text = self.captcha.resolve(True) if re.match("^[a-zA-Z0-9]{5}$", captcha_text): return captcha_text driver.refresh() return self.get_captcha(driver) def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%d-%m-%y') return date.strftime('%Y-%m-%d') def update_account(self, name, number, balance, payment_id): account = AcbAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def is_debug(self): return self.debug_mode
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. add summary ops 2. timing and logging 3. model saving 4. increment self.step ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) def train(self): start_step = self.step_op.eval() num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] self.env.new_random_episode(self.history) for self.step in range(start_step, self.max_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict(self.history.get()) # 2. act screen, reward, terminal = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal) if terminal: self.env.new_random_episode(self.history) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward def predict(self, s_t, test_ep=None): ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.env.action_size) else: action = self.q.action.eval({self.s_t: [s_t]})[0] return action def observe(self, screen, reward, action, terminal): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample( ) max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1}) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss = self.sess.run( [self.optimizer, self.q.values, self.loss], { self.target_q: target_q, self.action: action, self.s_t: s_t, self.learning_rate_step: self.step, }) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model(self.s_t) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model(self.t_s_t, train=False) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list()) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[ name].assign(self.q_weights_placeholders[name]) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.env.action_size, 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder( tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum( self.learning_rate_minimum, tf.train.exponential_decay(self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) # tf.initialize_all_variables().run() #initialize the q network and the target network with the same weights # self.update_target_network() def update_target_network(self): for name in self.q.weights.keys(): self.t_weights_assign_ops[name].eval({ self.q_weights_placeholders[name]: self.q.weights[name].eval() })
class Vietcombank: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.username = payment.get_username() self.password = payment.get_password() vietcombank = self.get_vietcombank_config() self.email_transport = EmailTransport() self.login_url = vietcombank['login_url'] self.captcha = CaptchaResolver() self.debug_mode = vietcombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 20 self.login_failed = 0 self.session = session self.proxy = proxy self.payment = payment def get_vietcombank_config(self): vietcombank = self.config.get_section_config('Vietcombank') return vietcombank def perform_login(self): login_url = self.login_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver() is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: driver.get(login_url) # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="username"]')) ) element.send_keys(username) # Get password input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="pass"]')) ) element.send_keys(password) while True: captcha = self.get_captcha(driver) # Get captcha input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="captcha"]')) ) element.send_keys(captcha) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//button[@id="btnLogin"]')) ) element.click() time.sleep(5) # click to account menu try: account_link = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//a[contains(@class,'has-link-arrow tk')]//div[contains(@class,'item-link-arrow')]")) ) account_link.click() time.sleep(5) self.login_failed = 0 break except: self.login_failed += 1 if self.login_failed > self.max_attempt_login: driver.close() self.history.set_current_update('vietcombank', "%d/%m/%Y") self.session.set_driver(None) # Update account information try: driver.execute_script("window.scrollTo(0, 800);") time.sleep(5) search_button = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//a[contains(@class,'ubtn ubg-primary ubtn-md ripple')]/span[contains(text(),'Tìm kiếm')]")) ) search_button.click() WebDriverWait(driver, 15).until( EC.visibility_of_element_located( (By.XPATH, "//form[@id='ChiTietTaiKhoan']")) ) info = driver.find_element_by_xpath( "//div[@class='list-info']") number = info.find_element_by_xpath("//select/option").get_attribute('value') name = info.find_elements_by_xpath( "//div[contains(@class,'list-info-txt-main')]")[1].text balance = float(info.find_elements_by_xpath( "//div[contains(@class,'list-info-txt-main')]")[2].text.replace( ',', '').replace(' VND', '')) account = self.update_account(name, number, balance, self.payment.get_id()) transactions = driver.find_elements_by_xpath( "//div[@id='toanbo']//div[contains(@class,'list-info-item')]") for row in transactions: columns = row.find_elements_by_xpath(".//div[contains(@class,'td-xs')]") self.save_transaction(account, columns) self.log.update_log('Vietcombank', self.username) self.log.log("Vcb " + self.payment.get_type() + self.payment.get_username() + ": " + str( self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log( "Vcb " + self.payment.get_type() + self.payment.get_username() + ": " + "Cannot load transactions", 'error') self.log.log(str(sys.exc_info()), 'error') self.session.set_changing_proxy(1) except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log(str(sys.exc_info()), 'debug') driver.close() self.history.set_current_update('vietcombank', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): trading_date = self.convert_trading_date( detail[0].find_element_by_xpath("div[contains(@class,'list-info-txt-sub')]").text) description = detail[0].find_element_by_xpath("div[contains(@class,'list-info-txt-main')]").text account_id = account.get_account_id() reference_number = detail[1].find_element_by_xpath("div[contains(@class,'list-info-txt-sub')]").text.replace( 'Số tham chiếu: ', '') balance = float( detail[1].find_element_by_xpath("div[contains(@class,'list-info-txt-main')]").text.replace(',', '').replace( ' ', '')) transaction = Transaction(account_id, reference_number, trading_date, balance, description) print(reference_number) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def get_captcha(self, driver): img_data = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//div[contains(@class,'captcha')]//img")) ) img_file = img_data.screenshot_as_png self.captcha.save_from_source(img_file, 'png') captcha_text = self.captcha.resolve(True) if re.match("^[a-zA-Z0-9]{5}$", captcha_text): return captcha_text ic_reload = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//img[contains(@src,'ic_reload')]")) ) ic_reload.click() return self.get_captcha(driver) def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%d/%m/%Y') return date.strftime('%Y-%m-%d') def update_account(self, name, number, balance, payment_id): account = VietcombankAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def get_config(self, name=None): if name is None: name = 'Vietcombank' vietcombank = self.config.get_section_config(name) return vietcombank def is_debug(self): return self.debug_mode
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. play ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) @property def summary_writer(self): return self._summary_writer def train(self): start_step = self.step_op.eval() num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] self.env.new_random_episode(self.history) for self.step in tqdm(range(start_step, self.max_step), ncols=70, initial=start_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict(self.history.get()) # 2. act screen, reward, terminal = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal) if terminal: self.env.new_random_episode(self.history) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward if self.step >= self.learn_start: if self.step % self.test_step == self.test_step - 1: avg_reward = total_reward / self.test_step avg_loss = self.total_loss / self.update_count avg_q = self.total_q / self.update_count try: max_ep_reward = np.max(ep_rewards) min_ep_reward = np.min(ep_rewards) avg_ep_reward = np.mean(ep_rewards) except: max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0 message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_game) print_and_log_message(message, self.logger) if max_avg_ep_reward * 0.9 <= avg_ep_reward: self.step_assign_op.eval( {self.step_input: self.step + 1}) self.save_model(self.step + 1) max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward) if self.step > 180: self.inject_summary( { 'average.reward': avg_reward, 'average.loss': avg_loss, 'average.q': avg_q, 'episode.max reward': max_ep_reward, 'episode.min reward': min_ep_reward, 'episode.avg reward': avg_ep_reward, 'episode.num of game': num_game, 'episode.rewards': ep_rewards, 'episode.actions': actions, 'training.learning_rate': self.learning_rate_op.eval( {self.learning_rate_step: self.step}), }, self.step) num_game = 0 total_reward = 0. self.total_loss = 0. self.total_q = 0. self.update_count = 0 ep_reward = 0. ep_rewards = [] actions = [] def predict(self, s_t, test_ep=None): ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.env.action_size) else: action = self.q.action.eval({self.s_t: [s_t]})[0] return action def observe(self, screen, reward, action, terminal): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample( ) max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1}) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss, avg_q_summary = self.sess.run( [ self.optimizer, self.q.values, self.loss, self.q.avg_q_summary ], { self.target_q: target_q, self.action: action, self.s_t: s_t, self.learning_rate_step: self.step, }) self.summary_writer.add_summary(avg_q_summary, self.step) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model(self.s_t) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model(self.t_s_t, train=False) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list()) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[ name].assign(self.q_weights_placeholders[name]) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.env.action_size, 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder( tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum( self.learning_rate_minimum, tf.train.exponential_decay(self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) with tf.variable_scope(SUMMARY): scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \ 'episode.max reward', 'episode.min reward', 'episode.avg reward', \ 'episode.num of game', 'training.learning_rate'] self.summary_placeholders = {} self.summary_ops = {} for tag in scalar_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.scalar( name="{}-{}".format(self.env_name, tag), tensor=self.summary_placeholders[tag] ) histogram_summary_tags = ['episode.rewards', 'episode.actions'] for tag in histogram_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.histogram( name=tag, self.summary_placeholders[tag] ) self._summary_writer = tf.summary.FileWriter( config[TENSORBOARD_LOG_DIR]) self._summary_writer.add_graph(sess.graph) tf.initialize_all_variables().run() self._saver = tf.train.Saver(self.q.weights.values + [self.step_op], max_to_keep=30) self.load_model() self.update_target_network() def update_target_network(self): for name in self.q.weights.keys(): self.t_weights_assign_ops[name].eval({ self.q_weights_placeholders[name]: self.q.weights[name].eval() }) def inject_summary(self, tag_dict, step): summary_str_lists = self.sess.run( [self.summary_ops[tag] for tag in tag_dict.keys()], { self.summary_placeholders[tag]: value for tag, value in tag_dict.items() }) for summary_str in summary_str_lists: self.writer.add_summary(summary_str, self.step)
class Techcombank: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.payment = payment self.config = Config() self.log = Log() techcombank = self.get_techcombank_config() self.email_transport = EmailTransport() self.login_url = techcombank['login_url'] self.username = payment.get_username() self.password = payment.get_password() self.debug_mode = techcombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() def perform_login(self): login_url = self.login_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver( ) is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: driver.get(login_url) # Get username input element element = WebDriverWait(driver, 60).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="signOnName"]'))) element.send_keys(username) # element.send_keys(Keys.RETURN) element = WebDriverWait(driver, 60).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="password"]'))) # Get password input element element.send_keys(password) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 60).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="btn_login"]'))) element.click() time.sleep(5) WebDriverWait(driver, 60).until( EC.visibility_of_element_located( (By.XPATH, "//table[contains(@id,'datadisplay_AcctBalance')]"))) sub_accounts = driver.find_elements_by_xpath( "//table[contains(@id,'datadisplay_AcctBalance')]//tr") for i in range(0, len(sub_accounts)): WebDriverWait(driver, 60).until( EC.visibility_of_element_located( (By.XPATH, "//table[contains(@id,'datadisplay_AcctBalance')]" ))) WebDriverWait(driver, 60).until( EC.visibility_of_element_located(( By.XPATH, "//table[contains(@id,'datadisplay_AcctBalance')]//a/img[contains(@src,'View.png')]" ))) sub_account = driver.find_elements_by_xpath( "//table[contains(@id,'datadisplay_AcctBalance')]//tr" )[i] search_btn = sub_account.find_element_by_xpath( ".//a/img[contains(@src,'View.png')]") search_btn.click() # Update account information try: WebDriverWait(driver, 120).until( EC.visibility_of_element_located( (By.XPATH, "//table[@id='enqheader']//tr"))) info = driver.find_elements_by_xpath( "//table[@id='enqheader']//tr") number = info[2].find_elements_by_xpath("td")[1].text try: balance = float(info[2].find_elements_by_xpath( "td")[3].text.replace(',', '')) except: balance = float(0) name = info[3].find_elements_by_xpath("td")[1].text account = self.update_account(name, number, balance, self.payment.get_id()) transactions = driver.find_elements_by_xpath( "//table[starts-with(@id,'datadisplay_MainBody')]//tr" ) count = 0 for row in transactions: count += 1 # Get first 10 records if count > 10: break columns = row.find_elements_by_xpath("td") self.save_transaction(account, columns) self.log.update_log('Techcombank', self.username) self.log.log( "Tcb " + self.payment.get_type() + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log( "Tcb " + self.payment.get_type() + self.payment.get_username() + ": " + "Cannot load transactions", 'error') self.log.log(str(sys.exc_info()), 'error') self.session.set_changing_proxy(1) home_btn = driver.find_element_by_xpath( '//a[@name="AI.QCK.HOME"]') home_btn.click() except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log(str(sys.exc_info()), 'debug') driver.close() self.history.set_current_update('techcombank', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): trading_date = self.convert_trading_date(detail[0].text) if trading_date is None: return 0 description = detail[1].text account_id = account.get_account_id() balance = float(detail[2].text.replace(',', '')) reference_number = self.convert_reference_number(description) transaction = Transaction(account_id, reference_number, trading_date, balance, description) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def convert_reference_number(self, description): description_array = description.split('/') reference_number = description_array[len(description_array) - 1].strip() return reference_number def convert_trading_date(self, trading_date): try: date = datetime.strptime(trading_date, '%d/%m/%Y') return date.strftime('%Y-%m-%d') except: return None def update_account(self, name, number, balance, payment_id): account = TechcombankAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def get_techcombank_config(self): techcombank = self.config.get_section_config('Techcombank') return techcombank def is_debug(self): return self.debug_mode
class VietcombankEnterprise: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.username = payment.get_username() self.password = payment.get_password() vietcombank = self.get_vietcombank_config() self.email_transport = EmailTransport() self.login_url = vietcombank['login_url'] self.captcha = CaptchaResolver() self.debug_mode = vietcombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 5 self.login_failed = 0 self.session = session self.proxy = proxy self.payment = payment def get_vietcombank_config(self): vietcombank = self.config.get_section_config('VietcombankEnterprise') return vietcombank def perform_login(self): login_url = self.login_url selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver( ) is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: driver.get(login_url) if self.solve_captcha(driver) is False: driver.close() return False time.sleep(5) # Update account information try: # driver.execute_script("window.scrollTo(0, 800);") WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//tbody[@id='dstkdd-tbody']"))) account_info = driver.find_elements_by_xpath( "//tbody[@id='dstkdd-tbody']/tr/td") account_name = self.payment.get_username() account_number = account_info[0].text.strip() account_balance = float( account_info[3].text.strip().replace(' VND', '').replace( ',', '')) account = self.update_account(account_name, account_number, account_balance, self.payment.get_id()) WebDriverWait(driver, 10).until( EC.presence_of_element_located(( By.XPATH, "//a[contains(@href,'/IbankingCorp/Accounts/TransactionDetail.aspx')]" ))) reference_links = driver.find_elements_by_xpath( "//a[contains(@href,'/IbankingCorp/Accounts/TransactionDetail.aspx')]" ) reference_links[2].click() #view_link = WebDriverWait(driver, 10).until( # EC.presence_of_element_located( # (By.XPATH, "//a[contains(@id,'bdsdList-tab')]")) #) #driver.execute_script("arguments[0].scrollIntoView();", view_link) #time.sleep(5) #view_link.click() tran_by_date = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//a[contains(@id,'TransByDate')]"))) #tran_by_date.click() driver.execute_script("arguments[0].click();", tran_by_date) time.sleep(5) WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//table[@id='tbTranHis']"))) trans_foot = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//div[@id='tbTranHis-footer']"))) driver.execute_script("arguments[0].scrollIntoView();", trans_foot) transactions = driver.find_elements_by_xpath( "//table[@id='tbTranHis']//tbody/tr[position() >= 0 and position() <= 25]" ) for row in transactions: columns = row.find_elements_by_xpath("td") self.save_transaction(account, columns) self.log.update_log('Vietcombank', self.username) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + "Cannot load transactions", 'error') exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] self.log.log( str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno), 'error') self.session.set_changing_proxy(1) except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ': ' + str(sys.exc_info()), 'debug') driver.close() self.history.set_current_update('vietcombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): trading_date = self.convert_trading_date(detail[0].text) reference_number = detail[1].text.replace('Số tham chiếu: ', '') account_id = account.get_account_id() if detail[2].text is not '': balance = -float(detail[2].text.replace(',', '').replace( ' ', '').replace('VND', '')) else: balance = float(detail[3].text.replace(',', '').replace( ' ', '').replace('VND', '')) description = detail[4].text transaction = Transaction(account_id, reference_number, trading_date, balance, description) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def solve_captcha(self, driver): img_data = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//img[@id='ctl00_Content_Login_Captcha_Captcha']"))) img_file = img_data.screenshot_as_png self.captcha.save_from_source(img_file, 'png') captcha_text = self.captcha.resolve(True) if re.match("^[a-zA-Z0-9]{6}$", captcha_text): captcha = captcha_text username = self.username password = self.password # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="ctl00$Content$Login$TenTC"]'))) element.send_keys(username) # Get password input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="ctl00$Content$Login$MatKH"]'))) element.send_keys(password) # Get captcha input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="ctl00$Content$Login$ImgStr"]'))) element.send_keys(captcha) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@id="ctl00_Content_Login_LoginBtn"]'))) element.click() # click to account menu try: list_account = WebDriverWait(driver, 5).until( EC.presence_of_element_located( (By.XPATH, "//a[contains(@href,'Accounts/ListAccounts.aspx')]"))) time.sleep(5) list_account.click() self.login_failed = 0 return True except: self.login_failed += 1 if self.login_failed > self.max_attempt_login: self.history.set_current_update('vietcombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) return False driver.back() return self.solve_captcha(driver) driver.refresh() return self.solve_captcha(driver) def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%Y-%m-%d') return date.strftime('%Y-%m-%d') def update_account(self, name, number, balance, payment_id): account = VietcombankAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def get_config(self, name=None): if name is None: name = 'Vietcombank' techcombank = self.config.get_section_config(name) return techcombank def is_debug(self): return self.debug_mode
def train(self, episodes, time_steps, update_steptime, env, path): path_save = './results/saved_models/' + path stats = EpisodeStats(episode_lengths=np.zeros(episodes), episode_rewards=np.zeros(episodes)) print("START!") collector = History() avg_length = 0 accum_reward = 0 state = env.reset() max_length_episode = time_steps episode_reward = 0 length_episode = 0 i_episode = 1 while i_episode < episodes: for _ in range(update_steptime): action, log_prob, state = self.get_action(state) next_state, reward, done, _ = env.step( action.cpu().data.numpy().flatten()) #Store the enviroment's reaction to action taken by Policy_old collector.rewards.append(reward) collector.dones.append(done) collector.states.append(state) collector.actions.append(action) collector.logprob_actions.append(log_prob) state = next_state episode_reward += reward length_episode += 1 if done or length_episode == max_length_episode: state = env.reset() done = True avg_length += length_episode accum_reward += episode_reward stats.episode_lengths[i_episode - 1] = length_episode stats.episode_rewards[i_episode - 1] = episode_reward length_episode = 0 episode_reward = 0 i_episode += 1 if i_episode % self._print_timesteps == 0: print( 'Episode {}/{} \tAvg length: {} \tAvg reward: {}'. format(i_episode + self.episode, episodes, int(avg_length / 50), int(accum_reward / 50))) avg_length = 0. accum_reward = 0. if i_episode % self._save_timestep == 0: torch.save( { 'episodes': i_episode + self.episode, 'model_state_dict': self._ac.state_dict(), 'optimizer_state_dict': self._opt.state_dict(), 'loss': self.loss_CLIP_VF_S }, path_save + str(i_episode + self.episode) + ".pkl") print("created and saved " + path + str(i_episode + self.episode) + " models") if i_episode % self._save_timestep == 0: np.save( './results/stats/' + path + str(i_episode + self.episode) + '_episodes', stats.episode_lengths) np.save( './results/stats/' + path + str(i_episode + self.episode) + '_reward', stats.episode_rewards) self.update_actorcritic(collector) collector.reset()
class Msb: def __init__(self, payment, session=None, proxy={}): self.payment = payment self.session = session self.proxy = {} self.captcha = CaptchaResolver() self.config = Config() self.log = Log() self.history = History() self.code = GenerateCode() self.email_transport = EmailTransport() section_config = self.get_msb_config() self.username = payment.get_username() self.password = payment.get_password() self.home_url = section_config['home_url'] self.login_url = section_config['login_url'] self.account_list_url = section_config['account_list_url'] self.account_info_url = section_config['account_info_url'] self.debug_mode = section_config['debug_mode'] self.total_transactions = 0 self.max_attempt_login = 10 self.max_validate_captcha = 10 self.login_failed = 0 self.captcha_failed = 0 def get_msb_config(self): msb = self.config.get_section_config('Msb') return msb def is_logged_in(self): session_requests = self.session token = session_requests.get_token() if token is None: return 0 try: account_list_url = self.account_list_url payload = { 'acctType': "'CA','SA','LN'", 'status': "'ACTV','DORM','MATU'", 'tokenNo': token, 'lang': "vi_VN", } list_post = session_requests.post(account_list_url, payload) response = list_post.json() if response['status'] == '200': return 1 return 0 except Exception as e: if self.debug_mode == 'true': self.log.log(str(e), 'debug') return 0 def is_debug(self): return self.debug_mode def perform_login(self): home_url = self.home_url login_url = self.login_url username = self.username password = self.password session_requests = self.session if self.is_logged_in() == 0: session_requests.set_tree(None) session_requests.init_session(self.proxy) session_requests.set_changing_proxy(1) result = session_requests.get(login_url) tree = html.fromstring(result.text) window_location = tree.xpath("//script/text()")[0] window_location = self.replace_all(window_location, { "window.location.href = '": "", "';": "" }) login_redirect_url = str(home_url + window_location.strip()) result = session_requests.get(login_redirect_url, dict(referer=login_redirect_url)) tree = html.fromstring(result.text) window_location = tree.xpath("//script/text()")[0] window_location = self.replace_all(window_location, { "window.location.href = '": "", "';": "" }) real_login_redirect_url = str(home_url + window_location.strip()) result = session_requests.get(real_login_redirect_url, dict(referer=login_redirect_url)) tree = html.fromstring(result.content) dse_session_id = tree.xpath( "//input[@name='dse_sessionId']/@value")[0] img_data = session_requests.get( home_url + '/IBSRetail/servlet/ImageServlet').content self.captcha.save_from_source(img_data) captcha_text = self.captcha.resolve(True) if self.validate_captcha(captcha_text) == 0: if self.captcha_failed < self.max_validate_captcha: self.captcha_failed = self.captcha_failed + 1 return self.perform_login() else: if self.debug_mode == 'true': self.log.log( 'Can\'t validate captcha after ' + str(self.captcha_failed) + ' times', 'debug') self.captcha_failed = 0 return 0 payload = { 'dse_sessionId': dse_session_id, 'dse_applicationId': -1, 'dse_pageId': 2, 'dse_operationName': 'retailUserLoginProc', 'dse_errorPage': 'index.jsp', 'dse_processorState': 'initial', 'dse_nextEventName': 'start', 'orderId': '', '_userNameEncode': username, '_userName': username, '_password': password, '_verifyCode': captcha_text } # Perform login login_post = session_requests.post(login_url + '/Request', payload) tree = html.fromstring(login_post.text) window_location = tree.xpath("//script/text()")[0] window_location = self.replace_all(window_location, { "window.location.href = '": "", "';": "" }) redirect_url = str(home_url + window_location.strip()) result = session_requests.get(redirect_url) tree = html.fromstring(result.content) username = tree.xpath("//p[@class='username']/text()") if len(username) > 0: token = self.get_token_from_string(str(result.content)) session_requests.set_token(token) session_requests.set_changing_proxy(0) self.login_failed = 0 else: self.login_failed = self.login_failed + 1 if self.login_failed >= self.max_attempt_login: if self.debug_mode == 'true': self.log.log( "Can not login Msb:" + self.payment.get_username + " with " + str(self.max_attempt_login) + " attempts", 'error') return 0 return self.perform_login() else: session_requests.set_changing_proxy(0) if self.debug_mode == 'true': self.log.log( "Login Msb:" + self.payment.get_username + " successfully by using old session", 'debug') token = session_requests.get_token() if token is None: return 0 # with open('msb.html', 'w') as f: # f.write(str(result.content)) self.update_transactions(session_requests, token) def update_transactions(self, session_requests, token): account_list_url = self.account_list_url payload = { 'acctType': "'CA','SA','LN'", 'status': "'ACTV','DORM','MATU'", 'tokenNo': token, 'lang': "vi_VN", } list_post = session_requests.post(account_list_url, payload) try: response = list_post.json() except: response = {'status': 401} if response['status'] == '200': accounts = response['data'] for account in accounts: account_name = account['acctName'] account_number = account['acctNo'] account_balance = account['availableBalance'] msb_account = MsbAccount(account_name, account_number, self.payment.get_id()) msb_account.set_balance(account_balance) msb_account.update_account() account_info_url = self.account_info_url current_date = self.history.get_current_update('maritimebank') payload = { 'acctNo': msb_account.get_number(), 'page': 1, 'tokenNo': token, 'lang': "vi_VN", 'fromDate': current_date.rstrip('\r\n'), 'toDate': time.strftime("%Y-%m-%d") } account_post = session_requests.post(account_info_url, payload) response = account_post.json() if response['status'] == '200': histories = response['data']['history'] for history in histories: self.save_transaction(msb_account, history) self.log.update_log('Maritimebank', self.username) self.history.set_current_update('maritimebank', "%Y-%m-%d") self.log.log( "Msb " + self.payment.get_type() + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') def save_transaction(self, account, history): trading_date = self.convert_trading_date(history['transferDate']) trading_time = history['transferTime'] description = history['remark'] if description is None: description = 'None' account_id = account.get_account_id() balance = float(history['amount']) if history['dcSign'] == 'D': balance = -balance reference_number = self.code.generate_code(description + history['transferDate']) created_at = trading_date + ' ' + trading_time transaction = Transaction(account_id, reference_number, trading_date, balance, description, created_at) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def replace_all(self, text, conditions): for i, j in conditions.items(): text = text.replace(i, j) return text def validate_captcha(self, captcha): if re.match("^[a-zA-Z0-9]{4}$", captcha): return 1 return 0 def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%d/%m/%Y') return date.strftime('%Y-%m-%d') def get_token_from_string(self, string): token = re.search( "\'[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}", string) return token[0].replace("'", "")
def __init__(self): self.nds = NodeStore() self.cursor_y = 0 self._load_data() self._update = True self.history = History(seed=self.nds)
class UserFile: DATA_FILE = os.path.expanduser("~/.cache/.wfclidata") root_node_id = "0" # SETUP METHODS def __init__(self): self.nds = NodeStore() self.cursor_y = 0 self._load_data() self._update = True self.history = History(seed=self.nds) # Decorator for funtions that need to force an update to our tree def update_visible_after(func): def do_update(self, *args, **kwargs): result = func(self, *args, **kwargs) self._update = True return result return do_update def update_visible_now(self): self._update = True def set_cursor_to_node(self, node_id): self.update_visible_now() for i, v in enumerate(self.visible): if v[0].uuid == node_id: self.cursor_y = i def current_node(self, depth=False): node_pair = self.visible[self.cursor_y] if depth: return node_pair else: return node_pair[0] # FILE METHODS def data_from_file_object(self, fo): data = json.load(fo) for node_def in data: node = Node(node_def=node_def) self.nds.add_node(node) @classmethod def _write_data_file(cls, data_obj): os.makedirs(os.path.dirname(cls.DATA_FILE), exist_ok=True) with open(cls.DATA_FILE, "x+") as f: json.dump(data_obj, f, indent=2) @classmethod def _create_empty_data_file(cls): empty_data = [ {"id": cls.root_node_id, "pa": None, "ch": ["1"]}, {"pa": cls.root_node_id, "id": "1", "nm": "Write down your thoughts"}, ] cls._write_data_file(empty_data) def _load_data(self): if not os.path.exists(self.DATA_FILE): self._create_empty_data_file() with open(self.DATA_FILE) as f: self.data_from_file_object(f) def save(self): with open(self.DATA_FILE, "w") as f: json.dump(self.nds.flat_format, f, indent=2) def commit(self): self.history.add(self.nds) # TREE TRAVERSAL @property def visible(self): if self._update: self._update = False return self.load_visible() else: return self._visible def load_visible(self): """ returns a list of tuples like this ( node, depth,) """ self._visible = [] for node in self.nds.get_node(self.root_node_id).children: if node is not None: self._traverse_node(node, 0) return self._visible def _traverse_node(self, node, depth): current_node = self.nds.get_node(node) self._visible.append((current_node, depth)) if not current_node.closed: for child in current_node.children: self._traverse_node(child, depth + 1) # NAVIGATION METHODS def nav_up(self): if self.cursor_y > 0: self.cursor_y -= 1 def nav_down(self): if self.cursor_y < len(self.visible) - 1: self.cursor_y += 1 def bottom(self): self.cursor_y = len(self.visible) - 1 def top(self): self.cursor_y = 0 # LINKING METHODS @update_visible_after def link_parent_child(self, parent, child, position=-1): self.nds.get_node(child).parent = parent if position >= 0: self.nds.get_node(parent).children.insert(position, child) else: self.nds.get_node(parent).children.append(child) @update_visible_after def unlink_relink(self, old_parent, child, new_parent, position): def unlink_parent_child(self, parent, child): assert child in self.nds assert parent in self.nds assert self.nds.get_node(child).parent == parent assert child in self.nds.get_node(parent).children self.nds.get_node(parent).children.remove(child) self.nds.get_node(child).parent = None unlink_parent_child(self, old_parent, child) self.link_parent_child(new_parent, child, position) # MANIPULATE NODES @update_visible_after def indent(self): current_node = self.current_node() parent_node = current_node.parent parents_child_list = self.nds.get_node(parent_node).children current_node_index = parents_child_list.index(current_node.uuid) if current_node_index == 0: raise ModelException("Indent of top child") else: new_parent = parents_child_list[current_node_index - 1] self.unlink_relink(parent_node, current_node.uuid, new_parent, -1) self.nds.get_node(new_parent).closed = False log.info("Nailed it") @update_visible_after def unindent(self): current_node = self.current_node() parent_id = current_node.parent if parent_id == self.root_node_id: raise ModelException("top level, can't unindent") else: super_parent_node = self.nds.get_node(self.nds.get_node(parent_id).parent) pos_in_parent_list = super_parent_node.children.index(parent_id) self.unlink_relink( parent_id, current_node.uuid, super_parent_node.uuid, pos_in_parent_list + 1, ) log.info("nailed it") @update_visible_after def open_above(self): current_node = self.current_node() parent_node = self.nds.get_node(current_node.parent) new_node = self.create_node(parent_node.uuid) pos_in_parent_list = parent_node.children.index(current_node.uuid) self.link_parent_child( parent_node.uuid, new_node.uuid, pos_in_parent_list, ) @update_visible_after def open_below(self): current_node = self.current_node() if current_node.state == "open": new_node = self.create_node(current_node.uuid) self.link_parent_child( current_node.uuid, new_node.uuid, 0, ) else: # new node is sibling of current node parent_node = self.nds.get_node(current_node.parent) new_node = self.create_node(parent_node.uuid) pos_in_parent_list = parent_node.children.index(current_node.uuid) self.link_parent_child( parent_node.uuid, new_node.uuid, pos_in_parent_list + 1, ) @update_visible_after def delete_item(self, node_id=None): current_node = self.current_node() if node_id is None else self.nds.get_node(node_id) for child_id in current_node.children[:]: self.delete_item(node_id=child_id) parent_id = current_node.parent self.nds.get_node(parent_id).children.remove(current_node.uuid) del self.nds[current_node.uuid] if node_id is None: # this is our top-level delete self.cursor_y = max(0, self.cursor_y - 1) if len(self.nds.get_node(self.root_node_id).children) == 0: new_node = self.create_node( self.root_node_id, nm="Ooops, you deleted the last item on the list", ) self.link_parent_child( self.root_node_id, new_node.uuid, 0, ) @update_visible_after def move_down(self): current_node = self.current_node() parent_id = current_node.parent parents_child_list = self.nds.get_node(parent_id).children current_node_index = parents_child_list.index(current_node.uuid) if current_node_index < len(parents_child_list) - 1: # swap with the one behind parents_child_list[current_node_index] = parents_child_list[current_node_index + 1] parents_child_list[current_node_index + 1] = current_node.uuid self.set_cursor_to_node(current_node.uuid) @update_visible_after def move_up(self): current_node = self.current_node() parent_id = current_node.parent parents_child_list = self.nds.get_node(parent_id).children current_node_index = parents_child_list.index(current_node.uuid) if current_node_index > 0: # swap with the one behind parents_child_list[current_node_index] = parents_child_list[current_node_index - 1] parents_child_list[current_node_index - 1] = current_node.uuid self.set_cursor_to_node(current_node.uuid) @update_visible_after def complete(self): current_node = self.current_node() current_node.complete = not current_node.complete @update_visible_after def create_node(self, parent, **kwargs): node = Node(pa=parent, **kwargs) self.nds.add_node(node) return node @update_visible_after def collapse_node(self): self.visible[self.cursor_y][0].closed = True @update_visible_after def expand_node(self): self.visible[self.cursor_y][0].closed = False @update_visible_after def undo(self): ret = self.history.undo() if ret is not None: self.nds = ret @update_visible_after def redo(self): ret = self.history.redo() if ret is not None: self.nds = ret # EDIT TEXT @update_visible_after def add_char(self, char, cursor_x): current_node = self.current_node() name = current_node.name[0:cursor_x] + char + current_node.name[cursor_x:] current_node.name = name @update_visible_after def delete_char(self, num, cursor_x): current_node = self.current_node() if cursor_x > 0: name = current_node.name[0:cursor_x - num] + current_node.name[cursor_x:] current_node.name = name
class SacombankEnterprise: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.username = payment.get_username() self.password = payment.get_password() sacombank = self.get_config() self.email_transport = EmailTransport() self.corp_url = sacombank['corp_url'] self.captcha = CaptchaResolver() self.debug_mode = sacombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 20 self.login_failed = 0 self.session = session self.proxy = proxy self.payment = payment def get_config(self): config = self.config.get_section_config('SacombankEnterprise') return config def perform_login(self): corp_url = self.corp_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver( ) is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: while True: driver.get(corp_url) online_banking = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//a[contains(@class,'btn-ebanking')]"))) online_banking.click() time.sleep(5) try: # Click to ignore banner element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//div[@id="pbanner"]'))) element.click() except: print('Banner not found') # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located(( By.XPATH, '//input[@id="AuthenticationFG.USER_PRINCIPAL"]'))) element.send_keys(username, Keys.ARROW_DOWN) # Get captcha input element captcha = self.get_captcha(driver) element = WebDriverWait(driver, 10).until( EC.presence_of_element_located(( By.XPATH, '//input[@name="AuthenticationFG.VERIFICATION_CODE"]' ))) element.send_keys(captcha) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@id="STU_VALIDATE_CREDENTIALS"]'))) element.click() time.sleep(5) try: element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//span[@id="LoginHDisplay.Ra4.C1"]'))) element.click() except: print('Can not login') driver.close() return self.perform_login() # Get password input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="AuthenticationFG.ACCESS_CODE"]'))) element.send_keys(password) element.send_keys(Keys.RETURN) time.sleep(5) # Update Account Information WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//table[@id='HWListTable10072682']//tr/td"))) account_info = driver.find_elements_by_xpath( "//table[@id='HWListTable10072682']//tr/td") account_number = account_info[0].text.strip() account_name = self.payment.get_username() account_balance = float( account_info[2].text.strip().replace('\n', '').replace( 'VND', '').replace('.', '')) account = self.update_account(account_name, account_number, account_balance, self.payment.get_id()) # click to transaction menu action_link = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//a[contains(@name,'HREF_Ti_khon')]"))) hover_action = ActionChains(driver).move_to_element( action_link) hover_action.perform() acc_link = WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.XPATH, "//a[contains(@id,'Ti-khon_Tng-quan-ti-khon')]"))) hover_trans = ActionChains(driver).move_to_element( acc_link) hover_trans.perform() acc_link.click() # time.sleep(5) trans_link = WebDriverWait(driver, 10).until( EC.element_to_be_clickable(( By.XPATH, "//a[contains(@id,'HREF_PageConfigurationMaster_CACCTSW__1:AccountSummaryFG')]" ))) trans_link.click() try: time.sleep(5) WebDriverWait(driver, 5).until( EC.visibility_of_element_located(( By.XPATH, "//a[contains(@id,'PageConfigurationMaster_CACCTSW__1:errorlink1')]" ))) driver.close() self.history.set_current_update( 'sacombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + "No transactions today", 'message') return False except: print('ok') time.sleep(1) try: WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//table[contains(@id,'HWListTable')]//tbody" ))) transactions = driver.find_elements_by_xpath( "//table[contains(@id,'HWListTable')]//tbody[position() >= 0 and position() <= 10]" ) for row in transactions: columns = row.find_elements_by_xpath(".//tr/td") self.save_transaction(account, columns) self.log.update_log('Sacombank', self.username) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + "Cannot load transactions", 'error') exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] self.log.log( str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno), 'error') self.session.set_changing_proxy(1) except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ': ' + str(sys.exc_info()), 'debug') driver.close() self.history.set_current_update('sacombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): reference_number = detail[0].text trading_date = self.convert_trading_date(detail[2].text) account_id = account.get_account_id() description = detail[3].text if detail[6].text is not '': balance = -float(detail[6].text.replace(',', '').replace( ' ', '').replace('VND', '')) else: balance = float(detail[7].text.replace(',', '').replace( ' ', '').replace('VND', '')) transaction = Transaction(account_id, reference_number, trading_date, balance, description) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def get_captcha(self, driver): img_data = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, "//img[@id='IMAGECAPTCHA']"))) img_file = img_data.screenshot_as_png self.captcha.save_from_source(img_file, 'png') captcha_text = self.captcha.resolve(True) if re.match("^[0-9]{4}$", captcha_text): return captcha_text reload_captcha = WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//img[@id='TEXTIMAGE']"))) reload_captcha.click() return self.get_captcha(driver) def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%d-%m-%Y') return date.strftime('%Y-%m-%d') def update_account(self, name, number, balance, payment_id): account = SacombankAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def is_debug(self): return self.debug_mode
class TechcombankEnterprise: def __init__(self, payment, session=None, proxy={}): self.session = session self.proxy = proxy self.payment = payment self.config = Config() self.log = Log() techcombank = self.get_techcombank_config() self.email_transport = EmailTransport() self.login_url = techcombank['login_url'] self.username = payment.get_username() self.password = payment.get_password() self.debug_mode = techcombank['debug_mode'] self.total_transactions = 0 self.history = History() self.code = GenerateCode() self.max_attempt_login = 20 self.login_failed = 0 def perform_login(self): login_url = self.login_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver( ) is None or self.session.get_last_driver() is 'Firefox': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Chrome') else: driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') self.session.set_driver(driver) try: while 1: driver.get(login_url) window_before = driver.window_handles[0] # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="signOnName"]'))) element.send_keys(username) time.sleep(2) # element.send_keys(Keys.RETURN) element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="password"]'))) # Get password input element element.send_keys(password) time.sleep(2) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@type="submit"]'))) element.click() try: WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//div[contains(@id,'pane__FRAME')]"))) self.login_failed = 0 break except: self.login_failed += 1 if self.login_failed > self.max_attempt_login: driver.close() self.history.set_current_update( 'techcombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) # Update account information try: search = WebDriverWait(driver, 30).until( EC.visibility_of_element_located( (By.XPATH, "//img[contains(@src,'drilldown/View.png')]"))) info = driver.find_elements_by_xpath( "//table[contains(@id,'datadisplay_ACCOUNT')]//tr/td") number = info[0].text.split('-')[0].strip() name = info[1].text balance = float(info[3].text.replace(',', '')) account = self.update_account(name, number, balance, self.payment.get_id()) search.click() window_after = driver.window_handles[1] driver.switch_to_window(window_after) WebDriverWait(driver, 60).until( EC.visibility_of_element_located( (By.XPATH, "//table[contains(@id,'datadisplay_DATA')]"))) transactions = driver.find_elements_by_xpath( "//table[contains(@id,'datadisplay_DATA')]//tr") for row in transactions: columns = row.find_elements_by_xpath("td") self.save_transaction(account, columns) self.log.update_log('Techcombank', self.username) self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + str(self.total_transactions) + ' transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log( self.payment.get_name() + '-' + self.payment.get_type() + '-' + self.payment.get_username() + ": " + "Cannot load transactions", 'error') exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] self.log.log( str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno), 'error') self.session.set_changing_proxy(1) except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log(str(sys.exc_info()), 'debug') driver.quit() self.history.set_current_update('techcombank_enterprise', "%d/%m/%Y") self.session.set_driver(None) def save_transaction(self, account, detail): trading_date = self.convert_trading_date(detail[0].text) reference_number = detail[1].text description = detail[3].text account_id = account.get_account_id() if detail[4].text is not '': balance = float(detail[4].text.replace(',', '').replace(' ', '')) else: balance = float(detail[5].text.replace(',', '').replace(' ', '')) transaction = Transaction(account_id, reference_number, trading_date, balance, description) if transaction.save() == 1: self.total_transactions = self.total_transactions + 1 self.email_transport.send_transaction_email(account, transaction) def convert_trading_date(self, trading_date): date = datetime.strptime(trading_date, '%d/%m/%Y') return date.strftime('%Y-%m-%d') def update_account(self, name, number, balance, payment_id): account = TechcombankAccount(name, number, payment_id) account.set_balance(balance) account.update_account() return account def get_techcombank_config(self): techcombank = self.config.get_section_config('TechcombankEnterprise') return techcombank def is_debug(self): return self.debug_mode
def __create_objects(query): return_list = [] for history in [i for i in query.cursor.fetchall()]: return_list.append(History.from_json(history)) return return_list
class Msb: def __init__(self, name=None, session=None, proxy={}): self.session = session self.proxy = proxy self.config = Config() self.log = Log() self.captcha = CaptchaResolver() section_config = self.get_section_config(name) self.email_transport = EmailTransport() self.home_url = section_config['home_url'] self.login_url = section_config['login_url'] self.username = section_config['username'] self.password = section_config['password'] self.debug_mode = section_config['debug_mode'] self.total_transactions = 0 self.max_attempt_login = 10 self.max_validate_captcha = 10 self.login_failed = 0 self.captcha_failed = 0 self.history = History() self.code = GenerateCode() def is_debug(self): return self.debug_mode def get_section_config(self, name=None): if name is None: name = 'Msb' section_config = self.config.get_section_config(name) return section_config def perform_login(self): login_url = self.login_url username = self.username password = self.password selenium = Selenium() if self.session.get_driver() is None: if self.session.get_last_driver() is None or self.session.get_last_driver() is 'Chrome': driver = selenium.get_firefox_driver(self.proxy) self.session.set_last_driver('Firefox') else: driver = selenium.get_chrome_driver(self.proxy) self.session.set_last_driver('Chrome') self.session.set_driver(driver) try: driver.get(login_url) # Get username input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="_userName"]')) ) element.send_keys(username) # Get password input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="_passwordS"]')) ) element.send_keys(password) captcha = self.get_captcha(driver) print(captcha) exit(111111) # Get captcha input element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//input[@name="_verifyCode"]')) ) element.send_keys(captcha) # element.send_keys(Keys.RETURN) # Get submit button element element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.XPATH, '//button[@id="loginBtn"]')) ) element.click() # click to account menu time.sleep(5) account_link = WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, '//a[@name="AI.QCK.ACCOUNT"]')) ) retry_account = 0 while retry_account < 3: try: account_link.click() # Click to view last 10 transactions time.sleep(5) current_date = self.history.get_current_update('techcombank') from_date = driver.find_elements_by_xpath( "//input[@id='fieldName:START.DATE']")[0] from_date.send_keys(current_date, Keys.ENTER) time.sleep(5) to_date = driver.find_elements_by_xpath( "//input[@id='fieldName:END.DATE']")[0] to_date.send_keys(current_date) time.sleep(5) button = driver.find_elements_by_xpath( '//a[@title="Xem giao dịch"]')[1] button.click() time.sleep(5) button = driver.find_elements_by_xpath( '//a[@title="Xem giao dịch"]')[1] button.click() break except: self.log.log("Techcombank: Re-click to account", 'debug') self.log.log(str(sys.exc_info()), 'debug') retry_account = retry_account + 1 # Update account information try: WebDriverWait(driver, 15).until( EC.visibility_of_element_located( (By.XPATH, "//table[@id='enqheader']//tr[position()>1 and position()<4]")) ) info = driver.find_elements_by_xpath( "//table[@id='enqheader']//tr[position()>1 and position()<4]") number = info[0].find_elements_by_xpath("td")[1].text balance = float(info[0].find_elements_by_xpath("td")[3].text.replace(',', '')) name = info[1].find_elements_by_xpath("td")[1].text account = self.update_account(name, number, balance) transactions = driver.find_elements_by_xpath( "//table[starts-with(@id,'datadisplay_MainBody')]//tr") for row in transactions: columns = row.find_elements_by_xpath("td") self.save_transaction(account, columns) self.log.update_log('Techcombank', self.username) self.log.log(str(self.total_transactions) + ' Tcb transaction(s) created', 'message') self.session.set_changing_proxy(0) except: self.log.log("Techcombank: Cannot load transactions", 'error') self.log.log(str(sys.exc_info()), 'error') self.session.set_changing_proxy(1) except: exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.log.log(str(sys.exc_info()), 'debug') driver.close() self.history.set_current_update('techcombank', "%d/%m/%Y") self.session.set_driver(None) def get_captcha(self, driver): img_data = driver.find_element_by_id('safecode').screenshot_as_png self.captcha.save_from_source(img_data) captcha_text = self.captcha.resolve(True) if re.match("^[a-zA-Z0-9]{4}$", captcha_text): return captcha_text return self.get_captcha(driver)