def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.payment = payment
self.config = Config()
self.log = Log()
techcombank = self.get_techcombank_config()
self.email_transport = EmailTransport()
self.login_url = techcombank['login_url']
self.username = payment.get_username()
self.password = payment.get_password()
self.debug_mode = techcombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
def __init__(self, sess, logger, config, env):
super(Agent, self).__init__(config, logger)
self.sess = sess
self.logger = logger
self.config = config
params = DeepSenseParams(config)
self.env = env
self.history = History(logger, config)
self.replay_memory = ReplayMemory(logger, config)
with tf.variable_scope(STEPS):
self.step_op = tf.Variable(0, trainable=False, name=STEP)
self.step_input = tf.placeholder('int32', None, name=STEP_INPUT)
self.step_assign_op = self.step_op.assign(self.step_input)
self.build_dqn(params)
def set_as_used(id_ticket, id_user):
t = Ticket.get_ticket(id_ticket)
if t == None:
raise ErrorCodeException(ErrorCode.TICKET_DOESNT_EXISTS)
if t.id_user != id_user:
raise ErrorCodeException(ErrorCode.TICKET_DOESNT_BELONG_TO_USER)
if User.get_user(id_user) == None:
raise ErrorCodeException(ErrorCode.USER_DOESNT_EXISTS)
if t.used == True:
raise ErrorCodeException(ErrorCode.TICKET_ALREADY_USED)
t.set_used()
History.add_entry(History(id_ticket=unhexlify(id_ticket), id_user=id_user))
def __init__(self, name=None, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.captcha = CaptchaResolver()
section_config = self.get_section_config(name)
self.email_transport = EmailTransport()
self.home_url = section_config['home_url']
self.login_url = section_config['login_url']
self.username = section_config['username']
self.password = section_config['password']
self.debug_mode = section_config['debug_mode']
self.total_transactions = 0
self.max_attempt_login = 10
self.max_validate_captcha = 10
self.login_failed = 0
self.captcha_failed = 0
self.history = History()
self.code = GenerateCode()
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.username = payment.get_username()
self.password = payment.get_password()
sacombank = self.get_config()
self.email_transport = EmailTransport()
self.corp_url = sacombank['corp_url']
self.captcha = CaptchaResolver()
self.debug_mode = sacombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 20
self.login_failed = 0
self.session = session
self.proxy = proxy
self.payment = payment
def register(wagon_id, camera_id):
current_camera = WagonDB.get_by_id(wagon_id)[0].get_last_camera()
if current_camera is not None and int(current_camera) is int(
camera_id):
return True
else:
w = Wagon(wagon_id, camera_id, datetime.datetime.now())
h = History(wagon_id, camera_id, datetime.datetime.now())
if WagonDB.update_last_camera_id(w) and HistoryDB.add_item(h):
return True
else:
return False
def get(self, id_user):
sender_id_user = get_jwt_identity()
if sender_id_user != id_user:
return unauthorized("Forbiden!"), 403
try:
user_hist = History.get_all(id_user)
return {
"history":
[he.to_json() for he in user_hist] if user_hist else []
}, 200
except ErrorCodeException as ec:
return error_code(ec), 500
def __init__(self, payment, session=None, proxy={}):
self.payment = payment
self.session = session
self.proxy = {}
self.captcha = CaptchaResolver()
self.config = Config()
self.log = Log()
self.history = History()
self.code = GenerateCode()
self.email_transport = EmailTransport()
section_config = self.get_msb_config()
self.username = payment.get_username()
self.password = payment.get_password()
self.home_url = section_config['home_url']
self.login_url = section_config['login_url']
self.account_list_url = section_config['account_list_url']
self.account_info_url = section_config['account_info_url']
self.debug_mode = section_config['debug_mode']
self.total_transactions = 0
self.max_attempt_login = 10
self.max_validate_captcha = 10
self.login_failed = 0
self.captcha_failed = 0
def __init__(self, sess, logger, config, env):
super(Agent, self).__init__(config, logger)
self.sess = sess
self.logger = logger
self.config = config
params = DeepSenseParams(config)
self.env = env
self.history = History(logger, config)
self.replay_memory = ReplayMemory(logger, config)
with tf.variable_scope(STEPS):
self.step_op = tf.Variable(0, trainable=False, name=STEP)
self.step_input = tf.placeholder('int32', None, name=STEP_INPUT)
self.step_assign_op = self.step_op.assign(self.step_input)
self.build_dqn(params)
def sync(cls):
now = datetime.now().strftime('%Y-%m-%d_%H')
log.info(f'begin sync {now}')
file_name = f"./data/{now}.json"
if not os.path.exists(file_name):
CrawlerService.crawler()
raw_data = json.load(open(file_name, 'r'))
data = raw_data['nCoV2019']
date = datetime.now().strftime('%Y%m%d')
world = World(date=date,
name='中国',
confirmed=data['totalConNum'],
suspected=data['totalSusNum'],
cured=data['totalCureNum'],
dead=data['totalDeathNum'],
add_confirmed=data['addCon'],
add_cured=data['addCure'],
add_dead=data['addDeath'],
add_suspected=data['addSus'])
history_list = data['historyList']
histories = []
for history in history_list:
histories.append(
History(date=history['date'],
confirmed=history['conNum'],
cured=history['cureNum'],
dead=history['deathNum'],
suspected=history['susNum']))
overseas_list = data['overseasList']
worlds = []
for overseas in overseas_list:
worlds.append(
World(date=date,
confirmed=overseas['conNum'],
cured=overseas['cureNum'],
suspected=overseas['susNum'],
dead=overseas['deathNum'],
name=overseas['name']))
domestic_list = data['domesticList']
provinces = []
cities = []
for domestic in domestic_list:
provinces.append(
Province(date=date,
province_name=domestic['name'],
short_name=domestic['name'],
confirmed=domestic['conNum'],
dead=domestic['deathNum'],
suspected=domestic['susNum'],
cured=domestic['cureNum']))
city_list = domestic['cities']
for city in city_list:
cities.append(
City(date=date,
province_name=domestic['name'],
city_name=city['name'],
confirmed=city['conNum'],
suspected=city['susNum'],
cured=city['cureNum'],
dead=city['deathNum']))
History.select().delete()
History.bulk_insert(histories)
World.select().filter(World.date == date).delete()
world.insert()
World.bulk_insert(worlds)
Province.select().filter(Province.date == date).delete()
Province.bulk_insert(provinces)
City.select().filter(City.date == date).delete()
City.bulk_insert(cities)
log.info(f'sync {now} done')
class Agent(BaseAgent):
'''Deep Trading Agent based on Deep Q Learning'''
'''TODO:
1. add `play` function to run tests in the simulated environment
'''
def __init__(self, sess, logger, config, env):
super(Agent, self).__init__(config, logger)
self.sess = sess
self.logger = logger
self.config = config
params = DeepSenseParams(config)
self.env = env
self.history = History(logger, config)
self.replay_memory = ReplayMemory(logger, config)
with tf.variable_scope(STEPS):
self.step_op = tf.Variable(0, trainable=False, name=STEP)
self.step_input = tf.placeholder('int32', None, name=STEP_INPUT)
self.step_assign_op = self.step_op.assign(self.step_input)
self.build_dqn(params)
@property
def summary_writer(self):
return self._summary_writer
def train(self):
start_step = self.sess.run(self.step_op)
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
max_avg_ep_reward = 0
ep_rewards, actions = [], []
trade_rem = self.env.new_random_episode(self.history, self.replay_memory)
for self.step in tqdm(range(start_step, self.max_step), ncols=70, initial=start_step):
if self.step == self.learn_start:
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
ep_rewards, actions = [], []
# 1. predict
action = self.predict((self.history.history, trade_rem))
# 2. act
screen, reward, terminal, trade_rem = self.env.act(action)
# 3. observe
self.observe(screen, reward, action, terminal, trade_rem)
if terminal:
self.env.new_random_episode(self.history, self.replay_memory)
num_episodes += 1
ep_rewards.append(ep_reward)
ep_reward = 0.
else:
ep_reward += reward
actions.append(action)
total_reward += reward
if self.step >= self.learn_start:
if self.step % self.test_step == self.test_step - 1:
avg_reward = total_reward / self.test_step
avg_loss = self.total_loss / self.update_count
avg_q = self.total_q / self.update_count
try:
max_ep_reward = np.max(ep_rewards)
min_ep_reward = np.min(ep_rewards)
avg_ep_reward = np.mean(ep_rewards)
except:
max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0
message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \
% (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_episodes)
self.logger.info(message)
if max_avg_ep_reward * 0.9 <= avg_ep_reward:
self.sess.run(
fetches=self.step_assign_op,
feed_dict={self.step_input: self.step + 1}
)
self.save_model(self.step + 1)
max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward)
if self.step > 180:
self.inject_summary({
'average.reward': avg_reward,
'average.loss': avg_loss,
'average.q': avg_q,
'episode.max reward': max_ep_reward,
'episode.min reward': min_ep_reward,
'episode.avg reward': avg_ep_reward,
'episode.num of episodes': num_episodes,
'episode.rewards': ep_rewards,
'episode.actions': actions,
'training.learning_rate': self.sess.run(
fetches=self.learning_rate_op,
feed_dict={self.learning_rate_step: self.step}
)
}, self.step)
num_episodes = 0
total_reward = 0.
self.total_loss = 0.
self.total_q = 0.
self.update_count = 0
ep_reward = 0.
ep_rewards = []
actions = []
def predict(self, state, test_ep=None):
s_t = state[0]
trade_rem_t = state[1]
ep = test_ep or (self.ep_end +
max(0., (self.ep_start - self.ep_end) \
* (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t))
if random.random() < ep:
action = random.randrange(self.config[NUM_ACTIONS])
else:
action = self.sess.run(
fetches=self.q.action,
feed_dict={
self.q.phase: 0,
self.s_t: [s_t],
self.trade_rem_t: [trade_rem_t],
self.q_conv_keep_prob: 1.0,
self.q_dense_keep_prob: 1.0,
self.q_gru_keep_prob: 1.0
}
)[0]
return action
def observe(self, screen, reward, action, terminal, trade_rem):
#clip reward in the range min to max
reward = max(self.min_reward, min(self.max_reward, reward))
self.history.add(screen)
self.replay_memory.add(screen, reward, action, terminal, trade_rem)
if self.step > self.learn_start:
if self.step % self.train_frequency == 0:
self.q_learning_mini_batch()
if self.step % self.target_q_update_step == self.target_q_update_step - 1:
self.update_target_network()
def q_learning_mini_batch(self):
if self.replay_memory.count >= self.replay_memory.history_length:
state_t, action, reward, state_t_plus_1, terminal = self.replay_memory.sample
s_t, trade_rem_t = state_t[0], state_t[1]
s_t_plus_1, trade_rem_t_plus_1 = state_t_plus_1[0], state_t_plus_1[1]
q_t_plus_1 = self.sess.run(
fetches=self.t_q.values,
feed_dict={
self.t_q.phase: 0,
self.t_s_t: s_t_plus_1,
self.t_trade_rem_t: trade_rem_t_plus_1
}
)
max_q_t_plus_1 = np.max(q_t_plus_1, axis=1)
terminal = np.array(terminal) + 0.
target_q = reward + (1 - terminal) * max_q_t_plus_1
_, q_t, loss, avg_q_summary = self.sess.run([self.optimizer, self.q.values, self.loss, self.q.avg_q_summary], {
self.q.phase: 1,
self.target_q: target_q,
self.action: action,
self.s_t: s_t,
self.trade_rem_t: trade_rem_t,
self.q_conv_keep_prob: self.config[CONV_KEEP_PROB],
self.q_dense_keep_prob: self.config[DENSE_KEEP_PROB],
self.q_gru_keep_prob: self.config[GRU_KEEP_PROB],
self.learning_rate_step: self.step
})
self.summary_writer.add_summary(avg_q_summary, self.step)
self.total_loss += loss
self.total_q += q_t.mean()
self.update_count += 1
def build_dqn(self, params):
with tf.variable_scope(PREDICTION):
self.s_t = tf.placeholder(
dtype=tf.float32,
shape=[None, self.replay_memory.history_length,
self.replay_memory.num_channels],
name=HISTORICAL_PRICES
)
self.trade_rem_t = tf.placeholder(
dtype=tf.float32,
shape=[None,],
name=TRADE_REM
)
with tf.variable_scope(DROPOUT_KEEP_PROBS):
self.q_conv_keep_prob = tf.placeholder(tf.float32)
self.q_dense_keep_prob = tf.placeholder(tf.float32)
self.q_gru_keep_prob = tf.placeholder(tf.float32)
params.dropoutkeepprobs = DropoutKeepProbs(
self.q_conv_keep_prob,
self.q_dense_keep_prob,
self.q_gru_keep_prob
)
self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK)
self.q.build_model((self.s_t, self.trade_rem_t))
with tf.variable_scope(TARGET):
self.t_s_t = tf.placeholder(
dtype=tf.float32,
shape=[None, self.replay_memory.history_length,
self.replay_memory.num_channels],
name=HISTORICAL_PRICES
)
self.t_trade_rem_t = tf.placeholder(
dtype=tf.float32,
shape=[None,],
name=TRADE_REM
)
params.dropoutkeepprobs = DropoutKeepProbs()
self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK)
self.t_q.build_model((self.t_s_t, self.t_trade_rem_t))
with tf.variable_scope(UPDATE_TARGET_NETWORK):
self.q_weights_placeholders = {}
self.t_weights_assign_ops = {}
for name in self.q.weights.keys():
self.q_weights_placeholders[name] = tf.placeholder(
tf.float32,
self.q.weights[name].get_shape().as_list()
)
for name in self.q.weights.keys():
self.t_weights_assign_ops[name] = self.t_q.weights[name].assign(
self.q_weights_placeholders[name]
)
with tf.variable_scope(TRAINING):
self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q)
self.action = tf.placeholder(tf.int64, [None], name=ACTION)
action_one_hot = tf.one_hot(self.action, self.config[NUM_ACTIONS],
1.0, 0.0, name=ACTION_ONE_HOT)
q_acted = tf.reduce_sum(self.q.values * action_one_hot,
reduction_indices=1, name=Q_ACTED)
with tf.variable_scope(LOSS):
self.delta = self.target_q - q_acted
self.global_step = tf.Variable(0, trainable=False)
self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS)
with tf.variable_scope(OPTIMIZER):
self.learning_rate_step = tf.placeholder(tf.int64, None, name=LEARNING_RATE_STEP)
self.learning_rate_op = tf.maximum(self.learning_rate_minimum,
tf.train.exponential_decay(
self.learning_rate,
self.learning_rate_step,
self.learning_rate_decay_step,
self.learning_rate_decay,
staircase=True))
self.optimizer = tf.train.RMSPropOptimizer(
self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss)
with tf.variable_scope(SUMMARY):
scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \
'episode.max reward', 'episode.min reward', 'episode.avg reward', \
'episode.num of episodes', 'training.learning_rate']
self.summary_placeholders = {}
self.summary_ops = {}
for tag in scalar_summary_tags:
self.summary_placeholders[tag] = \
tf.placeholder('float32', None, name=tag.replace(' ', '_'))
self.summary_ops[tag] = \
tf.summary.scalar(
name="{}-{}".format(self.env_name, tag.replace(' ', '_')),
tensor=self.summary_placeholders[tag]
)
histogram_summary_tags = ['episode.rewards', 'episode.actions']
for tag in histogram_summary_tags:
self.summary_placeholders[tag] = \
tf.placeholder('float32', None, name=tag)
self.summary_ops[tag] = \
tf.summary.histogram(
tag,
self.summary_placeholders[tag]
)
self.sess.run(tf.local_variables_initializer())
self.sess.run(tf.global_variables_initializer())
self._saver = tf.train.Saver(self.q.weights.values() + [self.step_op], max_to_keep=30)
self.load_model()
self.update_target_network()
self._summary_writer = tf.summary.FileWriter(self.config[TENSORBOARD_LOG_DIR])
self._summary_writer.add_graph(self.sess.graph)
def update_target_network(self):
for name in self.q.weights.keys():
self.sess.run(
fetches=self.t_weights_assign_ops[name],
feed_dict=
{self.q_weights_placeholders[name]: self.sess.run(
fetches=self.q.weights[name]
)}
)
def inject_summary(self, tag_dict, step):
summary_str_lists = self.sess.run([self.summary_ops[tag] for tag in tag_dict.keys()], {
self.summary_placeholders[tag]: value for tag, value in tag_dict.items()
})
for summary_str in summary_str_lists:
self.summary_writer.add_summary(summary_str, self.step)
def history(cls):
return History.select().order_by(History.date.asc()).all()
class AcbEnterprise:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.username = payment.get_username()
self.password = payment.get_password()
acb = self.get_config()
self.email_transport = EmailTransport()
self.corp_url = acb['corp_url']
self.captcha = CaptchaResolver()
self.debug_mode = acb['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 20
self.login_failed = 0
self.session = session
self.proxy = proxy
self.payment = payment
def get_config(self):
config = self.config.get_section_config('AcbEnterprise')
return config
def perform_login(self):
corp_url = self.corp_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver(
) is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
driver.get(corp_url)
online_banking = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//a[contains(@href,'obk/login.jsp')]")))
online_banking.click()
time.sleep(5)
captcha = self.get_captcha(driver)
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="UserName"]')))
element.send_keys(username, Keys.ARROW_DOWN)
# Get password input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="PassWord"]')))
driver.execute_script(
'arguments[0].style = ""; arguments[0].style.display = "block"; arguments[0].style.visibility = "visible";',
element)
element.send_keys(password)
# Get captcha input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="SecurityCode"]')))
driver.execute_script(
'arguments[0].style = ""; arguments[0].style.display = "block"; arguments[0].style.visibility = "visible";',
element)
element.send_keys(captcha)
time.sleep(2)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//a[@id="button-dangnhap"]')))
element.click()
time.sleep(3)
try:
# Update Account Information
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//table[@id='table']//tr/td")))
account_info = driver.find_elements_by_xpath(
"//table[@id='table']//tr/td")
account_number = account_info[0].text.strip()
account_name = self.payment.get_username()
account_balance = float(
account_info[2].text.strip().replace(' ', '').replace(
'(VND)', '').replace('.', ''))
account = self.update_account(account_name, account_number,
account_balance,
self.payment.get_id())
except:
driver.close()
return self.perform_login()
# click to transaction menu
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//a[contains(@class,'acc_bold')]")))
element.click()
time.sleep(3)
view_link = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH,
"//input[contains(@onclick,'OutPrintByDate')]")))
view_link.click()
try:
window_after = driver.window_handles[1]
driver.switch_to_window(window_after)
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//table[@id='table1']//tbody")))
transactions = driver.find_elements_by_xpath(
"//table[@id='table1']//tbody/tr[position() >= 0 and position() <= 10]"
)
for row in transactions:
columns = row.find_elements_by_xpath("td")
self.save_transaction(account, columns)
self.log.update_log('Acb', self.username)
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
str(self.total_transactions) +
' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
driver.close()
except:
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
"Cannot load transactions", 'error')
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ': ' +
str(sys.exc_info()), 'debug')
self.log.log(
str(exc_type) + '-' + fname + '-' +
str(exc_tb.tb_lineno), 'error')
self.session.set_changing_proxy(1)
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
self.log.log(
self.payment.get_name() + '-' + self.payment.get_type() +
'-' + self.payment.get_username() + ': ' +
str(sys.exc_info()), 'debug')
self.log.log(
str(exc_type) + '-' + fname + '-' + str(exc_tb.tb_lineno),
'error')
self.session.set_changing_proxy(1)
driver.quit()
self.history.set_current_update('acb_enterprise', "%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
if len(detail) < 4:
return False
trading_date = self.convert_trading_date(detail[0].text)
reference_number = detail[1].text
account_id = account.get_account_id()
if detail[3].text.strip() is not '':
balance = -float(detail[3].text.strip().replace('.', ''))
else:
balance = float(detail[4].text.strip().replace('.', ''))
description = detail[2].text
transaction = Transaction(account_id, reference_number, trading_date,
balance, description)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def get_captcha(self, driver):
img_data = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//img[@src='Captcha.jpg']")))
img_file = img_data.screenshot_as_png
self.captcha.save_from_source(img_file, 'png')
captcha_text = self.captcha.resolve(True)
if re.match("^[a-zA-Z0-9]{5}$", captcha_text):
return captcha_text
driver.refresh()
return self.get_captcha(driver)
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%d-%m-%y')
return date.strftime('%Y-%m-%d')
def update_account(self, name, number, balance, payment_id):
account = AcbAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def is_debug(self):
return self.debug_mode
class Agent(BaseAgent):
'''Deep Trading Agent based on Deep Q Learning'''
'''TODO:
1. add summary ops
2. timing and logging
3. model saving
4. increment self.step
'''
def __init__(self, sess, logger, config, env):
super(Agent, self).__init__(config)
self.sess = sess
self.logger = logger
self.config = config
params = DeepSenseParams(config)
self.env = env
self.history = History(logger, config)
self.replay_memory = ReplayMemory(logger, config)
with tf.variable_scope(STEPS):
self.step_op = tf.Variable(0, trainable=False, name=STEP)
self.step_input = tf.placeholder('int32', None, name=STEP_INPUT)
self.step_assign_op = self.step_op.assign(self.step_input)
self.build_dqn(params)
def train(self):
start_step = self.step_op.eval()
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
max_avg_ep_reward = 0
ep_rewards, actions = [], []
self.env.new_random_episode(self.history)
for self.step in range(start_step, self.max_step):
if self.step == self.learn_start:
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
ep_rewards, actions = [], []
# 1. predict
action = self.predict(self.history.get())
# 2. act
screen, reward, terminal = self.env.act(action)
# 3. observe
self.observe(screen, reward, action, terminal)
if terminal:
self.env.new_random_episode(self.history)
num_episodes += 1
ep_rewards.append(ep_reward)
ep_reward = 0.
else:
ep_reward += reward
actions.append(action)
total_reward += reward
def predict(self, s_t, test_ep=None):
ep = test_ep or (self.ep_end +
max(0., (self.ep_start - self.ep_end) \
* (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t))
if random.random() < ep:
action = random.randrange(self.env.action_size)
else:
action = self.q.action.eval({self.s_t: [s_t]})[0]
return action
def observe(self, screen, reward, action, terminal):
#clip reward in the range min to max
reward = max(self.min_reward, min(self.max_reward, reward))
self.history.add(screen)
self.replay_memory.add(screen, reward, action, terminal)
if self.step > self.learn_start:
if self.step % self.train_frequency == 0:
self.q_learning_mini_batch()
if self.step % self.target_q_update_step == self.target_q_update_step - 1:
self.update_target_network()
def q_learning_mini_batch(self):
if self.replay_memory.count >= self.replay_memory.history_length:
s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample(
)
max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1})
terminal = np.array(terminal) + 0.
target_q = reward + (1 - terminal) * max_q_t_plus_1
_, q_t, loss = self.sess.run(
[self.optimizer, self.q.values, self.loss], {
self.target_q: target_q,
self.action: action,
self.s_t: s_t,
self.learning_rate_step: self.step,
})
self.total_loss += loss
self.total_q += q_t.mean()
self.update_count += 1
def build_dqn(self, params):
with tf.variable_scope(PREDICTION):
self.s_t = tf.placeholder(dtype=tf.float32,
shape=[
None,
self.replay_memory.history_length,
self.replay_memory.num_channels
])
self.q = DeepSense(params,
self.logger,
self.sess,
self.config,
name=Q_NETWORK)
self.q.build_model(self.s_t)
with tf.variable_scope(TARGET):
self.t_s_t = tf.placeholder(dtype=tf.float32,
shape=[
None,
self.replay_memory.history_length,
self.replay_memory.num_channels
])
self.t_q = DeepSense(params,
self.logger,
self.sess,
self.config,
name=T_Q_NETWORK)
self.t_q.build_model(self.t_s_t, train=False)
with tf.variable_scope(UPDATE_TARGET_NETWORK):
self.q_weights_placeholders = {}
self.t_weights_assign_ops = {}
for name in self.q.weights.keys():
self.q_weights_placeholders[name] = tf.placeholder(
tf.float32, self.q.weights[name].get_shape().as_list())
for name in self.q.weights.keys():
self.t_weights_assign_ops[name] = self.t_q.weights[
name].assign(self.q_weights_placeholders[name])
with tf.variable_scope(TRAINING):
self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q)
self.action = tf.placeholder(tf.int64, [None], name=ACTION)
action_one_hot = tf.one_hot(self.action,
self.env.action_size,
1.0,
0.0,
name=ACTION_ONE_HOT)
q_acted = tf.reduce_sum(self.q.values * action_one_hot,
reduction_indices=1,
name=Q_ACTED)
with tf.variable_scope(LOSS):
self.delta = self.target_q - q_acted
self.global_step = tf.Variable(0, trainable=False)
self.loss = tf.reduce_mean(clipped_error(self.delta),
name=LOSS)
with tf.variable_scope(OPTIMIZER):
self.learning_rate_step = tf.placeholder(
tf.int64, None, name=LEARNING_RATE_STEP)
self.learning_rate_op = tf.maximum(
self.learning_rate_minimum,
tf.train.exponential_decay(self.learning_rate,
self.learning_rate_step,
self.learning_rate_decay_step,
self.learning_rate_decay,
staircase=True))
self.optimizer = tf.train.RMSPropOptimizer(
self.learning_rate_op, momentum=0.95,
epsilon=0.01).minimize(self.loss)
# tf.initialize_all_variables().run()
#initialize the q network and the target network with the same weights
# self.update_target_network()
def update_target_network(self):
for name in self.q.weights.keys():
self.t_weights_assign_ops[name].eval({
self.q_weights_placeholders[name]:
self.q.weights[name].eval()
})
class Vietcombank:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.username = payment.get_username()
self.password = payment.get_password()
vietcombank = self.get_vietcombank_config()
self.email_transport = EmailTransport()
self.login_url = vietcombank['login_url']
self.captcha = CaptchaResolver()
self.debug_mode = vietcombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 20
self.login_failed = 0
self.session = session
self.proxy = proxy
self.payment = payment
def get_vietcombank_config(self):
vietcombank = self.config.get_section_config('Vietcombank')
return vietcombank
def perform_login(self):
login_url = self.login_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver() is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
driver.get(login_url)
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="username"]'))
)
element.send_keys(username)
# Get password input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="pass"]'))
)
element.send_keys(password)
while True:
captcha = self.get_captcha(driver)
# Get captcha input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="captcha"]'))
)
element.send_keys(captcha)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//button[@id="btnLogin"]'))
)
element.click()
time.sleep(5)
# click to account menu
try:
account_link = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH,
"//a[contains(@class,'has-link-arrow tk')]//div[contains(@class,'item-link-arrow')]"))
)
account_link.click()
time.sleep(5)
self.login_failed = 0
break
except:
self.login_failed += 1
if self.login_failed > self.max_attempt_login:
driver.close()
self.history.set_current_update('vietcombank', "%d/%m/%Y")
self.session.set_driver(None)
# Update account information
try:
driver.execute_script("window.scrollTo(0, 800);")
time.sleep(5)
search_button = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH,
"//a[contains(@class,'ubtn ubg-primary ubtn-md ripple')]/span[contains(text(),'Tìm kiếm')]"))
)
search_button.click()
WebDriverWait(driver, 15).until(
EC.visibility_of_element_located(
(By.XPATH, "//form[@id='ChiTietTaiKhoan']"))
)
info = driver.find_element_by_xpath(
"//div[@class='list-info']")
number = info.find_element_by_xpath("//select/option").get_attribute('value')
name = info.find_elements_by_xpath(
"//div[contains(@class,'list-info-txt-main')]")[1].text
balance = float(info.find_elements_by_xpath(
"//div[contains(@class,'list-info-txt-main')]")[2].text.replace(
',', '').replace(' VND', ''))
account = self.update_account(name, number, balance, self.payment.get_id())
transactions = driver.find_elements_by_xpath(
"//div[@id='toanbo']//div[contains(@class,'list-info-item')]")
for row in transactions:
columns = row.find_elements_by_xpath(".//div[contains(@class,'td-xs')]")
self.save_transaction(account, columns)
self.log.update_log('Vietcombank', self.username)
self.log.log("Vcb " + self.payment.get_type() + self.payment.get_username() + ": " + str(
self.total_transactions) + ' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log(
"Vcb " + self.payment.get_type() + self.payment.get_username() + ": " + "Cannot load transactions",
'error')
self.log.log(str(sys.exc_info()), 'error')
self.session.set_changing_proxy(1)
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(str(sys.exc_info()), 'debug')
driver.close()
self.history.set_current_update('vietcombank', "%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
trading_date = self.convert_trading_date(
detail[0].find_element_by_xpath("div[contains(@class,'list-info-txt-sub')]").text)
description = detail[0].find_element_by_xpath("div[contains(@class,'list-info-txt-main')]").text
account_id = account.get_account_id()
reference_number = detail[1].find_element_by_xpath("div[contains(@class,'list-info-txt-sub')]").text.replace(
'Số tham chiếu: ', '')
balance = float(
detail[1].find_element_by_xpath("div[contains(@class,'list-info-txt-main')]").text.replace(',', '').replace(
' ', ''))
transaction = Transaction(account_id, reference_number, trading_date, balance, description)
print(reference_number)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def get_captcha(self, driver):
img_data = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//div[contains(@class,'captcha')]//img"))
)
img_file = img_data.screenshot_as_png
self.captcha.save_from_source(img_file, 'png')
captcha_text = self.captcha.resolve(True)
if re.match("^[a-zA-Z0-9]{5}$", captcha_text):
return captcha_text
ic_reload = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//img[contains(@src,'ic_reload')]"))
)
ic_reload.click()
return self.get_captcha(driver)
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%d/%m/%Y')
return date.strftime('%Y-%m-%d')
def update_account(self, name, number, balance, payment_id):
account = VietcombankAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def get_config(self, name=None):
if name is None:
name = 'Vietcombank'
vietcombank = self.config.get_section_config(name)
return vietcombank
def is_debug(self):
return self.debug_mode
class Agent(BaseAgent):
'''Deep Trading Agent based on Deep Q Learning'''
'''TODO:
1. play
'''
def __init__(self, sess, logger, config, env):
super(Agent, self).__init__(config, logger)
self.sess = sess
self.logger = logger
self.config = config
params = DeepSenseParams(config)
self.env = env
self.history = History(logger, config)
self.replay_memory = ReplayMemory(logger, config)
with tf.variable_scope(STEPS):
self.step_op = tf.Variable(0, trainable=False, name=STEP)
self.step_input = tf.placeholder('int32', None, name=STEP_INPUT)
self.step_assign_op = self.step_op.assign(self.step_input)
self.build_dqn(params)
@property
def summary_writer(self):
return self._summary_writer
def train(self):
start_step = self.step_op.eval()
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
max_avg_ep_reward = 0
ep_rewards, actions = [], []
self.env.new_random_episode(self.history)
for self.step in tqdm(range(start_step, self.max_step),
ncols=70,
initial=start_step):
if self.step == self.learn_start:
num_episodes, self.update_count, ep_reward = 0, 0, 0.
total_reward, self.total_loss, self.total_q = 0., 0., 0.
ep_rewards, actions = [], []
# 1. predict
action = self.predict(self.history.get())
# 2. act
screen, reward, terminal = self.env.act(action)
# 3. observe
self.observe(screen, reward, action, terminal)
if terminal:
self.env.new_random_episode(self.history)
num_episodes += 1
ep_rewards.append(ep_reward)
ep_reward = 0.
else:
ep_reward += reward
actions.append(action)
total_reward += reward
if self.step >= self.learn_start:
if self.step % self.test_step == self.test_step - 1:
avg_reward = total_reward / self.test_step
avg_loss = self.total_loss / self.update_count
avg_q = self.total_q / self.update_count
try:
max_ep_reward = np.max(ep_rewards)
min_ep_reward = np.min(ep_rewards)
avg_ep_reward = np.mean(ep_rewards)
except:
max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0
message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \
% (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_game)
print_and_log_message(message, self.logger)
if max_avg_ep_reward * 0.9 <= avg_ep_reward:
self.step_assign_op.eval(
{self.step_input: self.step + 1})
self.save_model(self.step + 1)
max_avg_ep_reward = max(max_avg_ep_reward,
avg_ep_reward)
if self.step > 180:
self.inject_summary(
{
'average.reward':
avg_reward,
'average.loss':
avg_loss,
'average.q':
avg_q,
'episode.max reward':
max_ep_reward,
'episode.min reward':
min_ep_reward,
'episode.avg reward':
avg_ep_reward,
'episode.num of game':
num_game,
'episode.rewards':
ep_rewards,
'episode.actions':
actions,
'training.learning_rate':
self.learning_rate_op.eval(
{self.learning_rate_step: self.step}),
}, self.step)
num_game = 0
total_reward = 0.
self.total_loss = 0.
self.total_q = 0.
self.update_count = 0
ep_reward = 0.
ep_rewards = []
actions = []
def predict(self, s_t, test_ep=None):
ep = test_ep or (self.ep_end +
max(0., (self.ep_start - self.ep_end) \
* (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t))
if random.random() < ep:
action = random.randrange(self.env.action_size)
else:
action = self.q.action.eval({self.s_t: [s_t]})[0]
return action
def observe(self, screen, reward, action, terminal):
#clip reward in the range min to max
reward = max(self.min_reward, min(self.max_reward, reward))
self.history.add(screen)
self.replay_memory.add(screen, reward, action, terminal)
if self.step > self.learn_start:
if self.step % self.train_frequency == 0:
self.q_learning_mini_batch()
if self.step % self.target_q_update_step == self.target_q_update_step - 1:
self.update_target_network()
def q_learning_mini_batch(self):
if self.replay_memory.count >= self.replay_memory.history_length:
s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample(
)
max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1})
terminal = np.array(terminal) + 0.
target_q = reward + (1 - terminal) * max_q_t_plus_1
_, q_t, loss, avg_q_summary = self.sess.run(
[
self.optimizer, self.q.values, self.loss,
self.q.avg_q_summary
], {
self.target_q: target_q,
self.action: action,
self.s_t: s_t,
self.learning_rate_step: self.step,
})
self.summary_writer.add_summary(avg_q_summary, self.step)
self.total_loss += loss
self.total_q += q_t.mean()
self.update_count += 1
def build_dqn(self, params):
with tf.variable_scope(PREDICTION):
self.s_t = tf.placeholder(dtype=tf.float32,
shape=[
None,
self.replay_memory.history_length,
self.replay_memory.num_channels
])
self.q = DeepSense(params,
self.logger,
self.sess,
self.config,
name=Q_NETWORK)
self.q.build_model(self.s_t)
with tf.variable_scope(TARGET):
self.t_s_t = tf.placeholder(dtype=tf.float32,
shape=[
None,
self.replay_memory.history_length,
self.replay_memory.num_channels
])
self.t_q = DeepSense(params,
self.logger,
self.sess,
self.config,
name=T_Q_NETWORK)
self.t_q.build_model(self.t_s_t, train=False)
with tf.variable_scope(UPDATE_TARGET_NETWORK):
self.q_weights_placeholders = {}
self.t_weights_assign_ops = {}
for name in self.q.weights.keys():
self.q_weights_placeholders[name] = tf.placeholder(
tf.float32, self.q.weights[name].get_shape().as_list())
for name in self.q.weights.keys():
self.t_weights_assign_ops[name] = self.t_q.weights[
name].assign(self.q_weights_placeholders[name])
with tf.variable_scope(TRAINING):
self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q)
self.action = tf.placeholder(tf.int64, [None], name=ACTION)
action_one_hot = tf.one_hot(self.action,
self.env.action_size,
1.0,
0.0,
name=ACTION_ONE_HOT)
q_acted = tf.reduce_sum(self.q.values * action_one_hot,
reduction_indices=1,
name=Q_ACTED)
with tf.variable_scope(LOSS):
self.delta = self.target_q - q_acted
self.global_step = tf.Variable(0, trainable=False)
self.loss = tf.reduce_mean(clipped_error(self.delta),
name=LOSS)
with tf.variable_scope(OPTIMIZER):
self.learning_rate_step = tf.placeholder(
tf.int64, None, name=LEARNING_RATE_STEP)
self.learning_rate_op = tf.maximum(
self.learning_rate_minimum,
tf.train.exponential_decay(self.learning_rate,
self.learning_rate_step,
self.learning_rate_decay_step,
self.learning_rate_decay,
staircase=True))
self.optimizer = tf.train.RMSPropOptimizer(
self.learning_rate_op, momentum=0.95,
epsilon=0.01).minimize(self.loss)
with tf.variable_scope(SUMMARY):
scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \
'episode.max reward', 'episode.min reward', 'episode.avg reward', \
'episode.num of game', 'training.learning_rate']
self.summary_placeholders = {}
self.summary_ops = {}
for tag in scalar_summary_tags:
self.summary_placeholders[tag] = \
tf.placeholder('float32', None, name=tag.replace(' ', '_'))
self.summary_ops[tag] = \
tf.summary.scalar(
name="{}-{}".format(self.env_name, tag),
tensor=self.summary_placeholders[tag]
)
histogram_summary_tags = ['episode.rewards', 'episode.actions']
for tag in histogram_summary_tags:
self.summary_placeholders[tag] = \
tf.placeholder('float32', None, name=tag.replace(' ', '_'))
self.summary_ops[tag] = \
tf.summary.histogram(
name=tag,
self.summary_placeholders[tag]
)
self._summary_writer = tf.summary.FileWriter(
config[TENSORBOARD_LOG_DIR])
self._summary_writer.add_graph(sess.graph)
tf.initialize_all_variables().run()
self._saver = tf.train.Saver(self.q.weights.values + [self.step_op],
max_to_keep=30)
self.load_model()
self.update_target_network()
def update_target_network(self):
for name in self.q.weights.keys():
self.t_weights_assign_ops[name].eval({
self.q_weights_placeholders[name]:
self.q.weights[name].eval()
})
def inject_summary(self, tag_dict, step):
summary_str_lists = self.sess.run(
[self.summary_ops[tag] for tag in tag_dict.keys()], {
self.summary_placeholders[tag]: value
for tag, value in tag_dict.items()
})
for summary_str in summary_str_lists:
self.writer.add_summary(summary_str, self.step)
class Techcombank:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.payment = payment
self.config = Config()
self.log = Log()
techcombank = self.get_techcombank_config()
self.email_transport = EmailTransport()
self.login_url = techcombank['login_url']
self.username = payment.get_username()
self.password = payment.get_password()
self.debug_mode = techcombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
def perform_login(self):
login_url = self.login_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver(
) is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
driver.get(login_url)
# Get username input element
element = WebDriverWait(driver, 60).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="signOnName"]')))
element.send_keys(username)
# element.send_keys(Keys.RETURN)
element = WebDriverWait(driver, 60).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="password"]')))
# Get password input element
element.send_keys(password)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 60).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="btn_login"]')))
element.click()
time.sleep(5)
WebDriverWait(driver, 60).until(
EC.visibility_of_element_located(
(By.XPATH,
"//table[contains(@id,'datadisplay_AcctBalance')]")))
sub_accounts = driver.find_elements_by_xpath(
"//table[contains(@id,'datadisplay_AcctBalance')]//tr")
for i in range(0, len(sub_accounts)):
WebDriverWait(driver, 60).until(
EC.visibility_of_element_located(
(By.XPATH,
"//table[contains(@id,'datadisplay_AcctBalance')]"
)))
WebDriverWait(driver, 60).until(
EC.visibility_of_element_located((
By.XPATH,
"//table[contains(@id,'datadisplay_AcctBalance')]//a/img[contains(@src,'View.png')]"
)))
sub_account = driver.find_elements_by_xpath(
"//table[contains(@id,'datadisplay_AcctBalance')]//tr"
)[i]
search_btn = sub_account.find_element_by_xpath(
".//a/img[contains(@src,'View.png')]")
search_btn.click()
# Update account information
try:
WebDriverWait(driver, 120).until(
EC.visibility_of_element_located(
(By.XPATH, "//table[@id='enqheader']//tr")))
info = driver.find_elements_by_xpath(
"//table[@id='enqheader']//tr")
number = info[2].find_elements_by_xpath("td")[1].text
try:
balance = float(info[2].find_elements_by_xpath(
"td")[3].text.replace(',', ''))
except:
balance = float(0)
name = info[3].find_elements_by_xpath("td")[1].text
account = self.update_account(name, number, balance,
self.payment.get_id())
transactions = driver.find_elements_by_xpath(
"//table[starts-with(@id,'datadisplay_MainBody')]//tr"
)
count = 0
for row in transactions:
count += 1
# Get first 10 records
if count > 10:
break
columns = row.find_elements_by_xpath("td")
self.save_transaction(account, columns)
self.log.update_log('Techcombank', self.username)
self.log.log(
"Tcb " + self.payment.get_type() +
self.payment.get_username() + ": " +
str(self.total_transactions) +
' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log(
"Tcb " + self.payment.get_type() +
self.payment.get_username() + ": " +
"Cannot load transactions", 'error')
self.log.log(str(sys.exc_info()), 'error')
self.session.set_changing_proxy(1)
home_btn = driver.find_element_by_xpath(
'//a[@name="AI.QCK.HOME"]')
home_btn.click()
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(str(sys.exc_info()), 'debug')
driver.close()
self.history.set_current_update('techcombank', "%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
trading_date = self.convert_trading_date(detail[0].text)
if trading_date is None:
return 0
description = detail[1].text
account_id = account.get_account_id()
balance = float(detail[2].text.replace(',', ''))
reference_number = self.convert_reference_number(description)
transaction = Transaction(account_id, reference_number, trading_date,
balance, description)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def convert_reference_number(self, description):
description_array = description.split('/')
reference_number = description_array[len(description_array) -
1].strip()
return reference_number
def convert_trading_date(self, trading_date):
try:
date = datetime.strptime(trading_date, '%d/%m/%Y')
return date.strftime('%Y-%m-%d')
except:
return None
def update_account(self, name, number, balance, payment_id):
account = TechcombankAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def get_techcombank_config(self):
techcombank = self.config.get_section_config('Techcombank')
return techcombank
def is_debug(self):
return self.debug_mode
class VietcombankEnterprise:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.username = payment.get_username()
self.password = payment.get_password()
vietcombank = self.get_vietcombank_config()
self.email_transport = EmailTransport()
self.login_url = vietcombank['login_url']
self.captcha = CaptchaResolver()
self.debug_mode = vietcombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 5
self.login_failed = 0
self.session = session
self.proxy = proxy
self.payment = payment
def get_vietcombank_config(self):
vietcombank = self.config.get_section_config('VietcombankEnterprise')
return vietcombank
def perform_login(self):
login_url = self.login_url
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver(
) is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
driver.get(login_url)
if self.solve_captcha(driver) is False:
driver.close()
return False
time.sleep(5)
# Update account information
try:
# driver.execute_script("window.scrollTo(0, 800);")
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//tbody[@id='dstkdd-tbody']")))
account_info = driver.find_elements_by_xpath(
"//tbody[@id='dstkdd-tbody']/tr/td")
account_name = self.payment.get_username()
account_number = account_info[0].text.strip()
account_balance = float(
account_info[3].text.strip().replace(' VND',
'').replace(
',', ''))
account = self.update_account(account_name, account_number,
account_balance,
self.payment.get_id())
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((
By.XPATH,
"//a[contains(@href,'/IbankingCorp/Accounts/TransactionDetail.aspx')]"
)))
reference_links = driver.find_elements_by_xpath(
"//a[contains(@href,'/IbankingCorp/Accounts/TransactionDetail.aspx')]"
)
reference_links[2].click()
#view_link = WebDriverWait(driver, 10).until(
# EC.presence_of_element_located(
# (By.XPATH, "//a[contains(@id,'bdsdList-tab')]"))
#)
#driver.execute_script("arguments[0].scrollIntoView();", view_link)
#time.sleep(5)
#view_link.click()
tran_by_date = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH, "//a[contains(@id,'TransByDate')]")))
#tran_by_date.click()
driver.execute_script("arguments[0].click();",
tran_by_date)
time.sleep(5)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//table[@id='tbTranHis']")))
trans_foot = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//div[@id='tbTranHis-footer']")))
driver.execute_script("arguments[0].scrollIntoView();",
trans_foot)
transactions = driver.find_elements_by_xpath(
"//table[@id='tbTranHis']//tbody/tr[position() >= 0 and position() <= 25]"
)
for row in transactions:
columns = row.find_elements_by_xpath("td")
self.save_transaction(account, columns)
self.log.update_log('Vietcombank', self.username)
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
str(self.total_transactions) +
' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
"Cannot load transactions", 'error')
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
self.log.log(
str(exc_type) + '-' + fname + '-' +
str(exc_tb.tb_lineno), 'error')
self.session.set_changing_proxy(1)
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(
self.payment.get_name() + '-' + self.payment.get_type() +
'-' + self.payment.get_username() + ': ' +
str(sys.exc_info()), 'debug')
driver.close()
self.history.set_current_update('vietcombank_enterprise',
"%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
trading_date = self.convert_trading_date(detail[0].text)
reference_number = detail[1].text.replace('Số tham chiếu: ', '')
account_id = account.get_account_id()
if detail[2].text is not '':
balance = -float(detail[2].text.replace(',', '').replace(
' ', '').replace('VND', ''))
else:
balance = float(detail[3].text.replace(',', '').replace(
' ', '').replace('VND', ''))
description = detail[4].text
transaction = Transaction(account_id, reference_number, trading_date,
balance, description)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def solve_captcha(self, driver):
img_data = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH,
"//img[@id='ctl00_Content_Login_Captcha_Captcha']")))
img_file = img_data.screenshot_as_png
self.captcha.save_from_source(img_file, 'png')
captcha_text = self.captcha.resolve(True)
if re.match("^[a-zA-Z0-9]{6}$", captcha_text):
captcha = captcha_text
username = self.username
password = self.password
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="ctl00$Content$Login$TenTC"]')))
element.send_keys(username)
# Get password input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="ctl00$Content$Login$MatKH"]')))
element.send_keys(password)
# Get captcha input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="ctl00$Content$Login$ImgStr"]')))
element.send_keys(captcha)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@id="ctl00_Content_Login_LoginBtn"]')))
element.click()
# click to account menu
try:
list_account = WebDriverWait(driver, 5).until(
EC.presence_of_element_located(
(By.XPATH,
"//a[contains(@href,'Accounts/ListAccounts.aspx')]")))
time.sleep(5)
list_account.click()
self.login_failed = 0
return True
except:
self.login_failed += 1
if self.login_failed > self.max_attempt_login:
self.history.set_current_update('vietcombank_enterprise',
"%d/%m/%Y")
self.session.set_driver(None)
return False
driver.back()
return self.solve_captcha(driver)
driver.refresh()
return self.solve_captcha(driver)
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%Y-%m-%d')
return date.strftime('%Y-%m-%d')
def update_account(self, name, number, balance, payment_id):
account = VietcombankAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def get_config(self, name=None):
if name is None:
name = 'Vietcombank'
techcombank = self.config.get_section_config(name)
return techcombank
def is_debug(self):
return self.debug_mode
def train(self, episodes, time_steps, update_steptime, env, path):
path_save = './results/saved_models/' + path
stats = EpisodeStats(episode_lengths=np.zeros(episodes),
episode_rewards=np.zeros(episodes))
print("START!")
collector = History()
avg_length = 0
accum_reward = 0
state = env.reset()
max_length_episode = time_steps
episode_reward = 0
length_episode = 0
i_episode = 1
while i_episode < episodes:
for _ in range(update_steptime):
action, log_prob, state = self.get_action(state)
next_state, reward, done, _ = env.step(
action.cpu().data.numpy().flatten())
#Store the enviroment's reaction to action taken by Policy_old
collector.rewards.append(reward)
collector.dones.append(done)
collector.states.append(state)
collector.actions.append(action)
collector.logprob_actions.append(log_prob)
state = next_state
episode_reward += reward
length_episode += 1
if done or length_episode == max_length_episode:
state = env.reset()
done = True
avg_length += length_episode
accum_reward += episode_reward
stats.episode_lengths[i_episode - 1] = length_episode
stats.episode_rewards[i_episode - 1] = episode_reward
length_episode = 0
episode_reward = 0
i_episode += 1
if i_episode % self._print_timesteps == 0:
print(
'Episode {}/{} \tAvg length: {} \tAvg reward: {}'.
format(i_episode + self.episode, episodes,
int(avg_length / 50),
int(accum_reward / 50)))
avg_length = 0.
accum_reward = 0.
if i_episode % self._save_timestep == 0:
torch.save(
{
'episodes': i_episode + self.episode,
'model_state_dict': self._ac.state_dict(),
'optimizer_state_dict': self._opt.state_dict(),
'loss': self.loss_CLIP_VF_S
},
path_save + str(i_episode + self.episode) + ".pkl")
print("created and saved " + path +
str(i_episode + self.episode) + " models")
if i_episode % self._save_timestep == 0:
np.save(
'./results/stats/' + path +
str(i_episode + self.episode) + '_episodes',
stats.episode_lengths)
np.save(
'./results/stats/' + path +
str(i_episode + self.episode) + '_reward',
stats.episode_rewards)
self.update_actorcritic(collector)
collector.reset()
class Msb:
def __init__(self, payment, session=None, proxy={}):
self.payment = payment
self.session = session
self.proxy = {}
self.captcha = CaptchaResolver()
self.config = Config()
self.log = Log()
self.history = History()
self.code = GenerateCode()
self.email_transport = EmailTransport()
section_config = self.get_msb_config()
self.username = payment.get_username()
self.password = payment.get_password()
self.home_url = section_config['home_url']
self.login_url = section_config['login_url']
self.account_list_url = section_config['account_list_url']
self.account_info_url = section_config['account_info_url']
self.debug_mode = section_config['debug_mode']
self.total_transactions = 0
self.max_attempt_login = 10
self.max_validate_captcha = 10
self.login_failed = 0
self.captcha_failed = 0
def get_msb_config(self):
msb = self.config.get_section_config('Msb')
return msb
def is_logged_in(self):
session_requests = self.session
token = session_requests.get_token()
if token is None:
return 0
try:
account_list_url = self.account_list_url
payload = {
'acctType': "'CA','SA','LN'",
'status': "'ACTV','DORM','MATU'",
'tokenNo': token,
'lang': "vi_VN",
}
list_post = session_requests.post(account_list_url, payload)
response = list_post.json()
if response['status'] == '200':
return 1
return 0
except Exception as e:
if self.debug_mode == 'true':
self.log.log(str(e), 'debug')
return 0
def is_debug(self):
return self.debug_mode
def perform_login(self):
home_url = self.home_url
login_url = self.login_url
username = self.username
password = self.password
session_requests = self.session
if self.is_logged_in() == 0:
session_requests.set_tree(None)
session_requests.init_session(self.proxy)
session_requests.set_changing_proxy(1)
result = session_requests.get(login_url)
tree = html.fromstring(result.text)
window_location = tree.xpath("//script/text()")[0]
window_location = self.replace_all(window_location, {
"window.location.href = '": "",
"';": ""
})
login_redirect_url = str(home_url + window_location.strip())
result = session_requests.get(login_redirect_url,
dict(referer=login_redirect_url))
tree = html.fromstring(result.text)
window_location = tree.xpath("//script/text()")[0]
window_location = self.replace_all(window_location, {
"window.location.href = '": "",
"';": ""
})
real_login_redirect_url = str(home_url + window_location.strip())
result = session_requests.get(real_login_redirect_url,
dict(referer=login_redirect_url))
tree = html.fromstring(result.content)
dse_session_id = tree.xpath(
"//input[@name='dse_sessionId']/@value")[0]
img_data = session_requests.get(
home_url + '/IBSRetail/servlet/ImageServlet').content
self.captcha.save_from_source(img_data)
captcha_text = self.captcha.resolve(True)
if self.validate_captcha(captcha_text) == 0:
if self.captcha_failed < self.max_validate_captcha:
self.captcha_failed = self.captcha_failed + 1
return self.perform_login()
else:
if self.debug_mode == 'true':
self.log.log(
'Can\'t validate captcha after ' +
str(self.captcha_failed) + ' times', 'debug')
self.captcha_failed = 0
return 0
payload = {
'dse_sessionId': dse_session_id,
'dse_applicationId': -1,
'dse_pageId': 2,
'dse_operationName': 'retailUserLoginProc',
'dse_errorPage': 'index.jsp',
'dse_processorState': 'initial',
'dse_nextEventName': 'start',
'orderId': '',
'_userNameEncode': username,
'_userName': username,
'_password': password,
'_verifyCode': captcha_text
}
# Perform login
login_post = session_requests.post(login_url + '/Request', payload)
tree = html.fromstring(login_post.text)
window_location = tree.xpath("//script/text()")[0]
window_location = self.replace_all(window_location, {
"window.location.href = '": "",
"';": ""
})
redirect_url = str(home_url + window_location.strip())
result = session_requests.get(redirect_url)
tree = html.fromstring(result.content)
username = tree.xpath("//p[@class='username']/text()")
if len(username) > 0:
token = self.get_token_from_string(str(result.content))
session_requests.set_token(token)
session_requests.set_changing_proxy(0)
self.login_failed = 0
else:
self.login_failed = self.login_failed + 1
if self.login_failed >= self.max_attempt_login:
if self.debug_mode == 'true':
self.log.log(
"Can not login Msb:" + self.payment.get_username +
" with " + str(self.max_attempt_login) +
" attempts", 'error')
return 0
return self.perform_login()
else:
session_requests.set_changing_proxy(0)
if self.debug_mode == 'true':
self.log.log(
"Login Msb:" + self.payment.get_username +
" successfully by using old session", 'debug')
token = session_requests.get_token()
if token is None:
return 0
# with open('msb.html', 'w') as f:
# f.write(str(result.content))
self.update_transactions(session_requests, token)
def update_transactions(self, session_requests, token):
account_list_url = self.account_list_url
payload = {
'acctType': "'CA','SA','LN'",
'status': "'ACTV','DORM','MATU'",
'tokenNo': token,
'lang': "vi_VN",
}
list_post = session_requests.post(account_list_url, payload)
try:
response = list_post.json()
except:
response = {'status': 401}
if response['status'] == '200':
accounts = response['data']
for account in accounts:
account_name = account['acctName']
account_number = account['acctNo']
account_balance = account['availableBalance']
msb_account = MsbAccount(account_name, account_number,
self.payment.get_id())
msb_account.set_balance(account_balance)
msb_account.update_account()
account_info_url = self.account_info_url
current_date = self.history.get_current_update('maritimebank')
payload = {
'acctNo': msb_account.get_number(),
'page': 1,
'tokenNo': token,
'lang': "vi_VN",
'fromDate': current_date.rstrip('\r\n'),
'toDate': time.strftime("%Y-%m-%d")
}
account_post = session_requests.post(account_info_url, payload)
response = account_post.json()
if response['status'] == '200':
histories = response['data']['history']
for history in histories:
self.save_transaction(msb_account, history)
self.log.update_log('Maritimebank', self.username)
self.history.set_current_update('maritimebank', "%Y-%m-%d")
self.log.log(
"Msb " + self.payment.get_type() + self.payment.get_username() +
": " + str(self.total_transactions) + ' transaction(s) created',
'message')
def save_transaction(self, account, history):
trading_date = self.convert_trading_date(history['transferDate'])
trading_time = history['transferTime']
description = history['remark']
if description is None:
description = 'None'
account_id = account.get_account_id()
balance = float(history['amount'])
if history['dcSign'] == 'D':
balance = -balance
reference_number = self.code.generate_code(description +
history['transferDate'])
created_at = trading_date + ' ' + trading_time
transaction = Transaction(account_id, reference_number, trading_date,
balance, description, created_at)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def replace_all(self, text, conditions):
for i, j in conditions.items():
text = text.replace(i, j)
return text
def validate_captcha(self, captcha):
if re.match("^[a-zA-Z0-9]{4}$", captcha):
return 1
return 0
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%d/%m/%Y')
return date.strftime('%Y-%m-%d')
def get_token_from_string(self, string):
token = re.search(
"\'[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}",
string)
return token[0].replace("'", "")
def __init__(self):
self.nds = NodeStore()
self.cursor_y = 0
self._load_data()
self._update = True
self.history = History(seed=self.nds)
class UserFile:
DATA_FILE = os.path.expanduser("~/.cache/.wfclidata")
root_node_id = "0"
# SETUP METHODS
def __init__(self):
self.nds = NodeStore()
self.cursor_y = 0
self._load_data()
self._update = True
self.history = History(seed=self.nds)
# Decorator for funtions that need to force an update to our tree
def update_visible_after(func):
def do_update(self, *args, **kwargs):
result = func(self, *args, **kwargs)
self._update = True
return result
return do_update
def update_visible_now(self):
self._update = True
def set_cursor_to_node(self, node_id):
self.update_visible_now()
for i, v in enumerate(self.visible):
if v[0].uuid == node_id:
self.cursor_y = i
def current_node(self, depth=False):
node_pair = self.visible[self.cursor_y]
if depth:
return node_pair
else:
return node_pair[0]
# FILE METHODS
def data_from_file_object(self, fo):
data = json.load(fo)
for node_def in data:
node = Node(node_def=node_def)
self.nds.add_node(node)
@classmethod
def _write_data_file(cls, data_obj):
os.makedirs(os.path.dirname(cls.DATA_FILE), exist_ok=True)
with open(cls.DATA_FILE, "x+") as f:
json.dump(data_obj, f, indent=2)
@classmethod
def _create_empty_data_file(cls):
empty_data = [
{"id": cls.root_node_id, "pa": None, "ch": ["1"]},
{"pa": cls.root_node_id, "id": "1", "nm": "Write down your thoughts"},
]
cls._write_data_file(empty_data)
def _load_data(self):
if not os.path.exists(self.DATA_FILE):
self._create_empty_data_file()
with open(self.DATA_FILE) as f:
self.data_from_file_object(f)
def save(self):
with open(self.DATA_FILE, "w") as f:
json.dump(self.nds.flat_format, f, indent=2)
def commit(self):
self.history.add(self.nds)
# TREE TRAVERSAL
@property
def visible(self):
if self._update:
self._update = False
return self.load_visible()
else:
return self._visible
def load_visible(self):
"""
returns a list of tuples like this ( node, depth,)
"""
self._visible = []
for node in self.nds.get_node(self.root_node_id).children:
if node is not None:
self._traverse_node(node, 0)
return self._visible
def _traverse_node(self, node, depth):
current_node = self.nds.get_node(node)
self._visible.append((current_node, depth))
if not current_node.closed:
for child in current_node.children:
self._traverse_node(child, depth + 1)
# NAVIGATION METHODS
def nav_up(self):
if self.cursor_y > 0:
self.cursor_y -= 1
def nav_down(self):
if self.cursor_y < len(self.visible) - 1:
self.cursor_y += 1
def bottom(self):
self.cursor_y = len(self.visible) - 1
def top(self):
self.cursor_y = 0
# LINKING METHODS
@update_visible_after
def link_parent_child(self, parent, child, position=-1):
self.nds.get_node(child).parent = parent
if position >= 0:
self.nds.get_node(parent).children.insert(position, child)
else:
self.nds.get_node(parent).children.append(child)
@update_visible_after
def unlink_relink(self, old_parent, child, new_parent, position):
def unlink_parent_child(self, parent, child):
assert child in self.nds
assert parent in self.nds
assert self.nds.get_node(child).parent == parent
assert child in self.nds.get_node(parent).children
self.nds.get_node(parent).children.remove(child)
self.nds.get_node(child).parent = None
unlink_parent_child(self, old_parent, child)
self.link_parent_child(new_parent, child, position)
# MANIPULATE NODES
@update_visible_after
def indent(self):
current_node = self.current_node()
parent_node = current_node.parent
parents_child_list = self.nds.get_node(parent_node).children
current_node_index = parents_child_list.index(current_node.uuid)
if current_node_index == 0:
raise ModelException("Indent of top child")
else:
new_parent = parents_child_list[current_node_index - 1]
self.unlink_relink(parent_node, current_node.uuid, new_parent, -1)
self.nds.get_node(new_parent).closed = False
log.info("Nailed it")
@update_visible_after
def unindent(self):
current_node = self.current_node()
parent_id = current_node.parent
if parent_id == self.root_node_id:
raise ModelException("top level, can't unindent")
else:
super_parent_node = self.nds.get_node(self.nds.get_node(parent_id).parent)
pos_in_parent_list = super_parent_node.children.index(parent_id)
self.unlink_relink(
parent_id,
current_node.uuid,
super_parent_node.uuid,
pos_in_parent_list + 1,
)
log.info("nailed it")
@update_visible_after
def open_above(self):
current_node = self.current_node()
parent_node = self.nds.get_node(current_node.parent)
new_node = self.create_node(parent_node.uuid)
pos_in_parent_list = parent_node.children.index(current_node.uuid)
self.link_parent_child(
parent_node.uuid,
new_node.uuid,
pos_in_parent_list,
)
@update_visible_after
def open_below(self):
current_node = self.current_node()
if current_node.state == "open":
new_node = self.create_node(current_node.uuid)
self.link_parent_child(
current_node.uuid,
new_node.uuid,
0,
)
else: # new node is sibling of current node
parent_node = self.nds.get_node(current_node.parent)
new_node = self.create_node(parent_node.uuid)
pos_in_parent_list = parent_node.children.index(current_node.uuid)
self.link_parent_child(
parent_node.uuid,
new_node.uuid,
pos_in_parent_list + 1,
)
@update_visible_after
def delete_item(self, node_id=None):
current_node = self.current_node() if node_id is None else self.nds.get_node(node_id)
for child_id in current_node.children[:]:
self.delete_item(node_id=child_id)
parent_id = current_node.parent
self.nds.get_node(parent_id).children.remove(current_node.uuid)
del self.nds[current_node.uuid]
if node_id is None: # this is our top-level delete
self.cursor_y = max(0, self.cursor_y - 1)
if len(self.nds.get_node(self.root_node_id).children) == 0:
new_node = self.create_node(
self.root_node_id,
nm="Ooops, you deleted the last item on the list",
)
self.link_parent_child(
self.root_node_id,
new_node.uuid,
0,
)
@update_visible_after
def move_down(self):
current_node = self.current_node()
parent_id = current_node.parent
parents_child_list = self.nds.get_node(parent_id).children
current_node_index = parents_child_list.index(current_node.uuid)
if current_node_index < len(parents_child_list) - 1:
# swap with the one behind
parents_child_list[current_node_index] = parents_child_list[current_node_index + 1]
parents_child_list[current_node_index + 1] = current_node.uuid
self.set_cursor_to_node(current_node.uuid)
@update_visible_after
def move_up(self):
current_node = self.current_node()
parent_id = current_node.parent
parents_child_list = self.nds.get_node(parent_id).children
current_node_index = parents_child_list.index(current_node.uuid)
if current_node_index > 0:
# swap with the one behind
parents_child_list[current_node_index] = parents_child_list[current_node_index - 1]
parents_child_list[current_node_index - 1] = current_node.uuid
self.set_cursor_to_node(current_node.uuid)
@update_visible_after
def complete(self):
current_node = self.current_node()
current_node.complete = not current_node.complete
@update_visible_after
def create_node(self, parent, **kwargs):
node = Node(pa=parent, **kwargs)
self.nds.add_node(node)
return node
@update_visible_after
def collapse_node(self):
self.visible[self.cursor_y][0].closed = True
@update_visible_after
def expand_node(self):
self.visible[self.cursor_y][0].closed = False
@update_visible_after
def undo(self):
ret = self.history.undo()
if ret is not None:
self.nds = ret
@update_visible_after
def redo(self):
ret = self.history.redo()
if ret is not None:
self.nds = ret
# EDIT TEXT
@update_visible_after
def add_char(self, char, cursor_x):
current_node = self.current_node()
name = current_node.name[0:cursor_x] + char + current_node.name[cursor_x:]
current_node.name = name
@update_visible_after
def delete_char(self, num, cursor_x):
current_node = self.current_node()
if cursor_x > 0:
name = current_node.name[0:cursor_x - num] + current_node.name[cursor_x:]
current_node.name = name
class SacombankEnterprise:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.username = payment.get_username()
self.password = payment.get_password()
sacombank = self.get_config()
self.email_transport = EmailTransport()
self.corp_url = sacombank['corp_url']
self.captcha = CaptchaResolver()
self.debug_mode = sacombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 20
self.login_failed = 0
self.session = session
self.proxy = proxy
self.payment = payment
def get_config(self):
config = self.config.get_section_config('SacombankEnterprise')
return config
def perform_login(self):
corp_url = self.corp_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver(
) is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
while True:
driver.get(corp_url)
online_banking = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH,
"//a[contains(@class,'btn-ebanking')]")))
online_banking.click()
time.sleep(5)
try:
# Click to ignore banner
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//div[@id="pbanner"]')))
element.click()
except:
print('Banner not found')
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((
By.XPATH,
'//input[@id="AuthenticationFG.USER_PRINCIPAL"]')))
element.send_keys(username, Keys.ARROW_DOWN)
# Get captcha input element
captcha = self.get_captcha(driver)
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((
By.XPATH,
'//input[@name="AuthenticationFG.VERIFICATION_CODE"]'
)))
element.send_keys(captcha)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH,
'//input[@id="STU_VALIDATE_CREDENTIALS"]')))
element.click()
time.sleep(5)
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH,
'//span[@id="LoginHDisplay.Ra4.C1"]')))
element.click()
except:
print('Can not login')
driver.close()
return self.perform_login()
# Get password input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH,
'//input[@name="AuthenticationFG.ACCESS_CODE"]')))
element.send_keys(password)
element.send_keys(Keys.RETURN)
time.sleep(5)
# Update Account Information
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH,
"//table[@id='HWListTable10072682']//tr/td")))
account_info = driver.find_elements_by_xpath(
"//table[@id='HWListTable10072682']//tr/td")
account_number = account_info[0].text.strip()
account_name = self.payment.get_username()
account_balance = float(
account_info[2].text.strip().replace('\n', '').replace(
'VND', '').replace('.', ''))
account = self.update_account(account_name, account_number,
account_balance,
self.payment.get_id())
# click to transaction menu
action_link = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH, "//a[contains(@name,'HREF_Ti_khon')]")))
hover_action = ActionChains(driver).move_to_element(
action_link)
hover_action.perform()
acc_link = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH,
"//a[contains(@id,'Ti-khon_Tng-quan-ti-khon')]")))
hover_trans = ActionChains(driver).move_to_element(
acc_link)
hover_trans.perform()
acc_link.click()
# time.sleep(5)
trans_link = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((
By.XPATH,
"//a[contains(@id,'HREF_PageConfigurationMaster_CACCTSW__1:AccountSummaryFG')]"
)))
trans_link.click()
try:
time.sleep(5)
WebDriverWait(driver, 5).until(
EC.visibility_of_element_located((
By.XPATH,
"//a[contains(@id,'PageConfigurationMaster_CACCTSW__1:errorlink1')]"
)))
driver.close()
self.history.set_current_update(
'sacombank_enterprise', "%d/%m/%Y")
self.session.set_driver(None)
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
"No transactions today", 'message')
return False
except:
print('ok')
time.sleep(1)
try:
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH,
"//table[contains(@id,'HWListTable')]//tbody"
)))
transactions = driver.find_elements_by_xpath(
"//table[contains(@id,'HWListTable')]//tbody[position() >= 0 and position() <= 10]"
)
for row in transactions:
columns = row.find_elements_by_xpath(".//tr/td")
self.save_transaction(account, columns)
self.log.update_log('Sacombank', self.username)
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
str(self.total_transactions) +
' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
"Cannot load transactions", 'error')
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
self.log.log(
str(exc_type) + '-' + fname + '-' +
str(exc_tb.tb_lineno), 'error')
self.session.set_changing_proxy(1)
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(
self.payment.get_name() + '-' + self.payment.get_type() +
'-' + self.payment.get_username() + ': ' +
str(sys.exc_info()), 'debug')
driver.close()
self.history.set_current_update('sacombank_enterprise', "%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
reference_number = detail[0].text
trading_date = self.convert_trading_date(detail[2].text)
account_id = account.get_account_id()
description = detail[3].text
if detail[6].text is not '':
balance = -float(detail[6].text.replace(',', '').replace(
' ', '').replace('VND', ''))
else:
balance = float(detail[7].text.replace(',', '').replace(
' ', '').replace('VND', ''))
transaction = Transaction(account_id, reference_number, trading_date,
balance, description)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def get_captcha(self, driver):
img_data = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//img[@id='IMAGECAPTCHA']")))
img_file = img_data.screenshot_as_png
self.captcha.save_from_source(img_file, 'png')
captcha_text = self.captcha.resolve(True)
if re.match("^[0-9]{4}$", captcha_text):
return captcha_text
reload_captcha = WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//img[@id='TEXTIMAGE']")))
reload_captcha.click()
return self.get_captcha(driver)
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%d-%m-%Y')
return date.strftime('%Y-%m-%d')
def update_account(self, name, number, balance, payment_id):
account = SacombankAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def is_debug(self):
return self.debug_mode
class TechcombankEnterprise:
def __init__(self, payment, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.payment = payment
self.config = Config()
self.log = Log()
techcombank = self.get_techcombank_config()
self.email_transport = EmailTransport()
self.login_url = techcombank['login_url']
self.username = payment.get_username()
self.password = payment.get_password()
self.debug_mode = techcombank['debug_mode']
self.total_transactions = 0
self.history = History()
self.code = GenerateCode()
self.max_attempt_login = 20
self.login_failed = 0
def perform_login(self):
login_url = self.login_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver(
) is None or self.session.get_last_driver() is 'Firefox':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Chrome')
else:
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
self.session.set_driver(driver)
try:
while 1:
driver.get(login_url)
window_before = driver.window_handles[0]
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="signOnName"]')))
element.send_keys(username)
time.sleep(2)
# element.send_keys(Keys.RETURN)
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="password"]')))
# Get password input element
element.send_keys(password)
time.sleep(2)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@type="submit"]')))
element.click()
try:
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH,
"//div[contains(@id,'pane__FRAME')]")))
self.login_failed = 0
break
except:
self.login_failed += 1
if self.login_failed > self.max_attempt_login:
driver.close()
self.history.set_current_update(
'techcombank_enterprise', "%d/%m/%Y")
self.session.set_driver(None)
# Update account information
try:
search = WebDriverWait(driver, 30).until(
EC.visibility_of_element_located(
(By.XPATH,
"//img[contains(@src,'drilldown/View.png')]")))
info = driver.find_elements_by_xpath(
"//table[contains(@id,'datadisplay_ACCOUNT')]//tr/td")
number = info[0].text.split('-')[0].strip()
name = info[1].text
balance = float(info[3].text.replace(',', ''))
account = self.update_account(name, number, balance,
self.payment.get_id())
search.click()
window_after = driver.window_handles[1]
driver.switch_to_window(window_after)
WebDriverWait(driver, 60).until(
EC.visibility_of_element_located(
(By.XPATH,
"//table[contains(@id,'datadisplay_DATA')]")))
transactions = driver.find_elements_by_xpath(
"//table[contains(@id,'datadisplay_DATA')]//tr")
for row in transactions:
columns = row.find_elements_by_xpath("td")
self.save_transaction(account, columns)
self.log.update_log('Techcombank', self.username)
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
str(self.total_transactions) +
' transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log(
self.payment.get_name() + '-' +
self.payment.get_type() + '-' +
self.payment.get_username() + ": " +
"Cannot load transactions", 'error')
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
self.log.log(
str(exc_type) + '-' + fname + '-' +
str(exc_tb.tb_lineno), 'error')
self.session.set_changing_proxy(1)
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(str(sys.exc_info()), 'debug')
driver.quit()
self.history.set_current_update('techcombank_enterprise',
"%d/%m/%Y")
self.session.set_driver(None)
def save_transaction(self, account, detail):
trading_date = self.convert_trading_date(detail[0].text)
reference_number = detail[1].text
description = detail[3].text
account_id = account.get_account_id()
if detail[4].text is not '':
balance = float(detail[4].text.replace(',', '').replace(' ', ''))
else:
balance = float(detail[5].text.replace(',', '').replace(' ', ''))
transaction = Transaction(account_id, reference_number, trading_date,
balance, description)
if transaction.save() == 1:
self.total_transactions = self.total_transactions + 1
self.email_transport.send_transaction_email(account, transaction)
def convert_trading_date(self, trading_date):
date = datetime.strptime(trading_date, '%d/%m/%Y')
return date.strftime('%Y-%m-%d')
def update_account(self, name, number, balance, payment_id):
account = TechcombankAccount(name, number, payment_id)
account.set_balance(balance)
account.update_account()
return account
def get_techcombank_config(self):
techcombank = self.config.get_section_config('TechcombankEnterprise')
return techcombank
def is_debug(self):
return self.debug_mode
def __create_objects(query):
return_list = []
for history in [i for i in query.cursor.fetchall()]:
return_list.append(History.from_json(history))
return return_list
class Msb:
def __init__(self, name=None, session=None, proxy={}):
self.session = session
self.proxy = proxy
self.config = Config()
self.log = Log()
self.captcha = CaptchaResolver()
section_config = self.get_section_config(name)
self.email_transport = EmailTransport()
self.home_url = section_config['home_url']
self.login_url = section_config['login_url']
self.username = section_config['username']
self.password = section_config['password']
self.debug_mode = section_config['debug_mode']
self.total_transactions = 0
self.max_attempt_login = 10
self.max_validate_captcha = 10
self.login_failed = 0
self.captcha_failed = 0
self.history = History()
self.code = GenerateCode()
def is_debug(self):
return self.debug_mode
def get_section_config(self, name=None):
if name is None:
name = 'Msb'
section_config = self.config.get_section_config(name)
return section_config
def perform_login(self):
login_url = self.login_url
username = self.username
password = self.password
selenium = Selenium()
if self.session.get_driver() is None:
if self.session.get_last_driver() is None or self.session.get_last_driver() is 'Chrome':
driver = selenium.get_firefox_driver(self.proxy)
self.session.set_last_driver('Firefox')
else:
driver = selenium.get_chrome_driver(self.proxy)
self.session.set_last_driver('Chrome')
self.session.set_driver(driver)
try:
driver.get(login_url)
# Get username input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="_userName"]'))
)
element.send_keys(username)
# Get password input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="_passwordS"]'))
)
element.send_keys(password)
captcha = self.get_captcha(driver)
print(captcha)
exit(111111)
# Get captcha input element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@name="_verifyCode"]'))
)
element.send_keys(captcha)
# element.send_keys(Keys.RETURN)
# Get submit button element
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//button[@id="loginBtn"]'))
)
element.click()
# click to account menu
time.sleep(5)
account_link = WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, '//a[@name="AI.QCK.ACCOUNT"]'))
)
retry_account = 0
while retry_account < 3:
try:
account_link.click()
# Click to view last 10 transactions
time.sleep(5)
current_date = self.history.get_current_update('techcombank')
from_date = driver.find_elements_by_xpath(
"//input[@id='fieldName:START.DATE']")[0]
from_date.send_keys(current_date, Keys.ENTER)
time.sleep(5)
to_date = driver.find_elements_by_xpath(
"//input[@id='fieldName:END.DATE']")[0]
to_date.send_keys(current_date)
time.sleep(5)
button = driver.find_elements_by_xpath(
'//a[@title="Xem giao dịch"]')[1]
button.click()
time.sleep(5)
button = driver.find_elements_by_xpath(
'//a[@title="Xem giao dịch"]')[1]
button.click()
break
except:
self.log.log("Techcombank: Re-click to account", 'debug')
self.log.log(str(sys.exc_info()), 'debug')
retry_account = retry_account + 1
# Update account information
try:
WebDriverWait(driver, 15).until(
EC.visibility_of_element_located(
(By.XPATH, "//table[@id='enqheader']//tr[position()>1 and position()<4]"))
)
info = driver.find_elements_by_xpath(
"//table[@id='enqheader']//tr[position()>1 and position()<4]")
number = info[0].find_elements_by_xpath("td")[1].text
balance = float(info[0].find_elements_by_xpath("td")[3].text.replace(',', ''))
name = info[1].find_elements_by_xpath("td")[1].text
account = self.update_account(name, number, balance)
transactions = driver.find_elements_by_xpath(
"//table[starts-with(@id,'datadisplay_MainBody')]//tr")
for row in transactions:
columns = row.find_elements_by_xpath("td")
self.save_transaction(account, columns)
self.log.update_log('Techcombank', self.username)
self.log.log(str(self.total_transactions) + ' Tcb transaction(s) created', 'message')
self.session.set_changing_proxy(0)
except:
self.log.log("Techcombank: Cannot load transactions", 'error')
self.log.log(str(sys.exc_info()), 'error')
self.session.set_changing_proxy(1)
except:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.log.log(str(sys.exc_info()), 'debug')
driver.close()
self.history.set_current_update('techcombank', "%d/%m/%Y")
self.session.set_driver(None)
def get_captcha(self, driver):
img_data = driver.find_element_by_id('safecode').screenshot_as_png
self.captcha.save_from_source(img_data)
captcha_text = self.captcha.resolve(True)
if re.match("^[a-zA-Z0-9]{4}$", captcha_text):
return captcha_text
return self.get_captcha(driver)