def do_upgrade(self):
try:
try:
self.http_handler.log_ok('Start upgrader')
self.copy_basic_conf()
self.stop_agent()
self.deploy_new_agent()
self.start_agent()
if not self.run_hooks():
self.roll_back()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
else:
nfs.remove(AGENT_BACK_DIR)
self.http_handler.log_ok('Upgrade successfully', done=True)
except Exception as e:
self.http_handler.log_error(str(e))
logger.error(str(e), exc_info=True)
self.roll_back()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
except Exception:
self.start_agent()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
def post_message(self, msg, done, status):
if done:
body = {
'task_id': self.task_id,
'result': msg,
'exit_code': status,
'is_timeout': False,
'is_aborted': False
}
type = 102
else:
body = {'task_id': self.task_id, 'log': msg}
type = 103
payload = {
'messages': [{
'id': uuid.uuid4().hex,
'type': type,
'body': body
}]
}
logger.info(payload)
try:
r = requests.post(self.message_url, json=payload, verify=False)
if r.status_code != 200:
logger.error('Post task log:{} failed, code {} !!!'.format(
payload, r.status_code))
except Exception as e:
logger.error('Post task log:{} failed, exception: {}!!!'.format(
payload, e))
def _load_config(self):
if os.path.exists(CONFIG_PATH):
try:
with open(CONFIG_PATH) as f:
self.conf_dict = yaml.load(f.read())
except yaml.YAMLError as exc:
logger.error('Invalid configuration: %s', exc)
def _init_http_handler(self):
try:
self.http_handler = HttpHandler(self.task_message['task_id'])
self.http_handler.initialization()
except Exception as e:
logger.error(str(e), exc_info=True)
return False
return True
def validate(self):
if not isinstance(self.task_message, dict):
logger.error('Info should be dict')
return False
for key in ('version', 'task_id', 'filename'):
if key not in self.task_message:
logger.error('{!r} is missing in info'.format(key))
return False
return True
def waiting_for_upgrade(self):
while 1:
try:
task_message = yield self.task_queue.get()
from preupgrade import PreUpgrade
pre_upgrade = PreUpgrade(task_message)
pre_upgrade.execute()
reloader.reload_module(self.modify_times)
except Exception as e:
logger.error(e)
def write_balance(session, balance_new, currency):
if currency == 'USD':
model = BalanceUSD
elif currency == 'CNY':
model = BalanceCNY
else:
logger.error('Wrong currency!')
logger.info('Writing to database: {}'.format(model.__tablename__))
session.bulk_insert_mappings(model, balance_new.to_dict('record'))
def run(self):
try:
upgrade_app = self.create_upgrade_app()
upgrade_server = httpserver.HTTPServer(upgrade_app)
upgrade_server.bind(UPGRADE_PORT, '127.0.0.1')
upgrade_server.start()
self.io_loop.spawn_callback(self.waiting_for_upgrade)
self.io_loop.start()
except Exception as e:
logger.error(str(e))
def get_historical_stock_data(ticker, start_date, end_date):
arr = []
try:
s = Share(ticker)
start_date, end_date = start_date.isoformat(), end_date.isoformat()
data = s.get_historical(start_date, end_date)
for day in data:
arr.append(DataFetcher._extract_data(day))
arr.reverse() # this makes sure that the oldest day is first so the day field is set to 0
except YQLQueryError:
logger.error("Historical data not found for {}".format(ticker))
return arr
def roll_back(self):
self.http_handler.log_ok('Rolling back...')
try:
self.stop_agent()
self._roll_back_files()
self.start_agent()
except Exception as e:
self.http_handler.log_error(str(e))
logger.error(str(e), exc_info=True)
self._roll_back_files()
self.http_handler.log_ok('Roll back done', done=True)
def get_historical_stock_data(ticker, start_date, end_date):
arr = []
try:
s = Share(ticker)
start_date, end_date = start_date.isoformat(), end_date.isoformat()
data = s.get_historical(start_date, end_date)
for day in data:
arr.append(DataFetcher._extract_data(day))
arr.reverse(
) # this makes sure that the oldest day is first so the day field is set to 0
except YQLQueryError:
logger.error("Historical data not found for {}".format(ticker))
return arr
def get_extra_field(ticker, date, field):
try:
s = Share(ticker)
date = date.isoformat()
data = s.get_historical(date, date)[0]
try:
value = data[field]
return value
except KeyError as e:
logger.error('KeyError: {}, while trying to access {} field'.format(e, field))
except YQLQueryError:
logger.error("Error gathering data for {}".format(ticker))
return None
def copy_basic_conf(self):
self.http_handler.log_ok('Coping basic conf')
try:
new_conf_path = nfs.join(self.new_agent_dir, CONFIG_NAME)
with open(CONFIG_PATH) as temp:
yaml.dump(yaml.load(temp.read()),
open(new_conf_path, 'a+'),
default_flow_style=False)
self.http_handler.log_ok('Copy conf done')
except IOError as e:
self.http_handler.log_error(str(e))
logger.error(str(e), exc_info=True)
self.http_handler.log_ok('Skip copy basic conf')
def get_last_balance(session, currency):
if currency == 'USD':
model = BalanceUSD
elif currency == 'CNY':
model = BalanceCNY
else:
logger.error('Wrong currency!')
df_query = session.query(model)
df = pd.read_sql(df_query.statement, df_query.session.bind)
df = df.sort_values(
['observation_date', 'last_update']
).drop_duplicates('account', keep='last')
return df[['observation_date', 'account', 'balance']]
def get_extra_field(ticker, date, field):
try:
s = Share(ticker)
date = date.isoformat()
data = s.get_historical(date, date)[0]
try:
value = data[field]
return value
except KeyError as e:
logger.error(
'KeyError: {}, while trying to access {} field'.format(
e, field))
except YQLQueryError:
logger.error("Error gathering data for {}".format(ticker))
return None
def get_today_stock_data(ticker):
try:
s = Share(ticker)
data = {
'Open': float(s.get_open()),
'Close': float(s.get_prev_close()),
'High': float(s.get_days_high()),
'Low': float(s.get_days_low()),
'Volume': int(s.get_volume()),
'Date': datetime.date.today(),
}
return data
except YQLQueryError:
logger.error("Daily data not found for {}".format(ticker))
except Exception as e:
logger.error("Unexpected error occurred: {}".format(e))
return {}
def get_company_info(ticker):
try:
s = Share(ticker)
data = {
'Market_cap': s.get_market_cap(),
'Average_volume': s.get_avg_daily_volume(),
'EPS': s.get_earnings_share(),
'Short_ratio': s.get_short_ratio(),
'PE': s.get_price_earnings_ratio(),
'PEG': s.get_price_earnings_growth_ratio(),
}
return DataFetcher._extract_company_info(data)
except YQLQueryError:
logger.error("Company info not found for {}".format(ticker))
except Exception as e:
logger.error("Unexpected error occured: {}".format(e))
return {}
def get_today_stock_data(ticker):
try:
s = Share(ticker)
data = {
'Open': float(s.get_open()),
'Close': float(s.get_prev_close()),
'High': float(s.get_days_high()),
'Low': float(s.get_days_low()),
'Volume': int(s.get_volume()),
'Date': datetime.date.today(),
}
return data
except YQLQueryError:
logger.error("Daily data not found for {}".format(ticker))
except Exception as e:
logger.error("Unexpected error occurred: {}".format(e))
return {}
def get_company_info(ticker):
try:
s = Share(ticker)
data = {
'Market_cap': s.get_market_cap(),
'Average_volume': s.get_avg_daily_volume(),
'EPS': s.get_earnings_share(),
'Short_ratio': s.get_short_ratio(),
'PE': s.get_price_earnings_ratio(),
'PEG': s.get_price_earnings_growth_ratio(),
}
return DataFetcher._extract_company_info(data)
except YQLQueryError:
logger.error("Company info not found for {}".format(ticker))
except Exception as e:
logger.error("Unexpected error occured: {}".format(e))
return {}
def execute(self):
try:
if not self.validate():
return
if not self._init_http_handler():
return
nfs.makedirs(AGENT_DOWNLOAD_DIR)
self.check_version(self.task_message['version'])
compress_agent_path = self.download()
uncompress_agent_path = self.umcompress(compress_agent_path)
# Backup files
self.backup_files()
# Run upgrader
from doupgrade import DoUpgrade
do_upgrade = DoUpgrade(uncompress_agent_path, self.http_handler)
do_upgrade.do_upgrade()
except Exception as e:
self.http_handler.log_error(str(e), done=True)
logger.error(str(e), exc_info=True)
def __init__(self, symbol, day, interval_lenght=1):
"""
Class that holds data for one trading day.
If collapse_open_close then the volume of the opening auction is added
to the first continuous trading interval, and the closing auction
to the last.
"""
if not interval_lenght in [1, 2, 5, 10, 15, 30]:
logger.error("unsupported interval_lenght")
raise Exception
self.interval_lenght = interval_lenght
self.symbol = symbol
self.day = day
# add p_0 and p_closing
self.volumes = np.zeros(390 / interval_lenght + 2)
self.prices = np.zeros(390 / interval_lenght + 2) * \
np.nan # set them to NaN
self.times_list = self.compute_times()
self.activeTradingIntervalIndex = 1
self.activeTradingIntervalTime = self.times_list[
self.activeTradingIntervalIndex]
self.activeTradingInterval = ContinuousTradingInterval(
self.activeTradingIntervalTime)
self.InvalidDay = False
