def do_upgrade(self): try: try: self.http_handler.log_ok('Start upgrader') self.copy_basic_conf() self.stop_agent() self.deploy_new_agent() self.start_agent() if not self.run_hooks(): self.roll_back() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True) else: nfs.remove(AGENT_BACK_DIR) self.http_handler.log_ok('Upgrade successfully', done=True) except Exception as e: self.http_handler.log_error(str(e)) logger.error(str(e), exc_info=True) self.roll_back() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True) except Exception: self.start_agent() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True)
def post_message(self, msg, done, status): if done: body = { 'task_id': self.task_id, 'result': msg, 'exit_code': status, 'is_timeout': False, 'is_aborted': False } type = 102 else: body = {'task_id': self.task_id, 'log': msg} type = 103 payload = { 'messages': [{ 'id': uuid.uuid4().hex, 'type': type, 'body': body }] } logger.info(payload) try: r = requests.post(self.message_url, json=payload, verify=False) if r.status_code != 200: logger.error('Post task log:{} failed, code {} !!!'.format( payload, r.status_code)) except Exception as e: logger.error('Post task log:{} failed, exception: {}!!!'.format( payload, e))
def _load_config(self): if os.path.exists(CONFIG_PATH): try: with open(CONFIG_PATH) as f: self.conf_dict = yaml.load(f.read()) except yaml.YAMLError as exc: logger.error('Invalid configuration: %s', exc)
def _init_http_handler(self): try: self.http_handler = HttpHandler(self.task_message['task_id']) self.http_handler.initialization() except Exception as e: logger.error(str(e), exc_info=True) return False return True
def validate(self): if not isinstance(self.task_message, dict): logger.error('Info should be dict') return False for key in ('version', 'task_id', 'filename'): if key not in self.task_message: logger.error('{!r} is missing in info'.format(key)) return False return True
def waiting_for_upgrade(self): while 1: try: task_message = yield self.task_queue.get() from preupgrade import PreUpgrade pre_upgrade = PreUpgrade(task_message) pre_upgrade.execute() reloader.reload_module(self.modify_times) except Exception as e: logger.error(e)
def write_balance(session, balance_new, currency): if currency == 'USD': model = BalanceUSD elif currency == 'CNY': model = BalanceCNY else: logger.error('Wrong currency!') logger.info('Writing to database: {}'.format(model.__tablename__)) session.bulk_insert_mappings(model, balance_new.to_dict('record'))
def run(self): try: upgrade_app = self.create_upgrade_app() upgrade_server = httpserver.HTTPServer(upgrade_app) upgrade_server.bind(UPGRADE_PORT, '127.0.0.1') upgrade_server.start() self.io_loop.spawn_callback(self.waiting_for_upgrade) self.io_loop.start() except Exception as e: logger.error(str(e))
def get_historical_stock_data(ticker, start_date, end_date): arr = [] try: s = Share(ticker) start_date, end_date = start_date.isoformat(), end_date.isoformat() data = s.get_historical(start_date, end_date) for day in data: arr.append(DataFetcher._extract_data(day)) arr.reverse() # this makes sure that the oldest day is first so the day field is set to 0 except YQLQueryError: logger.error("Historical data not found for {}".format(ticker)) return arr
def roll_back(self): self.http_handler.log_ok('Rolling back...') try: self.stop_agent() self._roll_back_files() self.start_agent() except Exception as e: self.http_handler.log_error(str(e)) logger.error(str(e), exc_info=True) self._roll_back_files() self.http_handler.log_ok('Roll back done', done=True)
def get_historical_stock_data(ticker, start_date, end_date): arr = [] try: s = Share(ticker) start_date, end_date = start_date.isoformat(), end_date.isoformat() data = s.get_historical(start_date, end_date) for day in data: arr.append(DataFetcher._extract_data(day)) arr.reverse( ) # this makes sure that the oldest day is first so the day field is set to 0 except YQLQueryError: logger.error("Historical data not found for {}".format(ticker)) return arr
def get_extra_field(ticker, date, field): try: s = Share(ticker) date = date.isoformat() data = s.get_historical(date, date)[0] try: value = data[field] return value except KeyError as e: logger.error('KeyError: {}, while trying to access {} field'.format(e, field)) except YQLQueryError: logger.error("Error gathering data for {}".format(ticker)) return None
def copy_basic_conf(self): self.http_handler.log_ok('Coping basic conf') try: new_conf_path = nfs.join(self.new_agent_dir, CONFIG_NAME) with open(CONFIG_PATH) as temp: yaml.dump(yaml.load(temp.read()), open(new_conf_path, 'a+'), default_flow_style=False) self.http_handler.log_ok('Copy conf done') except IOError as e: self.http_handler.log_error(str(e)) logger.error(str(e), exc_info=True) self.http_handler.log_ok('Skip copy basic conf')
def get_last_balance(session, currency): if currency == 'USD': model = BalanceUSD elif currency == 'CNY': model = BalanceCNY else: logger.error('Wrong currency!') df_query = session.query(model) df = pd.read_sql(df_query.statement, df_query.session.bind) df = df.sort_values( ['observation_date', 'last_update'] ).drop_duplicates('account', keep='last') return df[['observation_date', 'account', 'balance']]
def get_extra_field(ticker, date, field): try: s = Share(ticker) date = date.isoformat() data = s.get_historical(date, date)[0] try: value = data[field] return value except KeyError as e: logger.error( 'KeyError: {}, while trying to access {} field'.format( e, field)) except YQLQueryError: logger.error("Error gathering data for {}".format(ticker)) return None
def get_today_stock_data(ticker): try: s = Share(ticker) data = { 'Open': float(s.get_open()), 'Close': float(s.get_prev_close()), 'High': float(s.get_days_high()), 'Low': float(s.get_days_low()), 'Volume': int(s.get_volume()), 'Date': datetime.date.today(), } return data except YQLQueryError: logger.error("Daily data not found for {}".format(ticker)) except Exception as e: logger.error("Unexpected error occurred: {}".format(e)) return {}
def get_company_info(ticker): try: s = Share(ticker) data = { 'Market_cap': s.get_market_cap(), 'Average_volume': s.get_avg_daily_volume(), 'EPS': s.get_earnings_share(), 'Short_ratio': s.get_short_ratio(), 'PE': s.get_price_earnings_ratio(), 'PEG': s.get_price_earnings_growth_ratio(), } return DataFetcher._extract_company_info(data) except YQLQueryError: logger.error("Company info not found for {}".format(ticker)) except Exception as e: logger.error("Unexpected error occured: {}".format(e)) return {}
def execute(self): try: if not self.validate(): return if not self._init_http_handler(): return nfs.makedirs(AGENT_DOWNLOAD_DIR) self.check_version(self.task_message['version']) compress_agent_path = self.download() uncompress_agent_path = self.umcompress(compress_agent_path) # Backup files self.backup_files() # Run upgrader from doupgrade import DoUpgrade do_upgrade = DoUpgrade(uncompress_agent_path, self.http_handler) do_upgrade.do_upgrade() except Exception as e: self.http_handler.log_error(str(e), done=True) logger.error(str(e), exc_info=True)
def __init__(self, symbol, day, interval_lenght=1): """ Class that holds data for one trading day. If collapse_open_close then the volume of the opening auction is added to the first continuous trading interval, and the closing auction to the last. """ if not interval_lenght in [1, 2, 5, 10, 15, 30]: logger.error("unsupported interval_lenght") raise Exception self.interval_lenght = interval_lenght self.symbol = symbol self.day = day # add p_0 and p_closing self.volumes = np.zeros(390 / interval_lenght + 2) self.prices = np.zeros(390 / interval_lenght + 2) * \ np.nan # set them to NaN self.times_list = self.compute_times() self.activeTradingIntervalIndex = 1 self.activeTradingIntervalTime = self.times_list[ self.activeTradingIntervalIndex] self.activeTradingInterval = ContinuousTradingInterval( self.activeTradingIntervalTime) self.InvalidDay = False