def _sync(self) -> bool: try: str_date = DateTime(self.get_last_sync_date(), timezone_hours=0).to_str("%Y%m%d") market_data = self._get_market_data(str_date) if market_data is not None: with DBLock: try: # save to models self.save_market_data(market_data) # update sync date self.set_last_sync_date( DateTime(timezone_hours=0).date.timestamp) self._logger.log( "%s market data sync success. date: %s" % (self.__class__.__name__, DateTime(self.get_last_sync_date(), timezone_hours=0).to_str("%Y%m%d"))) commit() except Exception as e: rollback() raise e return True else: return False except Exception as e: self._logger.log_err(e) return False
def _start_timer(self): if self._timer is not None: self._timer.cancel() # 今天的数据,需要明天零点半后开始同步(utc+0) t = DateTime(timezone_hours=0).add_days(1).date.timestamp - DateTime().timestamp + 60 * 30 self._timer = Timer(t, self.start) self._timer.daemon = True self._timer.start()
def start(self): with self._lock: now = DateTime(timezone_hours=0).date.timestamp dt = self._begin_date if self._last_date != 0: dt = self._last_date while dt < now: self._dates.append(dt) dt = DateTime(dt, timezone_hours=0).date.add_days(1).timestamp self._last_date = dt self._check_and_req()
def _add_to_address(self, db_table, addresses: dict, nr: NrItem) -> str: a = nr.address item = self.new_pb_item(nr) now = DateTime().timestamp c = self.get_current_sync_date() if a in addresses.keys(): count = addresses[a].count + 1 total = addresses[a].total_nr if total is None: total = 0 else: total = float(total) total += float(item.score) i_bytes = b'|' + item.SerializeToString() i_hex = ''.join(['%02x' % b for b in i_bytes]) dates = self.get_valid_dates(addresses[a].dates) if float(item.score) >= 0.1 and now - c <= self._SEVEN_DAY_SECONDS: dates.append(str(c)) n = len(dates) str_dts = ','.join(dates) return 'UPDATE %s SET last_above_0_dates=\'%s\', last_above_0_num=\'%s\', `count`=\'%s\', total_nr=\'%s\', `data`=concat(`data`, X\'%s\') WHERE `address`=\'%s\';' % \ (db_table, str_dts, str(n), str(count), str(total), i_hex, a) else: count = 1 total = float(item.score) i_bytes = item.SerializeToString() i_hex = ''.join(['%02x' % b for b in i_bytes]) dates = [] if float(item.score) >= 0.1 and now - c <= self._SEVEN_DAY_SECONDS: dates.append(str(c)) n = len(dates) str_dts = ','.join(dates) return 'INSERT INTO %s (`address`, `last_above_0_dates`, `last_above_0_num`, `count`, total_nr, `data`) VALUES (\'%s\', \'%s\', \'%s\', \'%s\', \'%s\', X\'%s\');' % \ (db_table, a, str_dts, str(n), str(count), str(total), i_hex)
def _pass_market_data(self, content: str) -> list: if content is not None: tb = self._get_table(content) if tb is not None: items = [] trs = tb.find('tbody').findAll('tr') keys = [ 'date', 'opening', 'highest', 'lowest', 'closing', 'amount', 'total_circulation', 'total' ] for tr in trs: item = {} tds = tr.findAll('td') for i in range(len(tds)): key = keys[i] text: str = tds[i].text if key == 'date': item[key] = DateTime.from_str( text, '%Y年%m月%d日', timezone_hours=0).date.timestamp elif key == 'amount' or key == 'total_circulation': item[key] = text.replace(',', '') else: item[key] = float(text) item['total'] = format( self.currency_count() * item['closing'], '.0f') items.append(item) return items return None
def _begin(self): str_date = DateTime(self._date, timezone_hours=0).to_str('%Y%m%d') ok, data = _Request(str_date, self._is_neb).get_result() if ok and len(data) > 0: self.data = data self.did_load_data(self._date) else: time.sleep(10) self.start()
def get_valid_dates(self, dates: str): r = [] if dates is None or len(dates) == 0: return r now = DateTime().timestamp dts = dates.split(',') for d in dts: if now - int(d) <= self._SEVEN_DAY_SECONDS: r.append(d) return r
def _sync(self) -> bool: try: # str_date = DateTime(self.get_current_sync_date(), timezone_hours=0).to_str("%Y%m%d") dt = self.get_current_sync_date() date_table, _, _ = self.db_context() if self._exists(date_table, dt): self.set_last_sync_date(dt) self._logger.log_err('duplicate date: %s' % DateTime(dt, timezone_hours=0).to_str('%Y-%m-%d')) return True nr_data = self._req_queue.get(dt) # nr_data = self._get_daily_all_nr(str_date) if nr_data is not None: with DBLock: try: # save to models self._sort_nr_data(nr_data) self._save_nr_data(nr_data) # update sync date self.set_last_sync_date(dt) self._logger.log( "%s nr_data sync success. date: %s" % ( self.__class__.__name__, DateTime(self.get_last_sync_date(), timezone_hours=0).to_str("%Y%m%d") ) ) commit() self._req_queue.remove(dt) except Exception as e: rollback() raise e return True else: return False except Exception as e: self._logger.log_err(e) return False
def _check_can_sync(self) -> bool: return self.get_current_sync_date() < DateTime(timezone_hours=0).date.timestamp
def log_rewrite(f, log): with open(os.path.join(root_dir, f), 'w') as h: h.write('[' + DateTime().to_str() + '] ' + str(log) + '\n') h.flush()
def log_err(self, log): print('task error log:', log) self._e.write('[' + DateTime().to_str() + '] ' + str(log) + '\n') self._e.flush()
def log(self, log): print('task log:', log) self._f.write('[' + DateTime().to_str() + '] ' + str(log) + '\n') self._f.flush()
class NRState(object): """ 需要持久化的状态类型数据管理类(Cache & Db) """ _neb_key_last_sync_date = "neb_last_sync_date" _neb_key_market_last_sync_date = "neb_market_last_sync_date" _neb_begin_date = DateTime.from_str("20180508", "%Y%m%d", timezone_hours=0).timestamp # _neb_begin_date = DateTime.from_str("20181127", "%Y%m%d", timezone_hours=0).timestamp _eth_key_last_sync_date = "eth_last_sync_date" _eth_key_market_last_sync_date = "eth_market_last_sync_date" _eth_begin_date = DateTime.from_str("20170101", "%Y%m%d", timezone_hours=0).timestamp # neb nr date ------------------------------------------------------------------------------------------------------ @property def neb_last_sync_date(self): return self.get_last_date(self._neb_key_last_sync_date) @neb_last_sync_date.setter def neb_last_sync_date(self, value): self.set_last_date(self._neb_key_last_sync_date, value) @property def neb_current_sync_date(self): return self.get_current_date(self._neb_begin_date, self.neb_last_sync_date) # neb market date -------------------------------------------------------------------------------------------------- @property def neb_market_last_sync_date(self): d = self.get_last_date(self._neb_key_market_last_sync_date) if d == 0: d = self._neb_begin_date return d @neb_market_last_sync_date.setter def neb_market_last_sync_date(self, value): self.set_last_date(self._neb_key_market_last_sync_date, value) # eth nr date ------------------------------------------------------------------------------------------------------ @property def eth_last_sync_date(self): return self.get_last_date(self._eth_key_last_sync_date) @eth_last_sync_date.setter def eth_last_sync_date(self, value): self.set_last_date(self._eth_key_last_sync_date, value) @property def eth_current_sync_date(self): return self.get_current_date(self._eth_begin_date, self.eth_last_sync_date) # eth market date -------------------------------------------------------------------------------------------------- @property def eth_market_last_sync_date(self): d = self.get_last_date(self._eth_key_market_last_sync_date) if d == 0: d = self._eth_begin_date return d @eth_market_last_sync_date.setter def eth_market_last_sync_date(self, value): self.set_last_date(self._eth_key_market_last_sync_date, value) # private ---------------------------------------------------------------------------------------------------------- @staticmethod def get_last_date(key): ts = NRLocalMemCache.get(key) if ts is None: ts = DbStates.get(key) if ts is None: ts = 0 return ts @staticmethod def set_last_date(key, value): ts = DateTime(timestamp=value, timezone_hours=0).date.timestamp DbStates.set(key, ts) NRLocalMemCache.set(key, ts) @staticmethod def get_current_date(begin_date, last_date): if last_date == 0: return begin_date else: return DateTime(timestamp=last_date, timezone_hours=0).add_days(1).timestamp
def get_current_date(begin_date, last_date): if last_date == 0: return begin_date else: return DateTime(timestamp=last_date, timezone_hours=0).add_days(1).timestamp
def set_last_date(key, value): ts = DateTime(timestamp=value, timezone_hours=0).date.timestamp DbStates.set(key, ts) NRLocalMemCache.set(key, ts)
def url_with_date(self, date: str) -> str: end = DateTime(timezone_hours=0).date.to_str('%Y%m%d') return 'https://coinmarketcap.com/zh/currencies/nebulas-token/historical-data/?start=%s&end=%s' % (date, end)