def _sync(self) -> bool:
try:
str_date = DateTime(self.get_last_sync_date(),
timezone_hours=0).to_str("%Y%m%d")
market_data = self._get_market_data(str_date)
if market_data is not None:
with DBLock:
try:
# save to models
self.save_market_data(market_data)
# update sync date
self.set_last_sync_date(
DateTime(timezone_hours=0).date.timestamp)
self._logger.log(
"%s market data sync success. date: %s" %
(self.__class__.__name__,
DateTime(self.get_last_sync_date(),
timezone_hours=0).to_str("%Y%m%d")))
commit()
except Exception as e:
rollback()
raise e
return True
else:
return False
except Exception as e:
self._logger.log_err(e)
return False
def _start_timer(self):
if self._timer is not None:
self._timer.cancel()
# 今天的数据，需要明天零点半后开始同步（utc+0）
t = DateTime(timezone_hours=0).add_days(1).date.timestamp - DateTime().timestamp + 60 * 30
self._timer = Timer(t, self.start)
self._timer.daemon = True
self._timer.start()
def start(self):
with self._lock:
now = DateTime(timezone_hours=0).date.timestamp
dt = self._begin_date
if self._last_date != 0:
dt = self._last_date
while dt < now:
self._dates.append(dt)
dt = DateTime(dt, timezone_hours=0).date.add_days(1).timestamp
self._last_date = dt
self._check_and_req()
def _add_to_address(self, db_table, addresses: dict, nr: NrItem) -> str:
a = nr.address
item = self.new_pb_item(nr)
now = DateTime().timestamp
c = self.get_current_sync_date()
if a in addresses.keys():
count = addresses[a].count + 1
total = addresses[a].total_nr
if total is None:
total = 0
else:
total = float(total)
total += float(item.score)
i_bytes = b'|' + item.SerializeToString()
i_hex = ''.join(['%02x' % b for b in i_bytes])
dates = self.get_valid_dates(addresses[a].dates)
if float(item.score) >= 0.1 and now - c <= self._SEVEN_DAY_SECONDS:
dates.append(str(c))
n = len(dates)
str_dts = ','.join(dates)
return 'UPDATE %s SET last_above_0_dates=\'%s\', last_above_0_num=\'%s\', `count`=\'%s\', total_nr=\'%s\', `data`=concat(`data`, X\'%s\') WHERE `address`=\'%s\';' % \
(db_table, str_dts, str(n), str(count), str(total), i_hex, a)
else:
count = 1
total = float(item.score)
i_bytes = item.SerializeToString()
i_hex = ''.join(['%02x' % b for b in i_bytes])
dates = []
if float(item.score) >= 0.1 and now - c <= self._SEVEN_DAY_SECONDS:
dates.append(str(c))
n = len(dates)
str_dts = ','.join(dates)
return 'INSERT INTO %s (`address`, `last_above_0_dates`, `last_above_0_num`, `count`, total_nr, `data`) VALUES (\'%s\', \'%s\', \'%s\', \'%s\', \'%s\', X\'%s\');' % \
(db_table, a, str_dts, str(n), str(count), str(total), i_hex)
def _pass_market_data(self, content: str) -> list:
if content is not None:
tb = self._get_table(content)
if tb is not None:
items = []
trs = tb.find('tbody').findAll('tr')
keys = [
'date', 'opening', 'highest', 'lowest', 'closing',
'amount', 'total_circulation', 'total'
]
for tr in trs:
item = {}
tds = tr.findAll('td')
for i in range(len(tds)):
key = keys[i]
text: str = tds[i].text
if key == 'date':
item[key] = DateTime.from_str(
text, '%Y年%m月%d日',
timezone_hours=0).date.timestamp
elif key == 'amount' or key == 'total_circulation':
item[key] = text.replace(',', '')
else:
item[key] = float(text)
item['total'] = format(
self.currency_count() * item['closing'], '.0f')
items.append(item)
return items
return None
def _begin(self):
str_date = DateTime(self._date, timezone_hours=0).to_str('%Y%m%d')
ok, data = _Request(str_date, self._is_neb).get_result()
if ok and len(data) > 0:
self.data = data
self.did_load_data(self._date)
else:
time.sleep(10)
self.start()
def get_valid_dates(self, dates: str):
r = []
if dates is None or len(dates) == 0:
return r
now = DateTime().timestamp
dts = dates.split(',')
for d in dts:
if now - int(d) <= self._SEVEN_DAY_SECONDS:
r.append(d)
return r
def _sync(self) -> bool:
try:
# str_date = DateTime(self.get_current_sync_date(), timezone_hours=0).to_str("%Y%m%d")
dt = self.get_current_sync_date()
date_table, _, _ = self.db_context()
if self._exists(date_table, dt):
self.set_last_sync_date(dt)
self._logger.log_err('duplicate date: %s' % DateTime(dt, timezone_hours=0).to_str('%Y-%m-%d'))
return True
nr_data = self._req_queue.get(dt)
# nr_data = self._get_daily_all_nr(str_date)
if nr_data is not None:
with DBLock:
try:
# save to models
self._sort_nr_data(nr_data)
self._save_nr_data(nr_data)
# update sync date
self.set_last_sync_date(dt)
self._logger.log(
"%s nr_data sync success. date: %s" % (
self.__class__.__name__,
DateTime(self.get_last_sync_date(), timezone_hours=0).to_str("%Y%m%d")
)
)
commit()
self._req_queue.remove(dt)
except Exception as e:
rollback()
raise e
return True
else:
return False
except Exception as e:
self._logger.log_err(e)
return False
def _check_can_sync(self) -> bool:
return self.get_current_sync_date() < DateTime(timezone_hours=0).date.timestamp
def log_rewrite(f, log):
with open(os.path.join(root_dir, f), 'w') as h:
h.write('[' + DateTime().to_str() + '] ' + str(log) + '\n')
h.flush()
def log_err(self, log):
print('task error log:', log)
self._e.write('[' + DateTime().to_str() + '] ' + str(log) + '\n')
self._e.flush()
def log(self, log):
print('task log:', log)
self._f.write('[' + DateTime().to_str() + '] ' + str(log) + '\n')
self._f.flush()
class NRState(object):
"""
需要持久化的状态类型数据管理类(Cache & Db)
"""
_neb_key_last_sync_date = "neb_last_sync_date"
_neb_key_market_last_sync_date = "neb_market_last_sync_date"
_neb_begin_date = DateTime.from_str("20180508", "%Y%m%d",
timezone_hours=0).timestamp
# _neb_begin_date = DateTime.from_str("20181127", "%Y%m%d", timezone_hours=0).timestamp
_eth_key_last_sync_date = "eth_last_sync_date"
_eth_key_market_last_sync_date = "eth_market_last_sync_date"
_eth_begin_date = DateTime.from_str("20170101", "%Y%m%d",
timezone_hours=0).timestamp
# neb nr date ------------------------------------------------------------------------------------------------------
@property
def neb_last_sync_date(self):
return self.get_last_date(self._neb_key_last_sync_date)
@neb_last_sync_date.setter
def neb_last_sync_date(self, value):
self.set_last_date(self._neb_key_last_sync_date, value)
@property
def neb_current_sync_date(self):
return self.get_current_date(self._neb_begin_date,
self.neb_last_sync_date)
# neb market date --------------------------------------------------------------------------------------------------
@property
def neb_market_last_sync_date(self):
d = self.get_last_date(self._neb_key_market_last_sync_date)
if d == 0:
d = self._neb_begin_date
return d
@neb_market_last_sync_date.setter
def neb_market_last_sync_date(self, value):
self.set_last_date(self._neb_key_market_last_sync_date, value)
# eth nr date ------------------------------------------------------------------------------------------------------
@property
def eth_last_sync_date(self):
return self.get_last_date(self._eth_key_last_sync_date)
@eth_last_sync_date.setter
def eth_last_sync_date(self, value):
self.set_last_date(self._eth_key_last_sync_date, value)
@property
def eth_current_sync_date(self):
return self.get_current_date(self._eth_begin_date,
self.eth_last_sync_date)
# eth market date --------------------------------------------------------------------------------------------------
@property
def eth_market_last_sync_date(self):
d = self.get_last_date(self._eth_key_market_last_sync_date)
if d == 0:
d = self._eth_begin_date
return d
@eth_market_last_sync_date.setter
def eth_market_last_sync_date(self, value):
self.set_last_date(self._eth_key_market_last_sync_date, value)
# private ----------------------------------------------------------------------------------------------------------
@staticmethod
def get_last_date(key):
ts = NRLocalMemCache.get(key)
if ts is None:
ts = DbStates.get(key)
if ts is None:
ts = 0
return ts
@staticmethod
def set_last_date(key, value):
ts = DateTime(timestamp=value, timezone_hours=0).date.timestamp
DbStates.set(key, ts)
NRLocalMemCache.set(key, ts)
@staticmethod
def get_current_date(begin_date, last_date):
if last_date == 0:
return begin_date
else:
return DateTime(timestamp=last_date,
timezone_hours=0).add_days(1).timestamp
def get_current_date(begin_date, last_date):
if last_date == 0:
return begin_date
else:
return DateTime(timestamp=last_date,
timezone_hours=0).add_days(1).timestamp
def set_last_date(key, value):
ts = DateTime(timestamp=value, timezone_hours=0).date.timestamp
DbStates.set(key, ts)
NRLocalMemCache.set(key, ts)
def url_with_date(self, date: str) -> str:
end = DateTime(timezone_hours=0).date.to_str('%Y%m%d')
return 'https://coinmarketcap.com/zh/currencies/nebulas-token/historical-data/?start=%s&end=%s' % (date, end)