def run(self):
util.setup_logging('hp_create')
self.sakuya_db = sessionmaker(bind=engine_sakuya_db)()
data = []
for i in haopan.PRODS:
for j in haopan.CHARTS:
for k in haopan.CITIES:
for l in haopan.CHANNELS:
for m in haopan.ENTRIES:
data.append((0, i[0], j[0], k[0], l[0], m[0]))
chart_keys = haopan.fill_data(data).keys()
charts = None
try:
with file(CHARTFILE) as f:
charts = msgpack.unpackb(f.read())
except Exception:
pass
if not isinstance(charts, dict):
charts = {}
stats = [0, 0]
for key in chart_keys:
if key not in charts:
id = self.new_chart(key)
if id is not None:
charts[key] = id
stats[0] += 1
else:
stats[1] += 1
else:
logging.info('skip %s' % str(key))
if stats[0]:
if os.path.isfile(CHARTFILE):
shutil.copyfile(
CHARTFILE, CHARTFILE + '.' + time.strftime('%Y%m%d%H%M%S'))
with file(CHARTFILE, 'w') as f:
f.write(msgpack.packb(charts))
logging.info('%d charts created, %d failed.' % tuple(stats))
def run(self):
util.setup_logging('hp_create')
self.sakuya_db = sessionmaker(bind=engine_sakuya_db)()
data = []
for i in haopan.PRODS:
for j in haopan.CHARTS:
for k in haopan.CITIES:
for l in haopan.CHANNELS:
for m in haopan.ENTRIES:
data.append((0, i[0], j[0], k[0], l[0], m[0]))
chart_keys = haopan.fill_data(data).keys()
charts = None
try:
with file(CHARTFILE) as f:
charts = msgpack.unpackb(f.read())
except Exception:
pass
if not isinstance(charts, dict):
charts = {}
stats = [0, 0]
for key in chart_keys:
if key not in charts:
id = self.new_chart(key)
if id is not None:
charts[key] = id
stats[0] += 1
else:
stats[1] += 1
else:
logging.info('skip %s' % str(key))
if stats[0]:
if os.path.isfile(CHARTFILE):
shutil.copyfile(CHARTFILE, CHARTFILE + '.' + time.strftime('%Y%m%d%H%M%S'))
with file(CHARTFILE, 'w') as f:
f.write(msgpack.packb(charts))
logging.info('%d charts created, %d failed.' % tuple(stats))
def stats(self):
if not self.data:
return
try:
rows = haopan.fill_data(self.data)
logging.info('stats time: %s' %
self.next_dt.strftime('%Y-%m-%d %H:%M'))
d_charts = haopan.get_charts(True)
stats = [0, 0]
for k, v in rows.iteritems():
id = d_charts.get(k)
if id is None:
logging.info('key %s not found.' % str(k))
stats[1] += 1
continue
row = ChartdataTmpl.gen(self.next_dt.strftime('%Y%m%d'))()
row.ds_id = id
row.time = self.next_dt
row.data = v
self.sakuya_db.add(row)
stats[0] += 1
logging.info('key: %s, id: %d, data: %d' % (str(k), id, v))
self.sakuya_db.commit()
logging.info('%d rows inserted, %d failed.' % tuple(stats))
except Exception:
self.sakuya_db.rollback()
logging.exception('Fail to stats')
self.stats_extra()
def stats(self):
if not self.data:
return
try:
rows = haopan.fill_data(self.data)
logging.info('stats time: %s' % self.next_dt.strftime('%Y-%m-%d %H:%M'))
d_charts = haopan.get_charts(True)
stats = [0, 0]
for k, v in rows.iteritems():
id = d_charts.get(k)
if id is None:
logging.info('key %s not found.' % str(k))
stats[1] += 1
continue
row = ChartdataTmpl.gen(self.next_dt.strftime('%Y%m%d'))()
row.ds_id = id
row.time = self.next_dt
row.data = v
self.sakuya_db.add(row)
stats[0] += 1
logging.info('key: %s, id: %d, data: %d' % (str(k), id, v))
self.sakuya_db.commit()
logging.info('%d rows inserted, %d failed.' % tuple(stats))
except Exception:
self.sakuya_db.rollback()
logging.exception('Fail to stats')
self.stats_extra()
