def import_device_slices(input):
for i, row in enumerate(input):
data = json.loads(row)
data['time'] = datetime.datetime.utcfromtimestamp(data['time'])
data['stat'] = data.pop('_stat')
# Do a basic integrity check
floored = DeviceSlice.floor_time(data['time'], data['span'])
if data['time'] != floored:
current_app.logger.warn("{} != {}".format(data['time'], floored))
data['time'] = floored
db.session.add(DeviceSlice(**data))
# Print periodic progress
if i % 100 == 0:
db.session.commit()
print("{} inserted!".format(i))
def import_shares(input):
for i, line in enumerate(input):
data = json.loads(line)
data['time'] = datetime.datetime.utcfromtimestamp(data['time'])
slc = ShareSlice(algo="scrypt", **data)
floored = DeviceSlice.floor_time(data['time'], data['span'])
if data['time'] != floored:
current_app.logger.warn("{} != {}".format(data['time'], floored))
data['time'] = floored
db.session.add(slc)
if i % 100 == 0:
print "{} completed".format(i)
db.session.commit()
def import_shares(input):
for i, line in enumerate(input):
data = json.loads(line)
data['time'] = datetime.datetime.utcfromtimestamp(data['time'])
slc = ShareSlice(algo="scrypt", **data)
floored = DeviceSlice.floor_time(data['time'], data['span'])
if data['time'] != floored:
current_app.logger.warn("{} != {}".format(data['time'], floored))
data['time'] = floored
db.session.add(slc)
if i % 100 == 0:
print "{} completed".format(i)
db.session.commit()
def import_device_slices(input):
for i, row in enumerate(input):
data = json.loads(row)
data['time'] = datetime.datetime.utcfromtimestamp(data['time'])
data['stat'] = data.pop('_stat')
# Do a basic integrity check
floored = DeviceSlice.floor_time(data['time'], data['span'])
if data['time'] != floored:
current_app.logger.warn("{} != {}".format(data['time'], floored))
data['time'] = floored
db.session.add(DeviceSlice(**data))
# Print periodic progress
if i % 100 == 0:
db.session.commit()
print("{} inserted!".format(i))
def _grab_data(prefix, stat):
proc_name = "processing_{}".format(stat)
unproc_mins = redis_conn.keys(prefix)
for key in unproc_mins:
current_app.logger.info("Processing key {}".format(key))
try:
(stamp, ) = key.split("_")[1:]
except Exception:
current_app.logger.error("Error processing key {}".format(key),
exc_info=True)
continue
minute = datetime.datetime.utcfromtimestamp(float(stamp))
# To ensure invalid stampt don't get committed
minute = ShareSlice.floor_time(minute, 0)
if stamp < (time.time() - 30):
current_app.logger.info("Skipping timestamp {}, too young".format(minute))
continue
redis_conn.rename(key, proc_name)
for user, value in redis_conn.hgetall(proc_name).iteritems():
try:
address, worker, did = user.split("_")
try:
value = float(value)
except ValueError:
if value != "None":
current_app.logger.warn(
"Got bogus value {} from ppagent for stat {}"
.format(value, stat), exc_info=True)
continue
# Megahashes are was cgminer reports
if stat == "hashrate":
value *= 1000000
except Exception:
current_app.logger.error("Error processing key {} on hash {}"
.format(user, key), exc_info=True)
continue
try:
slc = DeviceSlice(user=address, time=minute, worker=worker,
device=did, stat=stat, value=value, span=0)
db.session.add(slc)
db.session.commit()
except sqlalchemy.exc.IntegrityError:
current_app.logger.warn("SQLAlchemy collision", exc_info=True)
db.session.rollback()
redis_conn.delete(proc_name)
def compress_five_minute():
ShareSlice.compress(1)
DeviceSlice.compress(1)
db.session.commit()
def compress_minute():
ShareSlice.compress(0)
DeviceSlice.compress(0)
db.session.commit()
def compress_five_minute():
ShareSlice.compress(1)
DeviceSlice.compress(1)
db.session.commit()
def compress_minute():
ShareSlice.compress(0)
DeviceSlice.compress(0)
db.session.commit()
