def import_device_slices(input): for i, row in enumerate(input): data = json.loads(row) data['time'] = datetime.datetime.utcfromtimestamp(data['time']) data['stat'] = data.pop('_stat') # Do a basic integrity check floored = DeviceSlice.floor_time(data['time'], data['span']) if data['time'] != floored: current_app.logger.warn("{} != {}".format(data['time'], floored)) data['time'] = floored db.session.add(DeviceSlice(**data)) # Print periodic progress if i % 100 == 0: db.session.commit() print("{} inserted!".format(i))
def import_shares(input): for i, line in enumerate(input): data = json.loads(line) data['time'] = datetime.datetime.utcfromtimestamp(data['time']) slc = ShareSlice(algo="scrypt", **data) floored = DeviceSlice.floor_time(data['time'], data['span']) if data['time'] != floored: current_app.logger.warn("{} != {}".format(data['time'], floored)) data['time'] = floored db.session.add(slc) if i % 100 == 0: print "{} completed".format(i) db.session.commit()
def _grab_data(prefix, stat): proc_name = "processing_{}".format(stat) unproc_mins = redis_conn.keys(prefix) for key in unproc_mins: current_app.logger.info("Processing key {}".format(key)) try: (stamp, ) = key.split("_")[1:] except Exception: current_app.logger.error("Error processing key {}".format(key), exc_info=True) continue minute = datetime.datetime.utcfromtimestamp(float(stamp)) # To ensure invalid stampt don't get committed minute = ShareSlice.floor_time(minute, 0) if stamp < (time.time() - 30): current_app.logger.info("Skipping timestamp {}, too young".format(minute)) continue redis_conn.rename(key, proc_name) for user, value in redis_conn.hgetall(proc_name).iteritems(): try: address, worker, did = user.split("_") try: value = float(value) except ValueError: if value != "None": current_app.logger.warn( "Got bogus value {} from ppagent for stat {}" .format(value, stat), exc_info=True) continue # Megahashes are was cgminer reports if stat == "hashrate": value *= 1000000 except Exception: current_app.logger.error("Error processing key {} on hash {}" .format(user, key), exc_info=True) continue try: slc = DeviceSlice(user=address, time=minute, worker=worker, device=did, stat=stat, value=value, span=0) db.session.add(slc) db.session.commit() except sqlalchemy.exc.IntegrityError: current_app.logger.warn("SQLAlchemy collision", exc_info=True) db.session.rollback() redis_conn.delete(proc_name)
def compress_five_minute(): ShareSlice.compress(1) DeviceSlice.compress(1) db.session.commit()
def compress_minute(): ShareSlice.compress(0) DeviceSlice.compress(0) db.session.commit()