def collect_minutes():
""" Grabs all the pending minute shares out of redis and puts them in the
database """
unproc_mins = redis_conn.keys("min_*")
for key in unproc_mins:
current_app.logger.info("Processing key {}".format(key))
share_type, algo, stamp = key.split("_")[1:]
minute = datetime.datetime.utcfromtimestamp(float(stamp))
# To ensure invalid stampt don't get committed
minute = ShareSlice.floor_time(minute, 0)
if stamp < (time.time() - 30):
current_app.logger.info("Skipping timestamp {}, too young"
.format(minute))
continue
redis_conn.rename(key, "processing_shares")
for user, shares in redis_conn.hgetall("processing_shares").iteritems():
shares = float(shares)
# messily parse out the worker/address combo...
parts = user.split(".")
if len(parts) > 0:
worker = parts[1]
else:
worker = ''
address = parts[0]
if not address.startswith("pool"):
try:
curr = currencies.lookup_payable_addr(address)
except InvalidAddressException:
curr = None
if curr is None:
address = global_config.pool_payout_currency.pool_payout_addr
try:
slc = ShareSlice(user=address, time=minute, worker=worker, algo=algo,
share_type=share_type, value=shares, span=0)
db.session.add(slc)
db.session.commit()
except sqlalchemy.exc.IntegrityError:
db.session.rollback()
slc = ShareSlice.query.with_lockmode('update').filter_by(
user=address, time=minute, worker=worker, algo=algo,
share_type=share_type).one()
slc.value += shares
db.session.commit()
redis_conn.delete("processing_shares")
def collect_minutes():
""" Grabs all the pending minute shares out of redis and puts them in the
database """
unproc_mins = redis_conn.keys("min_*")
for key in unproc_mins:
current_app.logger.info("Processing key {}".format(key))
share_type, algo, stamp = key.split("_")[1:]
minute = datetime.datetime.utcfromtimestamp(float(stamp))
# To ensure invalid stampt don't get committed
minute = ShareSlice.floor_time(minute, 0)
if stamp < (time.time() - 30):
current_app.logger.info("Skipping timestamp {}, too young"
.format(minute))
continue
redis_conn.rename(key, "processing_shares")
for user, shares in redis_conn.hgetall("processing_shares").iteritems():
shares = float(shares)
# messily parse out the worker/address combo...
parts = user.split(".")
if len(parts) > 0:
worker = parts[1]
else:
worker = ''
address = parts[0]
if address != "pool":
try:
curr = currencies.lookup_payable_addr(address)
except InvalidAddressException:
curr = None
if not curr:
address = global_config.pool_payout_currency.pool_payout_addr
try:
slc = ShareSlice(user=address, time=minute, worker=worker, algo=algo,
share_type=share_type, value=shares, span=0)
db.session.add(slc)
db.session.commit()
except sqlalchemy.exc.IntegrityError:
db.session.rollback()
slc = ShareSlice.query.with_lockmode('update').filter_by(
user=address, time=minute, worker=worker, algo=algo,
share_type=share_type).one()
slc.value += shares
db.session.commit()
redis_conn.delete("processing_shares")
def _grab_data(prefix, stat):
proc_name = "processing_{}".format(stat)
unproc_mins = redis_conn.keys(prefix)
for key in unproc_mins:
current_app.logger.info("Processing key {}".format(key))
try:
(stamp, ) = key.split("_")[1:]
except Exception:
current_app.logger.error("Error processing key {}".format(key),
exc_info=True)
continue
minute = datetime.datetime.utcfromtimestamp(float(stamp))
# To ensure invalid stampt don't get committed
minute = ShareSlice.floor_time(minute, 0)
if stamp < (time.time() - 30):
current_app.logger.info("Skipping timestamp {}, too young".format(minute))
continue
redis_conn.rename(key, proc_name)
for user, value in redis_conn.hgetall(proc_name).iteritems():
try:
address, worker, did = user.split("_")
try:
value = float(value)
except ValueError:
if value != "None":
current_app.logger.warn(
"Got bogus value {} from ppagent for stat {}"
.format(value, stat), exc_info=True)
continue
# Megahashes are was cgminer reports
if stat == "hashrate":
value *= 1000000
except Exception:
current_app.logger.error("Error processing key {} on hash {}"
.format(user, key), exc_info=True)
continue
try:
slc = DeviceSlice(user=address, time=minute, worker=worker,
device=did, stat=stat, value=value, span=0)
db.session.add(slc)
db.session.commit()
except sqlalchemy.exc.IntegrityError:
current_app.logger.warn("SQLAlchemy collision", exc_info=True)
db.session.rollback()
redis_conn.delete(proc_name)
def _grab_data(prefix, stat):
proc_name = "processing_{}".format(stat)
unproc_mins = redis_conn.keys(prefix)
for key in unproc_mins:
current_app.logger.info("Processing key {}".format(key))
try:
(stamp, ) = key.split("_")[1:]
except Exception:
current_app.logger.error("Error processing key {}".format(key),
exc_info=True)
continue
minute = datetime.datetime.utcfromtimestamp(float(stamp))
# To ensure invalid stampt don't get committed
minute = ShareSlice.floor_time(minute, 0)
if stamp < (time.time() - 30):
current_app.logger.info("Skipping timestamp {}, too young".format(minute))
continue
redis_conn.rename(key, proc_name)
for user, value in redis_conn.hgetall(proc_name).iteritems():
try:
address, worker, did = user.split("_")
try:
value = float(value)
except ValueError:
if value != "None":
current_app.logger.warn(
"Got bogus value {} from ppagent for stat {}"
.format(value, stat), exc_info=True)
continue
# Megahashes are was cgminer reports
if stat == "hashrate":
value *= 1000000
except Exception:
current_app.logger.error("Error processing key {} on hash {}"
.format(user, key), exc_info=True)
continue
try:
slc = DeviceSlice(user=address, time=minute, worker=worker,
device=did, stat=stat, value=value, span=0)
db.session.add(slc)
db.session.commit()
except sqlalchemy.exc.IntegrityError:
current_app.logger.warn("SQLAlchemy collision", exc_info=True)
db.session.rollback()
redis_conn.delete(proc_name)
def compress_slices():
for chain in chains.itervalues():
# Get the index of the last inserted share slice on this chain
last_complete_slice = redis_conn.get("chain_{}_slice_index".format(chain.id))
if last_complete_slice is None:
# Chain must not be in use....
current_app.logger.debug(
"No slice index for chain {}".format(chain))
continue
else:
last_complete_slice = int(last_complete_slice)
# Loop thorugh all possible share slice numbers
empty = 0
encoding_time = 0.0
retrieval_time = 0.0
entry_count = 0
encoded_size = 0
original_size = 0
last_slice = last_complete_slice
for slc_idx in xrange(last_complete_slice, 0, -1):
key = "chain_{}_slice_{}".format(chain.id, slc_idx)
key_type = redis_conn.type(key)
# Compress if it's a list. This is raw data from powerpools redis
# reporter
if key_type == "list":
# Reduce empty counter, but don't go negative
empty = max(0, empty - 1)
# Retrieve the enencoded information from redis
t = time.time()
slice_shares = redis_conn.lrange(key, 0, -1)
this_original_size = int(redis_conn.debug_object(key)['serializedlength'])
this_retrieval_time = time.time() - t
# Parse the list into proper python representation
data = []
total_shares = 0
for entry in slice_shares:
user, shares = entry.split(":")
shares = Decimal(shares)
data.append((user, shares))
total_shares += shares
this_entry_count = len(data)
# serialization and compression
t = time.time()
data = json.dumps(data, separators=(',', ':'), use_decimal=True)
data = bz2.compress(data)
this_encoding_time = time.time() - t
# Put all the new data into a temporary key, then atomically
# replace the old list key. ensures we never loose data, even
# on failures (exceptions)
key_compressed = key + "_compressed"
redis_conn.hmset(key_compressed,
dict(
date=int(time.time()),
data=data,
encoding="bz2json",
total_shares=total_shares)
)
redis_conn.rename(key_compressed, key)
this_encoded_size = int(redis_conn.debug_object(key)['serializedlength'])
last_slice = slc_idx
# Update all the aggregates
encoding_time += this_encoding_time
retrieval_time += this_retrieval_time
entry_count += this_entry_count
encoded_size += this_encoded_size
original_size += this_original_size
# Print progress
current_app.logger.info(
"Encoded slice #{:,} containing {:,} entries."
" retrieval_time: {}; encoding_time: {}; start_size: {:,}; end_size: {:,}; ratio: {}"
.format(slc_idx, this_entry_count,
time_format(this_retrieval_time),
time_format(this_encoding_time),
this_original_size, this_encoded_size,
float(this_original_size) / (this_encoded_size or 1)))
# Count an empty entry to detect the end of live slices
elif key_type == "none":
empty += 1
# If we've seen a lot of empty slices, probably nothing else to find!
if empty >= 20:
current_app.logger.info(
"Ended compression search at {}".format(slc_idx))
break
current_app.logger.info(
"Encoded from slice #{:,} -> #{:,} containing {:,} entries."
" retrieval_time: {}; encoding_time: {}; start_size: {:,}; end_size: {:,}; ratio: {}"
.format(last_complete_slice, last_slice, entry_count,
time_format(retrieval_time), time_format(encoding_time),
original_size, encoded_size,
float(original_size) / (encoded_size or 1)))
def compress_slices():
for chain in chains.itervalues():
# Get the index of the last inserted share slice on this chain
last_complete_slice = redis_conn.get("chain_{}_slice_index".format(chain.id))
if last_complete_slice is None:
# Chain must not be in use....
current_app.logger.debug(
"No slice index for chain {}".format(chain))
continue
else:
last_complete_slice = int(last_complete_slice)
# Loop thorugh all possible share slice numbers
empty = 0
encoding_time = 0.0
retrieval_time = 0.0
entry_count = 0
encoded_size = 0
original_size = 0
last_slice = last_complete_slice
for slc_idx in xrange(last_complete_slice, 0, -1):
key = "chain_{}_slice_{}".format(chain.id, slc_idx)
key_type = redis_conn.type(key)
# Compress if it's a list. This is raw data from powerpools redis
# reporter
if key_type == "list":
# Reduce empty counter, but don't go negative
empty = max(0, empty - 1)
# Retrieve the enencoded information from redis
t = time.time()
slice_shares = redis_conn.lrange(key, 0, -1)
this_original_size = int(redis_conn.debug_object(key)['serializedlength'])
this_retrieval_time = time.time() - t
# Parse the list into proper python representation
data = []
total_shares = 0
for entry in slice_shares:
user, shares = entry.split(":")
shares = Decimal(shares)
data.append((user, shares))
total_shares += shares
this_entry_count = len(data)
# serialization and compression
t = time.time()
data = json.dumps(data, separators=(',', ':'), use_decimal=True)
data = bz2.compress(data)
this_encoding_time = time.time() - t
# Put all the new data into a temporary key, then atomically
# replace the old list key. ensures we never loose data, even
# on failures (exceptions)
key_compressed = key + "_compressed"
redis_conn.hmset(key_compressed,
dict(
date=int(time.time()),
data=data,
encoding="bz2json",
total_shares=total_shares)
)
redis_conn.rename(key_compressed, key)
this_encoded_size = int(redis_conn.debug_object(key)['serializedlength'])
last_slice = slc_idx
# Update all the aggregates
encoding_time += this_encoding_time
retrieval_time += this_retrieval_time
entry_count += this_entry_count
encoded_size += this_encoded_size
original_size += this_original_size
# Print progress
current_app.logger.info(
"Encoded slice #{:,} containing {:,} entries."
" retrieval_time: {}; encoding_time: {}; start_size: {:,}; end_size: {:,}; ratio: {}"
.format(slc_idx, this_entry_count,
time_format(this_retrieval_time),
time_format(this_encoding_time),
this_original_size, this_encoded_size,
float(this_original_size) / (this_encoded_size or 1)))
# Count an empty entry to detect the end of live slices
elif key_type == "none":
empty += 1
# If we've seen a lot of empty slices, probably nothing else to find!
if empty >= 20:
current_app.logger.info(
"Ended compression search at {}".format(slc_idx))
break
current_app.logger.info(
"Encoded from slice #{:,} -> #{:,} containing {:,} entries."
" retrieval_time: {}; encoding_time: {}; start_size: {:,}; end_size: {:,}; ratio: {}"
.format(last_complete_slice, last_slice, entry_count,
time_format(retrieval_time), time_format(encoding_time),
original_size, encoded_size,
float(original_size) / (encoded_size or 1)))
