def process_block(height: int, blockhash: str):
print('processing block: height=%d hash=%s' % (height, blockhash))
height_bin = b'HGHT' + struct.pack('>I', height)
blockhash_bin = binascii.unhexlify(blockhash)
h = db.get(height_bin)
if h is not None and h == blockhash_bin and db.get(b'HASH' +
blockhash_bin):
# block already seen and processed
print('* already seen and processed')
return
if h is not None and h != blockhash_bin:
# seen another block with this height, delete old info
print('* reorg')
db.delete(b'HASH' + h)
else:
# this is new block
print('* new block')
db.put(height_bin, binascii.unhexlify(blockhash))
process_transactions(blockhash)
def create_quote_form():
ip = request.remote_addr
type = request.headers['Content-Type']
tags = []
tags_valid = True
body_valid = True
content = request.form['quote']
tags_raw = request.form['tags']
# a smidgen of validation
if len(tags_raw) > 100:
tags_valid = False
else:
tags = map(string.strip, tags_raw.split(','))
body_valid, tags_valid = validate_quote(content, tags)
quote = None
if body_valid and tags_valid:
quote = Quote(content, ip)
quote.tags = map(Tag, tags)
quote = db.put(quote) # grabbing return val isn't strictly necessary
if request.wants_json():
return create_quote_resp_json(quote, body_valid, tags_valid)
else:
return create_quote_resp_html(quote, body_valid, tags_valid)
def create_quote_form():
ip = request.remote_addr
type = request.headers['Content-Type']
tags = []
tags_valid = True
body_valid = True
content = request.form['quote']
# strip trailing whitespace at each line
content_lines = content.splitlines()
for i, l in enumerate(content_lines):
content_lines[i] = l.rstrip()
content = '\n'.join(content_lines)
logger.info(content)
tags_raw = request.form['tags']
# a smidgen of validation
if len(tags_raw) > 100:
tags_valid = False
else:
tags = map(string.strip, tags_raw.split(','))
body_valid, tags_valid = validate_quote(content, tags)
quote = None
if body_valid and tags_valid:
quote = Quote(content, ip)
quote.tags = map(Tag, tags)
quote = db.put(quote) # grabbing return val isn't strictly necessary
if request.wants_json():
return create_quote_resp_json(quote, body_valid, tags_valid)
else:
return create_quote_resp_html(quote, body_valid, tags_valid)
def create_quote_form():
ip = request.remote_addr
type = request.headers['Content-Type']
tags = []
tags_valid = True
body_valid = True
content = request.form['quote']
# strip trailing whitespace at each line
content_lines = content.splitlines()
for i,l in enumerate(content_lines):
content_lines[i] = l.rstrip()
content = '\n'.join(content_lines)
logger.info(content)
tags_raw = request.form['tags']
# a smidgen of validation
if len(tags_raw) > 100:
tags_valid = False
else:
tags = map(string.strip, tags_raw.split(','))
body_valid, tags_valid = validate_quote(content, tags)
quote = None
if body_valid and tags_valid:
quote = Quote(content, ip)
quote.tags = map(Tag, tags)
quote = db.put(quote) # grabbing return val isn't strictly necessary
if request.wants_json():
return create_quote_resp_json(quote, body_valid, tags_valid)
else:
return create_quote_resp_html(quote, body_valid, tags_valid)
def process_transactions(blockhash: str):
block = rpc.getblock(blockhash)
strippedsize = block['strippedsize']
size = block['size']
weight = block['weight']
txtotal = len(block['tx'])
txsegwit = 0
for i, txid in enumerate(block['tx']):
if i > 0 and i % 100 == 0:
print(' * tx: %d/%d (%.2f%%)' % (i, txtotal, 100.0 * i / txtotal))
txraw = rpc.getrawtransaction(txid, False)
if txraw[8:12] == '0001': # segwit tx has 0 inputs and 1 output
txsegwit += 1
k = b'HASH' + binascii.unhexlify(blockhash)
v = struct.pack('>IIIII', strippedsize, size, weight, txtotal, txsegwit)
db.put(k, v)
print('* data: strippedsize=%d size=%d weight=%d txtotal=%d txsegwit=%d' %
(strippedsize, size, weight, txtotal, txsegwit))
def create_quote_json():
ip = request.remote_addr
data = request.json
body_valid, tags_valid = validate_quote(data['body'], data['tags'])
if body_valid and tags_valid:
quote = Quote(data['body'], ip)
quote.tags = map(Tag, data['tags'])
quote = db.put(quote) # grabbing return val isn't strictly necessary
if request.wants_json():
return create_quote_resp_json(quote, body_valid, tags_valid)
else:
return create_quote_resp_html(quote, body_valid, tags_valid)
def create_quote_json():
ip = request.remote_addr
data = request.json
body_valid, tags_valid = validate_quote(data['body'], data['tags'])
if body_valid and tags_valid:
quote = Quote(data['body'], ip)
quote.tags = map(Tag, data['tags'])
quote = db.put(quote) # grabbing return val isn't strictly necessary
if request.wants_json():
return create_quote_resp_json(quote, body_valid, tags_valid)
else:
return create_quote_resp_html(quote, body_valid, tags_valid)
def process(obj,
id,
getter,
table,
name,
nopreserve=None,
nodiff=False,
nostore=False,
onchanged=None):
if nopreserve is None: nopreserve = []
# clear out empty values
obj = {k: v for k, v in obj.items() if v or v == False}
if nodiff:
now = datetime.utcnow().replace(microsecond=0)
if not 'meta' in obj: obj['meta'] = {}
log(3, 'adding %s (%s)' % (name, id))
obj['meta']['created'] = now
obj['changes'] = {}
if not nostore and not db.put(table, obj):
log(1, "failed to store updated obj {}".format(id))
raise ValueError
if onchanged is not None:
onchanged(obj, d)
return
# generate diff
prev = getter(id)
if prev is not None:
d = diff(
{
k: v
for k, v in prev.items()
if not k in ['meta', 'changes', '_id']
}, {
k: v
for k, v in obj.items() if not k in ['meta', 'changes', '_id']
})
# preserve some top level items
d1 = []
for c in d:
if c['type'] != 'deleted' or len(
c['path']) != 1 or c['path'][0] in nopreserve:
d1.append(c)
continue
if c['type'] == 'deleted' and len(
c['path']) == 1 and c['data'] in ({}, []):
d1.append(c)
continue
log(
3, "preserving deleted path {} for obj id: {}".format(
c['path'], id))
obj[c['path'][0]] = prev[c['path'][0]]
d = d1
else:
d = diff({}, {
k: v
for k, v in obj.items() if not k in ['meta', 'changes', '_id']
})
if d:
# attempt to recreate current version by applying d to prev
o2 = patch(prev or {}, json.loads(jdump(d)))
if not o2:
log(
1,
"failed to recreate {} record by patching previous version with diff"
.format(id))
raise ValueError
else:
# make a diff between current record, an recreated one
zero = diff(
{
k: v
for k, v in o2.items()
if not k in ['meta', 'changes', '_id']
}, {
k: v
for k, v in obj.items()
if not k in ['meta', 'changes', '_id']
})
if zero != []:
log(
1,
"id:{} diff between current record and patched previous one is not empty\n{!r}"
.format(id, zero))
raise ValueError(
"diff between new and patched old is not empty")
now = datetime.utcnow().replace(microsecond=0)
if not 'meta' in obj: obj['meta'] = {}
if not prev or nodiff:
log(3, 'adding %s (%s)' % (name, id))
obj['meta']['created'] = now
obj['changes'] = {}
else:
log(3, 'updating %s (%s)' % (name, id))
log(4, "changes for %s\n%s" % (id, jdump(d)))
obj['meta']['updated'] = now
obj['changes'] = prev.get('changes', {})
obj['changes'][now.isoformat()] = d
if not nostore and not db.put(table, obj):
log(1, "failed to store updated obj {}".format(id))
raise ValueError
if onchanged is not None:
onchanged(obj, d)
del prev
if __name__ == '__main__':
print(jdump(obj))
return obj
