def process_block(height: int, blockhash: str): print('processing block: height=%d hash=%s' % (height, blockhash)) height_bin = b'HGHT' + struct.pack('>I', height) blockhash_bin = binascii.unhexlify(blockhash) h = db.get(height_bin) if h is not None and h == blockhash_bin and db.get(b'HASH' + blockhash_bin): # block already seen and processed print('* already seen and processed') return if h is not None and h != blockhash_bin: # seen another block with this height, delete old info print('* reorg') db.delete(b'HASH' + h) else: # this is new block print('* new block') db.put(height_bin, binascii.unhexlify(blockhash)) process_transactions(blockhash)
def create_quote_form(): ip = request.remote_addr type = request.headers['Content-Type'] tags = [] tags_valid = True body_valid = True content = request.form['quote'] tags_raw = request.form['tags'] # a smidgen of validation if len(tags_raw) > 100: tags_valid = False else: tags = map(string.strip, tags_raw.split(',')) body_valid, tags_valid = validate_quote(content, tags) quote = None if body_valid and tags_valid: quote = Quote(content, ip) quote.tags = map(Tag, tags) quote = db.put(quote) # grabbing return val isn't strictly necessary if request.wants_json(): return create_quote_resp_json(quote, body_valid, tags_valid) else: return create_quote_resp_html(quote, body_valid, tags_valid)
def create_quote_form(): ip = request.remote_addr type = request.headers['Content-Type'] tags = [] tags_valid = True body_valid = True content = request.form['quote'] # strip trailing whitespace at each line content_lines = content.splitlines() for i, l in enumerate(content_lines): content_lines[i] = l.rstrip() content = '\n'.join(content_lines) logger.info(content) tags_raw = request.form['tags'] # a smidgen of validation if len(tags_raw) > 100: tags_valid = False else: tags = map(string.strip, tags_raw.split(',')) body_valid, tags_valid = validate_quote(content, tags) quote = None if body_valid and tags_valid: quote = Quote(content, ip) quote.tags = map(Tag, tags) quote = db.put(quote) # grabbing return val isn't strictly necessary if request.wants_json(): return create_quote_resp_json(quote, body_valid, tags_valid) else: return create_quote_resp_html(quote, body_valid, tags_valid)
def create_quote_form(): ip = request.remote_addr type = request.headers['Content-Type'] tags = [] tags_valid = True body_valid = True content = request.form['quote'] # strip trailing whitespace at each line content_lines = content.splitlines() for i,l in enumerate(content_lines): content_lines[i] = l.rstrip() content = '\n'.join(content_lines) logger.info(content) tags_raw = request.form['tags'] # a smidgen of validation if len(tags_raw) > 100: tags_valid = False else: tags = map(string.strip, tags_raw.split(',')) body_valid, tags_valid = validate_quote(content, tags) quote = None if body_valid and tags_valid: quote = Quote(content, ip) quote.tags = map(Tag, tags) quote = db.put(quote) # grabbing return val isn't strictly necessary if request.wants_json(): return create_quote_resp_json(quote, body_valid, tags_valid) else: return create_quote_resp_html(quote, body_valid, tags_valid)
def process_transactions(blockhash: str): block = rpc.getblock(blockhash) strippedsize = block['strippedsize'] size = block['size'] weight = block['weight'] txtotal = len(block['tx']) txsegwit = 0 for i, txid in enumerate(block['tx']): if i > 0 and i % 100 == 0: print(' * tx: %d/%d (%.2f%%)' % (i, txtotal, 100.0 * i / txtotal)) txraw = rpc.getrawtransaction(txid, False) if txraw[8:12] == '0001': # segwit tx has 0 inputs and 1 output txsegwit += 1 k = b'HASH' + binascii.unhexlify(blockhash) v = struct.pack('>IIIII', strippedsize, size, weight, txtotal, txsegwit) db.put(k, v) print('* data: strippedsize=%d size=%d weight=%d txtotal=%d txsegwit=%d' % (strippedsize, size, weight, txtotal, txsegwit))
def create_quote_json(): ip = request.remote_addr data = request.json body_valid, tags_valid = validate_quote(data['body'], data['tags']) if body_valid and tags_valid: quote = Quote(data['body'], ip) quote.tags = map(Tag, data['tags']) quote = db.put(quote) # grabbing return val isn't strictly necessary if request.wants_json(): return create_quote_resp_json(quote, body_valid, tags_valid) else: return create_quote_resp_html(quote, body_valid, tags_valid)
def process(obj, id, getter, table, name, nopreserve=None, nodiff=False, nostore=False, onchanged=None): if nopreserve is None: nopreserve = [] # clear out empty values obj = {k: v for k, v in obj.items() if v or v == False} if nodiff: now = datetime.utcnow().replace(microsecond=0) if not 'meta' in obj: obj['meta'] = {} log(3, 'adding %s (%s)' % (name, id)) obj['meta']['created'] = now obj['changes'] = {} if not nostore and not db.put(table, obj): log(1, "failed to store updated obj {}".format(id)) raise ValueError if onchanged is not None: onchanged(obj, d) return # generate diff prev = getter(id) if prev is not None: d = diff( { k: v for k, v in prev.items() if not k in ['meta', 'changes', '_id'] }, { k: v for k, v in obj.items() if not k in ['meta', 'changes', '_id'] }) # preserve some top level items d1 = [] for c in d: if c['type'] != 'deleted' or len( c['path']) != 1 or c['path'][0] in nopreserve: d1.append(c) continue if c['type'] == 'deleted' and len( c['path']) == 1 and c['data'] in ({}, []): d1.append(c) continue log( 3, "preserving deleted path {} for obj id: {}".format( c['path'], id)) obj[c['path'][0]] = prev[c['path'][0]] d = d1 else: d = diff({}, { k: v for k, v in obj.items() if not k in ['meta', 'changes', '_id'] }) if d: # attempt to recreate current version by applying d to prev o2 = patch(prev or {}, json.loads(jdump(d))) if not o2: log( 1, "failed to recreate {} record by patching previous version with diff" .format(id)) raise ValueError else: # make a diff between current record, an recreated one zero = diff( { k: v for k, v in o2.items() if not k in ['meta', 'changes', '_id'] }, { k: v for k, v in obj.items() if not k in ['meta', 'changes', '_id'] }) if zero != []: log( 1, "id:{} diff between current record and patched previous one is not empty\n{!r}" .format(id, zero)) raise ValueError( "diff between new and patched old is not empty") now = datetime.utcnow().replace(microsecond=0) if not 'meta' in obj: obj['meta'] = {} if not prev or nodiff: log(3, 'adding %s (%s)' % (name, id)) obj['meta']['created'] = now obj['changes'] = {} else: log(3, 'updating %s (%s)' % (name, id)) log(4, "changes for %s\n%s" % (id, jdump(d))) obj['meta']['updated'] = now obj['changes'] = prev.get('changes', {}) obj['changes'][now.isoformat()] = d if not nostore and not db.put(table, obj): log(1, "failed to store updated obj {}".format(id)) raise ValueError if onchanged is not None: onchanged(obj, d) del prev if __name__ == '__main__': print(jdump(obj)) return obj