def explorer(): #args = read_args() config = getConfig() result = [] ledger_data_dir = get_ledger_dir("", "") read_copy_ledger_data_dir = None try: # RocksDB supports real read-only mode and does not need to have a ledger copy. if config.hashStore['type'].lower() != HS_ROCKSDB: config.db_transactions_config = None # NOTE: such approach works well only for small ledgers. tmp = make_copy_of_ledger(ledger_data_dir) # Let's be paranoid to avoid removing of ledger instead of its copy. ledger_path = Path(ledger_data_dir) ledger_copy_path = Path(tmp) assert ledger_path != ledger_copy_path assert ledger_copy_path not in ledger_path.parents read_copy_ledger_data_dir = tmp ledger_data_dir = read_copy_ledger_data_dir elif config.db_transactions_config is not None: # This allows to avoid debug logs creation on each read_ledger run config.db_transactions_config['db_log_dir'] = '/dev/null' storage = get_storage("domain", ledger_data_dir) finally: if read_copy_ledger_data_dir: shutil.rmtree(read_copy_ledger_data_dir) for seqNo, txn in storage.iterator(start=0, end=100): txn = ledger_txn_serializer.deserialize(txn) serializer = JsonSerializer() x = serializer.serialize(txn, toBytes=False) result.append(x) return result
def migrate_txn_log(db_dir, db_name): def put_into_seq_no_db(txn): # If there is no reqId, then it's genesis txn if get_req_id(txn) is None: return txn_new = copy.deepcopy(txn) operation = get_payload_data(txn_new) operation[TXN_TYPE] = get_type(txn_new) dct = { f.IDENTIFIER.nm: get_from(txn_new), f.REQ_ID.nm: get_req_id(txn_new), OPERATION: operation, } if get_protocol_version(txn_new) is not None: dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new) digest = sha256(serialize_msg_for_signing(dct)).hexdigest().encode() seq_no = get_seq_no(txn_new) ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE]) line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no) dest_seq_no_db_storage.put(digest, line_to_record) return digest new_db_name = db_name + '_new' old_path = os.path.join(db_dir, db_name) new_path = os.path.join(db_dir, new_db_name) new_seqno_db_name = config.seqNoDbName + '_new' try: dest_seq_no_db_storage = initKeyValueStorage(config.reqIdToTxnStorage, db_dir, new_seqno_db_name) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open new seq_no_db storage") return False # open new and old ledgers try: src_storage = KeyValueStorageRocksdbIntKeys(db_dir, db_name, read_only=True) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open old ledger: {}".format( os.path.join(db_dir, db_name))) return False try: dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open new ledger: {}".format( os.path.join(db_dir, new_db_name))) return False # put values from old ledger to the new one try: for key, val in src_storage.iterator(): key = key.decode() val = ledger_txn_serializer.deserialize(val) new_val = transform_to_new_format(txn=val, seq_no=int(key)) digest = put_into_seq_no_db(new_val) # add digest into txn if get_req_id(new_val): new_val[TXN_PAYLOAD][TXN_PAYLOAD_METADATA][ TXN_PAYLOAD_METADATA_DIGEST] = digest new_val = ledger_txn_serializer.serialize(new_val) dest_storage.put(key, new_val) except Exception: logger.error(traceback.print_exc()) logger.error( "Could not put key/value to the new ledger '{}'".format(db_name)) return False src_storage.close() dest_storage.close() dest_seq_no_db_storage.close() # Remove old ledger try: shutil.rmtree(old_path) except Exception: logger.error(traceback.print_exc()) logger.error("Could not remove old ledger: {}".format(old_path)) return False # Rename new ledger to old one try: shutil.move(new_path, old_path) except Exception: logger.error(traceback.print_exc()) logger.error( "Could not rename temporary new ledger from '{}' to '{}'".format( new_path, old_path)) return False try: set_own_perm("indy", old_path) except Exception: pass return True
def migrate_txn_log(db_dir, db_name): def put_into_seq_no_db(txn): # If there is no reqId, then it's genesis txn if get_req_id(txn) is None: return txn_new = copy.deepcopy(txn) operation = get_payload_data(txn_new) operation[TXN_TYPE] = get_type(txn_new) dct = { f.IDENTIFIER.nm: get_from(txn_new), f.REQ_ID.nm: get_req_id(txn_new), OPERATION: operation, } if get_protocol_version(txn_new) is not None: dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new) digest = sha256(serialize_msg_for_signing(dct)).hexdigest() seq_no = get_seq_no(txn_new) ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE]) line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no) dest_seq_no_db_storage.put(digest, line_to_record) return digest new_db_name = db_name + '_new' old_path = os.path.join(db_dir, db_name) new_path = os.path.join(db_dir, new_db_name) new_seqno_db_name = config.seqNoDbName + '_new' try: dest_seq_no_db_storage = initKeyValueStorage(config.reqIdToTxnStorage, db_dir, new_seqno_db_name) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open new seq_no_db storage") return False # open new and old ledgers try: src_storage = KeyValueStorageRocksdbIntKeys(db_dir, db_name, read_only=True) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open old ledger: {}".format(os.path.join(db_dir, db_name))) return False try: dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open new ledger: {}".format(os.path.join(db_dir, new_db_name))) return False # put values from old ledger to the new one try: for key, val in src_storage.iterator(): key = key.decode() val = ledger_txn_serializer.deserialize(val) new_val = transform_to_new_format(txn=val, seq_no=int(key)) digest = put_into_seq_no_db(new_val) # add digest into txn if get_req_id(new_val): new_val[TXN_PAYLOAD][TXN_PAYLOAD_METADATA][TXN_PAYLOAD_METADATA_DIGEST] = digest new_val = ledger_txn_serializer.serialize(new_val) dest_storage.put(key, new_val) except Exception: logger.error(traceback.print_exc()) logger.error("Could not put key/value to the new ledger '{}'".format(db_name)) return False src_storage.close() dest_storage.close() dest_seq_no_db_storage.close() # Remove old ledger try: shutil.rmtree(old_path) except Exception: logger.error(traceback.print_exc()) logger.error("Could not remove old ledger: {}" .format(old_path)) return False # Rename new ledger to old one try: shutil.move(new_path, old_path) except Exception: logger.error(traceback.print_exc()) logger.error("Could not rename temporary new ledger from '{}' to '{}'" .format(new_path, old_path)) return False try: set_own_perm("indy", old_path) except Exception: pass return True
def _print_debug_data(found_data): serializer = JsonSerializer() txn = ledger_txn_serializer.deserialize(found_data) txn = serializer.serialize(txn, toBytes=False) logger.debug("txn: {}".format(txn))
def migrate_txn_log(db_dir, db_name): new_db_name = db_name + '_new' old_path = os.path.join(db_dir, db_name) new_path = os.path.join(db_dir, new_db_name) # open new and old ledgers try: src_storage = KeyValueStorageRocksdbIntKeys(db_dir, db_name, read_only=True) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open old ledger: {}".format( os.path.join(db_dir, db_name))) return False try: dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name) except Exception: logger.error(traceback.print_exc()) logger.error("Could not open new ledger: {}".format( os.path.join(db_dir, new_db_name))) return False # put values from old ledger to the new one try: for key, val in src_storage.iterator(): val = ledger_txn_serializer.deserialize(val) new_val = transform_to_new_format(txn=val, seq_no=int(key)) new_val = ledger_txn_serializer.serialize(new_val) dest_storage.put(key, new_val) except Exception: logger.error(traceback.print_exc()) logger.error( "Could not put key/value to the new ledger '{}'".format(db_name)) return False src_storage.close() dest_storage.close() # Remove old ledger try: shutil.rmtree(old_path) except Exception: logger.error(traceback.print_exc()) logger.error("Could not remove old ledger: {}".format(old_path)) return False # Rename new ledger to old one try: shutil.move(new_path, old_path) except Exception: logger.error(traceback.print_exc()) logger.error( "Could not rename temporary new ledger from '{}' to '{}'".format( new_path, old_path)) return False set_own_perm("indy", old_path) return True
def print_all(storage, serializer): frm = int(args.frm) if args.frm else None to = int(args.to) if args.to else None for seqNo, txn in storage.iterator(start=frm, end=to): txn = ledger_txn_serializer.deserialize(txn) print(serializer.serialize(txn, toBytes=False))