def explorer():
#args = read_args()
config = getConfig()
result = []
ledger_data_dir = get_ledger_dir("", "")
read_copy_ledger_data_dir = None
try:
# RocksDB supports real read-only mode and does not need to have a ledger copy.
if config.hashStore['type'].lower() != HS_ROCKSDB:
config.db_transactions_config = None
# NOTE: such approach works well only for small ledgers.
tmp = make_copy_of_ledger(ledger_data_dir)
# Let's be paranoid to avoid removing of ledger instead of its copy.
ledger_path = Path(ledger_data_dir)
ledger_copy_path = Path(tmp)
assert ledger_path != ledger_copy_path
assert ledger_copy_path not in ledger_path.parents
read_copy_ledger_data_dir = tmp
ledger_data_dir = read_copy_ledger_data_dir
elif config.db_transactions_config is not None:
# This allows to avoid debug logs creation on each read_ledger run
config.db_transactions_config['db_log_dir'] = '/dev/null'
storage = get_storage("domain", ledger_data_dir)
finally:
if read_copy_ledger_data_dir:
shutil.rmtree(read_copy_ledger_data_dir)
for seqNo, txn in storage.iterator(start=0, end=100):
txn = ledger_txn_serializer.deserialize(txn)
serializer = JsonSerializer()
x = serializer.serialize(txn, toBytes=False)
result.append(x)
return result
def migrate_txn_log(db_dir, db_name):
def put_into_seq_no_db(txn):
# If there is no reqId, then it's genesis txn
if get_req_id(txn) is None:
return
txn_new = copy.deepcopy(txn)
operation = get_payload_data(txn_new)
operation[TXN_TYPE] = get_type(txn_new)
dct = {
f.IDENTIFIER.nm: get_from(txn_new),
f.REQ_ID.nm: get_req_id(txn_new),
OPERATION: operation,
}
if get_protocol_version(txn_new) is not None:
dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new)
digest = sha256(serialize_msg_for_signing(dct)).hexdigest().encode()
seq_no = get_seq_no(txn_new)
ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE])
line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no)
dest_seq_no_db_storage.put(digest, line_to_record)
return digest
new_db_name = db_name + '_new'
old_path = os.path.join(db_dir, db_name)
new_path = os.path.join(db_dir, new_db_name)
new_seqno_db_name = config.seqNoDbName + '_new'
try:
dest_seq_no_db_storage = initKeyValueStorage(config.reqIdToTxnStorage,
db_dir, new_seqno_db_name)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open new seq_no_db storage")
return False
# open new and old ledgers
try:
src_storage = KeyValueStorageRocksdbIntKeys(db_dir,
db_name,
read_only=True)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open old ledger: {}".format(
os.path.join(db_dir, db_name)))
return False
try:
dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open new ledger: {}".format(
os.path.join(db_dir, new_db_name)))
return False
# put values from old ledger to the new one
try:
for key, val in src_storage.iterator():
key = key.decode()
val = ledger_txn_serializer.deserialize(val)
new_val = transform_to_new_format(txn=val, seq_no=int(key))
digest = put_into_seq_no_db(new_val)
# add digest into txn
if get_req_id(new_val):
new_val[TXN_PAYLOAD][TXN_PAYLOAD_METADATA][
TXN_PAYLOAD_METADATA_DIGEST] = digest
new_val = ledger_txn_serializer.serialize(new_val)
dest_storage.put(key, new_val)
except Exception:
logger.error(traceback.print_exc())
logger.error(
"Could not put key/value to the new ledger '{}'".format(db_name))
return False
src_storage.close()
dest_storage.close()
dest_seq_no_db_storage.close()
# Remove old ledger
try:
shutil.rmtree(old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not remove old ledger: {}".format(old_path))
return False
# Rename new ledger to old one
try:
shutil.move(new_path, old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error(
"Could not rename temporary new ledger from '{}' to '{}'".format(
new_path, old_path))
return False
try:
set_own_perm("indy", old_path)
except Exception:
pass
return True
def migrate_txn_log(db_dir, db_name):
def put_into_seq_no_db(txn):
# If there is no reqId, then it's genesis txn
if get_req_id(txn) is None:
return
txn_new = copy.deepcopy(txn)
operation = get_payload_data(txn_new)
operation[TXN_TYPE] = get_type(txn_new)
dct = {
f.IDENTIFIER.nm: get_from(txn_new),
f.REQ_ID.nm: get_req_id(txn_new),
OPERATION: operation,
}
if get_protocol_version(txn_new) is not None:
dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new)
digest = sha256(serialize_msg_for_signing(dct)).hexdigest()
seq_no = get_seq_no(txn_new)
ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE])
line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no)
dest_seq_no_db_storage.put(digest, line_to_record)
return digest
new_db_name = db_name + '_new'
old_path = os.path.join(db_dir, db_name)
new_path = os.path.join(db_dir, new_db_name)
new_seqno_db_name = config.seqNoDbName + '_new'
try:
dest_seq_no_db_storage = initKeyValueStorage(config.reqIdToTxnStorage,
db_dir,
new_seqno_db_name)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open new seq_no_db storage")
return False
# open new and old ledgers
try:
src_storage = KeyValueStorageRocksdbIntKeys(db_dir, db_name, read_only=True)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open old ledger: {}".format(os.path.join(db_dir, db_name)))
return False
try:
dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open new ledger: {}".format(os.path.join(db_dir, new_db_name)))
return False
# put values from old ledger to the new one
try:
for key, val in src_storage.iterator():
key = key.decode()
val = ledger_txn_serializer.deserialize(val)
new_val = transform_to_new_format(txn=val, seq_no=int(key))
digest = put_into_seq_no_db(new_val)
# add digest into txn
if get_req_id(new_val):
new_val[TXN_PAYLOAD][TXN_PAYLOAD_METADATA][TXN_PAYLOAD_METADATA_DIGEST] = digest
new_val = ledger_txn_serializer.serialize(new_val)
dest_storage.put(key, new_val)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not put key/value to the new ledger '{}'".format(db_name))
return False
src_storage.close()
dest_storage.close()
dest_seq_no_db_storage.close()
# Remove old ledger
try:
shutil.rmtree(old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not remove old ledger: {}"
.format(old_path))
return False
# Rename new ledger to old one
try:
shutil.move(new_path, old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not rename temporary new ledger from '{}' to '{}'"
.format(new_path, old_path))
return False
try:
set_own_perm("indy", old_path)
except Exception:
pass
return True
def _print_debug_data(found_data):
serializer = JsonSerializer()
txn = ledger_txn_serializer.deserialize(found_data)
txn = serializer.serialize(txn, toBytes=False)
logger.debug("txn: {}".format(txn))
def migrate_txn_log(db_dir, db_name):
new_db_name = db_name + '_new'
old_path = os.path.join(db_dir, db_name)
new_path = os.path.join(db_dir, new_db_name)
# open new and old ledgers
try:
src_storage = KeyValueStorageRocksdbIntKeys(db_dir,
db_name,
read_only=True)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open old ledger: {}".format(
os.path.join(db_dir, db_name)))
return False
try:
dest_storage = KeyValueStorageRocksdbIntKeys(db_dir, new_db_name)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not open new ledger: {}".format(
os.path.join(db_dir, new_db_name)))
return False
# put values from old ledger to the new one
try:
for key, val in src_storage.iterator():
val = ledger_txn_serializer.deserialize(val)
new_val = transform_to_new_format(txn=val, seq_no=int(key))
new_val = ledger_txn_serializer.serialize(new_val)
dest_storage.put(key, new_val)
except Exception:
logger.error(traceback.print_exc())
logger.error(
"Could not put key/value to the new ledger '{}'".format(db_name))
return False
src_storage.close()
dest_storage.close()
# Remove old ledger
try:
shutil.rmtree(old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error("Could not remove old ledger: {}".format(old_path))
return False
# Rename new ledger to old one
try:
shutil.move(new_path, old_path)
except Exception:
logger.error(traceback.print_exc())
logger.error(
"Could not rename temporary new ledger from '{}' to '{}'".format(
new_path, old_path))
return False
set_own_perm("indy", old_path)
return True
def print_all(storage, serializer):
frm = int(args.frm) if args.frm else None
to = int(args.to) if args.to else None
for seqNo, txn in storage.iterator(start=frm, end=to):
txn = ledger_txn_serializer.deserialize(txn)
print(serializer.serialize(txn, toBytes=False))