def __init__(self, path_or_file: typing.Union[str, typing.BinaryIO]):
self.logical_files: typing.List[LogicalFile] = []
# Low level index of Logical Records. A reference to this is given to every LogicalFile
self._logical_record_index = Index.LogicalRecordIndex(path_or_file)
# TODO: Created from the FILE-HEADER values
# {ID : {SEQUENCE-NUMBER : index_into_self.logical_files, ...}, ...}
self.sequence_map: typing.Dict[str, typing.Dict[int, int]] = {}
def test_logical_record_index_minimal_file_get_file_logical_data(index, offset, length, expected_bytes):
file = io.BytesIO(test_data.MINIMAL_FILE)
with Index.LogicalRecordIndex(file) as lr_index:
file_logical_data = lr_index.get_file_logical_data(index, offset, length)
assert file_logical_data.is_sealed()
assert file_logical_data.logical_data.index == 0
assert file_logical_data.logical_data.bytes == expected_bytes
def test_logical_record_index_pickle():
fobj = io.BytesIO(test_data.BASIC_FILE)
with Index.LogicalRecordIndex(fobj) as lr_index:
pickled_index = pickle.dumps(lr_index)
assert len(pickled_index) == 53106
new_lr_index = pickle.loads(pickled_index)
assert len(new_lr_index) == 660
def index_a_single_file(path_in: str, path_out: str, read_back: bool,
validate: bool) -> IndexResult:
"""Read a single file and return an IndexResult."""
file_type = bin_file_type.binary_file_type_from_path(path_in)
if file_type == 'RP66V1':
if path_out:
out_dir = os.path.dirname(path_out)
if not os.path.exists(out_dir):
logger.info(f'Making directory: {out_dir}')
os.makedirs(out_dir, exist_ok=True)
logger.info(f'Indexing {path_in}')
try:
t_start = time.perf_counter()
# Index the file
# process.add_message_to_queue(f'Start {os.path.basename(path_in)}')
with Index.LogicalRecordIndex(path_in) as logical_record_index:
index_time = time.perf_counter() - t_start
if validate:
try:
logical_record_index.validate()
logger.info(f'Validation OK for {path_in}')
except File.ExceptionFileReadPositionsInconsistent(
) as err:
logger.exception(
f'Validation of {path_in} failed with {err}')
t_start = time.perf_counter()
# process.add_message_to_queue(f'Pickle {os.path.basename(path_in)}')
# Pickle the index
if path_out:
pickle_path = path_out + '.pkl'
with open(pickle_path, 'wb') as pickle_file:
pickle.dump(logical_record_index, pickle_file)
len_pickled_index = os.path.getsize(pickle_path)
write_time = time.perf_counter() - t_start
if read_back:
t_start = time.perf_counter()
_read_index = unpickle(pickle_path)
read_back_time = time.perf_counter() - t_start
else:
read_back_time = 0.0
else:
pickled_index = pickle.dumps(logical_record_index)
write_time = time.perf_counter() - t_start
len_pickled_index = len(pickled_index)
read_back_time = 0.0
result = IndexResult(path_in, os.path.getsize(path_in),
len_pickled_index, index_time, write_time,
read_back_time, False, False)
return result
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(
f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0,
True, False) # pragma: no cover
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0,
False, True) # pragma: no cover
def test_logical_record_index_small_file_eflr_iflr_counts():
file = io.BytesIO(test_data.SMALL_FILE)
with Index.LogicalRecordIndex(file) as lr_index:
counts = {'EFLR': 0, 'IFLR': 0, }
for i in range(len(lr_index)):
if lr_index[i].description.attributes.is_eflr:
counts['EFLR'] += 1
else:
counts['IFLR'] += 1
assert counts == {'EFLR': 5, 'IFLR': 83,}
def test_logical_record_index_basic_file_with_two_vrs(index, vr_position, lrsh_position, attributes, lr_type, ld_length):
file = io.BytesIO(test_data.BASIC_FILE_WITH_TWO_VISIBLE_RECORDS_NO_IFLRS)
with Index.LogicalRecordIndex(file) as lr_index:
assert len(lr_index) == 9
assert lr_index[index].position.vr_position == vr_position
assert lr_index[index].position.lrsh_position == lrsh_position
assert lr_index[index].description.attributes.is_first
assert lr_index[index].description.attributes.attributes == attributes
assert lr_index[index].description.lr_type == lr_type
assert lr_index[index].description.ld_length == ld_length
def test_logical_record_index_minimal_file(index, vr_position, lrsh_position, attributes, lr_type, ld_length):
file = io.BytesIO(test_data.MINIMAL_FILE)
with Index.LogicalRecordIndex(file) as lr_index:
assert len(lr_index) == 2
assert lr_index[index].position.vr_position == vr_position
assert lr_index[index].position.lrsh_position == lrsh_position
assert lr_index[index].description.attributes.is_first
assert lr_index[index].description.attributes.attributes == attributes
assert lr_index[index].description.lr_type == lr_type
assert lr_index[index].description.ld_length == ld_length
def test_logical_record_index_file_256kb_eflr_iflr_counts():
file = io.BytesIO(test_data.FILE_256kb)
with Index.LogicalRecordIndex(file) as lr_index:
counts = {'EFLR': 0, 'IFLR': 0, }
for i in range(len(lr_index)):
if lr_index[i].description.attributes.is_eflr:
counts['EFLR'] += 1
else:
counts['IFLR'] += 1
assert counts == {'EFLR': 31, 'IFLR': 1852,}
def __init__(self, path_or_file: typing.Union[str, typing.BinaryIO]):
self.logical_files: typing.List[LogicalFile] = []
# A reference to this is given to every LogicalFile
self._logical_record_index = Index.LogicalRecordIndex(path_or_file)
def test_logical_record_index_basic_file():
file = io.BytesIO(test_data.BASIC_FILE)
with Index.LogicalRecordIndex(file) as lr_index:
assert len(lr_index) == 660
def test_logical_record_index_small_file():
file = io.BytesIO(test_data.SMALL_FILE)
with Index.LogicalRecordIndex(file) as lr_index:
assert len(lr_index) == 88
def test_logical_record_index_sul():
fobj = io.BytesIO(test_data.BASIC_FILE)
with Index.LogicalRecordIndex(fobj) as lr_index:
assert str(lr_index.sul) == r"""StorageUnitLabel:
def test_logical_record_index_all_files(by, expected):
fobj = io.BytesIO(by)
with Index.LogicalRecordIndex(fobj) as lr_index:
assert len(lr_index) == expected
def test_logical_record_index_file_256kb():
file = io.BytesIO(test_data.FILE_256kb)
with Index.LogicalRecordIndex(file) as lr_index:
assert len(lr_index) == 1883
def test_logical_record_index_ctor_empty():
file = io.BytesIO(test_data.BASIC_FILE)
index = Index.LogicalRecordIndex(file)
assert len(index) == 0