def test_archive_calculate_hash(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
archive = ArchiveFile.from_file(tmp_archive)
actual = archive.calculate_file_hash('file1.txt')
expected = 'a250369afb3eeaa96fb0df99e7755ba784dfd69c'
assert actual == expected
def test_verify_file(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
archive_file = ArchiveFile.from_file(tmp_archive)
assert archive_file.verify_file('file1.txt')
archive_file._manifest["file_list"][0]["hash"] = "nonsense"
assert not archive_file.verify_file('file1.txt')
def test_summarise_archive(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
archive_file = ArchiveFile.from_file(tmp_archive)
summary = archive_file.summarise()
assert isinstance(summary, dict)
assert summary['n_files'] == 2
def test_from_tar_gz(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
archive_file = ArchiveFile.from_file(tmp_archive)
assert isinstance(archive_file, ArchiveFile)
assert archive_file._tar_path == tmp_archive
assert 'dtool_version' in archive_file.admin_metadata
assert archive_file.admin_metadata['name'] == 'brassica_rnaseq_reads'
assert len(archive_file.admin_metadata['uuid']) == 36
assert 'file_list' in archive_file.manifest
def full(path):
click.secho("Performing full verification on:", nl=False)
click.secho(" {}".format(path), fg='green')
archive_file = ArchiveFile.from_path(path)
result = archive_file.verify_all()
click.secho("Verification ", nl=False)
if result:
click.secho("passed", fg='green')
else:
click.secho("failed", fg='red')
def summary(path):
archive_file = ArchiveFile.from_file(path)
summary_data = archive_file.summarise()
size_in_gibi = float(summary_data['total_size']) / (2**30)
click.secho("Archive contains", nl=False)
click.secho(" {} ".format(summary_data['n_files']), fg='green', nl=False)
click.secho("files.")
click.secho("Total uncompressed archive size is", nl=False)
click.secho(" {:.2f} GiB".format(size_in_gibi), fg='green', nl=False)
click.secho(".")
def test_from_tar(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
unzip_command = ["gunzip", tmp_archive]
subprocess.call(unzip_command)
tar_filename, _ = tmp_archive.rsplit('.', 1)
archive_file = ArchiveFile.from_file(tar_filename)
assert isinstance(archive_file, ArchiveFile)
assert archive_file._tar_path == tar_filename
assert 'dtool_version' in archive_file.admin_metadata
assert archive_file.admin_metadata['name'] == 'brassica_rnaseq_reads'
assert len(archive_file.admin_metadata['uuid']) == 36
assert 'file_list' in archive_file.manifest
def compress(path, cores, slurm):
path = os.path.abspath(path)
archive = ArchiveFile.from_file(path)
if not slurm:
click.secho('Compressing archive: ', nl=False)
click.secho(path, fg='green')
pre_log = {
'dataset_uuid': archive.admin_metadata["uuid"],
'archive_path': path,
'cores': cores,
'tar_size': os.stat(path).st_size
}
logger.emit('pre_compress_archive', pre_log)
compressed_archive_path = compress_archive(path, n_threads=cores)
click.secho('Created compressed file: ', nl=False)
click.secho(compressed_archive_path, fg='green')
post_log = {
'dataset_uuid': archive.admin_metadata["uuid"],
'compressed_archive_path': compressed_archive_path,
'gzip_size': os.stat(compressed_archive_path).st_size
}
logger.emit('post_compress_archive', post_log)
click.secho('Now:')
click.secho(
' Move {} to archive storage'.format(compressed_archive_path),
fg='yellow')
# WARNING - be VERY careful automating this to submit the job - if the
# logic fails, the job will repeatedly submit itself forever!
else:
job_parameters = dict(n_cores=cores, partition="rg-sv")
command_string = "arctool archive compress -c {} {}".format(
cores, path)
submit_string = generate_slurm_script(command_string, job_parameters)
print(submit_string)
def test_initialise_ArchiveFile():
from arctool.archive import ArchiveFile
archive = ArchiveFile()
assert archive._tar_path is None
assert archive._name is None
def test_archive_verify_all(tmp_archive): # NOQA
from arctool.archive import ArchiveFile
archive_file = ArchiveFile.from_file(tmp_archive)
assert archive_file.verify_all()