def test_archive_calculate_hash(tmp_archive): # NOQA from arctool.archive import ArchiveFile archive = ArchiveFile.from_file(tmp_archive) actual = archive.calculate_file_hash('file1.txt') expected = 'a250369afb3eeaa96fb0df99e7755ba784dfd69c' assert actual == expected
def test_verify_file(tmp_archive): # NOQA from arctool.archive import ArchiveFile archive_file = ArchiveFile.from_file(tmp_archive) assert archive_file.verify_file('file1.txt') archive_file._manifest["file_list"][0]["hash"] = "nonsense" assert not archive_file.verify_file('file1.txt')
def test_summarise_archive(tmp_archive): # NOQA from arctool.archive import ArchiveFile archive_file = ArchiveFile.from_file(tmp_archive) summary = archive_file.summarise() assert isinstance(summary, dict) assert summary['n_files'] == 2
def test_from_tar_gz(tmp_archive): # NOQA from arctool.archive import ArchiveFile archive_file = ArchiveFile.from_file(tmp_archive) assert isinstance(archive_file, ArchiveFile) assert archive_file._tar_path == tmp_archive assert 'dtool_version' in archive_file.admin_metadata assert archive_file.admin_metadata['name'] == 'brassica_rnaseq_reads' assert len(archive_file.admin_metadata['uuid']) == 36 assert 'file_list' in archive_file.manifest
def full(path): click.secho("Performing full verification on:", nl=False) click.secho(" {}".format(path), fg='green') archive_file = ArchiveFile.from_path(path) result = archive_file.verify_all() click.secho("Verification ", nl=False) if result: click.secho("passed", fg='green') else: click.secho("failed", fg='red')
def summary(path): archive_file = ArchiveFile.from_file(path) summary_data = archive_file.summarise() size_in_gibi = float(summary_data['total_size']) / (2**30) click.secho("Archive contains", nl=False) click.secho(" {} ".format(summary_data['n_files']), fg='green', nl=False) click.secho("files.") click.secho("Total uncompressed archive size is", nl=False) click.secho(" {:.2f} GiB".format(size_in_gibi), fg='green', nl=False) click.secho(".")
def test_from_tar(tmp_archive): # NOQA from arctool.archive import ArchiveFile unzip_command = ["gunzip", tmp_archive] subprocess.call(unzip_command) tar_filename, _ = tmp_archive.rsplit('.', 1) archive_file = ArchiveFile.from_file(tar_filename) assert isinstance(archive_file, ArchiveFile) assert archive_file._tar_path == tar_filename assert 'dtool_version' in archive_file.admin_metadata assert archive_file.admin_metadata['name'] == 'brassica_rnaseq_reads' assert len(archive_file.admin_metadata['uuid']) == 36 assert 'file_list' in archive_file.manifest
def compress(path, cores, slurm): path = os.path.abspath(path) archive = ArchiveFile.from_file(path) if not slurm: click.secho('Compressing archive: ', nl=False) click.secho(path, fg='green') pre_log = { 'dataset_uuid': archive.admin_metadata["uuid"], 'archive_path': path, 'cores': cores, 'tar_size': os.stat(path).st_size } logger.emit('pre_compress_archive', pre_log) compressed_archive_path = compress_archive(path, n_threads=cores) click.secho('Created compressed file: ', nl=False) click.secho(compressed_archive_path, fg='green') post_log = { 'dataset_uuid': archive.admin_metadata["uuid"], 'compressed_archive_path': compressed_archive_path, 'gzip_size': os.stat(compressed_archive_path).st_size } logger.emit('post_compress_archive', post_log) click.secho('Now:') click.secho( ' Move {} to archive storage'.format(compressed_archive_path), fg='yellow') # WARNING - be VERY careful automating this to submit the job - if the # logic fails, the job will repeatedly submit itself forever! else: job_parameters = dict(n_cores=cores, partition="rg-sv") command_string = "arctool archive compress -c {} {}".format( cores, path) submit_string = generate_slurm_script(command_string, job_parameters) print(submit_string)
def test_initialise_ArchiveFile(): from arctool.archive import ArchiveFile archive = ArchiveFile() assert archive._tar_path is None assert archive._name is None
def test_archive_verify_all(tmp_archive): # NOQA from arctool.archive import ArchiveFile archive_file = ArchiveFile.from_file(tmp_archive) assert archive_file.verify_all()