def test_single_file(self): self.make_file(b'Normal text\nand things\n', 'foobarbaz.txt') self.assertEqual( util.md5sum_directory(self.test_path), collections.OrderedDict([('foobarbaz.txt', '93b048d0202e4b06b658f3aef1e764d3')]))
def test_sorted_decent(self): nested_dir = self.test_path / 'beta' nested_dir.mkdir() filepath = (nested_dir / '10').relative_to(self.test_path) self.make_file(b'10', filepath) filepath = (nested_dir / '1').relative_to(self.test_path) self.make_file(b'1', filepath) filepath = (nested_dir / '2').relative_to(self.test_path) self.make_file(b'2', filepath) nested_dir = self.test_path / 'alpha' nested_dir.mkdir() filepath = (nested_dir / 'foo').relative_to(self.test_path) self.make_file(b'foo', filepath) filepath = (nested_dir / 'bar').relative_to(self.test_path) self.make_file(b'bar', filepath) self.make_file(b'z', 'z') self.assertEqual(list(util.md5sum_directory(self.test_path).items()), [ ('z', 'fbade9e36a3f36d3d676c1b808451dd7'), ('alpha/bar', '37b51d194a7513e45b56f6524f2d51f2'), ('alpha/foo', 'acbd18db4cc2f85cedef654fccc4a4d8'), ('beta/1', 'c4ca4238a0b923820dcc509a6f75849b'), ('beta/10', 'd3d9446802a44259755d38e6d163e820'), ('beta/2', 'c81e728d9d4c2f636f067f89cc14862c'), ])
def test_sorted_decent(self): nested_dir = self.test_path / 'beta' nested_dir.mkdir() filepath = (nested_dir / '10').relative_to(self.test_path) self.make_file(b'10', filepath) filepath = (nested_dir / '1').relative_to(self.test_path) self.make_file(b'1', filepath) filepath = (nested_dir / '2').relative_to(self.test_path) self.make_file(b'2', filepath) nested_dir = self.test_path / 'alpha' nested_dir.mkdir() filepath = (nested_dir / 'foo').relative_to(self.test_path) self.make_file(b'foo', filepath) filepath = (nested_dir / 'bar').relative_to(self.test_path) self.make_file(b'bar', filepath) self.make_file(b'z', 'z') self.assertEqual( list(util.md5sum_directory(self.test_path).items()), [ ('z', 'fbade9e36a3f36d3d676c1b808451dd7'), ('alpha/bar', '37b51d194a7513e45b56f6524f2d51f2'), ('alpha/foo', 'acbd18db4cc2f85cedef654fccc4a4d8'), ('beta/1', 'c4ca4238a0b923820dcc509a6f75849b'), ('beta/10', 'd3d9446802a44259755d38e6d163e820'), ('beta/2', 'c81e728d9d4c2f636f067f89cc14862c'), ])
def test_nested_empty_directories(self): (self.test_path / 'foo').mkdir() (self.test_path / 'foo' / 'bar').mkdir() (self.test_path / 'baz').mkdir() self.assertEqual(util.md5sum_directory(self.test_path), collections.OrderedDict())
def test_nested_empty_directories(self): (self.test_path / 'foo').mkdir() (self.test_path / 'foo' / 'bar').mkdir() (self.test_path / 'baz').mkdir() self.assertEqual(util.md5sum_directory(self.test_path), collections.OrderedDict())
def test_single_file(self): self.make_file(b'Normal text\nand things\n', 'foobarbaz.txt') self.assertEqual( util.md5sum_directory(self.test_path), collections.OrderedDict([ ('foobarbaz.txt', '93b048d0202e4b06b658f3aef1e764d3') ]))
def write(cls, archive_record, type, format, data_initializer, provenance_capture): super().write(archive_record, type, format, data_initializer, provenance_capture) checksums = md5sum_directory(str(archive_record.root)) with (archive_record.root / cls.CHECKSUM_FILE).open('w') as fh: for item in checksums.items(): fh.write(to_checksum_format(*item)) fh.write('\n')
def write(cls, archive_record, type, format, data_initializer, provenance_capture): super().write(archive_record, type, format, data_initializer, provenance_capture) checksums = md5sum_directory(str(archive_record.root)) with (archive_record.root / cls.CHECKSUM_FILE).open('w') as fh: for item in checksums.items(): fh.write(to_checksum_format(*item)) fh.write('\n')
def test_can_use_string(self): nested_dir = self.test_path / 'bar' nested_dir.mkdir() filepath = (nested_dir / 'foo.baz').relative_to(self.test_path) self.make_file(b'anything at all', filepath) self.assertEqual( util.md5sum_directory(str(self.test_path)), collections.OrderedDict([('bar/foo.baz', 'dcc0975b66728be0315abae5968379cb')]))
def test_can_use_string(self): nested_dir = self.test_path / 'bar' nested_dir.mkdir() filepath = (nested_dir / 'foo.baz').relative_to(self.test_path) self.make_file(b'anything at all', filepath) self.assertEqual( util.md5sum_directory(str(self.test_path)), collections.OrderedDict([ ('bar/foo.baz', 'dcc0975b66728be0315abae5968379cb') ]))
def import_data(cls, type, view, view_type=None): type_, type = type, __builtins__['type'] is_format = False if isinstance(type_, str): type_ = qiime2.sdk.parse_type(type_) if isinstance(view_type, str): view_type = qiime2.sdk.parse_format(view_type) is_format = True if view_type is None: if type(view) is str or isinstance(view, pathlib.PurePath): is_format = True pm = qiime2.sdk.PluginManager() output_dir_fmt = pm.get_directory_format(type_) if pathlib.Path(view).is_file(): if not issubclass(output_dir_fmt, model.SingleFileDirectoryFormatBase): raise qiime2.plugin.ValidationError( "Importing %r requires a directory, not %s" % (output_dir_fmt.__name__, view)) view_type = output_dir_fmt.file.format else: view_type = output_dir_fmt else: view_type = type(view) format_ = None md5sums = None if is_format: path = pathlib.Path(view) if path.is_file(): md5sums = {path.name: util.md5sum(path)} elif path.is_dir(): md5sums = util.md5sum_directory(path) else: raise qiime2.plugin.ValidationError( "Path '%s' does not exist." % path) format_ = view_type provenance_capture = archive.ImportProvenanceCapture(format_, md5sums) return cls._from_view(type_, view, view_type, provenance_capture, validate_level='max')
def validate_checksums(self): if not isinstance(self._fmt, self.get_format_class('5')): return ChecksumDiff({}, {}, {}) obs = dict(x for x in md5sum_directory(str(self.root_dir)).items() if x[0] != self._fmt.CHECKSUM_FILE) exp = dict(from_checksum_format(line) for line in (self.root_dir / self._fmt.CHECKSUM_FILE).open().readlines() ) obs_keys = set(obs) exp_keys = set(exp) added = {x: obs[x] for x in obs_keys - exp_keys} removed = {x: exp[x] for x in exp_keys - obs_keys} changed = {x: (exp[x], obs[x]) for x in exp_keys & obs_keys if exp[x] != obs[x]} return ChecksumDiff(added=added, removed=removed, changed=changed)
def validate_checksums(self): if not isinstance(self._fmt, self.get_format_class('5')): return ChecksumDiff({}, {}, {}) obs = dict(x for x in md5sum_directory(str(self.root_dir)).items() if x[0] != self._fmt.CHECKSUM_FILE) exp = dict(from_checksum_format(line) for line in (self.root_dir / self._fmt.CHECKSUM_FILE).open().readlines() ) obs_keys = set(obs) exp_keys = set(exp) added = {x: obs[x] for x in obs_keys - exp_keys} removed = {x: exp[x] for x in exp_keys - obs_keys} changed = {x: (exp[x], obs[x]) for x in exp_keys & obs_keys if exp[x] != obs[x]} return ChecksumDiff(added=added, removed=removed, changed=changed)
def import_data(cls, type, view, view_type=None): type_, type = type, __builtins__['type'] is_format = False if isinstance(type_, str): type_ = qiime2.sdk.parse_type(type_) if isinstance(view_type, str): view_type = qiime2.sdk.parse_format(view_type) is_format = True if view_type is None: if type(view) is str or isinstance(view, pathlib.PurePath): is_format = True pm = qiime2.sdk.PluginManager() output_dir_fmt = pm.get_directory_format(type_) if pathlib.Path(view).is_file(): if not issubclass(output_dir_fmt, model.SingleFileDirectoryFormatBase): raise qiime2.plugin.ValidationError( "Importing %r requires a directory, not %s" % (output_dir_fmt.__name__, view)) view_type = output_dir_fmt.file.format else: view_type = output_dir_fmt else: view_type = type(view) format_ = None md5sums = None if is_format: path = pathlib.Path(view) if path.is_file(): md5sums = {path.name: util.md5sum(path)} elif path.is_dir(): md5sums = util.md5sum_directory(path) else: raise qiime2.plugin.ValidationError( "Path '%s' does not exist." % path) format_ = view_type provenance_capture = archive.ImportProvenanceCapture(format_, md5sums) return cls._from_view(type_, view, view_type, provenance_capture)
def test_empty_directory(self): self.assertEqual(util.md5sum_directory(self.test_path), collections.OrderedDict())
def test_empty_directory(self): self.assertEqual(util.md5sum_directory(self.test_path), collections.OrderedDict())