def test_single_file(self):
self.make_file(b'Normal text\nand things\n', 'foobarbaz.txt')
self.assertEqual(
util.md5sum_directory(self.test_path),
collections.OrderedDict([('foobarbaz.txt',
'93b048d0202e4b06b658f3aef1e764d3')]))
def test_sorted_decent(self):
nested_dir = self.test_path / 'beta'
nested_dir.mkdir()
filepath = (nested_dir / '10').relative_to(self.test_path)
self.make_file(b'10', filepath)
filepath = (nested_dir / '1').relative_to(self.test_path)
self.make_file(b'1', filepath)
filepath = (nested_dir / '2').relative_to(self.test_path)
self.make_file(b'2', filepath)
nested_dir = self.test_path / 'alpha'
nested_dir.mkdir()
filepath = (nested_dir / 'foo').relative_to(self.test_path)
self.make_file(b'foo', filepath)
filepath = (nested_dir / 'bar').relative_to(self.test_path)
self.make_file(b'bar', filepath)
self.make_file(b'z', 'z')
self.assertEqual(list(util.md5sum_directory(self.test_path).items()), [
('z', 'fbade9e36a3f36d3d676c1b808451dd7'),
('alpha/bar', '37b51d194a7513e45b56f6524f2d51f2'),
('alpha/foo', 'acbd18db4cc2f85cedef654fccc4a4d8'),
('beta/1', 'c4ca4238a0b923820dcc509a6f75849b'),
('beta/10', 'd3d9446802a44259755d38e6d163e820'),
('beta/2', 'c81e728d9d4c2f636f067f89cc14862c'),
])
def test_sorted_decent(self):
nested_dir = self.test_path / 'beta'
nested_dir.mkdir()
filepath = (nested_dir / '10').relative_to(self.test_path)
self.make_file(b'10', filepath)
filepath = (nested_dir / '1').relative_to(self.test_path)
self.make_file(b'1', filepath)
filepath = (nested_dir / '2').relative_to(self.test_path)
self.make_file(b'2', filepath)
nested_dir = self.test_path / 'alpha'
nested_dir.mkdir()
filepath = (nested_dir / 'foo').relative_to(self.test_path)
self.make_file(b'foo', filepath)
filepath = (nested_dir / 'bar').relative_to(self.test_path)
self.make_file(b'bar', filepath)
self.make_file(b'z', 'z')
self.assertEqual(
list(util.md5sum_directory(self.test_path).items()),
[
('z', 'fbade9e36a3f36d3d676c1b808451dd7'),
('alpha/bar', '37b51d194a7513e45b56f6524f2d51f2'),
('alpha/foo', 'acbd18db4cc2f85cedef654fccc4a4d8'),
('beta/1', 'c4ca4238a0b923820dcc509a6f75849b'),
('beta/10', 'd3d9446802a44259755d38e6d163e820'),
('beta/2', 'c81e728d9d4c2f636f067f89cc14862c'),
])
def test_nested_empty_directories(self):
(self.test_path / 'foo').mkdir()
(self.test_path / 'foo' / 'bar').mkdir()
(self.test_path / 'baz').mkdir()
self.assertEqual(util.md5sum_directory(self.test_path),
collections.OrderedDict())
def test_nested_empty_directories(self):
(self.test_path / 'foo').mkdir()
(self.test_path / 'foo' / 'bar').mkdir()
(self.test_path / 'baz').mkdir()
self.assertEqual(util.md5sum_directory(self.test_path),
collections.OrderedDict())
def test_single_file(self):
self.make_file(b'Normal text\nand things\n', 'foobarbaz.txt')
self.assertEqual(
util.md5sum_directory(self.test_path),
collections.OrderedDict([
('foobarbaz.txt', '93b048d0202e4b06b658f3aef1e764d3')
]))
def write(cls, archive_record, type, format, data_initializer,
provenance_capture):
super().write(archive_record, type, format, data_initializer,
provenance_capture)
checksums = md5sum_directory(str(archive_record.root))
with (archive_record.root / cls.CHECKSUM_FILE).open('w') as fh:
for item in checksums.items():
fh.write(to_checksum_format(*item))
fh.write('\n')
def write(cls, archive_record, type, format, data_initializer,
provenance_capture):
super().write(archive_record, type, format, data_initializer,
provenance_capture)
checksums = md5sum_directory(str(archive_record.root))
with (archive_record.root / cls.CHECKSUM_FILE).open('w') as fh:
for item in checksums.items():
fh.write(to_checksum_format(*item))
fh.write('\n')
def test_can_use_string(self):
nested_dir = self.test_path / 'bar'
nested_dir.mkdir()
filepath = (nested_dir / 'foo.baz').relative_to(self.test_path)
self.make_file(b'anything at all', filepath)
self.assertEqual(
util.md5sum_directory(str(self.test_path)),
collections.OrderedDict([('bar/foo.baz',
'dcc0975b66728be0315abae5968379cb')]))
def test_can_use_string(self):
nested_dir = self.test_path / 'bar'
nested_dir.mkdir()
filepath = (nested_dir / 'foo.baz').relative_to(self.test_path)
self.make_file(b'anything at all', filepath)
self.assertEqual(
util.md5sum_directory(str(self.test_path)),
collections.OrderedDict([
('bar/foo.baz', 'dcc0975b66728be0315abae5968379cb')
]))
def import_data(cls, type, view, view_type=None):
type_, type = type, __builtins__['type']
is_format = False
if isinstance(type_, str):
type_ = qiime2.sdk.parse_type(type_)
if isinstance(view_type, str):
view_type = qiime2.sdk.parse_format(view_type)
is_format = True
if view_type is None:
if type(view) is str or isinstance(view, pathlib.PurePath):
is_format = True
pm = qiime2.sdk.PluginManager()
output_dir_fmt = pm.get_directory_format(type_)
if pathlib.Path(view).is_file():
if not issubclass(output_dir_fmt,
model.SingleFileDirectoryFormatBase):
raise qiime2.plugin.ValidationError(
"Importing %r requires a directory, not %s" %
(output_dir_fmt.__name__, view))
view_type = output_dir_fmt.file.format
else:
view_type = output_dir_fmt
else:
view_type = type(view)
format_ = None
md5sums = None
if is_format:
path = pathlib.Path(view)
if path.is_file():
md5sums = {path.name: util.md5sum(path)}
elif path.is_dir():
md5sums = util.md5sum_directory(path)
else:
raise qiime2.plugin.ValidationError(
"Path '%s' does not exist." % path)
format_ = view_type
provenance_capture = archive.ImportProvenanceCapture(format_, md5sums)
return cls._from_view(type_,
view,
view_type,
provenance_capture,
validate_level='max')
def validate_checksums(self):
if not isinstance(self._fmt, self.get_format_class('5')):
return ChecksumDiff({}, {}, {})
obs = dict(x for x in md5sum_directory(str(self.root_dir)).items()
if x[0] != self._fmt.CHECKSUM_FILE)
exp = dict(from_checksum_format(line) for line in
(self.root_dir / self._fmt.CHECKSUM_FILE).open().readlines()
)
obs_keys = set(obs)
exp_keys = set(exp)
added = {x: obs[x] for x in obs_keys - exp_keys}
removed = {x: exp[x] for x in exp_keys - obs_keys}
changed = {x: (exp[x], obs[x]) for x in exp_keys & obs_keys
if exp[x] != obs[x]}
return ChecksumDiff(added=added, removed=removed, changed=changed)
def validate_checksums(self):
if not isinstance(self._fmt, self.get_format_class('5')):
return ChecksumDiff({}, {}, {})
obs = dict(x for x in md5sum_directory(str(self.root_dir)).items()
if x[0] != self._fmt.CHECKSUM_FILE)
exp = dict(from_checksum_format(line) for line in
(self.root_dir / self._fmt.CHECKSUM_FILE).open().readlines()
)
obs_keys = set(obs)
exp_keys = set(exp)
added = {x: obs[x] for x in obs_keys - exp_keys}
removed = {x: exp[x] for x in exp_keys - obs_keys}
changed = {x: (exp[x], obs[x]) for x in exp_keys & obs_keys
if exp[x] != obs[x]}
return ChecksumDiff(added=added, removed=removed, changed=changed)
def import_data(cls, type, view, view_type=None):
type_, type = type, __builtins__['type']
is_format = False
if isinstance(type_, str):
type_ = qiime2.sdk.parse_type(type_)
if isinstance(view_type, str):
view_type = qiime2.sdk.parse_format(view_type)
is_format = True
if view_type is None:
if type(view) is str or isinstance(view, pathlib.PurePath):
is_format = True
pm = qiime2.sdk.PluginManager()
output_dir_fmt = pm.get_directory_format(type_)
if pathlib.Path(view).is_file():
if not issubclass(output_dir_fmt,
model.SingleFileDirectoryFormatBase):
raise qiime2.plugin.ValidationError(
"Importing %r requires a directory, not %s"
% (output_dir_fmt.__name__, view))
view_type = output_dir_fmt.file.format
else:
view_type = output_dir_fmt
else:
view_type = type(view)
format_ = None
md5sums = None
if is_format:
path = pathlib.Path(view)
if path.is_file():
md5sums = {path.name: util.md5sum(path)}
elif path.is_dir():
md5sums = util.md5sum_directory(path)
else:
raise qiime2.plugin.ValidationError(
"Path '%s' does not exist." % path)
format_ = view_type
provenance_capture = archive.ImportProvenanceCapture(format_, md5sums)
return cls._from_view(type_, view, view_type, provenance_capture)
def test_empty_directory(self):
self.assertEqual(util.md5sum_directory(self.test_path),
collections.OrderedDict())
def test_empty_directory(self):
self.assertEqual(util.md5sum_directory(self.test_path),
collections.OrderedDict())