def test_stdout_ok(self, data_mock, urljoin_mock, resolwe_mock):
# Configure mocks:
process_mock = MagicMock(type="data:index")
data_mock.configure_mock(id=123,
status="OK",
resolwe=resolwe_mock,
process=process_mock)
urljoin_mock.return_value = "some_url"
resolwe_mock.configure_mock(url="a", auth="b")
# If response.ok = True:
resolwe_mock.session.get.return_value = MagicMock(
ok=True, **{"iter_content.return_value": [b"abc", b"def"]})
out = Data.stdout(data_mock)
self.assertEqual(out, "abcdef")
urljoin_mock.assert_called_once_with("a", "data/123/stdout.txt")
resolwe_mock.session.get.assert_called_once_with("some_url",
stream=True,
auth="b")
# If response.ok = False:
response = MagicMock(ok=False)
resolwe_mock.session.get.return_value = response
out = Data.stdout(data_mock)
self.assertEqual(response.raise_for_status.call_count, 1)
def test_collections(self):
data = Data(id=1, resolwe=MagicMock())
# test getting collections attribute
data.api(data.id).get = MagicMock(return_value={'collections': [5]})
data.resolwe.collection.filter = MagicMock(return_value=[
Collection(data.resolwe, **{'id': 5, 'name': 'XYZ'})
])
self.assertEqual(len(data.collections), 1)
self.assertEqual(data.collections[0].id, 5)
self.assertEqual(data.collections[0].name, 'XYZ')
# test caching collections attribute
self.assertEqual(len(data.collections), 1)
self.assertEqual(data.resolwe.collection.filter.call_count, 1)
# cache is cleared at update
data.update()
self.assertEqual(data._collections, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.collections
def test_stdout_ok(self, data_mock, urljoin_mock, requests_mock):
# Configure mocks:
data_mock.configure_mock(id=123, resolwe=MagicMock(url="a", auth="b"))
urljoin_mock.return_value = "some_url"
# If response.ok = True:
response = MagicMock(ok=True,
**{'iter_content.return_value': [b"abc", b"def"]})
requests_mock.configure_mock(**{'get.return_value': response})
out = Data.stdout(data_mock)
self.assertEqual(out, "abcdef")
urljoin_mock.assert_called_once_with("a", 'data/123/stdout.txt')
requests_mock.get.assert_called_once_with("some_url",
stream=True,
auth="b")
# If response.ok = False:
response = MagicMock(ok=False)
requests_mock.configure_mock(**{'get.return_value': response})
out = Data.stdout(data_mock)
self.assertEqual(response.raise_for_status.call_count, 1)
def test_add_output(self, data_mock):
data_mock.configure_mock(
output={
'fastq': {
'file': 'reads.fq'
},
'fasta': {
'file': 'genome.fa'
},
},
process_output_schema=[
{
'type': 'basic:file:',
'name': 'fastq',
'label': 'FASTQ'
},
{
'type': 'basic:file:',
'name': 'fasta',
'label': 'FASTA'
},
],
)
files_list = Data._files_dirs(data_mock,
'file',
field_name="output.fastq")
self.assertEqual(files_list, ['reads.fq'])
files_list = Data._files_dirs(data_mock, 'file', field_name="fastq")
self.assertEqual(files_list, ['reads.fq'])
def test_download_ok(self, data_mock):
data_mock.configure_mock(id=123, **{'resolwe': MagicMock()})
data_mock.configure_mock(**{'files.return_value': ['file1.txt', 'file2.fq.gz']})
Data.download(data_mock)
data_mock.resolwe.download_files.assert_called_once_with([u'123/file1.txt', u'123/file2.fq.gz'], None)
data_mock.reset_mock()
Data.download(data_mock, download_dir="/some/path/")
data_mock.resolwe.download_files.assert_called_once_with([u'123/file1.txt', u'123/file2.fq.gz'], '/some/path/')
def test_dir_files(self, requests_mock):
data = Data(id=123, resolwe=MagicMock(url='http://resolwe.url'))
requests_mock.get = MagicMock(side_effect=[
MagicMock(content=b'[{"type": "file", "name": "file1.txt"}, '
b'{"type": "directory", "name": "subdir"}]'),
MagicMock(content=b'[{"type": "file", "name": "file2.txt"}]'),
])
files = data._get_dir_files('test_dir')
self.assertEqual(files, ['test_dir/file1.txt', 'test_dir/subdir/file2.txt'])
def test_dir_files(self, resolwe_mock):
resolwe_mock.url = "http://resolwe.url"
resolwe_mock.session.get.side_effect = [
MagicMock(content=b'[{"type": "file", "name": "file1.txt"}, '
b'{"type": "directory", "name": "subdir"}]'),
MagicMock(content=b'[{"type": "file", "name": "file2.txt"}]'),
]
data = Data(id=123, resolwe=resolwe_mock)
files = data._get_dir_files("test_dir")
self.assertEqual(files,
["test_dir/file1.txt", "test_dir/subdir/file2.txt"])
def test_dir_files(self, requests_mock):
data = Data(id=123, resolwe=MagicMock(url='http://resolwe.url'))
requests_mock.get = MagicMock(side_effect=[
MagicMock(content=b'[{"type": "file", "name": "file1.txt"}, '
b'{"type": "directory", "name": "subdir"}]'),
MagicMock(content=b'[{"type": "file", "name": "file2.txt"}]'),
])
files = data._get_dir_files('test_dir')
self.assertEqual(files, ['test_dir/file1.txt', 'test_dir/subdir/file2.txt'])
def test_add_output(self, data_mock):
data_mock.configure_mock(
annotation={'output.fastq': {'type': 'basic:file:', 'value': {'file': 'reads.fq'}},
'output.fasta': {'type': 'basic:file:', 'value': {'file': 'genome.fa'}}}
)
files_list = Data._files_dirs(data_mock, 'file', field_name="output.fastq")
self.assertEqual(files_list, ['reads.fq'])
files_list = Data._files_dirs(data_mock, 'file', field_name="fastq")
self.assertEqual(files_list, ['reads.fq'])
def test_collections(self):
resolwe_mock = MagicMock(**{
'collection.filter.return_value': ['collection_1', 'collection_2']
})
data = Data(id=1, resolwe=resolwe_mock)
self.assertEqual(data.collections, ['collection_1', 'collection_2'])
# cache is cleared at update
data.collections = MagicMock()
data.update()
self.assertEqual(data.collections.clear_cache.call_count, 1)
def test_files(self, data_mock):
data_annotation = {
'output.list': {
'value': [{
'file': "element.gz"
}],
'type': 'list:basic:file:'
},
'output.fastq': {
'value': {
'file': "file.fastq.gz"
},
'type': 'basic:file:fastq'
},
'output.fastq_archive': {
'value': {
'file': "archive.gz"
},
'type': 'basic:file:'
},
'input.fastq_url': {
'value': {
'file': "blah"
},
'type': 'basic:url:'
},
'input.blah': {
'value': "blah.gz",
'type': 'basic:file:'
}
}
bad_data_annotation = {
'output.list': {
'value': [{
'no_file_field_here': "element.gz"
}],
'type': 'list:basic:file:'
},
}
data_mock.configure_mock(annotation=data_annotation)
file_list = Data.files(data_mock)
self.assertEqual(set(file_list),
set(['element.gz', 'archive.gz', 'file.fastq.gz']))
file_list = Data.files(data_mock, file_name='element.gz')
self.assertEqual(file_list, ['element.gz'])
file_list = Data.files(data_mock, field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
data_mock.configure_mock(annotation=bad_data_annotation)
message = r"Item .* does not contain 'file' key."
with six.assertRaisesRegex(self, KeyError, message):
Data.files(data_mock)
def test_data_types(self):
resolwe = MagicMock()
data1 = Data(resolwe=resolwe, id=1)
data1._process = Process(resolwe=resolwe,
type="data:reads:fastq:single:")
collection = Collection(resolwe=resolwe, id=1)
collection._data = [data1]
types = collection.data_types()
self.assertEqual(types, ["data:reads:fastq:single:"])
def test_download_ok(self, data_mock):
data_mock.configure_mock(id=123, **{'resolwe': MagicMock()})
data_mock.configure_mock(
**{'files.return_value': ['file1.txt', 'file2.fq.gz']})
Data.download(data_mock)
data_mock.resolwe._download_files.assert_called_once_with(
[u'123/file1.txt', u'123/file2.fq.gz'], None)
data_mock.reset_mock()
Data.download(data_mock, download_dir="/some/path/")
data_mock.resolwe._download_files.assert_called_once_with(
[u'123/file1.txt', u'123/file2.fq.gz'], '/some/path/')
def test_children(self):
# Data with no id should fail.
data = Data(id=None, resolwe=MagicMock())
with self.assertRaisesRegex(ValueError, 'Instance must be saved *'):
data.children # pylint: disable=pointless-statement
# Core functionality should be checked with e2e tests.
# Check that cache is cleared at update.
data = Data(id=42, resolwe=MagicMock())
data._children = 'foo'
data.update()
self.assertEqual(data._children, None)
def test_download_ok(self, data_mock):
data_mock.configure_mock(id=123, **{"resolwe": MagicMock()})
data_mock.configure_mock(**{
"files.return_value": ["file1.txt", "file2.fq.gz"],
})
Data.download(data_mock)
data_mock.resolwe._download_files.assert_called_once_with(
["123/file1.txt", "123/file2.fq.gz"], None)
data_mock.reset_mock()
Data.download(data_mock, download_dir="/some/path/")
data_mock.resolwe._download_files.assert_called_once_with(
["123/file1.txt", "123/file2.fq.gz"], "/some/path/")
def test_files(self, data_mock):
data_annotation = {
'output.fastq': {'value': {'file': "file.fastq.gz"}, 'type': 'basic:file:fastq'},
'output.fastq_archive': {'value': {'file': "archive.gz"}, 'type': 'basic:file:'},
'input.fastq_url': {'value': {'file': "blah"}, 'type': 'basic:url:'},
'input.blah': {'value': "blah.gz", 'type': 'basic:file:'}
}
data_mock.configure_mock(annotation=data_annotation)
file_list = Data.files(data_mock)
self.assertEqual(set(file_list), set(['archive.gz', 'file.fastq.gz']))
file_list = Data.files(data_mock, file_name='archive.gz')
self.assertEqual(file_list, ['archive.gz'])
file_list = Data.files(data_mock, field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
def test_sample(self):
data = Data(id=1, resolwe=MagicMock())
data.resolwe.sample.filter = MagicMock(return_value=[])
self.assertEqual(data.sample, None)
data.resolwe.sample.filter = MagicMock(return_value=['sample'])
self.assertEqual(data.sample, 'sample')
# test caching
self.assertEqual(data.sample, 'sample')
self.assertEqual(data.resolwe.sample.filter.call_count, 1)
# cache is cleared at update
data._sample = 'sample'
data.update()
self.assertEqual(data._sample, None)
def test_add_output(self, data_mock):
data_mock.configure_mock(
output={
'fastq': {'file': 'reads.fq'},
'fasta': {'file': 'genome.fa'},
},
process_output_schema=[
{'type': 'basic:file:', 'name': 'fastq', 'label': 'FASTQ'},
{'type': 'basic:file:', 'name': 'fasta', 'label': 'FASTA'},
],
)
files_list = Data._files_dirs(data_mock, 'file', field_name="output.fastq")
self.assertEqual(files_list, ['reads.fq'])
files_list = Data._files_dirs(data_mock, 'file', field_name="fastq")
self.assertEqual(files_list, ['reads.fq'])
def test_descriptor_schema(self):
data = Data(id=1, resolwe=MagicMock())
data._descriptor_schema = 1
# test getting descriptor schema attribute
data.resolwe.descriptor_schema.get = MagicMock(return_value='descriptor_schema')
self.assertEqual(data.descriptor_schema, 'descriptor_schema')
_, get_kwargs = data.resolwe.descriptor_schema.get.call_args
self.assertEqual(get_kwargs['id'], 1)
# descriptor schema is not set
data._descriptor_schema = None
self.assertEqual(data.descriptor_schema, None)
# cache is cleared at update
data._hydrated_descriptor_schema = 'descriptor_schema'
data.update()
self.assertEqual(data._hydrated_descriptor_schema, None)
# new data object
data = Data(resolwe=MagicMock())
data.descriptor_schema = 'my-schema'
self.assertEqual(data._descriptor_schema, 'my-schema')
data.resolwe.descriptor_schema.get = MagicMock(return_value='descriptor_schema')
self.assertEqual(data.descriptor_schema, 'descriptor_schema')
_, get_kwargs = data.resolwe.descriptor_schema.get.call_args
self.assertEqual(get_kwargs['slug'], 'my-schema')
# hidrated descriptor schema
descriptor_schema = {
"slug": "test-schema",
"name": "Test schema",
"version": "1.0.0",
"schema": [
{
"default": "56G",
"type": "basic:string:",
"name": "description",
"label": "Object description"
}
],
"id": 1,
}
data = Data(id=1, descriptor_schema=descriptor_schema, resolwe=MagicMock())
self.assertTrue(isinstance(data.descriptor_schema, DescriptorSchema))
# pylint: disable=no-member
self.assertEqual(data.descriptor_schema.slug, 'test-schema')
def test_update_fields(self, data_mock):
payload = {
'id': 42,
'slug': 'foo',
'process_input_schema': [
{
'name': "x",
'type': "basic:integer:",
'label': "Input X",
},
{
'name': "group_input",
'group': [
{
'name': "y1",
'type': "basic:string:",
'label': "Input Y1",
},
],
}
],
'input': {
'x': 123,
'group_input': {
'y1': 'bar',
},
},
}
Data._update_fields(data_mock, payload)
expected = {
'input.x': {
'name': 'x',
'type': 'basic:integer:',
'label': 'Input X',
'value': 123
},
'input.group_input.y1': {
'name': 'y1',
'type': 'basic:string:',
'label': 'Input Y1',
'value': 'bar'
}
}
self.assertEqual(data_mock.annotation, expected)
def test_flatten_field(self, data_mock):
input_ = [{'src': "abc"}]
process_input_schema = [{'name': "src", 'type': "x", 'label': "y"}]
flat = Data._flatten_field(data_mock,
input_,
process_input_schema,
"p")
expected = {u'p.src': {u'type': 'x', u'name': 'src', u'value': None, u'label': 'y'}}
self.assertEqual(flat, expected)
def test_sample(self):
data = Data(id=1, resolwe=MagicMock())
data.api(data.id).get = MagicMock(return_value={'entities': [5]})
data.resolwe.sample.get = MagicMock(
return_value=Sample(data.resolwe, **{'id': 5, 'name': 'XYZ'}))
self.assertEqual(data.sample.id, 5)
self.assertEqual(data.sample.name, 'XYZ')
# test caching
self.assertEqual(data.sample.id, 5)
self.assertEqual(data.resolwe.sample.get.call_count, 1)
# cache is cleared at update
data.update()
self.assertEqual(data._sample, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.sample
def test_collection(self):
data = Data(resolwe=MagicMock(),
id=1,
collection={
"id": 5,
"name": "XYZ"
})
# test getting collections attribute
self.assertEqual(data.collection.id, 5)
self.assertEqual(data.collection.name, "XYZ")
def test_presample(self):
data = Data(id=1, resolwe=MagicMock())
data.resolwe.presample.filter = MagicMock(return_value=[])
self.assertEqual(data.presample, None)
data.resolwe.presample.filter = MagicMock(return_value=['presample'])
self.assertEqual(data.presample, 'presample')
# don't overwrite existing presample
data.resolwe.presample.filter = MagicMock(return_value=['presample 2'])
self.assertEqual(data.presample, 'presample')
# remove presample when gone
data.resolwe.presample.filter = MagicMock(return_value=[])
self.assertEqual(data.presample, None)
# cache is cleared at update
data._presample = 'presample'
data.update()
self.assertEqual(data._presample, None)
def test_descriptor_schema(self):
resolwe = MagicMock()
data = Data(id=1, resolwe=resolwe)
data._descriptor_schema = DescriptorSchema(resolwe=resolwe, id=2)
# test getting descriptor schema attribute
self.assertEqual(data.descriptor_schema.id, 2) # pylint: disable=no-member
# descriptor schema is not set
data._descriptor_schema = None
self.assertEqual(data.descriptor_schema, None)
# hydrated descriptor schema
descriptor_schema = {
"slug": "test-schema",
"name": "Test schema",
"version": "1.0.0",
"schema": [
{
"default": "56G",
"type": "basic:string:",
"name": "description",
"label": "Object description"
}
],
"id": 1,
}
data = Data(id=1, descriptor_schema=descriptor_schema, resolwe=MagicMock())
self.assertTrue(isinstance(data.descriptor_schema, DescriptorSchema))
# pylint: disable=no-member,unsubscriptable-object
self.assertEqual(data.descriptor_schema.slug, 'test-schema')
self.assertEqual(data.descriptor_schema.schema[0]['label'], 'Object description')
def test_collections(self):
data = Data(id=1, resolwe=MagicMock())
# test getting collections attribute
data.api(data.id).get = MagicMock(return_value={'collections': [5]})
data.resolwe.collection.filter = MagicMock(return_value=[
Collection(data.resolwe, **{'id': 5, 'name': 'XYZ'})
])
self.assertEqual(len(data.collections), 1)
self.assertEqual(data.collections[0].id, 5)
self.assertEqual(data.collections[0].name, 'XYZ')
# test caching collections attribute
self.assertEqual(len(data.collections), 1)
self.assertEqual(data.resolwe.collection.filter.call_count, 1)
# cache is cleared at update
data.update()
self.assertEqual(data._collections, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.collections
def test_stdout_ok(self, data_mock, urljoin_mock, requests_mock):
# Configure mocks:
data_mock.configure_mock(id=123, resolwe=MagicMock(url="a", auth="b"))
urljoin_mock.return_value = "some_url"
# If response.ok = True:
response = MagicMock(ok=True, **{'iter_content.return_value': [b"abc", b"def"]})
requests_mock.configure_mock(**{'get.return_value': response})
out = Data.stdout(data_mock)
self.assertEqual(out, "abcdef")
urljoin_mock.assert_called_once_with("a", 'data/123/stdout.txt')
requests_mock.get.assert_called_once_with("some_url", stream=True, auth="b")
# If response.ok = False:
response = MagicMock(ok=False)
requests_mock.configure_mock(**{'get.return_value': response})
out = Data.stdout(data_mock)
self.assertEqual(response.raise_for_status.call_count, 1)
def test_add_output(self, data_mock):
data_mock.configure_mock(
annotation={
'output.fastq': {
'type': 'basic:file:',
'value': {
'file': 'reads.fq'
}
},
'output.fasta': {
'type': 'basic:file:',
'value': {
'file': 'genome.fa'
}
}
})
files_list = Data.files(data_mock, field_name="output.fastq")
self.assertEqual(files_list, ['reads.fq'])
files_list = Data.files(data_mock, field_name="fastq")
self.assertEqual(files_list, ['reads.fq'])
def test_children(self):
# Data with no id should fail.
data = Data(id=None, resolwe=MagicMock())
with self.assertRaisesRegex(ValueError, "Instance must be saved *"):
data.children
# Core functionality should be checked with e2e tests.
# Check that cache is cleared at update.
data = Data(id=42, resolwe=MagicMock())
data._children = "foo"
data.update()
self.assertEqual(data._children, None)
def test_children(self):
# Data with no id should fail.
data = Data(id=None, resolwe=MagicMock())
with self.assertRaisesRegex(ValueError, 'Instance must be saved *'):
data.children # pylint: disable=pointless-statement
# Core functionality should be checked with e2e tests.
# Check that cache is cleared at update.
data = Data(id=42, resolwe=MagicMock())
data._children = 'foo'
data.update()
self.assertEqual(data._children, None)
def test_flatten_field(self, data_mock):
input_ = [{'src': "abc"}]
process_input_schema = [{'name': "src", 'type': "x", 'label': "y"}]
flat = Data._flatten_field(data_mock, input_, process_input_schema,
"p")
expected = {
u'p.src': {
u'type': 'x',
u'name': 'src',
u'value': None,
u'label': 'y'
}
}
self.assertEqual(flat, expected)
def test_sample(self):
data = Data(id=1, resolwe=MagicMock())
data.api(data.id).get = MagicMock(return_value={'entities': [5]})
data.resolwe.sample.get = MagicMock(
return_value=Sample(data.resolwe, **{'id': 5, 'name': 'XYZ'}))
self.assertEqual(data.sample.id, 5)
self.assertEqual(data.sample.name, 'XYZ')
# test caching
self.assertEqual(data.sample.id, 5)
self.assertEqual(data.resolwe.sample.get.call_count, 1)
# cache is cleared at update
data.update()
self.assertEqual(data._sample, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.sample
def test_collections(self):
data = Data(id=1, resolwe=MagicMock())
# test getting collections attribute
data.resolwe.collection.filter = MagicMock(return_value=['collection'])
self.assertEqual(data.collections, ['collection'])
# test caching collections attribute
self.assertEqual(data.collections, ['collection'])
self.assertEqual(data.resolwe.collection.filter.call_count, 1)
# cache is cleared at update
data._collections = ['collection']
data.update()
self.assertEqual(data._collections, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.collections
def test_sample(self):
data = Data(id=1, resolwe=MagicMock())
data.resolwe.sample.filter = MagicMock(return_value=[])
self.assertEqual(data.sample, None)
data.resolwe.sample.filter = MagicMock(return_value=['sample'])
self.assertEqual(data.sample, 'sample')
# test caching
self.assertEqual(data.sample, 'sample')
self.assertEqual(data.resolwe.sample.filter.call_count, 1)
# cache is cleared at update
data._sample = 'sample'
data.update()
self.assertEqual(data._sample, None)
# raising error if data object is not saved
data.id = None
with self.assertRaises(ValueError):
_ = data.sample
def test_print_annotation(self, data_mock):
with six.assertRaisesRegex(self, NotImplementedError, ""):
Data.print_annotation(data_mock)
def test_files(self):
resolwe = MagicMock()
data = Data(id=123, resolwe=resolwe)
data._get_dir_files = MagicMock(
side_effect=[['first_dir/file1.txt'], ['fastq_dir/file2.txt']])
data.output = {
'list': [{'file': "element.gz"}],
'dir_list': [{'dir': "first_dir"}],
'fastq': {'file': "file.fastq.gz"},
'fastq_archive': {'file': "archive.gz"},
'fastq_dir': {'dir': "fastq_dir"},
}
data.process = Process(resolwe=resolwe, output_schema=[
{'name': 'list', 'label': 'List', 'type': 'list:basic:file:'},
{'name': 'dir_list', 'label': 'Dir_list', 'type': 'list:basic:dir:'},
{'name': 'fastq', 'label': 'Fastq', 'type': 'basic:file:fastq:'},
{'name': 'fastq_archive', 'label': 'Fastq_archive', 'type': 'basic:file:'},
{'name': 'fastq_dir', 'label': 'Fastq_dir', 'type': 'basic:dir:'},
])
file_list = data.files()
self.assertCountEqual(file_list, [
'element.gz',
'archive.gz',
'file.fastq.gz',
'first_dir/file1.txt',
'fastq_dir/file2.txt'
])
file_list = data.files(file_name='element.gz')
self.assertEqual(file_list, ['element.gz'])
file_list = data.files(field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
data.output = {
'list': [{'no_file_field_here': "element.gz"}],
}
data.process.output_schema = [
{'name': 'list', 'label': 'List', 'type': 'list:basic:file:'},
]
with self.assertRaisesRegex(KeyError, "does not contain 'file' key."):
data.files()
data = Data(resolwe=MagicMock(), id=None)
with self.assertRaisesRegex(ValueError, "must be saved before"):
data.files()
def test_download_fail(self, data_mock):
message = "Only one of file_name or field_name may be given."
with six.assertRaisesRegex(self, ValueError, message):
Data.download(data_mock, file_name="a", field_name="b")
def test_update_fields(self, data_mock):
Data._update_fields(data_mock, DATA_SAMPLE[0])
self.assertEqual(data_mock._flatten_field.call_count, 2)
def test_files(self):
data = Data(id=123, resolwe=MagicMock())
data._get_dir_files = MagicMock(
side_effect=[['first_dir/file1.txt'], ['fastq_dir/file2.txt']])
data.annotation = {
'output.list': {
'value': [{
'file': "element.gz"
}],
'type': 'list:basic:file:'
},
'output.dir_list': {
'value': [{
'dir': "first_dir"
}],
'type': 'list:basic:dir:'
},
'output.fastq': {
'value': {
'file': "file.fastq.gz"
},
'type': 'basic:file:fastq'
},
'output.fastq_archive': {
'value': {
'file': "archive.gz"
},
'type': 'basic:file:'
},
'output.fastq_dir': {
'value': {
'dir': "fastq_dir"
},
'type': 'basic:dir:'
},
'input.fastq_url': {
'value': {
'file': "blah"
},
'type': 'basic:url:'
},
'input.blah': {
'value': "blah.gz",
'type': 'basic:file:'
}
}
file_list = data.files()
six.assertCountEqual(self, file_list, [
'element.gz', 'archive.gz', 'file.fastq.gz', 'first_dir/file1.txt',
'fastq_dir/file2.txt'
])
file_list = data.files(file_name='element.gz')
self.assertEqual(file_list, ['element.gz'])
file_list = data.files(field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
data.annotation = {
'output.list': {
'value': [{
'no_file_field_here': "element.gz"
}],
'type': 'list:basic:file:'
},
}
with six.assertRaisesRegex(self, KeyError,
"does not contain 'file' key."):
data.files()
data = Data(resolwe=MagicMock(), id=None)
with six.assertRaisesRegex(self, ValueError, "must be saved before"):
data.files()
def test_init(self, data_mock, log_mock):
data_mock.configure_mock(endpoint="a string")
Data.__init__(data_mock, id=1, resolwe=MagicMock())
log_mock.getLogger.assert_called_once_with('resdk.resources.data')
def test_sample(self):
data = Data(resolwe=MagicMock(), id=1)
data._original_values = {'entity': {'id': 5, 'name': 'XYZ'}}
self.assertEqual(data.sample.id, 5)
self.assertEqual(data.sample.name, 'XYZ')
def test_no_sample(self):
data = Data(id=1, resolwe=MagicMock())
# Data object does not belong to any sample:
data.api(data.id).get = MagicMock(return_value={'entities': []})
self.assertEqual(data.sample, None)
def test_print_annotation(self, data_mock):
with six.assertRaisesRegex(self, NotImplementedError, ""):
Data.print_annotation(data_mock)
def test_descriptor_schema(self):
data = Data(id=1, resolwe=MagicMock())
data._descriptor_schema = 1
# test getting descriptor schema attribute
data.resolwe.descriptor_schema.get = MagicMock(
return_value='descriptor_schema')
self.assertEqual(data.descriptor_schema, 'descriptor_schema')
_, get_kwargs = data.resolwe.descriptor_schema.get.call_args
self.assertEqual(get_kwargs['id'], 1)
# descriptor schema is not set
data._descriptor_schema = None
self.assertEqual(data.descriptor_schema, None)
# cache is cleared at update
data._hydrated_descriptor_schema = 'descriptor_schema'
data.update()
self.assertEqual(data._hydrated_descriptor_schema, None)
# new data object
data = Data(resolwe=MagicMock())
data.descriptor_schema = 'my-schema'
self.assertEqual(data._descriptor_schema, 'my-schema')
data.resolwe.descriptor_schema.get = MagicMock(
return_value='descriptor_schema')
self.assertEqual(data.descriptor_schema, 'descriptor_schema')
_, get_kwargs = data.resolwe.descriptor_schema.get.call_args
self.assertEqual(get_kwargs['slug'], 'my-schema')
# hidrated descriptor schema
descriptor_schema = {
"slug":
"test-schema",
"name":
"Test schema",
"version":
"1.0.0",
"schema": [{
"default": "56G",
"type": "basic:string:",
"name": "description",
"label": "Object description"
}],
"id":
1,
}
data = Data(id=1,
descriptor_schema=descriptor_schema,
resolwe=MagicMock())
self.assertTrue(isinstance(data.descriptor_schema, DescriptorSchema))
# pylint: disable=no-member
self.assertEqual(data.descriptor_schema.slug, 'test-schema')
def test_files(self):
data = Data(id=123, resolwe=MagicMock())
data._get_dir_files = MagicMock(
side_effect=[['first_dir/file1.txt'], ['fastq_dir/file2.txt']])
data.output = {
'list': [{'file': "element.gz"}],
'dir_list': [{'dir': "first_dir"}],
'fastq': {'file': "file.fastq.gz"},
'fastq_archive': {'file': "archive.gz"},
'fastq_dir': {'dir': "fastq_dir"},
}
data.process_output_schema = [
{'name': 'list', 'label': 'List', 'type': 'list:basic:file:'},
{'name': 'dir_list', 'label': 'Dir_list', 'type': 'list:basic:dir:'},
{'name': 'fastq', 'label': 'Fastq', 'type': 'basic:file:fastq:'},
{'name': 'fastq_archive', 'label': 'Fastq_archive', 'type': 'basic:file:'},
{'name': 'fastq_dir', 'label': 'Fastq_dir', 'type': 'basic:dir:'},
]
file_list = data.files()
self.assertCountEqual(file_list, [
'element.gz',
'archive.gz',
'file.fastq.gz',
'first_dir/file1.txt',
'fastq_dir/file2.txt'
])
file_list = data.files(file_name='element.gz')
self.assertEqual(file_list, ['element.gz'])
file_list = data.files(field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
data.output = {
'list': [{'no_file_field_here': "element.gz"}],
}
data.process_output_schema = [
{'name': 'list', 'label': 'List', 'type': 'list:basic:file:'},
]
with self.assertRaisesRegex(KeyError, "does not contain 'file' key."):
data.files()
data = Data(resolwe=MagicMock(), id=None)
with self.assertRaisesRegex(ValueError, "must be saved before"):
data.files()
def test_download_fail(self, data_mock):
message = "Only one of file_name or field_name may be given."
with six.assertRaisesRegex(self, ValueError, message):
Data.download(data_mock, file_name="a", field_name="b")
def test_collection(self):
data = Data(resolwe=MagicMock(), id=1, collection={'id': 5, 'name': 'XYZ'})
# test getting collections attribute
self.assertEqual(data.collection.id, 5)
self.assertEqual(data.collection.name, 'XYZ')
def test_update_fields(self, data_mock):
Data._update_fields(data_mock, DATA_SAMPLE[0])
self.assertEqual(data_mock._flatten_field.call_count, 2)
def test_files(self):
data = Data(id=123, resolwe=MagicMock())
data._get_dir_files = MagicMock(
side_effect=[['first_dir/file1.txt'], ['fastq_dir/file2.txt']])
data.annotation = {
'output.list': {'value': [{'file': "element.gz"}], 'type': 'list:basic:file:'},
'output.dir_list': {'value': [{'dir': "first_dir"}], 'type': 'list:basic:dir:'},
'output.fastq': {'value': {'file': "file.fastq.gz"}, 'type': 'basic:file:fastq'},
'output.fastq_archive': {'value': {'file': "archive.gz"}, 'type': 'basic:file:'},
'output.fastq_dir': {'value': {'dir': "fastq_dir"}, 'type': 'basic:dir:'},
'input.fastq_url': {'value': {'file': "blah"}, 'type': 'basic:url:'},
'input.blah': {'value': "blah.gz", 'type': 'basic:file:'}
}
file_list = data.files()
six.assertCountEqual(self, file_list, [
'element.gz',
'archive.gz',
'file.fastq.gz',
'first_dir/file1.txt',
'fastq_dir/file2.txt'
])
file_list = data.files(file_name='element.gz')
self.assertEqual(file_list, ['element.gz'])
file_list = data.files(field_name='output.fastq')
self.assertEqual(file_list, ['file.fastq.gz'])
data.annotation = {
'output.list': {'value': [{'no_file_field_here': "element.gz"}],
'type': 'list:basic:file:'},
}
with six.assertRaisesRegex(self, KeyError, "does not contain 'file' key."):
data.files()
data = Data(resolwe=MagicMock(), id=None)
with six.assertRaisesRegex(self, ValueError, "must be saved before"):
data.files()
