class DatasetResourceTest(MyTardisResourceTestCase): def setUp(self): super(DatasetResourceTest, self).setUp() self.ds_no_instrument = Dataset() self.ds_no_instrument.description = "Dataset no instrument" self.ds_no_instrument.save() self.ds_no_instrument.experiments.add(self.testexp) self.ds_with_instrument = Dataset() self.ds_with_instrument.description = "Dataset with instrument" self.ds_with_instrument.instrument = self.testinstrument self.ds_with_instrument.save() self.ds_with_instrument.experiments.add(self.testexp) def test_get_dataset_no_instrument(self): uri = '/api/v1/dataset/?description=%s' \ % urllib.quote(self.ds_no_instrument.description) output = self.api_client.get(uri, authentication=self.get_credentials()) returned_data = json.loads(output.content) self.assertEqual(returned_data['meta']['total_count'], 1) returned_object = returned_data['objects'][0] self.assertTrue('description' in returned_object) self.assertEqual(returned_object['description'], self.ds_no_instrument.description) self.assertTrue('instrument' in returned_object) self.assertIsNone(returned_object['instrument']) def test_get_dataset_with_instrument(self): uri = '/api/v1/dataset/?description=%s' \ % urllib.quote(self.ds_with_instrument.description) output = self.api_client.get(uri, authentication=self.get_credentials()) returned_data = json.loads(output.content) self.assertEqual(returned_data['meta']['total_count'], 1) returned_object = returned_data['objects'][0] self.assertTrue('description' in returned_object) self.assertEqual(returned_object['description'], self.ds_with_instrument.description) self.assertTrue('instrument' in returned_object) self.assertTrue('id' in returned_object['instrument']) self.assertEqual(returned_object['instrument']['id'], self.testinstrument.id) def test_post_dataset(self): exp_id = Experiment.objects.first().id post_data = { "description": "api test dataset", "experiments": [ "/api/v1/experiment/%d/" % exp_id, ], "immutable": False } dataset_count = Dataset.objects.count() self.assertHttpCreated( self.api_client.post('/api/v1/dataset/', data=post_data, authentication=self.get_credentials())) self.assertEqual(dataset_count + 1, Dataset.objects.count())
def _make_data_set(exp_id): # make datafile exp = Experiment.objects.get(id=exp_id) print "Experiment ----", exp dataset = Dataset(description="HRMC results") dataset.save() dataset.experiments.add(exp) dataset.save() return dataset
class DatasetResourceTest(MyTardisResourceTestCase): def setUp(self): super(DatasetResourceTest, self).setUp() self.ds_no_instrument = Dataset() self.ds_no_instrument.description = "Dataset no instrument" self.ds_no_instrument.save() self.ds_no_instrument.experiments.add(self.testexp) self.ds_with_instrument = Dataset() self.ds_with_instrument.description = "Dataset with instrument" self.ds_with_instrument.instrument = self.testinstrument self.ds_with_instrument.save() self.ds_with_instrument.experiments.add(self.testexp) def test_get_dataset_no_instrument(self): uri = '/api/v1/dataset/?description=%s' \ % urllib.quote(self.ds_no_instrument.description) output = self.api_client.get(uri, authentication=self.get_credentials()) returned_data = json.loads(output.content) self.assertEqual(returned_data['meta']['total_count'], 1) returned_object = returned_data['objects'][0] self.assertTrue('description' in returned_object) self.assertEqual(returned_object['description'], self.ds_no_instrument.description) self.assertTrue('instrument' in returned_object) self.assertIsNone(returned_object['instrument']) def test_get_dataset_with_instrument(self): uri = '/api/v1/dataset/?description=%s' \ % urllib.quote(self.ds_with_instrument.description) output = self.api_client.get(uri, authentication=self.get_credentials()) returned_data = json.loads(output.content) self.assertEqual(returned_data['meta']['total_count'], 1) returned_object = returned_data['objects'][0] self.assertTrue('description' in returned_object) self.assertEqual(returned_object['description'], self.ds_with_instrument.description) self.assertTrue('instrument' in returned_object) self.assertTrue('id' in returned_object['instrument']) self.assertEqual(returned_object['instrument']['id'], self.testinstrument.id) def test_post_dataset(self): exp_id = Experiment.objects.first().id post_data = { "description": "api test dataset", "experiments": [ "/api/v1/experiment/%d/" % exp_id, ], "immutable": False} dataset_count = Dataset.objects.count() self.assertHttpCreated(self.api_client.post( '/api/v1/dataset/', data=post_data, authentication=self.get_credentials())) self.assertEqual(dataset_count + 1, Dataset.objects.count())
class DataFileResourceTest(MyTardisResourceTestCase): def setUp(self): super(DataFileResourceTest, self).setUp() self.django_client = Client() self.django_client.login(username=self.username, password=self.password) self.testds = Dataset() self.testds.description = "test dataset" self.testds.save() self.testds.experiments.add(self.testexp) df_schema_name = "http://datafileshop.com/" self.test_schema = Schema(namespace=df_schema_name, type=Schema.DATAFILE) self.test_schema.save() self.test_parname1 = ParameterName(schema=self.test_schema, name="fileparameter1", data_type=ParameterName.STRING) self.test_parname1.save() self.test_parname2 = ParameterName(schema=self.test_schema, name="fileparameter2", data_type=ParameterName.NUMERIC) self.test_parname2.save() def test_post_single_file(self): ds_id = Dataset.objects.first().id post_data = """{ "dataset": "/api/v1/dataset/%d/", "filename": "mytestfile.txt", "md5sum": "930e419034038dfad994f0d2e602146c", "size": "8", "mimetype": "text/plain", "parameter_sets": [{ "schema": "http://datafileshop.com/", "parameters": [{ "name": "fileparameter1", "value": "123" }, { "name": "fileparameter2", "value": "123" }] }] }""" % ds_id post_file = tempfile.NamedTemporaryFile() file_content = "123test\n" post_file.write(file_content) post_file.flush() post_file.seek(0) datafile_count = DataFile.objects.count() dfo_count = DataFileObject.objects.count() self.assertHttpCreated( self.django_client.post('/api/v1/dataset_file/', data={ "json_data": post_data, "attached_file": post_file })) self.assertEqual(datafile_count + 1, DataFile.objects.count()) self.assertEqual(dfo_count + 1, DataFileObject.objects.count()) new_file = DataFile.objects.order_by('-pk')[0] self.assertEqual(file_content, new_file.get_file().read()) def test_shared_fs_single_file(self): pass def test_shared_fs_many_files(self): # noqa # TODO too complex ''' tests sending many files with known permanent location (useful for Australian Synchrotron ingestions) ''' files = [{'content': 'test123\n'}, {'content': 'test246\n'}] from django.conf import settings for file_dict in files: post_file = tempfile.NamedTemporaryFile( dir=settings.DEFAULT_STORAGE_BASE_DIR) file_dict['filename'] = os.path.basename(post_file.name) file_dict['full_path'] = post_file.name post_file.write(file_dict['content']) post_file.flush() post_file.seek(0) file_dict['object'] = post_file def clumsily_build_uri(res_type, dataset): return '/api/v1/%s/%d/' % (res_type, dataset.id) def md5sum(filename): import hashlib md5 = hashlib.md5() with open(filename, 'rb') as f: for chunk in iter(lambda: f.read(128 * md5.block_size), b''): md5.update(chunk) return md5.hexdigest() def guess_mime(filename): import magic mime = magic.Magic(mime=True) return mime.from_file(filename) json_data = {"objects": []} for file_dict in files: file_json = { 'dataset': clumsily_build_uri('dataset', self.testds), 'filename': os.path.basename(file_dict['filename']), 'md5sum': md5sum(file_dict['full_path']), 'size': os.path.getsize(file_dict['full_path']), 'mimetype': guess_mime(file_dict['full_path']), 'replicas': [{ 'url': file_dict['filename'], 'location': 'default', 'protocol': 'file', }], } json_data['objects'].append(file_json) datafile_count = DataFile.objects.count() dfo_count = DataFileObject.objects.count() self.assertHttpAccepted( self.api_client.patch('/api/v1/dataset_file/', data=json_data, authentication=self.get_credentials())) self.assertEqual(datafile_count + 2, DataFile.objects.count()) self.assertEqual(dfo_count + 2, DataFileObject.objects.count()) # fake-verify DFO, so we can access the file: for newdfo in DataFileObject.objects.order_by('-pk')[0:2]: newdfo.verified = True newdfo.save() for sent_file, new_file in zip(reversed(files), DataFile.objects.order_by('-pk')[0:2]): self.assertEqual(sent_file['content'], new_file.get_file().read())
class DataFileResourceTest(MyTardisResourceTestCase): def setUp(self): super(DataFileResourceTest, self).setUp() self.django_client = Client() self.django_client.login(username=self.username, password=self.password) self.testds = Dataset() self.testds.description = "test dataset" self.testds.save() self.testds.experiments.add(self.testexp) df_schema_name = "http://datafileshop.com/" self.test_schema = Schema(namespace=df_schema_name, type=Schema.DATAFILE) self.test_schema.save() self.test_parname1 = ParameterName(schema=self.test_schema, name="fileparameter1", data_type=ParameterName.STRING) self.test_parname1.save() self.test_parname2 = ParameterName(schema=self.test_schema, name="fileparameter2", data_type=ParameterName.NUMERIC) self.test_parname2.save() def test_post_single_file(self): post_data = """{ "dataset": "/api/v1/dataset/1/", "filename": "mytestfile.txt", "md5sum": "930e419034038dfad994f0d2e602146c", "size": "8", "mimetype": "text/plain", "parameter_sets": [{ "schema": "http://datafileshop.com/", "parameters": [{ "name": "fileparameter1", "value": "123" }, { "name": "fileparameter2", "value": "123" }] }] }""" post_file = tempfile.NamedTemporaryFile() file_content = "123test\n" post_file.write(file_content) post_file.flush() post_file.seek(0) datafile_count = DataFile.objects.count() dfo_count = DataFileObject.objects.count() self.assertHttpCreated(self.django_client.post( '/api/v1/dataset_file/', data={"json_data": post_data, "attached_file": post_file})) self.assertEqual(datafile_count + 1, DataFile.objects.count()) self.assertEqual(dfo_count + 1, DataFileObject.objects.count()) new_file = DataFile.objects.order_by('-pk')[0] self.assertEqual(file_content, new_file.get_file().read()) def test_shared_fs_single_file(self): pass def test_shared_fs_many_files(self): # noqa # TODO too complex ''' tests sending many files with known permanent location (useful for Australian Synchrotron ingestions) ''' files = [{'content': 'test123\n'}, {'content': 'test246\n'}] from django.conf import settings for file_dict in files: post_file = tempfile.NamedTemporaryFile( dir=settings.DEFAULT_STORAGE_BASE_DIR) file_dict['filename'] = os.path.basename(post_file.name) file_dict['full_path'] = post_file.name post_file.write(file_dict['content']) post_file.flush() post_file.seek(0) file_dict['object'] = post_file def clumsily_build_uri(res_type, dataset): return '/api/v1/%s/%d/' % (res_type, dataset.id) def md5sum(filename): import hashlib md5 = hashlib.md5() with open(filename, 'rb') as f: for chunk in iter(lambda: f.read(128*md5.block_size), b''): md5.update(chunk) return md5.hexdigest() def guess_mime(filename): import magic mime = magic.Magic(mime=True) return mime.from_file(filename) json_data = {"objects": []} for file_dict in files: file_json = { 'dataset': clumsily_build_uri('dataset', self.testds), 'filename': os.path.basename(file_dict['filename']), 'md5sum': md5sum(file_dict['full_path']), 'size': os.path.getsize(file_dict['full_path']), 'mimetype': guess_mime(file_dict['full_path']), 'replicas': [{ 'url': file_dict['filename'], 'location': 'default', 'protocol': 'file', }], } json_data['objects'].append(file_json) datafile_count = DataFile.objects.count() dfo_count = DataFileObject.objects.count() self.assertHttpAccepted(self.api_client.patch( '/api/v1/dataset_file/', data=json_data, authentication=self.get_credentials())) self.assertEqual(datafile_count + 2, DataFile.objects.count()) self.assertEqual(dfo_count + 2, DataFileObject.objects.count()) # fake-verify DFO, so we can access the file: for newdfo in DataFileObject.objects.order_by('-pk')[0:2]: newdfo.verified = True newdfo.save() for sent_file, new_file in zip( reversed(files), DataFile.objects.order_by('-pk')[0:2]): self.assertEqual(sent_file['content'], new_file.get_file().read())