class DatasetResourceTest(MyTardisResourceTestCase):
def setUp(self):
super(DatasetResourceTest, self).setUp()
self.ds_no_instrument = Dataset()
self.ds_no_instrument.description = "Dataset no instrument"
self.ds_no_instrument.save()
self.ds_no_instrument.experiments.add(self.testexp)
self.ds_with_instrument = Dataset()
self.ds_with_instrument.description = "Dataset with instrument"
self.ds_with_instrument.instrument = self.testinstrument
self.ds_with_instrument.save()
self.ds_with_instrument.experiments.add(self.testexp)
def test_get_dataset_no_instrument(self):
uri = '/api/v1/dataset/?description=%s' \
% urllib.quote(self.ds_no_instrument.description)
output = self.api_client.get(uri,
authentication=self.get_credentials())
returned_data = json.loads(output.content)
self.assertEqual(returned_data['meta']['total_count'], 1)
returned_object = returned_data['objects'][0]
self.assertTrue('description' in returned_object)
self.assertEqual(returned_object['description'],
self.ds_no_instrument.description)
self.assertTrue('instrument' in returned_object)
self.assertIsNone(returned_object['instrument'])
def test_get_dataset_with_instrument(self):
uri = '/api/v1/dataset/?description=%s' \
% urllib.quote(self.ds_with_instrument.description)
output = self.api_client.get(uri,
authentication=self.get_credentials())
returned_data = json.loads(output.content)
self.assertEqual(returned_data['meta']['total_count'], 1)
returned_object = returned_data['objects'][0]
self.assertTrue('description' in returned_object)
self.assertEqual(returned_object['description'],
self.ds_with_instrument.description)
self.assertTrue('instrument' in returned_object)
self.assertTrue('id' in returned_object['instrument'])
self.assertEqual(returned_object['instrument']['id'],
self.testinstrument.id)
def test_post_dataset(self):
exp_id = Experiment.objects.first().id
post_data = {
"description": "api test dataset",
"experiments": [
"/api/v1/experiment/%d/" % exp_id,
],
"immutable": False
}
dataset_count = Dataset.objects.count()
self.assertHttpCreated(
self.api_client.post('/api/v1/dataset/',
data=post_data,
authentication=self.get_credentials()))
self.assertEqual(dataset_count + 1, Dataset.objects.count())
class DatasetResourceTest(MyTardisResourceTestCase):
def setUp(self):
super(DatasetResourceTest, self).setUp()
self.ds_no_instrument = Dataset()
self.ds_no_instrument.description = "Dataset no instrument"
self.ds_no_instrument.save()
self.ds_no_instrument.experiments.add(self.testexp)
self.ds_with_instrument = Dataset()
self.ds_with_instrument.description = "Dataset with instrument"
self.ds_with_instrument.instrument = self.testinstrument
self.ds_with_instrument.save()
self.ds_with_instrument.experiments.add(self.testexp)
def test_get_dataset_no_instrument(self):
uri = '/api/v1/dataset/?description=%s' \
% urllib.quote(self.ds_no_instrument.description)
output = self.api_client.get(uri,
authentication=self.get_credentials())
returned_data = json.loads(output.content)
self.assertEqual(returned_data['meta']['total_count'], 1)
returned_object = returned_data['objects'][0]
self.assertTrue('description' in returned_object)
self.assertEqual(returned_object['description'],
self.ds_no_instrument.description)
self.assertTrue('instrument' in returned_object)
self.assertIsNone(returned_object['instrument'])
def test_get_dataset_with_instrument(self):
uri = '/api/v1/dataset/?description=%s' \
% urllib.quote(self.ds_with_instrument.description)
output = self.api_client.get(uri,
authentication=self.get_credentials())
returned_data = json.loads(output.content)
self.assertEqual(returned_data['meta']['total_count'], 1)
returned_object = returned_data['objects'][0]
self.assertTrue('description' in returned_object)
self.assertEqual(returned_object['description'],
self.ds_with_instrument.description)
self.assertTrue('instrument' in returned_object)
self.assertTrue('id' in returned_object['instrument'])
self.assertEqual(returned_object['instrument']['id'],
self.testinstrument.id)
def test_post_dataset(self):
exp_id = Experiment.objects.first().id
post_data = {
"description": "api test dataset",
"experiments": [
"/api/v1/experiment/%d/" % exp_id,
],
"immutable": False}
dataset_count = Dataset.objects.count()
self.assertHttpCreated(self.api_client.post(
'/api/v1/dataset/',
data=post_data,
authentication=self.get_credentials()))
self.assertEqual(dataset_count + 1, Dataset.objects.count())
def setUp(self):
super(DatasetResourceTest, self).setUp()
self.ds_no_instrument = Dataset()
self.ds_no_instrument.description = "Dataset no instrument"
self.ds_no_instrument.save()
self.ds_no_instrument.experiments.add(self.testexp)
self.ds_with_instrument = Dataset()
self.ds_with_instrument.description = "Dataset with instrument"
self.ds_with_instrument.instrument = self.testinstrument
self.ds_with_instrument.save()
self.ds_with_instrument.experiments.add(self.testexp)
def setUp(self):
super(DatasetResourceTest, self).setUp()
self.ds_no_instrument = Dataset()
self.ds_no_instrument.description = "Dataset no instrument"
self.ds_no_instrument.save()
self.ds_no_instrument.experiments.add(self.testexp)
self.ds_with_instrument = Dataset()
self.ds_with_instrument.description = "Dataset with instrument"
self.ds_with_instrument.instrument = self.testinstrument
self.ds_with_instrument.save()
self.ds_with_instrument.experiments.add(self.testexp)
def setUp(self):
super(DataFileResourceTest, self).setUp()
self.django_client = Client()
self.django_client.login(username=self.username,
password=self.password)
self.testds = Dataset()
self.testds.description = "test dataset"
self.testds.save()
self.testds.experiments.add(self.testexp)
df_schema_name = "http://datafileshop.com/"
self.test_schema = Schema(namespace=df_schema_name,
type=Schema.DATAFILE)
self.test_schema.save()
self.test_parname1 = ParameterName(schema=self.test_schema,
name="fileparameter1",
data_type=ParameterName.STRING)
self.test_parname1.save()
self.test_parname2 = ParameterName(schema=self.test_schema,
name="fileparameter2",
data_type=ParameterName.NUMERIC)
self.test_parname2.save()
def _make_data_set(exp_id):
# make datafile
exp = Experiment.objects.get(id=exp_id)
print "Experiment ----", exp
dataset = Dataset(description="HRMC results")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
return dataset
def setUp(self):
super(DataFileResourceTest, self).setUp()
self.django_client = Client()
self.django_client.login(username=self.username,
password=self.password)
self.testds = Dataset()
self.testds.description = "test dataset"
self.testds.save()
self.testds.experiments.add(self.testexp)
df_schema_name = "http://datafileshop.com/"
self.test_schema = Schema(namespace=df_schema_name,
type=Schema.DATAFILE)
self.test_schema.save()
self.test_parname1 = ParameterName(schema=self.test_schema,
name="fileparameter1",
data_type=ParameterName.STRING)
self.test_parname1.save()
self.test_parname2 = ParameterName(schema=self.test_schema,
name="fileparameter2",
data_type=ParameterName.NUMERIC)
self.test_parname2.save()
def setUp(self):
super(DatasetResourceTest, self).setUp()
self.extra_instrument = Instrument()
self.extra_instrument = Instrument(name="Extra Instrument",
facility=self.testfacility)
self.extra_instrument.save()
self.ds_no_instrument = Dataset()
self.ds_no_instrument.description = "Dataset no instrument"
self.ds_no_instrument.save()
self.ds_no_instrument.experiments.add(self.testexp)
self.ds_with_instrument = Dataset()
self.ds_with_instrument.description = "Dataset with instrument"
self.ds_with_instrument.instrument = self.testinstrument
self.ds_with_instrument.save()
self.ds_with_instrument.experiments.add(self.testexp)
self.ds_with_instrument2 = Dataset()
self.ds_with_instrument2.description = "Dataset with a different instrument"
self.ds_with_instrument2.instrument = self.extra_instrument
self.ds_with_instrument2.save()
self.ds_with_instrument2.experiments.add(self.testexp)
class DataFileResourceTest(MyTardisResourceTestCase):
def setUp(self):
super(DataFileResourceTest, self).setUp()
self.django_client = Client()
self.django_client.login(username=self.username,
password=self.password)
self.testds = Dataset()
self.testds.description = "test dataset"
self.testds.save()
self.testds.experiments.add(self.testexp)
df_schema_name = "http://datafileshop.com/"
self.test_schema = Schema(namespace=df_schema_name,
type=Schema.DATAFILE)
self.test_schema.save()
self.test_parname1 = ParameterName(schema=self.test_schema,
name="fileparameter1",
data_type=ParameterName.STRING)
self.test_parname1.save()
self.test_parname2 = ParameterName(schema=self.test_schema,
name="fileparameter2",
data_type=ParameterName.NUMERIC)
self.test_parname2.save()
def test_post_single_file(self):
ds_id = Dataset.objects.first().id
post_data = """{
"dataset": "/api/v1/dataset/%d/",
"filename": "mytestfile.txt",
"md5sum": "930e419034038dfad994f0d2e602146c",
"size": "8",
"mimetype": "text/plain",
"parameter_sets": [{
"schema": "http://datafileshop.com/",
"parameters": [{
"name": "fileparameter1",
"value": "123"
},
{
"name": "fileparameter2",
"value": "123"
}]
}]
}""" % ds_id
post_file = tempfile.NamedTemporaryFile()
file_content = "123test\n"
post_file.write(file_content)
post_file.flush()
post_file.seek(0)
datafile_count = DataFile.objects.count()
dfo_count = DataFileObject.objects.count()
self.assertHttpCreated(
self.django_client.post('/api/v1/dataset_file/',
data={
"json_data": post_data,
"attached_file": post_file
}))
self.assertEqual(datafile_count + 1, DataFile.objects.count())
self.assertEqual(dfo_count + 1, DataFileObject.objects.count())
new_file = DataFile.objects.order_by('-pk')[0]
self.assertEqual(file_content, new_file.get_file().read())
def test_shared_fs_single_file(self):
pass
def test_shared_fs_many_files(self): # noqa # TODO too complex
'''
tests sending many files with known permanent location
(useful for Australian Synchrotron ingestions)
'''
files = [{'content': 'test123\n'}, {'content': 'test246\n'}]
from django.conf import settings
for file_dict in files:
post_file = tempfile.NamedTemporaryFile(
dir=settings.DEFAULT_STORAGE_BASE_DIR)
file_dict['filename'] = os.path.basename(post_file.name)
file_dict['full_path'] = post_file.name
post_file.write(file_dict['content'])
post_file.flush()
post_file.seek(0)
file_dict['object'] = post_file
def clumsily_build_uri(res_type, dataset):
return '/api/v1/%s/%d/' % (res_type, dataset.id)
def md5sum(filename):
import hashlib
md5 = hashlib.md5()
with open(filename, 'rb') as f:
for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
md5.update(chunk)
return md5.hexdigest()
def guess_mime(filename):
import magic
mime = magic.Magic(mime=True)
return mime.from_file(filename)
json_data = {"objects": []}
for file_dict in files:
file_json = {
'dataset':
clumsily_build_uri('dataset', self.testds),
'filename':
os.path.basename(file_dict['filename']),
'md5sum':
md5sum(file_dict['full_path']),
'size':
os.path.getsize(file_dict['full_path']),
'mimetype':
guess_mime(file_dict['full_path']),
'replicas': [{
'url': file_dict['filename'],
'location': 'default',
'protocol': 'file',
}],
}
json_data['objects'].append(file_json)
datafile_count = DataFile.objects.count()
dfo_count = DataFileObject.objects.count()
self.assertHttpAccepted(
self.api_client.patch('/api/v1/dataset_file/',
data=json_data,
authentication=self.get_credentials()))
self.assertEqual(datafile_count + 2, DataFile.objects.count())
self.assertEqual(dfo_count + 2, DataFileObject.objects.count())
# fake-verify DFO, so we can access the file:
for newdfo in DataFileObject.objects.order_by('-pk')[0:2]:
newdfo.verified = True
newdfo.save()
for sent_file, new_file in zip(reversed(files),
DataFile.objects.order_by('-pk')[0:2]):
self.assertEqual(sent_file['content'], new_file.get_file().read())
class DataFileResourceTest(MyTardisResourceTestCase):
def setUp(self):
super(DataFileResourceTest, self).setUp()
self.django_client = Client()
self.django_client.login(username=self.username,
password=self.password)
self.testds = Dataset()
self.testds.description = "test dataset"
self.testds.save()
self.testds.experiments.add(self.testexp)
df_schema_name = "http://datafileshop.com/"
self.test_schema = Schema(namespace=df_schema_name,
type=Schema.DATAFILE)
self.test_schema.save()
self.test_parname1 = ParameterName(schema=self.test_schema,
name="fileparameter1",
data_type=ParameterName.STRING)
self.test_parname1.save()
self.test_parname2 = ParameterName(schema=self.test_schema,
name="fileparameter2",
data_type=ParameterName.NUMERIC)
self.test_parname2.save()
def test_post_single_file(self):
post_data = """{
"dataset": "/api/v1/dataset/1/",
"filename": "mytestfile.txt",
"md5sum": "930e419034038dfad994f0d2e602146c",
"size": "8",
"mimetype": "text/plain",
"parameter_sets": [{
"schema": "http://datafileshop.com/",
"parameters": [{
"name": "fileparameter1",
"value": "123"
},
{
"name": "fileparameter2",
"value": "123"
}]
}]
}"""
post_file = tempfile.NamedTemporaryFile()
file_content = "123test\n"
post_file.write(file_content)
post_file.flush()
post_file.seek(0)
datafile_count = DataFile.objects.count()
dfo_count = DataFileObject.objects.count()
self.assertHttpCreated(self.django_client.post(
'/api/v1/dataset_file/',
data={"json_data": post_data, "attached_file": post_file}))
self.assertEqual(datafile_count + 1, DataFile.objects.count())
self.assertEqual(dfo_count + 1, DataFileObject.objects.count())
new_file = DataFile.objects.order_by('-pk')[0]
self.assertEqual(file_content, new_file.get_file().read())
def test_shared_fs_single_file(self):
pass
def test_shared_fs_many_files(self): # noqa # TODO too complex
'''
tests sending many files with known permanent location
(useful for Australian Synchrotron ingestions)
'''
files = [{'content': 'test123\n'},
{'content': 'test246\n'}]
from django.conf import settings
for file_dict in files:
post_file = tempfile.NamedTemporaryFile(
dir=settings.DEFAULT_STORAGE_BASE_DIR)
file_dict['filename'] = os.path.basename(post_file.name)
file_dict['full_path'] = post_file.name
post_file.write(file_dict['content'])
post_file.flush()
post_file.seek(0)
file_dict['object'] = post_file
def clumsily_build_uri(res_type, dataset):
return '/api/v1/%s/%d/' % (res_type, dataset.id)
def md5sum(filename):
import hashlib
md5 = hashlib.md5()
with open(filename, 'rb') as f:
for chunk in iter(lambda: f.read(128*md5.block_size), b''):
md5.update(chunk)
return md5.hexdigest()
def guess_mime(filename):
import magic
mime = magic.Magic(mime=True)
return mime.from_file(filename)
json_data = {"objects": []}
for file_dict in files:
file_json = {
'dataset': clumsily_build_uri('dataset', self.testds),
'filename': os.path.basename(file_dict['filename']),
'md5sum': md5sum(file_dict['full_path']),
'size': os.path.getsize(file_dict['full_path']),
'mimetype': guess_mime(file_dict['full_path']),
'replicas': [{
'url': file_dict['filename'],
'location': 'default',
'protocol': 'file',
}],
}
json_data['objects'].append(file_json)
datafile_count = DataFile.objects.count()
dfo_count = DataFileObject.objects.count()
self.assertHttpAccepted(self.api_client.patch(
'/api/v1/dataset_file/',
data=json_data,
authentication=self.get_credentials()))
self.assertEqual(datafile_count + 2, DataFile.objects.count())
self.assertEqual(dfo_count + 2, DataFileObject.objects.count())
# fake-verify DFO, so we can access the file:
for newdfo in DataFileObject.objects.order_by('-pk')[0:2]:
newdfo.verified = True
newdfo.save()
for sent_file, new_file in zip(
reversed(files), DataFile.objects.order_by('-pk')[0:2]):
self.assertEqual(sent_file['content'], new_file.get_file().read())
