def test_data_create_path(self):
url = reverse('substrapp:data_sample-list')
dir_path = os.path.dirname(os.path.realpath(__file__))
data_zip_path = os.path.join(
dir_path,
'../../../../fixtures/chunantes/datasamples/datasample1/0024700.zip'
)
data_path = os.path.join(MEDIA_ROOT, '0024700')
with open(data_zip_path, 'rb') as data_zip:
uncompress_content(data_zip.read(), data_path)
checksum = '24fb12ff87485f6b0bc5349e5bf7f36ccca4eb1353395417fdae7d8d787f178c'
data = {
'path': data_path,
'data_manager_keys': [datamanager[0]['key']],
'test_only': False
}
with mock.patch.object(DataManager.objects, 'filter') as mdatamanager, \
mock.patch.object(LedgerDataSampleSerializer, 'create') as mcreate_ledger, \
mock.patch.object(DataSampleSerializer, 'create', wraps=DataSampleSerializer().create) as mcreate:
mdatamanager.return_value = FakeFilterDataManager(1)
mcreate_ledger.return_value = {'keys': ['some_key']}
response = self.client.post(url,
data=data,
format='json',
**self.extra)
self.assertEqual(mcreate.call_args_list[0][0][0]['checksum'], checksum)
self.assertIsNotNone(response.data[0]['key'])
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_data_create_parent_path(self):
url = reverse('substrapp:data_sample-list')
dir_path = os.path.dirname(os.path.realpath(__file__))
data_zip_path = os.path.join(
dir_path,
'../../../../fixtures/chunantes/datasamples/datasample1/0024700.zip'
)
data_parent_path = os.path.join(MEDIA_ROOT, 'data_samples')
data_path = os.path.join(data_parent_path, '0024700')
with open(data_zip_path, 'rb') as data_zip:
uncompress_content(data_zip.read(), data_path)
# dir hash
pkhash = '24fb12ff87485f6b0bc5349e5bf7f36ccca4eb1353395417fdae7d8d787f178c'
data_manager_keys = [
get_hash(
os.path.join(
dir_path,
'../../../../fixtures/chunantes/datamanagers/datamanager0/opener.py'
))
]
data = {
'path': data_parent_path,
'data_manager_keys': data_manager_keys,
'test_only': False,
'multiple': True,
}
with mock.patch.object(DataManager.objects, 'filter') as mdatamanager, \
mock.patch.object(LedgerDataSampleSerializer, 'create') as mcreate:
mdatamanager.return_value = FakeFilterDataManager(1)
mcreate.return_value = {'keys': [pkhash]}
response = self.client.post(url,
data=data,
format='multipart',
**self.extra)
self.assertEqual(response.data[0]['pkhash'], pkhash)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def data_sample_pre_save(sender, instance, **kwargs):
directory = path.join(getattr(settings, 'MEDIA_ROOT'), 'datasamples/{0}'.format(instance.pk))
# uncompress file if an archive
if isinstance(instance.path, File):
try:
content = instance.path.read()
instance.path.seek(0)
uncompress_content(content, directory)
except Exception as e:
logging.info(e)
raise e
else:
# calculate new hash
sha256hash = dirhash(directory, 'sha256')
# rename directory to new hash if does not exist
new_directory = path.join(getattr(settings, 'MEDIA_ROOT'), 'datasamples', sha256hash)
try:
rename(directory, new_directory)
except Exception as e:
# directory already exists with same exact data sample inside
# created by a previous save, delete directory entitled pkhash
# for avoiding duplicates
shutil.rmtree(directory)
logging.error(e, exc_info=True)
# override defaults
instance.pkhash = sha256hash
instance.path = new_directory
# make an hardlink on all files if a path
else:
try:
p = normpath(instance.path)
create_hard_links(p, directory)
except Exception as e:
pass
else:
# override path for getting our hardlink
instance.path = directory
def prepare_objective(channel_name, directory, tuple_):
"""Prepare objective for tuple execution."""
metrics_content = get_objective(channel_name, tuple_)
dst_path = path.join(directory, 'metrics/')
uncompress_content(metrics_content, dst_path)
def prepare_algo(channel_name, directory, tuple_type, tuple_):
"""Prepare algo for tuple execution."""
content = get_algo(channel_name, tuple_type, tuple_)
uncompress_content(content, directory)
def put_algo(subtuple_directory, algo_content):
try:
uncompress_content(algo_content, subtuple_directory)
except Exception as e:
logging.error('Fail to uncompress algo file')
raise e