def test_computeTask(self):
class FakeSettings(object):
def __init__(self):
self.LEDGER = {
'signcert': 'signcert',
'org': 'owkin',
'peer': 'peer'
}
self.MEDIA_ROOT = MEDIA_ROOT
subtuple_key = 'test_owkin'
subtuple = {'key': subtuple_key, 'inModels': None}
subtuple_directory = build_subtuple_folders(subtuple)
with mock.patch('substrapp.tasks.settings') as msettings, \
mock.patch('substrapp.tasks.getattr') as mgetattr, \
mock.patch('substrapp.tasks.invokeLedger') as minvokeLedger:
msettings.return_value = FakeSettings()
mgetattr.return_value = self.subtuple_path
minvokeLedger.return_value = 'data', 200
for name in ['opener', 'metrics']:
with open(
os.path.join(subtuple_directory, f'{name}/{name}.py'),
'w') as f:
f.write('Hello World')
perf = 0.3141592
with open(os.path.join(subtuple_directory, 'pred/perf.json'),
'w') as f:
f.write(f'{{"all": {perf}}}')
with open(os.path.join(subtuple_directory, 'model/model'),
'w') as f:
f.write("MODEL")
with mock.patch('substrapp.tasks.compute_docker') as mcompute_docker, \
mock.patch('substrapp.tasks.prepareMaterials') as mprepareMaterials, \
mock.patch('substrapp.tasks.invokeLedger') as minvokeLedger:
mcompute_docker.return_value = 'DONE'
mprepareMaterials.return_value = 'DONE'
minvokeLedger.return_value = 'data', 201
computeTask('traintuple', subtuple, 'inModels', None)
def test_build_subtuple_folders(self):
with mock.patch('substrapp.tasks.getattr') as getattr:
getattr.return_value = self.subtuple_path
subtuple_key = 'test1234'
subtuple = {'key': subtuple_key}
subtuple_directory = build_subtuple_folders(subtuple)
self.assertTrue(os.path.exists(subtuple_directory))
self.assertEqual(
os.path.join(self.subtuple_path,
f'subtuple/{subtuple["key"]}'),
subtuple_directory)
for root, dirs, files in os.walk(subtuple_directory):
nb_subfolders = len(dirs)
self.assertTrue(5, nb_subfolders)
def test_put_data_tar(self):
data_sample = DataSample(pkhash='foo', path=self.data_sample_tar)
data_sample.save()
subtuple = {'key': 'bar', 'dataset': {'keys': [data_sample.pk]}}
with mock.patch('substrapp.models.DataSample.objects.get') as mget:
mget.return_value = data_sample
subtuple_direcory = build_subtuple_folders(subtuple)
put_data_sample(subtuple, subtuple_direcory)
# check folder has been correctly renamed with pk of directory containing uncompressed data_sample
self.assertFalse(
os.path.exists(os.path.join(MEDIA_ROOT, 'datasamples', 'foo')))
dir_pkhash = '30f6c797e277451b0a08da7119ed86fb2986fa7fab2258bf3edbd9f1752ed553'
self.assertTrue(
os.path.exists(
os.path.join(MEDIA_ROOT, 'datasamples', dir_pkhash)))
# check subtuple folder has been created and sym links exists
self.assertTrue(
os.path.exists(
os.path.join(MEDIA_ROOT, 'subtuple/bar/data',
data_sample.pk)))
self.assertTrue(
os.path.islink(
os.path.join(MEDIA_ROOT, 'subtuple/bar/data',
data_sample.pk)))
self.assertTrue(
os.path.exists(
os.path.join(MEDIA_ROOT, 'subtuple/bar/data',
data_sample.pk, 'LABEL_0024900.csv')))
self.assertTrue(
os.path.exists(
os.path.join(MEDIA_ROOT, 'subtuple/bar/data',
data_sample.pk, 'IMG_0024900.jpg')))
def compute_dryrun(self, metrics_path, test_data_manager_key, pkhash):
dryrun_uuid = f'{pkhash}_{uuid.uuid4().hex}'
subtuple_directory = build_subtuple_folders({'key': dryrun_uuid})
metrics_path_dst = os.path.join(subtuple_directory, 'metrics/metrics.py')
if not os.path.exists(metrics_path_dst):
shutil.copy2(metrics_path,
os.path.join(subtuple_directory, 'metrics/metrics.py'))
os.remove(metrics_path)
if not test_data_manager_key:
raise Exception(
'Cannot do a objective dryrun without a data manager key.')
datamanager = getObjectFromLedger(test_data_manager_key,
'queryDataManager')
opener_content, opener_computed_hash = get_computed_hash(
datamanager['opener']['storageAddress'])
with open(os.path.join(subtuple_directory, 'opener/opener.py'),
'wb') as opener_file:
opener_file.write(opener_content)
# Launch verification
client = docker.from_env()
pred_path = os.path.join(subtuple_directory, 'pred')
opener_file = os.path.join(subtuple_directory, 'opener/opener.py')
metrics_file = os.path.join(subtuple_directory, 'metrics/metrics.py')
metrics_path = os.path.join(
getattr(settings, 'PROJECT_ROOT'),
'fake_metrics') # base metrics comes with substrabac
metrics_docker = 'metrics_dry_run' # tag must be lowercase for docker
metrics_docker_name = f'{metrics_docker}_{dryrun_uuid}'
volumes = {
pred_path: {
'bind': '/sandbox/pred',
'mode': 'rw'
},
metrics_file: {
'bind': '/sandbox/metrics/__init__.py',
'mode': 'ro'
},
opener_file: {
'bind': '/sandbox/opener/__init__.py',
'mode': 'ro'
}
}
client.images.build(path=metrics_path, tag=metrics_docker, rm=False)
job_args = {
'image': metrics_docker,
'name': metrics_docker_name,
'cpuset_cpus': '0-0',
'mem_limit': '1G',
'command': None,
'volumes': volumes,
'shm_size': '8G',
'labels': ['dryrun'],
'detach': False,
'auto_remove': False,
'remove': False
}
try:
client.containers.run(**job_args)
if not os.path.exists(os.path.join(pred_path, 'perf.json')):
raise Exception('Perf file not found')
except ContainerError as e:
raise Exception(e.stderr)
finally:
try:
container = client.containers.get(metrics_docker_name)
container.remove(force=True)
except BaseException as e:
logging.error(e, exc_info=True)
remove_subtuple_materials(subtuple_directory)
def compute_dryrun(self, data, data_manager_keys):
from shutil import copy
from substrapp.models import DataManager
client = docker.from_env()
# Name of the dry-run subtuple (not important)
pkhash = data[0]['pkhash']
dryrun_uuid = f'{pkhash}_{uuid.uuid4().hex}'
subtuple_directory = build_subtuple_folders({'key': dryrun_uuid})
data_path = os.path.join(subtuple_directory, 'data')
volumes = {}
try:
for data_sample in data:
# uncompress only for file
if 'file' in data_sample:
try:
uncompress_path(data_sample['file'], os.path.join(data_path, data_sample['pkhash']))
except Exception as e:
raise e
# for all data paths, we need to create symbolic links inside data_path
# and add real path to volume bind docker
elif 'path' in data_sample:
os.symlink(data_sample['path'], os.path.join(data_path, data_sample['pkhash']))
volumes.update({data_sample['path']: {'bind': data_sample['path'], 'mode': 'ro'}})
for datamanager_key in data_manager_keys:
datamanager = DataManager.objects.get(pk=datamanager_key)
copy(datamanager.data_opener.path, os.path.join(subtuple_directory, 'opener/opener.py'))
# Launch verification
opener_file = os.path.join(subtuple_directory, 'opener/opener.py')
data_sample_docker_path = os.path.join(getattr(settings, 'PROJECT_ROOT'), 'fake_data_sample') # fake_data comes with substrabac
data_docker = 'data_dry_run' # tag must be lowercase for docker
data_docker_name = f'{data_docker}_{dryrun_uuid}'
volumes.update({data_path: {'bind': '/sandbox/data', 'mode': 'rw'},
opener_file: {'bind': '/sandbox/opener/__init__.py', 'mode': 'ro'}})
client.images.build(path=data_sample_docker_path,
tag=data_docker,
rm=False)
job_args = {'image': data_docker,
'name': data_docker_name,
'cpuset_cpus': '0-0',
'mem_limit': '1G',
'command': None,
'volumes': volumes,
'shm_size': '8G',
'labels': ['dryrun'],
'detach': False,
'auto_remove': False,
'remove': False}
client.containers.run(**job_args)
except ContainerError as e:
raise Exception(e.stderr)
finally:
try:
container = client.containers.get(data_docker_name)
container.remove()
except:
logger.error('Could not remove containers')
remove_subtuple_materials(subtuple_directory)
for data_sample in data:
if 'file' in data_sample and os.path.exists(data_sample['file']):
os.remove(data_sample['file'])