def removeImages(names, user):
folderModel = Folder()
removed = []
for name in names:
image, tag = _split(name)
q = {'meta.slicerCLIType': 'image', 'name': image}
imageFolder = folderModel.findOne(q,
user=user,
level=AccessType.READ)
if not imageFolder:
continue
qt = {
'meta.slicerCLIType': 'tag',
'parentId': imageFolder['_id'],
'name': tag
}
tagFolder = folderModel.findOne(qt,
user=user,
level=AccessType.WRITE)
if not tagFolder:
continue
folderModel.remove(tagFolder)
removed.append(name)
if folderModel.hasAccess(imageFolder, user, AccessType.WRITE) and \
folderModel.countFolders(imageFolder) == 0:
# clean also empty image folders
folderModel.remove(imageFolder)
return removed
def execute_simulation(self, name, config, folder=None):
target_time = config.get('simulation', {}).get('run_time', 50)
user, token = self.getCurrentUser(returnToken=True)
folder_model = Folder()
job_model = Job()
if folder is None:
folder = folder_model.findOne(
{'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user'}
)
if folder is None:
raise RestException('Could not find the user\'s "public" folder.')
simulation_model = Simulation()
simulation = simulation_model.createSimulation(
folder,
name,
config,
user,
True,
)
girder_config = GirderConfig(
api=GIRDER_API, token=str(token['_id']), folder=str(folder['_id'])
)
simulation_config = SimulationConfig(NLI_CONFIG_FILE, config)
# TODO: This would be better stored as a dict, but it's easier once we change the
# config object format.
simulation_config_file = StringIO()
simulation_config.write(simulation_config_file)
job = job_model.createJob(
title='NLI Simulation',
type=NLI_JOB_TYPE,
kwargs={
'girder_config': attr.asdict(girder_config),
'simulation_config': simulation_config_file.getvalue(),
'config': config,
'simulation_id': simulation['_id'],
},
user=user,
)
simulation['nli']['job_id'] = job['_id']
simulation_model.save(simulation)
run_simulation.delay(
name=name,
girder_config=girder_config,
simulation_config=simulation_config,
target_time=target_time,
job=job,
simulation_id=simulation['_id'],
)
return job
def findOne(self, query=None, annotator_user=None, state=None, **kwargs):
study_query = self._find_query_filter(query, annotator_user, state)
return Folder.findOne(self, study_query, **kwargs)
def run_experiment(self, name, config, folder=None):
target_time = config.get('simulation', {}).get('run_time', 0)
# if there is no run_time, this is not a valid request
if target_time <= 0:
raise RestException(
'Invalid (or unprovided) run time for experiment.')
runs_per_config = config.get('simulation',
{}).get('runs_per_config', 1)
max_run_digit_len = math.floor(1 + math.log10(runs_per_config))
user, token = self.getCurrentUser(returnToken=True)
folder_model = Folder()
job_model = Job()
if folder is None:
folder = folder_model.findOne({
'parentId': user['_id'],
'name': 'Public',
'parentCollection': 'user'
})
if folder is None:
raise RestException(
'Could not find the user\'s "public" folder.')
# for each of the configuration values which are lists, we run the simulator with
# each of the possible values. (cartesian product)
configs = [dict()]
experimental_variables: List[Dict[str, Any]] = []
for module, module_config in config.items():
for parameter, parameter_values in module_config.items():
if isinstance(parameter_values, list):
# this will unpack lists appropriately, even of length 0 or 1,
# but those are not experimental variables
if len(parameter_values) > 1:
experimental_variables.append({
'module':
module,
'parameter':
parameter,
'values':
parameter_values
})
new_configs = []
for cfg in configs:
for val in parameter_values:
new_cfg = copy.deepcopy(cfg)
if module not in new_cfg:
new_cfg[module] = dict()
new_cfg[module][parameter] = val
new_configs.append(new_cfg)
configs = new_configs
else:
for cfg in configs:
if module not in cfg:
cfg[module] = dict()
cfg[module][parameter] = parameter_values
# create a folder to hold the various runs of the simulator
# TODO: what if this fails? how does it fail?
experiment_model = Experiment()
experiment_folder = experiment_model.createExperiment(
parentFolder=folder,
name=name,
config=config,
creator=user,
version=nlisim_version,
experimental_variables=experimental_variables,
runs_per_config=runs_per_config,
public=True,
)
jobs = []
for config_variant in configs:
for run_number in range(runs_per_config):
# create an informative name for the run, noting the run number and the values
# of the experimental variables
run_name = name + "-run-" + str(run_number).zfill(
max_run_digit_len)
for experimental_variable in experimental_variables:
run_name += (
'-' + str(experimental_variable['module']) + "." +
str(experimental_variable['parameter']) + "-" +
str(config_variant[experimental_variable['module']][
experimental_variable['parameter']]))
job, simulation = simulation_runner(
config=config_variant,
parent_folder=experiment_folder,
job_model=job_model,
run_name=run_name,
target_time=target_time,
token=token,
user=user,
experiment=experiment_folder,
)
jobs.append(job)
return jobs
class HdfsAssetstoreResource(Resource):
def __init__(self):
super(HdfsAssetstoreResource, self).__init__()
self.resourceName = 'hdfs_assetstore'
self.route('PUT', (':id', 'import'), self.importData)
self.folderModel = Folder() # Save to avoid many lookups
self.itemModel = Item()
self.fileModel = File()
def _importFile(self, parent, name, user, assetstore, node):
item = self.itemModel.findOne({
'folderId': parent['_id'],
'name': name
})
if item is None:
item = self.itemModel.createItem(
name=name, creator=user, folder=parent)
file = self.fileModel.findOne({
'name': name,
'itemId': item['_id']
})
if file is None:
file = self.fileModel.createFile(
creator=user, item=item, name=name, size=node['length'],
assetstore=assetstore, mimeType=None, saveFile=False)
file['hdfs'] = {
'imported': True,
'path': node['path']
}
self.fileModel.save(file)
def _importData(self, parentType, parent, assetstore, client, path, ctx,
user):
for node in client.ls([path]):
ctx.update(message='Importing ' + node['path'])
name = posixpath.basename(node['path'])
if node['file_type'] == 'd':
folder = self.folderModel.findOne({
'parentId': parent['_id'],
'name': name,
'parentCollection': parentType
})
if folder is None:
folder = self.folderModel.createFolder(
parent, name, parentType=parentType, creator=user)
self._importData('folder', folder, assetstore, client,
node['path'], ctx, user)
elif node['file_type'] == 'f' and parentType == 'folder':
self._importFile(parent, name, user, assetstore, node)
@access.admin
@loadmodel(model='assetstore')
@describeRoute(
Description('Import a data hierarchy from an HDFS instance.')
.notes('Only site administrators may use this endpoint.')
.param('id', 'The ID of the assetstore representing the HDFS instance.',
paramType='path')
.param('parentId', 'The ID of the parent object in the Girder data '
'hierarchy under which to import the files.')
.param('parentType', 'The type of the parent object to import into.',
enum=('folder', 'user', 'collection'), required=False)
.param('path', 'Root of the directory structure (relative to the root '
'of the HDFS) to import.')
.param('progress', 'Whether to record progress on this operation ('
'default=False)', required=False, dataType='boolean')
.errorResponse()
.errorResponse('You are not an administrator.', 403)
)
def importData(self, assetstore, params):
self.requireParams(('parentId', 'path'), params)
user = self.getCurrentUser()
parentType = params.get('parentType', 'folder')
if parentType not in ('user', 'collection', 'folder'):
raise RestException('Invalid parentType.')
parent = self.model(parentType).load(params['parentId'], force=True, exc=True)
progress = self.boolParam('progress', params, default=False)
client = HdfsClient(
host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False)
path = params['path']
with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx:
try:
self._importData(parentType, parent, assetstore, client, path, ctx, user)
except FileNotFoundException:
raise RestException('File not found: %s.' % path)
def findOne(self, query=None, **kwargs):
dataset_query = self._find_query_filter(query)
return Folder.findOne(self, dataset_query, **kwargs)
