def removeImages(names, user): folderModel = Folder() removed = [] for name in names: image, tag = _split(name) q = {'meta.slicerCLIType': 'image', 'name': image} imageFolder = folderModel.findOne(q, user=user, level=AccessType.READ) if not imageFolder: continue qt = { 'meta.slicerCLIType': 'tag', 'parentId': imageFolder['_id'], 'name': tag } tagFolder = folderModel.findOne(qt, user=user, level=AccessType.WRITE) if not tagFolder: continue folderModel.remove(tagFolder) removed.append(name) if folderModel.hasAccess(imageFolder, user, AccessType.WRITE) and \ folderModel.countFolders(imageFolder) == 0: # clean also empty image folders folderModel.remove(imageFolder) return removed
def execute_simulation(self, name, config, folder=None): target_time = config.get('simulation', {}).get('run_time', 50) user, token = self.getCurrentUser(returnToken=True) folder_model = Folder() job_model = Job() if folder is None: folder = folder_model.findOne( {'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user'} ) if folder is None: raise RestException('Could not find the user\'s "public" folder.') simulation_model = Simulation() simulation = simulation_model.createSimulation( folder, name, config, user, True, ) girder_config = GirderConfig( api=GIRDER_API, token=str(token['_id']), folder=str(folder['_id']) ) simulation_config = SimulationConfig(NLI_CONFIG_FILE, config) # TODO: This would be better stored as a dict, but it's easier once we change the # config object format. simulation_config_file = StringIO() simulation_config.write(simulation_config_file) job = job_model.createJob( title='NLI Simulation', type=NLI_JOB_TYPE, kwargs={ 'girder_config': attr.asdict(girder_config), 'simulation_config': simulation_config_file.getvalue(), 'config': config, 'simulation_id': simulation['_id'], }, user=user, ) simulation['nli']['job_id'] = job['_id'] simulation_model.save(simulation) run_simulation.delay( name=name, girder_config=girder_config, simulation_config=simulation_config, target_time=target_time, job=job, simulation_id=simulation['_id'], ) return job
def findOne(self, query=None, annotator_user=None, state=None, **kwargs): study_query = self._find_query_filter(query, annotator_user, state) return Folder.findOne(self, study_query, **kwargs)
def run_experiment(self, name, config, folder=None): target_time = config.get('simulation', {}).get('run_time', 0) # if there is no run_time, this is not a valid request if target_time <= 0: raise RestException( 'Invalid (or unprovided) run time for experiment.') runs_per_config = config.get('simulation', {}).get('runs_per_config', 1) max_run_digit_len = math.floor(1 + math.log10(runs_per_config)) user, token = self.getCurrentUser(returnToken=True) folder_model = Folder() job_model = Job() if folder is None: folder = folder_model.findOne({ 'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user' }) if folder is None: raise RestException( 'Could not find the user\'s "public" folder.') # for each of the configuration values which are lists, we run the simulator with # each of the possible values. (cartesian product) configs = [dict()] experimental_variables: List[Dict[str, Any]] = [] for module, module_config in config.items(): for parameter, parameter_values in module_config.items(): if isinstance(parameter_values, list): # this will unpack lists appropriately, even of length 0 or 1, # but those are not experimental variables if len(parameter_values) > 1: experimental_variables.append({ 'module': module, 'parameter': parameter, 'values': parameter_values }) new_configs = [] for cfg in configs: for val in parameter_values: new_cfg = copy.deepcopy(cfg) if module not in new_cfg: new_cfg[module] = dict() new_cfg[module][parameter] = val new_configs.append(new_cfg) configs = new_configs else: for cfg in configs: if module not in cfg: cfg[module] = dict() cfg[module][parameter] = parameter_values # create a folder to hold the various runs of the simulator # TODO: what if this fails? how does it fail? experiment_model = Experiment() experiment_folder = experiment_model.createExperiment( parentFolder=folder, name=name, config=config, creator=user, version=nlisim_version, experimental_variables=experimental_variables, runs_per_config=runs_per_config, public=True, ) jobs = [] for config_variant in configs: for run_number in range(runs_per_config): # create an informative name for the run, noting the run number and the values # of the experimental variables run_name = name + "-run-" + str(run_number).zfill( max_run_digit_len) for experimental_variable in experimental_variables: run_name += ( '-' + str(experimental_variable['module']) + "." + str(experimental_variable['parameter']) + "-" + str(config_variant[experimental_variable['module']][ experimental_variable['parameter']])) job, simulation = simulation_runner( config=config_variant, parent_folder=experiment_folder, job_model=job_model, run_name=run_name, target_time=target_time, token=token, user=user, experiment=experiment_folder, ) jobs.append(job) return jobs
class HdfsAssetstoreResource(Resource): def __init__(self): super(HdfsAssetstoreResource, self).__init__() self.resourceName = 'hdfs_assetstore' self.route('PUT', (':id', 'import'), self.importData) self.folderModel = Folder() # Save to avoid many lookups self.itemModel = Item() self.fileModel = File() def _importFile(self, parent, name, user, assetstore, node): item = self.itemModel.findOne({ 'folderId': parent['_id'], 'name': name }) if item is None: item = self.itemModel.createItem( name=name, creator=user, folder=parent) file = self.fileModel.findOne({ 'name': name, 'itemId': item['_id'] }) if file is None: file = self.fileModel.createFile( creator=user, item=item, name=name, size=node['length'], assetstore=assetstore, mimeType=None, saveFile=False) file['hdfs'] = { 'imported': True, 'path': node['path'] } self.fileModel.save(file) def _importData(self, parentType, parent, assetstore, client, path, ctx, user): for node in client.ls([path]): ctx.update(message='Importing ' + node['path']) name = posixpath.basename(node['path']) if node['file_type'] == 'd': folder = self.folderModel.findOne({ 'parentId': parent['_id'], 'name': name, 'parentCollection': parentType }) if folder is None: folder = self.folderModel.createFolder( parent, name, parentType=parentType, creator=user) self._importData('folder', folder, assetstore, client, node['path'], ctx, user) elif node['file_type'] == 'f' and parentType == 'folder': self._importFile(parent, name, user, assetstore, node) @access.admin @loadmodel(model='assetstore') @describeRoute( Description('Import a data hierarchy from an HDFS instance.') .notes('Only site administrators may use this endpoint.') .param('id', 'The ID of the assetstore representing the HDFS instance.', paramType='path') .param('parentId', 'The ID of the parent object in the Girder data ' 'hierarchy under which to import the files.') .param('parentType', 'The type of the parent object to import into.', enum=('folder', 'user', 'collection'), required=False) .param('path', 'Root of the directory structure (relative to the root ' 'of the HDFS) to import.') .param('progress', 'Whether to record progress on this operation (' 'default=False)', required=False, dataType='boolean') .errorResponse() .errorResponse('You are not an administrator.', 403) ) def importData(self, assetstore, params): self.requireParams(('parentId', 'path'), params) user = self.getCurrentUser() parentType = params.get('parentType', 'folder') if parentType not in ('user', 'collection', 'folder'): raise RestException('Invalid parentType.') parent = self.model(parentType).load(params['parentId'], force=True, exc=True) progress = self.boolParam('progress', params, default=False) client = HdfsClient( host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False) path = params['path'] with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx: try: self._importData(parentType, parent, assetstore, client, path, ctx, user) except FileNotFoundException: raise RestException('File not found: %s.' % path)
def findOne(self, query=None, **kwargs): dataset_query = self._find_query_filter(query) return Folder.findOne(self, dataset_query, **kwargs)