def attach_complete(self, volume, cluster, params): user = getCurrentUser() path = getBodyJson().get('path', None) if path is not None: cluster.setdefault('volumes', []) cluster['volumes'].append(volume['_id']) cluster['volumes'] = list(set(cluster['volumes'])) volume['status'] = VolumeState.INUSE volume['path'] = path # TODO: removing msg should be refactored into # a general purpose 'update_status' function # on the volume model. This way msg only referes # to the current status. try: del volume['msg'] except KeyError: pass # Add cluster id to volume volume['clusterId'] = cluster['_id'] self.model('cluster', 'cumulus').save(cluster) self._model.update_volume(user, volume) else: volume['status'] = VolumeState.ERROR volume['msg'] = 'Volume path was not communicated on complete' self._model.update_volume(user, volume)
def handle_log_record(self, id, params): user = self.getCurrentUser() if not self._model.load(id, user=user, level=AccessType.ADMIN): raise RestException('Cluster not found.', code=404) return self._model.append_to_log(user, id, getBodyJson())
def create_profile(user, params): body = getBodyJson() requireParams([ 'name', 'accessKeyId', 'secretAccessKey', 'regionName', 'availabilityZone' ], body) profile_type = 'ec2' if 'cloudProvider' not in body.keys() \ else body['cloudProvider'] model = ModelImporter.model('aws', 'cumulus') profile = model.create_profile(user['_id'], body['name'], profile_type, body['accessKeyId'], body['secretAccessKey'], body['regionName'], body['availabilityZone'], body.get('publicIPs', False)) # Now fire of a task to create a key pair for this profile try: cumulus.aws.ec2.tasks.key.generate_key_pair.delay( _filter(profile), get_task_token()['_id']) cherrypy.response.status = 201 cherrypy.response.headers['Location'] \ = '/user/%s/aws/profile/%s' % (str(user['_id']), str(profile['_id'])) return model.filter(profile, getCurrentUser()) except Exception: # Remove profile if error occurs fire of task model.remove(profile) raise
def update(self, id, params): user = self.getCurrentUser() body = getBodyJson() job = self._model.load(id, user=user, level=AccessType.WRITE) if not job: raise RestException('Job not found.', code=404) if 'status' in body: job['status'] = body['status'] if 'queueJobId' in body: job['queueJobId'] = body['queueJobId'] if 'output' in body: job['output'] = body['output'] if 'timings' in body: if 'timings' in job: job['timings'].update(body['timings']) else: job['timings'] = body['timings'] if 'dir' in body: job['dir'] = body['dir'] job = self._model.update_job(user, job) # Don't return the access object del job['access'] # Don't return the log del job['log'] return job
def create(self, params): body = getBodyJson() self.requireParams(['name', 'type', 'size', 'profileId'], body) if not VolumeType.is_valid_type(body['type']): raise RestException('Invalid volume type.', code=400) profile_id = parse('profileId').find(body) if not profile_id: raise RestException('A profile id must be provided', 400) profile_id = profile_id[0].value profile, secret_key = _get_profile(profile_id) if not profile: raise RestException('Invalid profile', 400) if 'zone' in body: zone = body['zone'] else: zone = profile['availabilityZone'] volume = self._create_ebs(body, zone) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/volumes/%s' % volume['_id'] return self._model.filter(volume, getCurrentUser())
def update_step(self, simulation, stepName, params): user = getCurrentUser() immutable = ['type', 'folderId'] updates = getBodyJson() if stepName not in simulation.get('steps', {}): raise RestException( 'Simulation %s doesn\'t contain step %s' % (simulation['_id'], stepName), 400) for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) try: ref_resolver = jsonschema.RefResolver.from_schema( schema.definitions) jsonschema.validate(updates, schema.definitions['stepUpdate'], resolver=ref_resolver) except jsonschema.ValidationError as ve: raise RestException(ve.message, 400) status = updates.get('status') metadata = updates.get('metadata') export = updates.get('export') view = updates.get('view') return self._model.update_step(user, simulation, stepName, status, metadata, export, view)
def update(self, simulation, params): immutable = [ 'projectId', 'folderId', 'access', 'userId', '_id', 'updated', 'created' ] updates = getBodyJson() for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) user = getCurrentUser() name = updates.get('name') description = updates.get('description') active = updates.get('active') disabled = updates.get('disabled') status = updates.get('status') metadata = updates.get('metadata') steps = updates.get('steps') return self._model.update(user, simulation, name=name, metadata=metadata, description=description, active=active, disabled=disabled, status=status, steps=steps)
def append_to_log(self, id, params): user = getCurrentUser() if not self._model.load(id, user=user, level=AccessType.ADMIN): raise RestException('Volume not found.', code=404) return self._model.append_to_log(user, id, getBodyJson())
def create(self, params): body = getBodyJson() # Default ec2 cluster cluster_type = 'ec2' if 'type' in body: if not ClusterType.is_valid_type(body['type']): raise RestException('Invalid cluster type.', code=400) cluster_type = body['type'] if cluster_type == ClusterType.EC2: cluster = self._create_ec2(params, body) elif cluster_type == ClusterType.ANSIBLE: cluster = self._create_ansible(params, body) elif cluster_type == ClusterType.TRADITIONAL: cluster = self._create_traditional(params, body) elif cluster_type == ClusterType.NEWT: cluster = self._create_newt(params, body) else: raise RestException('Invalid cluster type.', code=400) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/clusters/%s' % cluster['_id'] return self._model.filter(cluster, self.getCurrentUser())
def update_step(self, simulation, stepName, params): user = getCurrentUser() immutable = ['type', 'folderId'] updates = getBodyJson() if stepName not in simulation.get('steps', {}): raise RestException('Simulation %s doesn\'t contain step %s' % (simulation['_id'], stepName), 400) for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) try: ref_resolver = jsonschema.RefResolver.from_schema( schema.definitions) jsonschema.validate( updates, schema.definitions['stepUpdate'], resolver=ref_resolver) except jsonschema.ValidationError as ve: raise RestException(ve.message, 400) status = updates.get('status') metadata = updates.get('metadata') export = updates.get('export') view = updates.get('view') return self._model.update_step( user, simulation, stepName, status, metadata, export, view)
def create_file(self, assetstore, params): params = getBodyJson() self.requireParams(('name', 'itemId', 'size', 'path'), params) name = params['name'] item_id = params['itemId'] size = int(params['size']) path = params['path'] user = self.getCurrentUser() mime_type = params.get('mimeType') item = self.model('item').load(id=item_id, user=user, level=AccessType.WRITE, exc=True) file = self.model('file').createFile(name=name, creator=user, item=item, reuseExisting=True, assetstore=assetstore, mimeType=mime_type, size=size) file['path'] = path file['imported'] = True self.model('file').save(file) return self.model('file').filter(file)
def move_files(params): def load_file(file): return ModelImporter.model('file').load(file['_id'], user=getCurrentUser()) files = list(map(load_file, getBodyJson())) total_size = sum([file['size'] for file in files]) move_files_to_assetstore(files, total_size)
def handle_log_record(self, id, params): user = self.getCurrentUser() if not self._model.load(id, user=user, level=AccessType.ADMIN): raise RestException('Cluster not found.', code=404) return self._model.append_to_log( user, id, getBodyJson())
def append_to_log(self, id, params): user = getCurrentUser() if not self._model.load(id, user=user, level=AccessType.ADMIN): raise RestException('Volume not found.', code=404) return self._model.append_to_log( user, id, getBodyJson())
def create(self, params): project = getBodyJson() project = self.model('project', 'hpccloud').create(getCurrentUser(), project) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/projects/%s' % project['_id'] return project
def _loadJsonBody(self, name, info): val = None if cherrypy.request.body.length == 0 and info['required']: raise RestException('JSON parameter %s must be passed in request body.' % name) elif cherrypy.request.body.length > 0: val = getBodyJson() self._validateJsonType(name, info, val) return val
def refine(self, params): params = getBodyJson() r = requests.put(self.search_url + '/refine', data={ 'sid': params['sid'], 'pos_uuids': json.dumps(params['pos_uuids']), 'neg_uuids': json.dumps(params['neg_uuids']) }) return r.json()
def clone(self, simulation, params): props = getBodyJson() self.requireParams(('name', ), props) user = getCurrentUser() cloned = self._model.clone(user, simulation, props['name']) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/simulations/%s' \ % cloned['_id'] return cloned
def create_simulation(self, project, params): simulation = getBodyJson() user = getCurrentUser() simulation = self.model('simulation', 'hpccloud').create( user, project, simulation) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/simulations/%s' \ % simulation['_id'] return simulation
def create_task(self, taskflow, params): user = getCurrentUser() task = getBodyJson() self.requireParams(['celeryTaskId'], task) task = self.model('task', 'taskflow').create(user, taskflow, task) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/tasks/%s' % task['_id'] return task
def create_simulation(self, project, params): simulation = getBodyJson() user = getCurrentUser() simulation = self.model('simulation', 'hpccloud').create(user, project, simulation) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/simulations/%s' \ % simulation['_id'] return simulation
def update_profile(user, profile, params): body = getBodyJson() properties = [ 'accessKeyId', 'secretAccessKey', 'status', 'errorMessage', 'publicIPs', 'cloudProvider' ] for prop in properties: if prop in body: profile[prop] = body[prop] ModelImporter.model('aws', 'cumulus').update_aws_profile(getCurrentUser(), profile)
def create(self, params): user = self.getCurrentUser() body = getBodyJson() self.requireParams(['name'], body) if 'commands' not in body and 'scriptId' not in body: raise RestException('command or scriptId must be provided', code=400) if 'scriptId' in body: script = ModelImporter.model('script', 'cumulus').load(body['scriptId'], user=user, level=AccessType.READ) if not script: raise RestException('Script not found', 400) del body['scriptId'] body['commands'] = script['commands'] if 'onTerminate' in body and 'scriptId' in body['onTerminate']: script = ModelImporter.model('script', 'cumulus') \ .load(body['onTerminate']['scriptId'], user=user, level=AccessType.READ) if not script: raise RestException('onTerminate script not found', 400) del body['onTerminate']['scriptId'] body['onTerminate']['commands'] = script['commands'] if 'input' in body: if not isinstance(body['input'], list): raise RestException('input must be a list', 400) for i in body['input']: if i['path'] == body['name']: raise RestException('input can\'t be the same as job name', 400) if 'output' in body: if not isinstance(body['output'], list): raise RestException('output must be a list', 400) job = self._model.create(user, body) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/jobs/%s' % job['_id'] return self._clean(job)
def create_calc(self, params): body = getBodyJson() if 'cjson' not in body and ('fileId' not in body or 'format' not in body): raise RestException('Either cjson or fileId is required.') user = getCurrentUser() cjson = body.get('cjson') props = body.get('properties', {}) molecule_id = body.get('moleculeId', None) geometry_id = body.get('geometryId', None) public = body.get('public', True) notebooks = body.get('notebooks', []) image = body.get('image') input_parameters = body.get('input', {}).get('parameters') if input_parameters is None: input_parameters = body.get('inputParameters', {}) file_id = None file_format = body.get('format', 'cjson') if 'fileId' in body: file = File().load(body['fileId'], user=getCurrentUser()) file_id = file['_id'] cjson = self._file_to_cjson(file, file_format) if molecule_id is None: mol = create_molecule(json.dumps(cjson), 'cjson', user, public, parameters=input_parameters) molecule_id = mol['_id'] calc = CalculationModel().create_cjson( user, cjson, props, molecule_id, geometry_id=geometry_id, image=image, input_parameters=input_parameters, file_id=file_id, notebooks=notebooks, public=public) cherrypy.response.status = 201 cherrypy.response.headers['Location'] \ = '/calculations/%s' % (str(calc['_id'])) return CalculationModel().filter(calc, user)
def update(self, task, params): immutable = ['access', '_id', 'celeryTaskId', 'log'] user = getCurrentUser() updates = getBodyJson() if not updates: raise RestException('A body must be provided', code=400) for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) status = updates.get('status') return self._model.update_task(user, task, status=status)
def create(self, params): user = self.getCurrentUser() body = getBodyJson() self.requireParams(['name'], body) if 'commands' not in body and 'scriptId' not in body: raise RestException('command or scriptId must be provided', code=400) if 'scriptId' in body: script = self.model('script', 'cumulus').load(body['scriptId'], user=user, level=AccessType.READ) if not script: raise RestException('Script not found', 400) del body['scriptId'] body['commands'] = script['commands'] if 'onTerminate' in body and 'scriptId' in body['onTerminate']: script = self.model('script', 'cumulus') \ .load(body['onTerminate']['scriptId'], user=user, level=AccessType.READ) if not script: raise RestException('onTerminate script not found', 400) del body['onTerminate']['scriptId'] body['onTerminate']['commands'] = script['commands'] if 'input' in body: if not isinstance(body['input'], list): raise RestException('input must be a list', 400) for i in body['input']: if i['path'] == body['name']: raise RestException('input can\'t be the same as job name', 400) if 'output' in body: if not isinstance(body['output'], list): raise RestException('output must be a list', 400) job = self._model.create(user, body) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/jobs/%s' % job['_id'] return self._clean(job)
def append_to_log(self, id, params): user = self.getCurrentUser() job = self._model.load(id, user=user, level=AccessType.WRITE) if not job: raise RestException('Job not found.', code=404) body = getBodyJson() if not body: raise RestException('Log entry must be provided', code=400) return self._model.append_to_log(user, id, body)
def create_assetstore(self, params): """Create a new SFTP assetstore.""" params = getBodyJson() self.requireParams(('name', 'host', 'user'), params) return self.model('assetstore').save({ 'type': AssetstoreType.SFTP, 'name': params.get('name'), 'sftp': { 'host': params.get('host'), 'user': params.get('user'), 'authKey': params.get('authKey') } })
def update(self, project, params): immutable = ['type', 'steps', 'folderId', 'access', 'userId', '_id', 'created', 'updated'] updates = getBodyJson() for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) user = getCurrentUser() name = updates.get('name') metadata = updates.get('metadata') description = updates.get('description') return self._model.update(user, project, name=name, metadata=metadata, description=description)
def update(self, taskflow, params): user = self.getCurrentUser() immutable = [ 'access', '_id', 'taskFlowClass', 'log', 'activeTaskCount' ] updates = getBodyJson() if not updates: raise RestException('A body must be provided', code=400) for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) taskflow = self._model.update_taskflow(user, taskflow, updates) return taskflow
def start(self, taskflow, params): user = self.getCurrentUser() try: params = getBodyJson() except RestException: params = {} constructor = load_class(taskflow['taskFlowClass']) token = self.model('token').createToken(user=user, days=7) workflow = constructor(id=str(taskflow['_id']), girder_token=token['_id'], girder_api_url=cumulus.config.girder.baseUrl) workflow.start(**params)
def update(self, id, params): body = getBodyJson() user = self.getCurrentUser() cluster = self._model.load(id, user=user, level=AccessType.WRITE) if not cluster: raise RestException('Cluster not found.', code=404) if 'assetstoreId' in body: cluster['assetstoreId'] = body['assetstoreId'] if 'status' in body: if ClusterStatus.valid(body['status']): cluster['status'] = body['status'] else: raise RestException('%s is not a valid cluster status' % body['status'], code=400) if 'timings' in body: if 'timings' in cluster: cluster['timings'].update(body['timings']) else: cluster['timings'] = body['timings'] if 'config' in body: # Need to check we aren't try to update immutable fields immutable_paths = ['_id', 'ssh.user'] for path in immutable_paths: if parse(path).find(body['config']): raise RestException("The '%s' field can't be updated" % path) update_dict(cluster['config'], body['config']) cluster = self._model.update_cluster(user, cluster) # Now do any updates the adapter provides adapter = get_cluster_adapter(cluster) try: adapter.update(body) # Skip adapter.update if update not defined for this adapter except (NotImplementedError, ValidationException): pass return self._model.filter(cluster, user)
def share(self, project, params): body = getBodyJson() user = getCurrentUser() # Validate we have been given a value body try: ref_resolver = jsonschema.RefResolver.from_schema( schema.definitions) jsonschema.validate(body, schema.project['definitions']['share'], resolver=ref_resolver) except jsonschema.ValidationError as ve: raise RestException(ve.message, 400) users = body.get('users', []) groups = body.get('groups', []) return self._model.share(user, project, users, groups)
def attach(self, volume, cluster, params): body = getBodyJson() self.requireParams(['path'], body) path = body['path'] profile_id = parse('profileId').find(volume)[0].value profile, secret_key = _get_profile(profile_id) girder_callback_info = { 'girder_api_url': cumulus.config.girder.baseUrl, 'girder_token': get_task_token()['_id'] } log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl, volume['_id']) p = CloudProvider(dict(secretAccessKey=secret_key, **profile)) aws_volume = p.get_volume(volume) # If volume exists it needs to be available to be attached. If # it doesn't exist it will be created as part of the attach # playbook. if aws_volume is not None and \ aws_volume['state'] != VolumeState.AVAILABLE: raise RestException( 'This volume is not available to attach ' 'to a cluster', 400) master = p.get_master_instance(cluster['_id']) if master['state'] != InstanceState.RUNNING: raise RestException('Master instance is not running!', 400) cluster = ModelImporter.model('cluster', 'cumulus').filter(cluster, getCurrentUser(), passphrase=False) cumulus.ansible.tasks.volume.attach_volume\ .delay(profile, cluster, master, self._model.filter(volume, getCurrentUser()), path, secret_key, log_write_url, girder_callback_info) volume['status'] = VolumeState.ATTACHING volume = self._model.update_volume(getCurrentUser(), volume) return self._model.filter(volume, getCurrentUser())
def patch(self, volume, params): body = getBodyJson() if not volume: raise RestException('Volume not found.', code=404) if 'ec2' in body: if 'ec2' not in volume: volume['ec2'] = {} volume['ec2'].update(body['ec2']) mutable = ['status', 'msg', 'path'] for k in mutable: if k in body: volume[k] = body[k] user = getCurrentUser() volume = self._model.update_volume(user, volume) return self._model.filter(volume, user)
def attach(self, volume, cluster, params): body = getBodyJson() self.requireParams(['path'], body) path = body['path'] profile_id = parse('profileId').find(volume)[0].value profile, secret_key = _get_profile(profile_id) girder_callback_info = { 'girder_api_url': getApiUrl(), 'girder_token': get_task_token()['_id']} p = CloudProvider(dict(secretAccessKey=secret_key, **profile)) aws_volume = p.get_volume(volume) # If volume exists it needs to be available to be attached. If # it doesn't exist it will be created as part of the attach # playbook. if aws_volume is not None and \ aws_volume['state'] != VolumeState.AVAILABLE: raise RestException('This volume is not available to attach ' 'to a cluster', 400) master = p.get_master_instance(cluster['_id']) if master['state'] != InstanceState.RUNNING: raise RestException('Master instance is not running!', 400) cluster = self.model('cluster', 'cumulus').filter( cluster, getCurrentUser(), passphrase=False) cumulus.ansible.tasks.volume.attach_volume\ .delay(profile, cluster, master, self._model.filter(volume, getCurrentUser()), path, secret_key, girder_callback_info) volume['status'] = VolumeState.ATTACHING volume = self._model.update_volume(getCurrentUser(), volume) return self._model.filter(volume, getCurrentUser())
def update(self, simulation, params): immutable = ['projectId', 'folderId', 'access', 'userId', '_id', 'steps', 'updated', 'created'] updates = getBodyJson() for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) user = getCurrentUser() name = updates.get('name') description = updates.get('description') active = updates.get('active') disabled = updates.get('disabled') status = updates.get('status') metadata = updates.get('metadata') return self._model.update(user, simulation, name=name, metadata=metadata, description=description, active=active, disabled=disabled, status=status)
def update(self, project, params): immutable = [ 'type', 'steps', 'folderId', 'access', 'userId', '_id', 'created', 'updated' ] updates = getBodyJson() for p in updates: if p in immutable: raise RestException('\'%s\' is an immutable property' % p, 400) user = getCurrentUser() name = updates.get('name') metadata = updates.get('metadata') description = updates.get('description') return self._model.update(user, project, name=name, metadata=metadata, description=description)
def create(self, params): user = self.getCurrentUser() taskflow = getBodyJson() self.requireParams(['taskFlowClass'], taskflow) # Check that we can load the class try: load_class(taskflow['taskFlowClass']) except Exception as ex: msg = 'Unable to load taskflow class: %s (%s)' % \ (taskflow['taskFlowClass'], ex) logger.exception(msg) traceback.print_exc() raise RestException(msg, 400) taskflow = self._model.create(user, taskflow) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/taskflows/%s' % \ taskflow['_id'] return taskflow
def create_file(self, assetstore, params): params = getBodyJson() self.requireParams(('name', 'itemId', 'size', 'path'), params) name = params['name'] item_id = params['itemId'] size = int(params['size']) path = params['path'] user = self.getCurrentUser() mime_type = params.get('mimeType') item = self.model('item').load(id=item_id, user=user, level=AccessType.WRITE, exc=True) file = self.model('file').createFile( name=name, creator=user, item=item, reuseExisting=True, assetstore=assetstore, mimeType=mime_type, size=size) file['path'] = path file['imported'] = True self.model('file').save(file) return self.model('file').filter(file)
def add_entry(self, params): body = getBodyJson() if "key" not in body: raise RestException("key is required", code=400) if "host" not in body: raise RestException("host is required", code=400) if "port" not in body: raise RestException("port is required", code=400) key = body["key"] host = body["host"] port = body["port"] with LockFile(self._proxy_file_path): db = None try: db = dbm.open(self._proxy_file_path, "c") # Encode the slash db[key] = "%s:%s" % (host, port) finally: if db: db.close()
def add_entry(self, params): body = getBodyJson() if 'key' not in body: raise RestException('key is required', code=400) if 'host' not in body: raise RestException('host is required', code=400) if 'port' not in body: raise RestException('port is required', code=400) key = body['key'] host = body['host'] port = body['port'] with LockFile(self._proxy_file_path): db = None try: db = dbm.open(self._proxy_file_path, 'c') # Encode the slash db[key] = '%s:%s' % (host, port) finally: if db: db.close()
def create_calc(self, params): body = getBodyJson() if 'cjson' not in body and ('fileId' not in body or 'format' not in body): raise RestException('Either cjson or fileId is required.') user = getCurrentUser() cjson = body.get('cjson') props = body.get('properties', {}) molecule_id = body.get('moleculeId', None) public = body.get('public', False) notebooks = body.get('notebooks', []) image = body.get('image') input_parameters = body.get('input', {}).get('parameters') file_id = None file_format = body.get('format', 'cjson') if 'fileId' in body: file = File().load(body['fileId'], user=getCurrentUser()) file_id = file['_id'] cjson = self._file_to_cjson(file, file_format) if molecule_id is None: mol = create_molecule(json.dumps(cjson), 'cjson', user, public) molecule_id = mol['_id'] calc = CalculationModel().create_cjson(user, cjson, props, molecule_id, image=image, input_parameters=input_parameters, file_id=file_id, notebooks=notebooks, public=public) cherrypy.response.status = 201 cherrypy.response.headers['Location'] \ = '/calculations/%s' % (str(calc['_id'])) return CalculationModel().filter(calc, user)
def update_properties(self, calculation, params): props = getBodyJson() calculation['properties'] = props calculation = self._model.save(calculation) return calculation
def create(self, params): params = getBodyJson() self.requireParams(('name', 'machine'), params) return create_assetstore(params)