Пример #1
0
            "description": "The last time when the spec was modified."
        }
    },
    'example': {
        '_accessLevel': 2,
        '_id': '5873dcdbaec030000144d233',
        '_modelType': 'spec',
        'name': 'canopy',
        'description': 'Canopy model for fake plant',
        'creatorId': '18312dcdbaec030000144d233',
        'created': '2017-01-09T18:56:27.262000+00:00',
        'official': True,
        'updated': '2017-01-10T16:15:17.313000+00:00'
    },
}
addModel('spec', specDef, resources='spec')


class Spec(Resource):
    """Defines spec API."""

    def __init__(self):
        """Initialize spec API."""
        super(Spec, self).__init__()
        self.resourceName = 'spec'
        self._model = SpecModel()
        self.route('GET', (), self.listSpecs)
        self.route('GET', (':id',), self.getSpec)
        self.route('POST', (), self.createSpec)
        self.route('PUT', (':id',), self.updateSpec)
        self.route('DELETE', (':id',), self.deleteSpec)
Пример #2
0
        'status': {
            'type': 'integer',
            'format': 'int32',
            'allowEmptyValue': False,
            'maximum': 1,
            'minimum': 0
        },
        'taleId': {
            'type': 'string'
        },
        'url': {
            'type': 'string'
        }
    }
}
addModel('instance', instanceSchema, resources='instance')
instanceCapErrMsg = ('You have reached a limit for running instances ({}). '
                     'Please shutdown one of the running instances before '
                     'continuing.')


class Instance(Resource):
    def __init__(self):
        super(Instance, self).__init__()
        self.resourceName = 'instance'
        self._model = instanceModel()

        self.route('GET', (), self.listInstances)
        self.route('POST', (), self.createInstance)
        self.route('GET', (':id', ), self.getInstance)
        self.route('DELETE', (':id', ), self.deleteInstance)
Пример #3
0
                    }
                }]
            },
            'Arctic Slope Shoreline Change Susceptibility Spatial Data Model, 2015-16':
            {
                'fileList': [{
                    'science_metadata.xml': {
                        'size': 10491
                    }
                }]
            }
        }
    }
}

addModel('dataMap', dataMap)
addModel('fileMap', fileMap)


def _http_lookup(pid):
    url = urlparse(pid)
    if url.scheme not in ('http', 'https'):
        return
    headers = requests.head(pid).headers

    valid_target = headers.get('Content-Type') is not None
    valid_target = valid_target and ('Content-Length' in headers
                                     or 'Content-Range' in headers)
    if not valid_target:
        return
Пример #4
0
class NewtAssetstore(Resource):
    def __init__(self):
        super(NewtAssetstore, self).__init__()
        self.resourceName = 'newt_assetstores'

        self.route('POST', (), self.create)
        self.route('POST', (':id', 'files'), self.create_file)

    @access.user
    @loadmodel(model='assetstore')
    def create_file(self, assetstore, params):
        params = getBodyJson()
        self.requireParams(('name', 'itemId', 'size', 'path'), params)
        name = params['name']
        item_id = params['itemId']
        size = int(params['size'])
        path = params['path']
        user = self.getCurrentUser()

        mime_type = params.get('mimeType')
        item = Item().load(id=item_id, user=user,
                           level=AccessType.WRITE, exc=True)

        file = File().createFile(
                        name=name, creator=user, item=item, reuseExisting=True,
                        assetstore=assetstore, mimeType=mime_type, size=size)

        file['path'] = path
        file['imported'] = True
        File().save(file)

        return File().filter(file)

    addModel('CreateFileParams', {
        'id': 'CreateFileParams',
        'required': ['name', 'itemId', 'size', 'path'],
        'properties': {
            'name': {'type': 'string',
                     'description': 'The name of the file.'},
            'itemId':  {'type': 'string',
                          'description': 'The item to attach the file to.'},
            'size': {'type': 'number',
                       'description': 'The size of the file.'},
            'path': {'type': 'string',
                       'description': 'The full path to the file.'},
            'mimeType': {'type': 'string',
                       'description': 'The the mimeType of the file.'},

            }
        }, 'newt')

    create_file.description = (
         Description('Create a new file in this assetstore.')
        .param('id', 'The the assetstore to create the file in', required=True,
               paramType='path')
        .param('body', 'The parameter to create the file with.', required=True,
               paramType='body', dataType='CreateFileParams'))

    @access.user
    def create(self, params):
        params = getBodyJson()
        self.requireParams(('name', 'machine'), params)

        return create_assetstore(params)

    addModel('CreateAssetstoreParams', {
        'id': 'CreateAssetstoreParams',
        'required': ['name', 'itemId', 'size', 'path'],
        'properties': {
            'name': {'type': 'string',
                     'description': '.'},
            'machine':  {'type': 'string',
                          'description': 'The host where files are stored.'}
            }
        }, 'newt')

    create.description = (
     Description('Create a new NEWT assetstore.')
    .param('body', 'The parameter to create the assetstore', required=True,
               paramType='body', dataType='CreateAssetstoreParams'))
Пример #5
0
        '_modelType': {'type': 'string'},
        'containerId': {'type': 'string'},
        'containerPath': {'type': 'string'},
        'created': {'type': 'string', 'format': 'date'},
        'folderId': {'type': 'string'},
        'frontendId': {'type': 'string'},
        'lastActivity': {'type': 'string', 'format': 'date'},
        'mountPoint': {'type': 'string'},
        'status': {'type': 'integer', 'format': 'int32',
                   'allowEmptyValue': False,
                   'maximum': 1, 'minimum': 0},
        'userId': {'type': 'string'},
        'when': {'type': 'string', 'format': 'date'},
    }
}
addModel('notebook', notebookModel, resources='notebook')


class Notebook(Resource):

    def __init__(self):
        super(Notebook, self).__init__()
        self.resourceName = 'notebook'

        self.route('GET', (), self.listNotebooks)
        self.route('POST', (), self.createNotebook)
        self.route('GET', (':id',), self.getNotebook)
        self.route('DELETE', (':id',), self.deleteNotebook)

        self.lastCulling = datetime.datetime.utcnow()
Пример #6
0
class Molecule(Resource):
    output_formats = ['cml', 'xyz', 'inchikey', 'sdf', 'cjson']
    input_formats = [
        'cml', 'xyz', 'sdf', 'cjson', 'json', 'log', 'nwchem', 'pdb'
    ]
    mime_types = {
        'cml': 'chemical/x-cml',
        'xyz': 'chemical/x-xyz',
        'sdf': 'chemical/x-mdl-sdfile',
        'cjson': 'application/json'
    }

    def __init__(self):
        super(Molecule, self).__init__()
        self.resourceName = 'molecules'
        self.route('GET', (), self.find)
        self.route('GET', ('inchikey', ':inchikey'), self.find_inchikey)
        self.route('GET', (':id', ':output_format'), self.get_format)
        self.route('GET', (':id', ), self.find_id)
        self.route('GET', ('search', ), self.search)
        self.route('POST', (), self.create)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PATCH', (':id', ), self.update)
        self.route('PATCH', (':id', 'notebooks'), self.add_notebooks)
        self.route('POST', ('conversions', ':output_format'), self.conversions)

        self._model = self.model('molecule', 'molecules')
        self._calc_model = self.model('calculation', 'molecules')

    def _clean(self, doc):
        del doc['access']
        if 'sdf' in doc:
            del doc['sdf']
        doc['_id'] = str(doc['_id'])
        if 'cjson' in doc:
            if 'basisSet' in doc['cjson']:
                del doc['cjson']['basisSet']
            if 'vibrations' in doc['cjson']:
                del doc['cjson']['vibrations']

        return doc

    @access.public
    def find(self, params):
        return self._model.findmol(params)

    find.description = (Description('Find a molecule.').param(
        'name', 'The name of the molecule', paramType='query',
        required=False).param('inchi',
                              'The InChI of the molecule',
                              paramType='query',
                              required=False).param(
                                  'inchikey',
                                  'The InChI key of the molecule',
                                  paramType='query',
                                  required=False).errorResponse())

    @access.public
    def find_inchikey(self, inchikey, params):
        mol = self._model.find_inchikey(inchikey)
        if not mol:
            raise RestException('Molecule not found.', code=404)
        return self._clean(mol)

    find_inchikey.description = (
        Description('Find a molecule by InChI key.').param(
            'inchi', 'The InChI key of the molecule',
            paramType='path').errorResponse().errorResponse(
                'Molecule not found.', 404))

    @access.public
    def find_id(self, id, params):
        mol = self._model.load(id,
                               level=AccessType.READ,
                               user=getCurrentUser())
        if not mol:
            raise RestException('Molecule not found.', code=404)
        return self._clean(mol)

    def _process_experimental(self, doc):
        facility_used = parse('experiment.experimentalEnvironment.facilityUsed'
                              ).find(doc)[0].value
        experiments = parse('experiment.experiments').find(doc)[0].value

        experiment_model = self.model('experimental', 'molecules')

        experiments_list = []
        for experiment in experiments:
            spectrum_type = experiment['spectrumType']
            experimental_technique = experiment['experimentalTechnique']
            id = experiment['id']
            molecular_formula = experiment['molecularFormula']
            instenisty_units = parse('measuredSpectrum.unitsY').find(
                experiment)[0].value
            frequency_units = parse('measuredSpectrum.unitsX').find(
                experiment)[0].value
            data_points = parse('measuredSpectrum.dataPoints').find(
                experiment)[0].value
            frequencies = data_points[::2]
            intensities = data_points[1::2]
            measured_spectrum = {
                'frequencies': {
                    'units': frequency_units,
                    'values': frequencies
                },
                'intensities': {
                    'units': instenisty_units,
                    'values': intensities
                }
            }

            experiments_list.append(
                experiment_model.create(facility_used, spectrum_type,
                                        experimental_technique, id,
                                        molecular_formula, measured_spectrum))

        return experiments_list

    @access.user
    def create(self, params):
        body = self.getBodyJson()
        user = self.getCurrentUser()
        public = body.get('public', False)
        if 'fileId' in body:
            file_id = body['fileId']
            calc_id = body.get('calculationId')
            file = self.model('file').load(file_id, user=user)
            parts = file['name'].split('.')
            input_format = parts[-1]
            name = '.'.join(parts[:-1])

            if input_format not in Molecule.input_formats:
                raise RestException('Input format not supported.', code=400)

            contents = functools.reduce(
                lambda x, y: x + y,
                self.model('file').download(file, headers=False)())
            data_str = contents.decode()

            # For now piggy backing experimental results upload here!
            # This should be refactored ...
            json_data = json.loads(data_str)
            if 'experiment' in json_data:
                return self._process_experimental(json_data)

            # Use the SDF format as it is the one with bonding that 3Dmol uses.
            output_format = 'sdf'

            if input_format == 'pdb':
                (output, _) = openbabel.convert_str(data_str, input_format,
                                                    output_format)
            else:
                output = avogadro.convert_str(data_str, input_format,
                                              output_format)

            # Get some basic molecular properties we want to add to the database.
            props = avogadro.molecule_properties(data_str, input_format)
            pieces = props['spacedFormula'].strip().split(' ')
            atomCounts = {}
            for i in range(0, int(len(pieces) / 2)):
                atomCounts[pieces[2 * i]] = int(pieces[2 * i + 1])

            cjson = []
            if input_format == 'cjson':
                cjson = json.loads(data_str)
            elif input_format == 'pdb':
                cjson = json.loads(avogadro.convert_str(
                    output, 'sdf', 'cjson'))
            else:
                cjson = json.loads(
                    avogadro.convert_str(data_str, input_format, 'cjson'))

            atom_count = openbabel.atom_count(data_str, input_format)

            if atom_count > 1024:
                raise RestException(
                    'Unable to generate inchi, molecule has more than 1024 atoms .',
                    code=400)

            (inchi, inchikey) = openbabel.to_inchi(output, 'sdf')

            if not inchi:
                raise RestException('Unable to extract inchi', code=400)

            # Check if the molecule exists, only create it if it does.
            molExists = self._model.find_inchikey(inchikey)
            mol = {}
            if molExists:
                mol = molExists
            else:
                # Whitelist parts of the CJSON that we store at the top level.
                cjsonmol = {}
                cjsonmol['atoms'] = cjson['atoms']
                cjsonmol['bonds'] = cjson['bonds']
                cjsonmol['chemical json'] = cjson['chemical json']
                mol = self._model.create_xyz(
                    user, {
                        'name':
                        chemspider.find_common_name(inchikey,
                                                    props['formula']),
                        'inchi':
                        inchi,
                        'inchikey':
                        inchikey,
                        output_format:
                        output,
                        'cjson':
                        cjsonmol,
                        'properties':
                        props,
                        'atomCounts':
                        atomCounts
                    }, public)

                # Upload the molecule to virtuoso
                try:
                    semantic.upload_molecule(mol)
                except requests.ConnectionError:
                    print(
                        TerminalColor.warning(
                            'WARNING: Couldn\'t connect to virtuoso.'))

            if 'vibrations' in cjson or 'basisSet' in cjson:
                # We have some calculation data, let's add it to the calcs.
                sdf = output
                moleculeId = mol['_id']
                calc_props = {}

                if calc_id is not None:
                    calc = self._calc_model.load(calc_id,
                                                 user=user,
                                                 level=AccessType.ADMIN)
                    calc_props = calc['properties']
                    # The calculation is no longer pending
                    if 'pending' in calc_props:
                        del calc_props['pending']

                if input_format == 'json':
                    jsonInput = json.loads(data_str)
                    # Don't override existing properties
                    new_calc_props = avogadro.calculation_properties(jsonInput)
                    new_calc_props.update(calc_props)
                    calc_props = new_calc_props

                # Use basisSet from cjson if we don't already have one.
                if 'basisSet' in cjson and 'basisSet' not in calc_props:
                    calc_props['basisSet'] = cjson['basisSet']

                # Use functional from cjson properties if we don't already have
                # one.
                functional = parse('properties.functional').find(cjson)
                if functional and 'functional' not in calc_props:
                    calc_props['functional'] = functional[0].value

                # Add theory priority to 'sort' calculations
                theory = calc_props.get('theory')
                functional = calc_props.get('functional')
                if theory in constants.theory_priority:
                    priority = constants.theory_priority[theory]
                    calc_props['theoryPriority'] = priority

                if calc_id is not None:
                    calc['properties'] = calc_props
                    calc['cjson'] = cjson
                    calc['fileId'] = file_id
                    self._calc_model.save(calc)
                else:
                    self._calc_model.create_cjson(user, cjson, calc_props,
                                                  moleculeId, file_id, public)

        elif 'xyz' in body or 'sdf' in body:

            if 'xyz' in body:
                input_format = 'xyz'
                data = body['xyz']
            else:
                input_format = 'sdf'
                data = body['sdf']

            (inchi, inchikey) = openbabel.to_inchi(data, input_format)

            mol = {'inchi': inchi, 'inchikey': inchikey, input_format: data}

            if 'name' in body:
                mol['name'] = body['name']

            mol = self._model.create_xyz(user, mol, public)
        elif 'inchi' in body:
            inchi = body['inchi']
            mol = self._model.create(user, inchi, public)
        else:
            raise RestException('Invalid request', code=400)

        return self._clean(mol)

    addModel(
        'Molecule', 'MoleculeParams', {
            "id": "MoleculeParams",
            "required": ["name", "inchi"],
            "properties": {
                "name": {
                    "type": "string",
                    "description": "The common name of the molecule"
                },
                "inchi": {
                    "type": "string",
                    "description": "The InChI of the molecule."
                }
            }
        })
    create.description = (Description('Create a molecule').param(
        'body',
        'The molecule to be added to the database.',
        dataType='MoleculeParams',
        required=True,
        paramType='body').errorResponse('Input format not supported.',
                                        code=400))

    @access.user
    def delete(self, id, params):
        user = self.getCurrentUser()
        mol = self._model.load(id, user=user, level=AccessType.WRITE)

        if not mol:
            raise RestException('Molecule not found.', code=404)

        return self._model.remove(mol)

    delete.description = (Description('Delete a molecule by id.').param(
        'id', 'The id of the molecule',
        paramType='path').errorResponse().errorResponse(
            'Molecule not found.', 404))

    @access.user
    def update(self, id, params):
        user = self.getCurrentUser()

        mol = self._model.load(id, user=user, level=AccessType.WRITE)

        if not mol:
            raise RestException('Molecule not found.', code=404)

        body = self.getBodyJson()

        # TODO this should be refactored to use $addToSet
        if 'logs' in body:
            logs = mol.setdefault('logs', [])
            logs += body['logs']

        mol = self._model.update(mol)

        return self._clean(mol)

    addModel(
        'Molecule', 'UpdateMoleculeParams', {
            "id": "UpdateMoleculeParams",
            "properties": {
                "logs": {
                    "type": "array",
                    "description": "List of Girder file ids"
                }
            }
        })
    update.description = (Description('Update a molecule by id.').param(
        'id', 'The id of the molecule', paramType='path').param(
            'body',
            'The update to the molecule.',
            dataType='UpdateMoleculeParams',
            required=True,
            paramType='body').errorResponse('Molecule not found.', 404))

    @access.user
    @autoDescribeRoute(
        Description('Add notebooks ( file ids ) to molecule.').modelParam(
            'id',
            'The molecule id',
            model=MoleculeModel,
            destName='molecule',
            force=True,
            paramType='path').jsonParam('notebooks',
                                        'List of notebooks',
                                        required=True,
                                        paramType='body'))
    def add_notebooks(self, molecule, notebooks):
        notebooks = notebooks.get('notebooks')
        if notebooks is not None:
            MoleculeModel().add_notebooks(molecule, notebooks)

    @access.user
    def conversions(self, output_format, params):
        user = self.getCurrentUser()

        if output_format not in Molecule.output_formats:
            raise RestException('Output output_format not supported.',
                                code=404)

        body = self.getBodyJson()

        if 'fileId' not in body:
            raise RestException('Invalid request body.', code=400)

        file_id = body['fileId']

        file = self.model('file').load(file_id, user=user)

        input_format = file['name'].split('.')[-1]

        if input_format not in Molecule.input_formats:
            raise RestException('Input format not supported.', code=400)

        if file is None:
            raise RestException('File not found.', code=404)

        contents = functools.reduce(
            lambda x, y: x + y,
            self.model('file').download(file, headers=False)())
        data_str = contents.decode()

        if output_format.startswith('inchi'):
            atom_count = 0
            if input_format == 'pdb':
                atom_count = openbabel.atom_count(data_str, input_format)
            else:
                atom_count = avogadro.atom_count(data_str, input_format)

            if atom_count > 1024:
                raise RestException(
                    'Unable to generate InChI, molecule has more than 1024 atoms.',
                    code=400)

            if input_format == 'pdb':
                (inchi, inchikey) = openbabel.to_inchi(data_str, input_format)
            else:
                sdf = avogadro.convert_str(data_str, input_format, 'sdf')
                (inchi, inchikey) = openbabel.to_inchi(sdf, 'sdf')

            if output_format == 'inchi':
                return inchi
            else:
                return inchikey

        else:
            output = ''
            mime = 'text/plain'
            if input_format == 'pdb':
                (output, mime) = openbabel.convert_str(data_str, input_format,
                                                       output_format)
            else:
                output = avogadro.convert_str(data_str, input_format,
                                              output_format)

            def stream():
                cherrypy.response.headers['Content-Type'] = mime
                yield output

            return stream

    addModel(
        'Molecule', 'ConversionParams', {
            "id": "ConversionParams",
            "properties": {
                "fileId": {
                    "type": "string",
                    "description": "Girder file id to do conversion on"
                }
            }
        })
    conversions.description = (Description('Update a molecule by id.').param(
        'format', 'The format to convert to', paramType='path').param(
            'body',
            'Details of molecule data to perform conversion on',
            dataType='ConversionParams',
            required=True,
            paramType='body').errorResponse(
                'Output format not supported.',
                404).errorResponse('File not found.', 404).errorResponse(
                    'Invalid request body.',
                    400).errorResponse('Input format not supported.',
                                       code=400))

    @access.public
    def get_format(self, id, output_format, params):
        # For now will for force load ( i.e. ignore access control )
        # This will change when we have access controls.
        molecule = self._model.load(id, force=True)

        if output_format not in Molecule.output_formats:
            raise RestException('Format not supported.', code=400)

        data = json.dumps(molecule['cjson'])
        if output_format != 'cjson':
            data = avogadro.convert_str(data, 'cjson', output_format)

        def stream():
            cherrypy.response.headers['Content-Type'] = Molecule.mime_types[
                output_format]
            yield data

        return stream

    get_format.description = (
        Description('Get molecule in particular format.').param(
            'id', 'The id of the molecule',
            paramType='path').param('output_format',
                                    'The format to convert to',
                                    paramType='path').errorResponse(
                                        'Output format not supported.', 400))

    @access.public
    def search(self, params):

        query_string = params.get('q')
        formula = params.get('formula')
        cactus = params.get('cactus')
        if query_string is None and formula is None and cactus is None:
            raise RestException(
                'Either \'q\', \'formula\' or \'cactus\' is required.')

        if query_string is not None:
            try:
                mongo_query = query.to_mongo_query(query_string)
            except query.InvalidQuery:
                raise RestException('Invalid query', 400)

            mols = []
            for mol in self._model.find(query=mongo_query,
                                        fields=['_id', 'inchikey', 'name']):
                mol['id'] = mol['_id']
                del mol['_id']
                mols.append(mol)

            return mols

        elif formula:
            # Search using formula
            return list(self._model.find_formula(formula, getCurrentUser()))
        elif cactus:
            # Disable cert verification for now
            # TODO Ensure we have the right root certs so this just works.
            r = requests.get(
                'https://cactus.nci.nih.gov/chemical/structure/%s/sdf' %
                cactus,
                verify=False)

            if r.status_code == 404:
                return []
            else:
                r.raise_for_status()

            (inchi, inchikey) = openbabel.to_inchi(r.content.decode('utf8'),
                                                   'sdf')

            # See if we already have a molecule
            mol = self._model.find_inchikey(inchikey)

            # Create new molecule
            if mol is None:

                cjson_str = avogadro.convert_str(r.content, 'sdf', 'cjson')
                mol = {
                    'cjson': json.loads(cjson_str),
                    'inchikey': inchikey,
                    'origin': 'cactus'
                }

                user = getCurrentUser()
                if user is not None:
                    mol = self._model.create_xyz(getCurrentUser(),
                                                 mol,
                                                 public=True)

            return [mol]

    search.description = (Description(
        'Search for molecules using a query string or formula').param(
            'q',
            'The query string to use for this search',
            paramType='query',
            required=False).param('formula',
                                  'The formula to search for',
                                  paramType='query',
                                  required=False).param(
                                      'cactus',
                                      'The identifier to pass to cactus',
                                      paramType='query',
                                      required=False))
Пример #7
0
    'required': ['dataId', 'repository', 'doi', 'name', 'size'],
    'example': {
        'dataId':
        'urn:uuid:42969280-e11c-41a9-92dc-33964bf785c8',
        'doi':
        '10.5063/F1Z899CZ',
        'name': ('Data from a dynamically downscaled projection of past and '
                 'future microclimates covering North America from 1980-1999 '
                 'and 2080-2099'),
        'repository':
        'DataONE',
        'size':
        178679
    },
}
addModel('dataMap', dataMap)


def _http_lookup(pid):
    url = urlparse(pid)
    if url.scheme not in ('http', 'https'):
        return
    headers = requests.head(pid).headers

    valid_target = headers.get('Content-Type') is not None
    valid_target = valid_target and ('Content-Length' in headers
                                     or 'Content-Range' in headers)
    if not valid_target:
        return

    if 'Content-Disposition' in headers:
Пример #8
0
class Job(BaseResource):
    def __init__(self):
        super(Job, self).__init__()
        self.resourceName = 'jobs'
        self.route('POST', (), self.create)
        self.route('PATCH', (':id', ), self.update)
        self.route('GET', (':id', 'status'), self.status)
        self.route('PUT', (':id', 'terminate'), self.terminate)
        self.route('POST', (':id', 'log'), self.append_to_log)
        self.route('GET', (':id', 'log'), self.log)
        self.route('GET', (':id', 'output'), self.output)
        self.route('DELETE', (':id', ), self.delete)
        self.route('GET', (':id', ), self.get)
        self.route('GET', (), self.find)

        self._model = ModelImporter.model('job', 'cumulus')

    def _clean(self, job):
        del job['access']
        del job['log']
        job['_id'] = str(job['_id'])
        job['userId'] = str(job['userId'])

        return job

    @access.user
    def create(self, params):
        user = self.getCurrentUser()

        body = getBodyJson()

        self.requireParams(['name'], body)

        if 'commands' not in body and 'scriptId' not in body:
            raise RestException('command or scriptId must be provided',
                                code=400)

        if 'scriptId' in body:
            script = ModelImporter.model('script',
                                         'cumulus').load(body['scriptId'],
                                                         user=user,
                                                         level=AccessType.READ)
            if not script:
                raise RestException('Script not found', 400)

            del body['scriptId']
            body['commands'] = script['commands']

        if 'onTerminate' in body and 'scriptId' in body['onTerminate']:
            script = ModelImporter.model('script', 'cumulus') \
                .load(body['onTerminate']['scriptId'], user=user,
                      level=AccessType.READ)
            if not script:
                raise RestException('onTerminate script not found', 400)

            del body['onTerminate']['scriptId']
            body['onTerminate']['commands'] = script['commands']

        if 'input' in body:
            if not isinstance(body['input'], list):
                raise RestException('input must be a list', 400)

            for i in body['input']:
                if i['path'] == body['name']:
                    raise RestException('input can\'t be the same as job name',
                                        400)

        if 'output' in body:
            if not isinstance(body['output'], list):
                raise RestException('output must be a list', 400)

        job = self._model.create(user, body)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/jobs/%s' % job['_id']

        return self._clean(job)

    addModel(
        'JobOnCompleteParams', {
            'id': 'JobOnCompleteParams',
            'properties': {
                'cluster': {
                    'type':
                    'string',
                    'enum': ['terminate'],
                    'description':
                    'Cluster operation to perform when job '
                    'is complete.'
                }
            }
        }, 'jobs')

    addModel(
        'InputItem', {
            'id': 'InputItem',
            'properties': {
                'itemId': {
                    'type': 'string',
                    'description': 'The item id'
                },
                'path': {
                    'type': 'string',
                    'description': 'The path to download this item to'
                }
            }
        }, 'jobs')

    addModel(
        'OutputItem', {
            'id': 'OutputItem',
            'properties': {
                'itemId': {
                    'type': 'string',
                    'description': 'The item id'
                },
                'path': {
                    'type': 'string',
                    'description': 'The path to upload, may include * wildcard'
                }
            }
        }, 'jobs')

    addModel(
        'JobParameters', {
            'id': 'JobParameters',
            'required': ['name', 'outputCollectionId'],
            'properties': {
                'commands': {
                    'type': 'array',
                    'description': 'The commands to run.',
                    'items': {
                        'type': 'string'
                    }
                },
                'scriptId': {
                    'pattern': '^[0-9a-fA-F]{24}$',
                    'type': 'string'
                },
                'name': {
                    'type': 'string',
                    'description': 'The human readable job name.'
                },
                'input': {
                    'type': 'array',
                    'description': 'Input to the job.',
                    'items': {
                        '$ref': '#/definitions/InputItem'
                    }
                },
                'output': {
                    'type': 'array',
                    'description': 'The output to upload.',
                    'items': {
                        '$ref': '#/definitions/OutputItem'
                    }
                },
                'onComplete': {
                    '$ref': '#/definitions/JobOnCompleteParams'
                }
            }
        }, 'jobs')

    create.description = (Description('Create a new job').param(
        'body',
        'The job parameters in JSON format.',
        dataType='JobParameters',
        paramType='body',
        required=True))

    @access.user
    def terminate(self, id, params):
        (user, token) = self.getCurrentUser(returnToken=True)
        job = self._model.load(id, user=user, level=AccessType.ADMIN)

        if not job:
            raise RestException('Job not found.', code=404)

        cluster_model = ModelImporter.model('cluster', 'cumulus')
        cluster = cluster_model.load(job['clusterId'],
                                     user=user,
                                     level=AccessType.ADMIN)

        base_url = cumulus.config.girder.baseUrl
        self._model.update_status(user, id, JobState.TERMINATING)

        log_url = '%s/jobs/%s/log' % (base_url, id)

        # Clean up job
        job = self._clean(job)

        girder_token = self.get_task_token()['_id']
        tasks.job.terminate_job.delay(cluster,
                                      job,
                                      log_write_url=log_url,
                                      girder_token=girder_token)

        return job

    terminate.description = (Description('Terminate a job').param(
        'id', 'The job id', paramType='path'))

    @access.user
    def update(self, id, params):
        user = self.getCurrentUser()
        body = getBodyJson()

        job = self._model.load(id, user=user, level=AccessType.WRITE)
        if not job:
            raise RestException('Job not found.', code=404)

        if 'status' in body:
            job['status'] = body['status']

        if 'queueJobId' in body:
            job['queueJobId'] = body['queueJobId']

        if 'output' in body:
            job['output'] = body['output']

        if 'timings' in body:
            if 'timings' in job:
                job['timings'].update(body['timings'])
            else:
                job['timings'] = body['timings']

        if 'dir' in body:
            job['dir'] = body['dir']

        if 'metadata' in body:
            job['metadata'] = body['metadata']

        job = self._model.update_job(user, job)

        # Don't return the access object
        del job['access']
        # Don't return the log
        del job['log']

        return job

    addModel(
        'JobUpdateParameters', {
            'id': 'JobUpdateParameters',
            'properties': {
                'status': {
                    '$ref': '#/definitions/JobStatus',
                    'description': 'The new status. (optional)'
                },
                'queueJobId': {
                    'type': 'integer',
                    'description': 'The native queue job id. (optional)'
                },
                'metadata': {
                    'type': 'object',
                    'description': 'Application metadata. (optional)'
                }
            }
        }, 'jobs')

    update.description = (Description('Update the job').param(
        'id', 'The id of the job to update', paramType='path').param(
            'body',
            'The properties to update.',
            dataType='JobUpdateParameters',
            paramType='body').notes('Internal - Used by Celery tasks'))

    @access.user
    def status(self, id, params):
        user = self.getCurrentUser()

        job = self._model.load(id, user=user, level=AccessType.READ)

        if not job:
            raise RestException('Job not found.', code=404)

        return {'status': job['status']}

    addModel(
        'JobStatus', {
            'id': 'JobStatus',
            'required': ['status'],
            'properties': {
                'status': {
                    'type':
                    'string',
                    'enum': [
                        'created', 'downloading', 'queued', 'running',
                        'uploading', 'terminating', 'terminated', 'complete',
                        'error'
                    ]
                }
            }
        }, 'jobs')

    status.description = (Description('Get the status of a job').param(
        'id', 'The job id.', paramType='path').responseClass('JobStatus'))

    @access.user
    def append_to_log(self, id, params):
        user = self.getCurrentUser()

        job = self._model.load(id, user=user, level=AccessType.WRITE)

        if not job:
            raise RestException('Job not found.', code=404)

        body = getBodyJson()

        if not body:
            raise RestException('Log entry must be provided', code=400)

        return self._model.append_to_log(user, id, body)

    append_to_log.description = None

    @access.user
    def log(self, id, params):
        user = self.getCurrentUser()
        offset = 0
        if 'offset' in params:
            offset = int(params['offset'])

        job = self._model.load(id, user=user, level=AccessType.READ)

        if not job:
            raise RestException('Job not found.', code=404)

        return {'log': job['log'][offset:]}

    log.description = (Description('Get log entries for job').param(
        'id', 'The job to get log entries for.',
        paramType='path').param('offset',
                                'The offset to start getting entries at.',
                                required=False,
                                paramType='query'))

    @access.user
    def output(self, id, params):
        user = self.getCurrentUser()

        if 'path' not in params:
            raise RestException('path parameter is required.', code=400)

        path = params['path']

        offset = 0
        if 'offset' in params:
            offset = int(params['offset'])

        job = self._model.load(id, user=user, level=AccessType.READ)

        if not job:
            raise RestException('Job not found.', code=404)

        match = None
        # Find the correct file path
        for output in job['output']:
            if output['path'] == path:
                match = output

        if not match:
            raise RestException('Output path not found', code=404)

        if 'content' not in match:
            match['content'] = []

        return {'content': match['content'][offset:]}

    output.description = (Description('Get output entries for job').param(
        'id', 'The job to get output entries for.', paramType='path').param(
            'path',
            'The path for the output file.',
            required=True,
            paramType='query').param('offset',
                                     'The offset to start getting entries at.',
                                     required=False,
                                     paramType='query'))

    @access.user
    def get(self, id, params):
        user = self.getCurrentUser()
        job = self._model.load(id, user=user, level=AccessType.READ)

        if not job:
            raise RestException('Job not found.', code=404)

        job = self._clean(job)

        return job

    get.description = (Description('Get a job').param('id',
                                                      'The job id.',
                                                      paramType='path',
                                                      required=True))

    @access.user
    @loadmodel(model='job', plugin='cumulus', level=AccessType.ADMIN)
    @describeRoute(
        Description('Delete a job').param(
            'id', 'The job id.', paramType='path',
            required=True).notes('A running job can not be deleted.'))
    def delete(self, job, params):
        user = self.getCurrentUser()
        running_state = [
            JobState.RUNNING, JobState.QUEUED, JobState.TERMINATING,
            JobState.UPLOADING
        ]

        if job['status'] in running_state:
            raise RestException('Unable to delete running job.')

        # Clean up any job output
        if 'clusterId' in job:
            cluster_model = ModelImporter.model('cluster', 'cumulus')
            cluster = cluster_model.load(job['clusterId'],
                                         user=user,
                                         level=AccessType.READ)

            # Only try to clean up if cluster is still running
            if cluster and cluster['status'] == 'running':
                cluster = cluster_model.filter(cluster, user)
                girder_token = self.get_task_token(cluster)['_id']
                tasks.job.remove_output.delay(cluster,
                                              self._clean(job.copy()),
                                              girder_token=girder_token)

        self._model.remove(job)

    @access.user
    def find(self, params):
        user = self.getCurrentUser()

        query = {'userId': user['_id']}

        limit = int(params.get('limit', 0))
        offset = int(params.get('offset', 0))

        jobs = self._model.find(query=query,
                                offset=offset,
                                limit=limit,
                                sort=[('name', SortDir.ASCENDING)])

        return [self._clean(job) for job in jobs]

    find.description = (Description('List all jobs for a given user').param(
        'offset',
        'The offset into the results',
        paramType='query',
        required=False).param('limit',
                              'Maximum number of jobs to return',
                              paramType='query',
                              required=False))
Пример #9
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import requests

from girder.api import access
from girder.api.docs import addModel
from girder.api.describe import Description, autoDescribeRoute
from girder.api.rest import Resource, filtermodel, RestException,\
    setResponseHeader, setContentDisposition

from girder.constants import AccessType, SortDir, TokenScope
from girder.utility import ziputil
from ..schema.tale import taleModel

addModel('tale', taleModel, resources='tale')


class Tale(Resource):
    def __init__(self):
        super(Tale, self).__init__()
        self.resourceName = 'tale'

        self.route('GET', (), self.listTales)
        self.route('GET', (':id', ), self.getTale)
        self.route('PUT', (':id', ), self.updateTale)
        self.route('POST', (), self.createTale)
        self.route('DELETE', (':id', ), self.deleteTale)
        self.route('GET', (':id', 'access'), self.getTaleAccess)
        self.route('PUT', (':id', 'access'), self.updateTaleAccess)
        self.route('GET', (':id', 'export'), self.exportTale)
Пример #10
0
class Calculation(Resource):
    output_formats = ['cml', 'xyz', 'inchikey', 'sdf']
    input_formats = ['cml', 'xyz', 'pdb']

    def __init__(self):
        super(Calculation, self).__init__()
        self.resourceName = 'calculations'
        self.route('POST', (), self.create_calc)
        self.route('GET', (), self.find_calc)
        self.route('GET', ('types', ), self.find_calc_types)
        self.route('GET', (':id', 'vibrationalmodes'),
                   self.get_calc_vibrational_modes)
        self.route('GET', (':id', 'vibrationalmodes', ':mode'),
                   self.get_calc_vibrational_mode)
        self.route('GET', (':id', 'sdf'), self.get_calc_sdf)
        self.route('GET', (':id', 'cjson'), self.get_calc_cjson)
        self.route('GET', (':id', 'xyz'), self.get_calc_xyz)
        self.route('GET', (':id', 'cube', ':mo'), self.get_calc_cube)
        self.route('GET', (':id', ), self.find_id)
        self.route('PUT', (':id', 'properties'), self.update_properties)
        self.route('PATCH', (':id', 'notebooks'), self.add_notebooks)

        self._model = self.model('calculation', 'molecules')
        self._cube_model = self.model('cubecache', 'molecules')

    @access.public
    def get_calc_vibrational_modes(self, id, params):

        fields = [
            'cjson..vibrations.modes', 'cjson.vibrations.intensities',
            'cjson.vibrations.frequencies', 'access'
        ]

        calc = self._model.load(id,
                                fields=fields,
                                user=getCurrentUser(),
                                level=AccessType.READ)

        del calc['access']

        return calc['cjson']['vibrations']

    get_calc_vibrational_modes.description = (Description(
        'Get the vibrational modes associated with a calculation').param(
            'id',
            'The id of the calculation to get the modes from.',
            dataType='string',
            required=True,
            paramType='path'))

    @access.public
    def get_calc_vibrational_mode(self, id, mode, params):

        try:
            mode = int(mode)
        except ValueError:
            raise ValidationException('mode number be an integer', 'mode')

        fields = ['cjson.vibrations.modes', 'access']
        calc = self._model.load(id,
                                fields=fields,
                                user=getCurrentUser(),
                                level=AccessType.READ)

        vibrational_modes = calc['cjson.vibrations']
        #frames = vibrational_modes.get('modeFrames')
        modes = vibrational_modes.get('modes', [])

        index = modes.index(mode)
        if index < 0:
            raise RestException('No such vibrational mode', 400)

        # Now select the modeFrames directly this seems to be more efficient
        # than iterating in Python
        query = {'_id': calc['_id']}

        projection = {
            'cjson.vibrations.frequencies': {
                '$slice': [index, 1]
            },
            'cjson.vibrations.intensities': {
                '$slice': [index, 1]
            },
            'cjson.vibrations.eigenVectors': {
                '$slice': [index, 1]
            }
        }

        mode = self._model.findOne(query, fields=projection)

        return mode

    get_calc_vibrational_mode.description = (Description(
        'Get a vibrational mode associated with a calculation').param(
            'id',
            'The id of the calculation that the mode is associated with.',
            dataType='string',
            required=True,
            paramType='path').param(
                'mode',
                'The index of the vibrational model to get.',
                dataType='string',
                required=True,
                paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_sdf(self, calculation, params):
        def stream():
            cherrypy.response.headers['Content-Type'] = 'chemical/x-mdl-sdfile'
            yield calculation['sdf']

        return stream

    get_calc_sdf.description = (Description(
        'Get the molecular structure of a give calculation in SDF format'
    ).param('id',
            'The id of the calculation to return the structure for.',
            dataType='string',
            required=True,
            paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_cjson(self, calculation, params):
        return calculation['cjson']

    get_calc_cjson.description = (Description(
        'Get the molecular structure of a give calculation in CJSON format'
    ).param('id',
            'The id of the calculation to return the structure for.',
            dataType='string',
            required=True,
            paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_xyz(self, calculation, params):
        data = json.dumps(calculation['cjson'])
        data = avogadro.convert_str(data, 'cjson', 'xyz')

        def stream():
            cherrypy.response.headers['Content-Type'] = Molecule.mime_types[
                'xyz']
            yield data

        return stream

    get_calc_xyz.description = (Description(
        'Get the molecular structure of a give calculation in XYZ format'
    ).param('id',
            'The id of the calculation to return the structure for.',
            dataType='string',
            required=True,
            paramType='path'))

    @access.public
    def get_calc_cube(self, id, mo, params):
        try:
            mo = int(mo)
        except ValueError:

            # Check for h**o lumo
            mo = mo.lower()
            if mo in ['h**o', 'lumo']:
                cal = self._model.load(id, force=True)
                electron_count = parse('cjson.basisSet.electronCount').find(
                    cal)
                if electron_count:
                    electron_count = electron_count[0].value
                else:
                    # Look here as well.
                    electron_count = parse('properties.electronCount').find(
                        cal)
                    if electron_count:
                        electron_count = electron_count[0].value
                    else:
                        raise RestException('Unable to access electronCount',
                                            400)

                if mo == 'h**o':
                    mo = int(electron_count / 2)
                elif mo == 'lumo':
                    mo = int(electron_count / 2 + 1)
            else:
                raise ValidationException(
                    'mo number be an integer or \'h**o\'/\'lumo\'', 'mode')

        cached = self._cube_model.find_mo(id, mo)

        # If we have a cached cube file use that.
        if cached:
            return cached['cjson']

        fields = ['cjson', 'access', 'fileId']

        # Ignoring access control on file/data for now, all public.
        calc = self._model.load(id, fields=fields, force=True)

        file_id = calc['fileId']
        file = self.model('file').load(file_id, force=True)
        parts = file['name'].split('.')
        input_format = parts[-1]
        name = '.'.join(parts[:-1])

        with self.model('file').open(file) as fp:
            data_str = fp.read().decode()

        # This is where the cube gets calculated, should be cached in future.
        cjson = avogadro.calculate_mo(data_str, mo)

        # Remove the vibrational mode data from the cube - big, not needed here.
        if 'vibrations' in cjson:
            del cjson['vibrations']

        # Cache this cube for the next time, they can take a while to generate.
        self._cube_model.create(id, mo, cjson)

        return cjson

    get_calc_cube.description = (Description(
        'Get the cube for the supplied MO of the calculation in CJSON format'
    ).param('id',
            'The id of the calculation to return the structure for.',
            dataType='string',
            required=True,
            paramType='path').param(
                'mo',
                'The molecular orbital to get the cube for.',
                dataType='string',
                required=True,
                paramType='path'))

    @access.user
    def create_calc(self, params):
        body = getBodyJson()
        self.requireParams(['cjson'], body)
        user = getCurrentUser()

        cjson = body['cjson']
        props = body.get('properties', {})
        moleculeId = body.get('moleculeId', None)
        public = body.get('public', False)
        notebooks = body.get('notebooks', [])

        calc = self._model.create_cjson(user,
                                        cjson,
                                        props,
                                        moleculeId,
                                        notebooks=notebooks,
                                        public=public)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] \
            = '/molecules/%s/calc/%s' % (id, str(calc['_id']))

        return self._model.filter(calc, user)

    # Try and reuse schema for documentation, this only partially works!
    calc_schema = CalculationModel.schema.copy()
    calc_schema['id'] = 'CalculationData'
    addModel('Calculation', 'CalculationData', calc_schema)

    create_calc.description = (Description(
        'Get the molecular structure of a give calculation in SDF format').
                               param('body',
                                     'The calculation data',
                                     dataType='CalculationData',
                                     required=True,
                                     paramType='body'))

    @access.public
    def find_calc(self, params):
        user = getCurrentUser()

        query = {}

        if 'moleculeId' in params:
            query['moleculeId'] = ObjectId(params['moleculeId'])
        if 'calculationType' in params:
            calculation_type = params['calculationType']
            if not isinstance(calculation_type, list):
                calculation_type = [calculation_type]

            query['properties.calculationTypes'] = {'$all': calculation_type}

        if 'functional' in params:
            query['properties.functional'] = params.get('functional').lower()

        if 'theory' in params:
            query['properties.theory'] = params.get('theory').lower()

        if 'basis' in params:
            query['properties.basisSet.name'] = params.get('basis').lower()

        if 'pending' in params:
            pending = toBool(params['pending'])
            query['properties.pending'] = pending
            # The absence of the field mean the calculation is not pending ...
            if not pending:
                query['properties.pending'] = {'$ne': True}

        limit = params.get('limit', 50)

        fields = [
            'cjson.vibrations.modes', 'cjson.vibrations.intensities',
            'cjson.vibrations.frequencies', 'properties', 'fileId', 'access',
            'public'
        ]
        sort = None
        sort_by_theory = toBool(params.get('sortByTheory', False))
        if sort_by_theory:
            sort = [('properties.theoryPriority', pymongo.DESCENDING)]
            # Exclude calculations that don't have a theoryPriority,
            # otherwise they will appear first in the list.
            query['properties.theoryPriority'] = {'$exists': True}

        calcs = self._model.find(query, fields=fields, sort=sort)
        calcs = self._model.filterResultsByPermission(calcs,
                                                      user,
                                                      AccessType.READ,
                                                      limit=int(limit))
        calcs = [self._model.filter(x, user) for x in calcs]

        not_sortable = []
        if sort_by_theory and len(calcs) < int(limit):
            # Now select any calculations without theoryPriority
            query['properties.theoryPriority'] = {'$exists': False}
            not_sortable = self._model.find(query, fields=fields)
            not_sortable = self._model.filterResultsByPermission(
                not_sortable,
                user,
                AccessType.READ,
                limit=int(limit) - len(calcs))
            not_sortable = [self._model.filter(x, user) for x in not_sortable]

        return calcs + not_sortable

    find_calc.description = (
        Description('Search for particular calculation').param(
            'moleculeId',
            'The moleculeId the calculations should be associated with',
            dataType='string',
            paramType='query',
            required=False).param(
                'calculationType',
                'The type or types of calculation being searched for',
                dataType='string',
                paramType='query',
                required=False).param(
                    'basis',
                    'The basis set used for the calculations.',
                    dataType='string',
                    paramType='query',
                    required=False).param(
                        'functional',
                        'The functional used for the calculations.',
                        dataType='string',
                        paramType='query',
                        required=False).param(
                            'theory',
                            'The theory used for the calculations.',
                            dataType='string',
                            paramType='query',
                            required=False).
        param('pending',
              'Whether the calculation is currently running.',
              dataType='boolean',
              paramType='query',
              required=False).param(
                  'limit',
                  'The max number of calculations to return',
                  dataType='integer',
                  paramType='query',
                  default=50,
                  required=False).param(
                      'sortByTheory',
                      'Sort the result by theory "priority", "best" first.',
                      dataType='boolean',
                      paramType='query',
                      default=False,
                      required=False))

    @access.public
    def find_id(self, id, params):
        cal = self._model.load(id,
                               level=AccessType.READ,
                               user=getCurrentUser())
        if not cal:
            raise RestException('Calculation not found.', code=404)
        return cal

    find_id.description = (Description('Get the calculation by id').param(
        'id',
        'The id of calculatino.',
        dataType='string',
        required=True,
        paramType='path'))

    @access.public
    def find_calc_types(self, params):
        fields = ['access', 'properties.calculationTypes']

        query = {}
        if 'moleculeId' in params:
            query['moleculeId'] = ObjectId(params['moleculeId'])

        calcs = self._model.find(query, fields=fields)

        allTypes = []
        for types in calcs:
            calc_types = parse('properties.calculationTypes').find(types)
            if calc_types:
                calc_types = calc_types[0].value
                allTypes.extend(calc_types)

        typeSet = set(allTypes)

        return list(typeSet)

    find_calc_types.description = (Description(
        'Get the calculation types available for the molecule').param(
            'moleculeId',
            'The id of the molecule we are finding types for.',
            dataType='string',
            required=True,
            paramType='query'))

    @access.token
    @autoDescribeRoute(
        Description('Update the calculation properties.').notes(
            'Override the exist properties').modelParam(
                'id',
                'The ID of the calculation.',
                model='calculation',
                plugin='molecules',
                level=AccessType.ADMIN).param(
                    'body', 'The new set of properties',
                    paramType='body').errorResponse(
                        'ID was invalid.').errorResponse(
                            'Write access was denied for the calculation.',
                            403))
    def update_properties(self, calculation, params):
        props = getBodyJson()
        calculation['properties'] = props
        calculation = self._model.save(calculation)

        return calculation

    @access.user
    @autoDescribeRoute(
        Description('Add notebooks ( file ids ) to molecule.').modelParam(
            'id',
            'The calculation id',
            model=CalculationModel,
            destName='calculation',
            force=True,
            paramType='path').jsonParam('notebooks',
                                        'List of notebooks',
                                        required=True,
                                        paramType='body'))
    def add_notebooks(self, calculation, notebooks):
        notebooks = notebooks.get('notebooks')
        if notebooks is not None:
            CalculationModel().add_notebooks(calculation, notebooks)
Пример #11
0
class Projects(Resource):
    def __init__(self):
        super(Projects, self).__init__()
        self.resourceName = 'projects'
        self.route('POST', (), self.create)
        self.route('PATCH', (':id', ), self.update)
        self.route('GET', (), self.get_all)
        self.route('DELETE', (':id', ), self.delete)
        self.route('GET', (':id', 'access'), self.get_access)
        self.route('PUT', (':id', 'access'), self.set_access)
        self.route('PATCH', (':id', 'access'), self.patch_access)
        self.route('PATCH', (':id', 'access', 'revoke'), self.revoke_access)
        self.route('POST', (':id', 'simulations'), self.create_simulation)
        self.route('GET', (':id', 'simulations'), self.simulations)
        self.route('GET', (':id', ), self.get)

        self._model = self.model('project', 'hpccloud')

    addModel('ProjectProperties', schema.project, 'projects')

    @autoDescribeRoute(
        Description('Create a new project').jsonParam(
            'project',
            'The properies of the project.',
            dataType='ProjectProperties',
            required=True,
            paramType='body'))
    @access.user
    def create(self, project, params):
        project = self.model('project',
                             'hpccloud').create(getCurrentUser(), project)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/projects/%s' % project['_id']

        return project

    @autoDescribeRoute(
        Description('Update a project').modelParam(
            'id',
            'The project to update.',
            model='project',
            plugin='hpccloud',
            level=AccessType.WRITE).jsonParam(
                'updates',
                'The properties of the project to update.',
                required=True,
                paramType='body'))
    @access.user
    def update(self, project, updates, params):
        immutable = [
            'type', 'steps', 'folderId', 'access', 'userId', '_id', 'created',
            'updated'
        ]

        for p in updates:
            if p in immutable:
                raise RestException('\'%s\' is an immutable property' % p, 400)

        user = getCurrentUser()
        name = updates.get('name')
        metadata = updates.get('metadata')
        description = updates.get('description')

        return self._model.update_project(user,
                                          project,
                                          name=name,
                                          metadata=metadata,
                                          description=description)

    @autoDescribeRoute(
        Description('Get all projects this user has access to project').
        pagingParams(defaultSort='created'))
    @access.user
    def get_all(self, limit, offset, sort, params):
        user = getCurrentUser()

        cursor = self._model.find(limit=limit, offset=offset, sort=sort)
        return list(
            self._model.filterResultsByPermission(cursor=cursor,
                                                  user=user,
                                                  level=AccessType.READ))

    @autoDescribeRoute(
        Description('Delete a project').modelParam(
            'id',
            'The project to delete.',
            model='project',
            plugin='hpccloud',
            level=AccessType.WRITE).notes(
                'Will clean up any files, items or folders associated with '
                'the project.'))
    @access.user
    def delete(self, project, params):
        user = getCurrentUser()
        self._model.delete(user, project)

    @autoDescribeRoute(
        Description('Get a particular project').modelParam(
            'id',
            'The project to get.',
            model='project',
            plugin='hpccloud',
            level=AccessType.READ))
    @access.user
    def get(self, project, params):
        return project

    addModel('ShareProperties', schema.project['definitions']['share'],
             'projects')

    @autoDescribeRoute(
        Description(
            'Share a give project with a set of users or groups').modelParam(
                'id',
                'The project to be shared.',
                model='project',
                plugin='hpccloud',
                level=AccessType.READ))
    @access.user
    def get_access(self, project):
        return project.get('access', {'groups': [], 'users': []})

    @autoDescribeRoute(
        Description(
            'Share a given project with a set of users or groups').modelParam(
                'id',
                'The project to be shared.',
                model='project',
                plugin='hpccloud',
                level=AccessType.WRITE).jsonParam(
                    'share',
                    'The users and groups to share with.',
                    dataType='ShareProperties',
                    required=True,
                    paramType='body'))
    @access.user
    def set_access(self, project, share, params):
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.project['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])

        return self._model.set_access(user, project, users, groups)

    @autoDescribeRoute(
        Description(
            'Share a given project with a set of users or groups').modelParam(
                'id',
                'The project to be shared.',
                model='project',
                plugin='hpccloud',
                level=AccessType.WRITE).jsonParam(
                    'share',
                    'The users and groups to share with.',
                    dataType='ShareProperties',
                    required=True,
                    paramType='body'))
    @access.user
    def patch_access(self, project, share, params):
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.project['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])
        level = share.get('level', 0)
        flags = share.get('flags', [])

        return self._model.patch_access(user, project, users, groups, level,
                                        flags)

    @autoDescribeRoute(
        Description('Revoke permissions for project given a set of users \
                    or groups').modelParam(
            'id',
            'The project to be unshared.',
            model='project',
            plugin='hpccloud',
            level=AccessType.WRITE).jsonParam(
                'share',
                'The users and groups to share with.',
                dataType='ShareProperties',
                required=True,
                paramType='body'))
    @access.user
    def revoke_access(self, project, share, params):
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.project['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])

        return self._model.revoke_access(user, project, users, groups)

    addModel('SimProperties', schema.simulation, 'projects')

    @autoDescribeRoute(
        Description(
            'Create a simulation associated with a project.').modelParam(
                'id',
                'The project the simulation will be created in.',
                model='project',
                plugin='hpccloud',
                level=AccessType.WRITE).jsonParam(
                    'simulation',
                    'The properties of the simulation.',
                    dataType='SimProperties',
                    required=True,
                    paramType='body'))
    @access.user
    def create_simulation(self, project, simulation, params):
        user = getCurrentUser()

        simulation = self.model('simulation',
                                'hpccloud').create(user, project, simulation)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/simulations/%s' \
            % simulation['_id']

        return simulation

    @autoDescribeRoute(
        Description(
            'List all the simulations associated with a project.').modelParam(
                'id',
                'The project',
                model='project',
                plugin='hpccloud',
                level=AccessType.READ).pagingParams(defaultSort='created'))
    @access.user
    def simulations(self, project, limit, offset, sort, params):
        user = getCurrentUser()
        return self.model('project',
                          'hpccloud').simulations(user, project, limit, offset)
Пример #12
0
        '_accessLevel': 2,
        '_id': '5873dcdbaec030000144d233',
        '_modelType': 'recipe',
        'commitId': '1234abc',
        'creatorId': '18312dcdbaec030000144d233',
        'created': '2017-01-09T18:56:27.262000+00:00',
        'description': 'My fancy recipe',
        'name': 'Xarthisius/wt_recipe',
        'parentId': 'null',
        'public': True,
        'tags': ['latest', 'py3'],
        'url': 'https://github.com/Xarthisius/wt_recipe',
        'updated': '2017-01-10T16:15:17.313000+00:00',
    },
}
addModel('recipe', recipeModel, resources='recipe')


class Recipe(Resource):
    def __init__(self):
        super(Recipe, self).__init__()
        self.resourceName = 'recipe'

        self.route('GET', (), self.listRecipes)
        self.route('POST', (), self.createRecipe)
        self.route('GET', (':id', ), self.getRecipe)
        # self.route('POST', (':id',), self.copyRecipe)
        self.route('PUT', (':id', ), self.updateRecipe)
        self.route('DELETE', (':id', ), self.deleteRecipe)
        self.route('GET', (':id', 'access'), self.getRecipeAccess)
        self.route('PUT', (':id', 'access'), self.updateRecipeAccess)
Пример #13
0
        'status': {
            'type': 'integer',
            'format': 'int32',
            'allowEmptyValue': False,
            'maximum': 1,
            'minimum': 0
        },
        'taleId': {
            'type': 'string'
        },
        'url': {
            'type': 'string'
        }
    }
}
addModel('instance', instanceModel, resources='instance')


class Instance(Resource):
    def __init__(self):
        super(Instance, self).__init__()
        self.resourceName = 'instance'

        self.route('GET', (), self.listInstances)
        self.route('POST', (), self.createInstance)
        self.route('GET', (':id', ), self.getInstance)
        self.route('DELETE', (':id', ), self.deleteInstance)

    @access.user
    @filtermodel(model='instance', plugin='wholetale')
    @autoDescribeRoute(
Пример #14
0
class Script(BaseResource):
    def __init__(self):
        super(Script, self).__init__()
        self.resourceName = 'scripts'
        self.route('POST', (), self.create)
        self.route('GET', (':id', ), self.get)
        self.route('PATCH', (':id', 'import'), self.import_script)
        self.route('PUT', (':id', 'access'), self.update_access)
        self.route('DELETE', (':id', ), self.delete)
        self._model = self.model('script', 'cumulus')

    def _clean(self, config):
        del config['access']

        return config

    @access.user
    def import_script(self, id, params):
        user = self.getCurrentUser()
        lines = cherrypy.request.body.read().decode('utf8').splitlines()

        script = self._model.load(id, user=user, level=AccessType.ADMIN)

        if not script:
            raise RestException('Script doesn\'t exist', code=404)

        script['commands'] = lines
        self._model.save(script)

        return self._clean(script)

    import_script.description = (Description('Import script').param(
        'id', 'The script to upload lines to', required=True,
        paramType='path').param('body',
                                'The contents of the script',
                                required=True,
                                paramType='body').consumes('text/plain'))

    @access.user
    def create(self, params):
        user = self.getCurrentUser()

        script = json.loads(cherrypy.request.body.read().decode('utf8'))

        if 'name' not in script:
            raise RestException('Script name is required', code=400)

        script = self._model.create(user, script)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/scripts/%s' % script['_id']

        return self._clean(script)

    addModel(
        'Script', {
            'id': 'Script',
            'required': 'global',
            'properties': {
                'name': {
                    'type': 'string'
                },
                'commands': {
                    'type': 'array',
                    'items': {
                        'type': 'string'
                    }
                }
            }
        }, 'scripts')

    create.description = (Description('Create script').param(
        'body',
        'The JSON contain script parameters',
        required=True,
        paramType='body',
        dataType='Script'))

    @access.user
    def get(self, id, params):
        user = self.getCurrentUser()

        script = self._model.load(id, user=user, level=AccessType.READ)

        if not script:
            raise RestException('Script not found', code=404)

        return self._clean(script)

    get.description = (Description('Get script').param(
        'id', 'The id of the script to get', required=True, paramType='path'))

    @access.user
    def delete(self, id, params):
        user = self.getCurrentUser()
        script = self._model.load(id, user=user, level=AccessType.ADMIN)

        self._model.remove(script)

    delete.description = (Description('Delete a script').param(
        'id', 'The script id.', paramType='path', required=True))

    @access.user
    def update_access(self, id, params):
        user = self.getCurrentUser()

        body = cherrypy.request.body.read()

        if not body:
            raise RestException('No message body provided', code=400)

        body = json.loads(body.decode('utf8'))

        script = self._model.load(id, user=user, level=AccessType.WRITE)
        if not script:
            raise RestException('Script not found.', code=404)

        script = self._model.setAccessList(script, body, save=True)

        return script

    update_access.description = (Description('Update script access').param(
        'id', 'The script to update', required=True,
        paramType='path').param('body',
                                'The fields to update',
                                required=True,
                                paramType='body').consumes('application/json'))
Пример #15
0
class Cluster(BaseResource):
    def __init__(self):
        super(Cluster, self).__init__()
        self.resourceName = 'clusters'
        self.route('POST', (), self.create)
        self.route('POST', (':id', 'log'), self.handle_log_record)
        self.route('GET', (':id', 'log'), self.log)
        self.route('PUT', (':id', 'start'), self.start)
        self.route('PUT', (':id', 'launch'), self.launch)
        self.route('PUT', (':id', 'provision'), self.provision)
        self.route('PATCH', (':id', ), self.update)
        self.route('GET', (':id', 'status'), self.status)
        self.route('PUT', (':id', 'terminate'), self.terminate)
        self.route('PUT', (':id', 'job', ':jobId', 'submit'), self.submit_job)
        self.route('GET', (':id', ), self.get)
        self.route('DELETE', (':id', ), self.delete)
        self.route('GET', (), self.find)

        # TODO Findout how to get plugin name rather than hardcoding it
        self._model = ModelImporter.model('cluster', 'cumulus')

    @access.user(scope=TokenScope.DATA_WRITE)
    def handle_log_record(self, id, params):
        user = self.getCurrentUser()

        if not self._model.load(id, user=user, level=AccessType.ADMIN):
            raise RestException('Cluster not found.', code=404)

        return self._model.append_to_log(user, id, getBodyJson())

    handle_log_record.description = None

    def _create_ec2(self, params, body):
        return self._create_ansible(params, body, cluster_type=ClusterType.EC2)

    def _create_ansible(self, params, body, cluster_type=ClusterType.ANSIBLE):

        self.requireParams(['name', 'profileId'], body)

        name = body['name']
        playbook = get_property('config.launch.spec', body, default='default')
        launch_params = get_property('config.launch.params', body, default={})
        config = get_property('config', body, default={})
        profile_id = body['profileId']
        user = self.getCurrentUser()

        cluster = self._model.create_ansible(user,
                                             name,
                                             config,
                                             playbook,
                                             launch_params,
                                             profile_id,
                                             cluster_type=cluster_type)

        return cluster

    def _create_traditional(self, params, body):

        self.requireParams(['name', 'config'], body)
        self.requireParams(['ssh', 'host'], body['config'])
        self.requireParams(['user'], body['config']['ssh'])

        name = body['name']
        config = body['config']
        user = self.getCurrentUser()

        cluster = self._model.create_traditional(user, name, config)

        # Fire off job to create key pair for cluster
        girder_token = self.get_task_token()['_id']
        generate_key_pair.delay(self._model.filter(cluster, user),
                                girder_token)

        return cluster

    def _create_newt(self, params, body):

        self.requireParams(['name', 'config'], body)
        self.requireParams(['host'], body['config'])

        name = body['name']
        config = body['config']
        user = self.getCurrentUser()
        config['user'] = user['login']

        cluster = self._model.create_newt(user, name, config)

        return cluster

    @access.user(scope=TokenScope.DATA_WRITE)
    def create(self, params):
        body = getBodyJson()
        # Default ec2 cluster
        cluster_type = 'ec2'

        if 'type' in body:
            if not ClusterType.is_valid_type(body['type']):
                raise RestException('Invalid cluster type.', code=400)
            cluster_type = body['type']

        if cluster_type == ClusterType.EC2:
            cluster = self._create_ec2(params, body)
        elif cluster_type == ClusterType.ANSIBLE:
            cluster = self._create_ansible(params, body)
        elif cluster_type == ClusterType.TRADITIONAL:
            cluster = self._create_traditional(params, body)
        elif cluster_type == ClusterType.NEWT:
            cluster = self._create_newt(params, body)
        else:
            raise RestException('Invalid cluster type.', code=400)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/clusters/%s' % cluster['_id']

        return self._model.filter(cluster, self.getCurrentUser())

    addModel(
        'Id', {
            'id': 'Id',
            'properties': {
                '_id': {
                    'type': 'string',
                    'description': 'The id.'
                }
            }
        }, 'clusters')

    addModel(
        'UserNameParameter', {
            'id': 'UserNameParameter',
            'properties': {
                'user': {
                    'type': 'string',
                    'description': 'The ssh user id'
                }
            }
        }, 'clusters')

    addModel(
        'SshParameters', {
            'id': 'SshParameters',
            'properties': {
                'ssh': {
                    '$ref': '#/definitions/UserNameParameter'
                }
            }
        }, 'clusters')

    addModel(
        'ClusterParameters', {
            'id': 'ClusterParameters',
            'required': ['name', 'config', 'type'],
            'properties': {
                'name': {
                    'type': 'string',
                    'description': 'The name to give the cluster.'
                },
                'template': {
                    'type': 'string',
                    'description': 'The cluster template to use. '
                    '(ec2 only)'
                },
                'config': {
                    '$ref': '#/definitions/SshParameters',
                    'host': {
                        'type':
                        'string',
                        'description':
                        'The hostname of the head node '
                        '(trad only)'
                    }
                },
                'type': {
                    'type': 'string',
                    'description': 'The cluster type, either "ec2" or "trad"'
                }
            }
        }, 'clusters')

    create.description = (Description('Create a cluster').param(
        'body',
        'The name to give the cluster.',
        dataType='ClusterParameters',
        required=True,
        paramType='body'))

    def _get_body(self):
        body = {}
        if cherrypy.request.body:
            request_body = cherrypy.request.body.read().decode('utf8')
            if request_body:
                body = json.loads(request_body)

        return body

    @access.user(scope=TokenScope.DATA_WRITE)
    @loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
    def start(self, cluster, params):
        body = self._get_body()
        adapter = get_cluster_adapter(cluster)
        adapter.start(body)
        events.trigger('cumulus.cluster.started', info=cluster)

    addModel(
        'ClusterOnStartParms', {
            'id': 'ClusterOnStartParms',
            'properties': {
                'submitJob': {
                    'pattern':
                    '^[0-9a-fA-F]{24}$',
                    'type':
                    'string',
                    'description':
                    'The id of a Job to submit when the cluster '
                    'is started.'
                }
            }
        }, 'clusters')

    addModel(
        'ClusterStartParams', {
            'id': 'ClusterStartParams',
            'properties': {
                'onStart': {
                    '$ref': '#/definitions/ClusterOnStartParms'
                }
            }
        }, 'clusters')

    start.description = (Description('Start a cluster (ec2 only)').param(
        'id', 'The cluster id to start.', paramType='path',
        required=True).param('body',
                             'Parameter used when starting cluster',
                             paramType='body',
                             dataType='ClusterStartParams',
                             required=False))

    @access.user(scope=TokenScope.DATA_WRITE)
    @loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
    def launch(self, cluster, params):

        # Update any launch parameters passed in message body
        body = self._get_body()
        cluster['config']['launch']['params'].update(body)
        cluster = self._model.save(cluster)

        return self._model.filter(self._launch_or_provision('launch', cluster),
                                  self.getCurrentUser())

    launch.description = (Description('Start a cluster with ansible').param(
        'id', 'The cluster id to start.', paramType='path', required=True))

    @access.user(scope=TokenScope.DATA_WRITE)
    @loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
    def provision(self, cluster, params):

        if not ClusterStatus.valid_transition(cluster['status'],
                                              ClusterStatus.PROVISIONING):
            raise RestException(
                'Cluster status is %s and cannot be provisioned' %
                cluster['status'],
                code=400)

        body = self._get_body()
        provision_ssh_user = get_property('ssh.user', body)
        if provision_ssh_user:
            cluster['config'].setdefault('provision', {})['ssh'] = {
                'user': provision_ssh_user
            }
            del body['ssh']

        if 'spec' in body:
            cluster['config'].setdefault('provision', {})['spec'] \
                = body['spec']
            del body['spec']

        cluster['config'].setdefault('provision', {})\
            .setdefault('params', {}).update(body)
        cluster = self._model.save(cluster)

        return self._model.filter(
            self._launch_or_provision('provision', cluster),
            self.getCurrentUser())

    provision.description = (
        Description('Provision a cluster with ansible').param(
            'id',
            'The cluster id to provision.',
            paramType='path',
            required=True).param('body',
                                 'Parameter used when provisioning cluster',
                                 paramType='body',
                                 dataType='list',
                                 required=False))

    def _launch_or_provision(self, process, cluster):
        assert process in ['launch', 'provision']
        adapter = get_cluster_adapter(cluster)

        return getattr(adapter, process)()

    @access.user(scope=TokenScope.DATA_WRITE)
    def update(self, id, params):
        body = getBodyJson()
        user = self.getCurrentUser()

        cluster = self._model.load(id, user=user, level=AccessType.WRITE)

        if not cluster:
            raise RestException('Cluster not found.', code=404)

        if 'assetstoreId' in body:
            cluster['assetstoreId'] = body['assetstoreId']

        if 'status' in body:
            if ClusterStatus.valid(body['status']):
                cluster['status'] = body['status']
            else:
                raise RestException('%s is not a valid cluster status' %
                                    body['status'],
                                    code=400)

        if 'timings' in body:
            if 'timings' in cluster:
                cluster['timings'].update(body['timings'])
            else:
                cluster['timings'] = body['timings']

        if 'config' in body:
            # Need to check we aren't try to update immutable fields
            immutable_paths = ['_id', 'ssh.user']
            for path in immutable_paths:
                if parse(path).find(body['config']):
                    raise RestException("The '%s' field can't be updated" %
                                        path)

            update_dict(cluster['config'], body['config'])

        cluster = self._model.update_cluster(user, cluster)

        # Now do any updates the adapter provides
        adapter = get_cluster_adapter(cluster)
        try:
            adapter.update(body)
        # Skip adapter.update if update not defined for this adapter
        except (NotImplementedError, ValidationException):
            pass

        return self._model.filter(cluster, user)

    addModel(
        'ClusterUpdateParameters', {
            'id': 'ClusterUpdateParameters',
            'properties': {
                'status': {
                    'type': 'string',
                    'enum': ['created', 'running', 'stopped', 'terminated'],
                    'description': 'The new status. (optional)'
                }
            }
        }, 'clusters')

    update.description = (Description('Update the cluster').param(
        'id', 'The id of the cluster to update', paramType='path').param(
            'body',
            'The properties to update.',
            dataType='ClusterUpdateParameters',
            paramType='body').notes('Internal - Used by Celery tasks'))

    @access.user(scope=TokenScope.DATA_READ)
    def status(self, id, params):
        user = self.getCurrentUser()
        cluster = self._model.load(id, user=user, level=AccessType.READ)

        if not cluster:
            raise RestException('Cluster not found.', code=404)

        return {'status': cluster['status']}

    addModel(
        'ClusterStatus', {
            'id': 'ClusterStatus',
            'required': ['status'],
            'properties': {
                'status': {
                    'type': 'string',
                    'enum': [ClusterStatus.valid_transitions.keys()]
                }
            }
        }, 'clusters')

    status.description = (Description('Get the clusters current state').param(
        'id', 'The cluster id to get the status of.',
        paramType='path').responseClass('ClusterStatus'))

    @access.user(scope=TokenScope.DATA_WRITE)
    def terminate(self, id, params):
        user = self.getCurrentUser()
        cluster = self._model.load(id, user=user, level=AccessType.ADMIN)

        if not cluster:
            raise RestException('Cluster not found.', code=404)

        adapter = get_cluster_adapter(cluster)
        adapter.terminate()

    terminate.description = (Description('Terminate a cluster').param(
        'id', 'The cluster to terminate.', paramType='path'))

    @access.user(scope=TokenScope.DATA_READ)
    def log(self, id, params):
        user = self.getCurrentUser()
        offset = 0
        if 'offset' in params:
            offset = int(params['offset'])

        if not self._model.load(id, user=user, level=AccessType.READ):
            raise RestException('Cluster not found.', code=404)

        log_records = self._model.log_records(user, id, offset)

        return {'log': log_records}

    log.description = (Description('Get log entries for cluster').param(
        'id', 'The cluster to get log entries for.',
        paramType='path').param('offset',
                                'The offset to start getting entries at.',
                                required=False,
                                paramType='query'))

    @access.user(scope=TokenScope.DATA_WRITE)
    def submit_job(self, id, jobId, params):
        job_id = jobId
        user = self.getCurrentUser()
        cluster = self._model.load(id, user=user, level=AccessType.ADMIN)

        if not cluster:
            raise RestException('Cluster not found.', code=404)

        if cluster['status'] != ClusterStatus.RUNNING:
            raise RestException('Cluster is not running', code=400)

        job_model = ModelImporter.model('job', 'cumulus')
        job = job_model.load(job_id, user=user, level=AccessType.ADMIN)

        # Set the clusterId on the job for termination
        job['clusterId'] = ObjectId(id)

        # Add any job parameters to be used when templating job script
        body = cherrypy.request.body.read().decode('utf8')
        if body:
            job['params'] = json.loads(body)

        job_model.save(job)

        cluster_adapter = get_cluster_adapter(cluster)
        del job['access']
        del job['log']
        cluster_adapter.submit_job(job)

    submit_job.description = (Description('Submit a job to the cluster').param(
        'id',
        'The cluster to submit the job to.',
        required=True,
        paramType='path').param('jobId',
                                'The cluster to get log entries for.',
                                required=True,
                                paramType='path').param(
                                    'body',
                                    'The properties to template on submit.',
                                    dataType='object',
                                    paramType='body'))

    @access.user(scope=TokenScope.DATA_READ)
    def get(self, id, params):
        user = self.getCurrentUser()
        cluster = self._model.load(id, user=user, level=AccessType.ADMIN)

        if not cluster:
            raise RestException('Cluster not found.', code=404)

        return self._model.filter(cluster, user)

    get.description = (Description('Get a cluster').param('id',
                                                          'The cluster id.',
                                                          paramType='path',
                                                          required=True))

    @access.user(scope=TokenScope.DATA_WRITE)
    def delete(self, id, params):
        user = self.getCurrentUser()

        cluster = self._model.load(id, user=user, level=AccessType.ADMIN)
        if not cluster:
            raise RestException('Cluster not found.', code=404)

        adapter = get_cluster_adapter(cluster)
        adapter.delete()

        self._model.delete(user, id)

    delete.description = (
        Description('Delete a cluster and its configuration').param(
            'id', 'The cluster id.', paramType='path', required=True))

    @access.user(scope=TokenScope.DATA_READ)
    def find(self, params):
        return self._model.find_cluster(params, user=self.getCurrentUser())

    find.description = (
        Description('Search for clusters with certain properties').param(
            'type',
            'The cluster type to search for',
            paramType='query',
            required=False).param('limit',
                                  'The max number of clusters to return',
                                  paramType='query',
                                  required=False,
                                  default=50))
Пример #16
0
from girder.plugins.jobs.constants import JobStatus

from ..schema.tale import taleModel as taleSchema
from ..models.tale import Tale as taleModel
from ..models.image import Image as imageModel
from ..lib import pids_to_entities, IMPORT_PROVIDERS
from ..lib.dataone import DataONELocations  # TODO: get rid of it
from ..lib.manifest import Manifest
from ..lib.exporters.bag import BagTaleExporter
from ..lib.exporters.native import NativeTaleExporter

from girder.plugins.worker import getCeleryApp

from ..constants import ImageStatus, TaleStatus, PluginSettings

addModel('tale', taleSchema, resources='tale')


class Tale(Resource):
    def __init__(self):
        super(Tale, self).__init__()
        self.resourceName = 'tale'
        self._model = taleModel()

        self.route('GET', (), self.listTales)
        self.route('GET', (':id', ), self.getTale)
        self.route('PUT', (':id', ), self.updateTale)
        self.route('POST', ('import', ), self.createTaleFromDataset)
        self.route('POST', (), self.createTale)
        self.route('POST', (':id', 'copy'), self.copyTale)
        self.route('DELETE', (':id', ), self.deleteTale)
Пример #17
0
class Projects(Resource):
    def __init__(self):
        super(Projects, self).__init__()
        self.resourceName = 'projects'
        self.route('POST', (), self.create)
        self.route('PATCH', (':id', ), self.update)
        self.route('GET', (), self.get_all)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PUT', (':id', 'share'), self.share)
        self.route('POST', (':id', 'simulations'), self.create_simulation)
        self.route('GET', (':id', 'simulations'), self.simulations)
        self.route('GET', (':id', ), self.get)

        self._model = self.model('project', 'hpccloud')

    addModel('ProjectProperties', schema.project, 'projects')

    @describeRoute(
        Description('Create a new project').param(
            'body',
            'The properies of the project.',
            dataType='ProjectProperties',
            required=True,
            paramType='body'))
    @access.user
    def create(self, params):
        project = getBodyJson()
        project = self.model('project',
                             'hpccloud').create(getCurrentUser(), project)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/projects/%s' % project['_id']

        return project

    @describeRoute(
        Description('Update a project').param(
            'id',
            'The project to update.',
            dataType='string',
            required=True,
            paramType='path').param('body',
                                    'The properies of the project to update.',
                                    dataType='object',
                                    required=True,
                                    paramType='body'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
    def update(self, project, params):
        immutable = [
            'type', 'steps', 'folderId', 'access', 'userId', '_id', 'created',
            'updated'
        ]
        updates = getBodyJson()

        for p in updates:
            if p in immutable:
                raise RestException('\'%s\' is an immutable property' % p, 400)

        user = getCurrentUser()
        name = updates.get('name')
        metadata = updates.get('metadata')
        description = updates.get('description')

        return self._model.update(user,
                                  project,
                                  name=name,
                                  metadata=metadata,
                                  description=description)

    @describeRoute(
        Description('Get all projects this user has access to project').param(
            'limit',
            'Result set size limit.',
            dataType='integer',
            required=False,
            paramType='query').param('offset',
                                     'Offset into result set.',
                                     dataType='integer',
                                     required=False,
                                     paramType='query'))
    @access.user
    def get_all(self, params):
        user = getCurrentUser()
        limit, offset, _ = self.getPagingParameters(params)

        cursor = self._model.find(limit=limit, offset=offset)
        return list(
            self._model.filterResultsByPermission(cursor=cursor,
                                                  user=user,
                                                  level=AccessType.READ))

    @describeRoute(
        Description('Delete a project').param(
            'id',
            'The project to delete.',
            dataType='string',
            required=True,
            paramType='path').notes(
                'Will clean up any files, items or folders associated with '
                'the project.'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
    def delete(self, project, params):
        user = getCurrentUser()
        self._model.delete(user, project)

    @describeRoute(
        Description('Get a particular project').param('id',
                                                      'The project to get.',
                                                      dataType='string',
                                                      required=True,
                                                      paramType='path'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
    def get(self, project, params):
        return project

    addModel('ShareProperties', schema.project['definitions']['share'],
             'projects')

    @describeRoute(
        Description(
            'Share a give project with a set of users or groups').param(
                'id',
                'The project to shared.',
                dataType='string',
                required=True,
                paramType='path'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
    def share(self, project, params):
        body = getBodyJson()
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(body,
                                schema.project['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = body.get('users', [])
        groups = body.get('groups', [])

        return self._model.share(user, project, users, groups)

    addModel('SimProperties', schema.simulation, 'projects')

    @describeRoute(
        Description('Create a simulation associated with a project.').param(
            'id',
            'The project the simulation will be created in.',
            dataType='string',
            required=True,
            paramType='path').param('body',
                                    'The properties of the simulation.',
                                    dataType='SimProperties',
                                    required=True,
                                    paramType='body'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
    def create_simulation(self, project, params):
        simulation = getBodyJson()
        user = getCurrentUser()

        simulation = self.model('simulation',
                                'hpccloud').create(user, project, simulation)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/simulations/%s' \
            % simulation['_id']

        return simulation

    @describeRoute(
        Description(
            'List all the simulations associated with a project.').param(
                'id',
                'The project',
                dataType='string',
                required=True,
                paramType='path').param('limit',
                                        'Result set size limit.',
                                        dataType='integer',
                                        required=False,
                                        paramType='query').param(
                                            'offset',
                                            'Offset into result set.',
                                            dataType='integer',
                                            required=False,
                                            paramType='query'))
    @access.user
    @loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
    def simulations(self, project, params):
        user = getCurrentUser()
        limit, offset, _ = self.getPagingParameters(params)
        return self.model('project',
                          'hpccloud').simulations(user, project, limit, offset)
Пример #18
0
class Proxy(Resource):
    def __init__(self):
        super(Proxy, self).__init__()
        self.resourceName = 'proxy'
        self.route('POST', (), self.add_entry)
        self.route('DELETE', (':key', ), self.delete_entry)
        self._proxy_file_path = Setting().get(
            constants.PluginSettings.PROXY_FILE_PATH) or '/pvw/proxy'

    addModel(
        'ProxyEntry', {
            'id': 'ProxyEntry',
            'required': ['key', 'host', 'port'],
            'properties': {
                'key': {
                    'type': 'string'
                },
                'host': {
                    'type': 'string'
                },
                'port': {
                    'type': 'integer'
                }
            }
        }, 'proxy')

    @access.public
    @describeRoute(
        Description('Add entry to the proxy file').param(
            'body',
            'The proxy entry parameters.',
            dataType='ProxyEntry',
            paramType='body',
            required=True))
    def add_entry(self, params):
        body = getBodyJson()

        if 'key' not in body:
            raise RestException('key is required', code=400)
        if 'host' not in body:
            raise RestException('host is required', code=400)
        if 'port' not in body:
            raise RestException('port is required', code=400)

        key = body['key']
        host = body['host']
        port = body['port']

        with LockFile(self._proxy_file_path):
            db = None
            try:
                db = dbm.open(self._proxy_file_path, 'c')
                # Encode the slash
                db[key] = '%s:%s' % (host, port)
            finally:
                if db:
                    db.close()

    @access.public
    @describeRoute(
        Description('Delete entry').param('key',
                                          'The key to delete.',
                                          dataType='string',
                                          paramType='path',
                                          required=True))
    def delete_entry(self, key, params):

        with LockFile(self._proxy_file_path):
            db = None
            try:
                db = dbm.open(self._proxy_file_path, 'c')
                if key in db:
                    del db[key]
            finally:
                if db:
                    db.close()
Пример #19
0
class Volume(BaseResource):
    def __init__(self):
        super(Volume, self).__init__()
        self.resourceName = 'volumes'
        self.route('POST', (), self.create)
        self.route('GET', (':id', ), self.get)
        self.route('PATCH', (':id', ), self.patch)
        self.route('GET', (), self.find)
        self.route('GET', (':id', 'status'), self.get_status)
        self.route('POST', (':id', 'log'), self.append_to_log)
        self.route('GET', (':id', 'log'), self.log)
        self.route('PUT', (':id', 'clusters', ':clusterId', 'attach'),
                   self.attach)
        self.route('PUT',
                   (':id', 'clusters', ':clusterId', 'attach', 'complete'),
                   self.attach_complete)
        self.route('PUT', (':id', 'detach'), self.detach)
        self.route('PUT', (':id', 'detach', 'complete'), self.detach_complete)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PUT', (':id', 'delete', 'complete'), self.delete_complete)

        self._model = self.model('volume', 'cumulus')

    def _create_ebs(self, body, zone):
        user = getCurrentUser()
        name = body['name']
        size = body['size']
        fs = body.get('fs', None)
        profileId = body['profileId']

        return self._model.create_ebs(user, profileId, name, zone, size, fs)

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.WRITE)
    def patch(self, volume, params):
        body = getBodyJson()

        if not volume:
            raise RestException('Volume not found.', code=404)

        if 'ec2' in body:
            if 'ec2' not in volume:
                volume['ec2'] = {}
            volume['ec2'].update(body['ec2'])

        mutable = ['status', 'msg', 'path']
        for k in mutable:
            if k in body:
                volume[k] = body[k]

        user = getCurrentUser()
        volume = self._model.update_volume(user, volume)
        return self._model.filter(volume, user)

    patch.description = (Description('Patch a volume').param(
        'id', 'The volume id.', paramType='path',
        required=True).param('body',
                             'The properties to use to create the volume.',
                             required=True,
                             paramType='body'))

    @access.user
    def create(self, params):
        body = getBodyJson()

        self.requireParams(['name', 'type', 'size', 'profileId'], body)

        if not VolumeType.is_valid_type(body['type']):
            raise RestException('Invalid volume type.', code=400)

        profile_id = parse('profileId').find(body)
        if not profile_id:
            raise RestException('A profile id must be provided', 400)

        profile_id = profile_id[0].value

        profile, secret_key = _get_profile(profile_id)

        if not profile:
            raise RestException('Invalid profile', 400)

        if 'zone' in body:
            zone = body['zone']
        else:
            zone = profile['availabilityZone']

        volume = self._create_ebs(body, zone)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/volumes/%s' % volume['_id']

        return self._model.filter(volume, getCurrentUser())

    addModel(
        'VolumeParameters', {
            'id': 'VolumeParameters',
            'required': ['name', 'config', 'type', 'zone', 'size'],
            'properties': {
                'name': {
                    'type': 'string',
                    'description': 'The name to give the cluster.'
                },
                'profileId': {
                    'type': 'string',
                    'description': 'Id of profile to use'
                },
                'type': {
                    'type':
                    'string',
                    'description':
                    'The type of volume to create ( currently '
                    'only esb )'
                },
                'zone': {
                    'type': 'string',
                    'description': 'The availability region'
                },
                'size': {
                    'type': 'integer',
                    'description': 'The size of the volume to create'
                }
            },
        }, 'volumes')

    create.description = (Description('Create a volume').param(
        'body',
        'The properties to use to create the volume.',
        dataType='VolumeParameters',
        required=True,
        paramType='body'))

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.READ)
    def get(self, volume, params):

        return self._model.filter(volume, getCurrentUser())

    get.description = (Description('Get a volume').param('id',
                                                         'The volume id.',
                                                         paramType='path',
                                                         required=True))

    @access.user
    def find(self, params):
        user = getCurrentUser()
        query = {}

        if 'clusterId' in params:
            query['clusterId'] = ObjectId(params['clusterId'])

        limit = params.get('limit', 50)

        volumes = self._model.find(query=query)
        volumes = list(volumes)

        volumes = self._model \
            .filterResultsByPermission(volumes, user, AccessType.ADMIN,
                                       limit=int(limit))

        return [self._model.filter(volume, user) for volume in volumes]

    find.description = (Description('Search for volumes').param(
        'limit',
        'The max number of volumes to return',
        paramType='query',
        required=False,
        default=50))

    @access.user
    @loadmodel(map={'clusterId': 'cluster'},
               model='cluster',
               plugin='cumulus',
               level=AccessType.ADMIN)
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def attach_complete(self, volume, cluster, params):

        user = getCurrentUser()

        path = params.get('path', None)

        # Is path being passed in as apart of the body json?
        if path is None:
            path = getBodyJson().get('path', None)

        if path is not None:
            cluster.setdefault('volumes', [])
            cluster['volumes'].append(volume['_id'])
            cluster['volumes'] = list(set(cluster['volumes']))

            volume['status'] = VolumeState.INUSE
            volume['path'] = path

            # TODO: removing msg should be refactored into
            #       a general purpose 'update_status' function
            #       on the volume model. This way msg only referes
            #       to the current status.
            try:
                del volume['msg']
            except KeyError:
                pass

            # Add cluster id to volume
            volume['clusterId'] = cluster['_id']

            self.model('cluster', 'cumulus').save(cluster)
            self._model.update_volume(user, volume)
        else:
            volume['status'] = VolumeState.ERROR
            volume['msg'] = 'Volume path was not communicated on complete'
            self._model.update_volume(user, volume)

    attach_complete.description = None

    @access.user
    @loadmodel(map={'clusterId': 'cluster'},
               model='cluster',
               plugin='cumulus',
               level=AccessType.ADMIN)
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def attach(self, volume, cluster, params):
        body = getBodyJson()

        self.requireParams(['path'], body)
        path = body['path']

        profile_id = parse('profileId').find(volume)[0].value
        profile, secret_key = _get_profile(profile_id)

        girder_callback_info = {
            'girder_api_url': cumulus.config.girder.baseUrl,
            'girder_token': get_task_token()['_id']
        }
        log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
                                               volume['_id'])

        p = CloudProvider(dict(secretAccessKey=secret_key, **profile))

        aws_volume = p.get_volume(volume)

        # If volume exists it needs to be available to be attached. If
        # it doesn't exist it will be created as part of the attach
        # playbook.
        if aws_volume is not None and \
           aws_volume['state'] != VolumeState.AVAILABLE:
            raise RestException(
                'This volume is not available to attach '
                'to a cluster', 400)

        master = p.get_master_instance(cluster['_id'])
        if master['state'] != InstanceState.RUNNING:
            raise RestException('Master instance is not running!', 400)

        cluster = self.model('cluster', 'cumulus').filter(cluster,
                                                          getCurrentUser(),
                                                          passphrase=False)
        cumulus.ansible.tasks.volume.attach_volume\
            .delay(profile, cluster, master,
                   self._model.filter(volume, getCurrentUser()), path,
                   secret_key, log_write_url, girder_callback_info)

        volume['status'] = VolumeState.ATTACHING
        volume = self._model.update_volume(getCurrentUser(), volume)

        return self._model.filter(volume, getCurrentUser())

    addModel(
        'AttachParameters', {
            'id': 'AttachParameters',
            'required': ['path'],
            'properties': {
                'path': {
                    'type': 'string',
                    'description': 'The path to mount the volume'
                }
            }
        }, 'volumes')

    attach.description = (Description('Attach a volume to a cluster').param(
        'id',
        'The id of the volume to attach',
        required=True,
        paramType='path').param('clusterId',
                                'The cluster to attach the volume to.',
                                required=True,
                                paramType='path').param(
                                    'body',
                                    'The properties to template on submit.',
                                    dataType='AttachParameters',
                                    paramType='body'))

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def detach(self, volume, params):

        profile_id = parse('profileId').find(volume)[0].value
        profile, secret_key = _get_profile(profile_id)

        girder_callback_info = {
            'girder_api_url': cumulus.config.girder.baseUrl,
            'girder_token': get_task_token()['_id']
        }

        log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
                                               volume['_id'])

        p = CloudProvider(dict(secretAccessKey=secret_key, **profile))

        aws_volume = p.get_volume(volume)
        if aws_volume is None or aws_volume['state'] != VolumeState.INUSE:
            raise RestException('This volume is not attached '
                                'to a cluster', 400)

        if 'clusterId' not in volume:
            raise RestException('clusterId is not set on this volume!', 400)

        try:
            volume['path']
        except KeyError:
            raise RestException('path is not set on this volume!', 400)

        cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
                                                        user=getCurrentUser(),
                                                        level=AccessType.ADMIN)
        master = p.get_master_instance(cluster['_id'])
        if master['state'] != InstanceState.RUNNING:
            raise RestException('Master instance is not running!', 400)
        user = getCurrentUser()
        cluster = self.model('cluster', 'cumulus').filter(cluster,
                                                          user,
                                                          passphrase=False)
        cumulus.ansible.tasks.volume.detach_volume\
            .delay(profile, cluster, master,
                   self._model.filter(volume, user),
                   secret_key, log_write_url, girder_callback_info)

        volume['status'] = VolumeState.DETACHING
        volume = self._model.update_volume(user, volume)

        return self._model.filter(volume, user)

    detach.description = (Description('Detach a volume from a cluster').param(
        'id', 'The id of the attached volume', required=True,
        paramType='path'))

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def detach_complete(self, volume, params):

        # First remove from cluster
        user = getCurrentUser()
        cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
                                                        user=user,
                                                        level=AccessType.ADMIN)
        cluster.setdefault('volumes', []).remove(volume['_id'])

        del volume['clusterId']

        for attr in ['path', 'msg']:
            try:
                del volume[attr]
            except KeyError:
                pass

        volume['status'] = VolumeState.AVAILABLE

        self.model('cluster', 'cumulus').save(cluster)
        self._model.save(volume)
        send_status_notification('volume', volume)

    detach_complete.description = None

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def delete(self, volume, params):
        if 'clusterId' in volume:
            raise RestException('Unable to delete attached volume')

        # If the volume is in state created and it has no ec2 volume id
        # associated with it,  we should be able to just delete it
        if volume['status'] in (VolumeState.CREATED, VolumeState.ERROR):
            if 'id' in volume['ec2'] and volume['ec2']['id'] is not None:
                raise RestException('Unable to delete volume,  it is '
                                    'associated with an ec2 volume %s' %
                                    volume['ec2']['id'])

            self._model.remove(volume)
            return None

        log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
                                               volume['_id'])

        # Call EC2 to delete volume
        profile_id = parse('profileId').find(volume)[0].value

        profile, secret_key = _get_profile(profile_id)

        girder_callback_info = {
            'girder_api_url': cumulus.config.girder.baseUrl,
            'girder_token': get_task_token()['_id']
        }

        p = CloudProvider(dict(secretAccessKey=secret_key, **profile))

        aws_volume = p.get_volume(volume)
        if aws_volume['state'] != VolumeState.AVAILABLE:
            raise RestException(
                'Volume must be in an "%s" status to be deleted' %
                VolumeState.AVAILABLE, 400)

        user = getCurrentUser()
        cumulus.ansible.tasks.volume.delete_volume\
            .delay(profile, self._model.filter(volume, user),
                   secret_key, log_write_url, girder_callback_info)

        volume['status'] = VolumeState.DELETING
        volume = self._model.update_volume(user, volume)

        return self._model.filter(volume, user)

    delete.description = (Description('Delete a volume').param(
        'id', 'The volume id.', paramType='path', required=True))

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def delete_complete(self, volume, params):
        self._model.remove(volume)

    delete_complete.description = None

    @access.user
    @loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
    def get_status(self, volume, params):
        return {'status': volume['status']}

    get_status.description = (Description('Get the status of a volume').param(
        'id', 'The volume id.', paramType='path', required=True))

    @access.user
    def append_to_log(self, id, params):
        user = getCurrentUser()

        if not self._model.load(id, user=user, level=AccessType.ADMIN):
            raise RestException('Volume not found.', code=404)

        return self._model.append_to_log(user, id, getBodyJson())

    append_to_log.description = None

    @access.user
    def log(self, id, params):
        user = getCurrentUser()
        offset = 0
        if 'offset' in params:
            offset = int(params['offset'])

        if not self._model.load(id, user=user, level=AccessType.READ):
            raise RestException('Volume not found.', code=404)

        log_records = self._model.log_records(user, id, offset)

        return {'log': log_records}

    log.description = (Description('Get log entries for volume').param(
        'id', 'The volume to get log entries for.',
        paramType='path').param('offset',
                                'The offset to start getting entries at.',
                                required=False,
                                paramType='query'))
Пример #20
0
dataSetItemSchema = {
    'title': 'dataSetItem',
    '$schema': 'http://json-schema.org/draft-04/schema#',
    'description': 'A schema representing data elements used in DMS dataSets',
    'type': 'object',
    'properties': {
        'itemId': {
            'type': 'string',
            'description': 'ID of a Girder item or a Girder folder'
        },
        'mountPath': {
            'type':
            'string',
            'description':
            'An absolute path where the item/folder are mounted in the EFS'
        }
    },
    'required': ['itemId', 'mountPath']
}

dataSetSchema = {
    'title':
    'A list of resources with a corresponding mount points in the ESF',
    '$schema': 'http://json-schema.org/draft-04/schema#',
    'type': 'array',
    'items': dataSetItemSchema,
}

addModel('dataSet', dataSetSchema)
Пример #21
0
class Calculation(Resource):
    output_formats = ['cml', 'xyz', 'inchikey', 'sdf']
    input_formats = ['cml', 'xyz', 'pdb']

    def __init__(self):
        super(Calculation, self).__init__()
        self.resourceName = 'calculations'
        self.route('POST', (), self.create_calc)
        self.route('PUT', (':id', ), self.ingest_calc)
        self.route('DELETE', (':id',), self.delete)
        self.route('GET', (), self.find_calc)
        self.route('GET', ('types',), self.find_calc_types)
        self.route('GET', (':id', 'vibrationalmodes'),
            self.get_calc_vibrational_modes)
        self.route('GET', (':id', 'vibrationalmodes', ':mode'),
            self.get_calc_vibrational_mode)
        self.route('GET', (':id', 'sdf'),
            self.get_calc_sdf)
        self.route('GET', (':id', 'cjson'),
            self.get_calc_cjson)
        self.route('GET', (':id', 'xyz'),
            self.get_calc_xyz)
        self.route('GET', (':id', 'cube', ':mo'),
            self.get_calc_cube)
        self.route('GET', (':id',),
            self.find_id)
        self.route('PUT', (':id', 'properties'),
            self.update_properties)
        self.route('PATCH', (':id', 'notebooks'), self.add_notebooks)

        self._model = ModelImporter.model('calculation', 'molecules')
        self._cube_model = ModelImporter.model('cubecache', 'molecules')

    @access.public
    def get_calc_vibrational_modes(self, id, params):

        # TODO: remove 'cjson' once girder issue #2883 is resolved
        fields = ['cjson', 'cjson.vibrations.modes', 'cjson.vibrations.intensities',
                 'cjson.vibrations.frequencies', 'access']

        calc = self._model.load(id, fields=fields, user=getCurrentUser(),
                                 level=AccessType.READ)

        del calc['access']

        if 'cjson' in calc and 'vibrations' in calc['cjson']:
            return calc['cjson']['vibrations']
        else:
            return {'modes': [], 'intensities': [], 'frequencies': []}

    get_calc_vibrational_modes.description = (
        Description('Get the vibrational modes associated with a calculation')
        .param(
            'id',
            'The id of the calculation to get the modes from.',
            dataType='string', required=True, paramType='path'))

    @access.public
    def get_calc_vibrational_mode(self, id, mode, params):

        try:
            mode = int(mode)
        except ValueError:
            raise ValidationException('mode number be an integer', 'mode')

        # TODO: remove 'cjson' once girder issue #2883 is resolved
        fields = ['cjson', 'cjson.vibrations.modes', 'access']
        calc = self._model.load(id, fields=fields, user=getCurrentUser(),
                                 level=AccessType.READ)

        vibrational_modes = calc['cjson']['vibrations']
        #frames = vibrational_modes.get('modeFrames')
        modes = vibrational_modes.get('modes', [])

        index = modes.index(mode)
        if index < 0:
            raise RestException('No such vibrational mode', 400)

        # Now select the modeFrames directly this seems to be more efficient
        # than iterating in Python
        query = {
            '_id': calc['_id']
        }

        projection = {
            'cjson.vibrations.frequencies': {
                '$slice': [index, 1]
            },
            'cjson.vibrations.intensities': {
                '$slice': [index, 1]
            },
            'cjson.vibrations.eigenVectors': {
                '$slice': [index, 1]
            },
            'cjson.vibrations.modes': {
                '$slice': [index, 1]
            }
        }

        mode = self._model.findOne(query, fields=projection)

        return mode['cjson']['vibrations']

    get_calc_vibrational_mode.description = (
        Description('Get a vibrational mode associated with a calculation')
        .param(
            'id',
            'The id of the calculation that the mode is associated with.',
            dataType='string', required=True, paramType='path')
        .param(
            'mode',
            'The index of the vibrational model to get.',
            dataType='string', required=True, paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_sdf(self, calculation, params):

        def stream():
            cherrypy.response.headers['Content-Type'] = 'chemical/x-mdl-sdfile'
            yield calculation['sdf']

        return stream

    get_calc_sdf.description = (
        Description('Get the molecular structure of a give calculation in SDF format')
        .param(
            'id',
            'The id of the calculation to return the structure for.',
            dataType='string', required=True, paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_cjson(self, calculation, params):
        return calculation['cjson']

    get_calc_cjson.description = (
        Description('Get the molecular structure of a give calculation in CJSON format')
        .param(
            'id',
            'The id of the calculation to return the structure for.',
            dataType='string', required=True, paramType='path'))

    @access.public
    @loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
    def get_calc_xyz(self, calculation, params):
        data = json.dumps(calculation['cjson'])
        data = avogadro.convert_str(data, 'cjson', 'xyz')

        def stream():
            cherrypy.response.headers['Content-Type'] = Molecule.mime_types['xyz']
            yield data

        return stream

    get_calc_xyz.description = (
        Description('Get the molecular structure of a give calculation in XYZ format')
        .param(
            'id',
            'The id of the calculation to return the structure for.',
            dataType='string', required=True, paramType='path'))

    @access.public
    def get_calc_cube(self, id, mo, params):
        orig_mo = mo
        try:
            mo = int(mo)
        except ValueError:
            # Check for h**o lumo
            mo = mo.lower()
            if mo in ['h**o', 'lumo']:
                cal = self._model.load(id, force=True)
                # Electron count might be saved in several places...
                path_expressions = [
                    'cjson.orbitals.electronCount',
                    'cjson.basisSet.electronCount',
                    'properties.electronCount'
                ]
                matches = []
                for expr in path_expressions:
                    matches.extend(parse(expr).find(cal))
                if len(matches) > 0:
                    electron_count = matches[0].value
                else:
                    raise RestException('Unable to access electronCount', 400)

                # The index of the first orbital is 0, so h**o needs to be
                # electron_count // 2 - 1
                if mo == 'h**o':
                    mo = int(electron_count / 2) - 1
                elif mo == 'lumo':
                    mo = int(electron_count / 2)
            else:
                raise ValidationException('mo number be an integer or \'h**o\'/\'lumo\'', 'mode')

        cached = self._cube_model.find_mo(id, mo)

        # If we have a cached cube file use that.
        if cached:
            return cached['cjson']

        fields = ['cjson', 'access', 'fileId']

        # Ignoring access control on file/data for now, all public.
        calc = self._model.load(id, fields=fields, force=True)

        # This is where the cube gets calculated, should be cached in future.
        if 'async' in params and params['async']:
            async_requests.schedule_orbital_gen(
                calc['cjson'], mo, id, orig_mo)
            calc['cjson']['cube'] = {
                'dimensions': [0, 0, 0],
                'scalars': []
            }
            return calc['cjson']
        else:
            cjson = avogadro.calculate_mo(calc['cjson'], mo)

            # Remove the vibrational mode data from the cube - big, not needed here.
            if 'vibrations' in cjson:
                del cjson['vibrations']

            # Cache this cube for the next time, they can take a while to generate.
            self._cube_model.create(id, mo, cjson)

            return cjson

    get_calc_cube.description = (
        Description('Get the cube for the supplied MO of the calculation in CJSON format')
        .param(
            'id',
            'The id of the calculation to return the structure for.',
            dataType='string', required=True, paramType='path')
        .param(
            'mo',
            'The molecular orbital to get the cube for.',
            dataType='string', required=True, paramType='path'))

    @access.user
    def create_calc(self, params):
        body = getBodyJson()
        if 'cjson' not in body and ('fileId' not in body or 'format' not in body):
            raise RestException('Either cjson or fileId is required.')

        user = getCurrentUser()

        cjson = body.get('cjson')
        props = body.get('properties', {})
        molecule_id = body.get('moleculeId', None)
        public = body.get('public', False)
        notebooks = body.get('notebooks', [])
        image = body.get('image')
        input_parameters = body.get('input', {}).get('parameters')
        file_id = None
        file_format = body.get('format', 'cjson')

        if 'fileId' in body:
            file = File().load(body['fileId'], user=getCurrentUser())
            file_id = file['_id']
            cjson = self._file_to_cjson(file, file_format)

        if molecule_id is None:
            mol = create_molecule(json.dumps(cjson), 'cjson', user, public)
            molecule_id = mol['_id']

        calc = CalculationModel().create_cjson(user, cjson, props, molecule_id,
                                               image=image,
                                               input_parameters=input_parameters,
                                               file_id=file_id,
                                               notebooks=notebooks, public=public)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] \
            = '/calculations/%s' % (str(calc['_id']))

        return CalculationModel().filter(calc, user)

    # Try and reuse schema for documentation, this only partially works!
    calc_schema = CalculationModel.schema.copy()
    calc_schema['id'] = 'CalculationData'
    addModel('Calculation', 'CalculationData', calc_schema)

    create_calc.description = (
        Description('Get the molecular structure of a give calculation in SDF format')
        .param(
            'body',
            'The calculation data', dataType='CalculationData', required=True,
            paramType='body'))

    def _file_to_cjson(self, file, file_format):
        readers = {
            'cjson': oc.CjsonReader
        }

        if file_format not in readers:
            raise Exception('Unknown file format %s' % file_format)
        reader = readers[file_format]

        with File().open(file) as f:
            calc_data = f.read().decode()

        # SpooledTemporaryFile doesn't implement next(),
        # workaround in case any reader needs it
        tempfile.SpooledTemporaryFile.__next__ = lambda self: self.__iter__().__next__()

        with tempfile.SpooledTemporaryFile(mode='w+', max_size=10*1024*1024) as tf:
            tf.write(calc_data)
            tf.seek(0)
            cjson = reader(tf).read()

        return cjson

    @access.user(scope=TokenScope.DATA_WRITE)
    @autoDescribeRoute(
        Description('Update pending calculation with results.')
        .modelParam('id', 'The calculation id',
            model=CalculationModel, destName='calculation',
            level=AccessType.WRITE, paramType='path')
        .param('detectBonds',
               'Automatically detect bonds if they are not already present in the ingested molecule',
               required=False,
               dataType='boolean',
               default=False)
        .jsonParam('body', 'The calculation details', required=True, paramType='body')
    )
    def ingest_calc(self, calculation, body, detectBonds=None):
        self.requireParams(['fileId', 'format'], body)

        file = File().load(body['fileId'], user=getCurrentUser())
        cjson = self._file_to_cjson(file, body['format'])

        calc_props = calculation['properties']
        # The calculation is no longer pending
        if 'pending' in calc_props:
            del calc_props['pending']

        # Add bonds if they were not there already
        if detectBonds is None:
            detectBonds = False

        bonds = cjson.get('bonds')
        if bonds is None and detectBonds:
            new_cjson = openbabel.autodetect_bonds(cjson)
            if new_cjson.get('bonds') is not None:
                cjson['bonds'] = new_cjson['bonds']

        calculation['properties'] = calc_props
        calculation['cjson'] = cjson
        calculation['fileId'] = file['_id']

        image = body.get('image')
        if image is not None:
            calculation['image'] = image

        scratch_folder_id = body.get('scratchFolderId')
        if scratch_folder_id is not None:
            calculation['scratchFolderId'] = scratch_folder_id

        return CalculationModel().save(calculation)

    @access.public
    @autoDescribeRoute(
        Description('Search for particular calculation')
        .param('moleculeId', 'The molecule ID linked to this calculation', required=False)
        .param('imageName', 'The name of the Docker image that run this calculation', required=False)
        .param('inputParametersHash', 'The hash of the input parameters dictionary.', required=False)
        .param('inputGeometryHash', 'The hash of the input geometry.', required=False)
        .param('name', 'The name of the molecule', paramType='query',
                   required=False)
        .param('inchi', 'The InChI of the molecule', paramType='query',
                required=False)
        .param('inchikey', 'The InChI key of the molecule', paramType='query',
                required=False)
        .param('smiles', 'The SMILES of the molecule', paramType='query',
                required=False)
        .param('formula',
                'The formula (using the "Hill Order") to search for',
                paramType='query', required=False)
        .param('creatorId', 'The id of the user that created the calculation',
               required=False)
        .pagingParams(defaultSort='_id', defaultSortDir=SortDir.DESCENDING, defaultLimit=25)
    )
    def find_calc(self, moleculeId=None, imageName=None,
                  inputParametersHash=None, inputGeometryHash=None,
                  name=None, inchi=None, inchikey=None, smiles=None,
                  formula=None, creatorId=None, pending=None, limit=None,
                  offset=None, sort=None):
        return CalculationModel().findcal(
            molecule_id=moleculeId, image_name=imageName,
            input_parameters_hash=inputParametersHash,
            input_geometry_hash=inputGeometryHash, name=name, inchi=inchi,
            inchikey=inchikey, smiles=smiles, formula=formula,
            creator_id=creatorId, pending=pending, limit=limit, offset=offset,
            sort=sort, user=getCurrentUser())

    @access.public
    def find_id(self, id, params):
        user = getCurrentUser()
        cal = self._model.load(id, level=AccessType.READ, user=user)
        if not cal:
            raise RestException('Calculation not found.', code=404)

        return cal
    find_id.description = (
        Description('Get the calculation by id')
        .param(
            'id',
            'The id of calculatino.',
            dataType='string', required=True, paramType='path'))

    @access.user
    def delete(self, id, params):
        user = getCurrentUser()
        cal = self._model.load(id, level=AccessType.READ, user=user)
        if not cal:
            raise RestException('Calculation not found.', code=404)

        return self._model.remove(cal, user)
    delete.description = (
        Description('Delete a calculation by id.')
        .param(
            'id',
            'The id of calculatino.',
            dataType='string', required=True, paramType='path'))

    @access.public
    def find_calc_types(self, params):
        fields = ['access', 'properties.calculationTypes']

        query = {}
        if 'moleculeId' in params:
            query['moleculeId'] = ObjectId(params['moleculeId'])

        calcs = self._model.find(query, fields=fields)

        allTypes = []
        for types in calcs:
            calc_types = parse('properties.calculationTypes').find(types)
            if calc_types:
                calc_types = calc_types[0].value
                allTypes.extend(calc_types)

        typeSet = set(allTypes)

        return list(typeSet)

    find_calc_types.description = (
        Description('Get the calculation types available for the molecule')
        .param(
            'moleculeId',
            'The id of the molecule we are finding types for.',
            dataType='string', required=True, paramType='query'))

    @access.token
    @autoDescribeRoute(
        Description('Update the calculation properties.')
        .notes('Override the exist properties')
        .modelParam('id', 'The ID of the calculation.', model='calculation',
                    plugin='molecules', level=AccessType.ADMIN)
        .param('body', 'The new set of properties', paramType='body')
        .errorResponse('ID was invalid.')
        .errorResponse('Write access was denied for the calculation.', 403)
    )
    def update_properties(self, calculation, params):
        props = getBodyJson()
        calculation['properties'] = props
        calculation = self._model.save(calculation)

        return calculation

    @access.user
    @autoDescribeRoute(
        Description('Add notebooks ( file ids ) to molecule.')
        .modelParam('id', 'The calculation id',
                    model=CalculationModel, destName='calculation',
                    force=True, paramType='path')
        .jsonParam('notebooks', 'List of notebooks', required=True, paramType='body')
    )
    def add_notebooks(self, calculation, notebooks):
        notebooks = notebooks.get('notebooks')
        if notebooks is not None:
            CalculationModel().add_notebooks(calculation, notebooks)
Пример #22
0
            "type": "string",
            "description": "External, unique identifier"
        },
        "provider": {
            "type": "string",
            "description": "Name of the provider",
            "enum": [
                "DataONE",
                "HTTP",
                "Globus"
            ]
        }
    }
}
datasetModelKeys = set(datasetModel['properties'].keys())
addModel('dataset', datasetModel, resources='dataset')


def _itemOrFolderToDataset(obj):
    ds = {key: obj[key] for key in obj.keys() & datasetModelKeys}
    ds['provider'] = obj['meta'].get('provider', 'unknown')
    ds['identifier'] = obj['meta'].get('identifier', 'unknown')
    ds['modelType'] = obj['_modelType']
    return ds


class Dataset(Resource):

    def __init__(self):
        super(Dataset, self).__init__()
        self.resourceName = 'dataset'
Пример #23
0
class SftpAssetstoreResource(Resource):
    def __init__(self):
        super(SftpAssetstoreResource, self).__init__()
        self.resourceName = 'sftp_assetstores'

        self.route('POST', (), self.create_assetstore)
        self.route('POST', (':id', 'files'), self.create_file)

    @access.user
    def create_assetstore(self, params):
        """Create a new SFTP assetstore."""

        params = getBodyJson()
        self.requireParams(('name', 'host', 'user'), params)

        return self.model('assetstore').save({
            'type': AssetstoreType.SFTP,
            'name': params.get('name'),
            'sftp': {
                'host': params.get('host'),
                'user': params.get('user'),
                'authKey': params.get('authKey')
            }
        })

    addModel(
        'CreateAssetstoreParams', {
            'id': 'CreateAssetstoreParams',
            'required': ['name', 'itemId', 'size', 'path'],
            'properties': {
                'name': {
                    'type': 'string',
                    'description': '.'
                },
                'host': {
                    'type': 'string',
                    'description': 'The host where files are stored.'
                },
                'user': {
                    'type': 'number',
                    'description': 'The user to access the files.'
                },
                'authKey': {
                    'type':
                    'string',
                    'description':
                    'A key that can be used to lookup authentication credentials.'
                }
            }
        }, 'sftp')

    create_assetstore.description = (
        Description('Create a new sftp assetstore.').param(
            'body',
            'The parameter to create the assetstore',
            required=True,
            paramType='body',
            dataType='CreateAssetstoreParams'))

    @access.user
    @loadmodel(model='assetstore')
    def create_file(self, assetstore, params):
        params = getBodyJson()
        self.requireParams(('name', 'itemId', 'size', 'path'), params)
        name = params['name']
        item_id = params['itemId']
        size = int(params['size'])
        path = params['path']
        user = self.getCurrentUser()

        mime_type = params.get('mimeType')
        item = self.model('item').load(id=item_id,
                                       user=user,
                                       level=AccessType.WRITE,
                                       exc=True)

        file = self.model('file').createFile(name=name,
                                             creator=user,
                                             item=item,
                                             reuseExisting=True,
                                             assetstore=assetstore,
                                             mimeType=mime_type,
                                             size=size)

        file['path'] = path
        file['imported'] = True
        self.model('file').save(file)

        return self.model('file').filter(file)

    addModel(
        'CreateFileParams', {
            'id': 'CreateFileParams',
            'required': ['name', 'itemId', 'size', 'path'],
            'properties': {
                'name': {
                    'type': 'string',
                    'description': 'The name of the file.'
                },
                'itemId': {
                    'type': 'string',
                    'description': 'The item to attach the file to.'
                },
                'size': {
                    'type': 'number',
                    'description': 'The size of the file.'
                },
                'path': {
                    'type': 'string',
                    'description': 'The full path to the file.'
                },
                'mimeType': {
                    'type': 'string',
                    'description': 'The the mimeType of the file.'
                },
            }
        }, 'sftp')

    create_file.description = (
        Description('Create a new file in this assetstore.').param(
            'id',
            'The the assetstore to create the file in',
            required=True,
            paramType='path').param('body',
                                    'The parameter to create the file with.',
                                    required=True,
                                    paramType='body',
                                    dataType='CreateFileParams'))
Пример #24
0
addModel(
    'AwsParameters', {
        'id':
        'AwsParameters',
        'required': [
            'name', 'accessKeyId', 'secretAccessKey', 'regionName',
            'regionHost', 'availabilityZone'
        ],
        'properties': {
            'name': {
                'type': 'string',
                'description': 'The name of the profile.'
            },
            'accessKeyId': {
                'type': 'string',
                'description': 'The aws access key id'
            },
            'secretAccessKey': {
                'type': 'string',
                'description': 'The aws secret access key'
            },
            'regionName': {
                'type': 'string',
                'description': 'The aws region'
            },
            'availabilityZone': {
                'type': 'string',
                'description': 'The aws availablility zone'
            }
        }
    }, 'user')
Пример #25
0
class Simulations(Resource):
    def __init__(self):
        super(Simulations, self).__init__()
        self.resourceName = 'simulations'
        self.route('GET', (':id', ), self.get)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PATCH', (':id', ), self.update)
        self.route('POST', (':id', 'clone'), self.clone)
        self.route('GET', (':id', 'steps', ':stepName'), self.get_step)
        self.route('PATCH', (':id', 'steps', ':stepName'), self.update_step)
        self.route('GET', (':id', 'download'), self.download)
        self.route('GET', (':id', 'access'), self.get_access)
        self.route('PUT', (':id', 'access'), self.set_access)
        self.route('PATCH', (':id', 'access'), self.patch_access)
        self.route('PATCH', (':id', 'access', 'revoke'), self.revoke_access)

        self._model = self.model('simulation', 'hpccloud')

    @autoDescribeRoute(
        Description('Get a simulation').modelParam('id',
                                                   'The simulation to get.',
                                                   model='simulation',
                                                   plugin='hpccloud',
                                                   level=AccessType.READ))
    @access.user
    def get(self, simulation, params):
        return simulation

    @autoDescribeRoute(
        Description('Delete a simulation').modelParam(
            'id',
            'The simulation to delete.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.WRITE))
    @access.user
    def delete(self, simulation, params):
        user = getCurrentUser()
        self._model.delete(user, simulation)

    addModel('Steps', schema.simulation['properties']['steps'], 'simulations')
    addModel(
        'UpdateProperties', {
            'id': 'UpdateProperties',
            'properties': {
                'name': {
                    'type': 'string',
                    'description': 'The simulation name.'
                },
                'description': {
                    'type': 'string',
                    'description': 'The description of the simulation.'
                },
                'active': {
                    'type': 'string',
                    'items': {
                        'type': 'string'
                    },
                    'description': 'The name of the active step.'
                },
                'disabled': {
                    'type': 'array',
                    'items': {
                        'type': 'string'
                    },
                    'description': 'List of disabled steps.'
                },
            }
        }, 'simulations')

    @autoDescribeRoute(
        Description('Update a simulation').modelParam(
            'id',
            'The simulation to update.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.WRITE).jsonParam(
                'updates',
                'The properties of the simulation to update.',
                dataType='UpdateProperties',
                required=True,
                paramType='body'))
    @access.user
    def update(self, simulation, updates, params):
        immutable = [
            'projectId', 'folderId', 'access', 'userId', '_id', 'updated',
            'created'
        ]

        for p in updates:
            if p in immutable:
                raise RestException('\'%s\' is an immutable property' % p, 400)

        user = getCurrentUser()
        name = updates.get('name')
        description = updates.get('description')
        active = updates.get('active')
        disabled = updates.get('disabled')
        status = updates.get('status')
        metadata = updates.get('metadata')
        steps = updates.get('steps')

        return self._model.update_simulation(user, simulation, name, metadata,
                                             description, active, disabled,
                                             status, steps)

    addModel(
        'CloneParams', {
            'id': 'CloneParams',
            'properties': {
                'name': {
                    'type': 'string',
                    'description': 'The new name for the simulation'
                }
            }
        }, 'simulations')

    @autoDescribeRoute(
        Description('Clone a simulation').modelParam(
            'id',
            'The simulation to clone.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.READ).jsonParam(
                'props',
                'The properies of the simulation to update.',
                dataType='CloneParams',
                required=True,
                paramType='body'))
    @access.user
    def clone(self, simulation, props, params):
        self.requireParams(('name', ), props)
        user = getCurrentUser()

        cloned = self._model.clone(user, simulation, props['name'])

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/simulations/%s' \
            % cloned['_id']

        return cloned

    @autoDescribeRoute(
        Description('Get a particular step in a simulation').modelParam(
            'id',
            'The simulation containing the step.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.READ).param('stepName',
                                         'The step name to gets.',
                                         dataType='string',
                                         required=True,
                                         paramType='path'))
    @access.user
    def get_step(self, simulation, stepName, params):
        if stepName not in simulation.get('steps', {}):
            raise RestException(
                'Simulation %s doesn\'t contain step %s' %
                (simulation['_id'], stepName), 400)

        return simulation.get('steps', {}).get(stepName)

    addModel('Step', schema.definitions['stepUpdate'], 'simulations')

    @autoDescribeRoute(
        Description('Update a particular step in a simulation').modelParam(
            'id',
            'The simulation containing the step.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.WRITE).param(
                'stepName',
                'The step name to gets.',
                dataType='string',
                required=True,
                paramType='path').jsonParam(
                    'updates',
                    'The properies of the step to update.',
                    dataType='Step',
                    required=True,
                    paramType='body'))
    @access.user
    def update_step(self, simulation, stepName, updates, params):
        user = getCurrentUser()
        immutable = ['type', 'folderId']

        if stepName not in simulation.get('steps', {}):
            raise RestException(
                'Simulation %s doesn\'t contain step %s' %
                (simulation['_id'], stepName), 400)

        for p in updates:
            if p in immutable:
                raise RestException('\'%s\' is an immutable property' % p, 400)

        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(updates,
                                schema.definitions['stepUpdate'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        status = updates.get('status')
        metadata = updates.get('metadata')
        export = updates.get('export')
        view = updates.get('view')

        return self._model.update_step(user, simulation, stepName, status,
                                       metadata, export, view)

    @autoDescribeRoute(
        Description('Get access object of simulation').modelParam(
            'id',
            'id of the simulation',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.READ))
    @access.user
    def get_access(self, simulation, params):
        return simulation.get('access', {'groups': [], 'users': []})

    @autoDescribeRoute(
        Description(
            'Share a simulation with a set of users or groups').modelParam(
                'id',
                'The simulation to be shared.',
                model='simulation',
                plugin='hpccloud',
                level=AccessType.WRITE).jsonParam(
                    'share',
                    'Array of users to share the project with.',
                    dataType='object',
                    required=True,
                    paramType='body'))
    @access.user
    def set_access(self, simulation, share, params):
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.simulation['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])

        return self._model.set_access(user, simulation, users, groups)

    @autoDescribeRoute(
        Description(
            'Share a simulation with a set of users or groups').modelParam(
                'id',
                'The simulation to be shared.',
                model='simulation',
                plugin='hpccloud',
                level=AccessType.WRITE).jsonParam(
                    'share',
                    'Array of users to share the project with.',
                    dataType='object',
                    required=True,
                    paramType='body'))
    @access.user
    def patch_access(self, simulation, share, params):
        user = getCurrentUser()
        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.simulation['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])
        level = share.get('level', 0)
        flags = share.get('flags', [])

        return self._model.patch_access(user, simulation, users, groups, level,
                                        flags)

    @autoDescribeRoute(
        Description('Revoke permissions for asimulation given a set of users \
                    or groups').modelParam(
            'id',
            'The simulation to be unshared.',
            model='simulation',
            plugin='hpccloud',
            level=AccessType.WRITE).jsonParam(
                'share',
                'Array of users to share the project with.',
                dataType='object',
                required=True,
                paramType='body'))
    @access.user
    def revoke_access(self, simulation, share, params):
        user = getCurrentUser()

        # Validate we have been given a value body
        try:
            ref_resolver = jsonschema.RefResolver.from_schema(
                schema.definitions)
            jsonschema.validate(share,
                                schema.simulation['definitions']['share'],
                                resolver=ref_resolver)
        except jsonschema.ValidationError as ve:
            raise RestException(ve.message, 400)

        users = share.get('users', [])
        groups = share.get('groups', [])

        return self._model.revoke_access(user, simulation, users, groups)

    @autoDescribeRoute(
        Description(
            'Download all the asset associated with a simulation').modelParam(
                'id',
                'The simulation to download.',
                model='simulation',
                plugin='hpccloud',
                level=AccessType.READ))
    @access.user
    def download(self, simulation, params):
        user = self.getCurrentUser()
        cherrypy.response.headers['Content-Type'] = 'application/zip'
        cherrypy.response.headers['Content-Disposition'] = \
            u'attachment; filename="{}{}"'.format(simulation['name'], '.zip')

        def stream():
            zip = ziputil.ZipGenerator()
            for (path, file) in list_simulation_assets(user, simulation):
                for data in zip.addFile(file, path):
                    yield data
            yield zip.footer()

        return stream

    @autoDescribeRoute(
        Description('Set particular step in the simulation as the active step'
                    ).modelParam('id',
                                 'The simulation id.',
                                 model='simulation',
                                 plugin='hpccloud',
                                 level=AccessType.WRITE).param(
                                     'stepName',
                                     'The simulation step name.',
                                     dataType='string',
                                     required=True,
                                     paramType='path'))
    @access.user
    def set_active(self, simulation, stepName, params):

        if stepName not in simulation.get('steps', {}):
            raise RestException('Invalid step name', 400)

        simulation['active'] = stepName

        return self._model.save(simulation)
Пример #26
0
        '_accessLevel': 2,
        '_id': '5873dcdbaec030000144d233',
        '_modelType': 'image',
        'fullName': 'Xarthisius/wt_image',
        'creatorId': '18312dcdbaec030000144d233',
        'created': '2017-01-09T18:56:27.262000+00:00',
        'description': 'My fancy image',
        'digest': '123456',
        'parentId': 'null',
        'public': True,
        'tags': ['latest', 'py3'],
        'status': 'building',
        'updated': '2017-01-10T16:15:17.313000+00:00',
    },
}
addModel('image', imageModel, resources='image')


class Image(Resource):
    def __init__(self):
        super(Image, self).__init__()
        self.resourceName = 'image'

        self.route('GET', (), self.listImages)
        self.route('POST', (), self.createImage)
        self.route('GET', (':id', ), self.getImage)
        self.route('PUT', (':id', ), self.updateImage)
        self.route('DELETE', (':id', ), self.deleteImage)
        self.route('PUT', (':id', 'build'), self.buildImage)
        self.route('PUT', (':id', 'check'), self.checkImage)
        self.route('POST', (':id', 'copy'), self.copyImage)
Пример #27
0
containerInfoSchema = {
    'title': 'containerInfo',
    '$schema': 'http://json-schema.org/draft-04/schema#',
    'description': 'A subset of docker info parameters used by Tales',
    'type': 'object',
    'properties': {
        'created': {
            'type': 'string',
            'format': 'date-time',
        },
        'name': {
            'type': 'string',
        },
        'nodeId': {
            'type': 'string',
        },
        'mountPoint': {
            'type': 'string',
        },
        'volumeName': {
            'type': 'string',
        },
        'urlPath': {
            'type': 'string',
        }
    },
    'required': ['name', 'mountPoint', 'nodeId', 'volumeName'],
}

addModel('containerConfig', containerConfigSchema)
Пример #28
0
class TaskFlows(Resource):
    def __init__(self):
        super(TaskFlows, self).__init__()
        self.resourceName = 'taskflows'
        self.route('POST', (), self.create)
        self.route('PATCH', (':id', ), self.update)
        self.route('GET', (':id', 'status'), self.status)
        self.route('GET', (':id', ), self.get)
        self.route('POST', (':id', 'log'), self.log)
        self.route('PUT', (':id', 'terminate'), self.terminate)
        self.route('PUT', (':id', 'start'), self.start)
        self.route('POST', (':id', 'tasks'), self.create_task)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PUT', (':id', 'delete'), self.delete_finished)
        self.route('GET', (':id', 'access'), self.get_access)
        self.route('PUT', (':id', 'access'), self.set_access)
        self.route('PATCH', (':id', 'access'), self.patch_access)
        self.route('PATCH', (':id', 'access', 'revoke'), self.revoke_access)
        self.route('GET', (':id', 'tasks'), self.tasks)
        self.route('PUT', (':id', 'tasks', ':taskId', 'finished'),
                   self.task_finished)
        self.route('GET', (':id', 'log'), self.get_log)

        self._model = self.model('taskflow', 'taskflow')

    def _check_status(self, request):
        if request.status_code != 200:
            print >> sys.stderr, request.content
            request.raise_for_status()

    addModel(
        'CreateTaskFlowParams', {
            'id': 'CreateTaskFlowParams',
            'required': ['taskFlowClass'],
            'properties': {
                'taskFlowClass': {
                    'type': 'string'
                }
            }
        }, 'taskflows')

    @access.token
    @filtermodel(model='taskflow', plugin='taskflow')
    @describeRoute(
        Description('Create the taskflow').param(
            'body',
            'The properties to update',
            required=False,
            paramType='body',
            dataType='CreateTaskFlowParams'))
    def create(self, params):
        user = self.getCurrentUser()
        taskflow = getBodyJson()

        self.requireParams(['taskFlowClass'], taskflow)

        # Check that we can load the class
        try:
            load_class(taskflow['taskFlowClass'])
        except Exception as ex:
            msg = 'Unable to load taskflow class: %s (%s)' % \
                  (taskflow['taskFlowClass'], ex)
            logger.exception(msg)
            traceback.print_exc()
            raise RestException(msg, 400)
        taskflow = self._model.create(user, taskflow)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/taskflows/%s' % \
                                                taskflow['_id']

        return taskflow

    @access.token
    @filtermodel(model='taskflow', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.WRITE)
    @describeRoute(
        Description('Update the taskflow').param(
            'id', 'The id of taskflow', required=True,
            paramType='path').param('updates',
                                    'The properties to update',
                                    required=True,
                                    paramType='body',
                                    dataType='object'))
    def update(self, taskflow, params):
        user = self.getCurrentUser()
        immutable = [
            'access', '_id', 'taskFlowClass', 'log', 'activeTaskCount'
        ]
        updates = getBodyJson()
        if not updates:
            raise RestException('A body must be provided', code=400)

        for p in updates:
            if p in immutable:
                raise RestException('\'%s\' is an immutable property' % p, 400)

        taskflow = self._model.update_taskflow(user, taskflow, updates)

        return taskflow

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.READ)
    @describeRoute(
        Description('Get the taskflow status').param('id',
                                                     'The id of taskflow',
                                                     required=True,
                                                     paramType='path'))
    def status(self, taskflow, params):
        return {'status': taskflow['status']}

    @access.token
    @filtermodel(model='taskflow', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.READ)
    @describeRoute(
        Description('Get a taskflow').param(
            'id', 'The id of taskflow', required=True,
            paramType='path').param('path',
                                    'Option path to a particular property',
                                    required=False,
                                    paramType='query'))
    def get(self, taskflow, params):

        if 'path' in params:
            taskflow = self._model.get_path(taskflow, params['path'])

        return taskflow

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.WRITE)
    @describeRoute(None)
    def log(self, taskflow, params):
        body = cherrypy.request.body.read().decode('utf8')
        if not body:
            raise RestException('Log entry must be provided', code=400)

        self._model.append_to_log(taskflow, json.loads(body))

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.WRITE)
    @describeRoute(
        Description('Terminate the taskflow ').param('id',
                                                     'The id of taskflow',
                                                     required=True,
                                                     paramType='path'))
    def terminate(self, taskflow, params):
        user = getCurrentUser()
        taskflow['status'] = TaskFlowState.TERMINATING

        self._model.save(taskflow)
        constructor = load_class(taskflow['taskFlowClass'])
        token = self.model('token').createToken(user=user, days=7)
        taskflow_instance = constructor(
            id=str(taskflow['_id']),
            girder_token=token['_id'],
            girder_api_url=cumulus.config.girder.baseUrl)

        # Set the meta data
        taskflow_instance['meta'] = taskflow.get('meta', {})
        # Mark the taskflow as being used to termination
        taskflow_instance['terminate'] = True

        taskflow_instance.terminate()

    @access.token
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Start the taskflow ').param(
            'id', 'The id of task', required=True,
            paramType='path').param('body',
                                    'The input to the taskflow',
                                    required=False,
                                    paramType='body'))
    def start(self, taskflow, params):
        user = self.getCurrentUser()

        try:
            params = getBodyJson()
        except RestException:
            params = {}

        constructor = load_class(taskflow['taskFlowClass'])
        token = self.model('token').createToken(user=user, days=7)

        workflow = constructor(id=str(taskflow['_id']),
                               girder_token=token['_id'],
                               girder_api_url=cumulus.config.girder.baseUrl)

        workflow.start(**params)

    @access.token
    @filtermodel(model='taskflow', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Delete the taskflow').param('id',
                                                 'The id of taskflow',
                                                 required=True,
                                                 paramType='path'))
    def delete(self, taskflow, params):
        user = self.getCurrentUser()

        status = self._model.status(user, taskflow)
        if status == TaskFlowState.RUNNING:
            raise RestException('Taskflow is running', 400)

        constructor = load_class(taskflow['taskFlowClass'])
        token = self.model('token').createToken(user=user, days=7)

        if taskflow['status'] != TaskFlowState.DELETING:

            taskflow['status'] = TaskFlowState.DELETING
            self._model.save(taskflow)

            workflow = constructor(
                id=str(taskflow['_id']),
                girder_token=token['_id'],
                girder_api_url=cumulus.config.girder.baseUrl)

            workflow.delete()

            # Check if we have any active tasks, it not then we are done and
            # can delete the tasks and taskflows
            taskflow = self._model.load(taskflow['_id'],
                                        user=user,
                                        level=AccessType.ADMIN)
            if taskflow['activeTaskCount'] == 0:
                self._model.delete(taskflow)
                cherrypy.response.status = 200
                taskflow['status'] = TaskFlowState.DELETED

                return taskflow

        cherrypy.response.status = 202
        return taskflow

    @access.token
    @filtermodel(model='task', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.READ)
    @describeRoute(
        Description('Get all the tasks associated with this taskflow').param(
            'id', 'The id of taskflow', required=True, paramType='path'))
    def tasks(self, taskflow, params):
        user = getCurrentUser()

        states = params.get('states')
        if states:
            states = json.loads(states)

        cursor = self.model('task', 'taskflow').find_by_taskflow_id(
            user, ObjectId(taskflow['_id']), states=states)

        return [task for task in cursor]

    addModel(
        'CreateTaskParams', {
            'id': 'CreateTaskParams',
            'required': ['celeryTaskId'],
            'properties': {
                'celeryTaskId': {
                    'type': 'string'
                }
            }
        }, 'taskflows')

    @access.user
    @filtermodel(model='task', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.READ)
    @describeRoute(
        Description('Create a new task associated with this flow').param(
            'id', 'The id of taskflow', required=True,
            paramType='path').param('body',
                                    'The properties to update',
                                    required=False,
                                    paramType='body',
                                    dataType='CreateTaskParams'))
    def create_task(self, taskflow, params):
        user = getCurrentUser()
        task = getBodyJson()

        self.requireParams(['celeryTaskId'], task)

        task = self.model('task', 'taskflow').create(user, taskflow, task)

        cherrypy.response.status = 201
        cherrypy.response.headers['Location'] = '/tasks/%s' % task['_id']

        return task

    @access.token
    @filtermodel(model='taskflow', plugin='taskflow')
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.WRITE)
    @describeRoute(None)
    def task_finished(self, taskflow, taskId, params):
        # decrement the number of active tasks
        query = {'_id': ObjectId(taskflow['_id'])}
        update = {'$inc': {'activeTaskCount': -1}}

        return self._model.collection.find_one_and_update(
            query, update, return_document=ReturnDocument.AFTER)

    @access.token
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(None)
    def delete_finished(self, taskflow, params):
        self._model.delete(taskflow)

    @access.token
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.READ)
    @describeRoute(
        Description('Get log entries for task').param(
            'id', 'The task to get log entries for.',
            paramType='path').param('offset',
                                    'A offset in to the log.',
                                    required=False,
                                    paramType='query'))
    def get_log(self, taskflow, params):
        offset = 0
        if 'offset' in params:
            offset = int(params['offset'])

        return {'log': taskflow['log'][offset:]}

    addModel(
        'ShareProperties', {
            'id': 'ShareProperties',
            'properties': {
                'users': {
                    'type': 'array',
                    'items': {
                        'type': 'string'
                    },
                    'description': 'array of user id\'s'
                },
                'groups': {
                    'type': 'array',
                    'items': {
                        'type': 'string'
                    },
                    'description': 'array of group id\'s'
                }
            }
        }, 'taskflows')

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Get access list for a taskflow').param(
            'id', 'The id of taskflow', required=True, paramType='path'))
    def get_access(self, taskflow, params):
        return taskflow.get('access', {'groups': [], 'users': []})

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Set access list for a taskflow given a list of user and \
                    group ids').param(
            'id', 'The id of taskflow', required=True,
            paramType='path').param('body',
                                    'Users and group ID\'s to be authorized.',
                                    dataType='ShareProperties',
                                    required=True,
                                    paramType='body'))
    def set_access(self, taskflow, params):
        user = self.getCurrentUser()
        body = getBodyJson()
        return self._model.set_access(user, taskflow, body['users'],
                                      body['groups'], True)

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Append access to a taskflow and its tasks').param(
            'id', 'The id of taskflow', required=True, paramType='path').param(
                'body',
                'Users and group ID\'s to share taskflow with.',
                dataType='ShareProperties',
                required=True,
                paramType='body'))
    def patch_access(self, taskflow, params):
        user = self.getCurrentUser()
        body = getBodyJson()
        return self._model.patch_access(user, taskflow, body['users'],
                                        body['groups'])

    @access.user
    @loadmodel(model='taskflow', plugin='taskflow', level=AccessType.ADMIN)
    @describeRoute(
        Description('Revoke access to a taskflow and its tasks').param(
            'id', 'The id of taskflow', required=True, paramType='path').param(
                'body',
                'Users and group ID\'s to be unauthorized.',
                dataType='ShareProperties',
                required=True,
                paramType='body'))
    def revoke_access(self, taskflow, params):
        user = self.getCurrentUser()
        body = getBodyJson()
        return self._model.revoke_access(user, taskflow, body['users'],
                                         body['groups'])
Пример #29
0
            "description": "The last time when the graph was modified."
        }
    },
    'example': {
        '_accessLevel': 2,
        '_id': '5873dcdbaec030000144d233',
        '_modelType': 'graph',
        'name': 'Fake Plant',
        'description': 'Example fake plant model graph',
        'creatorId': '18312dcdbaec030000144d233',
        'created': '2017-01-09T18:56:27.262000+00:00',
        'official': True,
        'updated': '2017-01-10T16:15:17.313000+00:00'
    },
}
addModel('graph', graphDef, resources='graph')


class Graph(Resource):
    """Defines graph API."""
    def __init__(self):
        """Initialize the API."""
        super(Graph, self).__init__()
        self.resourceName = 'graph'
        self._model = GraphModel()
        self.route('GET', (), self.listGraphs)
        self.route('GET', (':id', ), self.getGraph)
        self.route('POST', (), self.createGraph)
        self.route('PUT', (':id', ), self.updateGraph)
        self.route('DELETE', (':id', ), self.deleteGraph)
Пример #30
0
class Molecule(Resource):
    output_formats_2d = ['smiles', 'inchi', 'inchikey']
    output_formats_3d = ['cml', 'xyz', 'sdf', 'cjson']
    output_formats = output_formats_2d + output_formats_3d

    input_formats = [
        'cml', 'xyz', 'sdf', 'cjson', 'json', 'log', 'nwchem', 'pdb', 'smi',
        'smiles'
    ]
    mime_types = {
        'smiles': 'chemical/x-daylight-smiles',
        'inchi': 'chemical/x-inchi',
        'inchikey': 'text/plain',
        'cml': 'chemical/x-cml',
        'xyz': 'chemical/x-xyz',
        'sdf': 'chemical/x-mdl-sdfile',
        'cjson': 'application/json',
        'svg': 'image/svg+xml'
    }

    def __init__(self):
        super(Molecule, self).__init__()
        self.resourceName = 'molecules'
        self.route('GET', (), self.find)
        self.route('GET', ('inchikey', ':inchikey'), self.find_inchikey)
        self.route('GET', (':id', ':output_format'), self.get_format)
        self.route('GET', (':id', ), self.find_id)
        self.route('GET', (':id', 'svg'), self.get_svg)
        self.route('GET', ('search', ), self.search)
        self.route('POST', (), self.create)
        self.route('DELETE', (':id', ), self.delete)
        self.route('PATCH', (':id', ), self.update)
        self.route('PATCH', (':id', 'notebooks'), self.add_notebooks)
        self.route('POST', ('conversions', ':output_format'), self.conversions)
        self.route('POST', (':id', '3d'), self.generate_3d_coords)

    def _clean(self, doc, cjson=True):
        del doc['access']
        if 'sdf' in doc:
            del doc['sdf']
        if 'svg' in doc:
            del doc['svg']
        doc['_id'] = str(doc['_id'])
        if 'cjson' in doc:
            if cjson:
                if 'basisSet' in doc['cjson']:
                    del doc['cjson']['basisSet']
                if 'vibrations' in doc['cjson']:
                    del doc['cjson']['vibrations']
            else:
                del doc['cjson']

        return doc

    @access.public
    def find(self, params):
        return MoleculeModel().findmol(params)

    find.description = (Description('Find a molecule.').param(
        'name', 'The name of the molecule', paramType='query',
        required=False).param(
            'inchi',
            'The InChI of the molecule',
            paramType='query',
            required=False).param(
                'inchikey',
                'The InChI key of the molecule',
                paramType='query',
                required=False).param(
                    'smiles',
                    'The SMILES of the molecule',
                    paramType='query',
                    required=False).param(
                        'formula',
                        'The formula (using the "Hill Order") to search for',
                        paramType='query',
                        required=False).param(
                            'creatorId',
                            'The id of the user that created the molecule',
                            paramType='query',
                            required=False).param(
                                'queryString',
                                'The query string to use for this search '
                                '(supercedes all other search parameters)',
                                paramType='query',
                                required=False).pagingParams(
                                    defaultSort='_id',
                                    defaultSortDir=SortDir.DESCENDING,
                                    defaultLimit=25).errorResponse())

    @access.public
    def find_inchikey(self, inchikey, params):
        mol = MoleculeModel().find_inchikey(inchikey)
        if not mol:
            raise RestException('Molecule not found.', code=404)
        return self._clean(mol)

    find_inchikey.description = (
        Description('Find a molecule by InChI key.').param(
            'inchikey', 'The InChI key of the molecule',
            paramType='path').errorResponse().errorResponse(
                'Molecule not found.', 404))

    @access.public
    def find_id(self, id, params):
        mol = MoleculeModel().load(id,
                                   level=AccessType.READ,
                                   user=getCurrentUser())
        if not mol:
            raise RestException('Molecule not found.', code=404)
        cjson = True
        cjsonParam = params.get('cjson')
        if cjsonParam is not None:
            cjson = cjsonParam.lower() == 'true'
        return self._clean(mol, cjson)

    find_id.description = (Description('Get a specific molecule by id').param(
        'id', 'The id of the molecule', paramType='path'
    ).param(
        'cjson',
        'Attach the cjson data of the molecule to the response (Default: true)',
        paramType='query',
        required=False))

    @access.user
    def create(self, params):
        body = self.getBodyJson()
        user = self.getCurrentUser()
        public = body.get('public', False)
        gen3d = body.get('generate3D', True)
        provenance = body.get('provenance', 'uploaded by user')
        mol = None
        if 'fileId' in body:
            file_id = body['fileId']
            file = ModelImporter.model('file').load(file_id, user=user)
            parts = file['name'].split('.')
            input_format = parts[-1]
            name = '.'.join(parts[:-1])

            if input_format not in Molecule.input_formats:
                raise RestException('Input format not supported.', code=400)

            with File().open(file) as f:
                data_str = f.read().decode()

            mol = create_molecule(data_str, input_format, user, public, gen3d,
                                  provenance)
        elif 'inchi' in body:
            input_format = 'inchi'
            data = body['inchi']
            if not data.startswith('InChI='):
                data = 'InChI=' + data

            mol = create_molecule(data, input_format, user, public, gen3d,
                                  provenance)

        for key in body:
            if key in Molecule.input_formats:
                input_format = key
                data = body[input_format]
                # Convert to str if necessary
                if isinstance(data, dict):
                    data = json.dumps(data)
                mol = create_molecule(data, input_format, user, public, gen3d,
                                      provenance)
                break

        if not mol:
            raise RestException('Invalid request', code=400)

        return self._clean(mol)

    addModel(
        'Molecule', 'MoleculeParams', {
            "id": "MoleculeParams",
            "required": ["name", "inchi"],
            "properties": {
                "name": {
                    "type": "string",
                    "description": "The common name of the molecule"
                },
                "inchi": {
                    "type": "string",
                    "description": "The InChI of the molecule."
                }
            }
        })
    create.description = (Description('Create a molecule').param(
        'body',
        'The molecule to be added to the database.',
        dataType='MoleculeParams',
        required=True,
        paramType='body').errorResponse('Input format not supported.',
                                        code=400))

    @access.user
    def delete(self, id, params):
        user = self.getCurrentUser()
        mol = MoleculeModel().load(id, user=user, level=AccessType.WRITE)

        if not mol:
            raise RestException('Molecule not found.', code=404)

        return MoleculeModel().remove(mol)

    delete.description = (Description('Delete a molecule by id.').param(
        'id', 'The id of the molecule',
        paramType='path').errorResponse().errorResponse(
            'Molecule not found.', 404))

    @access.user
    def update(self, id, params):
        user = self.getCurrentUser()

        mol = MoleculeModel().load(id, user=user, level=AccessType.WRITE)

        if not mol:
            raise RestException('Molecule not found.', code=404)

        body = self.getBodyJson()

        query = {'_id': mol['_id']}

        updates = {'$set': {}, '$addToSet': {}}

        if 'name' in body:
            updates['$set']['name'] = body['name']

        if 'logs' in body:
            updates['$addToSet']['logs'] = body['logs']

        # Remove unused keys
        updates = {k: v for k, v in updates.items() if v}

        super(MoleculeModel, MoleculeModel()).update(query, updates)

        # Reload the molecule
        mol = MoleculeModel().load(id, user=user)

        return self._clean(mol)

    addModel(
        'Molecule', 'UpdateMoleculeParams', {
            "id": "UpdateMoleculeParams",
            "properties": {
                "logs": {
                    "type": "array",
                    "description": "List of Girder file ids"
                }
            }
        })
    update.description = (Description('Update a molecule by id.').param(
        'id', 'The id of the molecule', paramType='path').param(
            'body',
            'The update to the molecule.',
            dataType='UpdateMoleculeParams',
            required=True,
            paramType='body').errorResponse('Molecule not found.', 404))

    @access.user
    @autoDescribeRoute(
        Description('Add notebooks ( file ids ) to molecule.').modelParam(
            'id',
            'The molecule id',
            model=MoleculeModel,
            destName='molecule',
            force=True,
            paramType='path').jsonParam('notebooks',
                                        'List of notebooks',
                                        required=True,
                                        paramType='body'))
    def add_notebooks(self, molecule, notebooks):
        notebooks = notebooks.get('notebooks')
        if notebooks is not None:
            MoleculeModel().add_notebooks(molecule, notebooks)

    @access.user
    def conversions(self, output_format, params):
        user = self.getCurrentUser()

        if output_format not in Molecule.output_formats:
            raise RestException('Output output_format not supported.',
                                code=404)

        body = self.getBodyJson()

        if 'fileId' not in body:
            raise RestException('Invalid request body.', code=400)

        file_id = body['fileId']

        file = ModelImporter.model('file').load(file_id, user=user)

        input_format = file['name'].split('.')[-1]

        if input_format not in Molecule.input_formats:
            raise RestException('Input format not supported.', code=400)

        if file is None:
            raise RestException('File not found.', code=404)

        with File().load(file) as f:
            data_str = f.read().decode()

        if output_format.startswith('inchi'):
            atom_count = 0
            if input_format == 'pdb':
                props = openbabel.properties(data_str, input_format)
                atom_count = props['atomCount']
            else:
                atom_count = int(avogadro.atom_count(data_str, input_format))

            if atom_count > 1024:
                raise RestException(
                    'Unable to generate InChI, molecule has more than 1024 atoms.',
                    code=400)

            if input_format == 'pdb':
                (inchi, inchikey) = openbabel.to_inchi(data_str, input_format)
            else:
                sdf = avogadro.convert_str(data_str, input_format, 'sdf')
                (inchi, inchikey) = openbabel.to_inchi(sdf, 'sdf')

            if output_format == 'inchi':
                return inchi
            else:
                return inchikey

        else:
            output = ''
            mime = 'text/plain'
            if input_format == 'pdb':
                (output, mime) = openbabel.convert_str(data_str, input_format,
                                                       output_format)
            else:
                output = avogadro.convert_str(data_str, input_format,
                                              output_format)

            def stream():
                cherrypy.response.headers['Content-Type'] = mime
                yield output

            return stream

    addModel(
        'Molecule', 'ConversionParams', {
            "id": "ConversionParams",
            "properties": {
                "fileId": {
                    "type": "string",
                    "description": "Girder file id to do conversion on"
                }
            }
        })
    conversions.description = (Description('Update a molecule by id.').param(
        'format', 'The format to convert to', paramType='path').param(
            'body',
            'Details of molecule data to perform conversion on',
            dataType='ConversionParams',
            required=True,
            paramType='body').errorResponse(
                'Output format not supported.',
                404).errorResponse('File not found.', 404).errorResponse(
                    'Invalid request body.',
                    400).errorResponse('Input format not supported.',
                                       code=400))

    @access.public
    def get_format(self, id, output_format, params):
        # For now will for force load ( i.e. ignore access control )
        # This will change when we have access controls.
        molecule = MoleculeModel().load(id, force=True)

        if output_format not in Molecule.output_formats:
            raise RestException('Format not supported.', code=400)

        if output_format in Molecule.output_formats_3d:
            # If it is a 3d output format, cjson is required
            if 'cjson' not in molecule:
                raise RestException('Molecule does not have 3D coordinates.',
                                    404)

            data = json.dumps(molecule['cjson'])
            if output_format != 'cjson':
                data = avogadro.convert_str(data, 'cjson', output_format)
        else:
            # Right now, all 2d output formats are stored in the molecule
            data = molecule[output_format]

        def stream():
            cherrypy.response.headers['Content-Type'] = Molecule.mime_types[
                output_format]
            yield data

        return stream

    get_format.description = (
        Description('Get molecule in particular format.').param(
            'id', 'The id of the molecule', paramType='path').param(
                'output_format',
                'The format to convert to', paramType='path').errorResponse(
                    'Output format not supported.', 400).errorResponse(
                        'Molecule does not have 3D coordinates.', 404))

    @access.public
    @autoDescribeRoute(
        Description('Get an SVG representation of a molecule.').param(
            'id', 'The id of the molecule', paramType='path').errorResponse(
                'Molecule not found.',
                404).errorResponse('Molecule does not have SVG data.', 404))
    def get_svg(self, id):
        # For now will for force load ( i.e. ignore access control )
        # This will change when we have access controls.
        mol = MoleculeModel().load(id, force=True)

        if not mol:
            raise RestException('Molecule not found.', code=404)

        if 'svg' not in mol:
            raise RestException('Molecule does not have SVG data.', code=404)

        data = mol['svg']

        cherrypy.response.headers['Content-Type'] = Molecule.mime_types['svg']

        def stream():
            yield data.encode()

        return stream

    @access.public
    def search(self, params):
        limit, offset, sort = parse_pagination_params(params)

        query_string = params.get('q')
        formula = params.get('formula')
        cactus = params.get('cactus')
        if query_string is None and formula is None and cactus is None:
            raise RestException(
                'Either \'q\', \'formula\' or \'cactus\' is required.')

        if query_string is not None:
            try:
                mongo_query = query.to_mongo_query(query_string)
            except query.InvalidQuery:
                raise RestException('Invalid query', 400)

            fields = ['inchikey', 'smiles', 'properties', 'name']
            cursor = MoleculeModel().find(query=mongo_query,
                                          fields=fields,
                                          limit=limit,
                                          offset=offset,
                                          sort=sort)
            mols = [x for x in cursor]
            num_matches = cursor.collection.count_documents(mongo_query)

            return search_results_dict(mols, num_matches, limit, offset, sort)

        elif formula:
            # Search using formula
            return MoleculeModel().findmol(params)

        elif cactus:
            if getCurrentUser() is None:
                raise RestException('Must be logged in to search with cactus.')

            # Disable cert verification for now
            # TODO Ensure we have the right root certs so this just works.
            r = requests.get(
                'https://cactus.nci.nih.gov/chemical/structure/%s/file?format=sdf'
                % cactus,
                verify=False)

            if r.status_code == 404:
                return []
            else:
                r.raise_for_status()

            sdf_data = r.content.decode('utf8')
            provenance = 'cactus: ' + cactus
            mol = create_molecule(sdf_data,
                                  'sdf',
                                  getCurrentUser(),
                                  True,
                                  provenance=provenance)

            return search_results_dict([mol], 1, limit, offset, sort)

    search.description = (Description(
        'Search for molecules using a query string, formula, or cactus').param(
            'q',
            'The query string to use for this search',
            paramType='query',
            required=False).param(
                'formula',
                'The formula (using the "Hill Order") to search for',
                paramType='query',
                required=False).param('cactus',
                                      'The identifier to pass to cactus',
                                      paramType='query',
                                      required=False).pagingParams(
                                          defaultSort='_id',
                                          defaultSortDir=SortDir.DESCENDING,
                                          defaultLimit=25))

    @access.user
    @autoDescribeRoute(
        Description('Generate 3D coordinates for a molecule.').modelParam(
            'id',
            'The id of the molecule',
            destName='mol',
            level=AccessType.WRITE,
            model=MoleculeModel).errorResponse('Molecule not found.', 404))
    def generate_3d_coords(self, mol):
        """Generate 3D coords if not present and not being generated"""

        if (MoleculeModel().has_3d_coords(mol)
                or mol.get('generating_3d_coords', False)):
            return self._clean(mol)

        user = self.getCurrentUser()

        async_requests.schedule_3d_coords_gen(mol, user)
        return self._clean(mol)
Пример #31
0
        return kwargs


testModel = {
    'id': 'Body',
    'require': 'bob',
    'properties': {
        'bob': {
            'type': 'array',
            'items': {
                'type': 'integer'
            }
        }
    }
}
docs.addModel('Body', testModel, resources='model')

globalModel = {
    'id': 'Global',
    'properties': {}
}
docs.addModel('Global', globalModel)


class ModelResource(Resource):
    def __init__(self):
        super(ModelResource, self).__init__()
        self.resourceName = 'model'
        self.route('POST', (), self.hasModel)

    @access.public
Пример #32
0
        return kwargs


testModel = {
    'id': 'Body',
    'require': 'bob',
    'properties': {
        'bob': {
            'type': 'array',
            'items': {
                'type': 'integer'
            }
        }
    }
}
docs.addModel('Body', testModel, resources='model')

globalModel = {'id': 'Global', 'properties': {}}
docs.addModel('Global', globalModel)


class ModelResource(Resource):
    def __init__(self):
        super(ModelResource, self).__init__()
        self.resourceName = 'model'
        self.route('POST', (), self.hasModel)

    @access.public
    @describe.describeRoute(
        describe.Description('What a model').param('body',
                                                   'Where its at!',
Пример #33
0
        'imageName': {'type': 'string', 'allowEmptyValue': False},
        'command': {'type': 'string', 'allowEmptyValue': True},
        'memLimit': {'type': 'string', 'allowEmptyValue': True},
        'cpuShares': {'type': 'string', 'allowEmptyValue': True},
        'description': {'type': 'string', 'allowEmptyValue': False},
        'port': {'type': 'integer', 'format': 'int32',
                 'allowEmptyValue': True,
                 'maximum': 65535, 'minimum': 1},
        'updated': {'type': 'string', 'format': 'date',
                    'allowEmptyValue': True},
        'created': {'type': 'string', 'format': 'date',
                    'allowEmptyValue': True},
        'public': {'type': 'boolean', 'allowEmptyValue': True},
    }
}
addModel('frontend', frontendModel, resources='frontend')


class Frontend(Resource):

    def __init__(self):
        super(Frontend, self).__init__()
        self.resourceName = 'frontend'

        self.route('GET', (), self.listFrontends)
        self.route('GET', (':id',), self.getFrontend)
        self.route('PUT', (':id',), self.updateFrontend)
        self.route('POST', (), self.createFrontend)
        self.route('DELETE', (':id',), self.deleteFrontend)

    @access.public