def test_run_experiment_twice(self, mock_run_script): # create experiment form = self.form.get_form() form.request.form.update({ 'form.buttons.save': 'Create and start', }) # update form with updated request form.update() # start experiment jt = IExperimentJobTracker(self.experiments['my-experiment']) self.assertEqual(jt.state, u'QUEUED') # error state = jt.start_job(form.request) self.assertEqual(state[0], 'error') # setup mock_run_script mock_run_script.side_effect = self.form.mock_run_script # finish current job transaction.commit() self.assertEqual(jt.state, u'COMPLETED') # TODO: after commit tasks cause site to disappear and the # following code will fail, bceause without site we can't find # a catalog without whchi we can't finde the toolkit by uuid jt.start_job(form.request) self.assertEqual(jt.state, u'PARTIAL') transaction.commit() self.assertEqual(jt.state, u'COMPLETED')
def test_run_experiment_twice(self, mock_run_script): # create experiment form = self.form.get_form() form.request.form.update({ 'form.buttons.save': 'Create and start', }) # update form with updated request form.update() # start experiment jt = IExperimentJobTracker(self.experiments['my-experiment']) self.assertEqual(jt.state, u'QUEUED') # error state = jt.start_job(form.request) self.assertEqual(state[0], 'error') # setup mock_run_script mock_run_script.side_effect = self.form.mock_run_script # finish current job transaction.commit() self.assertEqual(jt.state, u'COMPLETED') # TODO: after commit tasks cause site to disappear and the # following code will fail, bceause without site we can't find # a catalog without whchi we can't finde the toolkit by uuid jt.start_job(form.request) self.assertEqual(jt.state, u'PARTIAL') transaction.commit() self.assertEqual(jt.state, u'COMPLETED')
def pullOccurrenceFromALA(self, lsid, taxon, dataSrc='ala', common=None): # TODO: check permisions? # 1. create new dataset with taxon, lsid and common name set portal = getToolByName(self.context, 'portal_url').getPortalObject() if dataSrc == 'ala': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['ala'] elif dataSrc == 'gbif': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['gbif'] elif dataSrc == 'aekos': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['aekos'] else: raise BadRequest('Invalid data source {0}'.format(dataSrc)) title = [taxon] if common: title.append(u"({})".format(common)) # TODO: move content creation into IALAJobTracker? # remotedataset? swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' else: portal_type = 'org.bccvl.content.dataset' # TODO: make sure we get a better content id that dataset-x ds = createContentInContainer(dscontainer, portal_type, title=u' '.join(title)) ds.dataSource = dataSrc # Either ALA or GBIF as source # TODO: add number of occurences to description ds.description = u' '.join( title) + u' imported from ' + unicode(dataSrc.upper()) md = IBCCVLMetadata(ds) # TODO: provenance ... import url? # FIXME: verify input parameters before adding to graph md['genre'] = 'DataGenreSpeciesOccurrence' md['species'] = { 'scientificName': taxon, 'taxonID': lsid, } if common: md['species']['vernacularName'] = common IStatusMessage(self.request).add('New Dataset created', type='info') # 2. create and push alaimport job for dataset # TODO: make this named adapter jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex object to make sure everything is up to date ds.reindexObject() # Job submission state notifier IStatusMessage(self.request).add(message, type=status) return (status, message)
def __call__(self): # TODO: could also just submit current context (the experiment) # with all infos accessible from it # TODO: if submitted as admin, submit experiment as owner # also add option to restart single restart within experiment jt = IExperimentJobTracker(self.context) msgtype, msg = jt.start_job(self.request) if msgtype is not None: IStatusMessage(self.request).add(msg, type=msgtype) self.request.response.redirect(self.context.absolute_url())
def __call__(self): # TODO: could also just submit current context (the experiment) # with all infos accessible from it # TODO: if submitted as admin, submit experiment as owner # also add option to restart single restart within experiment jt = IExperimentJobTracker(self.context) msgtype, msg = jt.start_job(self.request) if msgtype is not None: IStatusMessage(self.request).add(msg, type=msgtype) self.request.response.redirect(self.context.absolute_url())
def handleAdd(self, action): data, errors = self.extractData() self.validateAction(data) if errors: self.status = self.formErrorsMessage return # TODO: this is prob. a bug in base form, because createAndAdd # does not return the wrapped object. obj = self.createAndAdd(data) if obj is None: # TODO: this is probably an error here? # object creation/add failed for some reason return # get wrapped instance fo new object (see above) obj = self.context[obj.id] # mark only as finished if we get the new object self._finishedAdd = True IStatusMessage(self.request).addStatusMessage(_(u"Item created"), "info") # auto start job here jt = IExperimentJobTracker(obj) msgtype, msg = jt.start_job(self.request) if msgtype is not None: IStatusMessage(self.request).add(msg, type=msgtype)
def handleAdd(self, action): data, errors = self.extractData() self.validateAction(data) if errors: self.status = self.formErrorsMessage return # TODO: this is prob. a bug in base form, because createAndAdd # does not return the wrapped object. obj = self.createAndAdd(data) if obj is None: # TODO: this is probably an error here? # object creation/add failed for some reason return # get wrapped instance fo new object (see above) obj = self.context[obj.id] # mark only as finished if we get the new object self._finishedAdd = True IStatusMessage(self.request).addStatusMessage(_(u"Item created"), "info") # auto start job here jt = IExperimentJobTracker(obj) msgtype, msg = jt.start_job(self.request) if msgtype is not None: IStatusMessage(self.request).add(msg, type=msgtype)
def submitsdm(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('occurrence_data', None): self.record_error('Bad Request', 400, 'Missing parameter occurrence_data', {'parameter': 'occurrence_data'}) else: # FIXME: should properly support source / id # for now only bccvl source is supported props['species_occurrence_dataset'] = params[ 'occurrence_data']['id'] # FIXME: should properly support source/id for onw only bccvl source is # supported props['species_absence_dataset'] = params.get( 'absence_data', {}).get('id', None) props['scale_down'] = params.get('scale_down', False) if not params.get('environmental_data', None): self.record_error('Bad Request', 400, 'Missing parameter environmental_data', {'parameter': 'environmental_data'}) else: props['environmental_datasets'] = params['environmental_data'] if params.get('modelling_region', ''): props['modelling_region'] = NamedBlobFile( data=json.dumps(params['modelling_region'])) else: props['modelling_region'] = None if not params.get('algorithms', None): self.record_error('Bad Request', 400, 'Missing parameter algorithms', {'parameter': 'algorithms'}) else: portal = ploneapi.portal.get() props['functions'] = {} # FIXME: make sure we get the default values from our func object for algo, algo_params in params['algorithms'].items(): if algo_params is None: algo_params = {} toolkit = portal[defaults.FUNCTIONS_FOLDER_ID][algo] toolkit_model = loadString(toolkit.schema) toolkit_schema = toolkit_model.schema func_props = {} for field_name in toolkit_schema.names(): field = toolkit_schema.get(field_name) value = algo_params.get(field_name, field.missing_value) if value == field.missing_value: func_props[field_name] = field.default else: func_props[field_name] = value props['functions'][IUUID(toolkit)] = func_props if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent("org.bccvl.content.sdmexperiment", **props) experiment = addContentToContainer(context, experiment) # TODO: check if props and algo params have been applied properly experiment.parameters = dict(props['functions']) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def submitcc(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('species_distribution_models', None): self.record_error('Bad Request', 400, 'Missing parameter species_distribution_models', {'parameter': 'species_distribution_models'}) else: props['species_distribution_models'] = params[ 'species_distribution_models'] if not params.get('future_climate_datasets', None): self.record_error('Bad Request', 400, 'Missing parameter future_climate_datasets', {'parameter': 'future_climate_datasets'}) else: props['future_climate_datasets'] = params[ 'future_climate_datasets'] if params.get('projection_region', ''): props['projection_region'] = NamedBlobFile( data=json.dumps(params['projection_region'])) else: props['projection_region'] = None if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent("org.bccvl.content.projectionexperiment", **props) experiment = addContentToContainer(context, experiment) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def submittraits(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if params.get('species_list', None): props['species_list'] = params['species_list'] else: self.record_error('Bad Request', 400, 'Missing parameter speciesList', {'parameter': 'speciesList'}) if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('traits_data', None): self.record_error('Bad Request', 400, 'Missing parameter traits_data', {'parameter': 'traits_data'}) else: # FIXME: should properly support source / id # for now only bccvl source is supported props['species_traits_dataset'] = params[ 'traits_data']['id'] props['species_traits_dataset_params'] = {} for col_name, col_val in params.get("columns", {}).items(): if col_val not in ('lat', 'lon', 'species', 'trait_con', 'trait_ord', 'trait_nom', 'env_var_con', 'env_var_cat', 'random_con', 'random_cat'): continue props['species_traits_dataset_params'][col_name] = col_val if not props['species_traits_dataset_params']: self.record_error('Bad Request', 400, 'Invalid values for columns', {'parameter': 'columns'}) # Check for species-level trait data i.e. species is not specified if 'species' not in props['species_traits_dataset_params'].values(): props['species_list'] = [] props['scale_down'] = params.get('scale_down', False) # env data is optional props['environmental_datasets'] = params.get('environmental_data', None) if not (props['environmental_datasets'] or 'env_var_con' not in props['species_traits_dataset_params'].values() or 'env_var_cat' not in props['species_traits_dataset_params'].values()): self.record_error('Bad Request', 400, 'No Environmental data selected', {'parameter': 'environmental_datasets'}) if params.get('modelling_region', ''): props['modelling_region'] = NamedBlobFile( data=json.dumps(params['modelling_region'])) else: props['modelling_region'] = None if not params.get('algorithms', None): self.record_error('Bad Request', 400, 'Missing parameter algorithms', {'parameter': 'algorithms'}) else: props['algorithms_species'] = {} props['algorithms_diff'] = {} funcs_env = getUtility( IVocabularyFactory, 'traits_functions_species_source')(context) funcs_species = getUtility( IVocabularyFactory, 'traits_functions_diff_source')(context) # FIXME: make sure we get the default values from our func object for algo_uuid, algo_params in params['algorithms'].items(): if algo_params is None: algo_params = {} toolkit = uuidToObject(algo_uuid) toolkit_model = loadString(toolkit.schema) toolkit_schema = toolkit_model.schema func_props = {} for field_name in toolkit_schema.names(): field = toolkit_schema.get(field_name) value = algo_params.get(field_name, field.missing_value) if value == field.missing_value: func_props[field_name] = field.default else: func_props[field_name] = value if algo_uuid in funcs_env: props['algorithms_species'][algo_uuid] = func_props elif algo_uuid in funcs_species: props['algorithms_diff'][algo_uuid] = func_props else: LOG.warn( 'Algorithm {} not in allowed list of functions'.format(toolkit.id)) if not (props['algorithms_species'] or props['algorithms_diff']): self.record_error('Bad Request', 400, 'Iinvalid algorithms selected', {'parameter': 'algorithms'}) if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent( "org.bccvl.content.speciestraitsexperiment", **props) experiment = addContentToContainer(context, experiment) experiment.parameters = dict(props['algorithms_species']) experiment.parameters.update(dict(props['algorithms_diff'])) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def pullOccurrenceFromALA(self, lsid, taxon, dataSrc='ala', common=None): # TODO: check permisions? # 1. create new dataset with taxon, lsid and common name set portal = getToolByName(self.context, 'portal_url').getPortalObject() if dataSrc == 'ala': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['ala'] elif dataSrc == 'gbif': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['gbif'] elif dataSrc == 'aekos': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['aekos'] elif dataSrc == 'obis': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['obis'] else: raise BadRequest('Invalid data source {0}'.format(dataSrc)) title = [taxon] if common: title.append(u"({})".format(common)) # TODO: move content creation into IALAJobTracker? # remotedataset? swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' else: portal_type = 'org.bccvl.content.dataset' # TODO: make sure we get a better content id that dataset-x title = u' '.join(title) ds = createContent(portal_type, title=title) ds.dataSource = dataSrc # Either ALA or GBIF as source # TODO: add number of occurences to description ds.description = u' '.join( (title, u'imported from', unicode(dataSrc.upper()))) ds = addContentToContainer(dscontainer, ds) md = IBCCVLMetadata(ds) # TODO: provenance ... import url? # FIXME: verify input parameters before adding to graph md['genre'] = 'DataGenreSpeciesOccurrence' md['species'] = { 'scientificName': taxon, 'taxonID': lsid, } if common: md['species']['vernacularName'] = common IStatusMessage(self.request).add('New Dataset created', type='info') # 2. create and push alaimport job for dataset # TODO: make this named adapter jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex object to make sure everything is up to date ds.reindexObject() # Job submission state notifier IStatusMessage(self.request).add(message, type=status) return (status, message)
def submitsdm(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('occurrence_data', None): self.record_error('Bad Request', 400, 'Missing parameter occurrence_data', {'parameter': 'occurrence_data'}) else: # FIXME: should properly support source / id # for now only bccvl source is supported props['species_occurrence_dataset'] = params[ 'occurrence_data']['id'] # FIXME: should properly support source/id for onw only bccvl source is # supported props['species_absence_dataset'] = params.get( 'absence_data', {}).get('id', None) props['scale_down'] = params.get('scale_down', False) if not params.get('environmental_data', None): self.record_error('Bad Request', 400, 'Missing parameter environmental_data', {'parameter': 'environmental_data'}) else: props['environmental_datasets'] = params['environmental_data'] if params.get('modelling_region', ''): props['modelling_region'] = NamedBlobFile( data=json.dumps(params['modelling_region'])) else: props['modelling_region'] = None if not params.get('algorithms', None): self.record_error('Bad Request', 400, 'Missing parameter algorithms', {'parameter': 'algorithms'}) else: portal = ploneapi.portal.get() props['functions'] = {} # FIXME: make sure we get the default values from our func object for algo, algo_params in params['algorithms'].items(): if algo_params is None: algo_params = {} toolkit = portal[defaults.FUNCTIONS_FOLDER_ID][algo] toolkit_model = loadString(toolkit.schema) toolkit_schema = toolkit_model.schema func_props = {} for field_name in toolkit_schema.names(): field = toolkit_schema.get(field_name) value = algo_params.get(field_name, field.missing_value) if value == field.missing_value: func_props[field_name] = field.default else: func_props[field_name] = value props['functions'][IUUID(toolkit)] = func_props if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent("org.bccvl.content.sdmexperiment", **props) experiment = addContentToContainer(context, experiment) # TODO: check if props and algo params have been applied properly experiment.parameters = dict(props['functions']) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def import_ala_data(self): if self.request.get("REQUEST_METHOD", "GET").upper() != "POST": self.record_error("Request must be POST", 400) raise BadRequest("Request must be POST") context = None # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse( "/".join((defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, "ala")) ) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { "id": member.getUserName(), "email": member.getProperty("email"), "fullname": member.getProperty("fullname"), } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission("org.bccvl.AddDataset", context): raise Unauthorized("User not allowed in this context") params = self.request.form.get("data") if not params: raise BadRequest("At least on of traits or environ has to be set") if params is None: self.record_error("Bad Request", 400, "Missing parameter data", {"parameter": "data"}) if not params: self.record_error("Bad Request", 400, "Empty parameter data", {"parameter": "data"}) # TODO: should validate objects inside as well? (or use json schema # validation?) # all good so far # pull dataset from aekos # TODO: get better name here title = params[0].get("name", "ALA import") # determine dataset type # 1. test if it is a multi species import species = set() for query in params: biocache_url = "{}/occurrences/search".format(query["url"]) query = { "q": query["query"], "pageSize": 0, "limit": 2, "facets": "species_guid", "fq": "species_guid:*", # skip results without species guid } res = requests.get(biocache_url, params=query) res = res.json() # FIXME: do we need to treat sandbox downloads differently? if res["facetResults"]: # do we have some results at all? for guid in res["facetResults"][0]["fieldResult"]: species.add(guid["label"]) if len(species) > 1: portal_type = "org.bccvl.content.multispeciesdataset" else: portal_type = "org.bccvl.content.dataset" swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = "org.bccvl.content.remotedataset" # create content ds = createContentInContainer(context, portal_type, title=title) ds.dataSource = "ala" ds.description = u" ".join([title, u" imported from ALA"]) ds.import_params = params md = IBCCVLMetadata(ds) if IMultiSpeciesDataset.providedBy(ds): md["genre"] = "DataGenreSpeciesCollection" else: # species dataset md["genre"] = "DataGenreSpeciesOccurrence" md["categories"] = ["occurrence"] # TODO: populate this correctly as well md["species"] = [{"scientificName": "qid", "taxonID": "qid"}] # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add("New Dataset created", type="info") # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, "portal_url").getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader("Location", nexturl) # FIXME: should return a nice json representation of success or error return {"status": status, "message": message, "jobid": IJobTracker(ds).get_job().id}
def import_trait_data(self): source = self.request.form.get("source", None) species = self.request.form.get("species", None) traits = self.request.form.get("traits", None) environ = self.request.form.get("environ", None) context = None # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse( "/".join((defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, "aekos")) ) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { "id": member.getUserName(), "email": member.getProperty("email"), "fullname": member.getProperty("fullname"), } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission("org.bccvl.AddDataset", context): raise Unauthorized("User not allowed in this context") # check parameters if not source or source not in ("aekos"): raise BadRequest("source parameter bust be 'aekos'") if not species or not isinstance(species, (basestring, list)): raise BadRequest("Missing or invalid species parameter") elif isinstance(species, basestring): species = [species] if not traits and not environ: raise BadRequest("At least on of traits or environ has to be set") if not traits: traits = [] elif isinstance(traits, basestring): traits = [traits] if not environ: environ = [] elif isinstance(environ, basestring): environ = [environ] # all good so far # pull dataset from aekos title = " ".join(species) # determine dataset type portal_type = "org.bccvl.content.dataset" swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = "org.bccvl.content.remotedataset" # create content ds = createContentInContainer(context, portal_type, title=title) ds.dataSource = source ds.description = u" ".join( [title, ",".join(traits), ",".join(environ), u" imported from {}".format(source.upper())] ) md = IBCCVLMetadata(ds) md["genre"] = "DataGenreTraits" md["categories"] = ["traits"] md["species"] = [{"scientificName": spec, "taxonID": spec} for spec in species] md["traits"] = traits md["environ"] = environ # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add("New Dataset created", type="info") # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, "portal_url").getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader("Location", nexturl) # FIXME: should return a nice json representation of success or error return {"status": status, "message": message, "jobid": IJobTracker(ds).get_job().id}
def pullOccurrenceFromALA(self, lsid, taxon, dataSrc='ala', common=None): # TODO: check permisions? # 1. create new dataset with taxon, lsid and common name set portal = getToolByName(self.context, 'portal_url').getPortalObject() if dataSrc == 'ala': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['ala'] elif dataSrc == 'gbif': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['gbif'] elif dataSrc == 'aekos': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['aekos'] elif dataSrc == 'obis': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['obis'] else: raise BadRequest('Invalid data source {0}'.format(dataSrc)) title = [taxon] if common: title.append(u"({})".format(common)) # determine dataset type # 1. test if it is a multi species import species = set() if dataSrc == 'ala': params = [{ 'query': 'lsid:{}'.format(lsid), 'url': 'http://biocache.ala.org.au/ws' }] for query in params: biocache_url = '{}/occurrences/search'.format(query['url']) query = { 'q': query['query'], 'pageSize': 0, 'limit': 2, 'facets': 'species_guid', 'fq': 'species_guid:*' # skip results without species guid } res = requests.get(biocache_url, params=query) res = res.json() if res.get('facetResults'): # do we have some results at all? for guid in res['facetResults'][0]['fieldResult']: species.add(guid['label']) elif dataSrc == 'gbif': genusChildren_url = 'https://api.gbif.org/v1/species/{}/children?offset=0&limit=40'.format(lsid) res = requests.get(genusChildren_url) res = res.json() if res.get('results'): for sp in res.get('results'): if sp.get('speciesKey'): species.add(sp['speciesKey']) elif dataSrc == 'obis': genusChildren_url = 'https://backend.iobis.org/children/{}'.format(lsid) res = requests.get(genusChildren_url) res = res.json() for sp in res: if sp.get('rank_name', '') != 'Species': continue if sp.get('valid_id'): species.add(sp['valid_id']) if len(species) > 1: portal_type = 'org.bccvl.content.multispeciesdataset' else: swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' else: portal_type = 'org.bccvl.content.dataset' # TODO: make sure we get a better content id that dataset-x title = u' '.join(title) ds = createContent(portal_type, title=title) ds.dataSource = dataSrc # Either ALA or GBIF as source # TODO: add number of occurences to description ds.description = u' '.join( (title, u'imported from', unicode(dataSrc.upper())) ) ds = addContentToContainer(dscontainer, ds) md = IBCCVLMetadata(ds) # TODO: provenance ... import url? # FIXME: verify input parameters before adding to graph if IMultiSpeciesDataset.providedBy(ds): md['genre'] = 'DataGenreSpeciesCollection' md['categories'] = ['multispecies'] else: md['genre'] = 'DataGenreSpeciesOccurrence' md['categories'] = ['occurrence'] md['species'] = { 'scientificName': taxon, 'taxonID': lsid, } if common: md['species']['vernacularName'] = common IStatusMessage(self.request).add('New Dataset created', type='info') # 2. create and push alaimport job for dataset # TODO: make this named adapter jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex object to make sure everything is up to date ds.reindexObject() # Job submission state notifier IStatusMessage(self.request).add(message, type=status) return (status, message)
def import_ala_data(self): if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') context = None # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse( "/".join((defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, 'ala'))) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { 'id': member.getUserName(), 'email': member.getProperty('email'), 'fullname': member.getProperty('fullname') } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission('org.bccvl.AddDataset', context): raise Unauthorized("User not allowed in this context") params = self.request.form.get('data') if not params: raise BadRequest("At least on of traits or environ has to be set") if params is None: self.record_error('Bad Request', 400, 'Missing parameter data', {'parameter': 'data'}) if not params: self.record_error('Bad Request', 400, 'Empty parameter data', {'parameter': 'data'}) # TODO: should validate objects inside as well? (or use json schema # validation?) # all good so far # pull dataset from aekos # TODO: get better name here title = params[0].get('name', 'ALA import') # determine dataset type # 1. test if it is a multi species import species = set() for query in params: biocache_url = '{}/occurrences/search'.format(query['url']) query = { 'q': query['query'], 'pageSize': 0, 'limit': 2, 'facets': 'species_guid', 'fq': 'species_guid:*' # skip results without species guid } res = requests.get(biocache_url, params=query) res = res.json() # FIXME: do we need to treat sandbox downloads differently? if res.get('facetResults'): # do we have some results at all? for guid in res['facetResults'][0]['fieldResult']: species.add(guid['label']) # Check of it is trait-data isTrait = any([p.get('trait', 0) for p in params]) if not isTrait and len(species) > 1: portal_type = 'org.bccvl.content.multispeciesdataset' else: portal_type = 'org.bccvl.content.dataset' swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' # create content ds = createContent(portal_type, title=title) ds.dataSource = 'ala' ds.description = u' '.join([title, u' imported from ALA']) ds.import_params = params ds = addContentToContainer(context, ds) md = IBCCVLMetadata(ds) if IMultiSpeciesDataset.providedBy(ds): md['genre'] = 'DataGenreSpeciesCollection' md['categories'] = ['multispecies'] else: if isTrait: # Trait dataset md['genre'] = 'DataGenreTraits' md['categories'] = ['traits'] else: # species dataset md['genre'] = 'DataGenreSpeciesOccurrence' md['categories'] = ['occurrence'] # TODO: populate this correctly as well md['species'] = [{ 'scientificName': 'qid', 'taxonID': 'qid'}] # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add('New Dataset created', type='info') # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, 'portal_url').getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader('Location', nexturl) # FIXME: should return a nice json representation of success or error return { 'status': status, 'message': message, 'jobid': IJobTracker(ds).get_job().id }
def import_trait_data(self): if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') source = self.request.form.get('source', None) species = self.request.form.get('species', None) traits = self.request.form.get('traits', None) environ = self.request.form.get('environ', None) dataurl = self.request.form.get('url', None) context = None if not source or source not in ('aekos', 'zoatrack'): raise BadRequest("source parameter must be 'aekos' or 'zoatrack'") # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse("/".join( (defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, str(source)))) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { 'id': member.getUserName(), 'email': member.getProperty('email'), 'fullname': member.getProperty('fullname') } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission('org.bccvl.AddDataset', context): raise Unauthorized("User not allowed in this context") # check parameters if not species or not isinstance(species, (basestring, list)): raise BadRequest("Missing or invalid species parameter") elif isinstance(species, basestring): species = [species] # for zoatrack, url needs to be set if source == 'zoatrack' and not dataurl: raise BadRequest("url has to be set") # for aekos, at least a trait or environment variable must be specified. if source == 'aekos' and not traits and not environ: raise BadRequest( "At least a trait or environent variable has to be set") if not traits: traits = [] elif isinstance(traits, basestring): traits = [traits] if not environ: environ = [] elif isinstance(environ, basestring): environ = [environ] # all good so far # pull dataset from aekos title = ' '.join(species) # determine dataset type portal_type = 'org.bccvl.content.dataset' swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' # create content ds = createContent(portal_type, title=title) ds.dataSource = source ds.description = u' '.join([ title, ','.join(traits), ','.join(environ), u' imported from {}'.format(source.upper()) ]) ds = addContentToContainer(context, ds) md = IBCCVLMetadata(ds) md['genre'] = 'DataGenreTraits' md['categories'] = ['traits'] md['species'] = [{ 'scientificName': spec, 'taxonID': spec } for spec in species] md['traits'] = traits md['environ'] = environ md['dataurl'] = dataurl # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add('New Dataset created', type='info') # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, 'portal_url').getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader('Location', nexturl) # FIXME: should return a nice json representation of success or error return { 'status': status, 'message': message, 'jobid': IJobTracker(ds).get_job().id }
def submittraits(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if params.get('species_list', None): props['species_list'] = params['species_list'] else: self.record_error('Bad Request', 400, 'Missing parameter speciesList', {'parameter': 'speciesList'}) if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('traits_data', None): self.record_error('Bad Request', 400, 'Missing parameter traits_data', {'parameter': 'traits_data'}) else: # FIXME: should properly support source / id # for now only bccvl source is supported props['species_traits_dataset'] = params[ 'traits_data']['id'] props['species_traits_dataset_params'] = {} for col_name, col_val in params.get("columns", {}).items(): if col_val not in ('lat', 'lon', 'species', 'trait_con', 'trait_ord', 'trait_nom', 'env_var_con', 'env_var_cat', 'random_con', 'random_cat'): continue props['species_traits_dataset_params'][col_name] = col_val if not props['species_traits_dataset_params']: self.record_error('Bad Request', 400, 'Invalid values for columns', {'parameter': 'columns'}) # Check for species-level trait data i.e. species is not specified if 'species' not in props['species_traits_dataset_params'].values(): props['species_list'] = [] props['scale_down'] = params.get('scale_down', False) # env data is optional props['environmental_datasets'] = params.get('environmental_data', None) if not (props['environmental_datasets'] or 'env_var_con' not in props['species_traits_dataset_params'].values() or 'env_var_cat' not in props['species_traits_dataset_params'].values()): self.record_error('Bad Request', 400, 'No Environmental data selected', {'parameter': 'environmental_datasets'}) if params.get('modelling_region', ''): props['modelling_region'] = NamedBlobFile( data=json.dumps(params['modelling_region'])) else: props['modelling_region'] = None if not params.get('algorithms', None): self.record_error('Bad Request', 400, 'Missing parameter algorithms', {'parameter': 'algorithms'}) else: props['algorithms_species'] = {} props['algorithms_diff'] = {} funcs_env = getUtility( IVocabularyFactory, 'traits_functions_species_source')(context) funcs_species = getUtility( IVocabularyFactory, 'traits_functions_diff_source')(context) # FIXME: make sure we get the default values from our func object for algo_uuid, algo_params in params['algorithms'].items(): if algo_params is None: algo_params = {} toolkit = uuidToObject(algo_uuid) toolkit_model = loadString(toolkit.schema) toolkit_schema = toolkit_model.schema func_props = {} for field_name in toolkit_schema.names(): field = toolkit_schema.get(field_name) value = algo_params.get(field_name, field.missing_value) if value == field.missing_value: func_props[field_name] = field.default else: func_props[field_name] = value if algo_uuid in funcs_env: props['algorithms_species'][algo_uuid] = func_props elif algo_uuid in funcs_species: props['algorithms_diff'][algo_uuid] = func_props else: LOG.warn( 'Algorithm {} not in allowed list of functions'.format(toolkit.id)) if not (props['algorithms_species'] or props['algorithms_diff']): self.record_error('Bad Request', 400, 'Iinvalid algorithms selected', {'parameter': 'algorithms'}) if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent( "org.bccvl.content.speciestraitsexperiment", **props) experiment = addContentToContainer(context, experiment) experiment.parameters = dict(props['algorithms_species']) experiment.parameters.update(dict(props['algorithms_diff'])) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def submitcc(self): # TODO: catch UNAuthorized correctly and return json error if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') # make sure we have the right context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # experiments location context = self.context.restrictedTraverse( defaults.EXPERIMENTS_FOLDER_ID) else: # custom context.... let's use in context = self.context # parse request body params = self.request.form # validate input # TODO: should validate type as well..... (e.g. string has to be # string) # TODO: validate dataset and layer id's existence if possible props = {} if not params.get('title', None): self.record_error('Bad Request', 400, 'Missing parameter title', {'parameter': 'title'}) else: props['title'] = params['title'] props['description'] = params.get('description', '') if not params.get('species_distribution_models', None): self.record_error('Bad Request', 400, 'Missing parameter species_distribution_models', {'parameter': 'species_distribution_models'}) else: props['species_distribution_models'] = params[ 'species_distribution_models'] if not params.get('future_climate_datasets', None): self.record_error('Bad Request', 400, 'Missing parameter future_climate_datasets', {'parameter': 'future_climate_datasets'}) else: props['future_climate_datasets'] = params[ 'future_climate_datasets'] if params.get('projection_region', ''): props['projection_region'] = NamedBlobFile( data=json.dumps(params['projection_region'])) else: props['projection_region'] = None if self.errors: raise BadRequest("Validation Failed") # create experiment with data as form would do # TODO: make sure self.context is 'experiments' folder? from plone.dexterity.utils import createContent, addContentToContainer experiment = createContent("org.bccvl.content.projectionexperiment", **props) experiment = addContentToContainer(context, experiment) # FIXME: need to get resolution from somewhere IBCCVLMetadata(experiment)['resolution'] = 'Resolution30m' # submit newly created experiment # TODO: handle background job submit .... at this stage we wouldn't # know the model run job ids # TODO: handle submit errors and other errors that may happen above? # generic exceptions could behandled in returnwrapper retval = { 'experiment': { 'url': experiment.absolute_url(), 'uuid': IUUID(experiment) }, 'jobs': [], } jt = IExperimentJobTracker(experiment) msgtype, msg = jt.start_job(self.request) if msgtype is not None: retval['message'] = { 'type': msgtype, 'message': msg } for result in experiment.values(): jt = IJobTracker(result) retval['jobs'].append(jt.get_job().id) return retval
def pullOccurrenceFromALA(self, lsid, taxon, dataSrc='ala', common=None): # TODO: check permisions? # 1. create new dataset with taxon, lsid and common name set portal = getToolByName(self.context, 'portal_url').getPortalObject() if dataSrc == 'ala': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['ala'] elif dataSrc == 'gbif': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['gbif'] elif dataSrc == 'aekos': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['aekos'] elif dataSrc == 'obis': dscontainer = portal[defaults.DATASETS_FOLDER_ID][ defaults.DATASETS_SPECIES_FOLDER_ID]['obis'] else: raise BadRequest('Invalid data source {0}'.format(dataSrc)) title = [taxon] if common: title.append(u"({})".format(common)) # determine dataset type # 1. test if it is a multi species import species = set() if dataSrc == 'ala': params = [{ 'query': 'lsid:{}'.format(lsid), 'url': 'https://biocache-ws.ala.org.au/ws' }] for query in params: biocache_url = '{}/occurrences/search'.format(query['url']) query = { 'q': query['query'], 'pageSize': 0, 'limit': 2, 'facets': 'species_guid', 'fq': 'species_guid:*' # skip results without species guid } res = requests.get(biocache_url, params=query) res = res.json() if res.get('facetResults'): # do we have some results at all? for guid in res['facetResults'][0]['fieldResult']: species.add(guid['label']) elif dataSrc == 'gbif': genusChildren_url = 'https://api.gbif.org/v1/species/{}/children?offset=0&limit=40'.format( lsid) res = requests.get(genusChildren_url) res = res.json() if res.get('results'): for sp in res.get('results'): if sp.get('speciesKey'): species.add(sp['speciesKey']) elif dataSrc == 'obis': genusChildren_url = 'https://backend.iobis.org/children/{}'.format( lsid) res = requests.get(genusChildren_url) res = res.json() for sp in res: if sp.get('rank_name', '') != 'Species': continue if sp.get('valid_id'): species.add(sp['valid_id']) if len(species) > 1: portal_type = 'org.bccvl.content.multispeciesdataset' else: swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' else: portal_type = 'org.bccvl.content.dataset' # TODO: make sure we get a better content id that dataset-x title = u' '.join(title) ds = createContent(portal_type, title=title) ds.dataSource = dataSrc # Either ALA or GBIF as source # TODO: add number of occurences to description ds.description = u' '.join( (title, u'imported from', unicode(dataSrc.upper()))) ds = addContentToContainer(dscontainer, ds) md = IBCCVLMetadata(ds) # TODO: provenance ... import url? # FIXME: verify input parameters before adding to graph if IMultiSpeciesDataset.providedBy(ds): md['genre'] = 'DataGenreSpeciesCollection' md['categories'] = ['multispecies'] else: md['genre'] = 'DataGenreSpeciesOccurrence' md['categories'] = ['occurrence'] md['species'] = { 'scientificName': taxon, 'taxonID': lsid, } if common: md['species']['vernacularName'] = common IStatusMessage(self.request).add('New Dataset created', type='info') # 2. create and push alaimport job for dataset # TODO: make this named adapter jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex object to make sure everything is up to date ds.reindexObject() # Job submission state notifier IStatusMessage(self.request).add(message, type=status) return (status, message)
def import_ala_data(self): if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') context = None # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse("/".join( (defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, 'ala'))) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { 'id': member.getUserName(), 'email': member.getProperty('email'), 'fullname': member.getProperty('fullname') } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission('org.bccvl.AddDataset', context): raise Unauthorized("User not allowed in this context") params = self.request.form.get('data') if not params: raise BadRequest("At least on of traits or environ has to be set") if params is None: self.record_error('Bad Request', 400, 'Missing parameter data', {'parameter': 'data'}) if not params: self.record_error('Bad Request', 400, 'Empty parameter data', {'parameter': 'data'}) # TODO: should validate objects inside as well? (or use json schema # validation?) # all good so far # pull dataset from aekos # TODO: get better name here title = params[0].get('name', 'ALA import') # determine dataset type # 1. test if it is a multi species import species = set() for query in params: biocache_url = '{}/occurrences/search'.format(query['url']) query = { 'q': query['query'], 'pageSize': 0, 'limit': 2, 'facets': 'species_guid', 'fq': 'species_guid:*' # skip results without species guid } res = requests.get(biocache_url, params=query) res = res.json() # FIXME: do we need to treat sandbox downloads differently? if res.get('facetResults'): # do we have some results at all? for guid in res['facetResults'][0]['fieldResult']: species.add(guid['label']) if len(species) > 1: portal_type = 'org.bccvl.content.multispeciesdataset' else: portal_type = 'org.bccvl.content.dataset' swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' # create content ds = createContent(portal_type, title=title) ds.dataSource = 'ala' ds.description = u' '.join([title, u' imported from ALA']) ds.import_params = params ds = addContentToContainer(context, ds) md = IBCCVLMetadata(ds) if IMultiSpeciesDataset.providedBy(ds): md['genre'] = 'DataGenreSpeciesCollection' md['categories'] = ['multispecies'] else: # species dataset md['genre'] = 'DataGenreSpeciesOccurrence' md['categories'] = ['occurrence'] # TODO: populate this correctly as well md['species'] = [{'scientificName': 'qid', 'taxonID': 'qid'}] # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add('New Dataset created', type='info') # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, 'portal_url').getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader('Location', nexturl) # FIXME: should return a nice json representation of success or error return { 'status': status, 'message': message, 'jobid': IJobTracker(ds).get_job().id }
def import_trait_data(self): if self.request.get('REQUEST_METHOD', 'GET').upper() != 'POST': self.record_error('Request must be POST', 400) raise BadRequest('Request must be POST') source = self.request.form.get('source', None) species = self.request.form.get('species', None) traits = self.request.form.get('traits', None) environ = self.request.form.get('environ', None) dataurl = self.request.form.get('url', None) context = None if not source or source not in ('aekos', 'zoatrack'): raise BadRequest("source parameter must be 'aekos' or 'zoatrack'") # get import context if ISiteRoot.providedBy(self.context): # we have been called at site root... let's traverse to default # import location context = self.context.restrictedTraverse( "/".join((defaults.DATASETS_FOLDER_ID, defaults.DATASETS_SPECIES_FOLDER_ID, str(source)))) else: # custom context.... let's use in context = self.context # do user check first member = ploneapi.user.get_current() if member.getId(): user = { 'id': member.getUserName(), 'email': member.getProperty('email'), 'fullname': member.getProperty('fullname') } else: # We need at least a valid user raise Unauthorized("Invalid user") # check permission if not checkPermission('org.bccvl.AddDataset', context): raise Unauthorized("User not allowed in this context") # check parameters if not species or not isinstance(species, (basestring, list)): raise BadRequest("Missing or invalid species parameter") elif isinstance(species, basestring): species = [species] # for zoatrack, url needs to be set if source == 'zoatrack' and not dataurl: raise BadRequest("url has to be set") # for aekos, at least a trait or environment variable must be specified. if source == 'aekos' and not traits and not environ: raise BadRequest("At least a trait or environent variable has to be set") if not traits: traits = [] elif isinstance(traits, basestring): traits = [traits] if not environ: environ = [] elif isinstance(environ, basestring): environ = [environ] # all good so far # pull dataset from aekos title = ' '.join(species) # determine dataset type portal_type = 'org.bccvl.content.dataset' swiftsettings = getUtility(IRegistry).forInterface(ISwiftSettings) if swiftsettings.storage_url: portal_type = 'org.bccvl.content.remotedataset' # create content ds = createContent(portal_type, title=title) ds.dataSource = source ds.description = u' '.join([ title, ','.join(traits), ','.join(environ), u' imported from {}'.format(source.upper())]) ds = addContentToContainer(context, ds) md = IBCCVLMetadata(ds) md['genre'] = 'DataGenreTraits' md['categories'] = ['traits'] md['species'] = [{ 'scientificName': spec, 'taxonID': spec} for spec in species] md['traits'] = traits md['environ'] = environ md['dataurl'] = dataurl # FIXME: IStatusMessage should not be in API call from Products.statusmessages.interfaces import IStatusMessage IStatusMessage(self.request).add('New Dataset created', type='info') # start import job jt = IExperimentJobTracker(ds) status, message = jt.start_job() # reindex ojebct to make sure everything is up to date ds.reindexObject() # FIXME: IStatutsMessage should not be in API call IStatusMessage(self.request).add(message, type=status) # FIXME: API should not return a redirect # 201: new resource created ... location may point to resource from Products.CMFCore.utils import getToolByName portal = getToolByName(self.context, 'portal_url').getPortalObject() nexturl = portal[defaults.DATASETS_FOLDER_ID].absolute_url() self.request.response.setStatus(201) self.request.response.setHeader('Location', nexturl) # FIXME: should return a nice json representation of success or error return { 'status': status, 'message': message, 'jobid': IJobTracker(ds).get_job().id }