def save_default_model(datasetname): dataset = get_dataset(datasetname) if not dataset.mapping or not dataset.source: return jsonify({"errors":["No mapping for this dataset"]}) if not dataset.dataorg: return jsonify({"errors":['Has no dataorg']}) #get the OR instructions from dataset ORinstructions = dataset.source.getORInstructions() #get the OR instructions from dataset mapping = dataset.mapping dataorg = dataset.dataorg dataorg.ORTemplate = {"data": ORinstructions} dataorg.mappingTemplate = mapping db.session.commit() return jsonify({"success":True})
def field(datasetname): """ get the column names and any existing info for them - add check for if source name does not exist """ dataset = get_dataset(datasetname) if dataset.mapping: #we have a model. Get the model info modeler = dataset.mapping['mapping'] refineproj = dataset.source.get_or_create_ORProject() columns = refineproj.refineproj.columns return jsonify({ "columns": columns, "modeler": modeler }, headers={'Cache-Control': 'no-cache'}) else: refineproj = dataset.source.get_or_create_ORProject() headers = {'Cache-Control': 'no-cache'} basemodeler = DEFAULT_SOURCE_MAPPING return jsonify( { "columns": refineproj.refineproj.columns, 'modeler': basemodeler }, headers=headers)
def field_polling_post(datasetname, columnkey): """ post to check to verify that the column is good """ #print request.get_json().get('columnval', None) ORcolumn = request.get_json().get('columnval', None) if not ORcolumn: return jsonify({"errors":["could not find the column name"]}) dataset = get_dataset(datasetname) if not require.dataset.update(dataset): return jsonify({"errors":["Permission denied"]}) try: columnsettings = api_form_data() #use this later if async run is necessary #runop = Run(columnsettings['columnval'], dataset, source) #db.session.add(runop) #db.session.commit() #check_column.apply_async(args=[source.id, columnkey, columnsettings['columnval'], runop.id], countdown=1) resultval = check_column(dataset.source.id, columnkey, columnsettings['columnval']) if len(resultval['errors']) == 0: return jsonify({"success":True}) else: return jsonify(resultval) except Exception, e: print "here is my error", e return jsonify({"errors":['Unknown Error has occurred']})
def field_polling_post(datasetname, columnkey): """ post to check to verify that the column is good """ #print request.get_json().get('columnval', None) ORcolumn = request.get_json().get('columnval', None) if not ORcolumn: return jsonify({"errors": ["could not find the column name"]}) dataset = get_dataset(datasetname) if not require.dataset.update(dataset): return jsonify({"errors": ["Permission denied"]}) try: columnsettings = api_form_data() #use this later if async run is necessary #runop = Run(columnsettings['columnval'], dataset, source) #db.session.add(runop) #db.session.commit() #check_column.apply_async(args=[source.id, columnkey, columnsettings['columnval'], runop.id], countdown=1) resultval = check_column(dataset.source.id, columnkey, columnsettings['columnval']) if len(resultval['errors']) == 0: return jsonify({"success": True}) else: return jsonify(resultval) except Exception, e: print "here is my error", e return jsonify({"errors": ['Unknown Error has occurred']})
def ORoperations(datasetname): try: dataset = get_dataset(datasetname) ORinstructions = dataset.source.getORInstructions() return jsonify(ORinstructions, headers= {'Cache-Control' : 'no-cache'}) except Exception, e: return jsonify({"error":"Could not fetch the ORinstructions" + str(e)})
def view(name): """ Get the dataset info to populate a form """ dataset = get_dataset(name) outputdict = dataset.detailed_dict() return jsonify(outputdict, headers={'Cache-Control': 'no-cache'})
def view(name): """ Get the dataset info to populate a form """ dataset = get_dataset(name) outputdict = dataset.detailed_dict() return jsonify(outputdict, headers= {'Cache-Control' : 'no-cache'})
def update_model(datasetname): #we just got everything now let's save it sourcemeta = request.get_json().get("meta", None) sourcemodeler = request.get_json().get("modeler", None) #validate that we have everything here r = {"mapping":sourcemodeler} #let's handle the compounds for item in r['mapping'].values(): if item['type'] in ("compound", "geometry"): for attitem in item['attributes'].values(): if attitem['column'] == 'countryid': pass attitem['column'] = item['column'] #if not hasattr(r['mapping'], 'theid'): r['mapping']['theid'] = { "default_value": "", "description": "Unique ID", "datatype": "string", "key": True, "label": "UniqueID", "column": "uniqueid", "type": "attribute", "form": { "label": "Unique Identifier" } } r['mapping']['geom_time_id'] = { "default_value": "", "description": "Geometry Time ID", "datatype": "integer", "label": "Geometry Time ID", "column": "geom_time_id", "type": "geom_time_id", "form": { "label": "Geometry-Time ID" } } dataset = get_dataset(datasetname) dataset.mapping = r dataset.ORoperations = {'data': dataset.source.getORInstructions()} dataset.source.addData(r) db.session.commit() load_source(dataset.source.id) cache.clear() #add async request to load data return jsonify({"success":True})
def ORoperations(datasetname): try: dataset = get_dataset(datasetname) ORinstructions = dataset.source.getORInstructions() return jsonify(ORinstructions, headers={'Cache-Control': 'no-cache'}) except Exception, e: return jsonify( {"error": "Could not fetch the ORinstructions" + str(e)})
def get_run(dataset, source, id): dataset = get_dataset(dataset) source = obj_or_404(Source.by_id(source)) if source.dataset != dataset: raise BadRequest("There was no source") run = obj_or_404(Run.by_id(id)) if run.source != source: raise BadRequest("There is no run %s" % str(id)) return dataset, source, run
def get_run(dataset, source, id): dataset = get_dataset(dataset) require.dataset.update(dataset) source = obj_or_404(Source.by_id(source)) if source.dataset != dataset: raise BadRequest("There was no source") run = obj_or_404(Run.by_id(id)) if run.source != source: raise BadRequest("There is no run '" + str(id) + '") return dataset, source, run
def update_model(datasetname): #we just got everything now let's save it sourcemeta = request.get_json().get("meta", None) sourcemodeler = request.get_json().get("modeler", None) #validate that we have everything here r = {"mapping": sourcemodeler} #let's handle the compounds for item in r['mapping'].values(): if item['type'] in ("compound", "geometry"): for attitem in item['attributes'].values(): if attitem['column'] == 'countryid': pass attitem['column'] = item['column'] #if not hasattr(r['mapping'], 'theid'): r['mapping']['theid'] = { "default_value": "", "description": "Unique ID", "datatype": "string", "key": True, "label": "UniqueID", "column": "uniqueid", "type": "attribute", "form": { "label": "Unique Identifier" } } r['mapping']['geom_time_id'] = { "default_value": "", "description": "Geometry Time ID", "datatype": "integer", "label": "Geometry Time ID", "column": "geom_time_id", "type": "geom_time_id", "form": { "label": "Geometry-Time ID" } } dataset = get_dataset(datasetname) dataset.mapping = r dataset.ORoperations = {'data': dataset.source.getORInstructions()} dataset.source.addData(r) db.session.commit() load_source(dataset.source.id) cache.clear() #add async request to load data return jsonify({"success": True})
def field_polling_check(datasetname, columnname): """ GET to check if the run is complete """ dataset = get_dataset(datasetname) if dataset.data: #we have a model. Get the model info return jsonify({"error":"not yet implemented"}) else: refineproj = dataset.source.get_or_create_ORProject() # this is awkward. the class should be extended return jsonify(refineproj.refineproj.columns)
def delete(datasetname): try: dataset = get_dataset(datasetname) db.session.delete(dataset.source) db.session.commit() clear_index_cache() #drop solr index #solr.drop_index(source.name) return jsonify(True) except Exception, e: return jsonify({"errors": [e]})
def field_polling_check(datasetname, columnname): """ GET to check if the run is complete """ dataset = get_dataset(datasetname) if dataset.data: #we have a model. Get the model info return jsonify({"error": "not yet implemented"}) else: refineproj = dataset.source.get_or_create_ORProject() # this is awkward. the class should be extended return jsonify(refineproj.refineproj.columns)
def delete(datasetname): try: dataset = get_dataset(datasetname) require.dataset.update(dataset) db.session.delete(dataset.source) db.session.commit() clear_index_cache() #drop solr index #solr.drop_index(source.name) return jsonify(True) except Exception, e: return jsonify({"errors":[e]})
def model(datasetname): #if not sourcename then we are saving the defaults for dataset dataset = get_dataset(datasetname) if not dataset.source: #then create one dataset_source = Source.by_source_name(dataset.name) if not dataset_source: dataset_source = Source(name=dataset.name, dataset=dataset) db.session.add(dataset_source) else: dataset_source.dataset = dataset db.session.commit() #figure out what they need over there? return jsonify(dataset.source)
def update(name): """ Update a dataset with a json object and name from the dataset form """ try: dataset = get_dataset(name) schema = dataset_schema(ValidationState(dataset)) data = schema.deserialize(api_form_data()) dataset.update(data) db.session.commit() #clear_index_cache() return jsonify({"success": True}) except Exception, e: print e return jsonify({"errors": ['Unknown Error has occurred']})
def update(name): """ Update a dataset with a json object and name from the dataset form """ try: dataset = get_dataset(name) require.dataset.update(dataset) schema = dataset_schema(ValidationState(dataset)) data = schema.deserialize(api_form_data()) dataset.update(data) db.session.commit() #clear_index_cache() return jsonify({"success":True}) except Exception, e: print e return jsonify({"errors":['Unknown Error has occurred']})
def slicer_model(): # options # get dataset info results = {"models": {}, "options": {}} cubesarg = request.args.get("cubes", []) cubes = cubesarg.split("|") for cube in cubes: dataset = get_dataset(cube) if dataset: results["models"][cube] = dataset.detailed_dict() results["options"] = GEO_MAPPING results["formats"] = FORMATOPTS resp = Response(response=to_json(results), status=200, mimetype="application/json") return resp
def apply_default_model(datasetname): dataset = get_dataset(datasetname) if not dataset.dataorg or not dataset: return jsonify({"errors": ["Invalid URL. Could not find dataorg"]}) dataorg = dataset.dataorg if not dataorg.ORTemplate or not dataorg.mappingTemplate: return jsonify({"errors": ["Dataorg has no template"]}) dataset.source.applyORInstructions(dataorg.ORTemplate) dataset.mapping = dataorg.mappingTemplate db.session.commit() return jsonify(dataset.source, headers={'Cache-Control': 'no-cache'})
def get_json_result(self): results = self._getcache() resultmodel = {"cells": results} tempmodel = {} for dataset in self.cubes: tempmodel[dataset] = get_dataset(dataset).detailed_dict() resultmodel['models'] = tempmodel resultmodel['attributes'] = self.drilldowntables if self.clusterparams['cluster']: resultmodel['cluster'] = self.get_clusters(resultmodel['cells']) resp = Response(response=to_json(resultmodel), status=200, \ mimetype="application/json") return resp
def update_model_createnew(datasetname): #refactor to include the update dataset = get_dataset(datasetname) #source will have name and URL sourceapi = api_form_data() if not sourceapi['name']: sourceapi['name'] = dataset.name #return jsonify({"errors":["You must enter a data name " + str(e)]}) #verify that name is unique and URL is real #model = {'source': source} schema = source_schema(ValidationState(sourceapi)) try: data = schema.deserialize(sourceapi) except Invalid, e: #print message in thefuture return jsonify({"errors": ["Invalid field " + str(e)]})
def slicer_model(): #options #get dataset info results = {"models": {}, "options": {}} cubesarg = request.args.get("cubes", []) cubes = cubesarg.split("|") for cube in cubes: dataset = get_dataset(cube) if dataset: results['models'][cube] = dataset.detailed_dict() results['options'] = GEO_MAPPING results['formats'] = FORMATOPTS resp = Response(response=to_json(results), status=200, \ mimetype="application/json") return resp
def field(datasetname): """ get the column names and any existing info for them - add check for if source name does not exist """ dataset = get_dataset(datasetname) if dataset.mapping: #we have a model. Get the model info modeler = dataset.mapping['mapping'] refineproj = dataset.source.get_or_create_ORProject() columns = refineproj.refineproj.columns return jsonify({"columns":columns, "modeler":modeler}, headers= {'Cache-Control' : 'no-cache'}) else: refineproj = dataset.source.get_or_create_ORProject() headers= {'Cache-Control' : 'no-cache'} basemodeler = DEFAULT_SOURCE_MAPPING return jsonify({"columns": refineproj.refineproj.columns, 'modeler':basemodeler}, headers=headers)
def update_model_createnew(datasetname): #refactor to include the update dataset = get_dataset(datasetname) #source will have name and URL sourceapi = api_form_data() if not sourceapi['name']: sourceapi['name'] = dataset.name #return jsonify({"errors":["You must enter a data name " + str(e)]}) #verify that name is unique and URL is real #model = {'source': source} schema = source_schema(ValidationState(sourceapi)) try: data = schema.deserialize(sourceapi) except Invalid, e: #print message in thefuture return jsonify({"errors":["Invalid field " + str(e)]})
def get_json_result(self): results = self._getcache() resultmodel = { "cells": results } tempmodel = {} for dataset in self.cubes: tempmodel[dataset] = get_dataset(dataset).detailed_dict() resultmodel['models'] = tempmodel resultmodel['attributes'] = self.drilldowntables if self.clusterparams['cluster']: resultmodel['cluster'] = self.get_clusters(resultmodel['cells']) resp = Response(response=to_json(resultmodel), status=200, \ mimetype="application/json") return resp
def apply_default_model(datasetname): dataset = get_dataset(datasetname) if not dataset.dataorg or not dataset: return jsonify({"errors":["Invalid URL. Could not find dataorg"]}) dataorg = dataset.dataorg if not dataorg.ORTemplate or not dataorg.mappingTemplate: return jsonify({"errors":["Dataorg has no template"]}) dataset.source.applyORInstructions(dataorg.ORTemplate) dataset.mapping = dataorg.mappingTemplate db.session.commit() return jsonify(dataset.source, headers= {'Cache-Control' : 'no-cache'})
def save_default_model(datasetname): dataset = get_dataset(datasetname) if not dataset.mapping or not dataset.source: return jsonify({"errors": ["No mapping for this dataset"]}) if not dataset.dataorg: return jsonify({"errors": ['Has no dataorg']}) #get the OR instructions from dataset ORinstructions = dataset.source.getORInstructions() #get the OR instructions from dataset mapping = dataset.mapping dataorg = dataset.dataorg dataorg.ORTemplate = {"data": ORinstructions} dataorg.mappingTemplate = mapping db.session.commit() return jsonify({"success": True})
def cube(self, name, locale=None, namespace=None, metaonly=None): if name == "geometry": return getGeomCube(self, metaonly) dataset = get_dataset(name) if name is None: raise NoSuchCubeError("Unknown dataset %s" % name, name) mappings = {} joins = [] fact_table = dataset.source.model.table.name aggregates = [MeasureAggregate('num_entries', label='Number of entries', function='count')] measures = [] # "wma": partial(_window_function_factory, window_function=weighted_moving_average, label='Weighted Moving Avg. of {measure}'), # "sma": partial(_window_function_factory, window_function=simple_moving_average, label='Simple Moving Avg. of {measure}'), # "sms": partial(_window_function_factory, window_function=simple_moving_sum, label='Simple Moving Sum of {measure}'), aggregation_funcs = ["sum", "min", "max", "avg"] for measure in dataset.source.model.measures: cubes_measure = Measure(measure.name, label=measure.label) measures.append(cubes_measure) for agg_func in aggregation_funcs: aggregate = MeasureAggregate(measure.name + "_" + agg_func, label=measure.label + agg_func, measure=measure.name, function=agg_func) aggregates.append(aggregate) dimensions = [] for dim in dataset.source.model.dimensions: meta = dim.to_cubes(mappings, joins) meta.update({'name': dim.name, 'label': dim.label}) dimensions.append(Dimension.from_metadata(meta)) cube_meta = {"name":dataset.name, "fact":fact_table, "aggregates":aggregates, "measures":measures, #change these when they get addeed to the model "label":dataset.label, "description":"non null description", "dimensions":dimensions, "store":self.store, "mappings":mappings, "joins":joins} if metaonly: return cube_meta else: return Cube(name=cube_meta['name'], fact=cube_meta['fact'], aggregates=cube_meta['aggregates'], measures=cube_meta['measures'], label=cube_meta['label'], description=cube_meta['description'], dimensions=cube_meta['dimensions'], store=cube_meta['store'], mappings=cube_meta['mappings'], joins=cube_meta['joins'])
def cube(self, name, locale=None, namespace=None, metaonly=None): if name == "geometry": return getGeomCube(self, metaonly) dataset = get_dataset(name) if name is None: raise NoSuchCubeError("Unknown dataset %s" % name, name) mappings = {} joins = [] fact_table = dataset.source.model.table.name aggregates = [ MeasureAggregate('num_entries', label='Number of entries', function='count') ] measures = [] # "wma": partial(_window_function_factory, window_function=weighted_moving_average, label='Weighted Moving Avg. of {measure}'), # "sma": partial(_window_function_factory, window_function=simple_moving_average, label='Simple Moving Avg. of {measure}'), # "sms": partial(_window_function_factory, window_function=simple_moving_sum, label='Simple Moving Sum of {measure}'), aggregation_funcs = ["sum", "min", "max", "avg"] for measure in dataset.source.model.measures: cubes_measure = Measure(measure.name, label=measure.label) measures.append(cubes_measure) for agg_func in aggregation_funcs: aggregate = MeasureAggregate(measure.name + "_" + agg_func, label=measure.label + agg_func, measure=measure.name, function=agg_func) aggregates.append(aggregate) dimensions = [] for dim in dataset.source.model.dimensions: meta = dim.to_cubes(mappings, joins) meta.update({'name': dim.name, 'label': dim.label}) dimensions.append(Dimension.from_metadata(meta)) cube_meta = { "name": dataset.name, "fact": fact_table, "aggregates": aggregates, "measures": measures, #change these when they get addeed to the model "label": dataset.label, "description": "non null description", "dimensions": dimensions, "store": self.store, "mappings": mappings, "joins": joins } if metaonly: return cube_meta else: return Cube(name=cube_meta['name'], fact=cube_meta['fact'], aggregates=cube_meta['aggregates'], measures=cube_meta['measures'], label=cube_meta['label'], description=cube_meta['description'], dimensions=cube_meta['dimensions'], store=cube_meta['store'], mappings=cube_meta['mappings'], joins=cube_meta['joins'])