def __init__(self, project_name, config_fields, project_root_name, es, myargs=None): self.myargs = myargs if myargs else dict() self.config = config_fields self.es = es self.project_root_name = project_root_name self.project_name = project_name self.cquery = ConjunctiveQueryProcessor(self.project_name, self.config, self.project_root_name, self.es, myargs=self.myargs) self.field = self.myargs.get('_group-by', "event_date") self.agg = self.myargs.get('_interval', "month") self.agg_field = self.myargs.get('_aggregation-field', None) if self.agg_field is not None and "." in self.agg_field: self.agg_field = self.convert_to_nested_field(self.agg_field) if self.field is not None and "." in self.field: self.field = self.convert_to_nested_field(self.field) self.percent_change = True if '_percent_change' in self.myargs else False self.impute_method = self.myargs.get('_impute_method', 'previous')
def process_ts_query(self): """ This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' """ valid_input = self.validate_input() if not valid_input: err_json = dict() err_json['message'] = "Please enter valid query params. Fields must exist for the given project. " \ "If not sure, please access http://mydigurl/projects/<project_name>/fields API for " \ "reference" return rest.bad_request(err_json) resp = self.cquery.process() if resp is None or resp[1] == 400: return resp try: resp = resp[0] if len(resp['hits']['hits']) > 0 and 'doc_id' in resp['hits'][ 'hits'][0]['_source'].keys(): docid = resp['hits']['hits'][0]['_source']['doc_id'] argmap = dict() argmap['measure/value'] = docid argmap['_group-by'] = 'event_date' argmap['_interval'] = self.agg self.myargs = argmap newquery = ConjunctiveQueryProcessor(self.project_name, self.config, self.project_root_name, self.es, myargs=self.myargs) resp = newquery.process() if resp[1] == 400: return resp else: resp = resp[0] logger.debug("Response for query is {}".format(resp)) isDateAggregation = True if "." not in self.field and self.config[ self.field]['type'] == "date" else False ts, dims = DigOutputProcessor(resp['aggregations'][self.field], self.agg_field, isDateAggregation).process() ts_obj = TimeSeries( ts, dict(), dims, percent_change=self.percent_change, impute_method=self.impute_method).to_dict() return rest.ok(ts_obj) else: return rest.not_found("Time series not found") except Exception as e: logger.exception( "Exception encountered while performing time series query") return rest.bad_request("Enter valid query")
def __init__(self, request, project_name, config_fields, project_root_name, es): self.request = request self.config = config_fields self.es = es self.project_root_name = project_root_name self.project_name = project_name self.cquery = ConjunctiveQueryProcessor(self.request, self.project_name, self.config, self.project_root_name, self.es) self.field = request.args.get('_group-by', "event_date") self.agg = request.args.get('_interval', "month")
def process_ts_query(self): ''' This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' ''' # valid_input = self.validate_input() # if not valid_input: # err_json = {} # err_json['message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference" # return rest.bad_request(err_json) resp = self.cquery.process()[0] try: if len(resp['hits']['hits']) > 0 and 'doc_id' in resp['hits'][ 'hits'][0]['_source'].keys(): docid = resp['hits']['hits'][0]['_source']['doc_id'] argmap = {} argmap['measure/value'] = docid argmap['_group-by'] = 'event_date' argmap['_interval'] = self.agg #meta = resp['hits']['hits'][0]['_source']['measure']['metadata'] self.request.args = argmap newquery = ConjunctiveQueryProcessor(self.request, self.project_name, self.config, self.project_root_name, self.es) resp = newquery.process()[0] print resp ts, dims = DigOutputProcessor( resp['aggregations'][self.field]).process() dimensions = [] dimensions.append("DATE") dimensions.append(dims) ts_obj = TimeSeries(ts, dict(), dimensions).to_dict() return rest.ok(ts_obj) else: return rest.not_found("Time series not found") except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) lines = ''.join(lines) print lines return rest.bad_request("Enter valid query for measure document")
def __init__(self, request, project_name, config_fields, project_root_name, es): self.request = request self.config = config_fields self.es = es self.project_root_name = project_root_name self.project_name = project_name self.cquery = ConjunctiveQueryProcessor(self.request, self.project_name, self.config, self.project_root_name, self.es) self.field = request.args.get('_group-by', "event_date") self.agg = request.args.get('_interval', "month") self.agg_field = request.args.get('_aggregation-field', None) if "." in self.agg_field: self.agg_field = self.convert_to_nested_field(self.agg_field) if "." in self.field: self.field = self.convert_to_nested_field(self.field)
class EventQueryProcessor(object): def __init__(self, project_name, config_fields, project_root_name, es, myargs=None): self.myargs = myargs if myargs else dict() self.config = config_fields self.es = es self.project_root_name = project_root_name self.project_name = project_name self.cquery = ConjunctiveQueryProcessor(self.project_name, self.config, self.project_root_name, self.es, myargs=self.myargs) self.field = self.myargs.get('_group-by', "event_date") self.agg = self.myargs.get('_interval', "month") self.agg_field = self.myargs.get('_aggregation-field', None) if self.agg_field is not None and "." in self.agg_field: self.agg_field = self.convert_to_nested_field(self.agg_field) if self.field is not None and "." in self.field: self.field = self.convert_to_nested_field(self.field) self.percent_change = True if '_percent_change' in self.myargs else False self.impute_method = self.myargs.get('_impute_method', 'previous') # def preprocess(self): # for arg in self.request.args: # arg = urllib.unquote(arg) # return def convert_to_nested_field(self, field): params = field.split('.') return params[0] + "__" + params[1] def process_ts_query(self): """ This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' """ valid_input = self.validate_input() if not valid_input: err_json = dict() err_json['message'] = "Please enter valid query params. Fields must exist for the given project. " \ "If not sure, please access http://mydigurl/projects/<project_name>/fields API for " \ "reference" return rest.bad_request(err_json) resp = self.cquery.process() if resp is None or resp[1] == 400: return resp try: resp = resp[0] if len(resp['hits']['hits']) > 0 and 'doc_id' in resp['hits'][ 'hits'][0]['_source'].keys(): docid = resp['hits']['hits'][0]['_source']['doc_id'] argmap = dict() argmap['measure/value'] = docid argmap['_group-by'] = 'event_date' argmap['_interval'] = self.agg self.myargs = argmap newquery = ConjunctiveQueryProcessor(self.project_name, self.config, self.project_root_name, self.es, myargs=self.myargs) resp = newquery.process() if resp[1] == 400: return resp else: resp = resp[0] logger.debug("Response for query is {}".format(resp)) isDateAggregation = True if "." not in self.field and self.config[ self.field]['type'] == "date" else False ts, dims = DigOutputProcessor(resp['aggregations'][self.field], self.agg_field, isDateAggregation).process() ts_obj = TimeSeries( ts, dict(), dims, percent_change=self.percent_change, impute_method=self.impute_method).to_dict() return rest.ok(ts_obj) else: return rest.not_found("Time series not found") except Exception as e: logger.exception( "Exception encountered while performing time series query") return rest.bad_request("Enter valid query") def process_event_query(self): """ This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' """ valid_input = self.validate_input() if not valid_input: err_json = dict() err_json['message'] = "Please enter valid query params. Fields must exist for the given project. " \ "If not sure, please access http://mydigurl/projects/<project_name>/fields " \ "API for reference" return rest.bad_request(err_json) try: resp = self.cquery.process() if resp[1] == 400: logger.warning( "Request generated 4xx response. Check request again") return resp else: resp = resp[0] if resp is not None and len( resp['aggregations'][self.field]['buckets']) > 0: isDateAggregation = True if "." not in self.field and self.config[ self.field]['type'] == "date" else False ts, dims = DigOutputProcessor(resp['aggregations'][self.field], self.agg_field, isDateAggregation).process() ts_obj = TimeSeries( ts, {}, dims, percent_change=self.percent_change, impute_method=self.impute_method).to_dict() return rest.ok(ts_obj) else: return rest.not_found("No Time series found for query") except Exception as e: logger.exception( "Exception encountered while performing Event query") return rest.internal_error("Internal Error occured") def validate_input(self): field = self.myargs.get('_group-by', None) if field is None: return False elif "." not in field and field not in self.config.keys(): return False return True
class EventQueryProcessor(object): def __init__(self, request, project_name, config_fields, project_root_name, es): self.request = request self.config = config_fields self.es = es self.project_root_name = project_root_name self.project_name = project_name self.cquery = ConjunctiveQueryProcessor(self.request, self.project_name, self.config, self.project_root_name, self.es) self.field = request.args.get('_group-by', "event_date") self.agg = request.args.get('_interval', "month") self.agg_field = request.args.get('_aggregation-field', None) if "." in self.agg_field: self.agg_field = self.convert_to_nested_field(self.agg_field) if "." in self.field: self.field = self.convert_to_nested_field(self.field) def convert_to_nested_field(self, field): params = field.split('.') return params[0] + "__" + params[1] def process_ts_query(self): ''' This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' ''' valid_input = self.validate_input() if not valid_input: err_json = {} err_json[ 'message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference" return rest.bad_request(err_json) resp = self.cquery.process()[0] try: if len(resp['hits']['hits']) > 0 and 'doc_id' in resp['hits'][ 'hits'][0]['_source'].keys(): docid = resp['hits']['hits'][0]['_source']['doc_id'] argmap = {} argmap['measure/value'] = docid argmap['_group-by'] = 'event_date' argmap['_interval'] = self.agg self.request.args = argmap newquery = ConjunctiveQueryProcessor(self.request, self.project_name, self.config, self.project_root_name, self.es) resp = newquery.process()[0] print resp ts, dims = DigOutputProcessor(resp['aggregations'][self.field], self.agg_field).process() ts_obj = TimeSeries(ts, dict(), dims).to_dict() return rest.ok(ts_obj) else: return rest.not_found("Time series not found") except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) lines = ''.join(lines) print lines return rest.bad_request("Enter valid query") def process_event_query(self): ''' This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines joined by a '\n' ''' valid_input = self.validate_input() if not valid_input: err_json = {} err_json[ 'message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference" return rest.bad_request(err_json) resp = self.cquery.process()[0] ts, dims = DigOutputProcessor(resp['aggregations'][self.field], self.agg_field).process() ts_obj = TimeSeries(ts, {}, dims).to_dict() return rest.ok(ts_obj) def validate_input(self): field = self.request.args.get('_group-by', None) if field is None: return False elif "." not in field and not self.config[field]['type'] == "date": return False return True