def get_post_agg(mconf): """ For a metric specified as `postagg` returns the kind of post aggregation for pydruid. """ if mconf.get('type') == 'javascript': return JavascriptPostAggregator(name=mconf.get('name', ''), field_names=mconf.get( 'fieldNames', []), function=mconf.get('function', '')) elif mconf.get('type') == 'quantile': return Quantile( mconf.get('name', ''), mconf.get('probability', ''), ) elif mconf.get('type') == 'quantiles': return Quantiles( mconf.get('name', ''), mconf.get('probabilities', ''), ) elif mconf.get('type') == 'fieldAccess': return Field(mconf.get('name')) elif mconf.get('type') == 'constant': return Const( mconf.get('value'), output_name=mconf.get('name', ''), ) elif mconf.get('type') == 'hyperUniqueCardinality': return HyperUniqueCardinality(mconf.get('name'), ) elif mconf.get('type') == 'arithmetic': return Postaggregator(mconf.get('fn', '/'), mconf.get('fields', []), mconf.get('name', '')) else: return CustomPostAggregator(mconf.get('name', ''), mconf)
def create_from_calculation(cls, calculation, new_id=None, original_id=None, query_filter=None): # Only need to apply a suffix if the calculation is being modified by a # query filter. suffix = '' if query_filter: suffix = new_id if new_id.startswith(original_id): suffix = new_id.replace(original_id, '') if not suffix.startswith('_'): suffix = '_%s' % suffix output = cls( aggregations=calculation.aggregations, post_aggregations=calculation.post_aggregations, query_filter=query_filter, suffix=suffix, ) # The ID will exist in the aggregations already if the previous # calculation contained only one aggregation and the suffix will convert # the original id into the new id exactly. The ID will exist in the post # aggregations only if a suffix has been applied and the value was # previously a post aggregation. This is because the modification of the # underlying aggregations / post aggregations already handled the ID # remap. if new_id not in output.aggregations and new_id not in output.post_aggregations: output.add_post_aggregation(new_id, Field(output.create_id(original_id))) return output
def _build_from_node(node): node_type = type(node).__name__ # Variables encountered are considered to be druid data fields if node_type == 'Name': return Field(node.id) if node_type == 'Num': return Const(node.n) if node_type == 'BinOp': node_op_type = type(node.op).__name__ if node_op_type not in ARITHMETIC_NODE_MAP: raise UnsupportedNodeError(node_op_type) fn = ARITHMETIC_NODE_MAP[node_op_type] # Recurse to get fields being operated on left = _build_from_node(node.left).post_aggregator right = _build_from_node(node.right).post_aggregator fields = [left, right] # Collapse equivalent calculations into a single calculation if left.get('fn') == fn: fields = left['fields'] + [right] elif right.get('fn') == fn: fields = [left] + right['fields'] return Postaggregator(fn, fields, 'bin_op') raise UnsupportedNodeError(node_type)
def _metrics_and_post_aggs(metrics, metrics_dict): all_metrics = [] post_aggs = {} def recursive_get_fields(_conf): _type = _conf.get('type') _field = _conf.get('field') _fields = _conf.get('fields') field_names = [] if _type in [ 'fieldAccess', 'hyperUniqueCardinality', 'quantile', 'quantiles' ]: field_names.append(_conf.get('fieldName', '')) if _field: field_names += recursive_get_fields(_field) if _fields: for _f in _fields: field_names += recursive_get_fields(_f) return list(set(field_names)) for metric_name in metrics: metric = metrics_dict[metric_name] if metric.metric_type != 'postagg': all_metrics.append(metric_name) else: mconf = metric.json_obj all_metrics += recursive_get_fields(mconf) all_metrics += mconf.get('fieldNames', []) if mconf.get('type') == 'javascript': post_aggs[metric_name] = JavascriptPostAggregator( name=mconf.get('name', ''), field_names=mconf.get('fieldNames', []), function=mconf.get('function', '')) elif mconf.get('type') == 'quantile': post_aggs[metric_name] = Quantile( mconf.get('name', ''), mconf.get('probability', ''), ) elif mconf.get('type') == 'quantiles': post_aggs[metric_name] = Quantiles( mconf.get('name', ''), mconf.get('probabilities', ''), ) elif mconf.get('type') == 'fieldAccess': post_aggs[metric_name] = Field(mconf.get('name')) elif mconf.get('type') == 'constant': post_aggs[metric_name] = Const( mconf.get('value'), output_name=mconf.get('name', ''), ) elif mconf.get('type') == 'hyperUniqueCardinality': post_aggs[metric_name] = HyperUniqueCardinality( mconf.get('name'), ) elif mconf.get('type') == 'arithmetic': post_aggs[metric_name] = Postaggregator( mconf.get('fn', '/'), mconf.get('fields', []), mconf.get('name', '')) else: post_aggs[metric_name] = CustomPostAggregator( mconf.get('name', ''), mconf) return all_metrics, post_aggs
def run_query( # noqa / druid self, groupby, metrics, granularity, from_dttm, to_dttm, filter=None, # noqa is_timeseries=True, timeseries_limit=None, timeseries_limit_metric=None, row_limit=None, inner_from_dttm=None, inner_to_dttm=None, orderby=None, extras=None, # noqa select=None, # noqa columns=None, phase=2, client=None, form_data=None): """Runs a query against Druid and returns a dataframe. """ # TODO refactor into using a TBD Query object client = client or self.cluster.get_pydruid_client() if not is_timeseries: granularity = 'all' inner_from_dttm = inner_from_dttm or from_dttm inner_to_dttm = inner_to_dttm or to_dttm # add tzinfo to native datetime with config from_dttm = from_dttm.replace(tzinfo=DRUID_TZ) to_dttm = to_dttm.replace(tzinfo=DRUID_TZ) timezone = from_dttm.tzname() query_str = "" metrics_dict = {m.metric_name: m for m in self.metrics} all_metrics = [] post_aggs = {} columns_dict = {c.column_name: c for c in self.columns} def recursive_get_fields(_conf): _fields = _conf.get('fields', []) field_names = [] for _f in _fields: _type = _f.get('type') if _type in ['fieldAccess', 'hyperUniqueCardinality']: field_names.append(_f.get('fieldName')) elif _type == 'arithmetic': field_names += recursive_get_fields(_f) return list(set(field_names)) for metric_name in metrics: metric = metrics_dict[metric_name] if metric.metric_type != 'postagg': all_metrics.append(metric_name) else: mconf = metric.json_obj all_metrics += recursive_get_fields(mconf) all_metrics += mconf.get('fieldNames', []) if mconf.get('type') == 'javascript': post_aggs[metric_name] = JavascriptPostAggregator( name=mconf.get('name', ''), field_names=mconf.get('fieldNames', []), function=mconf.get('function', '')) elif mconf.get('type') == 'quantile': post_aggs[metric_name] = Quantile( mconf.get('name', ''), mconf.get('probability', ''), ) elif mconf.get('type') == 'quantiles': post_aggs[metric_name] = Quantiles( mconf.get('name', ''), mconf.get('probabilities', ''), ) elif mconf.get('type') == 'fieldAccess': post_aggs[metric_name] = Field(mconf.get('name')) elif mconf.get('type') == 'constant': post_aggs[metric_name] = Const(mconf.get('value'), output_name=mconf.get( 'name', '')) elif mconf.get('type') == 'hyperUniqueCardinality': post_aggs[metric_name] = HyperUniqueCardinality( mconf.get('name')) else: post_aggs[metric_name] = Postaggregator( mconf.get('fn', "/"), mconf.get('fields', []), mconf.get('name', '')) aggregations = OrderedDict() for m in self.metrics: if m.metric_name in all_metrics: aggregations[m.metric_name] = m.json_obj rejected_metrics = [ m.metric_name for m in self.metrics if m.is_restricted and m.metric_name in aggregations.keys() and not sm.has_access('metric_access', m.perm) ] if rejected_metrics: raise MetricPermException("Access to the metrics denied: " + ', '.join(rejected_metrics)) # the dimensions list with dimensionSpecs expanded dimensions = [] groupby = [gb for gb in groupby if gb in columns_dict] for column_name in groupby: col = columns_dict.get(column_name) dim_spec = col.dimension_spec if dim_spec: dimensions.append(dim_spec) else: dimensions.append(column_name) qry = dict( datasource=self.datasource_name, dimensions=dimensions, aggregations=aggregations, granularity=DruidDatasource.granularity( granularity, timezone=timezone, origin=extras.get('druid_time_origin'), ), post_aggregations=post_aggs, intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(), ) filters = self.get_filters(filter) if filters: qry['filter'] = filters having_filters = self.get_having_filters(extras.get('having_druid')) if having_filters: qry['having'] = having_filters orig_filters = filters if len(groupby) == 0 and not having_filters: del qry['dimensions'] client.timeseries(**qry) if not having_filters and len(groupby) == 1: qry['threshold'] = timeseries_limit or 1000 if row_limit and granularity == 'all': qry['threshold'] = row_limit qry['dimension'] = list(qry.get('dimensions'))[0] del qry['dimensions'] qry['metric'] = list(qry['aggregations'].keys())[0] client.topn(**qry) elif len(groupby) > 1 or having_filters: # If grouping on multiple fields or using a having filter # we have to force a groupby query if timeseries_limit and is_timeseries: order_by = metrics[0] if metrics else self.metrics[0] if timeseries_limit_metric: order_by = timeseries_limit_metric # Limit on the number of timeseries, doing a two-phases query pre_qry = deepcopy(qry) pre_qry['granularity'] = "all" pre_qry['limit_spec'] = { "type": "default", "limit": timeseries_limit, 'intervals': (inner_from_dttm.isoformat() + '/' + inner_to_dttm.isoformat()), "columns": [{ "dimension": order_by, "direction": "descending", }], } client.groupby(**pre_qry) query_str += "// Two phase query\n// Phase 1\n" query_str += json.dumps( client.query_builder.last_query.query_dict, indent=2) query_str += "\n" if phase == 1: return query_str query_str += ( "//\nPhase 2 (built based on phase one's results)\n") df = client.export_pandas() if df is not None and not df.empty: dims = qry['dimensions'] filters = [] for unused, row in df.iterrows(): fields = [] for dim in dims: f = Dimension(dim) == row[dim] fields.append(f) if len(fields) > 1: filt = Filter(type="and", fields=fields) filters.append(filt) elif fields: filters.append(fields[0]) if filters: ff = Filter(type="or", fields=filters) if not orig_filters: qry['filter'] = ff else: qry['filter'] = Filter(type="and", fields=[ff, orig_filters]) qry['limit_spec'] = None if row_limit: qry['limit_spec'] = { "type": "default", "limit": row_limit, "columns": [{ "dimension": (metrics[0] if metrics else self.metrics[0]), "direction": "descending", }], } client.groupby(**qry) query_str += json.dumps(client.query_builder.last_query.query_dict, indent=2) return query_str
def get_druid_data(dimensions=None, filter_list=[], filter_type="and", order_by=["target_area_name"], datasource=settings.DRUID_SPRAYDAY_DATASOURCE): """ Runs a query against Druid, returns data with metrics Inputs: dimensions => list of dimensions to group by filter_list => list of list of things to filter with e.g. filter_list=[['target_area_id', operator.ne, 1], ['sprayable', operator.eq, "true"], ['dimension', operator, "value"]]) filter_type => type of Druid filter to perform, order_by => field(s) to order the data by """ query = PyDruid(get_druid_broker_url(), 'druid/v2') params = dict( datasource=datasource, granularity='all', intervals=settings.DRUID_INTERVAL, aggregations={ 'num_not_sprayable': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('sprayable') == 'false'] ), aggregators.longsum('count') ), 'num_not_sprayed': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('sprayable') == 'true', filters.Dimension('sprayed') == settings.MSPRAY_WAS_NOT_SPRAYED_VALUE] ), aggregators.longsum('count') ), 'num_sprayed': aggregators.filtered( filters.Dimension('sprayed') == settings.MSPRAY_WAS_SPRAYED_VALUE, aggregators.longsum('count') ), 'num_new': aggregators.filtered( filters.Dimension('is_new') == 'true', aggregators.longsum('count') ), 'num_new_no_duplicates': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'false', filters.Dimension('is_new') == 'true'] ), aggregators.longsum('count') ), 'num_duplicate': aggregators.filtered( filters.Dimension('is_duplicate') == 'true', aggregators.longsum('count') ), 'num_sprayed_no_duplicates': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'false', filters.Dimension('sprayed') == settings.MSPRAY_WAS_SPRAYED_VALUE] ), aggregators.longsum('count') ), 'num_not_sprayed_no_duplicates': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'false', filters.Dimension('sprayable') == 'true', filters.Dimension('sprayed') == settings.MSPRAY_WAS_NOT_SPRAYED_VALUE] ), aggregators.longsum('count') ), 'num_sprayed_duplicates': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'true', filters.Dimension('sprayable') == 'true', filters.Dimension('sprayed') == settings.MSPRAY_WAS_SPRAYED_VALUE] ), aggregators.longsum('count') ), 'num_not_sprayable_no_duplicates': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'false', filters.Dimension('sprayable') == 'false'] ), aggregators.longsum('count') ), 'num_refused': aggregators.filtered( filters.Filter( type='and', fields=[filters.Dimension('is_duplicate') == 'false', filters.Dimension('is_refused') == 'true', filters.Dimension('sprayed') == settings.MSPRAY_WAS_NOT_SPRAYED_VALUE] ), aggregators.longsum('count') ), }, post_aggregations={ 'num_found': Field('num_sprayed_no_duplicates') + Field('num_sprayed_duplicates') + Field('num_not_sprayed_no_duplicates') }, limit_spec={ "type": "default", "limit": 50000, "columns": order_by } ) if filter_list: fields = [] for this_filter in filter_list: compare_dim = filters.Dimension(this_filter[0]) comparison_operator = this_filter[1] # e.g. operator.eq compare_dim_value = this_filter[2] fields.append(comparison_operator(compare_dim, compare_dim_value)) params['filter'] = filters.Filter( type=filter_type, fields=fields ) if dimensions is None: params['dimensions'] = ['target_area_id', 'target_area_name', 'target_area_structures'] else: params['dimensions'] = dimensions try: request = query.groupby(**params) except OSError: return [] else: return request.result