def compute(): """Create PFA for kNN.""" # Read intermediate inputs from jobs logging.info("Fetching intermediate data...") inputs = io_helper.fetch_data() indep_vars = inputs["data"]["independent"] # Extract hyperparameters from ENV variables k = parameters.get_param('n_clusters', int, DEFAULT_N_CLUSTERS) # featurization featurizer = _create_featurizer(indep_vars) # convert variables into dataframe X = io_helper.fetch_dataframe(variables=indep_vars) X = utils.remove_nulls(X, errors='ignore') X = featurizer.transform(X) estimator = KMeans(n_clusters=k) estimator.fit(X) # Generate PFA for kmeans types = [(var['name'], var['type']['name']) for var in indep_vars] pfa = sklearn_to_pfa(estimator, types, featurizer.generate_pretty_pfa()) # Add centroids as metadata pfa['metadata'] = { 'centroids': json.dumps(estimator.cluster_centers_.tolist()) } # Save or update job_result logging.info('Saving PFA to job_results table') io_helper.save_results(json.dumps(pfa), shapes.Shapes.PFA) logging.info("DONE")
def set_coordination(df): for i, row in df.iterrows(): if i % 10 == 0: print i struc = Structure.from_dict(row['structure']) # using own edited Voronoi algorithm try: specie_meancoord = get_avg_cns(VoronoiCoordFinder_edited(struc).get_cns()) except Exception as e: print e continue df.set_value(i, 'VoronoiEd_cn', json.dumps(specie_meancoord)) df.set_value(i, 'Voronoi_cation_avgcn', get_cation_weighted_avg(specie_meancoord, struc)) # using own Effective coordination algorithm try: specie_meaneffcoord = get_avg_cns(EffectiveCoordFinder(struc).get_cns(radius=10.0)) except Exception as e: print e continue df.set_value(i, 'eff_cn', json.dumps(specie_meaneffcoord)) df.set_value(i, 'eff_cation_avgcn', get_cation_weighted_avg(specie_meaneffcoord, struc)) # using Tina's O'Keeffe coordination algorithm if row['metadata']['_structure']['is_ordered']: try: okeeffe_coord = okeeffe_get_avg_cn(struc) except Exception as e: print e continue df.set_value(i, 'okeeffe_cn', json.dumps(okeeffe_coord)) df.set_value(i, 'okeeffe_cn_avg', get_cation_weighted_avg(okeeffe_coord, struc)) return df
def on_get(self, req: Request, resp: Response): result = self._method_manager.count_methods() if isinstance(result, Exception): resp.body = json.dumps(dict(error=str(result))) resp.status = falcon.HTTP_500 else: resp.body = json.dumps(dict(count=result)) resp.status = falcon.HTTP_201
def test_tz_range_is_utc(self): from pandas.io.json import dumps exp = '["2013-01-01T05:00:00.000Z","2013-01-02T05:00:00.000Z"]' dfexp = ('{"DT":{' '"0":"2013-01-01T05:00:00.000Z",' '"1":"2013-01-02T05:00:00.000Z"}}') tz_range = pd.date_range('2013-01-01 05:00:00Z', periods=2) assert dumps(tz_range, iso_dates=True) == exp dti = pd.DatetimeIndex(tz_range) assert dumps(dti, iso_dates=True) == exp df = DataFrame({'DT': dti}) assert dumps(df, iso_dates=True) == dfexp tz_range = pd.date_range('2013-01-01 00:00:00', periods=2, tz='US/Eastern') assert dumps(tz_range, iso_dates=True) == exp dti = pd.DatetimeIndex(tz_range) assert dumps(dti, iso_dates=True) == exp df = DataFrame({'DT': dti}) assert dumps(df, iso_dates=True) == dfexp tz_range = pd.date_range('2013-01-01 00:00:00-0500', periods=2) assert dumps(tz_range, iso_dates=True) == exp dti = pd.DatetimeIndex(tz_range) assert dumps(dti, iso_dates=True) == exp df = DataFrame({'DT': dti}) assert dumps(df, iso_dates=True) == dfexp
def test_tz_range_is_utc(self): from pandas.io.json import dumps exp = '["2013-01-01T05:00:00.000Z","2013-01-02T05:00:00.000Z"]' dfexp = ('{"DT":{' '"0":"2013-01-01T05:00:00.000Z",' '"1":"2013-01-02T05:00:00.000Z"}}') tz_range = pd.date_range('2013-01-01 05:00:00Z', periods=2) self.assertEqual(exp, dumps(tz_range, iso_dates=True)) dti = pd.DatetimeIndex(tz_range) self.assertEqual(exp, dumps(dti, iso_dates=True)) df = DataFrame({'DT': dti}) self.assertEqual(dfexp, dumps(df, iso_dates=True)) tz_range = pd.date_range('2013-01-01 00:00:00', periods=2, tz='US/Eastern') self.assertEqual(exp, dumps(tz_range, iso_dates=True)) dti = pd.DatetimeIndex(tz_range) self.assertEqual(exp, dumps(dti, iso_dates=True)) df = DataFrame({'DT': dti}) self.assertEqual(dfexp, dumps(df, iso_dates=True)) tz_range = pd.date_range('2013-01-01 00:00:00-0500', periods=2) self.assertEqual(exp, dumps(tz_range, iso_dates=True)) dti = pd.DatetimeIndex(tz_range) self.assertEqual(exp, dumps(dti, iso_dates=True)) df = DataFrame({'DT': dti}) self.assertEqual(dfexp, dumps(df, iso_dates=True))
def _estimator_metadata(estimator, X, y, featurizer): """Serialize estimator and add score and other metadata.""" meta = { 'estimator': serialize_sklearn_estimator(estimator), } if len(X) and hasattr(estimator, 'score'): meta['score'] = json.dumps(estimator.score(X, y)) if hasattr(estimator, 'coef_'): meta['coef_'] = json.dumps(estimator.coef_.tolist()) if hasattr(estimator, 'intercept_'): meta['intercept_'] = json.dumps(estimator.intercept_.tolist()) if hasattr(estimator, 'feature_importances_') and hasattr(featurizer, 'columns'): meta['feature_importances_'] = json.dumps(dict(zip(featurizer.columns, estimator.feature_importances_))) return meta
def on_get(self, req: Request, resp: Response, model_id: UUID = None, git_commit_id: str = None): query = get_params_to_query(req, SORT_ATTRIBUTES, 'created', 'desc') if model_id and git_commit_id: snapshots = self._snapshot_manager.list_snapshots_filtered(model_id=model_id, git_commit_id=git_commit_id, query=query) else: snapshots = self._snapshot_manager.list_snapshots(query=query) if isinstance(snapshots, Exception): resp.body = json.dumps(dict(error=str(snapshots))) resp.status = falcon.HTTP_500 else: resp.body = json.dumps(dict(count=self._snapshot_manager.snapshot_count())) resp.status = falcon.HTTP_201
def get_json_data(self): df = self.get_df() chart_data = self.to_series(df) time_compare = self.form_data.get('time_compare') if time_compare: query_object = self.query_obj() delta = utils.parse_human_timedelta(time_compare) query_object['inner_from_dttm'] = query_object['from_dttm'] query_object['inner_to_dttm'] = query_object['to_dttm'] query_object['from_dttm'] -= delta query_object['to_dttm'] -= delta df2 = self.get_df(query_object) df2.index += delta chart_data += self.to_series(df2, classed='dashed', title_suffix="---") chart_data = sorted(chart_data, key=lambda x: x['key']) data = { 'chart_data': chart_data, 'query': self.results.query, 'duration': self.results.duration, } return dumps(data)
def on_delete(self, req: Request, resp: Response): result = self._method_manager.get_method(req.media['method_id']) if not type(result) == Method: resp.body = json.dumps(dict(error=str(result))) resp.status = falcon.HTTP_500 return resp result = self._method_manager.remove_method(result) if isinstance(result, Exception): resp.body = json.dumps(dict(error=str(result))) resp.status = falcon.HTTP_404 else: resp.body = result.json() resp.status = falcon.HTTP_202
def get_stock_em_hold_stock_df(): # 获取最近的交易日 year = time.strftime("%Y", time.localtime()) current_time = time.strftime("%Y-%m-%d", time.localtime()) ts.set_token('605894af201d877ec9108ba2fc05a95ea580f744ed640d2aa449623b') pro = ts.pro_api() data = pro.query('trade_cal', start_date=year + '0101', is_open='1') trade_days = data['cal_date'] today = time.strftime("%Y%m%d", time.localtime()) while today not in trade_days.values: today = str(int(today) - 1) stock_em_hold_stock_df = ak.stock_em_hsgt_stock_statistics( symbol="北向持股", start_date=today, end_date=today).head(50) result = [] for index, row in stock_em_hold_stock_df.iterrows(): result.append({ 'symbol': row["股票代码"], 'name': row["股票简称"], 'market_value': round(row["持股市值"] / 100000000), 'market_percent': row["持股数量占发行股百分比"] }) # return result resu = { 'code': 200, 'data': { 'current_time': current_time, 'result': result }, 'message': '成功' } return json.dumps(resu, ensure_ascii=False)
def get_em_add_stock_df(): date = request.values.get('date') current_time = time.strftime("%Y-%m-%d", time.localtime()) stock_em_add_stock_df = ak.stock_em_hsgt_hold_stock( market="北向", indicator=date).head(50) result = [] print(stock_em_add_stock_df) for index, row in stock_em_add_stock_df.iterrows(): result.append({ 'symbol': row["SCode"], 'name': row["SName"], 'plate': row["HYName"], 'incHoldMoney': round(row["ShareSZ_Chg_One"] / 100000000, 2), }) # # return result resu = { 'code': 200, 'data': { 'current_time': current_time, 'result': result }, 'message': '成功' } return json.dumps(resu, ensure_ascii=False)
def get_json_data(self): df = self.get_df() series = df.to_dict('series') chart_data = [] for name, ys in series.items(): if df[name].dtype.kind not in "biufc": continue df['timestamp'] = pd.to_datetime(df.index, utc=False) if isinstance(name, string_types): series_title = name elif len(self.metrics) > 1: series_title = ", ".join(name) else: series_title = ", ".join(name[1:]) d = { "key": series_title, "color": utils.color(series_title), "values": [{ 'x': ds, 'y': ys[i] } for i, ds in enumerate(df.timestamp)] } chart_data.append(d) return dumps({ 'chart_data': chart_data, 'query': self.results.query, 'duration': self.results.duration, })
def get_json_data(self): df = self.get_df() series = df.to_dict('series') chart_data = [] for name, ys in series.items(): if df[name].dtype.kind not in "biufc": continue df['timestamp'] = pd.to_datetime(df.index, utc=False) if isinstance(name, basestring): series_title = name elif len(self.metrics) > 1: series_title = ", ".join(name) else: series_title = ", ".join(name[1:]) d = { "key": series_title, "color": utils.color(series_title), "values": [ {'x': ds, 'y': ys[i]} for i, ds in enumerate(df.timestamp)] } chart_data.append(d) return dumps({ 'chart_data': chart_data, 'query': self.results.query, 'duration': self.results.duration, })
def get_group(self, key): new_query = 'select value t.grps from (%s) t where grp_id=%s;' % ( self.query[:-1], str(key)) results = json.dumps(af.AFrame.send_request(new_query)[0]) grp = json.read_json(results)['grp'] df = pd.DataFrame(grp.tolist()) return df
def compute(graph_type=None): """Perform both intermediate step and aggregation at once.""" # Read inputs logging.info("Fetching data...") inputs = io_helper.fetch_data() dep_var = inputs["data"]["dependent"][0] indep_vars = inputs["data"]["independent"] result = _compute_intermediate_result(inputs) corr, columns, crosstab = _aggregate_results([result]) graph_type = graph_type or parameters.get_parameter( 'graph', str, 'correlation_heatmap') if graph_type == 'correlation_heatmap': fig = _fig_corr_heatmap(corr, columns, crosstab) elif graph_type == 'pca': X = io_helper.fetch_dataframe([dep_var] + indep_vars) fig = _fig_pca(corr, columns, X) else: raise errors.UserError( 'MODEL_PARAM_graph only supports values `correlation_heatmap` and `pca`' ) logging.info("Results:\n{}".format(fig)) io_helper.save_results(json.dumps(fig), shapes.Shapes.PLOTLY) logging.info("DONE")
def compute(): """Create PFA for kNN.""" inputs = io_helper.fetch_data() dep_var = inputs["data"]["dependent"][0] indep_vars = inputs["data"]["independent"] params = parameters.fetch_parameters() if dep_var['type']['name'] in ('polynominal', 'binominal'): job_type = 'classification' else: job_type = 'regression' logging.info('Creating new estimator') estimator = _create_estimator(job_type, params) featurizer = _create_featurizer(indep_vars) # convert variables into dataframe X = io_helper.fetch_dataframe(variables=[dep_var] + indep_vars) X = utils.remove_nulls(X) y = X.pop(dep_var['name']) X = featurizer.transform(X) # Drop NaN values estimator.fit(X, y) # Create PFA from the estimator types = [(var['name'], var['type']['name']) for var in indep_vars] pfa = sklearn_to_pfa(estimator, types, featurizer.generate_pretty_pfa()) pfa['name'] = "kNN" # Save or update job_result logging.info('Saving PFA to job_results table...') pfa = json.dumps(pfa) io_helper.save_results(pfa, shapes.Shapes.PFA)
def javascript_cmd(self): js = dumps(self.chart) js = (js.replace('"{{TOOLTIP_FORMATTER}}"', self.tooltip_formatter).replace("\n", " ")) if self.stockchart: return "new Highcharts.StockChart(%s);" % js return "new Highcharts.Chart(%s);" % js
def scatter(): data_scatter_continuous = list() for var in variable_list: sub_data = dict() sub_data['key']= var sub_data['values'] = list() metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) scatter_plotx_list = metadata_variable.ix['plotx',0] scatter_ploty_list = metadata_variable.ix['ploty',0] # scatter_plotx_list.append(each) # scatter_ploty_list.append(None) for each in range(0, len(scatter_plotx_list)): sub_dict = dict() sub_dict['x'] = scatter_plotx_list[each] sub_dict['y'] = scatter_ploty_list[each] sub_data['values'].append(sub_dict) data_scatter_continuous.append(sub_data) data_json_scatter_continuous = json.dumps(data_scatter_continuous) return jsonify ({'data_json_scatter_continuous': data_json_scatter_continuous})
def get_json(self): """Handles caching around the json payload retrieval""" cache_key = self.cache_key payload = None if self.form_data.get('force') != 'true': payload = cache.get(cache_key) if payload: is_cached = True logging.info("Serving from cache") else: is_cached = False cache_timeout = self.cache_timeout payload = { 'data': self.get_data(), 'query': self.query, 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, 'csv_endpoint': self.csv_endpoint, 'standalone_endpoint': self.standalone_endpoint, 'cache_timeout': cache_timeout, } payload['cached_dttm'] = datetime.now().isoformat().split('.')[0] logging.info("Caching for the next {} seconds".format( cache_timeout)) cache.set(cache_key, payload, timeout=self.cache_timeout) payload['is_cached'] = is_cached return dumps(payload)
def get_json_data(self): from panoramix.data import countries df = self.get_df() cols = [self.form_data.get('entity')] metric = self.form_data.get('metric') secondary_metric = self.form_data.get('secondary_metric') if metric == secondary_metric: ndf = df[cols] ndf['m1'] = df[metric] ndf['m2'] = df[metric] else: cols += [metric, secondary_metric] ndf = df[cols] df = ndf df.columns = ['country', 'm1', 'm2'] d = df.to_dict(orient='records') for row in d: country = countries.get( self.form_data.get('country_fieldtype'), row['country']) if country: row['country'] = country['cca3'] row['latitude'] = country['lat'] row['longitude'] = country['lng'] row['name'] = country['name'] else: row['country'] = "XXX" return dumps(d)
def get_json_data(self): df = self.get_df() chart_data = self.to_series(df) time_compare = self.form_data.get('time_compare') if time_compare: query_object = self.query_obj() delta = utils.parse_human_timedelta(time_compare) query_object['inner_from_dttm'] = query_object['from_dttm'] query_object['inner_to_dttm'] = query_object['to_dttm'] query_object['from_dttm'] -= delta query_object['to_dttm'] -= delta df2 = self.get_df(query_object) df2.index += delta chart_data += self.to_series( df2, classed='dashed', title_suffix="---") chart_data = sorted(chart_data, key=lambda x: x['key']) data = { 'chart_data': chart_data, 'query': self.results.query, 'duration': self.results.duration, } return dumps(data)
def collect(self): results = af.AFrame.send_request(self._query) json_str = json.dumps(results) result = pd.DataFrame(data=json.read_json(json_str)) if '_uuid' in result.columns: result.drop('_uuid', axis=1, inplace=True) return result
def get_json_data(self): from panoramix.data import countries df = self.get_df() cols = [self.form_data.get('entity')] metric = self.form_data.get('metric') secondary_metric = self.form_data.get('secondary_metric') if metric == secondary_metric: ndf = df[cols] ndf['m1'] = df[metric] ndf['m2'] = df[metric] else: cols += [metric, secondary_metric] ndf = df[cols] df = ndf df.columns = ['country', 'm1', 'm2'] d = df.to_dict(orient='records') for row in d: country = countries.get(self.form_data.get('country_fieldtype'), row['country']) if country: row['country'] = country['cca3'] row['latitude'] = country['lat'] row['longitude'] = country['lng'] row['name'] = country['name'] else: row['country'] = "XXX" return dumps(d)
def aggregate_stats(job_ids, graph_type=None): """Get all partial statistics from all nodes and aggregate them. :input job_ids: list of job_ids with intermediate results """ # Read intermediate inputs from jobs logging.info("Fetching intermediate data...") results = io_helper.load_intermediate_json_results(map(str, job_ids)) corr, columns, crosstab = _aggregate_results(results) graph_type = graph_type or parameters.get_parameter( 'graph', str, 'correlation_heatmap') if graph_type == 'correlation_heatmap': fig = _fig_corr_heatmap(corr, columns, crosstab) elif graph_type == 'pca': # save PCA graphs, but leave out the one with PCA scores logging.warning( 'Sample scores graph is not yet implemented for distributed PCA.') fig = _fig_pca(corr, columns, X=None) else: raise errors.UserError( 'MODEL_PARAM_graph only supports values `correlation_heatmap` and `pca`' ) logging.info("Results:\n{}".format(fig)) io_helper.save_results(json.dumps(fig), shapes.Shapes.PLOTLY) logging.info("DONE")
def get_json(self): """Handles caching around the json payload retrieval""" cache_key = self.cache_key payload = None if self.form_data.get('force') != 'true': payload = cache.get(cache_key) if payload: is_cached = True logging.info("Serving from cache") else: is_cached = False cache_timeout = self.cache_timeout payload = { 'data': self.get_data(), 'query': self.query, 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, 'csv_endpoint': self.csv_endpoint, 'standalone_endpoint': self.standalone_endpoint, 'cache_timeout': cache_timeout, } payload['cached_dttm'] = datetime.now().isoformat().split('.')[0] logging.info( "Caching for the next {} seconds".format(cache_timeout)) cache.set(cache_key, payload, timeout=self.cache_timeout) payload['is_cached'] = is_cached return dumps(payload)
def get_json_data(self): from panoramix.data import countries df = self.get_df() cols = [self.form_data.get("entity")] metric = self.form_data.get("metric") secondary_metric = self.form_data.get("secondary_metric") if metric == secondary_metric: ndf = df[cols] ndf["m1"] = df[metric] ndf["m2"] = df[metric] else: cols += [metric, secondary_metric] ndf = df[cols] df = ndf df.columns = ["country", "m1", "m2"] d = df.to_dict(orient="records") for row in d: country = countries.get(self.form_data.get("country_fieldtype"), row["country"]) if country: row["country"] = country["cca3"] row["latitude"] = country["lat"] row["longitude"] = country["lng"] row["name"] = country["name"] else: row["country"] = "XXX" return dumps(d)
def scatter(): data_scatter_continuous = list() for var in variable_list: sub_data = dict() sub_data["key"] = var sub_data["values"] = list() metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) scatter_plotx_list = metadata_variable.ix["plotx", 0] scatter_ploty_list = metadata_variable.ix["ploty", 0] # scatter_plotx_list.append(each) # scatter_ploty_list.append(None) for each in range(0, len(scatter_plotx_list)): sub_dict = dict() sub_dict["x"] = scatter_plotx_list[each] sub_dict["y"] = scatter_ploty_list[each] sub_data["values"].append(sub_dict) data_scatter_continuous.append(sub_data) data_json_scatter_continuous = json.dumps(data_scatter_continuous) return jsonify({"data_json_scatter_continuous": data_json_scatter_continuous})
def response_as_json(data): json_str = json.dumps(data) response = HttpResponse( json_str, content_type="application/json", ) response["Access-Control-Allow-Origin"] = "*" return response
def head(self, num=5): new_query = self.query[:-1] + ' limit %d;' % num results = af.AFrame.send_request(new_query) json_str = json.dumps(results) result = pd.DataFrame(data=json.read_json(json_str)) if '_uuid' in result.columns: result.drop('_uuid', axis=1, inplace=True) return result
def get_data_attribute(self): content = { 'viz_name': self.viz_type, 'json_endpoint': self.json_endpoint, 'token': self.token, 'form_data': self.form_data, } return dumps(content)
def get_json_data(self): df = self.get_df() df = df.reset_index() df.columns = ["x", "y"] df["color"] = map(utils.color, df.x) return dumps( {"chart_data": df.to_dict(orient="records"), "query": self.results.query, "duration": self.results.duration} )
def error_histograms(dep_var, indep_vars): histograms = list() if len(dep_var) > 0: histograms.append(error_histogram(dep_var)) grouping_vars = [indep_var for indep_var in indep_vars if utils.is_nominal(indep_var)] for grouping_var in grouping_vars: histograms.append(error_histogram(dep_var, grouping_var)) return json.dumps(histograms)
def intermediate_stats(): """Calculate X*X^T, means and count for single node that will be later used to construct covariance matrix.""" # Read inputs logging.info("Fetching data...") inputs = io_helper.fetch_data() result = _compute_intermediate_result(inputs) io_helper.save_results(json.dumps(result), shapes.Shapes.JSON) logging.info("DONE")
def javascript_cmd(self): js = dumps(self.chart) js = ( js.replace('"{{TOOLTIP_FORMATTER}}"', self.tooltip_formatter) .replace("\n", " ") ) if self.stockchart: return "new Highcharts.StockChart(%s);" % js return "new Highcharts.Chart(%s);" %js
def get_json_data(self): df = self.get_df() series = defaultdict(list) for row in df.to_dict(orient='records'): series[row['group']].append(row) chart_data = [] for k, v in series.items(): chart_data.append({'key': k, 'values': v}) return dumps(chart_data)
def on_post(self, req: Request, resp: Response): snap = ModelSnapshot.from_json(req.media['ModelSnapshot']) result = self._snapshot_manager.create_snapshot(snap) if isinstance(result, Exception): resp.body = json.dumps(dict(error=str(result))) resp.status = falcon.HTTP_500 else: resp.body = result.json() resp.status = falcon.HTTP_201
def get_json_data(self): df = self.get_df() series = defaultdict(list) for row in df.to_dict(orient="records"): series[row["group"]].append(row) chart_data = [] for k, v in series.items(): chart_data.append({"key": k, "color": utils.color(k), "values": v}) return dumps({"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration})
def version(): """Returns version information Example: /version """ return json.dumps({"version": VERSION})
def stacked_area(): all_xvalues = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) plotx_list = metadata_variable.ix["plotx", 0] for each in plotx_list: all_xvalues.append(each) all_xvalues = list(set(all_xvalues)) data_stackedarea = list() for var in variable_list: sub_data = dict() sub_data["key"] = var metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) # save the 'plotx' and 'ploty' values individually as lists plotx_list = metadata_variable.ix["plotx", 0] ploty_list = metadata_variable.ix["ploty", 0] length = len(plotx_list) # start adding values sub_data["values"] = list() xval_list = list() for each in range(0, length): xval_list.append(plotx_list[each]) for each_x in all_xvalues: if each_x not in xval_list: plotx_list.append(each_x) ploty_list.append(None) xy = zip(plotx_list, ploty_list) xy.sort() length_xy = len(xy) for each in range(0, length_xy): sub_list = list() sub_list.append(xy[each][0]) sub_list.append(xy[each][1]) sub_data["values"].append(sub_list) data_stackedarea.append(sub_data) data_json_stackedarea = json.dumps(data_stackedarea) return jsonify({"data_json_stackedarea": data_json_stackedarea})
def stacked_area(): all_xvalues = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) plotx_list = metadata_variable.ix['plotx', 0] for each in plotx_list: all_xvalues.append(each) all_xvalues = list(set(all_xvalues)) data_stackedarea = list() for var in variable_list: sub_data = dict() sub_data['key'] = var metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) #save the 'plotx' and 'ploty' values individually as lists plotx_list = metadata_variable.ix['plotx', 0] ploty_list = metadata_variable.ix['ploty', 0] length = len(plotx_list) #start adding values sub_data['values'] = list() xval_list = list() for each in range(0, length): xval_list.append(plotx_list[each]) for each_x in all_xvalues: if each_x not in xval_list: plotx_list.append(each_x) ploty_list.append(None) xy = zip(plotx_list, ploty_list) xy.sort() length_xy = len(xy) for each in range(0, length_xy): sub_list = list() sub_list.append(xy[each][0]) sub_list.append(xy[each][1]) sub_data['values'].append(sub_list) data_stackedarea.append(sub_data) data_json_stackedarea = json.dumps(data_stackedarea) return jsonify({'data_json_stackedarea': data_json_stackedarea})
def on_get(self, req: Request, resp: Response): query = get_params_to_query(req, SORT_ATTRIBUTES, 'created', 'desc') methods = self._method_manager.list_methods(query=query) if isinstance(methods, Exception): resp.body = json.dumps(dict(error=str(methods))) resp.status = falcon.HTTP_500 else: resp.body = f"[{','.join(map(lambda x: x.json(), methods))}]" resp.status = falcon.HTTP_201
def on_post(self, req: Request, resp: Response): snapshot = self._snapshot_manager.get_snapshot(req.media['model_snapshot_id']) if not type(snapshot) == ModelSnapshot: resp.body = json.dumps(dict(error=str(snapshot))) resp.status = falcon.HTTP_500 return resp result = self._method_manager.create_method(name=req.media['name'], description=req.media['description'], snap=snapshot) if isinstance(result, Exception): resp.body = json.dumps(dict(error=str(result))) resp.status = falcon.HTTP_500 else: resp.body = result.json() resp.status = falcon.HTTP_201
def get_json_data(self): df = self.get_df() df = df.reset_index() df.columns = ['x', 'y'] df['color'] = map(utils.color, df.x) return dumps({ 'chart_data': df.to_dict(orient="records"), 'query': self.results.query, 'duration': self.results.duration, })
def continuous_multibar(): data_multibar = list() all_xvalues = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) plotx_list = metadata_variable.ix['plotx',0] for each in plotx_list: all_xvalues.append(each) all_xvalues = list(set(all_xvalues)) for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) #save the 'plotx' and 'ploty' values individually as lists plotx_list = metadata_variable.ix['plotx',0] ploty_list = metadata_variable.ix['ploty',0] for each in all_xvalues: if each not in plotx_list: plotx_list.append(each) ploty_list.append(None) xy = zip(plotx_list, ploty_list) xy.sort() length_xy = len(xy) sub_data = dict() sub_data['key'] = var sub_data['values'] = list() xval_list = list() for each in range(0, length_xy): sub_dict = dict() sub_dict['x'] = xy[each][0] sub_dict['y'] = xy[each][1] #xval_list.append(plotx_list[each]) sub_data['values'].append(sub_dict) data_multibar.append(sub_data) data_json_multibar = json.dumps(data_multibar) return jsonify({'data_json_multibar': data_json_multibar})
def get_json_data(self): df = self.get_df() series = defaultdict(list) for row in df.to_dict(orient='records'): series[row['group']].append(row) chart_data = [] for k, v in series.items(): chart_data.append({ 'key': k, 'values': v }) return dumps(chart_data)
def test_convert_dates_infer(self): # GH10747 from pandas.io.json import dumps infer_words = ['trade_time', 'date', 'datetime', 'sold_at', 'modified', 'timestamp', 'timestamps'] for infer_word in infer_words: data = [{'id': 1, infer_word: 1036713600000}, {'id': 2}] expected = DataFrame([[1, Timestamp('2002-11-08')], [2, pd.NaT]], columns=['id', infer_word]) result = read_json(dumps(data))[['id', infer_word]] assert_frame_equal(result, expected)
def get_json_data(self): form_data = self.form_data df = self.get_df() df = df.sort(columns=df.columns[0]) compare_lag = form_data.get("compare_lag", "") compare_lag = int(compare_lag) if compare_lag and compare_lag.isdigit() else 0 d = { 'data': df.values.tolist(), 'compare_lag': compare_lag, 'compare_suffix': form_data.get('compare_suffix', ''), } return dumps(d)
def test_default_handler_indirect(self): from pandas.io.json import dumps def default(obj): if isinstance(obj, complex): return [("mathjs", "Complex"), ("re", obj.real), ("im", obj.imag)] return str(obj) df_list = [ 9, DataFrame({"a": [1, "STR", complex(4, -5)], "b": [float("nan"), None, "N/A"]}, columns=["a", "b"]), ] expected = '[9,[[1,null],["STR",null],[[["mathjs","Complex"],' '["re",4.0],["im",-5.0]],"N\\/A"]]]' self.assertEqual(expected, dumps(df_list, default_handler=default, orient="values"))
def get_json_data(self): df = self.get_df() series = defaultdict(list) for row in df.to_dict(orient='records'): series[row['group']].append(row) chart_data = [] for k, v in series.items(): chart_data.append({ 'key': k, "color": utils.color(k), 'values': v }) return dumps({ 'chart_data': chart_data, 'query': self.results.query, 'duration': self.results.duration, })
def test_default_handler_indirect(self): from pandas.io.json import dumps def default(obj): if isinstance(obj, complex): return [('mathjs', 'Complex'), ('re', obj.real), ('im', obj.imag)] return str(obj) df_list = [9, DataFrame({'a': [1, 'STR', complex(4, -5)], 'b': [float('nan'), None, 'N/A']}, columns=['a', 'b'])] expected = ('[9,[[1,null],["STR",null],[[["mathjs","Complex"],' '["re",4.0],["im",-5.0]],"N\\/A"]]]') self.assertEqual(expected, dumps(df_list, default_handler=default, orient="values"))
def cont_line(): line_data = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) is_plottype = metadata_variable.index == "plottype" plottype_df = metadata_variable[is_plottype] line_plottype = str(plottype_df.ix[0,0]) all_xvalues = list() all_yvalues = list() line_plotx_list = metadata_variable.ix['plotx',0] line_ploty_list = metadata_variable.ix['ploty',0] length = len(line_plotx_list) for each in range(0, length): all_xvalues.append(line_plotx_list[each]) all_yvalues.append(line_ploty_list[each]) xy = zip(all_xvalues,all_yvalues) xy.sort() sub_data = dict() sub_data['key']= var sub_data['values'] = list() for each in range(0, len(xy)): sub_dict = dict() sub_dict['x'] = xy[each][0] sub_dict['y']= xy[each][1] sub_data['values'].append(sub_dict) line_data.append(sub_data) data_json_line = json.dumps(line_data) return jsonify({'data_json_line': data_json_line})
def pie_chart(): data_pie = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) is_plotvalues = metadata_variable.index == "plotvalues" plotvalues_df = metadata_variable[is_plotvalues] plotvalues_unicode = plotvalues_df.ix[0,0] plotvalues_dict = dict(plotvalues_unicode) for each in plotvalues_dict: sub_data = {'value':plotvalues_dict[each], 'label':each} data_pie.append(sub_data) data_json_pie = json.dumps(data_pie) return jsonify({'data_json_pie': data_json_pie})
def get_json_data(self): df = self.get_df() chart_data = self.to_series(df) time_compare = self.form_data.get("time_compare") if time_compare: query_object = self.query_obj() delta = utils.parse_human_timedelta(time_compare) query_object["inner_from_dttm"] = query_object["from_dttm"] query_object["inner_to_dttm"] = query_object["to_dttm"] query_object["from_dttm"] -= delta query_object["to_dttm"] -= delta df2 = self.get_df(query_object) df2.index += delta chart_data += self.to_series(df2, classed="dashed", title_suffix="---") chart_data = sorted(chart_data, key=lambda x: x["key"]) data = {"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration} return dumps(data)
def get_json_data(self): df = self.get_df() series = df.to_dict("series") chart_data = [] for name, ys in series.items(): if df[name].dtype.kind not in "biufc": continue df["timestamp"] = pd.to_datetime(df.index, utc=False) if isinstance(name, string_types): series_title = name elif len(self.metrics) > 1: series_title = ", ".join(name) else: series_title = ", ".join(name[1:]) d = { "key": series_title, "color": utils.color(series_title), "values": [{"x": ds, "y": ys[i]} for i, ds in enumerate(df.timestamp)], } chart_data.append(d) return dumps({"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration})
def get_json_data(self): df = self.get_df() series = df.to_dict('series') chart_data = [] for name, ys in series.items(): if df[name].dtype.kind not in "biufc": continue df['timestamp'] = pd.to_datetime(df.index, utc=False) if isinstance(name, string_types): series_title = name elif len(self.metrics) > 1: series_title = ", ".join(name) else: series_title = ", ".join(name[1:]) d = { "key": series_title, "values": [ {'x': ds, 'y': ys[i]} for i, ds in enumerate(df.timestamp)] } chart_data.append(d) return dumps(chart_data)
def get_json_data(self): df = self.get_df() series = df.to_dict('series') chart_data = [] for name, ys in series.items(): if df[name].dtype.kind not in "biufc": continue if isinstance(name, string_types): series_title = name elif len(self.metrics) > 1: series_title = ", ".join(name) else: l = [str(s) for s in name[1:]] series_title = ", ".join(l) d = { "key": series_title, "values": [ {'x': i, 'y': v} for i, v in ys.iteritems()] } chart_data.append(d) return dumps(chart_data)
def test_tz_is_utc(self): from pandas.io.json import dumps exp = '"2013-01-10T05:00:00.000Z"' ts = Timestamp('2013-01-10 05:00:00Z') assert dumps(ts, iso_dates=True) == exp dt = ts.to_pydatetime() assert dumps(dt, iso_dates=True) == exp ts = Timestamp('2013-01-10 00:00:00', tz='US/Eastern') assert dumps(ts, iso_dates=True) == exp dt = ts.to_pydatetime() assert dumps(dt, iso_dates=True) == exp ts = Timestamp('2013-01-10 00:00:00-0500') assert dumps(ts, iso_dates=True) == exp dt = ts.to_pydatetime() assert dumps(dt, iso_dates=True) == exp
def multi_bar(): warning_multibar = None all_xvalues = list() all_yvalues = list() list_of_domains_multibar = list() for var in variable_list: metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) is_plotvalues = metadata_variable.index == "plotvalues" plotvalues_df = metadata_variable[is_plotvalues] plotvalues_unicode = plotvalues_df.ix[0,0] plotvalues_dict = dict(plotvalues_unicode) individual_variable_domains= list() #used for the warning message for each in plotvalues_dict: all_xvalues.append(each) individual_variable_domains.append(each) all_yvalues.append(plotvalues_dict[each]) list_of_domains_multibar.append(individual_variable_domains) union_multibar = list() intersection_multibar = list_of_domains_multibar[0] for each in list_of_domains_multibar: union_multibar = list(set(union_multibar) | set(each)) intersection_multibar = list(set(intersection_multibar) & set(each)) difference_domains_multibar = 100.0-(100.0* len(intersection_multibar)/len(union_multibar)) #pdb.set_trace() all_xvalues = list(set(all_xvalues)) data_multibar = list() for var in variable_list: sub_data = dict() sub_data['values'] = list() metadata_variable_series = metadata_all[var] metadata_variable = pandas.DataFrame(metadata_variable_series) is_plotvalues = metadata_variable.index == "plotvalues" plotvalues_df = metadata_variable[is_plotvalues] plotvalues_unicode = plotvalues_df.ix[0,0] plotvalues_dict = dict(plotvalues_unicode) xvalues_list = list() yvalues_list = list() for each in plotvalues_dict: xvalues_list.append(each) yvalues_list.append(plotvalues_dict[each]) for each_x in all_xvalues: if each_x not in xvalues_list: xvalues_list.append(each_x) yvalues_list.append(None) xy = zip(xvalues_list, yvalues_list) xy.sort() length_xy = len(xy) for each in range(0,length_xy): sub_dict = {'x':xy[each][0],'y':xy[each][1],} sub_data['values'].append(sub_dict) data_multibar.append(sub_data) sub_data['key']=var if difference_domains_multibar != None and difference_domains_multibar > 30.: warning_multibar = '<div class="alert alert-warning alert-dismissible" role="alert"><button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">×</span></button><strong>Warning:</strong> chart created but may be misleading. <strong>'+ str(difference_domains_multibar) + "%</strong> of the selected variables's domains to do not intersect. Please be cautious before inferring any relationship between these variables.</div>" data_json_multibar = json.dumps(data_multibar) d = {'data_json_multibar': data_json_multibar, 'warning_multibar':warning_multibar} return jsonify(**d)