def compute():
"""Create PFA for kNN."""
# Read intermediate inputs from jobs
logging.info("Fetching intermediate data...")
inputs = io_helper.fetch_data()
indep_vars = inputs["data"]["independent"]
# Extract hyperparameters from ENV variables
k = parameters.get_param('n_clusters', int, DEFAULT_N_CLUSTERS)
# featurization
featurizer = _create_featurizer(indep_vars)
# convert variables into dataframe
X = io_helper.fetch_dataframe(variables=indep_vars)
X = utils.remove_nulls(X, errors='ignore')
X = featurizer.transform(X)
estimator = KMeans(n_clusters=k)
estimator.fit(X)
# Generate PFA for kmeans
types = [(var['name'], var['type']['name']) for var in indep_vars]
pfa = sklearn_to_pfa(estimator, types, featurizer.generate_pretty_pfa())
# Add centroids as metadata
pfa['metadata'] = {
'centroids': json.dumps(estimator.cluster_centers_.tolist())
}
# Save or update job_result
logging.info('Saving PFA to job_results table')
io_helper.save_results(json.dumps(pfa), shapes.Shapes.PFA)
logging.info("DONE")
def set_coordination(df):
for i, row in df.iterrows():
if i % 10 == 0:
print i
struc = Structure.from_dict(row['structure'])
# using own edited Voronoi algorithm
try:
specie_meancoord = get_avg_cns(VoronoiCoordFinder_edited(struc).get_cns())
except Exception as e:
print e
continue
df.set_value(i, 'VoronoiEd_cn', json.dumps(specie_meancoord))
df.set_value(i, 'Voronoi_cation_avgcn', get_cation_weighted_avg(specie_meancoord, struc))
# using own Effective coordination algorithm
try:
specie_meaneffcoord = get_avg_cns(EffectiveCoordFinder(struc).get_cns(radius=10.0))
except Exception as e:
print e
continue
df.set_value(i, 'eff_cn', json.dumps(specie_meaneffcoord))
df.set_value(i, 'eff_cation_avgcn', get_cation_weighted_avg(specie_meaneffcoord, struc))
# using Tina's O'Keeffe coordination algorithm
if row['metadata']['_structure']['is_ordered']:
try:
okeeffe_coord = okeeffe_get_avg_cn(struc)
except Exception as e:
print e
continue
df.set_value(i, 'okeeffe_cn', json.dumps(okeeffe_coord))
df.set_value(i, 'okeeffe_cn_avg', get_cation_weighted_avg(okeeffe_coord, struc))
return df
def on_get(self, req: Request, resp: Response):
result = self._method_manager.count_methods()
if isinstance(result, Exception):
resp.body = json.dumps(dict(error=str(result)))
resp.status = falcon.HTTP_500
else:
resp.body = json.dumps(dict(count=result))
resp.status = falcon.HTTP_201
def test_tz_range_is_utc(self):
from pandas.io.json import dumps
exp = '["2013-01-01T05:00:00.000Z","2013-01-02T05:00:00.000Z"]'
dfexp = ('{"DT":{'
'"0":"2013-01-01T05:00:00.000Z",'
'"1":"2013-01-02T05:00:00.000Z"}}')
tz_range = pd.date_range('2013-01-01 05:00:00Z', periods=2)
assert dumps(tz_range, iso_dates=True) == exp
dti = pd.DatetimeIndex(tz_range)
assert dumps(dti, iso_dates=True) == exp
df = DataFrame({'DT': dti})
assert dumps(df, iso_dates=True) == dfexp
tz_range = pd.date_range('2013-01-01 00:00:00', periods=2,
tz='US/Eastern')
assert dumps(tz_range, iso_dates=True) == exp
dti = pd.DatetimeIndex(tz_range)
assert dumps(dti, iso_dates=True) == exp
df = DataFrame({'DT': dti})
assert dumps(df, iso_dates=True) == dfexp
tz_range = pd.date_range('2013-01-01 00:00:00-0500', periods=2)
assert dumps(tz_range, iso_dates=True) == exp
dti = pd.DatetimeIndex(tz_range)
assert dumps(dti, iso_dates=True) == exp
df = DataFrame({'DT': dti})
assert dumps(df, iso_dates=True) == dfexp
def test_tz_range_is_utc(self):
from pandas.io.json import dumps
exp = '["2013-01-01T05:00:00.000Z","2013-01-02T05:00:00.000Z"]'
dfexp = ('{"DT":{'
'"0":"2013-01-01T05:00:00.000Z",'
'"1":"2013-01-02T05:00:00.000Z"}}')
tz_range = pd.date_range('2013-01-01 05:00:00Z', periods=2)
self.assertEqual(exp, dumps(tz_range, iso_dates=True))
dti = pd.DatetimeIndex(tz_range)
self.assertEqual(exp, dumps(dti, iso_dates=True))
df = DataFrame({'DT': dti})
self.assertEqual(dfexp, dumps(df, iso_dates=True))
tz_range = pd.date_range('2013-01-01 00:00:00',
periods=2,
tz='US/Eastern')
self.assertEqual(exp, dumps(tz_range, iso_dates=True))
dti = pd.DatetimeIndex(tz_range)
self.assertEqual(exp, dumps(dti, iso_dates=True))
df = DataFrame({'DT': dti})
self.assertEqual(dfexp, dumps(df, iso_dates=True))
tz_range = pd.date_range('2013-01-01 00:00:00-0500', periods=2)
self.assertEqual(exp, dumps(tz_range, iso_dates=True))
dti = pd.DatetimeIndex(tz_range)
self.assertEqual(exp, dumps(dti, iso_dates=True))
df = DataFrame({'DT': dti})
self.assertEqual(dfexp, dumps(df, iso_dates=True))
def _estimator_metadata(estimator, X, y, featurizer):
"""Serialize estimator and add score and other metadata."""
meta = {
'estimator': serialize_sklearn_estimator(estimator),
}
if len(X) and hasattr(estimator, 'score'):
meta['score'] = json.dumps(estimator.score(X, y))
if hasattr(estimator, 'coef_'):
meta['coef_'] = json.dumps(estimator.coef_.tolist())
if hasattr(estimator, 'intercept_'):
meta['intercept_'] = json.dumps(estimator.intercept_.tolist())
if hasattr(estimator, 'feature_importances_') and hasattr(featurizer, 'columns'):
meta['feature_importances_'] = json.dumps(dict(zip(featurizer.columns, estimator.feature_importances_)))
return meta
def on_get(self, req: Request, resp: Response, model_id: UUID = None, git_commit_id: str = None):
query = get_params_to_query(req, SORT_ATTRIBUTES, 'created', 'desc')
if model_id and git_commit_id:
snapshots = self._snapshot_manager.list_snapshots_filtered(model_id=model_id,
git_commit_id=git_commit_id,
query=query)
else:
snapshots = self._snapshot_manager.list_snapshots(query=query)
if isinstance(snapshots, Exception):
resp.body = json.dumps(dict(error=str(snapshots)))
resp.status = falcon.HTTP_500
else:
resp.body = json.dumps(dict(count=self._snapshot_manager.snapshot_count()))
resp.status = falcon.HTTP_201
def get_json_data(self):
df = self.get_df()
chart_data = self.to_series(df)
time_compare = self.form_data.get('time_compare')
if time_compare:
query_object = self.query_obj()
delta = utils.parse_human_timedelta(time_compare)
query_object['inner_from_dttm'] = query_object['from_dttm']
query_object['inner_to_dttm'] = query_object['to_dttm']
query_object['from_dttm'] -= delta
query_object['to_dttm'] -= delta
df2 = self.get_df(query_object)
df2.index += delta
chart_data += self.to_series(df2,
classed='dashed',
title_suffix="---")
chart_data = sorted(chart_data, key=lambda x: x['key'])
data = {
'chart_data': chart_data,
'query': self.results.query,
'duration': self.results.duration,
}
return dumps(data)
def on_delete(self, req: Request, resp: Response):
result = self._method_manager.get_method(req.media['method_id'])
if not type(result) == Method:
resp.body = json.dumps(dict(error=str(result)))
resp.status = falcon.HTTP_500
return resp
result = self._method_manager.remove_method(result)
if isinstance(result, Exception):
resp.body = json.dumps(dict(error=str(result)))
resp.status = falcon.HTTP_404
else:
resp.body = result.json()
resp.status = falcon.HTTP_202
def get_stock_em_hold_stock_df():
# 获取最近的交易日
year = time.strftime("%Y", time.localtime())
current_time = time.strftime("%Y-%m-%d", time.localtime())
ts.set_token('605894af201d877ec9108ba2fc05a95ea580f744ed640d2aa449623b')
pro = ts.pro_api()
data = pro.query('trade_cal', start_date=year + '0101', is_open='1')
trade_days = data['cal_date']
today = time.strftime("%Y%m%d", time.localtime())
while today not in trade_days.values:
today = str(int(today) - 1)
stock_em_hold_stock_df = ak.stock_em_hsgt_stock_statistics(
symbol="北向持股", start_date=today, end_date=today).head(50)
result = []
for index, row in stock_em_hold_stock_df.iterrows():
result.append({
'symbol': row["股票代码"],
'name': row["股票简称"],
'market_value': round(row["持股市值"] / 100000000),
'market_percent': row["持股数量占发行股百分比"]
})
# return result
resu = {
'code': 200,
'data': {
'current_time': current_time,
'result': result
},
'message': '成功'
}
return json.dumps(resu, ensure_ascii=False)
def get_em_add_stock_df():
date = request.values.get('date')
current_time = time.strftime("%Y-%m-%d", time.localtime())
stock_em_add_stock_df = ak.stock_em_hsgt_hold_stock(
market="北向", indicator=date).head(50)
result = []
print(stock_em_add_stock_df)
for index, row in stock_em_add_stock_df.iterrows():
result.append({
'symbol':
row["SCode"],
'name':
row["SName"],
'plate':
row["HYName"],
'incHoldMoney':
round(row["ShareSZ_Chg_One"] / 100000000, 2),
})
# # return result
resu = {
'code': 200,
'data': {
'current_time': current_time,
'result': result
},
'message': '成功'
}
return json.dumps(resu, ensure_ascii=False)
def get_json_data(self):
df = self.get_df()
series = df.to_dict('series')
chart_data = []
for name, ys in series.items():
if df[name].dtype.kind not in "biufc":
continue
df['timestamp'] = pd.to_datetime(df.index, utc=False)
if isinstance(name, string_types):
series_title = name
elif len(self.metrics) > 1:
series_title = ", ".join(name)
else:
series_title = ", ".join(name[1:])
d = {
"key":
series_title,
"color":
utils.color(series_title),
"values": [{
'x': ds,
'y': ys[i]
} for i, ds in enumerate(df.timestamp)]
}
chart_data.append(d)
return dumps({
'chart_data': chart_data,
'query': self.results.query,
'duration': self.results.duration,
})
def get_json_data(self):
df = self.get_df()
series = df.to_dict('series')
chart_data = []
for name, ys in series.items():
if df[name].dtype.kind not in "biufc":
continue
df['timestamp'] = pd.to_datetime(df.index, utc=False)
if isinstance(name, basestring):
series_title = name
elif len(self.metrics) > 1:
series_title = ", ".join(name)
else:
series_title = ", ".join(name[1:])
d = {
"key": series_title,
"color": utils.color(series_title),
"values": [
{'x': ds, 'y': ys[i]}
for i, ds in enumerate(df.timestamp)]
}
chart_data.append(d)
return dumps({
'chart_data': chart_data,
'query': self.results.query,
'duration': self.results.duration,
})
def get_group(self, key):
new_query = 'select value t.grps from (%s) t where grp_id=%s;' % (
self.query[:-1], str(key))
results = json.dumps(af.AFrame.send_request(new_query)[0])
grp = json.read_json(results)['grp']
df = pd.DataFrame(grp.tolist())
return df
def compute(graph_type=None):
"""Perform both intermediate step and aggregation at once."""
# Read inputs
logging.info("Fetching data...")
inputs = io_helper.fetch_data()
dep_var = inputs["data"]["dependent"][0]
indep_vars = inputs["data"]["independent"]
result = _compute_intermediate_result(inputs)
corr, columns, crosstab = _aggregate_results([result])
graph_type = graph_type or parameters.get_parameter(
'graph', str, 'correlation_heatmap')
if graph_type == 'correlation_heatmap':
fig = _fig_corr_heatmap(corr, columns, crosstab)
elif graph_type == 'pca':
X = io_helper.fetch_dataframe([dep_var] + indep_vars)
fig = _fig_pca(corr, columns, X)
else:
raise errors.UserError(
'MODEL_PARAM_graph only supports values `correlation_heatmap` and `pca`'
)
logging.info("Results:\n{}".format(fig))
io_helper.save_results(json.dumps(fig), shapes.Shapes.PLOTLY)
logging.info("DONE")
def compute():
"""Create PFA for kNN."""
inputs = io_helper.fetch_data()
dep_var = inputs["data"]["dependent"][0]
indep_vars = inputs["data"]["independent"]
params = parameters.fetch_parameters()
if dep_var['type']['name'] in ('polynominal', 'binominal'):
job_type = 'classification'
else:
job_type = 'regression'
logging.info('Creating new estimator')
estimator = _create_estimator(job_type, params)
featurizer = _create_featurizer(indep_vars)
# convert variables into dataframe
X = io_helper.fetch_dataframe(variables=[dep_var] + indep_vars)
X = utils.remove_nulls(X)
y = X.pop(dep_var['name'])
X = featurizer.transform(X)
# Drop NaN values
estimator.fit(X, y)
# Create PFA from the estimator
types = [(var['name'], var['type']['name']) for var in indep_vars]
pfa = sklearn_to_pfa(estimator, types, featurizer.generate_pretty_pfa())
pfa['name'] = "kNN"
# Save or update job_result
logging.info('Saving PFA to job_results table...')
pfa = json.dumps(pfa)
io_helper.save_results(pfa, shapes.Shapes.PFA)
def javascript_cmd(self):
js = dumps(self.chart)
js = (js.replace('"{{TOOLTIP_FORMATTER}}"',
self.tooltip_formatter).replace("\n", " "))
if self.stockchart:
return "new Highcharts.StockChart(%s);" % js
return "new Highcharts.Chart(%s);" % js
def scatter():
data_scatter_continuous = list()
for var in variable_list:
sub_data = dict()
sub_data['key']= var
sub_data['values'] = list()
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
scatter_plotx_list = metadata_variable.ix['plotx',0]
scatter_ploty_list = metadata_variable.ix['ploty',0]
# scatter_plotx_list.append(each)
# scatter_ploty_list.append(None)
for each in range(0, len(scatter_plotx_list)):
sub_dict = dict()
sub_dict['x'] = scatter_plotx_list[each]
sub_dict['y'] = scatter_ploty_list[each]
sub_data['values'].append(sub_dict)
data_scatter_continuous.append(sub_data)
data_json_scatter_continuous = json.dumps(data_scatter_continuous)
return jsonify ({'data_json_scatter_continuous': data_json_scatter_continuous})
def get_json(self):
"""Handles caching around the json payload retrieval"""
cache_key = self.cache_key
payload = None
if self.form_data.get('force') != 'true':
payload = cache.get(cache_key)
if payload:
is_cached = True
logging.info("Serving from cache")
else:
is_cached = False
cache_timeout = self.cache_timeout
payload = {
'data': self.get_data(),
'query': self.query,
'form_data': self.form_data,
'json_endpoint': self.json_endpoint,
'csv_endpoint': self.csv_endpoint,
'standalone_endpoint': self.standalone_endpoint,
'cache_timeout': cache_timeout,
}
payload['cached_dttm'] = datetime.now().isoformat().split('.')[0]
logging.info("Caching for the next {} seconds".format(
cache_timeout))
cache.set(cache_key, payload, timeout=self.cache_timeout)
payload['is_cached'] = is_cached
return dumps(payload)
def get_json_data(self):
from panoramix.data import countries
df = self.get_df()
cols = [self.form_data.get('entity')]
metric = self.form_data.get('metric')
secondary_metric = self.form_data.get('secondary_metric')
if metric == secondary_metric:
ndf = df[cols]
ndf['m1'] = df[metric]
ndf['m2'] = df[metric]
else:
cols += [metric, secondary_metric]
ndf = df[cols]
df = ndf
df.columns = ['country', 'm1', 'm2']
d = df.to_dict(orient='records')
for row in d:
country = countries.get(
self.form_data.get('country_fieldtype'), row['country'])
if country:
row['country'] = country['cca3']
row['latitude'] = country['lat']
row['longitude'] = country['lng']
row['name'] = country['name']
else:
row['country'] = "XXX"
return dumps(d)
def get_json_data(self):
df = self.get_df()
chart_data = self.to_series(df)
time_compare = self.form_data.get('time_compare')
if time_compare:
query_object = self.query_obj()
delta = utils.parse_human_timedelta(time_compare)
query_object['inner_from_dttm'] = query_object['from_dttm']
query_object['inner_to_dttm'] = query_object['to_dttm']
query_object['from_dttm'] -= delta
query_object['to_dttm'] -= delta
df2 = self.get_df(query_object)
df2.index += delta
chart_data += self.to_series(
df2, classed='dashed', title_suffix="---")
chart_data = sorted(chart_data, key=lambda x: x['key'])
data = {
'chart_data': chart_data,
'query': self.results.query,
'duration': self.results.duration,
}
return dumps(data)
def collect(self):
results = af.AFrame.send_request(self._query)
json_str = json.dumps(results)
result = pd.DataFrame(data=json.read_json(json_str))
if '_uuid' in result.columns:
result.drop('_uuid', axis=1, inplace=True)
return result
def get_json_data(self):
from panoramix.data import countries
df = self.get_df()
cols = [self.form_data.get('entity')]
metric = self.form_data.get('metric')
secondary_metric = self.form_data.get('secondary_metric')
if metric == secondary_metric:
ndf = df[cols]
ndf['m1'] = df[metric]
ndf['m2'] = df[metric]
else:
cols += [metric, secondary_metric]
ndf = df[cols]
df = ndf
df.columns = ['country', 'm1', 'm2']
d = df.to_dict(orient='records')
for row in d:
country = countries.get(self.form_data.get('country_fieldtype'),
row['country'])
if country:
row['country'] = country['cca3']
row['latitude'] = country['lat']
row['longitude'] = country['lng']
row['name'] = country['name']
else:
row['country'] = "XXX"
return dumps(d)
def aggregate_stats(job_ids, graph_type=None):
"""Get all partial statistics from all nodes and aggregate them.
:input job_ids: list of job_ids with intermediate results
"""
# Read intermediate inputs from jobs
logging.info("Fetching intermediate data...")
results = io_helper.load_intermediate_json_results(map(str, job_ids))
corr, columns, crosstab = _aggregate_results(results)
graph_type = graph_type or parameters.get_parameter(
'graph', str, 'correlation_heatmap')
if graph_type == 'correlation_heatmap':
fig = _fig_corr_heatmap(corr, columns, crosstab)
elif graph_type == 'pca':
# save PCA graphs, but leave out the one with PCA scores
logging.warning(
'Sample scores graph is not yet implemented for distributed PCA.')
fig = _fig_pca(corr, columns, X=None)
else:
raise errors.UserError(
'MODEL_PARAM_graph only supports values `correlation_heatmap` and `pca`'
)
logging.info("Results:\n{}".format(fig))
io_helper.save_results(json.dumps(fig), shapes.Shapes.PLOTLY)
logging.info("DONE")
def get_json(self):
"""Handles caching around the json payload retrieval"""
cache_key = self.cache_key
payload = None
if self.form_data.get('force') != 'true':
payload = cache.get(cache_key)
if payload:
is_cached = True
logging.info("Serving from cache")
else:
is_cached = False
cache_timeout = self.cache_timeout
payload = {
'data': self.get_data(),
'query': self.query,
'form_data': self.form_data,
'json_endpoint': self.json_endpoint,
'csv_endpoint': self.csv_endpoint,
'standalone_endpoint': self.standalone_endpoint,
'cache_timeout': cache_timeout,
}
payload['cached_dttm'] = datetime.now().isoformat().split('.')[0]
logging.info(
"Caching for the next {} seconds".format(cache_timeout))
cache.set(cache_key, payload, timeout=self.cache_timeout)
payload['is_cached'] = is_cached
return dumps(payload)
def get_json_data(self):
from panoramix.data import countries
df = self.get_df()
cols = [self.form_data.get("entity")]
metric = self.form_data.get("metric")
secondary_metric = self.form_data.get("secondary_metric")
if metric == secondary_metric:
ndf = df[cols]
ndf["m1"] = df[metric]
ndf["m2"] = df[metric]
else:
cols += [metric, secondary_metric]
ndf = df[cols]
df = ndf
df.columns = ["country", "m1", "m2"]
d = df.to_dict(orient="records")
for row in d:
country = countries.get(self.form_data.get("country_fieldtype"), row["country"])
if country:
row["country"] = country["cca3"]
row["latitude"] = country["lat"]
row["longitude"] = country["lng"]
row["name"] = country["name"]
else:
row["country"] = "XXX"
return dumps(d)
def scatter():
data_scatter_continuous = list()
for var in variable_list:
sub_data = dict()
sub_data["key"] = var
sub_data["values"] = list()
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
scatter_plotx_list = metadata_variable.ix["plotx", 0]
scatter_ploty_list = metadata_variable.ix["ploty", 0]
# scatter_plotx_list.append(each)
# scatter_ploty_list.append(None)
for each in range(0, len(scatter_plotx_list)):
sub_dict = dict()
sub_dict["x"] = scatter_plotx_list[each]
sub_dict["y"] = scatter_ploty_list[each]
sub_data["values"].append(sub_dict)
data_scatter_continuous.append(sub_data)
data_json_scatter_continuous = json.dumps(data_scatter_continuous)
return jsonify({"data_json_scatter_continuous": data_json_scatter_continuous})
def response_as_json(data):
json_str = json.dumps(data)
response = HttpResponse(
json_str,
content_type="application/json",
)
response["Access-Control-Allow-Origin"] = "*"
return response
def head(self, num=5):
new_query = self.query[:-1] + ' limit %d;' % num
results = af.AFrame.send_request(new_query)
json_str = json.dumps(results)
result = pd.DataFrame(data=json.read_json(json_str))
if '_uuid' in result.columns:
result.drop('_uuid', axis=1, inplace=True)
return result
def get_data_attribute(self):
content = {
'viz_name': self.viz_type,
'json_endpoint': self.json_endpoint,
'token': self.token,
'form_data': self.form_data,
}
return dumps(content)
def get_json_data(self):
df = self.get_df()
df = df.reset_index()
df.columns = ["x", "y"]
df["color"] = map(utils.color, df.x)
return dumps(
{"chart_data": df.to_dict(orient="records"), "query": self.results.query, "duration": self.results.duration}
)
def error_histograms(dep_var, indep_vars):
histograms = list()
if len(dep_var) > 0:
histograms.append(error_histogram(dep_var))
grouping_vars = [indep_var for indep_var in indep_vars if utils.is_nominal(indep_var)]
for grouping_var in grouping_vars:
histograms.append(error_histogram(dep_var, grouping_var))
return json.dumps(histograms)
def intermediate_stats():
"""Calculate X*X^T, means and count for single node that will be later used to construct covariance matrix."""
# Read inputs
logging.info("Fetching data...")
inputs = io_helper.fetch_data()
result = _compute_intermediate_result(inputs)
io_helper.save_results(json.dumps(result), shapes.Shapes.JSON)
logging.info("DONE")
def javascript_cmd(self):
js = dumps(self.chart)
js = (
js.replace('"{{TOOLTIP_FORMATTER}}"', self.tooltip_formatter)
.replace("\n", " ")
)
if self.stockchart:
return "new Highcharts.StockChart(%s);" % js
return "new Highcharts.Chart(%s);" %js
def get_json_data(self):
df = self.get_df()
series = defaultdict(list)
for row in df.to_dict(orient='records'):
series[row['group']].append(row)
chart_data = []
for k, v in series.items():
chart_data.append({'key': k, 'values': v})
return dumps(chart_data)
def on_post(self, req: Request, resp: Response):
snap = ModelSnapshot.from_json(req.media['ModelSnapshot'])
result = self._snapshot_manager.create_snapshot(snap)
if isinstance(result, Exception):
resp.body = json.dumps(dict(error=str(result)))
resp.status = falcon.HTTP_500
else:
resp.body = result.json()
resp.status = falcon.HTTP_201
def get_json_data(self):
df = self.get_df()
series = defaultdict(list)
for row in df.to_dict(orient="records"):
series[row["group"]].append(row)
chart_data = []
for k, v in series.items():
chart_data.append({"key": k, "color": utils.color(k), "values": v})
return dumps({"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration})
def version():
"""Returns version information
Example:
/version
"""
return json.dumps({"version": VERSION})
def stacked_area():
all_xvalues = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
plotx_list = metadata_variable.ix["plotx", 0]
for each in plotx_list:
all_xvalues.append(each)
all_xvalues = list(set(all_xvalues))
data_stackedarea = list()
for var in variable_list:
sub_data = dict()
sub_data["key"] = var
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
# save the 'plotx' and 'ploty' values individually as lists
plotx_list = metadata_variable.ix["plotx", 0]
ploty_list = metadata_variable.ix["ploty", 0]
length = len(plotx_list)
# start adding values
sub_data["values"] = list()
xval_list = list()
for each in range(0, length):
xval_list.append(plotx_list[each])
for each_x in all_xvalues:
if each_x not in xval_list:
plotx_list.append(each_x)
ploty_list.append(None)
xy = zip(plotx_list, ploty_list)
xy.sort()
length_xy = len(xy)
for each in range(0, length_xy):
sub_list = list()
sub_list.append(xy[each][0])
sub_list.append(xy[each][1])
sub_data["values"].append(sub_list)
data_stackedarea.append(sub_data)
data_json_stackedarea = json.dumps(data_stackedarea)
return jsonify({"data_json_stackedarea": data_json_stackedarea})
def stacked_area():
all_xvalues = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
plotx_list = metadata_variable.ix['plotx', 0]
for each in plotx_list:
all_xvalues.append(each)
all_xvalues = list(set(all_xvalues))
data_stackedarea = list()
for var in variable_list:
sub_data = dict()
sub_data['key'] = var
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
#save the 'plotx' and 'ploty' values individually as lists
plotx_list = metadata_variable.ix['plotx', 0]
ploty_list = metadata_variable.ix['ploty', 0]
length = len(plotx_list)
#start adding values
sub_data['values'] = list()
xval_list = list()
for each in range(0, length):
xval_list.append(plotx_list[each])
for each_x in all_xvalues:
if each_x not in xval_list:
plotx_list.append(each_x)
ploty_list.append(None)
xy = zip(plotx_list, ploty_list)
xy.sort()
length_xy = len(xy)
for each in range(0, length_xy):
sub_list = list()
sub_list.append(xy[each][0])
sub_list.append(xy[each][1])
sub_data['values'].append(sub_list)
data_stackedarea.append(sub_data)
data_json_stackedarea = json.dumps(data_stackedarea)
return jsonify({'data_json_stackedarea': data_json_stackedarea})
def on_get(self, req: Request, resp: Response):
query = get_params_to_query(req, SORT_ATTRIBUTES, 'created', 'desc')
methods = self._method_manager.list_methods(query=query)
if isinstance(methods, Exception):
resp.body = json.dumps(dict(error=str(methods)))
resp.status = falcon.HTTP_500
else:
resp.body = f"[{','.join(map(lambda x: x.json(), methods))}]"
resp.status = falcon.HTTP_201
def on_post(self, req: Request, resp: Response):
snapshot = self._snapshot_manager.get_snapshot(req.media['model_snapshot_id'])
if not type(snapshot) == ModelSnapshot:
resp.body = json.dumps(dict(error=str(snapshot)))
resp.status = falcon.HTTP_500
return resp
result = self._method_manager.create_method(name=req.media['name'],
description=req.media['description'],
snap=snapshot)
if isinstance(result, Exception):
resp.body = json.dumps(dict(error=str(result)))
resp.status = falcon.HTTP_500
else:
resp.body = result.json()
resp.status = falcon.HTTP_201
def get_json_data(self):
df = self.get_df()
df = df.reset_index()
df.columns = ['x', 'y']
df['color'] = map(utils.color, df.x)
return dumps({
'chart_data': df.to_dict(orient="records"),
'query': self.results.query,
'duration': self.results.duration,
})
def continuous_multibar():
data_multibar = list()
all_xvalues = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
plotx_list = metadata_variable.ix['plotx',0]
for each in plotx_list:
all_xvalues.append(each)
all_xvalues = list(set(all_xvalues))
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
#save the 'plotx' and 'ploty' values individually as lists
plotx_list = metadata_variable.ix['plotx',0]
ploty_list = metadata_variable.ix['ploty',0]
for each in all_xvalues:
if each not in plotx_list:
plotx_list.append(each)
ploty_list.append(None)
xy = zip(plotx_list, ploty_list)
xy.sort()
length_xy = len(xy)
sub_data = dict()
sub_data['key'] = var
sub_data['values'] = list()
xval_list = list()
for each in range(0, length_xy):
sub_dict = dict()
sub_dict['x'] = xy[each][0]
sub_dict['y'] = xy[each][1]
#xval_list.append(plotx_list[each])
sub_data['values'].append(sub_dict)
data_multibar.append(sub_data)
data_json_multibar = json.dumps(data_multibar)
return jsonify({'data_json_multibar': data_json_multibar})
def get_json_data(self):
df = self.get_df()
series = defaultdict(list)
for row in df.to_dict(orient='records'):
series[row['group']].append(row)
chart_data = []
for k, v in series.items():
chart_data.append({
'key': k,
'values': v })
return dumps(chart_data)
def test_convert_dates_infer(self):
# GH10747
from pandas.io.json import dumps
infer_words = ['trade_time', 'date', 'datetime', 'sold_at',
'modified', 'timestamp', 'timestamps']
for infer_word in infer_words:
data = [{'id': 1, infer_word: 1036713600000}, {'id': 2}]
expected = DataFrame([[1, Timestamp('2002-11-08')], [2, pd.NaT]],
columns=['id', infer_word])
result = read_json(dumps(data))[['id', infer_word]]
assert_frame_equal(result, expected)
def get_json_data(self):
form_data = self.form_data
df = self.get_df()
df = df.sort(columns=df.columns[0])
compare_lag = form_data.get("compare_lag", "")
compare_lag = int(compare_lag) if compare_lag and compare_lag.isdigit() else 0
d = {
'data': df.values.tolist(),
'compare_lag': compare_lag,
'compare_suffix': form_data.get('compare_suffix', ''),
}
return dumps(d)
def test_default_handler_indirect(self):
from pandas.io.json import dumps
def default(obj):
if isinstance(obj, complex):
return [("mathjs", "Complex"), ("re", obj.real), ("im", obj.imag)]
return str(obj)
df_list = [
9,
DataFrame({"a": [1, "STR", complex(4, -5)], "b": [float("nan"), None, "N/A"]}, columns=["a", "b"]),
]
expected = '[9,[[1,null],["STR",null],[[["mathjs","Complex"],' '["re",4.0],["im",-5.0]],"N\\/A"]]]'
self.assertEqual(expected, dumps(df_list, default_handler=default, orient="values"))
def get_json_data(self):
df = self.get_df()
series = defaultdict(list)
for row in df.to_dict(orient='records'):
series[row['group']].append(row)
chart_data = []
for k, v in series.items():
chart_data.append({
'key': k,
"color": utils.color(k),
'values': v })
return dumps({
'chart_data': chart_data,
'query': self.results.query,
'duration': self.results.duration,
})
def test_default_handler_indirect(self):
from pandas.io.json import dumps
def default(obj):
if isinstance(obj, complex):
return [('mathjs', 'Complex'),
('re', obj.real),
('im', obj.imag)]
return str(obj)
df_list = [9, DataFrame({'a': [1, 'STR', complex(4, -5)],
'b': [float('nan'), None, 'N/A']},
columns=['a', 'b'])]
expected = ('[9,[[1,null],["STR",null],[[["mathjs","Complex"],'
'["re",4.0],["im",-5.0]],"N\\/A"]]]')
self.assertEqual(expected, dumps(df_list, default_handler=default,
orient="values"))
def cont_line():
line_data = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
is_plottype = metadata_variable.index == "plottype"
plottype_df = metadata_variable[is_plottype]
line_plottype = str(plottype_df.ix[0,0])
all_xvalues = list()
all_yvalues = list()
line_plotx_list = metadata_variable.ix['plotx',0]
line_ploty_list = metadata_variable.ix['ploty',0]
length = len(line_plotx_list)
for each in range(0, length):
all_xvalues.append(line_plotx_list[each])
all_yvalues.append(line_ploty_list[each])
xy = zip(all_xvalues,all_yvalues)
xy.sort()
sub_data = dict()
sub_data['key']= var
sub_data['values'] = list()
for each in range(0, len(xy)):
sub_dict = dict()
sub_dict['x'] = xy[each][0]
sub_dict['y']= xy[each][1]
sub_data['values'].append(sub_dict)
line_data.append(sub_data)
data_json_line = json.dumps(line_data)
return jsonify({'data_json_line': data_json_line})
def pie_chart():
data_pie = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
is_plotvalues = metadata_variable.index == "plotvalues"
plotvalues_df = metadata_variable[is_plotvalues]
plotvalues_unicode = plotvalues_df.ix[0,0]
plotvalues_dict = dict(plotvalues_unicode)
for each in plotvalues_dict:
sub_data = {'value':plotvalues_dict[each], 'label':each}
data_pie.append(sub_data)
data_json_pie = json.dumps(data_pie)
return jsonify({'data_json_pie': data_json_pie})
def get_json_data(self):
df = self.get_df()
chart_data = self.to_series(df)
time_compare = self.form_data.get("time_compare")
if time_compare:
query_object = self.query_obj()
delta = utils.parse_human_timedelta(time_compare)
query_object["inner_from_dttm"] = query_object["from_dttm"]
query_object["inner_to_dttm"] = query_object["to_dttm"]
query_object["from_dttm"] -= delta
query_object["to_dttm"] -= delta
df2 = self.get_df(query_object)
df2.index += delta
chart_data += self.to_series(df2, classed="dashed", title_suffix="---")
chart_data = sorted(chart_data, key=lambda x: x["key"])
data = {"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration}
return dumps(data)
def get_json_data(self):
df = self.get_df()
series = df.to_dict("series")
chart_data = []
for name, ys in series.items():
if df[name].dtype.kind not in "biufc":
continue
df["timestamp"] = pd.to_datetime(df.index, utc=False)
if isinstance(name, string_types):
series_title = name
elif len(self.metrics) > 1:
series_title = ", ".join(name)
else:
series_title = ", ".join(name[1:])
d = {
"key": series_title,
"color": utils.color(series_title),
"values": [{"x": ds, "y": ys[i]} for i, ds in enumerate(df.timestamp)],
}
chart_data.append(d)
return dumps({"chart_data": chart_data, "query": self.results.query, "duration": self.results.duration})
def get_json_data(self):
df = self.get_df()
series = df.to_dict('series')
chart_data = []
for name, ys in series.items():
if df[name].dtype.kind not in "biufc":
continue
df['timestamp'] = pd.to_datetime(df.index, utc=False)
if isinstance(name, string_types):
series_title = name
elif len(self.metrics) > 1:
series_title = ", ".join(name)
else:
series_title = ", ".join(name[1:])
d = {
"key": series_title,
"values": [
{'x': ds, 'y': ys[i]}
for i, ds in enumerate(df.timestamp)]
}
chart_data.append(d)
return dumps(chart_data)
def get_json_data(self):
df = self.get_df()
series = df.to_dict('series')
chart_data = []
for name, ys in series.items():
if df[name].dtype.kind not in "biufc":
continue
if isinstance(name, string_types):
series_title = name
elif len(self.metrics) > 1:
series_title = ", ".join(name)
else:
l = [str(s) for s in name[1:]]
series_title = ", ".join(l)
d = {
"key": series_title,
"values": [
{'x': i, 'y': v}
for i, v in ys.iteritems()]
}
chart_data.append(d)
return dumps(chart_data)
def test_tz_is_utc(self):
from pandas.io.json import dumps
exp = '"2013-01-10T05:00:00.000Z"'
ts = Timestamp('2013-01-10 05:00:00Z')
assert dumps(ts, iso_dates=True) == exp
dt = ts.to_pydatetime()
assert dumps(dt, iso_dates=True) == exp
ts = Timestamp('2013-01-10 00:00:00', tz='US/Eastern')
assert dumps(ts, iso_dates=True) == exp
dt = ts.to_pydatetime()
assert dumps(dt, iso_dates=True) == exp
ts = Timestamp('2013-01-10 00:00:00-0500')
assert dumps(ts, iso_dates=True) == exp
dt = ts.to_pydatetime()
assert dumps(dt, iso_dates=True) == exp
def multi_bar():
warning_multibar = None
all_xvalues = list()
all_yvalues = list()
list_of_domains_multibar = list()
for var in variable_list:
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
is_plotvalues = metadata_variable.index == "plotvalues"
plotvalues_df = metadata_variable[is_plotvalues]
plotvalues_unicode = plotvalues_df.ix[0,0]
plotvalues_dict = dict(plotvalues_unicode)
individual_variable_domains= list() #used for the warning message
for each in plotvalues_dict:
all_xvalues.append(each)
individual_variable_domains.append(each)
all_yvalues.append(plotvalues_dict[each])
list_of_domains_multibar.append(individual_variable_domains)
union_multibar = list()
intersection_multibar = list_of_domains_multibar[0]
for each in list_of_domains_multibar:
union_multibar = list(set(union_multibar) | set(each))
intersection_multibar = list(set(intersection_multibar) & set(each))
difference_domains_multibar = 100.0-(100.0* len(intersection_multibar)/len(union_multibar))
#pdb.set_trace()
all_xvalues = list(set(all_xvalues))
data_multibar = list()
for var in variable_list:
sub_data = dict()
sub_data['values'] = list()
metadata_variable_series = metadata_all[var]
metadata_variable = pandas.DataFrame(metadata_variable_series)
is_plotvalues = metadata_variable.index == "plotvalues"
plotvalues_df = metadata_variable[is_plotvalues]
plotvalues_unicode = plotvalues_df.ix[0,0]
plotvalues_dict = dict(plotvalues_unicode)
xvalues_list = list()
yvalues_list = list()
for each in plotvalues_dict:
xvalues_list.append(each)
yvalues_list.append(plotvalues_dict[each])
for each_x in all_xvalues:
if each_x not in xvalues_list:
xvalues_list.append(each_x)
yvalues_list.append(None)
xy = zip(xvalues_list, yvalues_list)
xy.sort()
length_xy = len(xy)
for each in range(0,length_xy):
sub_dict = {'x':xy[each][0],'y':xy[each][1],}
sub_data['values'].append(sub_dict)
data_multibar.append(sub_data)
sub_data['key']=var
if difference_domains_multibar != None and difference_domains_multibar > 30.:
warning_multibar = '<div class="alert alert-warning alert-dismissible" role="alert"><button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">×</span></button><strong>Warning:</strong> chart created but may be misleading. <strong>'+ str(difference_domains_multibar) + "%</strong> of the selected variables's domains to do not intersect. Please be cautious before inferring any relationship between these variables.</div>"
data_json_multibar = json.dumps(data_multibar)
d = {'data_json_multibar': data_json_multibar, 'warning_multibar':warning_multibar}
return jsonify(**d)
