def ranks(self, **kwargs):
ranks = int(kwargs.get("limit", 1))
col = kwargs.get("col")
attr_type = kwargs.get("attr_type", self.attr_type)
params = {}
params[attr_type] = self.attr["id"]
params["required"] = col
params["show"] = kwargs.get("show", self.attr_type)
params["sumlevel"] = kwargs.get("sumlevel", self.sumlevel(**kwargs))
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
del params[attr_type]
params["limit"] = 1
params["order"] = col
params["sort"] = "desc"
if rank <= (ranks / 2 + 1):
results = range(1, ranks + 1)
else:
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
max_rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
if rank > (max_rank - ranks / 2 - 1):
results = range(max_rank - ranks + 1, max_rank + 1)
else:
results = range(int(math.ceil(rank - ranks / 2)), int(math.ceil(rank + ranks / 2) + 1))
if kwargs.get("key", False) == "id":
del params["limit"]
params[col] = ",".join([str(r) for r in results])
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
results = [d[params["show"]] for d in datafold(requests.get(url).json())]
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
return ",".join([str(r) for r in results])
def ranks(self, **kwargs):
ranks = int(kwargs.get("limit", 1))
col = kwargs.get("col")
attr_type = kwargs.get("attr_type", self.attr_type)
params = {}
params[attr_type] = self.attr["id"]
params["required"] = col
params["show"] = kwargs.get("show", self.attr_type)
params["sumlevel"] = kwargs.get("sumlevel", self.sumlevel(**kwargs))
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
del params[attr_type]
params["limit"] = 1
params["order"] = col
params["sort"] = "desc"
if rank <= (ranks/2 + 1):
results = range(1, ranks + 1)
else:
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
max_rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
if rank > (max_rank - ranks/2 - 1):
results = range(max_rank - ranks + 1, max_rank + 1)
else:
results = range(int(math.ceil(rank - ranks/2)), int(math.ceil(rank + ranks/2) + 1))
if kwargs.get("key", False) == "id":
del params["limit"]
params[col] = ",".join([str(r) for r in results])
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
results = [d[params["show"]] for d in datafold(requests.get(url).json())]
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
return ",".join([str(r) for r in results])
def _get_params_with_signature(self, **kwargs):
t = self._to_epoch_miliseconds(datetime.now())
kwargs['timestamp'] = t
body = RequestEncodingMixin._encode_params(kwargs)
sign = hmac.new(self.api_secret, bytes(body, 'utf-8'),
hashlib.sha256).hexdigest()
return {'signature': sign, **kwargs}
def build_signature(request, consumer_secret):
"""Uses the request object and consumer secret to build a signature
This is an internal function and it's highly unlikely than an end
user would ever need to call this
:param request : a twolegged.Request subclass
:param consumer_secret : string
:rtype : string
"""
headers = request.headers()
auth_headers = {k: v for k, v in headers.iteritems() if k == 'Authorization'}
# we are only interested in the POST data if it's passed as form
# parameters while raw POST body is ignored
body = request.form_data()
params = [(k, v) for k, v in request.params() if k != 'oauth_signature']
qs = RequestEncodingMixin._encode_params(params)
collected_params = collect_parameters(qs, body, auth_headers)
normalized_params = normalize_parameters(collected_params)
host = headers.get('Host', None)
normalized_uri = normalize_base_string_uri(request.base_url(), host)
base_string = construct_base_string(unicode(request.method()),
normalized_uri,
normalized_params)
return sign_hmac_sha1(base_string, consumer_secret, None)
def get_authorisation_url(self, scopes):
params = {
'client_id': self.client_id,
'response_type': 'code',
'redirect_uri': self.get_redirect_uri(),
'scope': ' '.join(scopes),
}
return requote_uri(AUTHORISATION_URL.format(params=RequestEncodingMixin._encode_params(params)))
def _pack_file_segment(self, variables, filedata):
# Use the requests module to prepare the file data for us.
files = [('file.data', filedata)]
body, mimetype = RequestEncodingMixin._encode_files(files, variables)
return {
'body': body,
'mimetype': mimetype
}
def call_http_request(url, method, req_headers=None, req_data=None, req_query_string=None, **kwargs):
if config.DEBUG:
from requests.models import RequestEncodingMixin
import os
from publish import ClientService
print 'calling http %s%s pid:%s now_client_count:%s' % (
url, '?' + RequestEncodingMixin._encode_params(req_query_string) if req_query_string else '',
os.getpid(), len(ClientService.clients))
return getattr(requests, method.lower())('%s' % url, params=req_query_string, data=req_data, headers=req_headers,
**kwargs)
def call_http_request(url, method, req_headers=None, req_data=None, req_query_string=None, **kwargs):
d1 = datetime.datetime.now()
result = getattr(requests, method.lower())('%s' % url, params=req_query_string, data=req_data, headers=req_headers,
**kwargs)
d2 = datetime.datetime.now()
from requests.models import RequestEncodingMixin
print 'call http %s%s time:%s' % (url,
'?' + RequestEncodingMixin._encode_params(req_query_string) if req_query_string else '',(d2-d1).total_seconds())
return result
def _request(self, method, url_or_endpoint, **kwargs):
http_client = AsyncHTTPClient()
if not url_or_endpoint.startswith(('http://', 'https://')):
api_base_url = kwargs.pop('api_base_url', self.API_BASE_URL)
url = '{base}{endpoint}'.format(base=api_base_url,
endpoint=url_or_endpoint)
else:
url = url_or_endpoint
headers = {}
params = kwargs.pop('params', {})
if 'access_token' not in params:
params['access_token'] = self.access_token
params = urlencode(dict((k, to_binary(v)) for k, v in params.items()))
url = '{0}?{1}'.format(url, params)
data = kwargs.get('data')
files = kwargs.get('files')
if files:
from requests.models import RequestEncodingMixin
from requests.utils import super_len
body, content_type = RequestEncodingMixin._encode_files(
files, data)
headers['Content-Type'] = content_type
headers['Content-Length'] = super_len(body)
else:
if isinstance(data, dict):
body = json.dumps(data, ensure_ascii=False)
body = body.encode('utf-8')
else:
body = data
result_processor = kwargs.pop('result_processor', None)
timeout = kwargs.get('timeout', self.timeout)
req = HTTPRequest(url=url,
method=method.upper(),
headers=headers,
body=body,
request_timeout=timeout)
res = yield http_client.fetch(req)
if res.error is not None:
raise WeChatClientException(errcode=None,
errmsg=None,
client=self,
request=req,
response=res)
result = self._handle_result(res, method, url, result_processor,
**kwargs)
raise Return(result)
def build_url(url, params):
"""
only for display purpose, parse the params and url to build the final api
:param str url: the url path to use
:param dict params: the dict with the GET parameters, as accepted by requests
:return: the url with the get part
:rtype: str
"""
if params is None:
result = url
else:
result = "%s?%s" % (url, RequestEncodingMixin._encode_params(params))
return result
def call_http_request(url,
method,
req_headers=None,
req_data=None,
req_query_string=None,
**kwargs):
d1 = datetime.datetime.now()
result = getattr(requests, method.lower())('%s' % url,
params=req_query_string,
data=req_data,
headers=req_headers,
**kwargs)
d2 = datetime.datetime.now()
from requests.models import RequestEncodingMixin
print 'call http %s%s time:%s' % (
url, '?' + RequestEncodingMixin._encode_params(req_query_string)
if req_query_string else '', (d2 - d1).total_seconds())
return result
def post(self):
_url = self.get_argument('url', default=None)
data = self.get_argument('data', default=None)
self.client.is_params = False if self.get_argument('is_params', default=False) == '0' else True
if not _url:
raise UnexpectedReuqestDataException
if not _url.startswith('ws://') and not _url.startswith('wss://'):
raise InvalidWebSocketURLException
url = urlparse.urlparse(_url)
# `query_str` is the query string of websocket
query_str = url.query
url = '%s://%s%s' % (url.scheme, url.netloc, url.path)
if self.request.body:
query_str = query_str + '&' + self.request.body
query_str = encoder._encode_params(parse_qs_bytes(query_str))
if not data and not query_str:
logging.warning('No query strings in url, is it HTTP headers injection?')
# raise UnexpectedReuqestDataException
if query_str:
logging.info('Request query string: %s' % query_str)
if data:
logging.info('Request message: %s' % data)
if not self.client.ws:
self.run_websocket('%s?%s' % (url, query_str))
else:
self.client.has_send = False
try:
self.client.ws.send(data if data else '')
except WebSocketException, e:
self.client.ws.close()
self.client.ws = None
logging.error('Error occur: %s' % str(e))
self.finish()
def multi_col_top(profile, params):
namespace = params.pop("namespace")
after = params.pop("after", None)
attr_type = params.get("attr_type", profile.attr_type)
rows = params.pop("rows", False)
cols = params.pop("required", [])
children = params.pop("children", False)
params["show"] = params.get("show", attr_type)
params["limit"] = params.get("limit", 1)
params["sumlevel"] = params.get("sumlevel", "all")
params["sort"] = params.get("sort", "desc")
if children:
params["prefix"] = True
params["where"] = "{}:{}".format(attr_type, profile.children(**params))
prefix_pop = params.pop("prefix")
elif attr_type not in params:
params[attr_type] = profile.id(**params)
dataset = params.pop("dataset", False)
params["required"] = ",".join(cols)
pivot = params.pop("pivot", False)
query = RequestEncodingMixin._encode_params(params)
url = u"{}/api?{}".format(API, query)
try:
r = requests.get(url).json()
except ValueError:
app.logger.info("VAR ERROR: {}".format(url))
return {}
return_obj = {namespace: {} if not rows else []}
if "error" in r or len(r["data"]) == 0:
return return_obj
headers = r["headers"]
if pivot:
base_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, "-".join(pivot["keys"]), dataset)
limit = pivot.get("limit", 1)
cols = pivot["cols"]
api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]
if rows:
myobject = {}
for index, data_row in enumerate(api_data):
myobject = {}
headers = data_row.keys()
values = data_row.values()
for col in cols:
stat_url = u"{}&rank={}".format(base_url, index + 1)
myobject[col] = render_col(values, headers, col, stat_url)
return_obj[namespace].append(myobject)
else:
values = api_data[0].values()
headers = api_data[0].keys()
stat_url = u"{}&rank=1".format(base_url)
for col in cols:
return_obj[namespace][col] = render_col(values, headers, col, stat_url, dataset)
elif not rows:
if not r["data"]:
return {}
api_data = r["data"][0]
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, stat_col, dataset)
return_obj[namespace][col] = render_col(api_data, headers, col, stat_url, dataset)
else:
if after:
new_order = after.get("order", None)
if new_order:
new_sort = after.get("sort", "desc")
reverse = True if new_sort == "desc" else False
r["data"] = sorted(r["data"], key=lambda k: k[headers.index(new_order)], reverse=reverse)
for index, data_row in enumerate(r["data"]):
myobject = {}
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(url_for("profile.statView"), query, stat_col, index + 1, dataset)
myobject[col] = render_col(data_row, headers, col, stat_url, dataset)
return_obj[namespace].append(myobject)
return return_obj
def percent(self, **kwargs):
"""str: 2 columns divided by one another """
attr_type = kwargs.get("attr_type", self.attr_type)
attr_id = kwargs.get("attr_id", self.attr["id"])
r = {"num": 1, "den": 1}
for t in r.keys():
key = kwargs.get(t)
params = {}
params["limit"] = 1
params["year"] = kwargs.get("year", "latest")
params = param_format(params)
t_type = kwargs.get("{}_type".format(t), attr_type)
params["show"] = kwargs.get("show", t_type)
params[t_type] = kwargs.get("{}_id".format(t), attr_id)
params["exclude"] = kwargs.get("exclude", kwargs.get("{}_exclude".format(t), ""))
if "top:" in key:
params["col"], params["force"] = key.split(":")[1].split(",")
r["{}_key".format(t)] = params["col"]
r[t] = self.top(**params)["data"][0]
elif "var:" in key:
keys = key.split(":")[1].split(",")
if len(keys) == 2:
keys.append(None)
ns, col, row = keys
r["{}_key".format(t)] = col
r[t] = self.var(namespace=ns, key=col, row=row, format="raw")
elif "," in key:
num, den = key.split(",")
subparams = {}
subparams["num"] = num
subparams["den"] = den
subparams["data_only"] = True
subparams["num_id"] = params[t_type]
subparams["den_id"] = params[t_type]
r["{}_key".format(t)] = None
r[t] = self.percent(**subparams)
else:
params["required"] = key
r["{}_key".format(t)] = key
# convert request arguments into a url query string
query = RequestEncodingMixin._encode_params(params)
url = u"{}/api?{}".format(API, query)
try:
r[t] = datafold(requests.get(url).json())
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return "N/A"
if len(r[t]) == 0:
return "N/A"
r[t] = r[t][0][key]
if r[t] in [None, "N/A"]:
return "N/A"
diff = kwargs.get("diff", False)
text = kwargs.get("text", False)
if text and text in TEXTCOMPARATORS:
r["num"] = float(num_format(r["num"], r["num_key"], False, suffix=False).replace(",", ""))
r["den"] = float(num_format(r["den"], r["den_key"], False, suffix=False).replace(",", ""))
if r["num"] == 0 or r["den"] == 0:
val = 0
elif diff:
val = r["num"] - r["den"]
else:
val = float(r["num"])/float(r["den"])
if kwargs.get("invert", False):
val = 1 - val
if kwargs.get("data_only", False):
return val
if text and text in TEXTCOMPARATORS:
text = TEXTCOMPARATORS[text]
if diff:
if val > 0:
return text[0]
elif val < 0:
return text[1]
else:
return text[2]
else:
if val > 1:
return text[0]
elif val < 1:
return text[1]
else:
return text[2]
elif diff or kwargs.get("ratio", False):
return num_format(abs(val))
else:
return "{}%".format(num_format(val * 100))
def parents(self, **kwargs):
id_only = kwargs.get("id_only", False)
limit = kwargs.pop("limit", None)
attr_id = self.id(**kwargs)
prefix = kwargs.get("prefix", None)
if (prefix or limit) and id_only == False:
top = get_parents(attr_id, self.attr_type)
if prefix:
top = [p for p in top if p["id"].startswith(prefix)]
if limit:
top = top[-int(limit):]
top = self.make_links(top)
return top
if self.attr["id"] == "01000US":
col = kwargs.get("col", False)
if col:
params = {
"show": "geo",
"required": col,
"sumlevel": "state",
"order": col,
"sort": "desc",
"year": "latest"
}
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
results = [r for r in datafold(requests.get(url).json()) if r[col]]
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
results = results[:2] + results[-2:]
if id_only:
return ",".join([r["geo"] for r in results])
else:
return [fetch(r["geo"], "geo") for r in results]
else:
results = [
{"id": "04000US06", "name": "California", "url_name": "california"},
{"id": "04000US48", "name": "Texas", "url_name": "texas"},
{"id": "04000US36", "name": "New York", "url_name": "new-york"},
{"id": "16000US1150000", "name":"Washington D.C.", "url_name": "washington-dc"},
{"id": "16000US3651000", "name":"New York, NY", "url_name": "new-york-ny"},
{"id": "16000US0644000", "name":"Los Angeles, CA", "url_name": "los-angeles-ca"},
{"id": "16000US1714000", "name":"Chicago, IL", "url_name": "chicago-il"}
]
if id_only:
return ",".join([r["id"] for r in results])
else:
return results
results = [p for p in get_parents(attr_id, self.attr_type) if p["id"] != attr_id]
results = self.get_uniques(results)
for p in results:
if self.attr_type == "geo":
level = p["id"][:3]
elif self.attr_type == "cip":
level = str(len(p["id"]))
else:
level = str(fetch(p["id"], self.attr_type)["level"])
p["sumlevel"] = SUMLEVELS[self.attr_type][level]["label"]
if prefix:
results = [r for r in results if r["id"].startswith(prefix)]
if limit:
results = results[-int(limit):]
if id_only:
return ",".join([r["id"] for r in results])
else:
return results
def __init__(self, params, highlight=False, profile=False, select=False, slug=False):
"""Initializes a new Viz class.
Args:
config (str): The YAML configuration file as one long string.
profile (Profile): The Profile class instance this Section will be a part of.
"""
self.highlight = params.pop("highlight", highlight)
self.profile = profile.attr
self.select = select
self.profile_type = profile.attr_type
self.className = params.pop("class", False)
self.slug = slug
# force the data of params into a list
data = params.pop("data") if isinstance(params["data"], list) else [params.pop("data")]
# remove sumlevel if it exists
sumlevel = params.pop("sumlevel", None)
# loop through each data and append to self.data
self.data = []
for d in data:
# create a new dict containing the 'split' and 'static' params
data_obj = {
"map": d.pop("map", None),
"split": d.pop("split", None),
"static": d.pop("static", None),
"share": d.pop("share", None)
}
# Set fallback API params
d = param_format(d)
# create the data URL
p = RequestEncodingMixin._encode_params(d)
data_obj["url"] = "{}/api/?{}".format(API, p)
# store the params in the return dict
data_obj["params"] = d
# self.data.append(data_obj)
if "limit" in d and "year" in d and d["year"] == "all":
for year in year_cache[requests.get(data_obj["url"].replace("/api/", "/api/logic/")).json()["tables"][0]["table"]]:
new_obj = copy.deepcopy(data_obj)
year = str(int(year))
new_obj["url"] = new_obj["url"].replace("year=all", "year={}".format(year))
new_obj["params"]["year"] = year
self.data.append(new_obj)
else:
# append the data dict to self.data
self.data.append(data_obj)
self.attrs = []
if "attrs" in params:
# force the attrs of params into a list
attrs = params.pop("attrs") if isinstance(params["attrs"], list) else [params.pop("attrs")]
# loop through each data and append to self.data
self.attrs = [{"type": a, "url": "{}/attrs/{}/".format(API, a)} for a in attrs]
# set self.config to the params
self.config = params
if "mouse" in params:
if params["mouse"] == "NO":
self.config["mouse"] = False
else:
self.config["mouse"] = True
# set the tooltip config using the function
self.config["tooltip"] = params.pop("tooltip", {})
self.config["tooltip"]["value"] = self.tooltip()
# set default depth to zero
self.config["depth"] = int(params["depth"]) if "depth" in params else 0
# set default text to "name"
self.config["text"] = params["text"] if "text" in params else "name"
def percent(self, **kwargs):
"""str: 2 columns divided by one another """
attr_type = kwargs.get("attr_type", self.attr_type)
attr_id = kwargs.get("attr_id", self.attr["id"])
r = {"num": 1, "den": 1}
for t in r.keys():
key = kwargs.get(t)
params = {}
params["limit"] = 1
params["year"] = kwargs.get("year", "latest")
params = param_format(params)
t_type = kwargs.get("{}_type".format(t), attr_type)
params["show"] = kwargs.get("show", t_type)
params[t_type] = kwargs.get("{}_id".format(t), attr_id)
params["exclude"] = kwargs.get("exclude", kwargs.get("{}_exclude".format(t), ""))
if "top:" in key:
params["col"], params["force"] = key.split(":")[1].split(",")
r["{}_key".format(t)] = params["col"]
r[t] = self.top(**params)["data"][0]
elif "var:" in key:
keys = key.split(":")[1].split(",")
if len(keys) == 2:
keys.append(None)
ns, col, row = keys
r["{}_key".format(t)] = col
r[t] = self.var(namespace=ns, key=col, row=row, format="raw")
elif "," in key:
num, den = key.split(",")
subparams = {}
subparams["num"] = num
subparams["den"] = den
subparams["data_only"] = True
subparams["num_id"] = params[t_type]
subparams["den_id"] = params[t_type]
r["{}_key".format(t)] = None
r[t] = self.percent(**subparams)
else:
params["required"] = key
r["{}_key".format(t)] = key
# convert request arguments into a url query string
query = RequestEncodingMixin._encode_params(params)
url = u"{}/api?{}".format(API, query)
try:
r[t] = datafold(requests.get(url).json())
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return "N/A"
if len(r[t]) == 0:
return "N/A"
r[t] = r[t][0][key]
if r[t] in [None, "N/A"]:
return "N/A"
diff = kwargs.get("diff", False)
text = kwargs.get("text", False)
if text and text in TEXTCOMPARATORS:
r["num"] = float(num_format(r["num"], r["num_key"], False, suffix=False).replace(",", ""))
r["den"] = float(num_format(r["den"], r["den_key"], False, suffix=False).replace(",", ""))
if r["num"] == 0 or r["den"] == 0:
val = 0
elif diff:
val = r["num"] - r["den"]
else:
val = float(r["num"])/float(r["den"])
if kwargs.get("invert", False):
val = 1 - val
if kwargs.get("data_only", False):
return val
if text and text in TEXTCOMPARATORS:
text = TEXTCOMPARATORS[text]
if diff:
if val > 0:
return text[0]
elif val < 0:
return text[1]
else:
return text[2]
else:
if val > 1:
return text[0]
elif val < 1:
return text[1]
else:
return text[2]
elif diff or kwargs.get("ratio", False):
return num_format(abs(val))
else:
return "{}%".format(num_format(val * 100))
def parents(self, **kwargs):
id_only = kwargs.get("id_only", False)
limit = kwargs.pop("limit", None)
attr_id = self.id(**kwargs)
prefix = kwargs.get("prefix", None)
if (prefix or limit) and id_only == False:
top = get_parents(attr_id, self.attr_type)
if prefix:
top = [p for p in top if p["id"].startswith(prefix)]
if limit:
top = top[-int(limit):]
top = self.make_links(top)
return top
if self.attr["id"] == "01000US":
col = kwargs.get("col", False)
if col:
params = {
"show": "geo",
"required": col,
"sumlevel": "state",
"order": col,
"sort": "desc",
"year": "latest"
}
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
results = [r for r in datafold(requests.get(url).json()) if r[col]]
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
results = results[:2] + results[-2:]
if id_only:
return ",".join([r["geo"] for r in results])
else:
return [fetch(r["geo"], "geo") for r in results]
else:
results = [
{"id": "04000US06", "name": "California", "url_name": "california"},
{"id": "04000US48", "name": "Texas", "url_name": "texas"},
{"id": "04000US36", "name": "New York", "url_name": "new-york"},
{"id": "16000US1150000", "name":"Washington D.C.", "url_name": "washington-dc"},
{"id": "16000US3651000", "name":"New York, NY", "url_name": "new-york-ny"},
{"id": "16000US0644000", "name":"Los Angeles, CA", "url_name": "los-angeles-ca"},
{"id": "16000US1714000", "name":"Chicago, IL", "url_name": "chicago-il"}
]
if id_only:
return ",".join([r["id"] for r in results])
else:
return results
results = [p for p in get_parents(attr_id, self.attr_type) if p["id"] != attr_id]
results = self.get_uniques(results)
if prefix:
results = [r for r in results if r["id"].startswith(prefix)]
if limit:
results = results[-int(limit):]
if id_only:
return ",".join([r["id"] for r in results])
else:
return results
def multi_col_top(profile, params):
namespace = params.pop("namespace")
after = params.pop("after", None)
attr_type = params.get("attr_type", profile.attr_type)
rows = params.pop("rows", False)
cols = params.pop("required", [])
children = params.pop("children", False)
params["show"] = params.get("show", attr_type)
params["limit"] = params.get("limit", 1)
params["sumlevel"] = params.get("sumlevel", "all")
params["sort"] = params.get("sort", "desc")
if children:
params["prefix"] = True
params["where"] = "{}:{}".format(attr_type, profile.children(**params))
prefix_pop = params.pop("prefix")
elif attr_type not in params:
params[attr_type] = profile.id(**params)
dataset = params.pop("dataset", False)
params["required"] = ",".join(cols)
pivot = params.pop("pivot", False)
query = RequestEncodingMixin._encode_params(params)
url = u"{}/api?{}".format(API, query)
try:
r = requests.get(url).json()
except ValueError:
app.logger.info("VAR ERROR: {}".format(url))
return {}
return_obj = {namespace: {} if not rows else []}
if "error" in r or len(r["data"]) == 0:
return return_obj
headers = r["headers"]
if pivot:
base_url = u"{}?{}&col={}&dataset={}".format(
url_for("profile.statView"), query, "-".join(pivot["keys"]),
dataset)
limit = pivot.get("limit", 1)
cols = pivot["cols"]
api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]
if rows:
myobject = {}
for index, data_row in enumerate(api_data):
myobject = {}
headers = data_row.keys()
values = data_row.values()
for col in cols:
stat_url = u"{}&rank={}".format(base_url, index + 1)
myobject[col] = render_col(values, headers, col, stat_url)
return_obj[namespace].append(myobject)
else:
values = api_data[0].values()
headers = api_data[0].keys()
stat_url = u"{}&rank=1".format(base_url)
for col in cols:
return_obj[namespace][col] = render_col(
values, headers, col, stat_url, dataset)
elif not rows:
if not r["data"]:
return {}
api_data = r["data"][0]
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&dataset={}".format(
url_for("profile.statView"), query, stat_col, dataset)
return_obj[namespace][col] = render_col(api_data, headers, col,
stat_url, dataset)
else:
if after:
new_order = after.get("order", None)
if new_order:
new_sort = after.get("sort", "desc")
mute = after.get("mute", [])
reverse = True if new_sort == "desc" else False
show = params["show"].split(",")[0]
r["data"] = sorted(r["data"],
key=lambda k: 0 if k[headers.index(show)] in
mute else k[headers.index(new_order)],
reverse=reverse)
aggregate = after.get("aggregate", None)
if aggregate:
method = after.get("method", "sum")
myobject = {}
for row in r["data"]:
aggid = row[headers.index(aggregate)]
if aggid in myobject:
prev = myobject[aggid]
else:
prev = [None] * len(headers)
for col in headers:
i = headers.index(col)
if prev[i] == None or isinstance(
row[i], basestring
) or col == aggregate or col == "year":
prev[i] = row[i]
elif col == aggregate or col:
if method == "max":
prev[i] = max([prev[i], row[i]])
else:
prev[i] = prev[i] + row[i]
myobject[aggid] = prev
reverse = False if params["sort"] == "asc" else True
r["data"] = sorted(
[myobject[k] for k in myobject],
key=lambda k: k[headers.index(params["order"])],
reverse=reverse)
for index, data_row in enumerate(r["data"]):
myobject = {}
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(
url_for("profile.statView"), query, stat_col, index + 1,
dataset)
myobject[col] = render_col(data_row, headers, col, stat_url,
dataset)
return_obj[namespace].append(myobject)
return return_obj
def _request(self, method, url_or_endpoint, **kwargs):
http_client = AsyncHTTPClient()
if not url_or_endpoint.startswith(('http://', 'https://')):
api_base_url = kwargs.pop('api_base_url', self.API_BASE_URL)
url = '{base}{endpoint}'.format(
base=api_base_url,
endpoint=url_or_endpoint
)
else:
url = url_or_endpoint
headers = {}
params = kwargs.pop('params', {})
if 'access_token' not in params:
params['access_token'] = self.access_token
params = urlencode(dict((k, to_binary(v)) for k, v in params.items()))
url = '{0}?{1}'.format(url, params)
data = kwargs.get('data')
files = kwargs.get('files')
if files:
from requests.models import RequestEncodingMixin
from requests.utils import super_len
body, content_type = RequestEncodingMixin._encode_files(
files,
data
)
headers['Content-Type'] = content_type
headers['Content-Length'] = super_len(body)
else:
if isinstance(data, dict):
body = json.dumps(data, ensure_ascii=False)
body = body.encode('utf-8')
else:
body = data
result_processor = kwargs.pop('result_processor', None)
timeout = kwargs.get('timeout', self.timeout)
req = HTTPRequest(
url=url,
method=method.upper(),
headers=headers,
body=body,
request_timeout=timeout
)
res = yield http_client.fetch(req)
if res.error is not None:
raise WeChatClientException(
errcode=None,
errmsg=None,
client=self,
request=req,
response=res
)
result = self._handle_result(
res, method, url, result_processor, **kwargs
)
raise Return(result)
def _pack_file_segment(self, variables, filedata):
# Use the requests module to prepare the file data for us.
files = [('file.data', filedata)]
body, mimetype = RequestEncodingMixin._encode_files(files, variables)
return {'body': body, 'mimetype': mimetype}
def __init__(self,
params,
highlight=False,
profile=False,
select=False,
slug=False,
topic=False):
"""Initializes a new Viz class.
Args:
config (str): The YAML configuration file as one long string.
profile (Profile): The Profile class instance this Section will be a part of.
"""
self.highlight = params.pop("highlight", highlight)
self.profile = profile.attr
self.select = select
self.profile_type = profile.attr_type
self.className = params.pop("class", False)
self.slug = slug
# force the data of params into a list
data = params.pop("data") if isinstance(
params["data"], list) else [params.pop("data")]
data = [d for d in data if self.allowed_levels(d)]
# remove sumlevel if it exists
sumlevel = params.pop("sumlevel", None)
# loop through each data and append to self.data
self.data = []
for d in data:
# create a new dict containing the 'split' and 'static' params
data_obj = {
"join": d.pop("join", None),
"divide": d.pop("divide", None),
"map": d.pop("map", None),
"merge": d.pop("merge", None),
"split": d.pop("split", None),
"static": d.pop("static", None),
"share": d.pop("share", None),
"sum": d.pop("sum", None),
}
# Set fallback API params
d = param_format(d)
# create the data URL
p = RequestEncodingMixin._encode_params(d)
join = "join/" if data_obj["join"] else ""
data_obj["url"] = "{}/api/{}?{}".format(API, join, p)
# store the params in the return dict
data_obj["params"] = d
# self.data.append(data_obj)
if "limit" in d and "year" in d and d["year"] == "all":
for year in year_cache[requests.get(data_obj["url"].replace(
"/api/", "/api/logic/")).json()["tables"][0]["table"]]:
new_obj = copy.deepcopy(data_obj)
year = str(int(year))
new_obj["url"] = new_obj["url"].replace(
"year=all", "year={}".format(year))
new_obj["params"]["year"] = year
self.data.append(new_obj)
else:
# append the data dict to self.data
self.data.append(data_obj)
self.attrs = []
if "attrs" in params:
# force the attrs of params into a list
attrs = params.pop("attrs") if isinstance(
params["attrs"], list) else [params.pop("attrs")]
# loop through each data and append to self.data
self.attrs = [{
"type": a,
"url": "{}/attrs/{}/".format(API, a)
} for a in attrs]
# set self.config to the params
self.config = params
if "mouse" in params:
if params["mouse"] == "NO":
self.config["mouse"] = False
else:
self.config["mouse"] = True
# set the tooltip config using the function
self.config["tooltip"] = params.pop("tooltip", {})
self.config["tooltip"]["value"] = self.tooltip()
tooltipValues = []
if "year" not in tooltipValues:
tooltipValues.append("year")
for value in self.config["tooltip"]["value"]:
tooltipValues.append(value)
if value in COLLECTIONYEARS:
tooltipValues.append("{}_collection".format(value))
self.config["tooltip"]["value"] = tooltipValues
# set default depth to zero
self.config["depth"] = int(params["depth"]) if "depth" in params else 0
# set default text to "name"
self.config["text"] = params["text"] if "text" in params else "name"
if topic and "cart" in topic:
self.cart = topic["cart"]
else:
self.cart = False
def ranks(self, **kwargs):
ranks = int(kwargs.get("limit", 1))
col = kwargs.get("col")
attr_type = kwargs.get("attr_type", self.attr_type)
params = {}
params[attr_type] = self.attr["id"]
params["required"] = col
params["show"] = kwargs.get("show", self.attr_type)
params["year"] = kwargs.get("year", "latest")
params["sumlevel"] = kwargs.get("sumlevel", self.sumlevel(**kwargs))
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
del params[attr_type]
params["limit"] = 1
params["order"] = col
params["sort"] = "desc"
if rank <= (ranks / 2 + 1):
results = range(1, ranks + 1)
else:
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
max_rank = int(datafold(requests.get(url).json())[0][col])
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
if rank > (max_rank - ranks / 2 - 1):
results = range(max_rank - ranks + 1, max_rank + 1)
else:
results = range(int(math.ceil(rank - ranks / 2)),
int(math.ceil(rank + ranks / 2) + 1))
prev = kwargs.get("prev", False)
next = kwargs.get("next", False)
if prev:
if rank == results[0]:
return "N/A"
else:
results = [results[results.index(rank) - 1]]
if next:
if rank == results[-1]:
return "N/A"
else:
results = [results[results.index(rank) + 1]]
key = kwargs.get("key", False)
if key == "id" or key == "name":
del params["limit"]
params[col] = ",".join([str(r) for r in results])
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
try:
results = datafold(requests.get(url).json())
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return ""
if key == "id":
results = [d[params["show"]] for d in results]
elif key == "name":
return self.make_links([
fetch(d[params["show"]], params["show"]) for d in results
], params["show"])
return ",".join([str(r) for r in results])
def multi_col_top(profile, params):
namespace = params.pop("namespace")
after = params.pop("after", None)
attr_type = params.get("attr_type", profile.attr_type)
rows = params.pop("rows", False)
cols = params.pop("required", [])
children = params.pop("children", False)
params["show"] = params.get("show", attr_type)
params["limit"] = params.get("limit", 1)
params["sumlevel"] = params.get("sumlevel", "all")
params["sort"] = params.get("sort", "desc")
if children:
params["prefix"] = True
params["where"] = "{}:{}".format(attr_type, profile.children(**params))
prefix_pop = params.pop("prefix")
elif attr_type not in params:
params[attr_type] = profile.id(**params)
dataset = params.pop("dataset", False)
params["required"] = ",".join(cols)
pivot = params.pop("pivot", False)
query = RequestEncodingMixin._encode_params(params)
url = u"{}/api?{}".format(API, query)
try:
r = requests.get(url).json()
except ValueError:
app.logger.info("VAR ERROR: {}".format(url))
return {}
headers = r["headers"]
return_obj = {namespace: {} if not rows else []}
if len(r["data"]) == 0:
return return_obj
if pivot:
base_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, "-".join(pivot["keys"]), dataset)
limit = pivot.get("limit", 1)
cols = pivot["cols"]
api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]
if rows:
myobject = {}
for index, data_row in enumerate(api_data):
myobject = {}
headers = data_row.keys()
values = data_row.values()
for col in cols:
stat_url = u"{}&rank={}".format(base_url, index + 1)
myobject[col] = render_col(values, headers, col, stat_url)
return_obj[namespace].append(myobject)
else:
values = api_data[0].values()
headers = api_data[0].keys()
stat_url = u"{}&rank=1".format(base_url)
for col in cols:
return_obj[namespace][col] = render_col(values, headers, col, stat_url, dataset)
elif not rows:
if not r["data"]:
return {}
api_data = r["data"][0]
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, stat_col, dataset)
return_obj[namespace][col] = render_col(api_data, headers, col, stat_url, dataset)
else:
if after:
new_order = after.get("order", None)
if new_order:
new_sort = after.get("sort", "desc")
reverse = True if new_sort == "desc" else False
r["data"] = sorted(r["data"], key=lambda k: k[headers.index(new_order)], reverse=reverse)
for index, data_row in enumerate(r["data"]):
myobject = {}
for col in cols:
format_col = col
if col == params["show"]:
stat_col = "name"
else:
stat_col = col
stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(url_for("profile.statView"), query, stat_col, index + 1, dataset)
myobject[col] = render_col(data_row, headers, col, stat_url, dataset)
return_obj[namespace].append(myobject)
return return_obj
def test_content_md5(self):
data = {'signature': "HPMOHRgPSMKdXrU6AqQs/i9S7alOakkHsJiqLGmInt05Cxj6b/WhS7kJxbIQxKmDW08YKzoFnbVZIoTI2qofEzk="}
assert RequestEncodingMixin._encode_params(data) == "signature=HPMOHRgPSMKdXrU6AqQs%2Fi9S7alOakkHsJiqLGmInt05Cxj6b%2FWhS7kJxbIQxKmDW08YKzoFnbVZIoTI2qofEzk%3D"
assert hashlib.md5(RequestEncodingMixin._encode_params(data).encode("utf-8")).hexdigest() == "fdfc1a717d2c97649f3b8b2142507129"
'force_loging': '0',
'device_model': 'Symphony w68',
'device_platform': 'Android',
'device_uuid': 'b86e58c13a7aae35',
'device_imsi': 'b86e58c13a7aae35',
'device_version': '4.2.2',
'operator': '',
'network_type': 'mobile',
'app_version': '4.0.1'
}
num = raw_input('number: ')
_pass = raw_input('password: '******'conn': str(num), 'password': str(_pass)})
resp = session.post(HOST + LOGIN + time_dc(), data=REM._encode_params(data))
del data['password'], data['force_loging']
data.update({
'session_key': resp.json()['sessionKey'],
'conn_type': resp.json()['connections'][0]['conn_type'],
'operator': resp.json()['connections'][0]['operator'],
'conn': resp.json()['connections'][0]['conn'],
'ref_number': resp.json()['user']['mobile'],
'lang': 'en',
'page': '1',
'start': '0',
'limit': '15'
})
resp = session.post(HOST + PACKAGES + time_dc(), data=data)
del data['page'], data['start'], data['limit']
def _encode_multipart_formdata(fields, files):
body, content_type = RequestEncodingMixin._encode_files(files, fields)
return content_type, body
def aiohttp_payload(data, encoding="utf8"):
payload = RequestEncodingMixin._encode_params(data).encode(encoding)
stream = StreamReader()
stream.feed_data(payload)
stream.feed_eof()
return stream
def stat(params, col="name", dataset=False, data_only=False, moe=False, truncate=0):
# convert request arguments into a url query string
rank = int(params.pop("rank", "1"))
if "limit" in params:
limit = int(params["limit"])
else:
limit = 1
if rank > 1 and limit == 1:
params["limit"] = rank
query = RequestEncodingMixin._encode_params(params)
url = "{}/api?{}".format(API, query)
stat_url = "{}?{}&col={}&dataset={}&moe={}&rank={}".format(url_for("profile.statView"), query, col, dataset, moe, str(rank))
try:
r = requests.get(url).json()
except ValueError:
app.logger.info("STAT ERROR: {}".format(url))
return {
"url": stat_url,
"value": "N/A"
}
if data_only:
return r
else:
r = datafold(r)
if dataset == "stat":
if isinstance(r[0][col], list):
r = [{params["show"]:x} for x in r[0][col]]
col = "name"
if len(r) == 0:
return {
"url": stat_url,
"value": "N/A"
}
# if the output key is 'name', fetch attributes for each return and create an array of 'name' values
# else create an array of the output key for each returned datapoint
vals = []
show = params["show"].split(",")[-1]
if col == "ratio":
if limit == 1:
vals = sorted([v for k, v in r[0].iteritems() if k in params["required"].split(",")], reverse=True)
if vals[0] == 0 or vals[1] == 0:
val = 0
else:
val = vals[0]/vals[1]
return num_format(val, key=col)
else:
denom = max([d[params["order"]] for d in r[1:]])
return num_format(r[0][params["order"]]/denom, key=col)
if col == "diff":
return num_format(r[0][params["order"]] - r[1][params["order"]], key=col)
if col in COLMAP or "-" in col:
vals = datapivot(r, col.split("-"), sort="desc")
vals = [v for v in vals[rank - 1:limit]]
if moe:
top = [v["moe"] for v in vals]
else:
top = [v["name"] for v in vals]
vals = [v["value"] for v in vals]
else:
if rank > 1:
r = r[rank-1:]
if moe:
top = [d[moe] for d in r]
elif col == "name":
if dataset in ["acs", "pums"]:
attr = "{}_{}".format(dataset, show)
else:
attr = show
top = [fetch(d[show], attr) for d in r]
if attr in PROFILES or attr in CROSSWALKS:
top = [(t["id"], t["display_name"] if "display_name" in t else t[col], t["url_name"] if "url_name" in t and t["url_name"] else t["id"]) for t in top]
top = [u"<a href='{}'>{}</a>".format(url_for("profile.profile", attr_type=attr, attr_id=t[2]), t[1]) if attr != "geo" or t[0][:3] != "140" else t[1] for t in top]
else:
top = [t["display_name"] if "display_name" in t else t[col] for t in top]
elif col == "id":
top = [d[show] for d in r]
else:
top = [d[col] for d in r]
if col != "id":
if moe:
col = moe
top = [num_format(t, col) if isinstance(t, (int, float)) else t for t in top]
# coerce all values to strings
top = [u"{}".format(t) if t != "" else u"N/A" for t in top]
if col == "id":
top = u",".join(top)
else:
num_items = len(top)
if truncate and num_items > truncate:
top, rest = top[:int(truncate)], top[int(truncate):]
# now stringify
top = u"{}; <a href='#' class='show-more pri-link'>& {} more</a>".format(u"; ".join(top), len(rest))
if len(rest) > 1:
rest = u"; ".join(rest)
else:
rest = u"and {}".join(rest[-1])
top = u"<span>{}</span><span class='the_rest'>{}</span>".format(top, rest)
else:
if num_items > 1:
top[-1] = u"and {}".format(top[-1])
if num_items == 2:
top = u" ".join(top)
else:
top = u"; ".join(top)
# otherwise, return the list joined with commans
return {
"url": stat_url,
"value": top,
"data": vals
}
def __init__(self, params, highlight=False, profile=False):
"""Initializes a new Viz class.
Args:
config (str): The YAML configuration file as one long string.
profile (Profile): The Profile class instance this Section will be a part of.
"""
self.highlight = params.pop("highlight", highlight)
self.profile = profile.attr
self.profile_type = profile.attr_type
# force the data of params into a list
data = params.pop("data") if isinstance(
params["data"], list) else [params.pop("data")]
# remove sumlevel if it exists
sumlevel = params.pop("sumlevel", None)
# loop through each data and append to self.data
self.data = []
for d in data:
# create a new dict containing the 'split' and 'static' params
data_obj = {
"map": d.pop("map", None),
"split": d.pop("split", None),
"static": d.pop("static", None),
"share": d.pop("share", None)
}
# Set fallback API params
d = param_format(d)
# create the data URL
p = RequestEncodingMixin._encode_params(d)
data_obj["url"] = "{}/api/?{}".format(API, p)
# store the params in the return dict
data_obj["params"] = d
# append the data dict to self.data
self.data.append(data_obj)
self.attrs = []
if "attrs" in params:
# force the attrs of params into a list
attrs = params.pop("attrs") if isinstance(
params["attrs"], list) else [params.pop("attrs")]
# loop through each data and append to self.data
self.attrs = [{
"type": a,
"url": "{}/attrs/{}/".format(API, a)
} for a in attrs]
# set self.config to the params
self.config = params
if "mouse" in params:
if params["mouse"] == "NO":
self.config["mouse"] = False
else:
self.config["mouse"] = True
# set the tooltip config using the function
self.config["tooltip"] = params.pop("tooltip", {})
self.config["tooltip"]["value"] = self.tooltip()
# set default depth to zero
self.config["depth"] = int(params["depth"]) if "depth" in params else 0
# set default text to "name"
self.config["text"] = params["text"] if "text" in params else "name"