Example #1
0
def tileAPI(column):
    url = "{}{}&limit={}".format(API, column["url"], TILEMAX)
    r = requests.get(url).json()

    show = re.findall(r"show=([a-z_0-9]+)", url)[0]
    order = re.findall(r"order=([a-z_0-9]+)", url)[0]
    sumlevel = re.findall(r"sumlevel=([a-z_0-9]+)", url)[0]
    if show == "geo":
        sumlevel = sumlevelMap[sumlevel]
    data = datafold(r)

    for d in data:
        attr_id = d[show]
        attr = fetch(attr_id, show)
        slug = attr["url_name"] if attr["url_name"] else attr_id
        d["title"] = attr["display_name"] if "display_name" in attr else attr[
            "name"]
        # d["subtitle"] = "{}: {}".format(DICTIONARY[order], num_format(d[order], order))
        d["subtitle"] = SUMLEVELS[show][sumlevel]["label"]
        d["link"] = "/profile/{}/{}".format(show, slug)
        d["image"] = "/search/{}/{}/img".format(show, attr_id)
        d["type"] = {
            "icon": "/static/img/icons/{}.svg".format(show),
            "title": SUMLEVELS[show][sumlevel]["label"],
            "type": TYPEMAP[show],
            "depth": "{}".format(sumlevel).replace("_", " ")
        }

    column["icon"] = "/static/img/icons/{}.svg".format(show)
    column["data"] = data
    column["source"] = r["source"]
    if show in footMap:
        column["footer"] = {
            "link": "/search/?kind={}".format(show),
            "text": "{} more".format(num_format(footMap[show]))
        }
    return column
Example #2
0
def stat(params, col="name", dataset=False, data_only=False, moe=False, truncate=0):

    # convert request arguments into a url query string
    rank = int(params.pop("rank", "1"))
    if "limit" in params:
        limit = int(params["limit"])
    else:
        limit = 1
    if rank > 1 and limit == 1:
        params["limit"] = rank
    query = RequestEncodingMixin._encode_params(params)
    url = "{}/api?{}".format(API, query)
    stat_url = "{}?{}&col={}&dataset={}&moe={}&rank={}".format(url_for("profile.statView"), query, col, dataset, moe, str(rank))

    try:
        r = requests.get(url).json()
    except ValueError:
        app.logger.info("STAT ERROR: {}".format(url))
        return {
            "url": stat_url,
            "value": "N/A"
        }

    if data_only:
        return r
    else:
        r = datafold(r)

    if dataset == "stat":
        if isinstance(r[0][col], list):
            r = [{params["show"]:x} for x in r[0][col]]
            col = "name"

    if len(r) == 0:
        return {
            "url": stat_url,
            "value": "N/A"
        }

    # if the output key is 'name', fetch attributes for each return and create an array of 'name' values
    # else create an array of the output key for each returned datapoint
    vals = []
    show = params["show"].split(",")[-1]
    if col == "ratio":
        if limit == 1:
            vals = sorted([v for k, v in r[0].iteritems() if k in params["required"].split(",")], reverse=True)
            if vals[0] == 0 or vals[1] == 0:
                val = 0
            else:
                val = vals[0]/vals[1]
            return num_format(val, key=col)
        else:
            denom = max([d[params["order"]] for d in r[1:]])
            return num_format(r[0][params["order"]]/denom, key=col)


    if col == "diff":
        return num_format(r[0][params["order"]] - r[1][params["order"]], key=col)

    if col in COLMAP or "-" in col:
        vals = datapivot(r, col.split("-"), sort="desc")

        vals = [v for v in vals[rank - 1:limit]]

        if moe:
            top = [v["moe"] for v in vals]
        else:
            top = [v["name"] for v in vals]
            vals = [v["value"] for v in vals]

    else:
        if rank > 1:
            r = r[rank-1:]

        if moe:
            top = [d[moe] for d in r]
        elif col == "name":
            if dataset in ["acs", "pums"]:
                attr = "{}_{}".format(dataset, show)
            else:
                attr = show

            top = [fetch(d[show], attr) for d in r]

            if attr in PROFILES or attr in CROSSWALKS:
                top = [(t["id"], t["display_name"] if "display_name" in t else t[col], t["url_name"] if "url_name" in t and t["url_name"] else t["id"]) for t in top]
                top = [u"<a href='{}'>{}</a>".format(url_for("profile.profile", attr_type=attr, attr_id=t[2]), t[1]) if attr != "geo" or t[0][:3] != "140" else t[1] for t in top]
            else:
                top = [t["display_name"] if "display_name" in t else t[col] for t in top]

        elif col == "id":
            top = [d[show] for d in r]
        else:
            top = [d[col] for d in r]

    if col != "id":
        if moe:
            col = moe
        top = [num_format(t, col) if isinstance(t, (int, float)) else t for t in top]

    # coerce all values to strings
    top = [u"{}".format(t) if t != "" else u"N/A" for t in top]

    if col == "id":
        top = u",".join(top)
    else:
        num_items = len(top)

        if truncate and num_items > truncate:
            top, rest = top[:int(truncate)], top[int(truncate):]
            # now stringify
            top = u"{}; <a href='#' class='show-more pri-link'>& {} more</a>".format(u"; ".join(top), len(rest))
            if len(rest) > 1:
                rest = u"; ".join(rest)
            else:
                rest = u"and {}".join(rest[-1])
            top = u"<span>{}</span><span class='the_rest'>{}</span>".format(top, rest)

        else:
            if num_items > 1:
                top[-1] = u"and {}".format(top[-1])
            if num_items == 2:
                top = u" ".join(top)
            else:
                top = u"; ".join(top)

    # otherwise, return the list joined with commans
    return {
        "url": stat_url,
        "value": top,
        "data": vals
    }
Example #3
0
def multi_col_top(profile, params):
    namespace = params.pop("namespace")
    after = params.pop("after", None)
    attr_type = params.get("attr_type", profile.attr_type)
    rows = params.pop("rows", False)
    cols = params.pop("required", [])
    children = params.pop("children", False)
    params["show"] = params.get("show", attr_type)
    params["limit"] = params.get("limit", 1)
    params["sumlevel"] = params.get("sumlevel", "all")
    params["sort"] = params.get("sort", "desc")

    if children:
        params["prefix"] = True
        params["where"] = "{}:{}".format(attr_type, profile.children(**params))
        prefix_pop = params.pop("prefix")
    elif attr_type not in params:
        params[attr_type] = profile.id(**params)

    dataset = params.pop("dataset", False)
    params["required"] = ",".join(cols)
    pivot = params.pop("pivot", False)
    query = RequestEncodingMixin._encode_params(params)
    url = u"{}/api?{}".format(API, query)

    try:
        r = requests.get(url).json()
    except ValueError:
        app.logger.info("VAR ERROR: {}".format(url))
        return {}

    headers = r["headers"]
    return_obj = {namespace: {} if not rows else []}

    if len(r["data"]) == 0:
        return return_obj

    if pivot:

        base_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, "-".join(pivot["keys"]), dataset)

        limit = pivot.get("limit", 1)
        cols = pivot["cols"]
        api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]

        if rows:
            myobject = {}
            for index, data_row in enumerate(api_data):
                myobject = {}
                headers = data_row.keys()
                values = data_row.values()
                for col in cols:
                    stat_url = u"{}&rank={}".format(base_url, index + 1)
                    myobject[col] = render_col(values, headers, col, stat_url)
                return_obj[namespace].append(myobject)
        else:
            values = api_data[0].values()
            headers = api_data[0].keys()
            stat_url = u"{}&rank=1".format(base_url)
            for col in cols:
                return_obj[namespace][col] = render_col(values, headers, col, stat_url, dataset)

    elif not rows:
        if not r["data"]:
            return {}
        api_data = r["data"][0]
        for col in cols:
            format_col = col
            if col == params["show"]:
                stat_col = "name"
            else:
                stat_col = col
            stat_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, stat_col, dataset)
            return_obj[namespace][col] = render_col(api_data, headers, col, stat_url, dataset)
    else:
        if after:
            new_order = after.get("order", None)
            if new_order:
                new_sort = after.get("sort", "desc")
                reverse = True if new_sort == "desc" else False
                r["data"] = sorted(r["data"], key=lambda k: k[headers.index(new_order)], reverse=reverse)
        for index, data_row in enumerate(r["data"]):
            myobject = {}
            for col in cols:
                format_col = col
                if col == params["show"]:
                    stat_col = "name"
                else:
                    stat_col = col
                stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(url_for("profile.statView"), query, stat_col, index + 1, dataset)
                myobject[col] = render_col(data_row, headers, col, stat_url, dataset)
            return_obj[namespace].append(myobject)

    return return_obj
Example #4
0
def multi_col_top(profile, params):
    namespace = params.pop("namespace")
    after = params.pop("after", None)
    attr_type = params.get("attr_type", profile.attr_type)
    rows = params.pop("rows", False)
    cols = params.pop("required", [])
    children = params.pop("children", False)
    params["show"] = params.get("show", attr_type)
    params["limit"] = params.get("limit", 1)
    params["sumlevel"] = params.get("sumlevel", "all")
    params["sort"] = params.get("sort", "desc")

    if children:
        params["prefix"] = True
        params["where"] = "{}:{}".format(attr_type, profile.children(**params))
        prefix_pop = params.pop("prefix")
    elif attr_type not in params:
        params[attr_type] = profile.id(**params)

    dataset = params.pop("dataset", False)
    params["required"] = ",".join(cols)
    pivot = params.pop("pivot", False)
    query = RequestEncodingMixin._encode_params(params)
    url = u"{}/api?{}".format(API, query)

    try:
        r = requests.get(url).json()
    except ValueError:
        app.logger.info("VAR ERROR: {}".format(url))
        return {}

    return_obj = {namespace: {} if not rows else []}

    if "error" in r or len(r["data"]) == 0:
        return return_obj

    headers = r["headers"]

    if pivot:

        base_url = u"{}?{}&col={}&dataset={}".format(
            url_for("profile.statView"), query, "-".join(pivot["keys"]),
            dataset)

        limit = pivot.get("limit", 1)
        cols = pivot["cols"]
        api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]

        if rows:
            myobject = {}
            for index, data_row in enumerate(api_data):
                myobject = {}
                headers = data_row.keys()
                values = data_row.values()
                for col in cols:
                    stat_url = u"{}&rank={}".format(base_url, index + 1)
                    myobject[col] = render_col(values, headers, col, stat_url)
                return_obj[namespace].append(myobject)
        else:
            values = api_data[0].values()
            headers = api_data[0].keys()
            stat_url = u"{}&rank=1".format(base_url)
            for col in cols:
                return_obj[namespace][col] = render_col(
                    values, headers, col, stat_url, dataset)

    elif not rows:
        if not r["data"]:
            return {}
        api_data = r["data"][0]
        for col in cols:
            format_col = col
            if col == params["show"]:
                stat_col = "name"
            else:
                stat_col = col
            stat_url = u"{}?{}&col={}&dataset={}".format(
                url_for("profile.statView"), query, stat_col, dataset)
            return_obj[namespace][col] = render_col(api_data, headers, col,
                                                    stat_url, dataset)
    else:
        if after:
            new_order = after.get("order", None)
            if new_order:
                new_sort = after.get("sort", "desc")
                mute = after.get("mute", [])
                reverse = True if new_sort == "desc" else False
                show = params["show"].split(",")[0]
                r["data"] = sorted(r["data"],
                                   key=lambda k: 0 if k[headers.index(show)] in
                                   mute else k[headers.index(new_order)],
                                   reverse=reverse)
            aggregate = after.get("aggregate", None)
            if aggregate:
                method = after.get("method", "sum")
                myobject = {}
                for row in r["data"]:
                    aggid = row[headers.index(aggregate)]
                    if aggid in myobject:
                        prev = myobject[aggid]
                    else:
                        prev = [None] * len(headers)
                    for col in headers:
                        i = headers.index(col)
                        if prev[i] == None or isinstance(
                                row[i], basestring
                        ) or col == aggregate or col == "year":
                            prev[i] = row[i]
                        elif col == aggregate or col:
                            if method == "max":
                                prev[i] = max([prev[i], row[i]])
                            else:
                                prev[i] = prev[i] + row[i]
                    myobject[aggid] = prev
                reverse = False if params["sort"] == "asc" else True
                r["data"] = sorted(
                    [myobject[k] for k in myobject],
                    key=lambda k: k[headers.index(params["order"])],
                    reverse=reverse)
        for index, data_row in enumerate(r["data"]):
            myobject = {}
            for col in cols:
                format_col = col
                if col == params["show"]:
                    stat_col = "name"
                else:
                    stat_col = col
                stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(
                    url_for("profile.statView"), query, stat_col, index + 1,
                    dataset)
                myobject[col] = render_col(data_row, headers, col, stat_url,
                                           dataset)
            return_obj[namespace].append(myobject)

    return return_obj
Example #5
0
def multi_col_top(profile, params):
    namespace = params.pop("namespace")
    after = params.pop("after", None)
    attr_type = params.get("attr_type", profile.attr_type)
    rows = params.pop("rows", False)
    cols = params.pop("required", [])
    children = params.pop("children", False)
    params["show"] = params.get("show", attr_type)
    params["limit"] = params.get("limit", 1)
    params["sumlevel"] = params.get("sumlevel", "all")
    params["sort"] = params.get("sort", "desc")

    if children:
        params["prefix"] = True
        params["where"] = "{}:{}".format(attr_type, profile.children(**params))
        prefix_pop = params.pop("prefix")
    elif attr_type not in params:
        params[attr_type] = profile.id(**params)

    dataset = params.pop("dataset", False)
    params["required"] = ",".join(cols)
    pivot = params.pop("pivot", False)
    query = RequestEncodingMixin._encode_params(params)
    url = u"{}/api?{}".format(API, query)

    try:
        r = requests.get(url).json()
    except ValueError:
        app.logger.info("VAR ERROR: {}".format(url))
        return {}

    return_obj = {namespace: {} if not rows else []}

    if "error" in r or len(r["data"]) == 0:
        return return_obj

    headers = r["headers"]

    if pivot:

        base_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, "-".join(pivot["keys"]), dataset)

        limit = pivot.get("limit", 1)
        cols = pivot["cols"]
        api_data = datapivot(datafold(r)[0], pivot["keys"])[:limit]

        if rows:
            myobject = {}
            for index, data_row in enumerate(api_data):
                myobject = {}
                headers = data_row.keys()
                values = data_row.values()
                for col in cols:
                    stat_url = u"{}&rank={}".format(base_url, index + 1)
                    myobject[col] = render_col(values, headers, col, stat_url)
                return_obj[namespace].append(myobject)
        else:
            values = api_data[0].values()
            headers = api_data[0].keys()
            stat_url = u"{}&rank=1".format(base_url)
            for col in cols:
                return_obj[namespace][col] = render_col(values, headers, col, stat_url, dataset)

    elif not rows:
        if not r["data"]:
            return {}
        api_data = r["data"][0]
        for col in cols:
            format_col = col
            if col == params["show"]:
                stat_col = "name"
            else:
                stat_col = col
            stat_url = u"{}?{}&col={}&dataset={}".format(url_for("profile.statView"), query, stat_col, dataset)
            return_obj[namespace][col] = render_col(api_data, headers, col, stat_url, dataset)
    else:
        if after:
            new_order = after.get("order", None)
            if new_order:
                new_sort = after.get("sort", "desc")
                reverse = True if new_sort == "desc" else False
                r["data"] = sorted(r["data"], key=lambda k: k[headers.index(new_order)], reverse=reverse)
        for index, data_row in enumerate(r["data"]):
            myobject = {}
            for col in cols:
                format_col = col
                if col == params["show"]:
                    stat_col = "name"
                else:
                    stat_col = col
                stat_url = u"{}?{}&col={}&rank={}&dataset={}".format(url_for("profile.statView"), query, stat_col, index + 1, dataset)
                myobject[col] = render_col(data_row, headers, col, stat_url, dataset)
            return_obj[namespace].append(myobject)

    return return_obj