def api_dropdown_products(request, product_class="hs4"): """API to dynamically fill in a product dropdown, product name to code. Can also set lang=foo to get a specific language, but it'll default to the django user locale. """ lang = helpers.get_language(request)['code'] if product_class == "sitc4": products = Sitc4.objects.get_all(lang) elif product_class == "hs4": products = Hs4.objects.get_all(lang) products = [x for x in products if len(x["code"]) > 3] return HttpResponse(json.dumps([(p["name"], p["code"]) for p in products]), content_type="application/json")
def api_dropdown_countries(request): """API to dynamically fill in a country dropdown, product name to code. Can also set lang=foo to get a specific language, but it'll default to the django user locale. """ lang = helpers.get_language(request)['code'] countries = Country.objects\ .filter_lang(lang)\ .values("name", "name_3char") country_info = list( (c["name"], c["name_3char"].lower()) for c in countries) return HttpResponse(json.dumps(country_info), content_type="application/json")
def api_dropdown_countries(request): """API to dynamically fill in a country dropdown, product name to code. Can also set lang=foo to get a specific language, but it'll default to the django user locale. """ lang = helpers.get_language(request)['code'] countries = Country.objects\ .filter_lang(lang)\ .values("name", "name_3char") country_info = list((c["name"], c["name_3char"].lower()) for c in countries) return HttpResponse(json.dumps(country_info), content_type="application/json")
def api_casy(request, trade_flow, country1, year): """<COUNTRY> / all / show / <YEAR>""" # Get session / request vars prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET country1 = Country.objects.get(name_3char=country1) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class world_trade = helpers.get_world_trade(prod_class=prod_class) attr = helpers.get_attrs(prod_class=prod_class, name=name) years_available = helpers.get_years_available(prod_class=prod_class) magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) if prod_class == "sitc4": items = Sitc4_cpy.objects else: items = Hs4_cpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = calculate_rca(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list( 'year', 'product__id', 'product__code', 'product__name', 'product__community_id', 'product__community__color', 'product__community__name', 'val', 'rca', 'distance', 'opp_gain', 'product_year__pci', ) if single_year: items = items.filter(year=year) items = items.filter(country_id=country1.id) filter_expr = "%s > 0" % trade_type(trade_flow) items = items.extra(where=[filter_expr]) json_response = {} # Generate cache key key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class, trade_flow) if single_year: key += ":%d" % int(year) # Check cache cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[7] for r in rows]) rows = [{ "year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[7], "rca": r[8], "distance": r[9], "opp_gain": r[10], "pci": r[11], "share": (r[7] / total_val) * 100, "community_id": r[4], "color": r[5], "community_name": r[6], "code": r[2], "id": r[2] } for r in rows] # Save in cache cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows # Add in remaining metadata json_response["attr"] = attr json_response["attr_data"] = Sitc4.objects.get_all( lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) json_response["country1"] = country1.to_json() json_response["title"] = "What does %s %s?" % ( country1.name, trade_flow.replace("_", " ")) json_response["year"] = year json_response["item_type"] = "product" json_response["app_type"] = "casy" json_response["magic_numbers"] = magic_numbers json_response["world_trade"] = world_trade json_response["prod_class"] = prod_class json_response["other"] = query_params # Check the request data type if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): return HttpResponse(json.dumps(json_response))
def api_cspy(request, trade_flow, country1, product, year): """ country / show / product / year """ prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang product = helpers.get_product_by_code(product, prod_class) country1 = Country.objects.get(name_3char=country1) single_year = "single_year" in request.GET query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class region = helpers.get_region_list() continents = helpers.get_continent_list() if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'destination__id', 'destination__name_3char', 'name', 'destination__region_id', 'destination__continent', 'val') items = items.filter(origin_id=country1.id, product_id=product.id) if single_year: items = items.filter(year=year) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id, prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response['data'] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[6] for r in rows]) rows = [{ "year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[6], "share": (r[6] / total_val) * 100, "region_id": r[4], "continent": r[5], "id": r[1] } for r in rows] json_response["data"] = rows cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) years_available = helpers.get_years_available(prod_class=prod_class) inflation_adjustment = helpers.get_inflation_adjustment( country1, years_available[0], years_available[-1]) json_response["magic_numbers"] = inflation_adjustment json_response["attr_data"] = Country.objects.get_all(lang) article = "to" if trade_flow == "export" else "from" json_response["title"] = "Where does %s %s %s %s?" % ( country1.name, trade_flow, product.name_en, article) json_response["country1"] = country1.to_json() json_response["product"] = product.to_json() json_response["year"] = year json_response["item_type"] = "country" json_response["continents"] = continents json_response["region"] = region json_response["app_type"] = "cspy" json_response["class"] = prod_class json_response["other"] = query_params if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): return HttpResponse(json.dumps(json_response))
def api_ccsy(request, trade_flow, country1, country2, year): # Get session / request vars prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET country1 = Country.objects.get(name_3char=country1) country2 = Country.objects.get(name_3char=country2) article = "to" if trade_flow == "export" else "from" query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class attr = helpers.get_attrs(prod_class=prod_class, name=name) if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'product__id', 'product__code', 'name', 'product__community_id', 'product__community__name', 'product__community__color', 'val') if single_year: items = items.filter(year=year) items = items.filter(origin_id=country1.id, destination_id=country2.id) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show", prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[7] for r in rows]) """Add percentage value to return vals""" rows = [{ "year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[7], "share": (r[7] / total_val) * 100, "community_id": r[4], "community_name": r[5], "color": r[6], "code": r[2], "id": r[2] } for r in rows] json_response["data"] = rows cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) years_available = helpers.get_years_available(prod_class=prod_class) magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) json_response["magic_numbers"] = magic_numbers json_response["attr_data"] = Sitc4.objects.get_all( lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) json_response["country1"] = country1.to_json() json_response["country2"] = country2.to_json() json_response["title"] = "What does %s %s %s %s?" % ( country1.name, trade_flow, article, country2.name) json_response["year"] = year json_response["item_type"] = "product" json_response["app_type"] = "ccsy" json_response["prod_class"] = prod_class json_response["attr"] = attr json_response["class"] = prod_class json_response["other"] = query_params # Check the request data type if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def api_csay(request, trade_flow, country1, year): """<COUNTRY> / show / all / <YEAR>""" prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang country1 = Country.objects.get(name_3char=country1) region = helpers.get_region_list() continents = helpers.get_continent_list() if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow=trade_flow, sum_val=True) items = items.extra(select={'name': name}) items = items.values_list('year', 'destination__id', 'destination__name_3char', 'name', 'destination__region_id', 'destination__continent', 'val') items = items.filter(origin_id=country1.id) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: # This might possibly be the most disgusting hack ever made, simply # because when doing an aggregate (like SUM()) in extra, django does # not add that stuff correctly into the group by. It's also not # possible to use annotate() here because it's a complex aggregate that # uses addition / subtraction. C'est la vie :( cursor = connection.cursor() if trade_flow == "net_export": val = '`export_value` - `import_value`' elif trade_flow == "net_import": val = '`import_value` - `export_value`' elif trade_flow == "export": val = '`export_value`' else: val = '`import_value`' complete_query = str(items.query) + \ " group by `year`, `destination_id`" + \ " HAVING sum(%s) > 0" % val cursor.execute(complete_query) rows = cursor.fetchall() total_val = sum([r[0] for r in rows]) rows = [{ "year": r[2], "item_id": r[3], "abbrv": r[4], "name": r[1], "value": r[0], "share": (r[0] / total_val) * 100, "id": r[3], "region_id": r[5], "continent": r[6] } for r in rows] cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows years_available = helpers.get_years_available() magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class json_response["attr_data"] = Country.objects.get_all(lang) json_response["country1"] = country1.to_json() article = "to" if trade_flow == "export" else "from" json_response["title"] = "Where does %s %s %s?" % (country1.name, trade_flow, article) json_response["year"] = year json_response["item_type"] = "country" json_response["app_type"] = "csay" json_response["region"] = region json_response["continents"] = continents json_response["prod_class"] = prod_class json_response["magic_numbers"] = magic_numbers json_response["other"] = query_params # raise Exception(time.time() - start) # Check the request data type if (request.GET.get('data_type', None) is None): #"""Return to browser as JSON for AJAX request""" return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def api_sapy(request, trade_flow, product, year): """show / all / <product> / <year>""" # Get session / request vars prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET product = helpers.get_product_by_code(product, prod_class) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class region = helpers.get_region_list() continents = helpers.get_continent_list() attr = helpers.get_attrs(prod_class=prod_class, name=name) if prod_class == "sitc4": items = Sitc4_cpy.objects else: items = Hs4_cpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = calculate_rca(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'country__id', 'country__name_3char', 'name', 'country__region_id', 'country__continent', 'val', 'rca') if single_year: items = items.filter(year=year) items = items.filter(product_id=product.id) json_response = {} # Generate cache key key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class, trade_flow) if single_year: key += ":%d" % int(year) # Check cache cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[6] for r in rows]) rows = [{ "year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[6], "rca": r[7], "share": (r[6] / total_val) * 100, "id": r[1], "region_id": r[4], "continent": r[5] } for r in rows] # Save in cache cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows json_response["attr_data"] = Country.objects.get_all(lang) json_response["product"] = product.to_json() json_response["title"] = "Who %sed %s?" % (trade_flow.replace( "_", " "), product.name_en) json_response["year"] = year json_response["item_type"] = "country" json_response["app_type"] = "sapy" json_response["region"] = region json_response["continents"] = continents json_response["other"] = query_params json_response["prod_class"] = prod_class if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def explore( request, app_name, trade_flow, country1, country2, product, year="2011"): request.session['app_name'] = app_name was_redirected = request.GET.get("redirect", False) lang = helpers.get_language(request)['code'] crawler = request.GET.get("_escaped_fragment_", False) # Get session / request vars prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) options = request.GET.copy() options["lang"] = lang options["product_classification"] = prod_class options = options.urlencode() # Setup the hash dictionary request_hash_dictionary = collections.OrderedDict() # Add prod class to request hash dictionary request_hash_dictionary['app_name'] = app_name request_hash_dictionary['lang'] = lang request_hash_dictionary['prod_class'] = prod_class # Add the arguments to the request hash dictionary request_hash_dictionary['trade_flow'] = trade_flow request_hash_dictionary['country1'] = country1 request_hash_dictionary['country2'] = country2 request_hash_dictionary['product_type'] = product request_hash_dictionary['year'] = year # We are here, so let us store this data somewhere request_hash_string = "_".join(request_hash_dictionary.values()) # Code for showing a static image or not static_image_name = helpers.url_to_hash(request.path, request.GET) if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH, static_image_name + ".png")): displayviz = True displayImage = static_image_name + ".png" else: displayviz = False displayImage = settings.STATIC_URL + "img/all/loader.gif" # Verify countries from DB countries = [None, None] alert = None for i, country in enumerate([country1, country2]): if country != "show" and country != "all": try: countries[i] = Country.objects.get(name_3char=country) except Country.DoesNotExist: alert = { "title": "Country could not be found", "text": """There was no country with the 3 letter abbreviation <strong>%s</strong>. Please double check the <a href='about/data/country/'>list of countries</a>.""" % (country)} # The years of data available tends to vary based on the dataset used (Hs4 # vs Sitc4) and the specific country. years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy years_available = years_available_model.objects\ .values_list("year", flat=True)\ .order_by("year")\ .distinct() # Sometimes the query is not about a specific country (e.g. "all countries" # queries) in which case filtering by country is not necessary if countries[0]: years_available = years_available.filter(country=countries[0].id) # Force lazy queryset to hit the DB to reduce number of DB queries later years_available = list(years_available) if len(years_available) == 0: alert = {"title": """The product classification you're using (%s) does not seem to include data for the country code (%s) you selected. Please try a different product classification or country.""" % (prod_class, countries[0].name)} years_available = range(1995, 2012) # Dummy year1_list, year2_list = None, None warning, title = None, None data_as_text = {} prod_or_partner = "partner" # To make sure it cannot be another product class if prod_class != "hs4" and prod_class != "sitc4": prod_class = "sitc4" # Test for country exceptions if prod_class == "hs4": # redirect if and exception country if country1 == "bel" or country1 == "lux": return redirect( HTTP_HOST+'explore/%s/%s/blx/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz": return redirect( HTTP_HOST+'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) if was_redirected: # display warning is redirected from exception if country1 == "blx": warning = { "title": "Country Substitution", "text": "In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported as 'Belgium-Luxembourg'."} if country1 == "zaf": warning = { "title": "Country Substitution", "text": "In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."} trade_flow_list = [ ("export", _("Export")), ("import", _("Import")), ("net_export", _("Net Export")), ("net_import", _("Net Import"))] if (app_name == "product_space" or app_name == "country_space" or app_name == "rings"): trade_flow_list = [trade_flow_list[0]] year1_list = range( years_available[0], years_available[ len(years_available) - 1] + 1, 1) if app_name == "stacked" and year == "2009": year = "1969.2011.10" if "." in year: y = [int(x) for x in year.split(".")] year_start = y[0] year_end = y[1] year2_list = year1_list else: year_start, year_end = None, None year = int(year) # Check that year is within bounds if year > years_available[len(years_available)-1]: year = years_available[len(years_available)-1] elif year < years_available[0]: year = years_available[0] api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2, product, year, options) redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class, trade_flow, country1, country2, product, year) country_code = None if country1 != "show" and country1 != "all": country_code = country1 if crawler == "": view, args, kwargs = resolve( "api/%s/%s/%s/%s/%s/" % (trade_flow, country1, country2, product, year)) kwargs['request'] = request view_response = view(*args, **kwargs) raise Exception(view_response) data_as_text["data"] = view_response[0] data_as_text["total_value"] = view_response[1] data_as_text["columns"] = view_response[2] app_type = helpers.get_app_type(country1, country2, product, year) # What is actually being shown on the page if app_type == "csay" or app_type == "sapy": item_type = "country" else: item_type = "product" # Some countries need "the" before their names list_countries_the = set( ("Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States")) if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name if product not in ("show", "all"): p_code = product product = helpers.get_product_by_code(p_code, prod_class) if not alert: # Generate page title depending on visualization being used years = [year_start, year_end] if year_start is not None else [year] product_name = product.name_en if not isinstance( product, basestring) else product country_names = [getattr(x, "name", None) for x in countries] title = helpers.get_title(app_type, app_name, country_names=country_names, trade_flow=trade_flow.replace('_', ' '), years=years, product_name=product_name ) if app_type in ("ccsy", "cspy"): if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] # Should we show the product or partner tab pane? # quick fix should be merged with item_type if app_type in ["cspy", "sapy"]: prod_or_partner = "product" elif app_type == "casy": if app_name in ("stacked", "map", "tree_map", "pie_scatter", "product_space", "country_space", "rankings", "scatterplot"): prod_or_partner = "product" # Record views in redis for "newest viewed pages" visualization if isinstance(cache, RedisCache): views_image_path = settings.STATIC_URL + \ "data/" + request_hash_string + ".png" view_data = { "timestamp": time.time(), "image": views_image_path, "title": title, "url": request.build_absolute_uri() } r = cache.raw_client r.rpush("views", json.dumps(view_data)) previous_page = request.META.get('HTTP_REFERER', None), if previous_page[0] is not None: previous_page = previous_page[0] previous_image = helpers.url_to_hash(urlparse(previous_page).path, {}) if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH, previous_image + ".png")): previous_image = previous_image + ".png" else: previous_image = settings.STATIC_URL + "img/all/loader.gif" else: previous_image = settings.STATIC_URL + "img/all/loader.gif" previous_page = None return render_to_response( "explore/index.html", {"lang": lang, "warning": warning, "alert": alert, "prod_class": prod_class, "data_as_text": data_as_text, "app_name": app_name, "title": title, "trade_flow": trade_flow, "country1": countries[0] or country1, "country2": countries[1] or country2, "country1_3char": countries[0].name_3char if countries[0] else "", "country2_3char": countries[1].name_3char if countries[1] else "", "product": product, "product_code": product.code if not isinstance(product, basestring) else product, "years_available": years_available, "year": year, "year_start": year_start, "year_end": year_end, "year1_list": year1_list, "year2_list": year2_list, "trade_flow_list": trade_flow_list, "api_uri": api_uri, "app_type": app_type, "redesign_api_uri": redesign_api_uri, "country_code": country_code, "prod_or_partner": prod_or_partner, "version": VERSION, "previous_page": previous_page, "previous_image": previous_image, "item_type": item_type, "displayviz": displayviz, "displayImage": displayImage, "display_iframe": request.GET.get("display_iframe", False), "static_image": request.GET.get("static_image", "loading"), }, context_instance=RequestContext(request))
def api_casy(request, trade_flow, country1, year): """<COUNTRY> / all / show / <YEAR>""" # Get session / request vars prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET country1 = Country.objects.get(name_3char=country1) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class world_trade = helpers.get_world_trade(prod_class=prod_class) attr = helpers.get_attrs(prod_class=prod_class, name=name) years_available = helpers.get_years_available(prod_class=prod_class) magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) if prod_class == "sitc4": items = Sitc4_cpy.objects else: items = Hs4_cpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = calculate_rca(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'product__id', 'product__code', 'product__name', 'product__community_id', 'product__community__color', 'product__community__name', 'val', 'rca', 'distance', 'opp_gain', 'product_year__pci',) if single_year: items = items.filter(year=year) items = items.filter(country_id=country1.id) filter_expr = "%s > 0" % trade_type(trade_flow) items = items.extra(where=[filter_expr]) json_response = {} # Generate cache key key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class, trade_flow) if single_year: key += ":%d" % int(year) # Check cache cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[7] for r in rows]) rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[7], "rca": r[8], "distance": r[9], "opp_gain": r[10], "pci": r[11], "share": (r[7] / total_val) * 100, "community_id": r[4], "color": r[5], "community_name": r[6], "code": r[2], "id": r[2]} for r in rows] # Save in cache cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows # Add in remaining metadata json_response["attr"] = attr json_response["attr_data"] = Sitc4.objects.get_all( lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) json_response["country1"] = country1.to_json() json_response["title"] = "What does %s %s?" % ( country1.name, trade_flow.replace("_", " ")) json_response["year"] = year json_response["item_type"] = "product" json_response["app_type"] = "casy" json_response["magic_numbers"] = magic_numbers json_response["world_trade"] = world_trade json_response["prod_class"] = prod_class json_response["other"] = query_params # Check the request data type if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): return HttpResponse(json.dumps(json_response))
def api_cspy(request, trade_flow, country1, product, year): """ country / show / product / year """ prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang product = helpers.get_product_by_code(product, prod_class) country1 = Country.objects.get(name_3char=country1) single_year = "single_year" in request.GET query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class region = helpers.get_region_list() continents = helpers.get_continent_list() if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'destination__id', 'destination__name_3char', 'name', 'destination__region_id', 'destination__continent', 'val') items = items.filter(origin_id=country1.id, product_id=product.id) if single_year: items = items.filter(year=year) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id, prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response['data'] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[6] for r in rows]) rows = [ {"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[6], "share": (r[6] / total_val) * 100, "region_id": r[4], "continent": r[5], "id": r[1]} for r in rows] json_response["data"] = rows cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) years_available = helpers.get_years_available(prod_class=prod_class) inflation_adjustment = helpers.get_inflation_adjustment( country1, years_available[0], years_available[-1]) json_response["magic_numbers"] = inflation_adjustment json_response["attr_data"] = Country.objects.get_all(lang) article = "to" if trade_flow == "export" else "from" json_response["title"] = "Where does %s %s %s %s?" % ( country1.name, trade_flow, product.name_en, article) json_response["country1"] = country1.to_json() json_response["product"] = product.to_json() json_response["year"] = year json_response["item_type"] = "country" json_response["continents"] = continents json_response["region"] = region json_response["app_type"] = "cspy" json_response["class"] = prod_class json_response["other"] = query_params if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): return HttpResponse(json.dumps(json_response))
def api_ccsy(request, trade_flow, country1, country2, year): # Get session / request vars prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET country1 = Country.objects.get(name_3char=country1) country2 = Country.objects.get(name_3char=country2) article = "to" if trade_flow == "export" else "from" query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class attr = helpers.get_attrs(prod_class=prod_class, name=name) if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'product__id', 'product__code', 'name', 'product__community_id', 'product__community__name', 'product__community__color', 'val') if single_year: items = items.filter(year=year) items = items.filter(origin_id=country1.id, destination_id=country2.id) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show", prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[7] for r in rows]) """Add percentage value to return vals""" rows = [{"year": r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "share": (r[7] / total_val)*100, "community_id":r[4], "community_name":r[5], "color":r[6], "code":r[2], "id": r[2]} for r in rows] json_response["data"] = rows cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) years_available = helpers.get_years_available(prod_class=prod_class) magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) json_response["magic_numbers"] = magic_numbers json_response["attr_data"] = Sitc4.objects.get_all( lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) json_response["country1"] = country1.to_json() json_response["country2"] = country2.to_json() json_response["title"] = "What does %s %s %s %s?" % ( country1.name, trade_flow, article, country2.name) json_response["year"] = year json_response["item_type"] = "product" json_response["app_type"] = "ccsy" json_response["prod_class"] = prod_class json_response["attr"] = attr json_response["class"] = prod_class json_response["other"] = query_params # Check the request data type if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def api_csay(request, trade_flow, country1, year): """<COUNTRY> / show / all / <YEAR>""" prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang country1 = Country.objects.get(name_3char=country1) region = helpers.get_region_list() continents = helpers.get_continent_list() if prod_class == "sitc4": items = Sitc4_ccpy.objects else: items = Hs4_ccpy.objects items = calculate_volume(items, trade_flow=trade_flow, sum_val=True) items = items.extra(select={'name': name}) items = items.values_list('year', 'destination__id', 'destination__name_3char', 'name', 'destination__region_id', 'destination__continent', 'val') items = items.filter(origin_id=country1.id) json_response = {} key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class, trade_flow) cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: # This might possibly be the most disgusting hack ever made, simply # because when doing an aggregate (like SUM()) in extra, django does # not add that stuff correctly into the group by. It's also not # possible to use annotate() here because it's a complex aggregate that # uses addition / subtraction. C'est la vie :( cursor = connection.cursor() if trade_flow == "net_export": val = '`export_value` - `import_value`' elif trade_flow == "net_import": val = '`import_value` - `export_value`' elif trade_flow == "export": val = '`export_value`' else: val = '`import_value`' complete_query = str(items.query) + \ " group by `year`, `destination_id`" + \ " HAVING sum(%s) > 0" % val cursor.execute(complete_query) rows = cursor.fetchall() total_val = sum([r[0] for r in rows]) rows = [ {"year": r[2], "item_id": r[3], "abbrv": r[4], "name": r[1], "value": r[0], "share": (r[0] / total_val) * 100, "id": r[3], "region_id": r[5], "continent": r[6]} for r in rows] cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows years_available = helpers.get_years_available() magic_numbers = helpers.get_inflation_adjustment(country1, years_available[0], years_available[-1]) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class json_response["attr_data"] = Country.objects.get_all(lang) json_response["country1"] = country1.to_json() article = "to" if trade_flow == "export" else "from" json_response["title"] = "Where does %s %s %s?" % ( country1.name, trade_flow, article) json_response["year"] = year json_response["item_type"] = "country" json_response["app_type"] = "csay" json_response["region"] = region json_response["continents"] = continents json_response["prod_class"] = prod_class json_response["magic_numbers"] = magic_numbers json_response["other"] = query_params # raise Exception(time.time() - start) # Check the request data type if (request.GET.get('data_type', None) is None): #"""Return to browser as JSON for AJAX request""" return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def api_sapy(request, trade_flow, product, year): """show / all / <product> / <year>""" # Get session / request vars prod_class = request.GET.get("prod_class", request.session.get('product_classification', 'hs4')) lang = helpers.get_language(request)['code'] name = "name_%s" % lang single_year = 'single_year' in request.GET product = helpers.get_product_by_code(product, prod_class) query_params = request.GET.copy() query_params["lang"] = lang query_params["product_classification"] = prod_class region = helpers.get_region_list() continents = helpers.get_continent_list() attr = helpers.get_attrs(prod_class=prod_class, name=name) if prod_class == "sitc4": items = Sitc4_cpy.objects else: items = Hs4_cpy.objects items = calculate_volume(items, trade_flow=trade_flow) items = calculate_rca(items, trade_flow=trade_flow) items = items.extra(select={'name': name}) items = items.values_list('year', 'country__id', 'country__name_3char', 'name', 'country__region_id', 'country__continent', 'val', 'rca') if single_year: items = items.filter(year=year) items = items.filter(product_id=product.id) json_response = {} # Generate cache key key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class, trade_flow) if single_year: key += ":%d" % int(year) # Check cache cached_data = cache.get(key) if cached_data is not None: json_response["data"] = msgpack.loads(cached_data) else: rows = list(items) total_val = sum([r[6] for r in rows]) rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], "value": r[6], "rca": r[7], "share": (r[6] / total_val) * 100, "id": r[1], "region_id": r[4], "continent": r[5]} for r in rows] # Save in cache cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) json_response["data"] = rows json_response["attr_data"] = Country.objects.get_all(lang) json_response["product"] = product.to_json() json_response["title"] = "Who %sed %s?" % ( trade_flow.replace("_", " "), product.name_en) json_response["year"] = year json_response["item_type"] = "country" json_response["app_type"] = "sapy" json_response["region"] = region json_response["continents"] = continents json_response["other"] = query_params json_response["prod_class"] = prod_class if (request.GET.get('data_type', None) is None): return HttpResponse("") elif (request.GET.get('data_type', '') == 'json'): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response))
def explore(request, app_name, trade_flow, country1, country2, product, year="2011"): request.session['app_name'] = app_name was_redirected = request.GET.get("redirect", False) lang = helpers.get_language(request)['code'] crawler = request.GET.get("_escaped_fragment_", False) # Get session / request vars prod_class = request.GET.get( "prod_class", request.session.get('product_classification', 'hs4')) options = request.GET.copy() options["lang"] = lang options["product_classification"] = prod_class options = options.urlencode() # Setup the hash dictionary request_hash_dictionary = collections.OrderedDict() # Add prod class to request hash dictionary request_hash_dictionary['app_name'] = app_name request_hash_dictionary['lang'] = lang request_hash_dictionary['prod_class'] = prod_class # Add the arguments to the request hash dictionary request_hash_dictionary['trade_flow'] = trade_flow request_hash_dictionary['country1'] = country1 request_hash_dictionary['country2'] = country2 request_hash_dictionary['product_type'] = product request_hash_dictionary['year'] = year # We are here, so let us store this data somewhere request_hash_string = "_".join(request_hash_dictionary.values()) # Code for showing a static image or not static_image_name = helpers.url_to_hash(request.path, request.GET) if os.path.exists( os.path.join(settings.STATIC_IMAGE_PATH, static_image_name + ".png")): displayviz = True displayImage = static_image_name + ".png" else: displayviz = False displayImage = settings.STATIC_URL + "img/all/loader.gif" # Verify countries from DB countries = [None, None] alert = None for i, country in enumerate([country1, country2]): if country != "show" and country != "all": try: countries[i] = Country.objects.get(name_3char=country) except Country.DoesNotExist: alert = { "title": "Country could not be found", "text": """There was no country with the 3 letter abbreviation <strong>%s</strong>. Please double check the <a href='about/data/country/'>list of countries</a>.""" % (country) } # The years of data available tends to vary based on the dataset used (Hs4 # vs Sitc4) and the specific country. years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy years_available = years_available_model.objects\ .values_list("year", flat=True)\ .order_by("year")\ .distinct() # Sometimes the query is not about a specific country (e.g. "all countries" # queries) in which case filtering by country is not necessary if countries[0]: years_available = years_available.filter(country=countries[0].id) # Force lazy queryset to hit the DB to reduce number of DB queries later years_available = list(years_available) if len(years_available) == 0: alert = { "title": """The product classification you're using (%s) does not seem to include data for the country code (%s) you selected. Please try a different product classification or country.""" % (prod_class, countries[0].name) } years_available = range(1995, 2012) # Dummy year1_list, year2_list = None, None warning, title = None, None data_as_text = {} prod_or_partner = "partner" # To make sure it cannot be another product class if prod_class != "hs4" and prod_class != "sitc4": prod_class = "sitc4" # Test for country exceptions if prod_class == "hs4": # redirect if and exception country if country1 == "blx" or country1 == "lux": return redirect(HTTP_HOST + 'explore/%s/%s/bel/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz": return redirect(HTTP_HOST + 'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) if was_redirected: # display warning is redirected from exception if country1 == "bel": warning = { "title": "Country Substitution", "text": "In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported under 'BEL' ." } if country1 == "zaf": warning = { "title": "Country Substitution", "text": "In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'." } trade_flow_list = [("export", _("Export")), ("import", _("Import")), ("net_export", _("Net Export")), ("net_import", _("Net Import"))] if (app_name == "product_space" or app_name == "country_space" or app_name == "rings"): trade_flow_list = [trade_flow_list[0]] year1_list = range(years_available[0], years_available[len(years_available) - 1] + 1, 1) if app_name == "stacked" and year == "2009": year = "1969.2011.10" if "." in year: y = [int(x) for x in year.split(".")] year_start = y[0] year_end = y[1] year2_list = year1_list else: year_start, year_end = None, None year = int(year) # Check that year is within bounds if year > years_available[len(years_available) - 1]: year = years_available[len(years_available) - 1] elif year < years_available[0]: year = years_available[0] api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2, product, year, options) redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % ( prod_class, trade_flow, country1, country2, product, year) country_code = None if country1 != "show" and country1 != "all": country_code = country1 if crawler == "": view, args, kwargs = resolve( "api/%s/%s/%s/%s/%s/" % (trade_flow, country1, country2, product, year)) kwargs['request'] = request view_response = view(*args, **kwargs) raise Exception(view_response) data_as_text["data"] = view_response[0] data_as_text["total_value"] = view_response[1] data_as_text["columns"] = view_response[2] app_type = helpers.get_app_type(country1, country2, product, year) # What is actually being shown on the page if app_type == "csay" or app_type == "sapy": item_type = "country" else: item_type = "product" # Some countries need "the" before their names list_countries_the = set( ("Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States")) if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the " + countries[0].name if product not in ("show", "all"): p_code = product product = helpers.get_product_by_code(p_code, prod_class) if not alert: # Generate page title depending on visualization being used years = [year_start, year_end] if year_start is not None else [year] product_name = product.name_en if not isinstance( product, basestring) else product country_names = [getattr(x, "name", None) for x in countries] title = helpers.get_title(app_type, app_name, country_names=country_names, trade_flow=trade_flow.replace('_', ' '), years=years, product_name=product_name) if app_type in ("ccsy", "cspy"): if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] # Should we show the product or partner tab pane? # quick fix should be merged with item_type if app_type in ["cspy", "sapy"]: prod_or_partner = "product" elif app_type == "casy": if app_name in ("stacked", "map", "tree_map", "pie_scatter", "product_space", "country_space", "rankings", "scatterplot"): prod_or_partner = "product" # Record views in redis for "newest viewed pages" visualization if isinstance(cache, RedisCache): views_image_path = settings.STATIC_URL + \ "data/" + request_hash_string + ".png" view_data = { "timestamp": time.time(), "image": views_image_path, "title": title, "url": request.build_absolute_uri() } r = cache.raw_client r.rpush("views", json.dumps(view_data)) previous_page = request.META.get('HTTP_REFERER', None), if previous_page[0] is not None: previous_page = previous_page[0] previous_image = helpers.url_to_hash(urlparse(previous_page).path, {}) if os.path.exists( os.path.join(settings.STATIC_IMAGE_PATH, previous_image + ".png")): previous_image = previous_image + ".png" else: previous_image = settings.STATIC_URL + "img/all/loader.gif" else: previous_image = settings.STATIC_URL + "img/all/loader.gif" previous_page = None return render_to_response("explore/index.html", { "lang": lang, "warning": warning, "alert": alert, "prod_class": prod_class, "data_as_text": data_as_text, "app_name": app_name, "title": title, "trade_flow": trade_flow, "country1": countries[0] or country1, "country2": countries[1] or country2, "country1_3char": countries[0].name_3char if countries[0] else "", "country2_3char": countries[1].name_3char if countries[1] else "", "product": product, "product_code": product.code if not isinstance(product, basestring) else product, "years_available": years_available, "year": year, "year_start": year_start, "year_end": year_end, "year1_list": year1_list, "year2_list": year2_list, "trade_flow_list": trade_flow_list, "api_uri": api_uri, "app_type": app_type, "redesign_api_uri": redesign_api_uri, "country_code": country_code, "prod_or_partner": prod_or_partner, "version": VERSION, "previous_page": previous_page, "previous_image": previous_image, "item_type": item_type, "displayviz": displayviz, "displayImage": displayImage, "display_iframe": request.GET.get("display_iframe", False), "static_image": request.GET.get("static_image", "loading"), }, context_instance=RequestContext(request))