def api_dropdown_products(request, product_class="hs4"):
"""API to dynamically fill in a product dropdown, product name to code. Can
also set lang=foo to get a specific language, but it'll default to the
django user locale. """
lang = helpers.get_language(request)['code']
if product_class == "sitc4":
products = Sitc4.objects.get_all(lang)
elif product_class == "hs4":
products = Hs4.objects.get_all(lang)
products = [x for x in products if len(x["code"]) > 3]
return HttpResponse(json.dumps([(p["name"], p["code"]) for p in products]),
content_type="application/json")
def api_dropdown_products(request, product_class="hs4"):
"""API to dynamically fill in a product dropdown, product name to code. Can
also set lang=foo to get a specific language, but it'll default to the
django user locale. """
lang = helpers.get_language(request)['code']
if product_class == "sitc4":
products = Sitc4.objects.get_all(lang)
elif product_class == "hs4":
products = Hs4.objects.get_all(lang)
products = [x for x in products if len(x["code"]) > 3]
return HttpResponse(json.dumps([(p["name"], p["code"]) for p in products]),
content_type="application/json")
def api_dropdown_countries(request):
"""API to dynamically fill in a country dropdown, product name to code. Can
also set lang=foo to get a specific language, but it'll default to the
django user locale. """
lang = helpers.get_language(request)['code']
countries = Country.objects\
.filter_lang(lang)\
.values("name", "name_3char")
country_info = list(
(c["name"], c["name_3char"].lower()) for c in countries)
return HttpResponse(json.dumps(country_info),
content_type="application/json")
def api_dropdown_countries(request):
"""API to dynamically fill in a country dropdown, product name to code. Can
also set lang=foo to get a specific language, but it'll default to the
django user locale. """
lang = helpers.get_language(request)['code']
countries = Country.objects\
.filter_lang(lang)\
.values("name", "name_3char")
country_info = list((c["name"], c["name_3char"].lower())
for c in countries)
return HttpResponse(json.dumps(country_info),
content_type="application/json")
def api_casy(request, trade_flow, country1, year):
"""<COUNTRY> / all / show / <YEAR>"""
# Get session / request vars
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
country1 = Country.objects.get(name_3char=country1)
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
world_trade = helpers.get_world_trade(prod_class=prod_class)
attr = helpers.get_attrs(prod_class=prod_class, name=name)
years_available = helpers.get_years_available(prod_class=prod_class)
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
if prod_class == "sitc4":
items = Sitc4_cpy.objects
else:
items = Hs4_cpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = calculate_rca(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list(
'year',
'product__id',
'product__code',
'product__name',
'product__community_id',
'product__community__color',
'product__community__name',
'val',
'rca',
'distance',
'opp_gain',
'product_year__pci',
)
if single_year:
items = items.filter(year=year)
items = items.filter(country_id=country1.id)
filter_expr = "%s > 0" % trade_type(trade_flow)
items = items.extra(where=[filter_expr])
json_response = {}
# Generate cache key
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class,
trade_flow)
if single_year:
key += ":%d" % int(year)
# Check cache
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[7] for r in rows])
rows = [{
"year": r[0],
"item_id": r[1],
"abbrv": r[2],
"name": r[3],
"value": r[7],
"rca": r[8],
"distance": r[9],
"opp_gain": r[10],
"pci": r[11],
"share": (r[7] / total_val) * 100,
"community_id": r[4],
"color": r[5],
"community_name": r[6],
"code": r[2],
"id": r[2]
} for r in rows]
# Save in cache
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
# Add in remaining metadata
json_response["attr"] = attr
json_response["attr_data"] = Sitc4.objects.get_all(
lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang)
json_response["country1"] = country1.to_json()
json_response["title"] = "What does %s %s?" % (
country1.name, trade_flow.replace("_", " "))
json_response["year"] = year
json_response["item_type"] = "product"
json_response["app_type"] = "casy"
json_response["magic_numbers"] = magic_numbers
json_response["world_trade"] = world_trade
json_response["prod_class"] = prod_class
json_response["other"] = query_params
# Check the request data type
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
return HttpResponse(json.dumps(json_response))
def api_cspy(request, trade_flow, country1, product, year):
""" country / show / product / year """
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
product = helpers.get_product_by_code(product, prod_class)
country1 = Country.objects.get(name_3char=country1)
single_year = "single_year" in request.GET
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
region = helpers.get_region_list()
continents = helpers.get_continent_list()
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'destination__id',
'destination__name_3char', 'name',
'destination__region_id',
'destination__continent', 'val')
items = items.filter(origin_id=country1.id, product_id=product.id)
if single_year:
items = items.filter(year=year)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id,
prod_class, trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response['data'] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[6] for r in rows])
rows = [{
"year": r[0],
"item_id": r[1],
"abbrv": r[2],
"name": r[3],
"value": r[6],
"share": (r[6] / total_val) * 100,
"region_id": r[4],
"continent": r[5],
"id": r[1]
} for r in rows]
json_response["data"] = rows
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
years_available = helpers.get_years_available(prod_class=prod_class)
inflation_adjustment = helpers.get_inflation_adjustment(
country1, years_available[0], years_available[-1])
json_response["magic_numbers"] = inflation_adjustment
json_response["attr_data"] = Country.objects.get_all(lang)
article = "to" if trade_flow == "export" else "from"
json_response["title"] = "Where does %s %s %s %s?" % (
country1.name, trade_flow, product.name_en, article)
json_response["country1"] = country1.to_json()
json_response["product"] = product.to_json()
json_response["year"] = year
json_response["item_type"] = "country"
json_response["continents"] = continents
json_response["region"] = region
json_response["app_type"] = "cspy"
json_response["class"] = prod_class
json_response["other"] = query_params
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
return HttpResponse(json.dumps(json_response))
def api_ccsy(request, trade_flow, country1, country2, year):
# Get session / request vars
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
country1 = Country.objects.get(name_3char=country1)
country2 = Country.objects.get(name_3char=country2)
article = "to" if trade_flow == "export" else "from"
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
attr = helpers.get_attrs(prod_class=prod_class, name=name)
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'product__id', 'product__code', 'name',
'product__community_id',
'product__community__name',
'product__community__color', 'val')
if single_year:
items = items.filter(year=year)
items = items.filter(origin_id=country1.id, destination_id=country2.id)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show",
prod_class, trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[7] for r in rows])
"""Add percentage value to return vals"""
rows = [{
"year": r[0],
"item_id": r[1],
"abbrv": r[2],
"name": r[3],
"value": r[7],
"share": (r[7] / total_val) * 100,
"community_id": r[4],
"community_name": r[5],
"color": r[6],
"code": r[2],
"id": r[2]
} for r in rows]
json_response["data"] = rows
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
years_available = helpers.get_years_available(prod_class=prod_class)
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
json_response["magic_numbers"] = magic_numbers
json_response["attr_data"] = Sitc4.objects.get_all(
lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang)
json_response["country1"] = country1.to_json()
json_response["country2"] = country2.to_json()
json_response["title"] = "What does %s %s %s %s?" % (
country1.name, trade_flow, article, country2.name)
json_response["year"] = year
json_response["item_type"] = "product"
json_response["app_type"] = "ccsy"
json_response["prod_class"] = prod_class
json_response["attr"] = attr
json_response["class"] = prod_class
json_response["other"] = query_params
# Check the request data type
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def api_csay(request, trade_flow, country1, year):
"""<COUNTRY> / show / all / <YEAR>"""
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
country1 = Country.objects.get(name_3char=country1)
region = helpers.get_region_list()
continents = helpers.get_continent_list()
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow=trade_flow, sum_val=True)
items = items.extra(select={'name': name})
items = items.values_list('year', 'destination__id',
'destination__name_3char', 'name',
'destination__region_id',
'destination__continent', 'val')
items = items.filter(origin_id=country1.id)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class,
trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
# This might possibly be the most disgusting hack ever made, simply
# because when doing an aggregate (like SUM()) in extra, django does
# not add that stuff correctly into the group by. It's also not
# possible to use annotate() here because it's a complex aggregate that
# uses addition / subtraction. C'est la vie :(
cursor = connection.cursor()
if trade_flow == "net_export":
val = '`export_value` - `import_value`'
elif trade_flow == "net_import":
val = '`import_value` - `export_value`'
elif trade_flow == "export":
val = '`export_value`'
else:
val = '`import_value`'
complete_query = str(items.query) + \
" group by `year`, `destination_id`" + \
" HAVING sum(%s) > 0" % val
cursor.execute(complete_query)
rows = cursor.fetchall()
total_val = sum([r[0] for r in rows])
rows = [{
"year": r[2],
"item_id": r[3],
"abbrv": r[4],
"name": r[1],
"value": r[0],
"share": (r[0] / total_val) * 100,
"id": r[3],
"region_id": r[5],
"continent": r[6]
} for r in rows]
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
years_available = helpers.get_years_available()
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
json_response["attr_data"] = Country.objects.get_all(lang)
json_response["country1"] = country1.to_json()
article = "to" if trade_flow == "export" else "from"
json_response["title"] = "Where does %s %s %s?" % (country1.name,
trade_flow, article)
json_response["year"] = year
json_response["item_type"] = "country"
json_response["app_type"] = "csay"
json_response["region"] = region
json_response["continents"] = continents
json_response["prod_class"] = prod_class
json_response["magic_numbers"] = magic_numbers
json_response["other"] = query_params
# raise Exception(time.time() - start)
# Check the request data type
if (request.GET.get('data_type', None) is None):
#"""Return to browser as JSON for AJAX request"""
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def api_sapy(request, trade_flow, product, year):
"""show / all / <product> / <year>"""
# Get session / request vars
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
product = helpers.get_product_by_code(product, prod_class)
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
region = helpers.get_region_list()
continents = helpers.get_continent_list()
attr = helpers.get_attrs(prod_class=prod_class, name=name)
if prod_class == "sitc4":
items = Sitc4_cpy.objects
else:
items = Hs4_cpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = calculate_rca(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'country__id', 'country__name_3char',
'name', 'country__region_id',
'country__continent', 'val', 'rca')
if single_year:
items = items.filter(year=year)
items = items.filter(product_id=product.id)
json_response = {}
# Generate cache key
key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class,
trade_flow)
if single_year:
key += ":%d" % int(year)
# Check cache
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[6] for r in rows])
rows = [{
"year": r[0],
"item_id": r[1],
"abbrv": r[2],
"name": r[3],
"value": r[6],
"rca": r[7],
"share": (r[6] / total_val) * 100,
"id": r[1],
"region_id": r[4],
"continent": r[5]
} for r in rows]
# Save in cache
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
json_response["attr_data"] = Country.objects.get_all(lang)
json_response["product"] = product.to_json()
json_response["title"] = "Who %sed %s?" % (trade_flow.replace(
"_", " "), product.name_en)
json_response["year"] = year
json_response["item_type"] = "country"
json_response["app_type"] = "sapy"
json_response["region"] = region
json_response["continents"] = continents
json_response["other"] = query_params
json_response["prod_class"] = prod_class
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def explore(
request,
app_name,
trade_flow,
country1,
country2,
product,
year="2011"):
request.session['app_name'] = app_name
was_redirected = request.GET.get("redirect", False)
lang = helpers.get_language(request)['code']
crawler = request.GET.get("_escaped_fragment_", False)
# Get session / request vars
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
options = request.GET.copy()
options["lang"] = lang
options["product_classification"] = prod_class
options = options.urlencode()
# Setup the hash dictionary
request_hash_dictionary = collections.OrderedDict()
# Add prod class to request hash dictionary
request_hash_dictionary['app_name'] = app_name
request_hash_dictionary['lang'] = lang
request_hash_dictionary['prod_class'] = prod_class
# Add the arguments to the request hash dictionary
request_hash_dictionary['trade_flow'] = trade_flow
request_hash_dictionary['country1'] = country1
request_hash_dictionary['country2'] = country2
request_hash_dictionary['product_type'] = product
request_hash_dictionary['year'] = year
# We are here, so let us store this data somewhere
request_hash_string = "_".join(request_hash_dictionary.values())
# Code for showing a static image or not
static_image_name = helpers.url_to_hash(request.path, request.GET)
if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH,
static_image_name + ".png")):
displayviz = True
displayImage = static_image_name + ".png"
else:
displayviz = False
displayImage = settings.STATIC_URL + "img/all/loader.gif"
# Verify countries from DB
countries = [None, None]
alert = None
for i, country in enumerate([country1, country2]):
if country != "show" and country != "all":
try:
countries[i] = Country.objects.get(name_3char=country)
except Country.DoesNotExist:
alert = {
"title": "Country could not be found",
"text": """There was no country with the 3 letter
abbreviation <strong>%s</strong>. Please double check the
<a href='about/data/country/'>list of countries</a>.""" %
(country)}
# The years of data available tends to vary based on the dataset used (Hs4
# vs Sitc4) and the specific country.
years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy
years_available = years_available_model.objects\
.values_list("year", flat=True)\
.order_by("year")\
.distinct()
# Sometimes the query is not about a specific country (e.g. "all countries"
# queries) in which case filtering by country is not necessary
if countries[0]:
years_available = years_available.filter(country=countries[0].id)
# Force lazy queryset to hit the DB to reduce number of DB queries later
years_available = list(years_available)
if len(years_available) == 0:
alert = {"title": """The product classification you're using (%s) does
not seem to include data for the country code (%s) you selected.
Please try a different product classification or country.""" %
(prod_class, countries[0].name)}
years_available = range(1995, 2012) # Dummy
year1_list, year2_list = None, None
warning, title = None, None
data_as_text = {}
prod_or_partner = "partner"
# To make sure it cannot be another product class
if prod_class != "hs4" and prod_class != "sitc4":
prod_class = "sitc4"
# Test for country exceptions
if prod_class == "hs4":
# redirect if and exception country
if country1 == "bel" or country1 == "lux":
return redirect(
HTTP_HOST+'explore/%s/%s/blx/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz":
return redirect(
HTTP_HOST+'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if was_redirected:
# display warning is redirected from exception
if country1 == "blx":
warning = {
"title": "Country Substitution", "text":
"In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported as 'Belgium-Luxembourg'."}
if country1 == "zaf":
warning = {
"title": "Country Substitution", "text":
"In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."}
trade_flow_list = [
("export", _("Export")), ("import", _("Import")),
("net_export", _("Net Export")), ("net_import", _("Net Import"))]
if (app_name == "product_space" or app_name == "country_space" or app_name == "rings"):
trade_flow_list = [trade_flow_list[0]]
year1_list = range(
years_available[0],
years_available[
len(years_available) -
1] +
1,
1)
if app_name == "stacked" and year == "2009":
year = "1969.2011.10"
if "." in year:
y = [int(x) for x in year.split(".")]
year_start = y[0]
year_end = y[1]
year2_list = year1_list
else:
year_start, year_end = None, None
year = int(year)
# Check that year is within bounds
if year > years_available[len(years_available)-1]:
year = years_available[len(years_available)-1]
elif year < years_available[0]:
year = years_available[0]
api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow,
country1,
country2,
product,
year,
options)
redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class,
trade_flow,
country1,
country2,
product,
year)
country_code = None
if country1 != "show" and country1 != "all":
country_code = country1
if crawler == "":
view, args, kwargs = resolve(
"api/%s/%s/%s/%s/%s/" %
(trade_flow, country1, country2, product, year))
kwargs['request'] = request
view_response = view(*args, **kwargs)
raise Exception(view_response)
data_as_text["data"] = view_response[0]
data_as_text["total_value"] = view_response[1]
data_as_text["columns"] = view_response[2]
app_type = helpers.get_app_type(country1, country2, product, year)
# What is actually being shown on the page
if app_type == "csay" or app_type == "sapy":
item_type = "country"
else:
item_type = "product"
# Some countries need "the" before their names
list_countries_the = set(
("Cayman Islands",
"Central African Republic",
"Channel Islands",
"Congo, Dem. Rep.",
"Czech Republic",
"Dominican Republic",
"Faeroe Islands",
"Falkland Islands",
"Fm Yemen Dm",
"Lao PDR",
"Marshall Islands",
"Philippines",
"Seychelles",
"Slovak Republic",
"Syrian Arab Republic",
"Turks and Caicos Islands",
"United Arab Emirates",
"United Kingdom",
"Virgin Islands, U.S.",
"United States"))
if countries[0] and countries[0].name in list_countries_the:
countries[0].name = "the "+countries[0].name
if product not in ("show", "all"):
p_code = product
product = helpers.get_product_by_code(p_code, prod_class)
if not alert:
# Generate page title depending on visualization being used
years = [year_start, year_end] if year_start is not None else [year]
product_name = product.name_en if not isinstance(
product,
basestring) else product
country_names = [getattr(x, "name", None) for x in countries]
title = helpers.get_title(app_type, app_name,
country_names=country_names,
trade_flow=trade_flow.replace('_', ' '),
years=years,
product_name=product_name
)
if app_type in ("ccsy", "cspy"):
if _("net_export") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_export"))]
if _("net_import") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_import"))]
# Should we show the product or partner tab pane?
# quick fix should be merged with item_type
if app_type in ["cspy", "sapy"]:
prod_or_partner = "product"
elif app_type == "casy":
if app_name in ("stacked", "map", "tree_map", "pie_scatter", "product_space", "country_space", "rankings", "scatterplot"):
prod_or_partner = "product"
# Record views in redis for "newest viewed pages" visualization
if isinstance(cache, RedisCache):
views_image_path = settings.STATIC_URL + \
"data/" + request_hash_string + ".png"
view_data = {
"timestamp": time.time(),
"image": views_image_path,
"title": title,
"url": request.build_absolute_uri()
}
r = cache.raw_client
r.rpush("views", json.dumps(view_data))
previous_page = request.META.get('HTTP_REFERER',
None),
if previous_page[0] is not None:
previous_page = previous_page[0]
previous_image = helpers.url_to_hash(urlparse(previous_page).path, {})
if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH,
previous_image + ".png")):
previous_image = previous_image + ".png"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
previous_page = None
return render_to_response(
"explore/index.html",
{"lang": lang,
"warning": warning,
"alert": alert,
"prod_class": prod_class,
"data_as_text": data_as_text,
"app_name": app_name,
"title": title,
"trade_flow": trade_flow,
"country1": countries[0] or country1,
"country2": countries[1] or country2,
"country1_3char": countries[0].name_3char if countries[0] else "",
"country2_3char": countries[1].name_3char if countries[1] else "",
"product": product,
"product_code": product.code if not isinstance(product, basestring) else product,
"years_available": years_available,
"year": year,
"year_start": year_start,
"year_end": year_end,
"year1_list": year1_list,
"year2_list": year2_list,
"trade_flow_list": trade_flow_list,
"api_uri": api_uri,
"app_type": app_type,
"redesign_api_uri": redesign_api_uri,
"country_code": country_code,
"prod_or_partner": prod_or_partner,
"version": VERSION,
"previous_page": previous_page,
"previous_image": previous_image,
"item_type": item_type,
"displayviz": displayviz,
"displayImage": displayImage,
"display_iframe": request.GET.get("display_iframe", False),
"static_image": request.GET.get("static_image", "loading"),
},
context_instance=RequestContext(request))
def api_casy(request, trade_flow, country1, year):
"""<COUNTRY> / all / show / <YEAR>"""
# Get session / request vars
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
country1 = Country.objects.get(name_3char=country1)
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
world_trade = helpers.get_world_trade(prod_class=prod_class)
attr = helpers.get_attrs(prod_class=prod_class, name=name)
years_available = helpers.get_years_available(prod_class=prod_class)
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
if prod_class == "sitc4":
items = Sitc4_cpy.objects
else:
items = Hs4_cpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = calculate_rca(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'product__id', 'product__code',
'product__name', 'product__community_id',
'product__community__color',
'product__community__name', 'val', 'rca',
'distance', 'opp_gain', 'product_year__pci',)
if single_year:
items = items.filter(year=year)
items = items.filter(country_id=country1.id)
filter_expr = "%s > 0" % trade_type(trade_flow)
items = items.extra(where=[filter_expr])
json_response = {}
# Generate cache key
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show",
prod_class, trade_flow)
if single_year:
key += ":%d" % int(year)
# Check cache
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[7] for r in rows])
rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3],
"value": r[7], "rca": r[8], "distance": r[9], "opp_gain":
r[10], "pci": r[11], "share": (r[7] / total_val) * 100,
"community_id": r[4], "color": r[5], "community_name": r[6],
"code": r[2], "id": r[2]} for r in rows]
# Save in cache
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
# Add in remaining metadata
json_response["attr"] = attr
json_response["attr_data"] = Sitc4.objects.get_all(
lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang)
json_response["country1"] = country1.to_json()
json_response["title"] = "What does %s %s?" % (
country1.name, trade_flow.replace("_", " "))
json_response["year"] = year
json_response["item_type"] = "product"
json_response["app_type"] = "casy"
json_response["magic_numbers"] = magic_numbers
json_response["world_trade"] = world_trade
json_response["prod_class"] = prod_class
json_response["other"] = query_params
# Check the request data type
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
return HttpResponse(json.dumps(json_response))
def api_cspy(request, trade_flow, country1, product, year):
""" country / show / product / year """
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
product = helpers.get_product_by_code(product, prod_class)
country1 = Country.objects.get(name_3char=country1)
single_year = "single_year" in request.GET
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
region = helpers.get_region_list()
continents = helpers.get_continent_list()
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'destination__id',
'destination__name_3char', 'name',
'destination__region_id',
'destination__continent', 'val')
items = items.filter(origin_id=country1.id, product_id=product.id)
if single_year:
items = items.filter(year=year)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id,
prod_class, trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response['data'] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[6] for r in rows])
rows = [
{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3],
"value": r[6], "share": (r[6] / total_val) * 100,
"region_id": r[4], "continent": r[5], "id": r[1]}
for r in rows]
json_response["data"] = rows
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
years_available = helpers.get_years_available(prod_class=prod_class)
inflation_adjustment = helpers.get_inflation_adjustment(
country1, years_available[0], years_available[-1])
json_response["magic_numbers"] = inflation_adjustment
json_response["attr_data"] = Country.objects.get_all(lang)
article = "to" if trade_flow == "export" else "from"
json_response["title"] = "Where does %s %s %s %s?" % (
country1.name, trade_flow, product.name_en, article)
json_response["country1"] = country1.to_json()
json_response["product"] = product.to_json()
json_response["year"] = year
json_response["item_type"] = "country"
json_response["continents"] = continents
json_response["region"] = region
json_response["app_type"] = "cspy"
json_response["class"] = prod_class
json_response["other"] = query_params
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
return HttpResponse(json.dumps(json_response))
def api_ccsy(request, trade_flow, country1, country2, year):
# Get session / request vars
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
country1 = Country.objects.get(name_3char=country1)
country2 = Country.objects.get(name_3char=country2)
article = "to" if trade_flow == "export" else "from"
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
attr = helpers.get_attrs(prod_class=prod_class, name=name)
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'product__id', 'product__code',
'name', 'product__community_id',
'product__community__name',
'product__community__color', 'val')
if single_year:
items = items.filter(year=year)
items = items.filter(origin_id=country1.id, destination_id=country2.id)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show",
prod_class, trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[7] for r in rows])
"""Add percentage value to return vals"""
rows = [{"year": r[0],
"item_id":r[1],
"abbrv":r[2],
"name":r[3],
"value":r[7],
"share": (r[7] / total_val)*100,
"community_id":r[4],
"community_name":r[5],
"color":r[6],
"code":r[2],
"id": r[2]} for r in rows]
json_response["data"] = rows
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
years_available = helpers.get_years_available(prod_class=prod_class)
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
json_response["magic_numbers"] = magic_numbers
json_response["attr_data"] = Sitc4.objects.get_all(
lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang)
json_response["country1"] = country1.to_json()
json_response["country2"] = country2.to_json()
json_response["title"] = "What does %s %s %s %s?" % (
country1.name, trade_flow, article, country2.name)
json_response["year"] = year
json_response["item_type"] = "product"
json_response["app_type"] = "ccsy"
json_response["prod_class"] = prod_class
json_response["attr"] = attr
json_response["class"] = prod_class
json_response["other"] = query_params
# Check the request data type
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def api_csay(request, trade_flow, country1, year):
"""<COUNTRY> / show / all / <YEAR>"""
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
country1 = Country.objects.get(name_3char=country1)
region = helpers.get_region_list()
continents = helpers.get_continent_list()
if prod_class == "sitc4":
items = Sitc4_ccpy.objects
else:
items = Hs4_ccpy.objects
items = calculate_volume(items, trade_flow=trade_flow,
sum_val=True)
items = items.extra(select={'name': name})
items = items.values_list('year', 'destination__id',
'destination__name_3char', 'name',
'destination__region_id',
'destination__continent', 'val')
items = items.filter(origin_id=country1.id)
json_response = {}
key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class,
trade_flow)
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
# This might possibly be the most disgusting hack ever made, simply
# because when doing an aggregate (like SUM()) in extra, django does
# not add that stuff correctly into the group by. It's also not
# possible to use annotate() here because it's a complex aggregate that
# uses addition / subtraction. C'est la vie :(
cursor = connection.cursor()
if trade_flow == "net_export":
val = '`export_value` - `import_value`'
elif trade_flow == "net_import":
val = '`import_value` - `export_value`'
elif trade_flow == "export":
val = '`export_value`'
else:
val = '`import_value`'
complete_query = str(items.query) + \
" group by `year`, `destination_id`" + \
" HAVING sum(%s) > 0" % val
cursor.execute(complete_query)
rows = cursor.fetchall()
total_val = sum([r[0] for r in rows])
rows = [
{"year": r[2],
"item_id": r[3],
"abbrv": r[4],
"name": r[1],
"value": r[0],
"share": (r[0] / total_val) * 100,
"id": r[3],
"region_id": r[5],
"continent": r[6]}
for r in rows]
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
years_available = helpers.get_years_available()
magic_numbers = helpers.get_inflation_adjustment(country1,
years_available[0],
years_available[-1])
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
json_response["attr_data"] = Country.objects.get_all(lang)
json_response["country1"] = country1.to_json()
article = "to" if trade_flow == "export" else "from"
json_response["title"] = "Where does %s %s %s?" % (
country1.name, trade_flow, article)
json_response["year"] = year
json_response["item_type"] = "country"
json_response["app_type"] = "csay"
json_response["region"] = region
json_response["continents"] = continents
json_response["prod_class"] = prod_class
json_response["magic_numbers"] = magic_numbers
json_response["other"] = query_params
# raise Exception(time.time() - start)
# Check the request data type
if (request.GET.get('data_type', None) is None):
#"""Return to browser as JSON for AJAX request"""
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def api_sapy(request, trade_flow, product, year):
"""show / all / <product> / <year>"""
# Get session / request vars
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
lang = helpers.get_language(request)['code']
name = "name_%s" % lang
single_year = 'single_year' in request.GET
product = helpers.get_product_by_code(product, prod_class)
query_params = request.GET.copy()
query_params["lang"] = lang
query_params["product_classification"] = prod_class
region = helpers.get_region_list()
continents = helpers.get_continent_list()
attr = helpers.get_attrs(prod_class=prod_class, name=name)
if prod_class == "sitc4":
items = Sitc4_cpy.objects
else:
items = Hs4_cpy.objects
items = calculate_volume(items, trade_flow=trade_flow)
items = calculate_rca(items, trade_flow=trade_flow)
items = items.extra(select={'name': name})
items = items.values_list('year', 'country__id', 'country__name_3char',
'name', 'country__region_id',
'country__continent', 'val', 'rca')
if single_year:
items = items.filter(year=year)
items = items.filter(product_id=product.id)
json_response = {}
# Generate cache key
key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class,
trade_flow)
if single_year:
key += ":%d" % int(year)
# Check cache
cached_data = cache.get(key)
if cached_data is not None:
json_response["data"] = msgpack.loads(cached_data)
else:
rows = list(items)
total_val = sum([r[6] for r in rows])
rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3],
"value": r[6], "rca": r[7], "share": (r[6] / total_val) * 100,
"id": r[1], "region_id": r[4], "continent": r[5]}
for r in rows]
# Save in cache
cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG)
json_response["data"] = rows
json_response["attr_data"] = Country.objects.get_all(lang)
json_response["product"] = product.to_json()
json_response["title"] = "Who %sed %s?" % (
trade_flow.replace("_", " "), product.name_en)
json_response["year"] = year
json_response["item_type"] = "country"
json_response["app_type"] = "sapy"
json_response["region"] = region
json_response["continents"] = continents
json_response["other"] = query_params
json_response["prod_class"] = prod_class
if (request.GET.get('data_type', None) is None):
return HttpResponse("")
elif (request.GET.get('data_type', '') == 'json'):
"""Return to browser as JSON for AJAX request"""
return HttpResponse(json.dumps(json_response))
def explore(request,
app_name,
trade_flow,
country1,
country2,
product,
year="2011"):
request.session['app_name'] = app_name
was_redirected = request.GET.get("redirect", False)
lang = helpers.get_language(request)['code']
crawler = request.GET.get("_escaped_fragment_", False)
# Get session / request vars
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
options = request.GET.copy()
options["lang"] = lang
options["product_classification"] = prod_class
options = options.urlencode()
# Setup the hash dictionary
request_hash_dictionary = collections.OrderedDict()
# Add prod class to request hash dictionary
request_hash_dictionary['app_name'] = app_name
request_hash_dictionary['lang'] = lang
request_hash_dictionary['prod_class'] = prod_class
# Add the arguments to the request hash dictionary
request_hash_dictionary['trade_flow'] = trade_flow
request_hash_dictionary['country1'] = country1
request_hash_dictionary['country2'] = country2
request_hash_dictionary['product_type'] = product
request_hash_dictionary['year'] = year
# We are here, so let us store this data somewhere
request_hash_string = "_".join(request_hash_dictionary.values())
# Code for showing a static image or not
static_image_name = helpers.url_to_hash(request.path, request.GET)
if os.path.exists(
os.path.join(settings.STATIC_IMAGE_PATH,
static_image_name + ".png")):
displayviz = True
displayImage = static_image_name + ".png"
else:
displayviz = False
displayImage = settings.STATIC_URL + "img/all/loader.gif"
# Verify countries from DB
countries = [None, None]
alert = None
for i, country in enumerate([country1, country2]):
if country != "show" and country != "all":
try:
countries[i] = Country.objects.get(name_3char=country)
except Country.DoesNotExist:
alert = {
"title":
"Country could not be found",
"text":
"""There was no country with the 3 letter
abbreviation <strong>%s</strong>. Please double check the
<a href='about/data/country/'>list of countries</a>.""" %
(country)
}
# The years of data available tends to vary based on the dataset used (Hs4
# vs Sitc4) and the specific country.
years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy
years_available = years_available_model.objects\
.values_list("year", flat=True)\
.order_by("year")\
.distinct()
# Sometimes the query is not about a specific country (e.g. "all countries"
# queries) in which case filtering by country is not necessary
if countries[0]:
years_available = years_available.filter(country=countries[0].id)
# Force lazy queryset to hit the DB to reduce number of DB queries later
years_available = list(years_available)
if len(years_available) == 0:
alert = {
"title":
"""The product classification you're using (%s) does
not seem to include data for the country code (%s) you selected.
Please try a different product classification or country.""" %
(prod_class, countries[0].name)
}
years_available = range(1995, 2012) # Dummy
year1_list, year2_list = None, None
warning, title = None, None
data_as_text = {}
prod_or_partner = "partner"
# To make sure it cannot be another product class
if prod_class != "hs4" and prod_class != "sitc4":
prod_class = "sitc4"
# Test for country exceptions
if prod_class == "hs4":
# redirect if and exception country
if country1 == "blx" or country1 == "lux":
return redirect(HTTP_HOST +
'explore/%s/%s/bel/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz":
return redirect(HTTP_HOST +
'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if was_redirected:
# display warning is redirected from exception
if country1 == "bel":
warning = {
"title":
"Country Substitution",
"text":
"In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported under 'BEL' ."
}
if country1 == "zaf":
warning = {
"title":
"Country Substitution",
"text":
"In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."
}
trade_flow_list = [("export", _("Export")), ("import", _("Import")),
("net_export", _("Net Export")),
("net_import", _("Net Import"))]
if (app_name == "product_space" or app_name == "country_space"
or app_name == "rings"):
trade_flow_list = [trade_flow_list[0]]
year1_list = range(years_available[0],
years_available[len(years_available) - 1] + 1, 1)
if app_name == "stacked" and year == "2009":
year = "1969.2011.10"
if "." in year:
y = [int(x) for x in year.split(".")]
year_start = y[0]
year_end = y[1]
year2_list = year1_list
else:
year_start, year_end = None, None
year = int(year)
# Check that year is within bounds
if year > years_available[len(years_available) - 1]:
year = years_available[len(years_available) - 1]
elif year < years_available[0]:
year = years_available[0]
api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2,
product, year, options)
redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (
prod_class, trade_flow, country1, country2, product, year)
country_code = None
if country1 != "show" and country1 != "all":
country_code = country1
if crawler == "":
view, args, kwargs = resolve(
"api/%s/%s/%s/%s/%s/" %
(trade_flow, country1, country2, product, year))
kwargs['request'] = request
view_response = view(*args, **kwargs)
raise Exception(view_response)
data_as_text["data"] = view_response[0]
data_as_text["total_value"] = view_response[1]
data_as_text["columns"] = view_response[2]
app_type = helpers.get_app_type(country1, country2, product, year)
# What is actually being shown on the page
if app_type == "csay" or app_type == "sapy":
item_type = "country"
else:
item_type = "product"
# Some countries need "the" before their names
list_countries_the = set(
("Cayman Islands", "Central African Republic", "Channel Islands",
"Congo, Dem. Rep.", "Czech Republic", "Dominican Republic",
"Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR",
"Marshall Islands", "Philippines", "Seychelles", "Slovak Republic",
"Syrian Arab Republic", "Turks and Caicos Islands",
"United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.",
"United States"))
if countries[0] and countries[0].name in list_countries_the:
countries[0].name = "the " + countries[0].name
if product not in ("show", "all"):
p_code = product
product = helpers.get_product_by_code(p_code, prod_class)
if not alert:
# Generate page title depending on visualization being used
years = [year_start, year_end] if year_start is not None else [year]
product_name = product.name_en if not isinstance(
product, basestring) else product
country_names = [getattr(x, "name", None) for x in countries]
title = helpers.get_title(app_type,
app_name,
country_names=country_names,
trade_flow=trade_flow.replace('_', ' '),
years=years,
product_name=product_name)
if app_type in ("ccsy", "cspy"):
if _("net_export") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_export"))]
if _("net_import") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_import"))]
# Should we show the product or partner tab pane?
# quick fix should be merged with item_type
if app_type in ["cspy", "sapy"]:
prod_or_partner = "product"
elif app_type == "casy":
if app_name in ("stacked", "map", "tree_map", "pie_scatter",
"product_space", "country_space", "rankings",
"scatterplot"):
prod_or_partner = "product"
# Record views in redis for "newest viewed pages" visualization
if isinstance(cache, RedisCache):
views_image_path = settings.STATIC_URL + \
"data/" + request_hash_string + ".png"
view_data = {
"timestamp": time.time(),
"image": views_image_path,
"title": title,
"url": request.build_absolute_uri()
}
r = cache.raw_client
r.rpush("views", json.dumps(view_data))
previous_page = request.META.get('HTTP_REFERER', None),
if previous_page[0] is not None:
previous_page = previous_page[0]
previous_image = helpers.url_to_hash(urlparse(previous_page).path, {})
if os.path.exists(
os.path.join(settings.STATIC_IMAGE_PATH,
previous_image + ".png")):
previous_image = previous_image + ".png"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
previous_page = None
return render_to_response("explore/index.html", {
"lang":
lang,
"warning":
warning,
"alert":
alert,
"prod_class":
prod_class,
"data_as_text":
data_as_text,
"app_name":
app_name,
"title":
title,
"trade_flow":
trade_flow,
"country1":
countries[0] or country1,
"country2":
countries[1] or country2,
"country1_3char":
countries[0].name_3char if countries[0] else "",
"country2_3char":
countries[1].name_3char if countries[1] else "",
"product":
product,
"product_code":
product.code if not isinstance(product, basestring) else product,
"years_available":
years_available,
"year":
year,
"year_start":
year_start,
"year_end":
year_end,
"year1_list":
year1_list,
"year2_list":
year2_list,
"trade_flow_list":
trade_flow_list,
"api_uri":
api_uri,
"app_type":
app_type,
"redesign_api_uri":
redesign_api_uri,
"country_code":
country_code,
"prod_or_partner":
prod_or_partner,
"version":
VERSION,
"previous_page":
previous_page,
"previous_image":
previous_image,
"item_type":
item_type,
"displayviz":
displayviz,
"displayImage":
displayImage,
"display_iframe":
request.GET.get("display_iframe", False),
"static_image":
request.GET.get("static_image", "loading"),
},
context_instance=RequestContext(request))