def process_response(self, request, response):
if not settings.STATIC_IMAGE:
return response
# If this is not an explore page, no need to prerender
if request.resolver_match and ("observatory.views.explore"
!= request.resolver_match.url_name):
return response
# Deal with django streaming responses, these don't need to be
# prerendered anyway.
if response.streaming:
return response
# TODO: AFAIK there is no better way to send this html to phantomjs
temp = tempfile.NamedTemporaryFile(dir=settings.PHANTOM_JS_TEMP,
prefix="celery-temp-file-",
suffix=".html",
delete=False)
temp.write(response.content)
temp.flush()
os.chmod(temp.name, 0755)
url = settings.STATIC_IMAGE_PHANTOMJS_URL + \
"%s" % os.path.basename(temp.name)
# Try to prerender, if it times out just return the original
try:
prerender = celery_tasks.prerender.s(url)
filename = helpers.url_to_hash(request.path, request.GET) + ".png"
get_image = celery_tasks\
.prerendered_html_to_image.s(name=filename,
path=settings.STATIC_IMAGE_PATH)
if hasattr(request, "bot_crawl") and request.bot_crawl:
# If prerender is needed, wait for phantomjs
result = celery.chain(prerender, get_image)()
response.content = result.parent.get(timeout=15)
return response
else:
# Otherwise, generate static image in the background, if it
# isn't already there
if not os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH,
filename)):
result = celery.chain(prerender, get_image)()
return response
except Exception:
if "result" in locals():
result.forget()
logger.exception("Error occurred while prerendering page.")
raise
finally:
temp.close()
def process_response(self, request, response):
if not settings.STATIC_IMAGE:
return response
# If this is not an explore page, no need to prerender
if request.resolver_match and ("observatory.views.explore" !=
request.resolver_match.url_name):
return response
# Deal with django streaming responses, these don't need to be
# prerendered anyway.
if response.streaming:
return response
# TODO: AFAIK there is no better way to send this html to phantomjs
temp = tempfile.NamedTemporaryFile(dir=settings.PHANTOM_JS_TEMP,
prefix="celery-temp-file-",
suffix=".html",
delete=False)
temp.write(response.content)
temp.flush()
os.chmod(temp.name, 0755)
url = settings.STATIC_IMAGE_PHANTOMJS_URL + \
"%s" % os.path.basename(temp.name)
# Try to prerender, if it times out just return the original
try:
prerender = celery_tasks.prerender.s(url)
filename = helpers.url_to_hash(request.path, request.GET) + ".png"
get_image = celery_tasks\
.prerendered_html_to_image.s(name=filename,
path=settings.STATIC_IMAGE_PATH)
if hasattr(request, "bot_crawl") and request.bot_crawl:
# If prerender is needed, wait for phantomjs
result = celery.chain(prerender, get_image)()
response.content = result.parent.get(timeout=15)
return response
else:
# Otherwise, generate static image in the background, if it
# isn't already there
if not os.path.exists(
os.path.join(settings.STATIC_IMAGE_PATH, filename)):
result = celery.chain(prerender, get_image)()
return response
except Exception:
if "result" in locals():
result.forget()
logger.exception("Error occurred while prerendering page.")
raise
finally:
temp.close()
def explore(
request,
app_name,
trade_flow,
country1,
country2,
product,
year="2011"):
request.session['app_name'] = app_name
was_redirected = request.GET.get("redirect", False)
lang = helpers.get_language(request)['code']
crawler = request.GET.get("_escaped_fragment_", False)
# Get session / request vars
prod_class = request.GET.get("prod_class",
request.session.get('product_classification',
'hs4'))
options = request.GET.copy()
options["lang"] = lang
options["product_classification"] = prod_class
options = options.urlencode()
# Setup the hash dictionary
request_hash_dictionary = collections.OrderedDict()
# Add prod class to request hash dictionary
request_hash_dictionary['app_name'] = app_name
request_hash_dictionary['lang'] = lang
request_hash_dictionary['prod_class'] = prod_class
# Add the arguments to the request hash dictionary
request_hash_dictionary['trade_flow'] = trade_flow
request_hash_dictionary['country1'] = country1
request_hash_dictionary['country2'] = country2
request_hash_dictionary['product_type'] = product
request_hash_dictionary['year'] = year
# We are here, so let us store this data somewhere
request_hash_string = "_".join(request_hash_dictionary.values())
# Code for showing a static image or not
static_image_name = helpers.url_to_hash(request.path, request.GET)
if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH,
static_image_name + ".png")):
displayviz = True
displayImage = static_image_name + ".png"
else:
displayviz = False
displayImage = settings.STATIC_URL + "img/all/loader.gif"
# Verify countries from DB
countries = [None, None]
alert = None
for i, country in enumerate([country1, country2]):
if country != "show" and country != "all":
try:
countries[i] = Country.objects.get(name_3char=country)
except Country.DoesNotExist:
alert = {
"title": "Country could not be found",
"text": """There was no country with the 3 letter
abbreviation <strong>%s</strong>. Please double check the
<a href='about/data/country/'>list of countries</a>.""" %
(country)}
# The years of data available tends to vary based on the dataset used (Hs4
# vs Sitc4) and the specific country.
years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy
years_available = years_available_model.objects\
.values_list("year", flat=True)\
.order_by("year")\
.distinct()
# Sometimes the query is not about a specific country (e.g. "all countries"
# queries) in which case filtering by country is not necessary
if countries[0]:
years_available = years_available.filter(country=countries[0].id)
# Force lazy queryset to hit the DB to reduce number of DB queries later
years_available = list(years_available)
if len(years_available) == 0:
alert = {"title": """The product classification you're using (%s) does
not seem to include data for the country code (%s) you selected.
Please try a different product classification or country.""" %
(prod_class, countries[0].name)}
years_available = range(1995, 2012) # Dummy
year1_list, year2_list = None, None
warning, title = None, None
data_as_text = {}
prod_or_partner = "partner"
# To make sure it cannot be another product class
if prod_class != "hs4" and prod_class != "sitc4":
prod_class = "sitc4"
# Test for country exceptions
if prod_class == "hs4":
# redirect if and exception country
if country1 == "bel" or country1 == "lux":
return redirect(
HTTP_HOST+'explore/%s/%s/blx/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz":
return redirect(
HTTP_HOST+'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if was_redirected:
# display warning is redirected from exception
if country1 == "blx":
warning = {
"title": "Country Substitution", "text":
"In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported as 'Belgium-Luxembourg'."}
if country1 == "zaf":
warning = {
"title": "Country Substitution", "text":
"In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."}
trade_flow_list = [
("export", _("Export")), ("import", _("Import")),
("net_export", _("Net Export")), ("net_import", _("Net Import"))]
if (app_name == "product_space" or app_name == "country_space" or app_name == "rings"):
trade_flow_list = [trade_flow_list[0]]
year1_list = range(
years_available[0],
years_available[
len(years_available) -
1] +
1,
1)
if app_name == "stacked" and year == "2009":
year = "1969.2011.10"
if "." in year:
y = [int(x) for x in year.split(".")]
year_start = y[0]
year_end = y[1]
year2_list = year1_list
else:
year_start, year_end = None, None
year = int(year)
# Check that year is within bounds
if year > years_available[len(years_available)-1]:
year = years_available[len(years_available)-1]
elif year < years_available[0]:
year = years_available[0]
api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow,
country1,
country2,
product,
year,
options)
redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class,
trade_flow,
country1,
country2,
product,
year)
country_code = None
if country1 != "show" and country1 != "all":
country_code = country1
if crawler == "":
view, args, kwargs = resolve(
"api/%s/%s/%s/%s/%s/" %
(trade_flow, country1, country2, product, year))
kwargs['request'] = request
view_response = view(*args, **kwargs)
raise Exception(view_response)
data_as_text["data"] = view_response[0]
data_as_text["total_value"] = view_response[1]
data_as_text["columns"] = view_response[2]
app_type = helpers.get_app_type(country1, country2, product, year)
# What is actually being shown on the page
if app_type == "csay" or app_type == "sapy":
item_type = "country"
else:
item_type = "product"
# Some countries need "the" before their names
list_countries_the = set(
("Cayman Islands",
"Central African Republic",
"Channel Islands",
"Congo, Dem. Rep.",
"Czech Republic",
"Dominican Republic",
"Faeroe Islands",
"Falkland Islands",
"Fm Yemen Dm",
"Lao PDR",
"Marshall Islands",
"Philippines",
"Seychelles",
"Slovak Republic",
"Syrian Arab Republic",
"Turks and Caicos Islands",
"United Arab Emirates",
"United Kingdom",
"Virgin Islands, U.S.",
"United States"))
if countries[0] and countries[0].name in list_countries_the:
countries[0].name = "the "+countries[0].name
if product not in ("show", "all"):
p_code = product
product = helpers.get_product_by_code(p_code, prod_class)
if not alert:
# Generate page title depending on visualization being used
years = [year_start, year_end] if year_start is not None else [year]
product_name = product.name_en if not isinstance(
product,
basestring) else product
country_names = [getattr(x, "name", None) for x in countries]
title = helpers.get_title(app_type, app_name,
country_names=country_names,
trade_flow=trade_flow.replace('_', ' '),
years=years,
product_name=product_name
)
if app_type in ("ccsy", "cspy"):
if _("net_export") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_export"))]
if _("net_import") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_import"))]
# Should we show the product or partner tab pane?
# quick fix should be merged with item_type
if app_type in ["cspy", "sapy"]:
prod_or_partner = "product"
elif app_type == "casy":
if app_name in ("stacked", "map", "tree_map", "pie_scatter", "product_space", "country_space", "rankings", "scatterplot"):
prod_or_partner = "product"
# Record views in redis for "newest viewed pages" visualization
if isinstance(cache, RedisCache):
views_image_path = settings.STATIC_URL + \
"data/" + request_hash_string + ".png"
view_data = {
"timestamp": time.time(),
"image": views_image_path,
"title": title,
"url": request.build_absolute_uri()
}
r = cache.raw_client
r.rpush("views", json.dumps(view_data))
previous_page = request.META.get('HTTP_REFERER',
None),
if previous_page[0] is not None:
previous_page = previous_page[0]
previous_image = helpers.url_to_hash(urlparse(previous_page).path, {})
if os.path.exists(os.path.join(settings.STATIC_IMAGE_PATH,
previous_image + ".png")):
previous_image = previous_image + ".png"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
previous_page = None
return render_to_response(
"explore/index.html",
{"lang": lang,
"warning": warning,
"alert": alert,
"prod_class": prod_class,
"data_as_text": data_as_text,
"app_name": app_name,
"title": title,
"trade_flow": trade_flow,
"country1": countries[0] or country1,
"country2": countries[1] or country2,
"country1_3char": countries[0].name_3char if countries[0] else "",
"country2_3char": countries[1].name_3char if countries[1] else "",
"product": product,
"product_code": product.code if not isinstance(product, basestring) else product,
"years_available": years_available,
"year": year,
"year_start": year_start,
"year_end": year_end,
"year1_list": year1_list,
"year2_list": year2_list,
"trade_flow_list": trade_flow_list,
"api_uri": api_uri,
"app_type": app_type,
"redesign_api_uri": redesign_api_uri,
"country_code": country_code,
"prod_or_partner": prod_or_partner,
"version": VERSION,
"previous_page": previous_page,
"previous_image": previous_image,
"item_type": item_type,
"displayviz": displayviz,
"displayImage": displayImage,
"display_iframe": request.GET.get("display_iframe", False),
"static_image": request.GET.get("static_image", "loading"),
},
context_instance=RequestContext(request))
def explore(request,
app_name,
trade_flow,
country1,
country2,
product,
year="2011"):
request.session['app_name'] = app_name
was_redirected = request.GET.get("redirect", False)
lang = helpers.get_language(request)['code']
crawler = request.GET.get("_escaped_fragment_", False)
# Get session / request vars
prod_class = request.GET.get(
"prod_class", request.session.get('product_classification', 'hs4'))
options = request.GET.copy()
options["lang"] = lang
options["product_classification"] = prod_class
options = options.urlencode()
# Setup the hash dictionary
request_hash_dictionary = collections.OrderedDict()
# Add prod class to request hash dictionary
request_hash_dictionary['app_name'] = app_name
request_hash_dictionary['lang'] = lang
request_hash_dictionary['prod_class'] = prod_class
# Add the arguments to the request hash dictionary
request_hash_dictionary['trade_flow'] = trade_flow
request_hash_dictionary['country1'] = country1
request_hash_dictionary['country2'] = country2
request_hash_dictionary['product_type'] = product
request_hash_dictionary['year'] = year
# We are here, so let us store this data somewhere
request_hash_string = "_".join(request_hash_dictionary.values())
# Code for showing a static image or not
static_image_name = helpers.url_to_hash(request.path, request.GET)
if os.path.exists(
os.path.join(settings.STATIC_IMAGE_PATH,
static_image_name + ".png")):
displayviz = True
displayImage = static_image_name + ".png"
else:
displayviz = False
displayImage = settings.STATIC_URL + "img/all/loader.gif"
# Verify countries from DB
countries = [None, None]
alert = None
for i, country in enumerate([country1, country2]):
if country != "show" and country != "all":
try:
countries[i] = Country.objects.get(name_3char=country)
except Country.DoesNotExist:
alert = {
"title":
"Country could not be found",
"text":
"""There was no country with the 3 letter
abbreviation <strong>%s</strong>. Please double check the
<a href='about/data/country/'>list of countries</a>.""" %
(country)
}
# The years of data available tends to vary based on the dataset used (Hs4
# vs Sitc4) and the specific country.
years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy
years_available = years_available_model.objects\
.values_list("year", flat=True)\
.order_by("year")\
.distinct()
# Sometimes the query is not about a specific country (e.g. "all countries"
# queries) in which case filtering by country is not necessary
if countries[0]:
years_available = years_available.filter(country=countries[0].id)
# Force lazy queryset to hit the DB to reduce number of DB queries later
years_available = list(years_available)
if len(years_available) == 0:
alert = {
"title":
"""The product classification you're using (%s) does
not seem to include data for the country code (%s) you selected.
Please try a different product classification or country.""" %
(prod_class, countries[0].name)
}
years_available = range(1995, 2012) # Dummy
year1_list, year2_list = None, None
warning, title = None, None
data_as_text = {}
prod_or_partner = "partner"
# To make sure it cannot be another product class
if prod_class != "hs4" and prod_class != "sitc4":
prod_class = "sitc4"
# Test for country exceptions
if prod_class == "hs4":
# redirect if and exception country
if country1 == "blx" or country1 == "lux":
return redirect(HTTP_HOST +
'explore/%s/%s/bel/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz":
return redirect(HTTP_HOST +
'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' %
(app_name, trade_flow, country2, product, year))
if was_redirected:
# display warning is redirected from exception
if country1 == "bel":
warning = {
"title":
"Country Substitution",
"text":
"In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported under 'BEL' ."
}
if country1 == "zaf":
warning = {
"title":
"Country Substitution",
"text":
"In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."
}
trade_flow_list = [("export", _("Export")), ("import", _("Import")),
("net_export", _("Net Export")),
("net_import", _("Net Import"))]
if (app_name == "product_space" or app_name == "country_space"
or app_name == "rings"):
trade_flow_list = [trade_flow_list[0]]
year1_list = range(years_available[0],
years_available[len(years_available) - 1] + 1, 1)
if app_name == "stacked" and year == "2009":
year = "1969.2011.10"
if "." in year:
y = [int(x) for x in year.split(".")]
year_start = y[0]
year_end = y[1]
year2_list = year1_list
else:
year_start, year_end = None, None
year = int(year)
# Check that year is within bounds
if year > years_available[len(years_available) - 1]:
year = years_available[len(years_available) - 1]
elif year < years_available[0]:
year = years_available[0]
api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2,
product, year, options)
redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (
prod_class, trade_flow, country1, country2, product, year)
country_code = None
if country1 != "show" and country1 != "all":
country_code = country1
if crawler == "":
view, args, kwargs = resolve(
"api/%s/%s/%s/%s/%s/" %
(trade_flow, country1, country2, product, year))
kwargs['request'] = request
view_response = view(*args, **kwargs)
raise Exception(view_response)
data_as_text["data"] = view_response[0]
data_as_text["total_value"] = view_response[1]
data_as_text["columns"] = view_response[2]
app_type = helpers.get_app_type(country1, country2, product, year)
# What is actually being shown on the page
if app_type == "csay" or app_type == "sapy":
item_type = "country"
else:
item_type = "product"
# Some countries need "the" before their names
list_countries_the = set(
("Cayman Islands", "Central African Republic", "Channel Islands",
"Congo, Dem. Rep.", "Czech Republic", "Dominican Republic",
"Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR",
"Marshall Islands", "Philippines", "Seychelles", "Slovak Republic",
"Syrian Arab Republic", "Turks and Caicos Islands",
"United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.",
"United States"))
if countries[0] and countries[0].name in list_countries_the:
countries[0].name = "the " + countries[0].name
if product not in ("show", "all"):
p_code = product
product = helpers.get_product_by_code(p_code, prod_class)
if not alert:
# Generate page title depending on visualization being used
years = [year_start, year_end] if year_start is not None else [year]
product_name = product.name_en if not isinstance(
product, basestring) else product
country_names = [getattr(x, "name", None) for x in countries]
title = helpers.get_title(app_type,
app_name,
country_names=country_names,
trade_flow=trade_flow.replace('_', ' '),
years=years,
product_name=product_name)
if app_type in ("ccsy", "cspy"):
if _("net_export") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_export"))]
if _("net_import") in trade_flow_list:
del trade_flow_list[trade_flow_list.index(_("net_import"))]
# Should we show the product or partner tab pane?
# quick fix should be merged with item_type
if app_type in ["cspy", "sapy"]:
prod_or_partner = "product"
elif app_type == "casy":
if app_name in ("stacked", "map", "tree_map", "pie_scatter",
"product_space", "country_space", "rankings",
"scatterplot"):
prod_or_partner = "product"
# Record views in redis for "newest viewed pages" visualization
if isinstance(cache, RedisCache):
views_image_path = settings.STATIC_URL + \
"data/" + request_hash_string + ".png"
view_data = {
"timestamp": time.time(),
"image": views_image_path,
"title": title,
"url": request.build_absolute_uri()
}
r = cache.raw_client
r.rpush("views", json.dumps(view_data))
previous_page = request.META.get('HTTP_REFERER', None),
if previous_page[0] is not None:
previous_page = previous_page[0]
previous_image = helpers.url_to_hash(urlparse(previous_page).path, {})
if os.path.exists(
os.path.join(settings.STATIC_IMAGE_PATH,
previous_image + ".png")):
previous_image = previous_image + ".png"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
else:
previous_image = settings.STATIC_URL + "img/all/loader.gif"
previous_page = None
return render_to_response("explore/index.html", {
"lang":
lang,
"warning":
warning,
"alert":
alert,
"prod_class":
prod_class,
"data_as_text":
data_as_text,
"app_name":
app_name,
"title":
title,
"trade_flow":
trade_flow,
"country1":
countries[0] or country1,
"country2":
countries[1] or country2,
"country1_3char":
countries[0].name_3char if countries[0] else "",
"country2_3char":
countries[1].name_3char if countries[1] else "",
"product":
product,
"product_code":
product.code if not isinstance(product, basestring) else product,
"years_available":
years_available,
"year":
year,
"year_start":
year_start,
"year_end":
year_end,
"year1_list":
year1_list,
"year2_list":
year2_list,
"trade_flow_list":
trade_flow_list,
"api_uri":
api_uri,
"app_type":
app_type,
"redesign_api_uri":
redesign_api_uri,
"country_code":
country_code,
"prod_or_partner":
prod_or_partner,
"version":
VERSION,
"previous_page":
previous_page,
"previous_image":
previous_image,
"item_type":
item_type,
"displayviz":
displayviz,
"displayImage":
displayImage,
"display_iframe":
request.GET.get("display_iframe", False),
"static_image":
request.GET.get("static_image", "loading"),
},
context_instance=RequestContext(request))