def get_estimates_map(request): """ Generates a Map page containing the routes and traffic flow. :param request: The web request that specifies search criteria. :return: The web page. """ data = {'tab_selected': 'map'} results = get_estimates_common(request, data) data['map_year'] = globals.get_map_year(data['query']["year__gte"], data['query']["year__lte"]) # Group estimates by embarkation and disembarkation geocodes. regions = {} flows = {} cache = EstimateManager.cache() for pk in get_pks_from_haystack_results(results): result = cache[int(pk)] dregion = result.disembarkation_region.name eregion = result.embarkation_region.name regions[dregion] = (result.disembarkation_region.latitude, result.disembarkation_region.longitude, result.disembarkation_region.import_area) regions[eregion] = (result.embarkation_region.latitude, result.embarkation_region.longitude, result.embarkation_region.export_area) key = eregion + "_" + dregion item = (eregion, dregion, 0, 0) if key in flows: item = flows[key] flows[key] = (eregion, dregion, item[2] + result.embarked_slaves, item[3] + result.disembarked_slaves) data['regions'] = regions data['flows'] = flows return render(request, 'assessment/estimates.html', data)
def get_results_map_flow(request, results): map_ports = {} map_flows = {} def add_port(geo): result = geo is not None and len(geo) == 3 and geo[0].show and geo[1].show and geo[2].show and \ geo[0].lat and geo[0].lng and geo[1].lat and geo[1].lng and geo[2].lat and geo[2].lng if result and not geo[0].pk in map_ports: map_ports[geo[0].pk] = geo return result def add_flow(source, destination, embarked, disembarked): result = embarked is not None and disembarked is not None and add_port( source) and add_port(destination) if result: flow_key = long(source[0].pk) * 2147483647 + long( destination[0].pk) current = map_flows.get(flow_key) if current is not None: embarked += current[2] disembarked += current[3] map_flows[flow_key] = (source[0].name, destination[0].name, embarked, disembarked) return result # Ensure cache is loaded. VoyageCache.load() all_voyages = VoyageCache.voyages missed_embarked = 0 missed_disembarked = 0 # Set up an unspecified source that will be used if the appropriate setting is enabled geo_unspecified = CachedGeo(-1, -1, _('Africa, port unspecified'), 0.05, 9.34, True, None) source_unspecified = (geo_unspecified, geo_unspecified, geo_unspecified) keys = get_pks_from_haystack_results(results) for pk in keys: voyage = all_voyages.get(pk) if voyage is None: continue source = CachedGeo.get_hierarchy(voyage.emb_pk) if source is None and settings.MAP_MISSING_SOURCE_ENABLED: source = source_unspecified destination = CachedGeo.get_hierarchy(voyage.dis_pk) add_flow(source, destination, voyage.embarked, voyage.disembarked) if missed_embarked > 0 or missed_disembarked > 0: import logging logging.getLogger('voyages').info('Missing flow: (' + str(missed_embarked) + ', ' + str(missed_disembarked) + ')') return render(request, "search_maps.datatemplate", { 'map_ports': map_ports, 'map_flows': map_flows }, content_type='text/javascript')
def get_results_map_animation(results, allow_no_numbers=False): VoyageCache.load() all_voyages = VoyageCache.voyages keys = get_pks_from_haystack_results(results) items = [] for pk in keys: voyage = all_voyages.get(pk) if voyage is None: print "Missing voyage with PK" + str(pk) continue def can_show(port_pk): ph = CachedGeo.get_hierarchy(port_pk) return ph is not None and (ph[0].show or ph[1].show) if can_show(voyage.emb_pk) and can_show(voyage.dis_pk) and \ voyage.year is not None and \ (allow_no_numbers or \ (voyage.embarked is not None and voyage.embarked > 0 and voyage.disembarked is not None)): flag = VoyageCache.nations.get(voyage.ship_nat_pk) if flag is None: flag = '' hsrc = CachedGeo.get_hierarchy(voyage.emb_pk) hdst = CachedGeo.get_hierarchy(voyage.dis_pk) items.append({ "voyage_id": voyage.voyage_id, "src": voyage.emb_pk, "dst": voyage.dis_pk, "regsrc": hsrc[1].pk, "bregsrc": hsrc[2].pk, "bregdst": hdst[2].pk, "embarked": voyage.embarked or 0, "disembarked": voyage.disembarked or 0, "year": voyage.year, "month": voyage.month, "ship_ton": voyage.ship_ton if voyage.ship_ton is not None else 0, "nat_id": voyage.ship_nat_pk if voyage.ship_nat_pk is not None else 0, "ship_name": unicode(voyage.ship_name) if voyage.ship_name is not None else '', }) return JsonResponse(items, safe=False)
def get_estimates_timeline(request): """ Generates a Time-line page with total traffic volume per year. :param request: The web request that specifies search criteria. :return: The web page or an XLS spreadsheet with timeline data. """ data = {'tab_selected': 'timeline'} results = get_estimates_common(request, data) data['show_events'] = data['all_nations_selected'] and data['all_embarkations_selected'] and\ data['all_disembarkations_selected'] # Group estimates by year and sum embarked and disembarked for year. # The following dict has keys corresponding to years and entries # formed by tuples (embarked_count, disembarked_count) timeline = {} cache = EstimateManager.cache() for pk in get_pks_from_haystack_results(results): result = cache[pk] item = (0, 0) if result.year in timeline: item = timeline[result.year] timeline[result.year] = (item[0] + result.embarked_slaves, item[1] + result.disembarked_slaves) query = data['query'] data['min_year'] = query['year__gte'] if query['year__gte'] > globals.default_first_year\ else globals.default_first_year data['max_year'] = query['year__lte'] if query['year__lte'] < globals.default_last_year\ else globals.default_last_year data['timeline'] = timeline data['min_year'] -= 1 data['max_year'] += 1 post = data['post'] if post is None or "download" not in post: return render(request, 'assessment/estimates.html', data) else: rows = [[k, int(round(t[0])), int(round(t[1]))] for k, t in timeline.iteritems()] rows = sorted(rows, key=lambda a: a[0]) return download_xls([[('Year', 1), ('Embarked Slaves', 1), ('Disembarked Slaves', 1)]], rows)
def get_estimates_table(request): """ Generate tabular data summarizing the estimates. :param request: The web request containing search data and tabular layout :return: The rendered page or an XLS spreadsheet containing table data. """ data = {'tab_selected': 'table'} results = get_estimates_common(request, data) def year_mod(year, mod): """ Helper function that groups years according to a given modulus. For instance, year_mod(1501, 5) = year_mod(1502, 5) = ... = year_mod(1505, 5). :param year: The year. :param mod: The length of each year interval. :return: A pair (mod, index of year interval). """ year -= 1 return mod, (year - (year % mod)) / mod # key_functions dictionary specifies grouping functions. key_functions = { '0': lambda e: e.nation, '1': lambda e: e.embarkation_region, '2': lambda e: e.disembarkation_region.import_area, '3': lambda e: e.disembarkation_region, '4': lambda e: e.year, '5': lambda e: year_mod(e.year, 5), '6': lambda e: year_mod(e.year, 10), '7': lambda e: year_mod(e.year, 25), '8': lambda e: year_mod(e.year, 50), '9': lambda e: year_mod(e.year, 100), } # Select key functions based on post data or use default values if this is the initial GET request. row_key_index = '7' col_key_index = '0' cell_key_index = '1' include_empty = False; estimate_selection_form = None post = data['post'] # Fetch EstimateSelectionForm either from Post (if any), from Session (if any) or a default form. if post is not None: estimate_selection_form = EstimateSelectionForm(post) if (estimate_selection_form is None or not estimate_selection_form.is_valid()) \ and "estimate_selection_form" in request.session: estimate_selection_form = request.session["estimate_selection_form"] if post is not None and post.get("table_options") == "Reset to default": estimate_selection_form = None if estimate_selection_form is not None and estimate_selection_form.is_valid(): cell_key_index = estimate_selection_form.cleaned_data["cells"] col_key_index = estimate_selection_form.cleaned_data["columns"] row_key_index = estimate_selection_form.cleaned_data["rows"] include_empty = estimate_selection_form.cleaned_data["include_empty"] else: estimate_selection_form = EstimateSelectionForm(initial={ 'rows': row_key_index, 'columns': col_key_index, 'cells': cell_key_index}) row_key_function = key_functions[row_key_index] col_key_function = key_functions[col_key_index] data['table_form'] = estimate_selection_form # Save form to session so that if the user navigates elsewhere and then returns, the form is unchanged. request.session["estimate_selection_form"] = estimate_selection_form # Aggregate results according to the row and column keys. # Each result is a pair (tuple) containing total embarked and total disembarked. table_dict = {} cache = EstimateManager.cache() if include_empty: # Force loading entire rows or columns which have zero value. # This is done by projecting the entire set of data on each key function # and forming the Cartesian product of the two. query = data['query'] year_filter = lambda e: query['year__gte'] <= e.year <= query['year__lte'] filter_functions = { '0': lambda e: e.nation.name in query['nation__in'], '1': lambda e: e.embarkation_region.name in query['embarkation_region__in'], '2': lambda e: e.disembarkation_region.name in query['disembarkation_region__in'], '3': lambda e: e.disembarkation_region.name in query['disembarkation_region__in'], '4': year_filter, '5': year_filter, '6': year_filter, '7': year_filter, '8': year_filter, '9': year_filter, } row_filter = filter_functions[row_key_index] col_filter = filter_functions[col_key_index] estimates = cache.values() all_row_keys = set([row_key_function(e) for e in estimates if row_filter(e)]) all_col_keys = set([col_key_function(e) for e in estimates if col_filter(e)]) table_dict = {(rk, ck): (0, 0) for rk in all_row_keys for ck in all_col_keys} for pk in get_pks_from_haystack_results(results): result = cache.get(pk) if result is not None: key = (row_key_function(result), col_key_function(result)) cell = (0, 0) if key in table_dict: cell = table_dict[key] cell = (cell[0] + result.embarked_slaves, cell[1] + result.disembarked_slaves) table_dict[key] = cell def header_with_name(x): return x.name def header_year_range(x): return str(x[0] * x[1] + 1) + '-' + str(x[0] * (1 + x[1])) # header_functions specifies how the row/column headers are generated. header_functions = { '0': header_with_name, '1': header_with_name, '2': header_with_name, '3': header_with_name, '4': lambda x: x, '5': header_year_range, '6': header_year_range, '7': header_year_range, '8': header_year_range, '9': header_year_range } row_header_function = header_functions[row_key_index] col_header_function = header_functions[col_key_index] # Order columns and rows by their respective headers. default_sort_fun = lambda x: x.order_num row_sort_fun = default_sort_fun if int(row_key_index) < 4 else row_header_function row_set = sorted(set([k[0] for k in table_dict.keys()]), key=row_sort_fun) column_set = sorted(set([k[1] for k in table_dict.keys()]), key=default_sort_fun) # How many cells a single piece of data spans # (1 for either embarked or disembarked only and 2 for both). cell_span = 2 if cell_key_index == '0' else 1 def specific_region_grouping(original_set, header_list, span_multiplier=1): """ Helper function that allows grouping column or row headers when the headers are composed of Area and Region. :param original_set - the original set of columns or rows. :param header_list - the list of (row or column) headers that will be appended by area headers. :param span_multiplier - a multiplicity factor for the columnspan or rowspan of each cell. :return: original_set sorted according to area. """ key_map = lambda region: region.import_area.name modified_set = sorted(original_set, key=lambda region: region.import_area.order_num) from itertools import groupby header_list.append([(k, span_multiplier * sum(1 for x in g)) for k, g in groupby(modified_set, key_map)]) return modified_set # Header rows are encoded as an array of pairs (tuples), where each # pair consists of the header's label and column span. header_rows = [] if col_key_index == '3': column_set = specific_region_grouping(column_set, header_rows, cell_span) header_rows[0].append(('', cell_span)) header_rows.append([(col_header_function(x), cell_span) for x in column_set]) header_rows[-1].append(('Totals', cell_span)) # Generate row headers (left-most columns). row_headers = [] if row_key_index == '3': row_set = specific_region_grouping(row_set, row_headers) row_headers[0].append(('', 1)) row_headers.append([(row_header_function(r), 1) for r in row_set] + [('Totals', 1)]) cell_display_list = [] if cell_key_index == '0': # Use list comprehensions to repeat the pair of cells # Embarked, Disembarked for each column in column_set. helper = ['Embarked', 'Disembarked'] header_rows.append([(s, 1) for i in range(1 + len(column_set)) for s in helper]) cell_display_list = [0, 1] elif cell_key_index == '1': cell_display_list = [0] elif cell_key_index == '2': cell_display_list = [1] data['header_rows'] = header_rows data['header_rows_len'] = len(header_rows) data['totals_header_rows_len'] = len(header_rows) - (1 if cell_key_index == '0' else 0) data['totals_header_cols_span'] = len(cell_display_list) # Generate tabular data from table_dict filling any missing entries with (0, 0). # At this point each entry of the table is a pair (embarked_count, disembarked_count). full_data_set = [[table_dict[(r, c)] if (r, c) in table_dict else (0, 0) for c in column_set] for r in row_set] # Round numbers to integers. full_data_set = [[tuple(int(round(pair[i])) for i in range(2)) for pair in r] for r in full_data_set] # Append row totals (last column). full_data_set = [r + [tuple(sum([x[i] for x in r]) for i in range(2))] for r in full_data_set] # Append column totals (last row). full_data_set.append([tuple(sum([full_data_set[i][j][k] for i in range(len(row_set))]) for k in range(2)) for j in range(1 + len(column_set))]) # Transform pairs(embarked_count, disembarked_count) by projecting their values # into single integers using the cell_display_list to determine which numbers # (embarked, disembarked, or both) should appear in the final table. full_data_set = [[pair[i] for pair in r for i in cell_display_list] for r in full_data_set] data['rows'] = full_data_set if post is None or "download" not in post: # Alter row_headers so that it is simpler to use in the template. tmp = [[None for x in row_headers] for r in full_data_set] col_number = 0 for row_header_col in row_headers: row_number = 0 for cell in row_header_col: tmp[row_number][col_number] = cell row_number += cell[1] col_number += 1 data['row_headers'] = tmp data['row_headers_count'] = len(row_headers) return render(request, 'assessment/estimates.html', data) else: return download_xls(header_rows, full_data_set, row_headers)
def get_estimates_table(request): """ Generate tabular data summarizing the estimates. :param request: The web request containing search data and tabular layout :return: The rendered page or an XLS spreadsheet containing table data. """ data = {'tab_selected': 'table'} results = get_estimates_common(request, data) def year_mod(year, mod): """ Helper function that groups years according to a given modulus. For instance, year_mod(1501, 5) = year_mod(1502, 5) = ... = year_mod(1505, 5). :param year: The year. :param mod: The length of each year interval. :return: A pair (mod, index of year interval). """ year -= 1 return mod, (year - (year % mod)) / mod # key_functions dictionary specifies grouping functions. key_functions = { '0': lambda e: e.nation, '1': lambda e: e.embarkation_region, '2': lambda e: e.disembarkation_region.import_area, '3': lambda e: e.disembarkation_region, '4': lambda e: e.year, '5': lambda e: year_mod(e.year, 5), '6': lambda e: year_mod(e.year, 10), '7': lambda e: year_mod(e.year, 25), '8': lambda e: year_mod(e.year, 50), '9': lambda e: year_mod(e.year, 100), } # Select key functions based on post data or use default values if this is the initial GET request. row_key_index = '7' col_key_index = '0' cell_key_index = '1' include_empty = False estimate_selection_form = None post = data['post'] # Fetch EstimateSelectionForm either from Post (if any), from Session (if any) or a default form. if post is not None: estimate_selection_form = EstimateSelectionForm(post) if (estimate_selection_form is None or not estimate_selection_form.is_valid()) \ and "estimate_selection_form" in request.session: estimate_selection_form = request.session["estimate_selection_form"] if post is not None and post.get("table_options") == "Reset to default": estimate_selection_form = None if estimate_selection_form is not None and estimate_selection_form.is_valid( ): cell_key_index = estimate_selection_form.cleaned_data["cells"] col_key_index = estimate_selection_form.cleaned_data["columns"] row_key_index = estimate_selection_form.cleaned_data["rows"] include_empty = estimate_selection_form.cleaned_data["include_empty"] else: estimate_selection_form = EstimateSelectionForm( initial={ 'rows': row_key_index, 'columns': col_key_index, 'cells': cell_key_index }) row_key_function = key_functions[row_key_index] col_key_function = key_functions[col_key_index] data['table_form'] = estimate_selection_form # Save form to session so that if the user navigates elsewhere and then returns, the form is unchanged. request.session["estimate_selection_form"] = estimate_selection_form # Aggregate results according to the row and column keys. # Each result is a pair (tuple) containing total embarked and total disembarked. table_dict = {} cache = EstimateManager.cache() if include_empty: # Force loading entire rows or columns which have zero value. # This is done by projecting the entire set of data on each key function # and forming the Cartesian product of the two. query = data['query'] year_filter = lambda e: query['year__gte'] <= e.year <= query[ 'year__lte'] filter_functions = { '0': lambda e: e.nation.name in query['nation__in'], '1': lambda e: e.embarkation_region.name in query[ 'embarkation_region__in'], '2': lambda e: e.disembarkation_region.name in query[ 'disembarkation_region__in'], '3': lambda e: e.disembarkation_region.name in query[ 'disembarkation_region__in'], '4': year_filter, '5': year_filter, '6': year_filter, '7': year_filter, '8': year_filter, '9': year_filter, } row_filter = filter_functions[row_key_index] col_filter = filter_functions[col_key_index] estimates = cache.values() all_row_keys = set( [row_key_function(e) for e in estimates if row_filter(e)]) all_col_keys = set( [col_key_function(e) for e in estimates if col_filter(e)]) table_dict = {(rk, ck): (0, 0) for rk in all_row_keys for ck in all_col_keys} for pk in get_pks_from_haystack_results(results): result = cache.get(pk) if result is not None: key = (row_key_function(result), col_key_function(result)) cell = (0, 0) if key in table_dict: cell = table_dict[key] cell = (cell[0] + result.embarked_slaves, cell[1] + result.disembarked_slaves) table_dict[key] = cell def header_with_name(x): return x.name def header_year_range(x): return str(x[0] * x[1] + 1) + '-' + str(x[0] * (1 + x[1])) # header_functions specifies how the row/column headers are generated. header_functions = { '0': header_with_name, '1': header_with_name, '2': header_with_name, '3': header_with_name, '4': lambda x: x, '5': header_year_range, '6': header_year_range, '7': header_year_range, '8': header_year_range, '9': header_year_range } row_header_function = header_functions[row_key_index] col_header_function = header_functions[col_key_index] # Order columns and rows by their respective headers. default_sort_fun = lambda x: x.order_num row_sort_fun = default_sort_fun if int( row_key_index) < 4 else row_header_function row_set = sorted(set([k[0] for k in table_dict.keys()]), key=row_sort_fun) column_set = sorted(set([k[1] for k in table_dict.keys()]), key=default_sort_fun) # How many cells a single piece of data spans # (1 for either embarked or disembarked only and 2 for both). cell_span = 2 if cell_key_index == '0' else 1 def specific_region_grouping(original_set, header_list, span_multiplier=1): """ Helper function that allows grouping column or row headers when the headers are composed of Area and Region. :param original_set - the original set of columns or rows. :param header_list - the list of (row or column) headers that will be appended by area headers. :param span_multiplier - a multiplicity factor for the columnspan or rowspan of each cell. :return: original_set sorted according to area. """ key_map = lambda region: region.import_area.name modified_set = sorted(original_set, key=lambda region: region.import_area.order_num) from itertools import groupby header_list.append([(k, span_multiplier * sum(1 for x in g)) for k, g in groupby(modified_set, key_map)]) return modified_set # Header rows are encoded as an array of pairs (tuples), where each # pair consists of the header's label and column span. header_rows = [] if col_key_index == '3': column_set = specific_region_grouping(column_set, header_rows, cell_span) header_rows[0].append(('', cell_span)) header_rows.append([(col_header_function(x), cell_span) for x in column_set]) header_rows[-1].append(('Totals', cell_span)) # Generate row headers (left-most columns). row_headers = [] if row_key_index == '3': row_set = specific_region_grouping(row_set, row_headers) row_headers[0].append(('', 1)) row_headers.append([(row_header_function(r), 1) for r in row_set] + [('Totals', 1)]) cell_display_list = [] if cell_key_index == '0': # Use list comprehensions to repeat the pair of cells # Embarked, Disembarked for each column in column_set. helper = ['Embarked', 'Disembarked'] header_rows.append([(s, 1) for i in range(1 + len(column_set)) for s in helper]) cell_display_list = [0, 1] elif cell_key_index == '1': cell_display_list = [0] elif cell_key_index == '2': cell_display_list = [1] data['header_rows'] = header_rows data['header_rows_len'] = len(header_rows) data['totals_header_rows_len'] = len(header_rows) - (1 if cell_key_index == '0' else 0) data['totals_header_cols_span'] = len(cell_display_list) # Generate tabular data from table_dict filling any missing entries with (0, 0). # At this point each entry of the table is a pair (embarked_count, disembarked_count). full_data_set = [[ table_dict[(r, c)] if (r, c) in table_dict else (0, 0) for c in column_set ] for r in row_set] # Round numbers to integers. full_data_set = [[ tuple(int(round(pair[i])) for i in range(2)) for pair in r ] for r in full_data_set] # Append row totals (last column). full_data_set = [ r + [tuple(sum([x[i] for x in r]) for i in range(2))] for r in full_data_set ] # Append column totals (last row). full_data_set.append([ tuple( sum([full_data_set[i][j][k] for i in range(len(row_set))]) for k in range(2)) for j in range(1 + len(column_set)) ]) # Transform pairs(embarked_count, disembarked_count) by projecting their values # into single integers using the cell_display_list to determine which numbers # (embarked, disembarked, or both) should appear in the final table. full_data_set = [[pair[i] for pair in r for i in cell_display_list] for r in full_data_set] data['rows'] = full_data_set if post is None or "download" not in post: # Alter row_headers so that it is simpler to use in the template. tmp = [[None for x in row_headers] for r in full_data_set] col_number = 0 for row_header_col in row_headers: row_number = 0 for cell in row_header_col: tmp[row_number][col_number] = cell row_number += cell[1] col_number += 1 data['row_headers'] = tmp data['row_headers_count'] = len(row_headers) return render(request, 'assessment/estimates.html', data) else: return download_xls(header_rows, full_data_set, row_headers)