def handleAtlasFile(self):
atlasFileData = open(self.curFilePath)
dataLines = atlasFileData.readlines()
frames = {}
for dataLine in dataLines:
dataLine = str(dataLine)
frameInfos = dataLine.split(" ")
frames[str(frameInfos[0])] = frameInfos[1:]
splitPos = self.curFilePath.rfind("/")
fileDir = self.curFilePath[0:splitPos + 1]
fileName = self.curFilePath[splitPos + 1:]
dotPos = fileName.rfind(".")
textureFileName = fileName[0:dotPos] + ".png"
texture = Texture()
texture.texDir = fileDir
texture.texName = textureFileName
texPath = str(texture.texDir + texture.texName)
if not os.path.exists(texPath):
print "texture %s not exist!" % texPath
return texPath, None
texFile = Image.open(texPath)
texSize = texFile.size
w, h = texSize[0], texSize[0]
texture.init(fileDir, textureFileName, w, h)
frameArr = []
for key in frames:
frame = Frame()
frame.name = str(key)
frame.tex = texture
self.decodeFrame(frame, frames[key])
frameArr.append(frame)
return texture, frameArr
def get_regionalized_dataframe(data, label_field):
# We will store the final result here
ndf = Frame()
# Now we get the data that we want to average by region (defined in the groups)
# and also total
for g in groups:
# Data points
ctrs = c.get_groups(g)
for ctry in ctrs:
for r in data.rows:
lv = r.get_by_column_name(label_field).value.get()
if lv == ctry.name or lv in ctry.alias:
r.get_by_column_name(label_field).value.value = ctry.name
ndf.rows.append(r)
return ndf
def extract_countries(g, column_name, df):
n_df = Frame()
c = CountryList()
c.load_wb()
names = []
for ctr_name in g:
ctr = c.get_country_by_name(ctr_name)
names.append(ctr.name)
for i in ctr.alias:
names.append(i)
for r in df.rows:
if (r.get_by_column_name(column_name).value.value in names):
n_df.rows.append(r)
return (n_df)
def get(data, groups, country_field):
# We will store the final result here
ndf = Frame()
cl = CountryList()
cl.load_wb()
# Now we get the data that we want to average by region (defined in the groups)
# and also total
# Let's conflate
cs = []
for cnt in cl.get_groups(groups):
if cnt.name not in cs:
cs.append(cnt.name)
for r in data.rows:
if r.get_by_column_name(country_field).value.get() in cs:
ndf.rows.append(r)
return ndf
def json_to_frame(j):
df = Frame()
for r in j:
df.add_row(r)
return df
import re
import requests
from data.Frame import Frame
from countries.list import CountryList
from sources.sdgs.utils import json_to_frame
series = Frame()
def load_series():
series = json_to_frame(requests.get(url).json())
return series
def get_series(series_code,
area_code,
time_period,
dimensions,
page_size=10000):
url = 'https://unstats.un.org/SDGAPI/v1/sdg/Series/Data'
q = "%s?%s&%s&%s&%s&%s" % (url, series_code, area_code, time_period,
dimensions, page_size)
series = json_to_frame(requests.get(url).json())
return series
import re
import requests
from data.Frame import Frame
from geo.list import CountryList
from sources.sdgs.utils import json_to_frame
areas = Frame()
def load_areas():
url = 'https://unstats.un.org/SDGAPI/v1/sdg/GeoArea/List'
areas = json_to_frame(requests.get(url).json())
return areas
import re
import requests
from data.Frame import Frame
from countries.list import CountryList
__url_datasets = 'http://urbandata.unhabitat.org/wp-admin/admin-ajax.php?action=load_from_oipa&call=indicator-filter-options&format=json'
regions = Frame()
indicators = Frame()
cities = Frame()
countries = Frame()
def load_datasets():
url = __url_datasets
domains = json_to_frame(requests.get(url).json())
return domains
''' Download FAO data '''
import re
import requests
from data.Frame import Frame
from sources.fao.utils import json_to_frame
from geo.list import CountryList
__version__ = '0.1.0'
__url_datasets__ = "http://fenixservices.fao.org/faostat/static/bulkdownloads/datasets_E.json"
datasets = []
domains = Frame()
countries = Frame()
def get_domains():
url = 'http://fenixservices.fao.org/faostat/api/v1/en/groupsanddomains'
domains = json_to_frame(requests.get(url).json())
return domains
def load_countries():
url = 'http://fenixservices.fao.org/faostat/api/v1/en/codes/countries/IG/?show_lists=true'
countries = json_to_frame(requests.get(url).json())
return countries
# Get the code of a country
# needed to perform a query
def calculate_weighted_average(df_data, df_weight, label_field):
df_w = Frame()
# Go through all data rows
for d in df_data.rows:
dr = Row()
# Now search data for weights based in the label field
# We have to also search in a row for the label_field
# and just the first result as the list will only contain
# one value
dr.add_cell(
Cell(column=label_field,
value=d.get_by_column_name(label_field).value.value))
# If we are in a column that is not the label column
# Now we go through all columns
for c in d.cells:
if c.column.name != label_field:
data_value = d.get_by_column_name(c.column.name).value.value
dr.add_cell(
Cell(column="%s_%s" % (c.column.name, 'value'),
value=data_value))
# in the new row we have already created the label field that we
# can use here
# We have to check that the data point exists for the value that we are trying to weight
if (len(
df_weight.search(
label_field,
dr.get_by_column_name(
label_field).value.value).rows) > 0):
weight_value = df_weight.search(
label_field,
dr.get_by_column_name(label_field).value.value
).rows[0].get_by_column_name(c.column.name).value.value
else:
weight_value = None
if data_value is None or weight_value is None:
data_value = None
weight_value = None
dr.add_cell(
Cell(column="%s_%s" % (c.column.name, 'weight'),
value=weight_value))
# We will store the weighted value here
dr.add_cell(
Cell(column="%s_%s" % (c.column.name, 'weighted_value'),
value=None))
# We add a column where the weighted value will be stored
df_w.rows.append(dr)
# Now let's calculated the weighted values
for d in df_data.rows:
for c in d.cells:
if c.column.name != label_field:
# Initialize variables to avoid carried-over values
# from previous iteration
data_value = None
weight_value = None
weighted_value = None
# Check if weight value exists for data point
if len(
df_weight.search(
label_field,
d.get_by_column_name(
label_field).value.value).rows) > 0:
if c.value.value is not None and df_weight.search(
label_field,
d.get_by_column_name(label_field).value.value
).rows[0].get_by_column_name(
c.column.name).value.value is not None:
data_value = c.value.value
weight_value = df_weight.search(
label_field,
d.get_by_column_name(label_field).value.value
).rows[0].get_by_column_name(c.column.name).value.value
total_weight = df_w.get_column(
"%s_%s" % (c.column.name, 'weight')).sum()
if data_value is not None and weight_value is not None:
if is_number(data_value) == True and is_number(
weight_value) == True:
weighted_value = (
float(weight_value) /
float(total_weight)) * float(data_value)
else:
weighted_value = None
else:
weighted_value = None
# If weight value does not exists for the data point
else:
weighted_value = None
df_w.search(
label_field,
d.get_by_column_name(
label_field).value.value).rows[0].get_by_column_name(
"%s_%s" %
(c.column.name,
'weighted_value')).value.value = weighted_value
dr = Row()
dr.add_cell(Cell(column=label_field, value='Average'))
for c in df_w.get_column_names():
total_weight = None
if df_weight.get_column(c).get_type() == 'number':
total_weight = df_w.get_column(c).sum()
dr.add_cell(Cell(column=c, value=float(total_weight)))
df_w.rows.append(dr)
df_w.id = df_data.id + " - " + df_weight.id
df_w.description = df_data.description + " weighted against " + df_weight.description
df_w.source = df_data.id + " : " + df_data.source + " and " + df_weight.id + " : " + df_weight.source
df_w.source_url = df_data.id + " : " + df_data.source_url + " and " + df_weight.id + " : " + df_weight.source_url
return df_w
''' Download SDGs data '''
import re
import requests
from data.Frame import Frame
from geo.list import CountryList
from sources.sdgs.utils import json_to_frame
__version__ = '0.1.0'
__url__ = "https://unstats.un.org/SDGAPI/v1/"
datasets = []
domains = Frame()
countries = Frame()
indicators = Frame()
def load_indicators():
url = 'https://unstats.un.org/SDGAPI/v1/sdg/indicator/List'
indicators = json_to_frame(requests.get(url).json())
return countries