def download():
setproperties.load_properties()
for i in xrange(35):
print "page %s" % i
response = requests.get(sources[1] + "?page=%s" % i, verify=False)
properties = get_url_title_and_description_from_html(response.text)
for filename in properties:
print filename
filepath = os.path.join("data", "raw", filename)
if not os.path.exists(filepath):
with open(filepath, "wb") as datafile:
r = requests.get(properties[filename]["url"])
for chunk in r.iter_content(1024):
datafile.write(chunk)
setproperties.update_for_file(filename, properties[filename])
setproperties.save_properties()
def get_args(form_dict):
setproperties.load_properties()
group_list = setproperties.groups.keys()
group_list.sort()
group_datasets = {}
for group in group_list:
group_datasets[group] = []
for fname in setproperties.properties:
p = setproperties.properties[fname]
if not p.get("title"):
p["title"] = fname
p["file_name"] = fname
if p.get("groups"):
for group in p.get("groups"):
group_datasets[group].append(p)
else:
group_datasets["Other"].append(p)
return {
"group_list": group_list,
"groups": setproperties.groups,
"group_datasets": group_datasets
}
