def download(): setproperties.load_properties() for i in xrange(35): print "page %s" % i response = requests.get(sources[1] + "?page=%s" % i, verify=False) properties = get_url_title_and_description_from_html(response.text) for filename in properties: print filename filepath = os.path.join("data", "raw", filename) if not os.path.exists(filepath): with open(filepath, "wb") as datafile: r = requests.get(properties[filename]["url"]) for chunk in r.iter_content(1024): datafile.write(chunk) setproperties.update_for_file(filename, properties[filename]) setproperties.save_properties()
def get_args(form_dict): setproperties.load_properties() group_list = setproperties.groups.keys() group_list.sort() group_datasets = {} for group in group_list: group_datasets[group] = [] for fname in setproperties.properties: p = setproperties.properties[fname] if not p.get("title"): p["title"] = fname p["file_name"] = fname if p.get("groups"): for group in p.get("groups"): group_datasets[group].append(p) else: group_datasets["Other"].append(p) return { "group_list": group_list, "groups": setproperties.groups, "group_datasets": group_datasets }