def list_cubes(self): cubes = [] for dataset in Dataset.all(): if not len(dataset.mapping): continue cubes.append({ #change here too 'name': dataset.name, 'label': dataset.label }) return cubes
def list_cubes(self): cubes = [] for dataset in Dataset.all(): if not len(dataset.mapping): continue cubes.append({ #change here too 'name': dataset.name, 'label': dataset.label }) return cubes
def reindex(): """ Grant admin privileges to given user """ from openspending.core import db from openspending.model import Dataset from openspending.model.country import Country index = whoo.whoosh_index(current_app, Dataset) with index.writer() as writer: for dataset in Dataset.all(): primary_field = dataset.pure_whoosh.primary_key_name searchable = dataset.__searchable__ attrs = {} for key in searchable: try: attrs[key] = unicode(getattr(dataset, key)) except AttributeError: raise AttributeError('{0} does not have {1} field {2}' .format("Dataset", __searchable__, key)) attrs[primary_field] = unicode(getattr(dataset, primary_field)) writer.update_document(**attrs) #make sure we ahve all of the geometry tables in there createCountries(silent=True) index = whoo.whoosh_index(current_app, Country) with index.writer() as writer: for country in Country.all(): primary_field = country.pure_whoosh.primary_key_name searchable = country.__searchable__ attrs = {} for key in searchable: try: attrs[key] = unicode(getattr(country, key)) except AttributeError: raise AttributeError('{0} does not have {1} field {2}' .format("Country", __searchable__, key)) attrs[primary_field] = unicode(getattr(country, primary_field)) writer.update_document(**attrs)
def reindex(): """ Grant admin privileges to given user """ from openspending.core import db from openspending.model import Dataset from openspending.model.country import Country index = whoo.whoosh_index(current_app, Dataset) with index.writer() as writer: for dataset in Dataset.all(): primary_field = dataset.pure_whoosh.primary_key_name searchable = dataset.__searchable__ attrs = {} for key in searchable: try: attrs[key] = unicode(getattr(dataset, key)) except AttributeError: raise AttributeError( '{0} does not have {1} field {2}'.format( "Dataset", __searchable__, key)) attrs[primary_field] = unicode(getattr(dataset, primary_field)) writer.update_document(**attrs) #make sure we ahve all of the geometry tables in there createCountries(silent=True) index = whoo.whoosh_index(current_app, Country) with index.writer() as writer: for country in Country.all(): primary_field = country.pure_whoosh.primary_key_name searchable = country.__searchable__ attrs = {} for key in searchable: try: attrs[key] = unicode(getattr(country, key)) except AttributeError: raise AttributeError( '{0} does not have {1} field {2}'.format( "Country", __searchable__, key)) attrs[primary_field] = unicode(getattr(country, primary_field)) writer.update_document(**attrs)
def output_json(**args): """ Output JSON data """ outputfile = args.get('outputfile', None) if len(outputfile) != 1: print "You need to specific one and only one output file" return outputfile = outputfile[0] #need to load in this order for relations #metadataorg #dataorg #source #sourcefile #wrap up files #dataset outputobj = [] for metadataorg in MetadataOrg.all().all(): outputobj.append(metadataorg.to_json_dump()) for dataorg in DataOrg.all().all(): outputobj.append(dataorg.to_json_dump()) for source in Source.all().all(): outputobj.append(source.to_json_dump()) for sourcefile in SourceFile.all().all(): outputobj.append(sourcefile.to_json_dump()) for dataset in Dataset.all().all(): outputobj.append(dataset.to_json_dump()) with open(outputfile, 'wb') as f: json.dump(outputobj, f) print "success" print "written to ", outputfile
def output_json(**args): """ Output JSON data """ outputfile = args.get('outputfile', None) if len(outputfile) != 1: print "You need to specific one and only one output file" return outputfile = outputfile[0] #need to load in this order for relations #metadataorg #dataorg #source #sourcefile #wrap up files #dataset outputobj = [] for metadataorg in MetadataOrg.all().all(): outputobj.append(metadataorg.to_json_dump()) for dataorg in DataOrg.all().all(): outputobj.append(dataorg.to_json_dump()) for source in Source.all().all(): outputobj.append(source.to_json_dump()) for sourcefile in SourceFile.all().all(): outputobj.append(sourcefile.to_json_dump()) for dataset in Dataset.all().all(): outputobj.append(dataset.to_json_dump()) with open(outputfile, 'wb') as f: json.dump(outputobj, f) print "success" print "written to ", outputfile
def reload_all(**args): """Reload all sources with mapping. This will take a while""" datasets = Dataset.all().all() ids = [] for dataset in datasets: ids.append(dataset.id) total = 0 ran = 0 for id in ids: dataset = Dataset.by_id(id) total += 1 #has mapping and source if dataset.mapping and dataset.source: print "working on ", dataset load_source(dataset.source.id) ran += 1 print "Ran", ran, "out of", total
def reload_all(**args): """Reload all sources with mapping. This will take a while""" datasets = Dataset.all().all() ids = [] for dataset in datasets: ids.append(dataset.id) total = 0 ran = 0 for id in ids: dataset = Dataset.by_id(id) total +=1 #has mapping and source if dataset.mapping and dataset.source: print "working on ", dataset load_source(dataset.source.id) ran +=1 print "Ran", ran, "out of", total
def categories_list(): page_num = request.args.get('page', None) perpage = request.args.get('perpage', 25) includesubs = request.args.get('includesubs', True) limit = request.args.get('limit', None) query_all = Dataset.all(order=True) if limit: query_all = query_all.limit(int(limit)) numpages = 1 page = 1 if page_num: total_indicators = query_all.count() query_all = query_all.offset(int(page_num) * int(perpage)).limit( int(perpage)) numpages = int(float(total_indicators) / float(perpage)) + 1 page = page_num outputschema = { "page": page, "numpages": numpages, "data": { "categories": { "total": 0, "data": collections.OrderedDict({}) }, "subcategories": { #"total":0, "data": collections.OrderedDict({}) }, "sources": { "total": 0, "data": collections.OrderedDict({}) }, "colls": { "total": 0, "data": collections.OrderedDict({}) }, "indicators": { "total": 0, "data": collections.OrderedDict({}) } } } for indicator in query_all.all(): if not getattr(indicator, "mapping", None): continue keyname = indicator.name years = getattr(indicator, "years", None) if years: the_years = [] the_years = years.split(",") the_years = map(int, the_years) the_years.sort() else: the_years = [] dataorg = getattr(indicator, "dataorg", None) if not dataorg: dataorg = "None" else: if outputschema['data']['sources']['data'].get( dataorg.label, None): outputschema['data']['sources']['data'][ dataorg.label]['indicators'].append(indicator.name) else: outputschema['data']['sources']['data'][dataorg.label] = { 'label': dataorg.label, 'indicators': [indicator.name] } outputschema['data']['sources']['total'] += 1 dataorg = dataorg.label tags = getattr(indicator, "tags", []) subcategory = "None" category = "None" colls = "None" for tag in tags: if tag.category == "spsd": if outputschema['data']['categories']['data'].get( tag.slug_label, None): outputschema['data']['categories']['data'][ tag.slug_label]['indicators'].append(indicator.name) else: outputschema['data']['categories']['data'][ tag.slug_label] = { 'label': tag.label, 'indicators': [indicator.name] #"subcategories": {} } outputschema['data']['categories']['total'] += 1 category = tag.slug_label elif tag.category == "subspsd": #if outputschema['data']['categories']['data'].get(tag.slug_label, None): outputschema['data']['subcategories']['data'][ tag.slug_label] = { 'label': tag.label } subcategory = tag.slug_label elif tag.category == "colls": if outputschema['data']['colls']['data'].get( tag.slug_label, None): outputschema['data']['colls']['data'][ tag.slug_label]['indicators'].append(indicator.name) else: outputschema['data']['colls']['data'][tag.slug_label] = { 'label': tag.label, 'indicators': [indicator.name] #"subcategories": {} } outputschema['data']['colls']['total'] += 1 colls = tag.slug_label #outputschema['data']['colls']['data'][tag.slug_label]= {'label':tag.label} #colls = "test" #tag.slug_label + "test" #print "colls: " + colls indicatorschema = { "label": getattr(indicator, "label", "No Label"), "description": getattr(indicator, "description", "No Description"), "source": dataorg, "category": category, "subcategory": subcategory, "colls": colls, "years": the_years } outputschema['data']['indicators']['data'][keyname] = indicatorschema outputschema['data']['indicators']['total'] += 1 #outputschema['data']['indicators'] = list(sorted(outputschema['data']['indicators'].items(), key=lambda x: x)) # resp = Response(response=json.dumps({'data':outputschema}), # status=200, \ # mimetype="application/json") # return resp return json.dumps(outputschema)
def dataorgs(): page_num = request.args.get('page', None) perpage = request.args.get('perpage', 25) includesubs = request.args.get('includesubs', True) limit = request.args.get('limit', None) query_all = Dataset.all(order=True) if limit: query_all = query_all.limit(int(limit)) numpages = 1 page = 1 if page_num: total_indicators = query_all.count() query_all = query_all.offset(int(page_num) * int(perpage)).limit(int(perpage)) numpages = int(float(total_indicators)/float(perpage)) + 1 page = page_num outputschema = { "page":page, "numpages": numpages, "data": { "categories":{ "total":0, "data":collections.OrderedDict({}) }, "subcategories":{ #"total":0, "data":collections.OrderedDict({}) }, "sources":{ "total":0, "data":collections.OrderedDict({}) }, "indicators":{ "total":0, "data":collections.OrderedDict({}) } } } for indicator in query_all.all(): if not getattr(indicator, "mapping", None): continue keyname = indicator.name years = indicator.years the_years=[] the_years=years.split(",") the_years=map(int,the_years) the_years.sort() dataorg = getattr(indicator, "dataorg", None) if not dataorg: dataorg = "None" else: if outputschema['data']['sources']['data'].get(dataorg.label, None): outputschema['data']['sources']['data'][dataorg.label]['indicators'].append(indicator.name) else: outputschema['data']['sources']['data'][dataorg.label] = { 'label': dataorg.label, 'indicators': [indicator.name] } outputschema['data']['sources']['total'] += 1 dataorg = dataorg.label tags = getattr(indicator, "tags", []) subcategory = "None" category = "None" for tag in tags: if tag.category == "spsd": if outputschema['data']['categories']['data'].get(tag.slug_label, None): outputschema['data']['categories']['data'][tag.slug_label]['indicators'].append(indicator.name) else: outputschema['data']['categories']['data'][tag.slug_label] = { 'label': tag.label, 'indicators': [indicator.name] #"subcategories": {} } outputschema['data']['categories']['total'] += 1 category = tag.slug_label elif tag.category == "subspsd": #if outputschema['data']['categories']['data'].get(tag.slug_label, None): outputschema['data']['subcategories']['data'][tag.slug_label]= {'label':tag.label} # if outputschema['data']['categories']['data'].get(tag.slug_label, None): # if outputschema['data']['categories']['data'][tag.slug_label]['subcategories'].get(tag.slug_label, None): # outputschema['data']['categories']['data'][tag.slug_label]['subcategories'][tag.slug_label]['indicators'].append(indicator.name) # else: # outputschema['data']['categories']['data'][tag.slug_label]['subcategories'][tag.slug_label] = { # "label": tag.label, # "indicators": [indicator.name] # } # ['indicators'].append(indicator.name) # else: # outputschema['data']['categories']['data'][tag.slug_label] = { # 'label': tag.label, # 'indicators': [indicator.name], # "subcategories": {} # } # outputschema['data']['categories']['total'] += 1 subcategory = tag.slug_label indicatorschema = { "label":getattr(indicator, "label", "No Label"), "description":getattr(indicator, "description", "No Description"), "source":dataorg, "category":category, "subcategory":subcategory, "years":the_years } outputschema['data']['indicators']['data'][keyname] = indicatorschema outputschema['data']['indicators']['total'] += 1 #outputschema['data']['indicators'] = list(sorted(outputschema['data']['indicators'].items(), key=lambda x: x)) return json.dumps(outputschema)
def add_import_commands(manager): @manager.option('-f', '--file-dir', dest='file_dir', help='File Dir to output the files') @manager.command def output_logs(**args): filedir = args.get("file_dir", None) log.info("Using filedir: %s", filedir) if not filedir: log.warn("Please specify an output dir") sys.exit() try: f = open(os.path.join(filedir, "LogFiles.zip"), 'wb') except Exception, e: log.warn("Could not open directory : %s", e) zf = zipfile.ZipFile(f, "w") for dataset in Dataset.all(): if dataset.source and dataset.source.runs.first(): datalogs = dataset.source.runs.first().records_as_json() else: log.info("Skipping : %s", dataset.name) continue if not len(datalogs): log.info("No Datalog for : %s", dataset.name) zf.writestr(dataset.name + "/loadinglog.csv", "All is well") continue outputfile = io.BytesIO() #figureout the headers dw = csv.DictWriter(outputfile, delimiter=',', extrasaction='ignore', fieldnames=datalogs[0].keys()) dw.writeheader() for row in datalogs: dw.writerow(row) zf.writestr(dataset.name + "/loadinglog.csv", outputfile.getvalue()) #write openrefine output preloadvalue = dataset.source.getORFile().getvalue() zf.writestr(dataset.name + "/preloadvalue.csv", preloadvalue) url = "http://*****:*****@name&format=csv" # Fill in your details here to be posted to the login form. LOCKDOWN_FORCE = current_app.config.get("LOCKDOWNUSER", False) LOCKDOWNUSER = current_app.config.get("LOCKDOWNUSER") LOCKDOWNPASSWORD = current_app.config.get("LOCKDOWNUSER") if LOCKDOWN_FORCE: payload = { 'username': LOCKDOWNUSER, 'password': LOCKDOWNPASSWORD } # Use 'with' to ensure the session context is closed after use. with requests.Session() as s: try: p = s.post('http://localhost:5000/lockdown', data=payload) # An authorised request. postloadvalue = s.get(url).content except Exception, e: log.warn("could not get authorized postload value " + str(e)) else: try: postloadvalue = requests.get(url).content except Exception, e: log.warn("Could Not find post load content for " + dataset.name)
def add_import_commands(manager): @manager.option('-f', '--file-dir', dest='file_dir', help='File Dir to output the files') @manager.command def output_logs(**args): filedir = args.get("file_dir", None) log.info("Using filedir: %s", filedir) if not filedir: log.warn("Please specify an output dir") sys.exit() try: f = open(os.path.join(filedir, "LogFiles.zip"), 'wb' ) except Exception, e: log.warn("Could not open directory : %s", e) zf = zipfile.ZipFile(f, "w") for dataset in Dataset.all(): if dataset.source and dataset.source.runs.first(): datalogs = dataset.source.runs.first().records_as_json() else: log.info("Skipping : %s", dataset.name) continue if not len(datalogs): log.info("No Datalog for : %s", dataset.name) zf.writestr(dataset.name + "/loadinglog.csv", "All is well") continue outputfile = io.BytesIO() #figureout the headers dw = csv.DictWriter(outputfile, delimiter= ',', extrasaction='ignore', fieldnames=datalogs[0].keys()) dw.writeheader() for row in datalogs: dw.writerow(row) zf.writestr(dataset.name + "/loadinglog.csv", outputfile.getvalue()) #write openrefine output preloadvalue = dataset.source.getORFile().getvalue() zf.writestr(dataset.name + "/preloadvalue.csv", preloadvalue) url = "http://*****:*****@name&format=csv" # Fill in your details here to be posted to the login form. LOCKDOWN_FORCE = current_app.config.get("LOCKDOWNUSER", False) LOCKDOWNUSER = current_app.config.get("LOCKDOWNUSER") LOCKDOWNPASSWORD = current_app.config.get("LOCKDOWNUSER") if LOCKDOWN_FORCE: payload = { 'username': LOCKDOWNUSER, 'password': LOCKDOWNPASSWORD } # Use 'with' to ensure the session context is closed after use. with requests.Session() as s: try: p = s.post('http://localhost:5000/lockdown', data=payload) # An authorised request. postloadvalue = s.get(url).content except Exception, e: log.warn("could not get authorized postload value " + str(e)) else: try: postloadvalue = requests.get(url).content except Exception, e: log.warn("Could Not find post load content for " + dataset.name)