def geocoder(): config = configuration() config['remote'] = '' remote = 'on' # Geocoder handle = config['geocoderhandle'] (classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle']) fromyear = 1500 toyear = 2016 cfilter = '' if request.args.get('name'): cfilter = request.args.get('name') if request.args.get('name'): cfilter = request.args.get('name') if fromyear: historical = '' if historical == 'old': api = config['apiroot'] + "/collabs/static/data/historical.json" (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, cfilter) else: (geocoder, geolist, oecd) = buildgeocoder(geodataset, config, cfilter) data = json.dumps(geocoder, encoding="utf-8", sort_keys=True, indent=4) return Response(data, mimetype='application/json')
def tableapis(handle, customcountrycodes, fromyear, toyear, customyear, logflag): # years in filter config = {} indicator = '' config = configuration() DEBUG = 0 try: (dataset, revid, cliopid, clearpid) = findpid(handle) except: dataset = handle try: apifile = str(dataset) + ".json" jsonapi = config['apiroot'] + "/collabs/static/data/" + apifile dataframe = load_api_data(jsonapi, '') except: jsonapi = config['apiroot'] + "/api/datasets?handle=Panel[" + handle + "]" datajson = load_api_data(jsonapi, '') for handledata in datajson: dataframe = handledata['data'] # DEBUG2 #print dataframe loccodes = loadcodes(dataframe) (ctr, header) = countryset(customcountrycodes, loccodes) (frame, years, values, dates, original) = createframe(indicator, loccodes, dataframe, customyear, fromyear, toyear, ctr, logflag, DEBUG) names = ['indicator', 'm', 'ctrcode', 'country', 'year', 'intcode', 'value', 'id'] (csvdata, aggrdata) = combinedata(ctr, frame, loccodes) return (years, frame, csvdata, aggrdata, original)
def indicators(): #data = load_indicators("indicators.csv") config = configuration() pid = config['topicindex'] if pid: (handles, pidslist) = pidfrompanel(pid) hquery = formdatasetquery(handles,'') datainfo = readdatasets('datasets', json.loads(hquery)) csvio = StringIO(str(datainfo[0]['csvframe'])) data = pd.read_csv(csvio, sep='\t', dtype='unicode',quoting=csv.QUOTE_NONE) columns = [] for item in data.columns: col = re.sub(r"\"", "", item) columns.append(col) data.columns = columns storeddata = readdatasets('datasets', '') linking = {} for item in storeddata: try: linking[item['title']] = item['handle'] except: skip = 'yes' data['handle'] = '' data = data.drop('ID', axis=1) for row in data.index: title = data.ix[row]['Name'] try: data.ix[row]['handle'] = linking[title] except: data.ix[row]['handle'] = '' return Response(data.to_csv(orient='records'), mimetype='application/json') else: return 'No data'
def totalstatistics(settings=''): config = configuration() if config['error']: return config['error'] handles = [] if request.args.get('handle'): handledataset = request.args.get('handle') (dataset, revid, cliopid, clearpid) = findpid(handledataset) handles.append(dataset) if request.args.get('dataset'): dataset = request.args.get('dataset') handles.append(dataset) if request.args.get('yearmin'): fromyear = request.args.get('yearmin') if request.args.get('yearmax'): toyear = request.args.get('yearmax') html = '' for dataset in handles: jsonapi = config['apiroot'] + "/collabs/static/data/" + str(dataset) + ".json" data = createdata(jsonapi) d = data.describe() show = d.transpose() stats = show.to_html() html = html = stats + '<br>' return html
def save_serial_list(task, serial_list_dict): configs = configuration() root = configs.get_log_analysis_path() serial_list_path = os.path.join(root, task, __MY_VAR_ROOT_PATH__,__FN_RESULTS_LIST__) serial_display_path = os.path.join(root, task, __MY_VAR_ROOT_PATH__, __FN_DISPLAY_RESULTS_LIST__) file_result = open(serial_list_path, 'w') file_result_display = open(serial_display_path, 'w') for key, value in serial_list_dict.items(): 'write serial result into file /serial.txt/ for later dispaly' linekey = key '''print linekey''' file_result_display.write(linekey) '''print '-----''' file_result_display.write('\n-----\n') linevalue = '' if (type(value) is types.ListType): for v in value: linevalue = linevalue + str(v) + '|' linevalue = linevalue.strip('|') else: linevalue = str(value) '''print linevalue''' file_result_display.write(linevalue) '''print '==========''' file_result_display.write('\n==========\n') 'write serial result into file /serial/ for later use' serial_to_write = key + '=' + linevalue file_result.write(serial_to_write + '\n') file_result_display.close() file_result.close()
def get(settings=''): config = configuration() if config['error']: return config['error'] (year, code, website, server, imagepathloc, imagepathweb, viewerpath, path, geojson, datarange, custom) = readglobalvars() image = request.args.get('image') gzip = request.args.get('nlgis') svg = request.args.get('svg') pdf = request.args.get('pdf') outfile = '' thismimetype='image' if image: outfile = image if gzip: thismimetype = 'application/x-gzip' outfile = gzip if svg: thismimetype = 'text/plain' outfile = svg if pdf: thismimetype = 'application/pdf' outfile = pdf if image: return send_from_directory(imagepathloc, outfile, mimetype=thismimetype) else: return send_from_directory(imagepathloc, outfile, as_attachment=True)
def compiledataset(csvfile): handles = [] remote = 'on' # Geocoder config = configuration() config['remote'] = 'yes' dataframe = loaddataset_fromfile(config, csvfile) dataset = dataframe title = dataframe.columns[1] units = dataframe.ix[0][1] metadata = {} switch = 'modern' switch = 'historical' #dataset = dataset.convert_objects(convert_numeric=True) dataset.columns = dataset.ix[1] dataset.index = dataset[config['moderncode']] if '1' in dataset.columns: dataset = dataset.drop('1', axis=1) #dataset = dataset[2:] (intcodes, notint) = selectint(dataset.columns) #for colname in notint: # dataset = dataset.drop(colname, axis=1) dataset.columns = notint + intcodes return (dataset, title, units)
def advanced_statistics(): (yearmin, yearmax, ctrlist) = (1500, 2020, '') config = configuration() handles = [] if request.args.get('handle'): handledataset = request.args.get('handle') handledataset = handledataset.replace(" ", '') if request.args.get('dataset'): dataset = request.args.get('dataset') handles.append(dataset) if request.args.get('yearmin'): yearmin = request.args.get('yearmin') if request.args.get('yearmax'): yearmax = request.args.get('yearmax') if request.args.get('ctrlist'): ctrlist = request.args.get('ctrlist') modern = moderncodes(config['modernnames'], config['apiroot']) jsonapi = config['apiroot'] + '/api/datasets?handle=' + str(handledataset) (panel, cleanedpanel, names) = loadpanel(jsonapi, yearmin, yearmax, ctrlist) (header, data, countries, handles, vhandles) = advpanel2dict(cleanedpanel) ctrlimit = 200 #result = panel2csv(header, data, thisyear, countries, handles, vhandles, ctrlimit, modern) #maindataframe = data2statistics(handles, cleanedpanel) #showhtml = statistics_tojson(maindataframe, modern) data = handle2statistics(handles, cleanedpanel) showhtml = statistics2table(data) return showhtml
def testdownload(): config = configuration() DEBUG = 0 API_TOKEN=config['key'] cmd = "--insecure -u " + API_TOKEN + ": " + config['dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/" tmpdir = "/tmp/test" filename = randomword(10) arc = "data" + filename + ".zip" finaldir = "/home/dpe/tmp" if filename: finaldir = str(finaldir) + '/' + str(filename) tmpdir = str(tmpdir) + '/' + str(filename) pid = "hdl:10622/73BBBI" pid = "hdl:10622/4X6NCK" pid = "hdl:10622/JPIRSD" try: os.mkdir(tmpdir) os.mkdir(finaldir) except OSError as e: if e.errno != errno.EEXIST: raise e pass zipfile = get_papers(config['dataverseroot'], config['key'], cmd, pid, tmpdir, arc, finaldir) print zipfile return
def predict_task(task, serials): global lut_api_key config = configuration() lut_api_key = config.read_LUT(1) task_log_path = os.path.join(config.get_log_analysis_path(), task, task) try: task_log_file = open(task_log_path, 'r') except: print 'OPEN LOG FILE FAILED! (' + task_log_file + ')' else: api_serial = [] last_time = 0 predict_count = 0 predict_hitted = 0 total_api_count = 0 next_api_key = '' random_hitted = 0 for line in task_log_file: linearray = line.split('\t') api = linearray[5] 'check whether it is a api' if (lut_api_key.has_key(api)): apikey = lut_api_key[api] else: continue 'check whether time out' if (last_time > 0): if (int(linearray[3]) - last_time > __VAR_TIME_INTERVAL__*1000): api_serial = [] else: 'it is a valid api, check whether predict hitted!' total_api_count = total_api_count + 1 if (next_api_key != ''): random_next_api = chr(random.randint(ord('A'), ord('A')+len(lut_api_key)-1)) if (apikey == random_next_api): random_hitted = random_hitted + 1 if (apikey == next_api_key): predict_hitted = predict_hitted + 1 '''print 'PREDICT HITTED!''' next_api_key = '' last_time = int(linearray[3]) api_serial.append(apikey) predict = prediction(api_serial, serials) if (predict != False): predict_count = predict_count + 1 next_api_key = predict task_log_file.close() print '===== PREDICT TASK COMPLETED =====' time.sleep(2) print 'TOTAL API COUNT:' + str(total_api_count) print 'PREDICTED COUNT:' + str(predict_count) print 'PREDICTED HITTED COUNT:' + str(predict_hitted) print 'RANDOM HITTED COUNT:' + str(random_hitted)
def webmapper(): config = configuration() api = config['apiroot'] + "/collabs/static/data/historical.json" (regions, countries, ctr2reg, webmapper, geocoder) = histo(api) data = json.dumps(webmapper, encoding="utf-8", sort_keys=True, indent=4) return Response(data, mimetype='application/json')
def advanced(settings=''): config = configuration() if config['error']: return config['error'] (year, code, website, server, imagepathloc, imagepathweb, viewerpath, path, geojson, datarange, custom) = readglobalvars() for name in request.cookies: settings = settings + ' ' + name + '=' + request.cookies[name] #return settings image = imagepathweb + '/' + year + '.png'; settings = '' resp = make_response(render_template('advanced.html', image=image, settings=settings, r=request.cookies)) # Cookie revision for name in request.cookies: on = request.cookies[name] try: if request.args[name]: i = 1 except: if on == 'on': erase[name] = on resp.set_cookie(name, '') for name in request.args: resp.set_cookie(name, request.args[name]) return resp
def printme(): config = configuration() if config['error']: return config['error'] year = request.args.get("year") handle = request.args.get("handle") handles = [] handles.append(handle) hquery = formdatasetquery(handles,'') datainfo = readdatasets('datasets', json.loads(hquery)) try: for item in datainfo: datasetID = item['datasetID'] except: datasetID = 228 root = config['dataverseroot'] + "/api/datasets/" + str(datasetID) + "/versions/?key=" + config['key'] + "&show_entity_ids=true&q=authorName:*" data = load_api_data(root, 1) (title, citation) = get_citation(data['data']) uhandle = handle uhandle = uhandle.replace('hdl:', '') mapcopyright = config['cshapes_copyright'] if int(year) < 1946: mapcopyright = config['geacron_copyright'] # "Note: Map polygons provided by Geacron <a href=\"geacron.com\">http://www.geacron.com</a>" resp = make_response(render_template('printall.html', title=title, citation=citation, mapcopyright=mapcopyright, year=year, handle=handle, uhandle=uhandle)) return resp
def datasets(settings=''): config = configuration() if config['error']: return config['error'] (year, code, website, server, imagepathloc, imagepathweb, viewerpath, path, geojson, datarange, custom) = readglobalvars() topicapiurl = website + "/api/topicslist" topicstats = load_api_data(topicapiurl, '', '', '', '', '') localfile = 'index.csv' filename = imagepathloc + '/' + localfile f = csv.writer(open(filename, "wb+")) varlist = [] firstline = 0 for code in sorted(topicstats): dataset = topicstats[code] mapurl = website + "/site?code=" + dataset['topic_code'] + "&year=" + str(dataset['startyear']) dataurl = website + '/api/data?code=' + dataset['topic_code'] topicstats[code]['urlmap'] = mapurl topicstats[code]['urldata'] = dataurl datarow = [] if firstline == 0: for row in sorted(dataset): varlist.append(row) f.writerow(varlist) firstline = 1 for row in sorted(dataset): datarow.append(dataset[row]) f.writerow(datarow) return send_from_directory(imagepathloc, localfile, as_attachment=True)
def finalizeOrder(self,cursor,cashier): try: owner=self.book.owner except: cfg = configuration() owner = cfg.get("default_owner") cursor.execute (""" INSERT INTO transactionLog SET action = "SALE", amount = %s, cashier = %s, date = NOW(), info = %s, schedule = %s, owner = %s, cart_id='' """,(self.price,cashier,"[%s] %s" % (self.getDistributor(),self.getName()),self.price_schedule,owner)) try: self.book.sellme() # marks as sold except: print "sellme failed" pass cursor.close()
def update_pages(self,event): master_page_name=[m for m in self.pages.keys() if self.pages[m].master][0] master_page=self.pages[master_page_name] new_master_price=float(string.replace(master_page.GetValue(),"$","")) cfg = configuration() for mp in cfg.get("multiple_prices"): if mp[0] != master_page_name: #print mp[1] self.pages[mp[0]].SetValue("%s" % (mp[1]*new_master_price))
def boundaries(settings=''): config = configuration() if config['error']: return config['error'] root = config['apiroot'] dataset = 'dataframe66_' jsondataset = getboundaries(root, dataset) return Response(json.dumps(jsondataset), mimetype='application/json')
def treemap(settings=''): (years, ctrlist) = ([], '') showpanel = 'yes' config = configuration() if config['error']: return config['error'] (historical, handle, handles, thisyear) = ('', '', [], '') if request.args.get('face'): facehandle = request.args.get('face') if facehandle not in handles: handles.append(facehandle) handle = facehandle if request.args.get('handle'): handledataset = request.args.get('handle') try: (pids, pidslist) = pidfrompanel(handledataset) handle = pids[0] handles.append(handle) except: handles.append(handledataset) handle = handledataset nopanel = 'yes' if request.args.get('historical'): historical = request.args.get('historical') if request.args.get('year'): thisyear = request.args.get('year') if request.args.get('hist'): historical = request.args.get('hist') if request.args.get('ctrlist'): ctrlist = request.args.get('ctrlist') if ctrlist == config['ctrlist']: ctrlist = '' mainlink = '&handle=' + str(handle) try: (title, units, years) = dpemetadata(config, handle) except: (title, units, years) = ('Panel Data', '', []) if historical: mainlink = str(mainlink) + '&historical=on' if thisyear: mainlink = str(mainlink) + '&year=' + str(thisyear) if ctrlist: mainlink = str(mainlink) + '&ctrlist=' + str(ctrlist) links = graphlinks(mainlink) apitreemap = config['apiroot'] + "/api/treemap?action=showyears&handle=" + str(handles[0]) + "&ctrlist=" + str(ctrlist) years = load_api_data(apitreemap, 1) total = len(years) lastyear = years[-1] resp = make_response(render_template('treemap.html', handle=handle, chartlib=links['chartlib'], barlib=links['barlib'], panellib=links['panellib'], treemaplib=links['treemaplib'], q=handle, showpanel=showpanel, historical=historical, title=title, thisyear=thisyear, years=years, total=total, lastyear=lastyear, ctrlist=ctrlist)) return resp
def browse(settings=''): activepage = 'Dashboard' config = configuration() if config['error']: return config['error'] pages = getindex(activepage) dataverse = config['dataverseroot'] resp = make_response(render_template('dataverse.html', active=activepage, pages=pages, dataverse=dataverse)) return resp
def _init_ctrls(self, prnt): self.cfg = configuration() title = self.cfg.get("title") wxFrame.__init__(self, id=-1, name='', parent=prnt, pos=wxPoint(35, 47), style=wxDEFAULT_FRAME_STYLE, title=title) self._init_utils() self.SetSizer(self.build_GUI()) self.Fit() self.global_sizer.SetSizeHints(self)
def metadata(dataset): #return ('xxx', '', '') config = configuration() (pid, fileid, revid, clearpid) = findpid(dataset) #return ('xxx', '', '') data = {} if pid: query = pid apiurl = config['dataverseroot'] + "/api/search?q=" + query + '&key=' + config['key'] + '&type=dataset' data = load_dataverse(apiurl) return (data, pid, fileid)
def dialog(): pid = '' root = '' config = configuration() if request.args.get('pid'): pid = request.args.get('pid') zipfile = downloadzip(pid) root = config['clearance'] + "/collabs/static/tmp/" + zipfile resp = make_response(render_template('dialog.html', download=root)) return resp
def datasets(): config = configuration() (jsondata, pid) = ('', '') handles = [] combineddataset = [] resultdataset = '' datainfo = [] outformat = 'json' if request.args.get('format'): outformat = request.args.get('format') if request.args.get('handle'): pid = request.args.get('handle') if request.args.get('latest'): dataset = config['defaulthandle'] return dataset if pid: (handles, pidslist) = pidfrompanel(pid) hquery = formdatasetquery(handles,'') datainfo = readdatasets('datasets', json.loads(hquery)) #if not datainfo: #datainfo.append(pid) for dataset in datainfo: data = {} handle = dataset['handle'] if outformat == 'json': jsondata = str(dataset['data']) jsondata = jsondata.replace(".0,", ",") json_dict = ast.literal_eval(jsondata.strip()) data['handle'] = handle try: data['title'] = dataset['title'] data['units'] = dataset['units'] data['datasetID'] = dataset['datasetID'] except: data['title'] = 'Title' data['units'] = 'Units' data['datasetID'] = 228 data['data'] = json_dict combineddataset.append(data) elif outformat == 'csv': data['data'] = dataset['csvframe'] resultdataset = data['data'] if outformat == 'json': if combineddataset: finaldata = json.dumps(combineddataset, encoding="utf-8", sort_keys=True, indent=4) return Response(finaldata, mimetype='application/json') elif outformat == 'csv': return Response(resultdataset, mimetype='text/plain')
def export(settings=''): config = configuration() if config['error']: return config['error'] activepage = 'Dashboard' config = configuration() perlbin = "/usr/bin/perl " path = config['datapath'] varproject = request.args.get('project') varbase = request.args.get('base') dataset = request.args.get('dataset') fileID = request.args.get('fileID') maincontent = config['error'] cmd = "/bin/cat " + path + fileID + '.csv' docheck = re.match(r'.*[;|\/\:\(\)]', cmd) if not docheck: p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) maincontent = p.communicate()[0] else: maincontent = 'Something went wrong..' return maincontent
def search(qurl): # Global settings config = configuration() root = config['dataverseroot'] # Load topics topicurl = config['apiroot'] + "/collabs/static/data/dataframe100_0.json" topics = loadjson(topicurl) # Input IDS = getindicators(qurl) datasets = dataset_search(root, IDS, topics) return datasets
def read_serial_list(task): serial_list_dict = {} configs = configuration() root = configs.get_log_analysis_path() serial_list_path = os.path.join(root, task, __MY_VAR_ROOT_PATH__,__FN_RESULTS_LIST__) if (os.path.isfile(serial_list_path) == True): file_serial_list = open(serial_list_path, 'r') for line in file_serial_list: arrays = line.split('=') serial = arrays[0] values = arrays[1].split('|') serial_list_dict[serial] = values file_serial_list.close() return serial_list_dict
def load_indicators(filename): config = configuration() csvfile = config['clearance'] + "/collabs/static/data/" + filename ufile = urlopen(csvfile) data = pd.read_csv(ufile, delimiter='\t') df = data if csvfile: d = [ dict([ (colname, row[i]) for i,colname in enumerate(df.columns) ]) for row in df.values ] return json.dumps(d)
def visualize(): config = configuration() resp = 'visualize' view = 'panel' if request.args.get('view'): view = request.args.get('view') if config['error']: return config['error'] if view == 'panel': resp = panel() elif view == 'time-series': resp = chartlib() elif view == 'treemap': resp = treemap() return resp
def dataverse_search(apiurl): dataframe = loadjson(apiurl) config = configuration() topics = topicscoder(config) info = [] tmpinfo = [] panel = {} panel['url'] = 'url' panel['indicator'] = '<b> Panel data</b>' panel['description'] = '<b> All datasets in panel format</b>' panel['name'] = ' Panel data' panel['topic'] = '<b> Selected topics</b>' panel['pid'] = 'Panel' panel['citation'] = 'citation' link = config['apiroot'] + "/collabs/dashboard?dataset=" handles = [] for item in dataframe['data']['items']: datasets = {} datasets['url'] = item['url'] datasets['pid'] = item['global_id'] handles.append(datasets['pid']) datasets['indicator'] = ' ' + item['name'] try: datasets['topic'] = ' ' + topics[item['name']] except: datasets['topic'] = ' ' + item['description'] abstract = str(item['description']) resume = re.search(r'^(.+?\.)\s+', str(item['description'])) if resume: abstract = resume.group(1) datasets['description'] = ' ' + abstract datasets['startyear'] = 1500 datasets['endyear'] = 2010 datasets['analyze'] = link + item['global_id'] + '&action=visualize' tmpinfo.append(datasets) panel['pid'] = 'Panel' + str(handles) info.append(panel) for data in tmpinfo: info.append(data) return info
def chartonprint(webpage, fileformat, year, code, proxy): # Print function (cmd, size, fileonweb) = ('', '1024', '') code = str(randomword(10)) # Configuration config = configuration() path = config['path'] phantompath = config['phantompath'] imagepathloc = config['imagepath'] filesvg = config['tmpdir'] + '/' + year + '_' + code + '.svg' print filesvg if fileformat == 'shapefile': year = year else: cmd = phantompath + "/phantomjs/lib/phantom/bin/phantomjs --ssl-protocol=any --disk-cache=true " + path + "/static/renderHTML.js '" + webpage + "'" p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) html = p.communicate()[0] result = re.findall(r'<svg.+?</svg>', html, re.DOTALL) if year: svgfile = open(filesvg, "w") svgfile.write(result[0]) svgfile.close() if fileformat == 'SVG': svgfileout = '/get?svg=' + year + '_' + code + '_' + "map.svg" return "<a href=\"" + svgfileout + "\">Download SVG file</a>" fileonweb = '' if fileformat == 'png': outfile = year + '_' + code + '_' + 'map.png' outdirfile = imagepathloc + '/' + outfile cmd = "/usr/bin/inkscape " + filesvg + " -e " + outdirfile + " -h " + size + " -D -b '#ffffff'" fileonweb = config['proxy'] + config['imagepathonweb'] + '/' + outfile if cmd: p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) result = p.communicate()[0] image = outfile return fileonweb
def __init__(self, configlocation): self.config = configuration(configlocation)
from ynab import ynabapi from bunq2ynab import sync LOGGER = configure_logger(__name__) # Add commandline switches: parser = argparse.ArgumentParser() parser.add_argument('-l', action='store_true', help="Run in list mode") args = parser.parse_args() # Set configuration file path config_file_path = os.path.join(os.path.dirname(__file__), 'config.json') config_location = 'file:' + config_file_path # Initialize Config, Bunq and YNAB config = configuration(config_location) b = bunqapi(config_location) y = ynabapi(config_location) # Start program if args.l: LOGGER.info('Running in LIST mode, no accounts will be synced.') b.list_users() y.list_budget() else: for i in config.value['bunq2ynab']: for key, value in i.items(): assert value, '{0} cannot be an empty string. Setup your sync pairs in your config'.format( key) try:
def __init__(self, configlocation): self.config = configuration(configlocation) self.url = 'https://api.youneedabudget.com/'
preserving state from one request to the next. In some situations, hidden fields can be a useful alternative to using cookies (the session variable). """ import flask import logging # Our own modules import config ### # Globals ### app = flask.Flask(__name__) CONFIG = config.configuration(proxied=(__name__ != "__main__")) app.secret_key = CONFIG.SECRET_KEY # Should allow using session variables ### # Pages ### @app.route("/") @app.route("/index") def index(): """The main page of the application. In this app, we will carry state from screen to screen in hidden fields. Initially we know nothing. """
from PIL import Image import numpy as np import cv2 from torchvision import transforms as trans from config import configuration MEAN = [0.5, 0.5, 0.5] STD = [0.5, 0.5, 0.5] TARGET_IMAGE_SIZE = configuration().inputSize def image_norm(fname): img = Image.open(fname, 'r') if img.size[0] == 0 | img.size[1] == 0: return 0 resize_img = img.resize((TARGET_IMAGE_SIZE, TARGET_IMAGE_SIZE)) if len(resize_img.split()) == 1: gray_img = np.asarray(((np.float32(resize_img) / 255.0)-MEAN[0])/STD[0]) elif len(resize_img.split()) == 3: resize_img = resize_img.convert('L') gray_img = np.asarray(((np.float32(resize_img) / 255.0)-MEAN[0])/STD[0]) else: return 0 normImg = np.asarray([gray_img]) return normImg
Build an application and launch it. This is the entry point that creates the flaks application. This should be passed to FLASK_APP:: export FLASK_APP=app flask run """ import logging import atexit import config import signal import sys from webapp import create_app, requestManager, db from webapp.models import Monitor config_class = config.configuration() def configure_logging(config): """Configure application logging for `app`""" from logging.config import dictConfig default_format = \ '[%(asctime)s] %(levelname)s ' \ 'in %(module)s (%(threadName)s): %(message)s' logging_config = { 'version': 1, 'formatters': { 'default': { 'format': default_format, } },
def main(): handle = '' dataverse = '' customkey = '' config = configuration() try: myopts, args = getopt.getopt(sys.argv[1:], "H:r:d:k:D:") except getopt.GetoptError as e: print(str(e)) print( "Usage: %s -y year -d datatype -r region -f filename -DDEBUG -o output" % sys.argv[0]) sys.exit(2) (handle, rhandle, customdv) = ('', '', '') for o, a in myopts: if o == '-H': handle = a if o == '-r': rhandle = a if o == '-d': dataverse = a if o == '-k': customkey = a if o == '-D': customdv = a dataset = {} DEBUG = '' path = config['path'] # Default dataverse root = config['dataverseroot'] key = config['key'] dvname = config['branch'] title = 'Title' units = 'Units' if dataverse: root = dataverse if customkey: key = customkey if customdv: dvname = customdv files = [] if rhandle: contentsapi = root + "/api/dataverses/" + dvname + "/contents?key=" + key print contentsapi newdata = load_api_data(contentsapi, '') metadata = newdata['data'] for item in metadata: dv = item['id'] files = getfiles(root, dv, key) if handle: print handle (datahandle, datasetID, fileID) = parsehandle(handle) files.append(fileID) for fileID in files: fullpath = downloadfile(root, path, fileID, key) print fullpath (pid, revid, cliohandle, clearpid) = findpid(handle) (jsonfile, csvfile) = ('', '') #try: if pid: handle = pid try: (jsonfile, csvfile, tmptitle, tmpunits) = dataextractor(fullpath, path, pid, fileID) except: resultfile = config['tmpdir'] + "/" + fileID (jsonfile, csvfile, tmptitle, tmpunits) = excelvalidator(config['phantompath'], fullpath, resultfile, config['tmpdir']) if jsonfile: remove = removedata('datasets', 'handle', clearpid) try: title = str(tmptitle) units = str(tmpunits) except: donothing = 1 print "ADD " + str(jsonfile) datasetadd(jsonfile, csvfile, clearpid, handle, title, units, datasetID) print handle print clearpid print datasetID
def initialization(): """ This function reads the user-defined parameters and paths from :mod:`config.py`, then adds additional parameters related to the shapefiles. First, it saves the spatial scope of the problem. Then, it distinguishes between countries, exclusive economic zones and subregions. For each one of them, it saves the geodataframes, the number of features, and the coordinates of the bounding boxes of each feature. Finally, it saves the number of rows and columns in the low and righ resolution, and a georeference dictionary used for saving tif files. :return: The updated dictionaries param and paths. :rtype: tuple(dict, dict) """ timecheck("Start") # import param and paths paths, param = configuration() # Read shapefile of scope scope_shp = gpd.read_file(paths["spatial_scope"]) param["spatial_scope"] = define_spatial_scope(scope_shp) res_weather = param["res_weather"] res_desired = param["res_desired"] Crd_all = crd_merra(param["spatial_scope"], res_weather)[0] param["Crd_all"] = Crd_all ymax, xmax, ymin, xmin = Crd_all bounds_box = Polygon([(xmin, ymin), (xmin, ymax), (xmax, ymax), (xmax, ymin)]) timecheck("Read shapefile of countries") # Extract land areas countries_shp = gpd.read_file(paths["Countries"], bbox=scope_shp) countries_shp = countries_shp.to_crs({"init": "epsg:4326"}) # Crop all polygons and take the part inside the bounding box countries_shp["geometry"] = countries_shp["geometry"].buffer(0) countries_shp["geometry"] = countries_shp["geometry"].intersection(bounds_box) countries_shp = countries_shp[countries_shp.geometry.area > 0] param["regions_land"] = countries_shp param["nRegions_land"] = len(param["regions_land"]) Crd_regions_land = np.zeros((param["nRegions_land"], 4)) for reg in range(0, param["nRegions_land"]): # Box coordinates for MERRA2 data r = countries_shp.bounds.iloc[reg] box = np.array([r["maxy"], r["maxx"], r["miny"], r["minx"]])[np.newaxis] Crd_regions_land[reg, :] = crd_merra(box, res_weather) timecheck("Read shapefile of EEZ") # Extract sea areas eez_shp = gpd.read_file(paths["EEZ_global"], bbox=scope_shp) eez_shp = eez_shp.to_crs({"init": "epsg:4326"}) # Crop all polygons and take the part inside the bounding box eez_shp["geometry"] = eez_shp["geometry"].buffer(0) eez_shp["geometry"] = eez_shp["geometry"].intersection(bounds_box) eez_shp = eez_shp[eez_shp.geometry.area > 0] param["regions_sea"] = eez_shp param["nRegions_sea"] = len(param["regions_sea"]) Crd_regions_sea = np.zeros((param["nRegions_sea"], 4)) for reg in range(0, param["nRegions_sea"]): # Box coordinates for MERRA2 data r = eez_shp.bounds.iloc[reg] box = np.array([r["maxy"], r["maxx"], r["miny"], r["minx"]])[np.newaxis] Crd_regions_sea[reg, :] = crd_merra(box, res_weather) timecheck("Read shapefile of subregions") # Read shapefile of regions regions_shp = gpd.read_file(paths["subregions"], bbox=scope_shp) regions_shp = regions_shp.to_crs({"init": "epsg:4326"}) # Crop all polygons and take the part inside the bounding box regions_shp["geometry"] = regions_shp["geometry"].intersection(bounds_box) regions_shp = regions_shp[regions_shp.geometry.area > 0] regions_shp.sort_values(by=["NAME_0"], inplace=True) regions_shp.reset_index(inplace=True) param["regions_sub"] = regions_shp param["nRegions_sub"] = len(param["regions_sub"]) Crd_regions_sub = np.zeros((param["nRegions_sub"], 4)) for reg in range(0, param["nRegions_sub"]): # Box coordinates for MERRA2 data r = regions_shp.bounds.iloc[reg] box = np.array([r["maxy"], r["maxx"], r["miny"], r["minx"]])[np.newaxis] Crd_regions_sub[reg, :] = crd_merra(box, res_weather) # Saving parameters param["Crd_subregions"] = Crd_regions_sub param["Crd_regions"] = np.concatenate((Crd_regions_land, Crd_regions_sea), axis=0) # Indices and matrix dimensions Ind_all_low = ind_merra(Crd_all, Crd_all, res_weather) Ind_all_high = ind_merra(Crd_all, Crd_all, res_desired) param["m_high"] = int((Ind_all_high[:, 0] - Ind_all_high[:, 2] + 1)[0]) # number of rows param["n_high"] = int((Ind_all_high[:, 1] - Ind_all_high[:, 3] + 1)[0]) # number of columns param["m_low"] = int((Ind_all_low[:, 0] - Ind_all_low[:, 2] + 1)[0]) # number of rows param["n_low"] = int((Ind_all_low[:, 1] - Ind_all_low[:, 3] + 1)[0]) # number of columns param["GeoRef"] = calc_geotiff(Crd_all, res_desired) timecheck("End") # Display initial information print("\nRegion: " + param["region_name"] + " - Year: " + str(param["year"])) print("Folder Path: " + paths["region"] + "\n") return paths, param
def __init__(self): current_configuration = configuration() current_configuration.parse("config.ini")
def download(): (classification, pid, root, switch, datafile) = ('modern', '', '', 'modern', '') handle = '' config = configuration() cmd = "--insecure -u " + config['key'] + ": " + config[ 'dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/" config['remote'] = '' datafilter = {} datafilter['startyear'] = '1500' datafilter['endyear'] = '2010' datafilter['ctrlist'] = '' tmpdir = config['tmpdir'] filerandom = randomword(10) #filerandom = '12345' arc = "data" + filerandom + ".zip" filename = filerandom finaldir = config['path'] + '/static/tmp' # ToDO if filename: finaldir = str(finaldir) + '/' + str(filename) tmpdir = str(tmpdir) + '/' + str(filename) try: os.mkdir(tmpdir) os.mkdir(finaldir) except: donothing = 'ok' if request.args.get('handle'): handle = request.args.get('handle') if request.args.get('type[0]') == 'historical': classification = request.args.get('type[0]') switch = classification if request.args.get('y[min]'): datafilter['startyear'] = request.args.get('y[min]') if request.args.get('y[max]'): datafilter['endyear'] = request.args.get('y[max]') # Select countries customcountrycodes = '' f = request.args for key in f.keys(): if is_location(key): for value in sorted(f.getlist(key)): customcountrycodes = str(customcountrycodes) + str(value) + ',' if customcountrycodes: customcountrycodes = customcountrycodes[:-1] datafilter['ctrlist'] = customcountrycodes if request.args.get('ctrlist'): datafilter['ctrlist'] = request.args.get('ctrlist') if request.args.get('pid'): pid = request.args.get('pid') ispanel = '' try: (pids, pidslist) = pidfrompanel(pid) handles = pids handle = pids[0] match = re.match(r'Panel\[(.+)\]', pid) if match: ispanel = 'yes' except: handles = pid handle = pids[0] if ispanel: dirforzip = '' for handle in handles: dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir) (header, panelcells, metadata, totalpanel) = build_panel(config, switch, handles, datafilter) filename = "paneldata.xlsx" metadata = [] datadir = config['webtest'] localoutfile = panel2excel(dirforzip, filename, header, panelcells, metadata) arc = 'dataarchive.zip' compile2zip(dirforzip, arc) root = config['apiroot'] + "/collabs/static/tmp/" + str(arc) return redirect(root, code=301) if classification: outfile = "clioinfra.xlsx" dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir) #fullpath = config['webtest'] + "/" + str(outfile) fullpath = dirforzip # Check selection isselection = 'yes' if datafilter['startyear'] == '1500': if datafilter['ctrlist'] == '': isselection = 'yes' if isselection: (datafile, outfilefinal, finalsubset) = dataframe_compiler(config, fullpath, handle, classification, datafilter) #return datafile.to_html() else: # Copy original dataset source = os.listdir(tmpdir) for excelfile in source: shutil.copy(tmpdir + '/' + excelfile, dirforzip) #return outfilefinal arc = 'dataarchive.zip' if datafile: arc = "%s_%s.zip" % (datafile, switch) compile2zip(dirforzip, arc) root = config['apiroot'] + "/collabs/static/tmp/" + str(arc) #root = config['apiroot'] + "/collabs/static/tmp/" + str(outfile) return redirect(root, code=301) else: zipfile = downloadzip(pid) # CHANGE #return zipfile # DEBUG1 root = config['apiroot'] + "/collabs/static/tmp/" + zipfile # HTML #resp = make_response(render_template('progress.html', download=root)) #return "<a href=\"" + str(root) + "\">Download dataset(s) with all papers (zip archive)</a>" #return resp return redirect(root, code=301)
# import datetime # But we still need time from dateutil import tz # For interpreting local times # OAuth2 - Google library implementation for convenience from oauth2client import client import httplib2 # used in oauth2 flow # Google API for services from apiclient import discovery ### # Globals ### import config if __name__ == "__main__": CONFIG = config.configuration() else: CONFIG = config.configuration(proxied=True) app = flask.Flask(__name__) app.debug = CONFIG.DEBUG app.logger.setLevel(logging.DEBUG) app.secret_key = CONFIG.SECRET_KEY SCOPES = 'https://www.googleapis.com/auth/calendar.readonly' CLIENT_SECRET_FILE = CONFIG.GOOGLE_KEY_FILE ## You'll need this APPLICATION_NAME = 'MeetMe class project' checked = {} #############################
sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), '../modules'))) from config import configuration def loadjson(apiurl): jsondataurl = apiurl req = urllib2.Request(jsondataurl) opener = urllib2.build_opener() f = opener.open(req) dataframe = simplejson.load(f) return dataframe config = configuration() connection = Connection(host, token) dataverse = connection.get_dataverse('clioinfra') for item in dataverse.get_contents(): # u'protocol': u'hdl', u'authority': u'10622' u'identifier': u'R8EJJF' try: handle = str(item['protocol']) + ':' + str( item['authority']) + "/" + str(item['identifier']) datasetid = item['id'] url = "https://" + str(host) + "/api/datasets/" + str( datasetid) + "/versions/1.0?&key=" + str(token) print item dataframe = loadjson(url) for fileitem in dataframe['data']['files']:
""" import flask import logging import arrow # Replacement for datetime, based on moment.js # Our own modules import pre # Preprocess schedule file import config # Configure from configuration files or command line ### # Globals ### app = flask.Flask(__name__) if __name__ == "__main__": configuration = config.configuration() else: # If we aren't main, the command line doesn't belong to us configuration = config.configuration(proxied=True) if configuration.DEBUG: app.logger.setLevel(logging.DEBUG) # Pre-processed schedule is global, so be careful to update # it atomically in the view functions. # schedule = pre.process(open(configuration.SYLLABUS)) ### # Pages # Each of these transmits the default "200/OK" header
def downloadzip(pid): DEBUG = 0 (fullpath) = ('') fullmetadata = {} logscale = 0 config = configuration() config['remote'] = 'on' API_TOKEN = config['key'] HOSTNAME = config['dataverseroot'] cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/" tmpdir = config['tmpdir'] filerandom = randomword(10) #filerandom = '12345' arc = "data" + filerandom + ".zip" filename = filerandom finaldir = config['path'] + '/static/tmp' # ToDO if filename: finaldir = str(finaldir) + '/' + str(filename) tmpdir = str(tmpdir) + '/' + str(filename) try: os.mkdir(tmpdir) os.mkdir(finaldir) except: donothing = 'ok' customyear = '' fromyear = request.args.get('y[min]') toyear = request.args.get('y[max]') historical = request.args.get('type[0]') (handles, pidslist) = pidfrompanel(pid) try: if pidslist: fullmetadata = load_fullmetadata(pidslist) except: showwarning = 1 # Log scales switch if request.args.get('logscale'): logscale = 1 # Select countries customcountrycodes = '' f = request.args for key in f.keys(): if is_location(key): for value in sorted(f.getlist(key)): customcountrycodes = str(customcountrycodes) + str(value) + ',' if customcountrycodes: customcountrycodes = customcountrycodes[:-1] if handles: if historical: api = config['apiroot'] + "/collabs/static/data/historical.json" (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '') hist = countries else: hist = '' (classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle']) #geocoder = buildgeocoder(dataset, config) (modern, historical) = loadgeocoder(config, dataset, 'geocoder') for handle in handles: #if remote: # (class1, dataset) = loaddataset_fromurl(config, handle) #else: # dataset = loaddataset(handles) #(cfilter, notint) = selectint(activeindex.values) #(moderndata, historicaldata) = loadgeocoder(dataset, '') # CHANGE #return str(dataset.index) (header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale) filename = filename + '.xls' fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata) else: # Clio format download zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir) (alonepid, revid, cliohandle, clearpid) = findpid(pid) if alonepid: handles = [clearpid] for pid in handles: if historical: api = config['apiroot'] + "/collabs/static/data/historical.json" (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '') hist = countries else: hist = '' filename = filename + '.xls' # 2DEBUG (header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale) #codes = hist #return str(fullmetadata) metadata = fullmetadata result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata) try: for everypid in handles: # Download papers zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir) except: nopapers = 1 compile2zip(finaldir, arc) filename = arc return filename
import os from flask import Flask, redirect, url_for, request, render_template from pymongo import MongoClient import flask from flask import request import arrow # Replacement for datetime, based on moment.js import acp_times # Brevet time calculations import config import time import logging import datetime app = flask.Flask(__name__) CONFIG = config.configuration() app.secret_key = CONFIG.SECRET_KEY client = MongoClient("db", 27017) db = client.tododb db.tododb.drop() '''@app.route('/') def todo(): _items = db.tododb.find() items = [item for item in _items] return render_template('todo.html', items=items) @app.route('/new', methods=['POST']) def new(): item_doc = { 'name': request.form['name'],
def dataapi(): (datafilter, handles) = ({}, []) datafilter['ctrlist'] = '' logscale = '' config = configuration() customyear = '' fromyear = '1500' toyear = '2012' categoriesMax = 6 countriesNum = 200 geocoder = '' (special, getrange, colormap, pallette, customcountrycodes, switch) = ('', '', '', '', '', 'modern') if request.args.get('special'): special = request.args.get('special') if request.args.get('logscale'): logscale = request.args.get('logscale') if request.args.get('year'): customyear = request.args.get('year') datafilter['startyear'] = customyear datafilter['endyear'] = customyear if request.args.get('catmax'): categoriesMax = int(request.args.get('catmax')) if request.args.get('getrange'): getrange = request.args.get('getrange') if request.args.get('colors'): pallette = request.args.get('colors') if request.args.get('colormap'): colormap = request.args.get('colormap') if request.args.get('geocoder'): switch = request.args.get('geocoder') if switch == 'on': switch = 'modern' if request.args.get('handle'): handlestring = request.args.get('handle') ishandle = re.search(r'(hdl:\d+\/\w+)', handlestring) if ishandle: handle = ishandle.group(1) handle = handle.replace("'", "") else: handle = handlestring handles.append(handle) if request.args.get('ctrlist'): customcountrycodes = '' tmpcustomcountrycodes = request.args.get('ctrlist') c = tmpcustomcountrycodes.split(',') for ids in sorted(c): if ids: customcountrycodes = str(customcountrycodes) + str(ids) + ',' customcountrycodes = customcountrycodes[:-1] datafilter['ctrlist'] = tmpcustomcountrycodes hist = {} config = configuration() try: if len(customcountrycodes): countriesNum = len(customcountrycodes.split(',')) if countriesNum < categoriesMax: if countriesNum >= 1: categoriesMax = countriesNum except: nothing = 1 # Old version of panel data #(header, panelcells, codes, x1, x2, x3, x4, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale) panelcells = [] # New version is fast if config: (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') (subsets, panel) = ({}, []) try: (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) if not datasubset.empty: datasubset = datasubset.dropna(how='all') panel.append(datasubset) subsets[handle] = datasubset except: subsets[handles[0]] = get_iishvisitors_frame(int(customyear)) (panelcells, originalvalues) = dataset2panel(config, subsets[handles[0]], modern, logscale) #(header, panelcells, codes, x1, x2, x3, x4, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale) #modern = moderncodes(config['modernnames'], config['apiroot']) #jsondata = data2json(modern, codes, panelcells) #data = json.dumps(jsondata, ensure_ascii=False, sort_keys=True, indent=4) # SCALES if switch: if switch == 'historical': geocoder = historical else: geocoder = modern #geocoder = '' (defaultcolor, colors) = getcolors(categoriesMax, pallette, colormap) (catlimit, ranges, dataset) = getscales(config, panelcells, colors, categoriesMax, geocoder, originalvalues, switch, logscale) if getrange: (showrange, tmprange) = combinerange(ranges) webscale = webscales(showrange, colors, defaultcolor) data = json.dumps(webscale, ensure_ascii=False, sort_keys=True, indent=4) return Response(data, mimetype='application/json') else: data = json.dumps(dataset, ensure_ascii=False, sort_keys=True, indent=4) return Response(data, mimetype='application/json')
import os from flask import Flask, render_template, session, request from flaskext.mysql import MySQL from config import configuration import logging logging.basicConfig(format=' %(levelname)s - %(asctime)s - %(message)s ', level=logging.DEBUG) app = Flask(__name__) mysql = MySQL() app.config['MYSQL_DATABASE_USER'] = configuration().getDbUser() app.config['MYSQL_DATABASE_PASSWORD'] = configuration().getDbPass() app.config['MYSQL_DATABASE_DB'] = configuration().getDatabase() app.config['MYSQL_DATABASE_HOST'] = configuration().getLocalhost() mysql.init_app(app) @app.route('/', methods=['GET']) def home(): if not session.get('logged_in'): return render_template('login.html') else: configuration().getAppUser() logging.info(getTotalFlights()) logging.info(getAllFlights2020()) logging.info(getAllFlights2019()) logging.info(getAllFlights2018())
def tableapi(): # years in filter config = configuration() switch = 'modern' datafilter = {} datafilter['ctrlist'] = '' customyear = '' fromyear = '1500' datafilter['startyear'] = fromyear toyear = '2012' datafilter['endyear'] = toyear customcountrycodes = '' (aggr, logscale, dataset, handles) = ('', '', '', []) # Select countries f = request.args for key in f.keys(): if key == 'loc': for value in sorted(f.getlist(key)): if value: customcountrycodes = str(customcountrycodes) + str( value) + ',' if customcountrycodes: customcountrycodes = customcountrycodes[:-1] #handle = "F16UDU" # HANDLE if request.args.get('handle'): handledataset = request.args.get('handle') try: (pids, pidslist) = pidfrompanel(handledataset) handles.append(pids[0]) except: handles.append(handledataset) nopanel = 'yes' if request.args.get('dataset'): dataset = request.args.get('dataset') if request.args.get('hist'): switch = 'historical' if request.args.get('ctrlist'): customcountrycodes = '' tmpcustomcountrycodes = request.args.get('ctrlist') c = tmpcustomcountrycodes.split(',') for ids in sorted(c): if ids: customcountrycodes = str(customcountrycodes) + str(ids) + ',' customcountrycodes = customcountrycodes[:-1] datafilter['ctrlist'] = customcountrycodes if not customcountrycodes: customcountrycodes = '528' if request.args.get('yearmin'): fromyear = request.args.get('yearmin') datafilter['startyear'] = fromyear if request.args.get('yearmax'): toyear = request.args.get('yearmax') datafilter['endyear'] = toyear if request.args.get('aggr'): aggr = request.args.get('aggr') # Log scales switch if request.args.get('logscale'): logscale = request.args.get('logscale') DEBUG = 0 old = '' if old: apifile = str(dataset) + ".json" jsonapi = config['apiroot'] + "/collabs/static/data/" + apifile dataframe = load_api_data(jsonapi, '') loccodes = loadcodes(dataframe) (ctr, header) = countryset(customcountrycodes, loccodes) indicator = '' (frame, years, values, dates, original) = createframe(indicator, loccodes, dataframe, customyear, fromyear, toyear, ctr, logscale, DEBUG) names = [ 'indicator', 'm', 'ctrcode', 'country', 'year', 'intcode', 'value', 'id' ] (csvdata, aggrdata) = combinedata(ctr, frame, loccodes) # New version is fast else: (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) (subsets, panel) = ({}, []) for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) if not datasubset.empty: datasubset = datasubset.dropna(how='all') panel.append(datasubset) subsets[handle] = datasubset classification = modern if switch == 'historical': classification = historical (csvdata, aggrdata) = dataset_to_csv(config, subsets[handles[0]], classification) if aggr: csvdata = aggrdata return (csvdata, aggrdata)
def initialization(): """ This function reads the user-defined parameters and paths from :mod:`config.py`, then checks the validity of the input files. If they are missing or are not rasters, a warning is thrown and the code is exited. The same applies if the rasters do not have the same resolution or scope. If the input files are valid, the CSV file in *input_stats* is generated and filled with the information that is already available. It will be called by other functions and eventually filled by them. :return: The updated dictionaries param and paths. :rtype: tuple(dict, dict) """ timecheck("Start") # import param and paths paths, param = configuration() # Check whether the inputs are correct if not len(paths["inputs"]): warn("No input file given", UserWarning) sys.exit(0) for input_file in paths["inputs"]: if not os.path.isfile(input_file): warn("File does not exist: " + input_file, UserWarning) sys.exit(0) elif not input_file.endswith(".tif"): warn("File is not raster: " + input_file, UserWarning) sys.exit(0) # Check that all rasters have the same scope and resolution for input_file in paths["inputs"]: dataset = gdal.Open(input_file) (upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = dataset.GetGeoTransform() Crd_all = np.array( [[upper_left_y], [upper_left_x + x_size * dataset.RasterXSize], [upper_left_y + y_size * dataset.RasterYSize], [upper_left_x]] ) if input_file == paths["inputs"][0]: Crd_all_old = Crd_all x_size_old = x_size y_size_old = y_size elif (Crd_all_old != Crd_all).any() or (x_size_old != x_size) or (y_size_old != y_size): warn("Not the same scope / resolution!", UserWarning) sys.exit(0) param["Crd_all"] = Crd_all param["res_desired"] = np.array([abs(x_size), abs(y_size)]) param["GeoRef"] = calc_geotiff(Crd_all, param["res_desired"]) # Create dataframe for input stats df = pd.DataFrame( index=[ "map_parts_total", "output_raster_columns", "output_raster_rows", "ref_part_name", "size_max", "std_max", "max_no_of_cl_ref", "max_no_of_cl_total", ], columns=["value"], ) df.loc[["output_raster_columns", "output_raster_rows"], "value"] = (dataset.RasterXSize, dataset.RasterYSize) if not os.path.exists(paths["input_stats"]): df.to_csv(paths["input_stats"], sep=";", decimal=",") timecheck("End") return paths, param
def searchdata(query): config = configuration() searchapi = config[ 'dataverseroot'] + "/api/search?q=" + query + "&key=" + config['key'] dataframe = load_api_data(searchapi, '') return json.dumps(dataframe)
""" John Doe's Flask API. """ import config import os from flask import Flask, render_template, abort, send_from_directory app = Flask(__name__) options = config.configuration() ILLEGAL_CHARS = ["//", "~", ".."] DOCROOT = options.DOCROOT @app.route("/") def hello(): return "UOCIS docker demo!\n" @app.route("/<path:filename>") def get_page(filename): if any((char in filename) for char in ILLEGAL_CHARS): abort(403) if filename not in os.listdir(DOCROOT): abort(404) return send_from_directory(DOCROOT, filename), 200 @app.errorhandler(404)
def treemapweb(): (thisyear, datafilter, yearmin, lastyear, handles) = (0, {}, 1500, 2010, []) (action, switch, geodataset) = ('', 'modern', '') config = configuration() datafilter['startyear'] = yearmin datafilter['endyear'] = lastyear datafilter['ctrlist'] = '' handle = '' if request.args.get('handle'): handledataset = request.args.get('handle') try: (pids, pidslist) = pidfrompanel(handledataset) handle = pids[0] handles.append(handle) except: handles.append(handledataset) nopanel = 'yes' if request.args.get('face'): handle = request.args.get('face') handles.append(handle) if request.args.get('year'): thisyear = request.args.get('year') if request.args.get('action'): action = request.args.get('action') if request.args.get('ctrlist'): datafilter['ctrlist'] = request.args.get('ctrlist') if int(thisyear) > 0: datafilter['startyear'] = int(thisyear) datafilter['endyear'] = int(thisyear) if request.args.get('historical'): switch = 'historical' # Geocoder (classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle']) #(modern, historical) = loadgeocoder(config, geodataset, 'geocoder') (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') if switch == 'modern': activeindex = modern.index coder = modern class1 = switch else: activeindex = historical.index coder = historical class1 = switch # Loading dataset in dataframe try: (class1, dataset, title, units) = content2dataframe(config, handle) except: return 'No dataset ' + handle (cfilter, notint) = selectint(activeindex.values) (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) (subsets, panel) = ({}, []) # Show only available years if action == 'showyears': years = [] datafilter['startyear'] = yearmin datafilter['endyear'] = lastyear (datasubset, ctrlist) = datasetfilter(maindata[handles[0]], datafilter) # Remove years without any values if not datafilter['ctrlist']: if np.nan in datasubset.index: datasubset = datasubset.drop(np.nan, axis=0) for colyear in datasubset.columns: if datasubset[colyear].count() == 0: datasubset = datasubset.drop(colyear, axis=1) (years, notyears) = selectint(datasubset.columns) # YEARS return Response(json.dumps(years), mimetype='application/json') # Process all indicators for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) if not datasubset.empty: #datasubset = datasubset.dropna(how='all') if not datafilter['ctrlist']: if np.nan in datasubset.index: datasubset = datasubset.drop(np.nan, axis=0) panel.append(datasubset) subsets[handle] = datasubset maindata = subsets[handles[0]] treemapdata = buildtreemap(config, maindata, switch, cfilter, coder) return Response(treemapdata, mimetype='application/json')
import falcon from falcon_multipart.middleware import MultipartMiddleware from config import configuration configuration() from views import RecognitionView, LabelsView, RecognitionOldView api = application = falcon.API(middleware=[MultipartMiddleware()]) recognition_view = RecognitionView() api.add_route('/api/v2/recognition', recognition_view) labels_view = LabelsView() api.add_route('/api/v1/labels', labels_view) recognition_old_view = RecognitionOldView() api.add_route('/api/v1/recognition', recognition_old_view)
def panel(): (thisyear, datafilter, handle, yearmin, yearmax, thisyear, ctrlist, lastyear, logscale) = (0, {}, '', '1500', '2020', 1950, '', 2010, '') handles = [] config = configuration() datafilter['startyear'] = yearmin datafilter['endyear'] = lastyear datafilter['ctrlist'] = config['ctrlist'] #modern = moderncodes(config['modernnames'], config['apiroot']) if request.args.get('handle'): handle = str(request.args.get('handle')) handle = handle.replace(" ", "") handle = handle.replace("'", "") try: (pids, pidslist) = pidfrompanel(handle) handles = pids except: nopanel = 'yes' handles.append(handle) if request.args.get('face'): facehandle = request.args.get('face') if facehandle not in handles: handles.append(facehandle) if request.args.get('dataset'): dataset = request.args.get('dataset') if request.args.get('ctrlist'): customcountrycodes = '' ctrlist = request.args.get('ctrlist') datafilter['ctrlist'] = ctrlist if request.args.get('logscale'): logscale = request.args.get('logscale') if request.args.get('year'): thisyear = request.args.get('year') datafilter['startyear'] = int(thisyear) datafilter['endyear'] = int(thisyear) if request.args.get('yearmin'): fromyear = request.args.get('yearmin') datafilter['startyear'] = fromyear if request.args.get('yearmax'): toyear = request.args.get('yearmax') datafilter['endyear'] = toyear if request.args.get('hist'): switch = 'historical' if datafilter['ctrlist'] == '': datafilter['ctrlist'] = config['histctrlist'] else: switch = 'modern' (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) (subsets, subsetyears, panel) = ({}, [], []) for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) datasubset['handle'] = handle if not datasubset.empty: datasubset = datasubset.dropna(how='all') try: if np.nan in datasubset.index: datasubset = datasubset.drop(np.nan, axis=0) except: skip = 'yes' for year in datasubset: if datasubset[year].count() == 0: datasubset = datasubset.drop(year, axis=1) (datayears, notyears) = selectint(datasubset.columns) panel.append(datasubset) subsets[handle] = datasubset subsetyears.append(datayears) dataframe = subsets ctrlimit = 10 # Trying to find the best year with most filled data values try: bestyearlist = subsetyears[0] for i in range(1, len(subsetyears)): bestyearlist = list(set(bestyearlist) & set(subsetyears[i])) #bestyearlist = bestyearlist.sort() thisyear = bestyearlist[0] except: bestyearlist = [] allcodes = {} panel = [] names = {} for handle in dataframe: try: names[handle] = metadata[handle]['title'] except: names[handle] = 'title' try: #(dataset, codes) = paneldatafilter(dataframe[handle], int(yearmin), int(yearmax), ctrlist, handle) dataset = dataframe[handle] if not dataset.empty: panel.append(dataset) except: nodata = 0 if panel: totalpanel = pd.concat(panel) cleanedpanel = totalpanel.dropna(axis=1, how='any') cleanedpanel = totalpanel #return str(cleanedpanel.to_html()) totalpanel = cleanedpanel if int(thisyear) <= 0: thisyear = totalpanel.columns[-2] result = '' original = {} if thisyear: if switch == 'historical': geocoder = historical if switch == 'hist': geocoder = historical else: geocoder = modern result = 'Country,' for handle in handles: result = result + str(metadata[handle]['title']) + ',' result = result[:-1] known = {} for code in totalpanel.index: if str(code) not in known: result = result + '\n' + str( geocoder.ix[int(code)][config['webmappercountry']]) for handle in handles: tmpframe = totalpanel.loc[totalpanel['handle'] == handle] try: (thisval, original) = value2scale(tmpframe.ix[code][thisyear], logscale, original) except: thisval = 'NaN' result = result + ',' + str(thisval) known[str(code)] = code return Response(result, mimetype='text/plain') (allyears, notyears) = selectint(cleanedpanel.columns) (codes, notcodes) = selectint(cleanedpanel.index) cleanedpanel.index = codes (header, data, countries, handles, vhandles) = panel2dict(config, cleanedpanel, names) #return str(data) #thisyear = 1882 #return str(countries) #return str(countries) years = [] for year in sorted(data): try: years.append(int(year)) lastyear = year except: skip = 1 # Return only years if request.args.get('showyears'): yearsdata = {} yearsdata['years'] = years yearsdata['latestyear'] = lastyear #yearsdata['data'] = data yearsjson = json.dumps(yearsdata, ensure_ascii=False, sort_keys=True, indent=4) return Response(yearsjson, mimetype='application/json') return Response(result, mimetype='text/plain')
def main(): handle = '' dataverse = '' customkey = '' config = configuration() try: myopts, args = getopt.getopt(sys.argv[1:], "H:r:d:k:D:") except getopt.GetoptError as e: print(str(e)) print( "Usage: %s -y year -d datatype -r region -f filename -DDEBUG -o output" % sys.argv[0]) sys.exit(2) (handle, rhandle, customdv) = ('', '', '') for o, a in myopts: if o == '-H': handle = a if o == '-r': rhandle = a if o == '-d': dataverse = a if o == '-k': customkey = a if o == '-D': customdv = a dataset = {} DEBUG = '' path = config['path'] # Default dataverse root = config['dataverseroot'] key = config['key'] dvname = config['branch'] if dataverse: root = dataverse if customkey: key = customkey if customdv: dvname = customdv files = [] if rhandle: contentsapi = root + "/api/dataverses/" + dvname + "/contents?key=" + key print contentsapi newdata = load_api_data(contentsapi, '') metadata = newdata['data'] for item in metadata: dv = item['id'] files = getfiles(root, dv, key) if handle: print handle (datahandle, datasetID, fileID) = parsehandle(handle) files.append(fileID) for fileID in files: fullpath = downloadfile(root, path, fileID, key) print fullpath (pid, revid, cliohandle, clearpid) = findpid(handle) #try: if pid: handle = pid jsonfile = dataextractor(fullpath, path, pid, fileID) if jsonfile: title = 'Test' datasetadd(jsonfile, clearpid, handle, title) print handle print clearpid
def test_boundaries(self): config = configuration() root = config['apiroot'] dataset = 'dataframe66_' countries = getboundaries(root, dataset) self.assertEqual(countries[1879][0], 'Austria-Hungary')