def getAll(url):
data = Flow.getData(url, Flow.getToken())
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
print(
checkTime(time.time()) + ' GET DATA FROM[' + INSTANCE + "," +
url.split("?")[1].replace("&", ",") + ']')
url = data.get('nextPageUrl')
getAll(url)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
def getFile(iName, dataName):
instanceURL = "https://" + iName + ".akvolumen.org/api/"
dashboards = Flow.getData(instanceURL + 'library', token)
datasets = pd.DataFrame(dashboards['datasets'])
datasets['api'] = datasets['id'].apply(
lambda x: instanceURL + 'datasets/' + x)
dataPath = list(datasets.loc[datasets['name'] == dataName]['api'])[0]
data = Flow.getData(dataPath, token)
columns = list(pd.DataFrame(data['columns'])['title'])
rows = pd.DataFrame(data['rows'], columns=columns)
downloadName = iName + '_' + dataName + '.csv'
downloadName = downloadName.replace(' ', '_')
dir_path = os.path.dirname(os.path.realpath(__file__))
rows.to_csv(downloadName)
print("Downloaded to" + dir_path + '/' + downloadName)
def download():
apiData = Flow.getResponse(requestURI, FlowToken).get('forms')
questions = lambda x : [{'id':a['id'],'name':a['name'],'questions':details(a['questions'])} for a in x]
details = lambda x : [{'id':a['id'],'name':a['name'].replace(' ','_'),'type':a['type']} for a in x]
meta = questions(apiData[0]['questionGroups'])
mt = pd.DataFrame(meta)
groupID = mt['id'][0]
metadata = mt['questions'][0]
getAll(formURI)
output = pd.DataFrame(dataPoints)
number_columns = []
for qst in metadata:
qName = 'q' + str(qst['id'])
qId = str(qst['id'])
qType = qst['type']
output[qName] = output['responses'].apply(lambda x: FlowHandler(x[groupID],qId,qType))
if qType == 'NUMBER':
number_columns.append(qName)
if qType == 'GEO':
output['latitude'] = output[qName].apply(lambda x: x[0] if x is not None else x)
output['longitude'] = output[qName].apply(lambda x: x[1] if x is not None else x)
output = output.drop([qName], axis=1)
else:
question_columns.append(qName)
output['collection_date'] = output['submissionDate'].apply(getTime)
output = output.drop(['responses','submissionDate','id','modifiedAt','createdAt','displayName','surveyalTime','deviceIdentifier'], axis=1)
output = output.rename(columns={"dataPointId":"instance"})
output = output[question_columns]
return output
def getList(iName):
instanceURL = "https://" + iName + ".akvolumen.org/api/"
dashboards = Flow.getData(instanceURL + 'library', token)
datasets = pd.DataFrame(dashboards['datasets'])
datasets['api'] = datasets['id'].apply(
lambda x: instanceURL + 'datasets/' + x)
dataName = list(datasets['name'])
return dataName
def execute(folder_id):
urls = Flow.getResponse(requestURI + '/surveys?folder_id=' +
folder_id).get('surveys')
urls = [a['surveyUrl'] for a in urls if 'surveyUrl' in a]
for url in urls:
try:
get_data(url)
except:
pass
return True
def getAll(url):
data = Flow.getResponse(url, FlowToken)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
url = data.get('nextPageUrl')
getAll(url)
except:
return "done"
def getAll(url):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
print(checkTime(time.time()) + ' GET DATA FROM[' + url + ']')
url = data.get('nextPageUrl')
getAll(url)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
def get_data(survey_url):
mt = Flow.getResponse(survey_url)
meta = pd.DataFrame(mt['forms'][0]['questionGroups'][0]['questions'])
name = mt['forms'][0]['questionGroups']
form = mt['forms'][0]['formInstancesUrl']
submitter_id = mt['name'].split('_')[1]
submitter_name = mt['name'].split('_')[0]
meta_id = name[0]['id']
try:
push_data(form, submitter_id, submitter_name, meta, meta_id)
except:
pass
return True
def push_data(datapoints_url, submitter_id, submitter_name, meta, meta_id):
src = Flow.getResponse(datapoints_url)
sources = src.get('formInstances')
if len(sources) > 0:
data = [d['responses'] for d in sources if 'responses' in d]
data = [d[meta_id][0] for d in data if meta_id in d]
submit_date = [
d['submissionDate'] for d in sources if 'submissionDate' in d
]
appending(meta, data, submitter_id, submit_date, submitter_name)
try:
next_page = src.get('nextPageUrl')
push_data(next_page, submitter_id, submitter_name, meta, meta_id)
except:
pass
return True
def getAll(url, page_count):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
psource = url.replace(
requestURI + '/form_instances?survey_id=',
'').replace('&form_id=', '|')
if '&cursor=' in psource:
psource = psource.split('&cursor=')[0]
page_count += 1
print(
checkTime(time.time()) + ' GET DATA FROM [' + psource +
'|PAGE ' + str(page_count) + ']')
url = data.get('nextPageUrl')
getAll(url, page_count)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
data = Flow.getData(url, Flow.getToken())
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
print(
checkTime(time.time()) + ' GET DATA FROM[' + INSTANCE + "," +
url.split("?")[1].replace("&", ",") + ']')
url = data.get('nextPageUrl')
getAll(url)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
apiData = Flow.getData(surveyURI, Flow.getToken()).get("forms")
questions = lambda x: [{
'id': a['id'],
'name': a['name'],
'questions': details(a['questions'])
} for a in x]
details = lambda x: [{
'id': a['id'],
'name': a['name'].replace(' ', '_'),
'type': a['type'],
'code': a['variableName']
} for a in x]
meta = questions(apiData[0]['questionGroups'])
mt = pd.DataFrame(meta)
groupID = mt['id'][0]
metadata = mt['questions'][0]
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
instanceURI = 'spiceup'
requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI
EMAIL_RECEPIENTS = [
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**'
]
EMAIL_SENDER = os.environ['G4AW_EMAIL']
EMAIL_PASSWORD = os.environ['G4AW_PWD']
apiData = Flow.getResponse(requestURI + '/surveys/227030237')
forms = apiData.get('forms')
RegistrationForm = apiData['registrationFormId']
start_time = time.time()
date_format = '%Y-%m-%dT%H:%M:%SZ'
def checkTime(x):
total_time = x - start_time
spent = time.strftime("%H:%M:%S", time.gmtime(total_time))
return spent
questions = lambda x: [{
'id': a['id'],
'name': a['name'],
def getResponse(url,rtype):
if rtype == "post":
response = Flow.postData(url, token)
else:
response = Flow.getData(url, token)
return response
print('INFO : PENDING...WAITING NEXT RESPONSE IN 10 SECONDS')
sleep(5)
checkUpdate(url)
def updateDataset(instance, dataset):
api = 'https://' + instance + '.akvolumen.org/api/'
try:
data = getResponse(api+ 'datasets/' + dataset + '/update', "post")
check = api + "job_executions/" + data['updateId']
sleep(5)
checkUpdate(check)
except:
return
try:
token = Flow.getToken()
with open('datasets.csv', newline='') as csvfile:
reader = csv.DictReader(csvfile)
print('--- STARTING TO UPDATE ---')
for idx, row in enumerate(reader):
instance = row['instance']
dataset = row['dataset']
print('INFO : UPDATING INSTANCE https://' + instance + '.akvolumen.org' + ' DATASET ID[' + dataset + ']')
if idx <= 5:
updateDataset(instance, dataset)
else:
print('WARNING:USAGE LIMIT EXCEEDED')
print('--- JOB IS DONE ---')
except:
print('---TOKEN IS INVALID----')
from Akvo import Flow instanceURI = 'wwfid' requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI print(Flow.getResponse(requestURI + '/folders'))
import time
import numpy as np
from Akvo import Flow
from FlowHandler import FlowHandler
import pandas as pd
import shapefile as sf
import geojson
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
instanceURI = 'spiceup'
requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI
FarmDetails = Flow.getResponse(requestURI + '/surveys/245360436')
FarmerProfiles = Flow.getResponse(requestURI + '/surveys/249380481')
start_time = time.time()
date_format = '%Y-%m-%dT%H:%M:%SZ'
def checkTime(x):
total_time = x - start_time
spent = time.strftime("%H:%M:%S", time.gmtime(total_time))
return spent
def fileDate():
return datetime.now().strftime("_%Y_%m_%d_%H%M")
questions = lambda x : [{'id':a['id'],'name':a['name'],'questions':details(a['questions'])} for a in x]
details = lambda x : [{'id':a['id'],'name':a['name'].replace(' ','_'),'type':a['type']} for a in x]
def getAll(url, dataPoints):
# # Starting Process
if (day == '01'):
instanceURI = 'spiceup'
requestURI = "https://api-auth0.akvo.org/flow/orgs/{}".format(instanceURI)
EMAIL_RECEPIENTS = os.environ['EMAIL_RECEIVER'].split(',')
EMAIL_BCC = os.environ['EMAIL_BCC'].split(',')
print(EMAIL_RECEPIENTS, EMAIL_BCC)
MAILJET_APIKEY = os.environ['MAILJET_APIKEY']
MAILJET_SECRET = os.environ['MAILJET_SECRET']
mailjet = Client(auth=(MAILJET_SECRET, MAILJET_APIKEY), version='v3.1')
token = Flow.getAccessToken()
apiData = Flow.getResponse(requestURI + '/surveys/227030237', token)
forms = apiData.get('forms')
RegistrationForm = apiData['registrationFormId']
start_time = time.time()
date_format = '%Y-%m-%dT%H:%M:%SZ'
def formatDate(dt):
return dt.strftime("%d/%m/%Y")
def checkTime(x):
total_time = x - start_time
spent = time.strftime("%H:%M:%S", time.gmtime(total_time))
return spent
from Akvo import Flow
from FlowHandler import FlowHandler
APIKEY='&api_key='+os.environ['CARTO_KEY']
CARTOURL='https://akvo.cartodb.com/api/v2/sql?q='
#DATABASEID = "test_iucn_tof"
DATABASEID = "tof_28030003"
INSTANCE='iucn'
SURVEYID='550001'
FORMID='28030003'
requestURI = 'https://api-auth0.akvo.org/flow/orgs/' + INSTANCE + '/surveys/' + SURVEYID
formURI = 'https://api-auth0.akvo.org/flow/orgs/' + INSTANCE + '/form_instances?survey_id=' + SURVEYID + '&form_id=' + FORMID
FlowToken = Flow.getAccessToken()
def getTime(x):
return int(datetime.strptime(x, '%Y-%m-%dT%H:%M:%SZ').strftime("%s%f")) / 1000
def getAll(url):
data = Flow.getResponse(url, FlowToken)
print(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
url = data.get('nextPageUrl')
getAll(url)
except:
def getList():
token = Flow.getToken()
dashboards = Flow.getData(lumenInstance + '/api/library', token)
ds = pd.DataFrame(dashboards['datasets'])
return ds.to_dict('records')
def getUpdatedData():
apiData = Flow.getResponse(requestURI + '/surveys/225170023')
forms = apiData.get('forms')
questions = lambda x: [{
'id': a['id'],
'name': a['name'],
'questions': details(a['questions'])
} for a in x]
details = lambda x: [{
'id': a['id'],
'name': a['name'].replace(' ', '_'),
'type': a['type']
} for a in x]
allResponses = {}
for form in forms:
questionGroups = questions(form['questionGroups'])
metas = pd.DataFrame(questionGroups)
formURI = form['formInstancesUrl']
allGroups = {}
for index, questionGroup in enumerate(questionGroups):
dataPoints = []
groupID = questionGroup['id']
metadata = metas['questions'][index]
print(
checkTime(time.time()) + ' +++ ' +
questionGroup['name'].upper() + ' +++')
def getAll(url, page_count):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
psource = url.replace(
requestURI + '/form_instances?survey_id=',
'').replace('&form_id=', '|')
if '&cursor=' in psource:
psource = psource.split('&cursor=')[0]
page_count += 1
print(
checkTime(time.time()) + ' GET DATA FROM [' + psource +
'|PAGE ' + str(page_count) + ']')
url = data.get('nextPageUrl')
getAll(url, page_count)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
getAll(formURI, 0)
output = pd.DataFrame(dataPoints)
print(checkTime(time.time()) + ' TRANSFORMING')
for qst in metadata:
qName = qst['name'].replace('_', ' ')
qId = str(qst['id'])
qType = qst['type']
try:
output[qName] = output['responses'].apply(
lambda x: FlowHandler(x[groupID], qId, qType))
if qType == 'GEO':
output[qName + '_lat'] = output[qName].apply(
lambda x: x[0] if x is not None else x)
output[qName + '_long'] = output[qName].apply(
lambda x: x[1] if x is not None else x)
output = output.drop([qName], axis=1)
except:
pass
try:
output = output.drop(['responses'], axis=1)
except:
pass
allGroups.update(
{questionGroup['name']: output.to_dict('records')})
allResponses.update({form['name']: allGroups})
updatedData = TransformData(allResponses)
return updatedData