def download():
apiData = Flow.getResponse(requestURI, FlowToken).get('forms')
questions = lambda x : [{'id':a['id'],'name':a['name'],'questions':details(a['questions'])} for a in x]
details = lambda x : [{'id':a['id'],'name':a['name'].replace(' ','_'),'type':a['type']} for a in x]
meta = questions(apiData[0]['questionGroups'])
mt = pd.DataFrame(meta)
groupID = mt['id'][0]
metadata = mt['questions'][0]
getAll(formURI)
output = pd.DataFrame(dataPoints)
number_columns = []
for qst in metadata:
qName = 'q' + str(qst['id'])
qId = str(qst['id'])
qType = qst['type']
output[qName] = output['responses'].apply(lambda x: FlowHandler(x[groupID],qId,qType))
if qType == 'NUMBER':
number_columns.append(qName)
if qType == 'GEO':
output['latitude'] = output[qName].apply(lambda x: x[0] if x is not None else x)
output['longitude'] = output[qName].apply(lambda x: x[1] if x is not None else x)
output = output.drop([qName], axis=1)
else:
question_columns.append(qName)
output['collection_date'] = output['submissionDate'].apply(getTime)
output = output.drop(['responses','submissionDate','id','modifiedAt','createdAt','displayName','surveyalTime','deviceIdentifier'], axis=1)
output = output.rename(columns={"dataPointId":"instance"})
output = output[question_columns]
return output
def getAll(url):
data = Flow.getResponse(url, FlowToken)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
url = data.get('nextPageUrl')
getAll(url)
except:
return "done"
def execute(folder_id):
urls = Flow.getResponse(requestURI + '/surveys?folder_id=' +
folder_id).get('surveys')
urls = [a['surveyUrl'] for a in urls if 'surveyUrl' in a]
for url in urls:
try:
get_data(url)
except:
pass
return True
def getAll(url):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
print(checkTime(time.time()) + ' GET DATA FROM[' + url + ']')
url = data.get('nextPageUrl')
getAll(url)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
def get_data(survey_url):
mt = Flow.getResponse(survey_url)
meta = pd.DataFrame(mt['forms'][0]['questionGroups'][0]['questions'])
name = mt['forms'][0]['questionGroups']
form = mt['forms'][0]['formInstancesUrl']
submitter_id = mt['name'].split('_')[1]
submitter_name = mt['name'].split('_')[0]
meta_id = name[0]['id']
try:
push_data(form, submitter_id, submitter_name, meta, meta_id)
except:
pass
return True
def push_data(datapoints_url, submitter_id, submitter_name, meta, meta_id):
src = Flow.getResponse(datapoints_url)
sources = src.get('formInstances')
if len(sources) > 0:
data = [d['responses'] for d in sources if 'responses' in d]
data = [d[meta_id][0] for d in data if meta_id in d]
submit_date = [
d['submissionDate'] for d in sources if 'submissionDate' in d
]
appending(meta, data, submitter_id, submit_date, submitter_name)
try:
next_page = src.get('nextPageUrl')
push_data(next_page, submitter_id, submitter_name, meta, meta_id)
except:
pass
return True
def getAll(url, page_count):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
psource = url.replace(
requestURI + '/form_instances?survey_id=',
'').replace('&form_id=', '|')
if '&cursor=' in psource:
psource = psource.split('&cursor=')[0]
page_count += 1
print(
checkTime(time.time()) + ' GET DATA FROM [' + psource +
'|PAGE ' + str(page_count) + ']')
url = data.get('nextPageUrl')
getAll(url, page_count)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
from Akvo import Flow instanceURI = 'wwfid' requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI print(Flow.getResponse(requestURI + '/folders'))
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
instanceURI = 'spiceup'
requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI
EMAIL_RECEPIENTS = [
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**'
]
EMAIL_SENDER = os.environ['G4AW_EMAIL']
EMAIL_PASSWORD = os.environ['G4AW_PWD']
apiData = Flow.getResponse(requestURI + '/surveys/227030237')
forms = apiData.get('forms')
RegistrationForm = apiData['registrationFormId']
start_time = time.time()
date_format = '%Y-%m-%dT%H:%M:%SZ'
def checkTime(x):
total_time = x - start_time
spent = time.strftime("%H:%M:%S", time.gmtime(total_time))
return spent
questions = lambda x: [{
'id': a['id'],
'name': a['name'],
def getUpdatedData():
apiData = Flow.getResponse(requestURI + '/surveys/225170023')
forms = apiData.get('forms')
questions = lambda x: [{
'id': a['id'],
'name': a['name'],
'questions': details(a['questions'])
} for a in x]
details = lambda x: [{
'id': a['id'],
'name': a['name'].replace(' ', '_'),
'type': a['type']
} for a in x]
allResponses = {}
for form in forms:
questionGroups = questions(form['questionGroups'])
metas = pd.DataFrame(questionGroups)
formURI = form['formInstancesUrl']
allGroups = {}
for index, questionGroup in enumerate(questionGroups):
dataPoints = []
groupID = questionGroup['id']
metadata = metas['questions'][index]
print(
checkTime(time.time()) + ' +++ ' +
questionGroup['name'].upper() + ' +++')
def getAll(url, page_count):
data = Flow.getResponse(url)
formInstances = data.get('formInstances')
for dataPoint in formInstances:
dataPoints.append(dataPoint)
try:
psource = url.replace(
requestURI + '/form_instances?survey_id=',
'').replace('&form_id=', '|')
if '&cursor=' in psource:
psource = psource.split('&cursor=')[0]
page_count += 1
print(
checkTime(time.time()) + ' GET DATA FROM [' + psource +
'|PAGE ' + str(page_count) + ']')
url = data.get('nextPageUrl')
getAll(url, page_count)
except:
print(checkTime(time.time()) + ' DOWNLOAD COMPLETE')
return "done"
getAll(formURI, 0)
output = pd.DataFrame(dataPoints)
print(checkTime(time.time()) + ' TRANSFORMING')
for qst in metadata:
qName = qst['name'].replace('_', ' ')
qId = str(qst['id'])
qType = qst['type']
try:
output[qName] = output['responses'].apply(
lambda x: FlowHandler(x[groupID], qId, qType))
if qType == 'GEO':
output[qName + '_lat'] = output[qName].apply(
lambda x: x[0] if x is not None else x)
output[qName + '_long'] = output[qName].apply(
lambda x: x[1] if x is not None else x)
output = output.drop([qName], axis=1)
except:
pass
try:
output = output.drop(['responses'], axis=1)
except:
pass
allGroups.update(
{questionGroup['name']: output.to_dict('records')})
allResponses.update({form['name']: allGroups})
updatedData = TransformData(allResponses)
return updatedData
import time
import numpy as np
from Akvo import Flow
from FlowHandler import FlowHandler
import pandas as pd
import shapefile as sf
import geojson
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
instanceURI = 'spiceup'
requestURI = 'https://api.akvo.org/flow/orgs/' + instanceURI
FarmDetails = Flow.getResponse(requestURI + '/surveys/245360436')
FarmerProfiles = Flow.getResponse(requestURI + '/surveys/249380481')
start_time = time.time()
date_format = '%Y-%m-%dT%H:%M:%SZ'
def checkTime(x):
total_time = x - start_time
spent = time.strftime("%H:%M:%S", time.gmtime(total_time))
return spent
def fileDate():
return datetime.now().strftime("_%Y_%m_%d_%H%M")
questions = lambda x : [{'id':a['id'],'name':a['name'],'questions':details(a['questions'])} for a in x]
details = lambda x : [{'id':a['id'],'name':a['name'].replace(' ','_'),'type':a['type']} for a in x]
def getAll(url, dataPoints):