def getTable(self, tableID):
if not self.authorization:
self.get_authorization()
#step1 - verify if form exists:
url = 'https://docs.google.com/spreadsheets/d/%s/export?format=csv&id=%s&gid=0' % (tableID,self.getValue("data collection table ID"))
headers = {'Authorization': 'Bearer {}'.format(self.authorization['access_token']),
'Content-Type': 'application/json'}
response = requests.get(url, headers=headers, proxies = getProxiesConf(), verify=False)
if response.status_code == 200:
csvIO = StringIO.StringIO(response.text)
csvIn = csv.DictReader(csvIO, delimiter=',', quotechar='"')
csvList = []
for row in csvIn:
csvList.append(row)
geometryField = ''
for field in csvList[0].keys():
if 'GEOMETRY' in field.upper():
geometryField = field
prefix = field[:-8]
len_prefix = len(field)-8
newCsvList = []
for row in csvList:
newRow = {}
for field in row.keys():
newRow[field[len_prefix:]] = row[field] # remap field
newCsvList.append(newRow)
return newCsvList
else:
pass #print "getTable", response, response.text
def getAvailableDataCollections(self):
method='GET'
url=self.getValue('url')+'//formList'
response= requests.request(method,url,proxies=getProxiesConf(),auth=self.getAuth(),verify=False)
root=ET.fromstring(response.content)
keylist=[form.attrib['url'].split('=')[1] for form in root.findall('form')]
return keylist,response
def sendForm(self, xForm_id, xForm):
# step1 - verify if form exists:
formList, response = self.getAvailableDataCollections()
form_key = xForm_id in formList
if response.status_code != requests.codes.ok:
return response
message = ''
if form_key:
message = 'Form Updated'
method = 'POST'
url = self.getValue('url') + '//formUpload'
else:
message = 'Created new form'
method = 'POST'
url = self.getValue('url') + '//formUpload'
files = open(xForm, 'r')
files = {'form_def_file': files}
response = requests.request(method, url, files=files, proxies=getProxiesConf(), auth=self.getAuth(), verify=False)
if response.status_code == 201:
self.iface.messageBar().pushMessage(self.tr("QgisODK plugin"),
self.tr('Layer is online(' + message + '), Collect data from App'),
level=QgsMessageBar.SUCCESS, duration=6)
elif response.status_code == 409:
self.iface.messageBar().pushMessage(self.tr("QgisODK plugin"),
self.tr("Form exist and can not be updated"),
level=QgsMessageBar.CRITICAL, duration=6)
else:
self.iface.messageBar().pushMessage(self.tr("QgisODK plugin"),
self.tr("Form is not sent "),
level=QgsMessageBar.CRITICAL, duration=6)
return response
def getTable(self, XFormKey):
method = 'GET'
#detect topelement
url = self.getValue('url') + '/formXml?formId=' + XFormKey
response = requests.request(method, url, proxies=getProxiesConf(), auth=self.getAuth(), verify=False)
root = ET.fromstring(response.content)
topElement = ''
for sub_elem in root.iter():
if sub_elem.get('id'):
print sub_elem.tag, sub_elem.get('id')
if sub_elem.get('id') == XFormKey:
topElement = re.sub("[\{].*?[\}]", "",sub_elem.tag)
print "topElement", topElement
url = self.getValue('url') + '/view/submissionList?formId=' + XFormKey
table = []
response = requests.request(method, url, proxies=getProxiesConf(), auth=self.getAuth(), verify=False)
if not response.status_code == 200:
return response, table
#try:
root = ET.fromstring(response.content)
ns = '{http://opendatakit.org/submissions}'
instance_ids = [child.text for child in root[0].findall(ns + 'id')]
for id in instance_ids:
if id:
url = self.getValue(
'url') + '/view/downloadSubmission?formId={}[@version=null and @uiVersion=null]/{}[@key={}]'.format(
XFormKey,topElement, id)
print (url)
response = requests.request(method, url, proxies=getProxiesConf() ,auth=self.getAuth(), verify=False)
if not response.status_code == 200:
print response, table
return response, table
root1 = ET.fromstring(response.content)
data = root1[0].findall(ns + topElement)
if data:
dict = {child.tag.replace(ns, ''): child.text for child in data[0]}
dict['UUID'] = id
print(id,dict)
mediaFile = root1.findall(ns + 'mediaFile')
if len(mediaFile) > 0:
mediaDict = {child.tag.replace(ns, ''): child.text for child in mediaFile[0]}
for key, value in dict.iteritems():
if value == mediaDict['filename']:
dict[key] = mediaDict['downloadUrl']
table.append(dict)
return response, table
def getJSON(self,xForm_id):
#step1 - verify if form exists:
form_key, response = self.formIDToPk(xForm_id)
if response.status_code != requests.codes.ok:
return response
if form_key:
url = 'https://api.ona.io/api/v1/data/%s' % form_key
response = requests.get(url, auth=self.getAuth(), proxies = getProxiesConf())
return response
def getMetadataFromID(self,fileID):
if not self.authorization:
self.get_authorization()
url = 'https://www.googleapis.com/drive/v3/files/'+fileID
headers = { 'Authorization':'Bearer {}'.format(self.authorization['access_token'])}
response = requests.get( url, headers = headers, proxies = getProxiesConf(), verify=False)
if response.status_code == requests.codes.ok:
return response.json()
else:
return None
def getAvailableDataCollections(self):
url = 'https://api.ona.io/api/v1/projects/%s/forms' % self.getValue("project_id")
response = requests.get(url, auth=self.getAuth(), proxies = getProxiesConf())
if response.status_code != requests.codes.ok:
return None, response
forms = response.json()
availableDataCollections = []
for form in forms:
availableDataCollections.append(form["id_string"])
return availableDataCollections,response
def formIDToPk(self,xForm_id):
#verify if form exists:
url = 'https://api.ona.io/api/v1/projects/%s/forms' % self.getValue("project_id")
response = requests.get(url, auth=self.getAuth(), proxies = getProxiesConf())
if response.status_code != requests.codes.ok:
return None, response
forms = response.json()
form_key = None
for form in forms:
if form['sms_id_string'] == xForm_id:
form_key = form['formid']
break
return form_key, response
def setDataSubmissionTable(self,xForm_id):
if not self.authorization:
self.get_authorization()
headers = {'Authorization': 'Bearer {}'.format(self.authorization['access_token'])}
folderId = self.createNew(self.getValue('folder'),'application/vnd.google-apps.folder')
self.shareFileWithCollectors(folderId, role='reader')
content_table_id = self.createNew(xForm_id[:-4]+"-collect-table",'application/vnd.google-apps.spreadsheet', parentsId=folderId)
self.shareFileWithCollectors(content_table_id,role='writer')
if content_table_id:
url = 'https://www.googleapis.com/drive/v3/files/'+content_table_id
response = requests.get(url,headers = headers, proxies = getProxiesConf(), verify=False)
self.notify(xForm_id,self.getValue('folder'),response.json()['id'],response.json()['name'])
return 'https://docs.google.com/spreadsheets/d/%s/edit' % response.json()['id']
else:
return None
def getAvailableDataCollections(self):
if not self.authorization:
self.get_authorization()
url = 'https://www.googleapis.com/drive/v3/files'
headers = {'Authorization':'Bearer {}'.format(self.authorization['access_token'])}
folderID = self.getIdFromName(self.getValue('folder'), mimeType = 'application/vnd.google-apps.folder')
params = {"q": "mimeType = 'application/vnd.google-apps.spreadsheet' and '%s' in parents" % folderID, "spaces": "drive"}
response = requests.get( url, headers = headers, params = params, proxies = getProxiesConf(), verify=False )
if response.status_code == requests.codes.ok:
files = response.json()["files"]
filesList = []
for file in files:
filesList.append(file["name"])
if filesList != []:
return filesList, response
else:
return None, response
else:
return None, response
def getIdFromName(self,fileName, mimeType = None):
if not self.authorization:
self.get_authorization()
url = 'https://www.googleapis.com/drive/v3/files'
headers = { 'Authorization':'Bearer {}'.format(self.authorization['access_token'])}
params = {"q": "name = '%s'" % fileName, "spaces": "drive"}
response = requests.get( url, headers = headers, params = params, proxies = getProxiesConf(), verify=False )
if response.status_code == requests.codes.ok:
found = response.json()
files = found['files']
if len(files) > 0:
if mimeType:
if files[0]['mimeType'] == mimeType:
return files[0]['id']
else:
return None
else:
return files[0]['id']
else:
return None
def downloadMedia(self, URI, downloadDir):
#fileName = URI.split('/')[-1]
response = requests.get(URI, allow_redirects=True, stream=True, proxies=getProxiesConf(),
auth=self.getAuth())
fileName = re.findall('filename="([^"]*)"', response.headers['content-disposition'])[0]
print ("content", fileName,response.headers['content-disposition'], response.status_code)
localAttachmentPath = os.path.abspath(os.path.join(downloadDir, fileName))
if response.status_code == 200:
with open(localAttachmentPath, 'wb') as f:
for chunk in response:
f.write(chunk)
localURI = localAttachmentPath
try:
if self.relativePathsCheckBox.isChecked():
localURI = os.path.relpath(localURI, QgsProject.instance().readPath("./"))
except:
pass
return localURI
else:
return "can't download media: " + str(response.reason)
def getTable(self,form_key):
#step1 - verify if form exists:
#form_key, response = self.formIDToPk(xForm_id)
#if response.status_code != requests.codes.ok:
# self.iface.messageBar().pushMessage(self.tr("QgisODK plugin"), self.tr("error loading csv table %s, %s.") % (
# response.status_code, response.reason), level=QgsMessageBar.CRITICAL, duration=6)
# return response, None
def UnicodeDictReader(utf8_data, **kwargs):
csv_reader = csv.DictReader(utf_8_encoder(utf8_data), **kwargs)
for row in csv_reader:
yield {unicode(key, 'utf-8'):unicode(value, 'utf-8') for key, value in row.iteritems()}
def utf_8_encoder(unicode_csv_data):
for line in unicode_csv_data:
yield line.encode('utf-8')
if form_key:
url = 'https://api.ona.io/api/v1/data/%s.csv' % form_key
response = requests.get(url, auth=self.getAuth(), proxies = getProxiesConf())
if response.status_code == 200:
csvIO = StringIO.StringIO(response.text)
csvIn = UnicodeDictReader(csvIO, delimiter=',', quotechar='"')
csvList = []
for row in csvIn:
remappedRow = {}
for key,value in row.iteritems():
if '/' in key:
cleanedKey = key.split('/')[-1]
remappedRow[cleanedKey] = value
else:
remappedRow[key] = value
csvList.append(remappedRow)
return response, csvList
else:
self.iface.messageBar().pushMessage(self.tr("QGISODK plugin"), self.tr("error loading csv table %s, %s.") % (
response.status_code, response.reason), level=QgsMessageBar.CRITICAL, duration=6)
return response, None
def getLayerFromTable(self,remoteData, downloadAttachements = None):
#remoteResponse = self.getJSON(xForm_id)
#response, remoteData = self.getTable(xForm_id)
geojson = {
"type": "FeatureCollection",
"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
"features": []
}
for record in remoteData:
if 'GEOMETRY' in record:
geomType, geomCoordinates = self.guessGeomType(record['GEOMETRY'])
feature = {
"type": "Feature",
"properties": {},
"geometry": {
"type": geomType
}
}
#build geojson geometry
jsonCoordinates = []
for geomCoordinate in geomCoordinates:
jsonCoordinates.append([float(geomCoordinate[1]),float(geomCoordinate[0])])
if geomType == 'Point':
feature["geometry"]["coordinates"] = [jsonCoordinates[0][0],jsonCoordinates[0][1]]
if geomType == 'LineString':
feature["geometry"]["coordinates"] = jsonCoordinates
if geomType == 'Polygon':
feature["geometry"]["coordinates"] = [jsonCoordinates]
else:
feature = {
"type": "Feature",
"properties": {},
"geometry": None
}
#recode attachments:
attachements = {}
if '_attachments' in record:
for attachment in record['_attachments']:
fileKey = attachment["download_url"].split('/')[-1]
#todo local download attachment files if option is checked
attachements[fileKey] = 'https://api.ona.io' + attachment["download_url"]
if downloadAttachements and QgsProject.instance().readPath("./") != "./":
downloadDir = os.path.join(QgsProject.instance().readPath("./"),'attachments_%s_%s' % (self.getValue("name"),self.getValue("project_id")))
if not os.path.exists(downloadDir):
os.makedirs(downloadDir)
response = requests.get(attachements[fileKey], stream=True, proxies = getProxiesConf())
localAttachmentPath = os.path.abspath(os.path.join(downloadDir,fileKey))
if response.status_code == 200:
with open(localAttachmentPath, 'wb') as f:
for chunk in response:
f.write(chunk)
attachements[fileKey] = localAttachmentPath
else:
attachements[fileKey] = ''
#build geojson properties
for fieldKey, fieldValue in record.iteritems():
if not fieldKey in ('GEOMETRY','_attachments','_tags','_notes','_bamboo_dataset_id','_geolocation'): # field exclusion to verify
if fieldValue in attachements.keys():
fieldValue = attachements[fieldValue]
if "/" in fieldKey: #check if grouped Field
cleanedKey = fieldKey.split("/")[-1]
else:
cleanedKey = fieldKey
fieldRemap = self.module.dlg.treeView.mapNameTofield(cleanedKey) #try to remap field name to existing field using map to property
feature["properties"][fieldRemap] = fieldValue
geojson["features"].append(feature)
return geojson
def get_authorization(self): #phase 2 oauth2
if not self.verification:
self.get_verification()
authorization_params = {
'client_id': self.client_id,
'client_secret': self.client_secret,
'code': self.verification,
'grant_type': 'authorization_code',
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob:auto'
}
response = requests.post('https://www.googleapis.com/oauth2/v4/token', params=authorization_params, proxies = getProxiesConf(), verify=False)
if response.status_code == requests.codes.ok:
authorization = response.json()
if 'error' in authorization:
QMessageBox().warning(None,"Google authorization error", "Google authorization error: %s" % authorization['error'])
self.verification = None
self.authorization = None
else:
self.authorization = authorization
elif response.status_code == 401: # token expired
refresh_params = {
'client_id': self.client_id,
'client_secret': self.client_secret,
'refresh_token': self.authorization['refresh_token'],
'grant_type': 'refresh_token'
}
else:
pass #print response.reason
def get_verification(self): #phase 1 oauth2
verification_params = {
'response_type': 'code',
'client_id': self.client_id,
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob:auto', #
'scope': 'https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/gmail.send', #'https://www.googleapis.com/auth/drive.file',
'login_hint': self.getValue('google drive login')
}
response = requests.post('https://accounts.google.com/o/oauth2/v2/auth', params=verification_params, proxies = getProxiesConf(), verify=False)
if response.status_code == requests.codes.ok:
self.verification = internalBrowser.getCode(response.text, self.getValue('google drive login'))
def createNew(self, name, mimeType, parentsId = None):
if not self.authorization:
self.get_authorization()
if mimeType == 'application/vnd.google-apps.folder' and name == '':
return 'root'
foundId = self.getIdFromName(name, mimeType = mimeType)
if foundId:
return foundId
else:
url = 'https://www.googleapis.com/drive/v3/files'
headers = { 'Authorization':'Bearer {}'.format(self.authorization['access_token']), 'Content-Type': 'application/json'}
metadata = {
"name": name,
"mimeType": mimeType
}
if parentsId:
metadata['parents'] = [parentsId]
response = requests.post( url, headers = headers, data = json.dumps(metadata), proxies = getProxiesConf(), verify=False)
if response.status_code != 200 or 'error' in response.json():
return None
return response.json()['id']
def shareFileWithCollectors(self, id, role = 'reader', type = 'user'):
self.getCollectors()
url = 'https://www.googleapis.com/drive/v3/files/%s/permissions' % id
headers = {'Authorization': 'Bearer {}'.format(self.authorization['access_token']), 'Content-Type': 'application/json'}
for email in self.collectors:
metadata = {
"role": role,
"type": type,
"emailAddress": email
}
response = requests.post(url, headers=headers, data=json.dumps(metadata), proxies = getProxiesConf(), verify=False)
def sendForm(self, xForm_id, xForm):
#step1 - verify if form exists:
form_key, response = self.formIDToPk(xForm_id)
if response.status_code != requests.codes.ok:
return response
if form_key:
method = 'PATCH'
url = 'https://api.ona.io/api/v1/forms/%s' % form_key
else:
method = 'POST'
url = 'https://api.ona.io/api/v1/projects/%s/forms' % self.getValue("project_id")
#step1 - upload form: POST if new PATCH if exixtent
files = {'xls_file': (xForm, open(xForm, 'rb'), 'application/vnd.ms-excel', {'Expires': '0'})}
response = requests.request(method, url, files=files, auth=self.getAuth(), proxies = getProxiesConf())#, proxies = proxyDict,headers={'Content-Type': 'application/octet-stream'})
return response
def sendForm(self, xForm_id, xForm):
if not self.authorization:
self.get_authorization()
xForm_id += '.xml'
fileId = self.getIdFromName(xForm_id)
folderId = self.createNew(self.getValue('folder'),'application/vnd.google-apps.folder')
self.shareFileWithCollectors(folderId, role='reader')
metadata = {
"name": xForm_id,
"description": 'uploaded by QGISODK plugin'
}
if fileId:
method = "PATCH"
url = 'https://www.googleapis.com/upload/drive/v3/files/%s?uploadType=multipart' % fileId
else:
method = "POST"
url = 'https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart'
metadata['parents'] = [folderId]
headers = { 'Authorization':'Bearer {}'.format(self.authorization['access_token']) }
data = ('metadata', json.dumps(metadata),'application/json; charset=UTF-8')
file = (xForm,open(xForm,'r'),'text/xml')
files = {'data':data, 'file':file }
response = requests.request(method, url, headers = headers, files = files, proxies = getProxiesConf(), verify=False )
self.shareFileWithCollectors(response.json()['id'],role='reader')
return response
def send_message(self, FROM , TO, SUBJECT, MSG):
if not self.authorization:
self.get_authorization()
# create a message to send
message = MIMEText(MSG.encode('utf-8'), 'plain', 'utf-8')
message['to'] = TO
message['from'] = FROM
message['subject'] = SUBJECT.encode('utf-8')
body = {'raw': base64.b64encode(message.as_string())}
url = 'https://www.googleapis.com/gmail/v1/users/me/messages/send'
headers = {'Authorization': 'Bearer {}'.format(self.authorization['access_token']), 'Content-Type': 'application/json'}
response = requests.post(url,headers = headers, data = json.dumps(body), proxies = getProxiesConf(), verify=False)