def QueryAllFeatures(self,url,sql,out_fields="*",chunksize=1000,saveLocation="",outName=""):
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
qRes = fl.query(where=sql, returnIDsOnly=True)
if 'error' in qRes:
print (qRes)
return qRes
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return {'success':True, 'message':"No features matched the query"}
print ("%s features to be downloaded" % total)
chunksize = min(chunksize, fl.maxRecordCount)
combinedResults = None
totalQueried = 0
for chunk in chunklist(l=oids, n=chunksize):
oidsQuery = ",".join(map(str, chunk))
if not oidsQuery:
continue
else:
results = fl.query(objectIds=oidsQuery,
returnGeometry=True,
out_fields=out_fields)
if isinstance(results,FeatureSet):
if combinedResults is None:
combinedResults = results
else:
for feature in results.features:
combinedResults.features.append(feature)
totalQueried += len(results.features)
print("{:.0%} Completed: {}/{}".format(totalQueried / float(total), totalQueried, total))
else:
print (results)
if saveLocation == "" or outName == "":
return combinedResults
else:
return combinedResults.save(saveLocation=saveLocation, outName=outName)
else:
print (qRes)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "QueryFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
def AddFeaturesToFeatureLayer(self,url,pathToFeatureClass,chunksize=0,lowerCaseFieldNames=False):
if arcpyFound == False:
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "ArcPy required for this function"
})
fl = None
try:
fl = FeatureLayer(
url=url,
securityHandler=self._securityHandler)
if chunksize > 0:
fc = os.path.basename(pathToFeatureClass)
inDesc = arcpy.Describe(pathToFeatureClass)
oidName = arcpy.AddFieldDelimiters(pathToFeatureClass,inDesc.oidFieldName)
arr = arcpy.da.FeatureClassToNumPyArray(pathToFeatureClass, (oidName))
syncSoFar = 0
messages = {'addResults':[],'errors':[]}
total = len(arr)
errorCount = 0
if total == '0':
print ("0 features in %s" % pathToFeatureClass)
return "0 features in %s" % pathToFeatureClass
print ("%s features in layer" % (total))
arcpy.env.overwriteOutput = True
if int(total) < int(chunksize):
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
else:
newArr = chunklist(arr,chunksize)
exprList = ["{0} >= {1} AND {0} <= {2}".format(oidName, nArr[0][0], nArr[len(nArr)-1][0])
for nArr in newArr]
for expr in exprList:
UploadLayer = arcpy.MakeFeatureLayer_management(pathToFeatureClass, 'TEMPCOPY', expr).getOutput(0)
#print(arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0) + " features in the chunk")
results = fl.addFeatures(fc=UploadLayer,lowerCaseFieldNames=lowerCaseFieldNames)
chunkCount = arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0)
print(chunkCount + " features in the chunk")
if chunkCount > 0:
if results is not None and 'addResults' in results and results['addResults'] is not None:
featSucces = 0
for result in results['addResults']:
if 'success' in result:
if result['success'] == False:
if 'error' in result:
errorCount = errorCount + 1
print ("\tError info: %s" % (result))
else:
featSucces = featSucces + 1
syncSoFar = syncSoFar + featSucces
print ("%s features added in this chunk" % (featSucces))
print ("%s/%s features added, %s errors" % (syncSoFar,total,errorCount ))
if 'addResults' in messages:
messages['addResults'] = messages['addResults'] + results['addResults']
else:
messages['addResults'] = results['addResults']
else:
messages['errors'] = result
return messages
else:
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
}
)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
def QueryAllFeatures(self,
url=None,
where="1=1",
out_fields="*",
timeFilter=None,
geometryFilter=None,
returnFeatureClass=False,
out_fc=None,
outSR=None,
chunksize=1000,
printIndent="",
useAGSFeatureService=False):
"""Performs an SQL query against a hosted feature service layer
and returns all features regardless of service limit.
Args:
url (str): The URL of the feature service layer.
where - the selection sql statement
out_fields - the attribute fields to return
timeFilter - a TimeFilter object where either the start time
or start and end time are defined to limit the
search results for a given time. The values in
the timeFilter should be as UTC timestampes in
milliseconds. No checking occurs to see if they
are in the right format.
geometryFilter - a GeometryFilter object to parse down a given
query by another spatial dataset.
returnFeatureClass - Default False. If true, query will be
returned as feature class
chunksize (int): The maximum amount of features to query at a time. Defaults to 1000.
out_fc - only valid if returnFeatureClass is set to True.
Output location of query.
useAGSFeatureService (boolean) - Whether to use an AGOL FeatureLayer
(default or AGS-hosted FeatureService.
Output:
A list of Feature Objects (default) or a path to the output featureclass if
returnFeatureClass is set to True.
"""
if (url is None):
return
fl = None
try:
if useAGSFeatureService:
fl = FeatureService(url=url,
securityHandler=self._securityHandler)
qRes = fl.query(where=where,
returnIdsOnly=True,
timeFilter=timeFilter,
geometryFilter=geometryFilter)
else:
fl = FeatureLayer(url=url,
securityHandler=self._securityHandler)
qRes = fl.query(where=where,
returnIDsOnly=True,
timeFilter=timeFilter,
geometryFilter=geometryFilter)
if 'error' in qRes:
if isinstance(qRes, dict):
qRes = str(qRes)
print(printIndent + qRes)
return []
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return fl.query(where=where,
returnGeometry=True,
out_fields=out_fields,
timeFilter=timeFilter,
geometryFilter=geometryFilter,
outSR=outSR)
print(printIndent + "%s features to be downloaded" % total)
chunksize = min(chunksize, fl.maxRecordCount)
combinedResults = None
totalQueried = 0
for chunk in chunklist(l=oids, n=chunksize):
oidsQuery = ",".join(map(str, chunk))
if not oidsQuery:
continue
else:
results = fl.query(objectIds=oidsQuery,
returnGeometry=True,
out_fields=out_fields,
timeFilter=timeFilter,
geometryFilter=geometryFilter,
outSR=outSR)
if isinstance(results, FeatureSet):
if combinedResults is None:
combinedResults = results
else:
for feature in results.features:
combinedResults.features.append(feature)
totalQueried += len(results.features)
print(printIndent +
"{:.0%} Completed: {}/{}".format(
totalQueried /
float(total), totalQueried, total))
else:
print(printIndent + results)
if returnFeatureClass == True:
return combinedResults.save(*os.path.split(out_fc))
else:
return combinedResults
else:
print(printIndent + qRes)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "QueryAllFeatures",
"line": line,
"filename": filename,
"synerror": synerror,
})
finally:
fl = None
del fl
gc.collect()
def AddFeaturesToFeatureLayer(self,
url,
pathToFeatureClass,
chunksize=0,
lowerCaseFieldNames=False):
"""Appends local features to a hosted feature service layer.
Args:
url (str): The URL of the feature service layer.
pathToFeatureClass (str): The path of the feature class on disk.
chunksize (int): The maximum amount of features to upload at a time. Defaults to 0.
lowerCaseFieldNames (bool): A boolean value indicating if field names should be converted
to lowercase before uploading. Defaults to ``False``.
Returns:
The result from :py:func:`arcrest.agol.services.FeatureLayer.addFeatures`.
Raises:
ArcRestHelperError: if ``arcpy`` can't be found.
Notes:
If publishing to a PostgreSQL database, it is suggested to to set ``lowerCaseFieldNames`` to ``True``.
"""
if arcpyFound == False:
raise common.ArcRestHelperError({
"function":
"AddFeaturesToFeatureLayer",
"line":
inspect.currentframe().f_back.f_lineno,
"filename":
'featureservicetools',
"synerror":
"ArcPy required for this function"
})
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
if chunksize > 0:
fc = os.path.basename(pathToFeatureClass)
inDesc = arcpy.Describe(pathToFeatureClass)
oidName = arcpy.AddFieldDelimiters(pathToFeatureClass,
inDesc.oidFieldName)
arr = arcpy.da.FeatureClassToNumPyArray(
pathToFeatureClass, (oidName))
syncSoFar = 0
messages = {'addResults': [], 'errors': []}
total = len(arr)
errorCount = 0
if total == '0':
print("0 features in %s" % pathToFeatureClass)
return "0 features in %s" % pathToFeatureClass
print("%s features in layer" % (total))
arcpy.env.overwriteOutput = True
if int(total) < int(chunksize):
return fl.addFeatures(
fc=pathToFeatureClass,
lowerCaseFieldNames=lowerCaseFieldNames)
else:
newArr = chunklist(arr, chunksize)
exprList = [
"{0} >= {1} AND {0} <= {2}".format(
oidName, nArr[0][0], nArr[len(nArr) - 1][0])
for nArr in newArr
]
for expr in exprList:
UploadLayer = arcpy.MakeFeatureLayer_management(
pathToFeatureClass, 'TEMPCOPY', expr).getOutput(0)
#print(arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0) + " features in the chunk")
results = fl.addFeatures(
fc=UploadLayer,
lowerCaseFieldNames=lowerCaseFieldNames)
chunkCount = arcpy.GetCount_management(
in_rows=UploadLayer).getOutput(0)
print(chunkCount + " features in the chunk")
if chunkCount > 0:
if results is not None and 'addResults' in results and results[
'addResults'] is not None:
featSucces = 0
for result in results['addResults']:
if 'success' in result:
if result['success'] == False:
if 'error' in result:
errorCount = errorCount + 1
print("\tError info: %s" %
(result))
else:
featSucces = featSucces + 1
syncSoFar = syncSoFar + featSucces
print("%s features added in this chunk" %
(featSucces))
print("%s/%s features added, %s errors" %
(syncSoFar, total, errorCount))
if 'addResults' in messages:
messages['addResults'] = messages[
'addResults'] + results['addResults']
else:
messages['addResults'] = results[
'addResults']
else:
messages['errors'] = result
return messages
else:
return fl.addFeatures(fc=pathToFeatureClass,
lowerCaseFieldNames=lowerCaseFieldNames)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
})
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
})
finally:
fl = None
del fl
gc.collect()
def QueryAllFeatures(self, url=None,
where="1=1",
out_fields="*",
timeFilter=None,
geometryFilter=None,
returnFeatureClass=False,
out_fc=None,
outSR=None,
chunksize=1000,
printIndent=""):
"""Performs an SQL query against a hosted feature service layer
and returns all features regardless of service limit.
Args:
url (str): The URL of the feature service layer.
where - the selection sql statement
out_fields - the attribute fields to return
timeFilter - a TimeFilter object where either the start time
or start and end time are defined to limit the
search results for a given time. The values in
the timeFilter should be as UTC timestampes in
milliseconds. No checking occurs to see if they
are in the right format.
geometryFilter - a GeometryFilter object to parse down a given
query by another spatial dataset.
returnFeatureClass - Default False. If true, query will be
returned as feature class
chunksize (int): The maximum amount of features to query at a time. Defaults to 1000.
out_fc - only valid if returnFeatureClass is set to True.
Output location of query.
Output:
A list of Feature Objects (default) or a path to the output featureclass if
returnFeatureClass is set to True.
"""
if (url is None):
return
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
qRes = fl.query(where=where,
returnIDsOnly=True,
timeFilter=timeFilter,
geometryFilter=geometryFilter)
if 'error' in qRes:
print (printIndent + qRes)
return []
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return fl.query(where=where,
returnGeometry=True,
out_fields=out_fields,
timeFilter=timeFilter,
geometryFilter=geometryFilter,
outSR=outSR)
print (printIndent + "%s features to be downloaded" % total)
chunksize = min(chunksize, fl.maxRecordCount)
combinedResults = None
totalQueried = 0
for chunk in chunklist(l=oids, n=chunksize):
oidsQuery = ",".join(map(str, chunk))
if not oidsQuery:
continue
else:
results = fl.query(objectIds=oidsQuery,
returnGeometry=True,
out_fields=out_fields,
timeFilter=timeFilter,
geometryFilter=geometryFilter,
outSR=outSR)
if isinstance(results,FeatureSet):
if combinedResults is None:
combinedResults = results
else:
for feature in results.features:
combinedResults.features.append(feature)
totalQueried += len(results.features)
print(printIndent + "{:.0%} Completed: {}/{}".format(totalQueried / float(total), totalQueried, total))
else:
print (printIndent + results)
if returnFeatureClass == True:
return combinedResults.save(*os.path.split(out_fc))
else:
return combinedResults
else:
print (printIndent + qRes)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "QueryAllFeatures",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
def AddFeaturesToFeatureLayer(self, url, pathToFeatureClass, chunksize=0, lowerCaseFieldNames=False):
"""Appends local features to a hosted feature service layer.
Args:
url (str): The URL of the feature service layer.
pathToFeatureClass (str): The path of the feature class on disk.
chunksize (int): The maximum amount of features to upload at a time. Defaults to 0.
lowerCaseFieldNames (bool): A boolean value indicating if field names should be converted
to lowercase before uploading. Defaults to ``False``.
Returns:
The result from :py:func:`arcrest.agol.services.FeatureLayer.addFeatures`.
Raises:
ArcRestHelperError: if ``arcpy`` can't be found.
Notes:
If publishing to a PostgreSQL database, it is suggested to to set ``lowerCaseFieldNames`` to ``True``.
"""
if arcpyFound == False:
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "ArcPy required for this function"
})
fl = None
try:
fl = FeatureLayer(
url=url,
securityHandler=self._securityHandler)
if chunksize > 0:
fc = os.path.basename(pathToFeatureClass)
inDesc = arcpy.Describe(pathToFeatureClass)
oidName = inDesc.oidFieldName
arr = arcpy.da.FeatureClassToNumPyArray(pathToFeatureClass, (oidName))
syncSoFar = 0
messages = {'addResults':[],'errors':[]}
total = len(arr)
errorCount = 0
if total == '0':
print ("0 features in %s" % pathToFeatureClass)
return "0 features in %s" % pathToFeatureClass
print ("%s features in layer" % (total))
arcpy.env.overwriteOutput = True
if int(total) < int(chunksize):
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
else:
newArr = chunklist(arr,chunksize)
exprList = ["{0} >= {1} AND {0} <= {2}".format(oidName, nArr[0][0], nArr[len(nArr)-1][0])
for nArr in newArr]
for expr in exprList:
UploadLayer = arcpy.MakeFeatureLayer_management(pathToFeatureClass, 'TEMPCOPY', expr).getOutput(0)
#print(arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0) + " features in the chunk")
results = fl.addFeatures(fc=UploadLayer,lowerCaseFieldNames=lowerCaseFieldNames)
chunkCount = arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0)
print(chunkCount + " features in the chunk")
if chunkCount > 0:
if results is not None and 'addResults' in results and results['addResults'] is not None:
featSucces = 0
for result in results['addResults']:
if 'success' in result:
if result['success'] == False:
if 'error' in result:
errorCount = errorCount + 1
print ("\tError info: %s" % (result))
else:
featSucces = featSucces + 1
syncSoFar = syncSoFar + featSucces
print ("%s features added in this chunk" % (featSucces))
print ("%s/%s features added, %s errors" % (syncSoFar,total,errorCount ))
if 'addResults' in messages:
messages['addResults'] = messages['addResults'] + results['addResults']
else:
messages['addResults'] = results['addResults']
else:
messages['errors'] = result
return messages
else:
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
}
)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
def QueryAllFeatures(self, url, sql, out_fields="*", chunksize=1000, savePath=None):
"""Performs an SQL query against a hosted feature service layer.
Args:
url (str): The URL of the feature service layer.
sql (str): The SQL query to apply against the feature service.
Those features that satisfy the query will be returned.
out_fields (str): A comma delimited list of field names to return.
Defaults to ``"*"``, i.e., return all fields
chunksize (int): The maximum amount of features to query at a time. Defaults to 1000.
savePath (str): The full path on disk where the features will be saved. Defaults to ``None``.
Returns:
When ``savePath`` is not provided (``None``), the result from
:py:func:`arcrest.agol.services.FeatureLayer.query`.
When ``savePath`` is provided, the result from
:py:func:`arcrest.common.general.FeatureSet.save`.
"""
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
qRes = fl.query(where=sql, returnIDsOnly=True)
if 'error' in qRes:
print (qRes)
return qRes
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return {'success':True, 'message':"No features matched the query"}
print ("%s features to be downloaded" % total)
chunksize = min(chunksize, fl.maxRecordCount)
combinedResults = None
totalQueried = 0
for chunk in chunklist(l=oids, n=chunksize):
oidsQuery = ",".join(map(str, chunk))
if not oidsQuery:
continue
else:
results = fl.query(objectIds=oidsQuery,
returnGeometry=True,
out_fields=out_fields)
if isinstance(results,FeatureSet):
if combinedResults is None:
combinedResults = results
else:
for feature in results.features:
combinedResults.features.append(feature)
totalQueried += len(results.features)
print("{:.0%} Completed: {}/{}".format(totalQueried / float(total), totalQueried, total))
else:
print (results)
if savePath is None or savePath == '':
return combinedResults
else:
return combinedResults.save(*os.path.split(savePath))
else:
print (qRes)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "QueryFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
def AddFeaturesToFeatureLayer(self,
url,
pathToFeatureClass,
chunksize=0,
lowerCaseFieldNames=False):
if arcpyFound == False:
raise common.ArcRestHelperError({
"function":
"AddFeaturesToFeatureLayer",
"line":
inspect.currentframe().f_back.f_lineno,
"filename":
'featureservicetools',
"synerror":
"ArcPy required for this function"
})
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
if chunksize > 0:
fc = os.path.basename(pathToFeatureClass)
inDesc = arcpy.Describe(pathToFeatureClass)
oidName = arcpy.AddFieldDelimiters(pathToFeatureClass,
inDesc.oidFieldName)
arr = arcpy.da.FeatureClassToNumPyArray(
pathToFeatureClass, (oidName))
syncSoFar = 0
messages = {'addResults': [], 'errors': []}
total = len(arr)
errorCount = 0
if total == '0':
print("0 features in %s" % pathToFeatureClass)
return "0 features in %s" % pathToFeatureClass
print("%s features in layer" % (total))
arcpy.env.overwriteOutput = True
if int(total) < int(chunksize):
return fl.addFeatures(
fc=pathToFeatureClass,
lowerCaseFieldNames=lowerCaseFieldNames)
else:
newArr = chunklist(arr, chunksize)
exprList = [
"{0} >= {1} AND {0} <= {2}".format(
oidName, nArr[0][0], nArr[len(nArr) - 1][0])
for nArr in newArr
]
for expr in exprList:
UploadLayer = arcpy.MakeFeatureLayer_management(
pathToFeatureClass, 'TEMPCOPY', expr).getOutput(0)
#print(arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0) + " features in the chunk")
results = fl.addFeatures(
fc=UploadLayer,
lowerCaseFieldNames=lowerCaseFieldNames)
chunkCount = arcpy.GetCount_management(
in_rows=UploadLayer).getOutput(0)
print(chunkCount + " features in the chunk")
if chunkCount > 0:
if results is not None and 'addResults' in results and results[
'addResults'] is not None:
featSucces = 0
for result in results['addResults']:
if 'success' in result:
if result['success'] == False:
if 'error' in result:
errorCount = errorCount + 1
print("\tError info: %s" %
(result))
else:
featSucces = featSucces + 1
syncSoFar = syncSoFar + featSucces
print("%s features added in this chunk" %
(featSucces))
print("%s/%s features added, %s errors" %
(syncSoFar, total, errorCount))
if 'addResults' in messages:
messages['addResults'] = messages[
'addResults'] + results['addResults']
else:
messages['addResults'] = results[
'addResults']
else:
messages['errors'] = result
return messages
else:
return fl.addFeatures(fc=pathToFeatureClass,
lowerCaseFieldNames=lowerCaseFieldNames)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
})
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
})
finally:
fl = None
del fl
gc.collect()