def main():
# sign into the active portal of the ArcGIS Pro Project this script is run from
gis = GIS('pro')
arcpy.AddMessage("Logged into {} as {}".format(arcpy.GetActivePortalURL(), gis.properties['user']['username']))
# get output file path as text
output_file_path = arcpy.GetParameterAsText(0)
arcpy.AddMessage("Scanning services in {}".format(arcpy.GetActivePortalURL()))
# creates list of all service items in active portal
services = (gis.content.search(query="", item_type="Map Service", max_items=10000) +
gis.content.search(query="", item_type="Feature Service", max_items=10000) +
gis.content.search(query="", item_type="Vector Tile Service", max_items=10000) +
gis.content.search(query="", item_type="Image Service", max_items=10000))
# initialise empty dictionary to store service information
service_info = {}
arcpy.AddMessage("Accessing service properties")
# iterate over services in service list and populate info into service_info dictionary
for service in services:
layers = []
try:
for layer in service.layers:
if layer.properties['type'] != 'Group Layer':
layers.append(layer.properties['name'])
except:
layers.append("No layers")
groups = []
if len(service.shared_with['groups']) != 0:
for group in service.shared_with['groups']:
groups.append(group.title)
else:
groups.append("Not shared to groups")
service_info[service.id] = {'service name': service.title,
'service type': service.type,
'service layers': ", ".join(layers),
'service url': service.url,
'sharing': service.access,
'shared groups': ", ".join(groups)}
arcpy.AddMessage("Writing csv with service information")
# writes a csv using the csv module and the information gathered iterating over portal services
with open(output_file_path, mode='w', newline="") as serviceReport:
fieldnames = ['service name', 'service type', 'service layers', 'service url', 'sharing', 'shared groups']
reportwriter = csv.DictWriter(serviceReport, fieldnames=fieldnames)
reportwriter.writeheader()
for k in service_info.keys():
reportwriter.writerow(service_info[k])
# opens file upon completion
os.startfile(output_file_path)
def create_and_append(feature_list=None,
token=None,
portal_url=None,
service_url=None,
matching=None):
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
layer = FeatureLayer(service_url)
features_to_append = []
for feature in feature_list:
new_feature = {'attributes': {}, 'geometry': feature['geometry']}
#Find fields and it's value
for field in matching:
new_feature['attributes'][field[0]] = [
x[1] for x in feature['attributes'].items() if x[0] == field[1]
][0]
features_to_append.append(new_feature.copy())
if len(features_to_append) > 500:
result = layer.edit_features(adds=features_to_append)
features_to_append = []
if features_to_append:
layer.edit_features(adds=features_to_append)
def app_search(services):
arcpy.AddMessage('Searching webapp configured searches in ' +
arcpy.GetActivePortalURL())
web_apps = gis.content.search(query='',
item_type='Web Mapping Application',
max_items=10000)
for app in web_apps:
for service in services:
try:
for widget in app.get_data()['widgetOnScreen']['widgets']:
if 'uri' in widget.keys(
) and widget['uri'] == 'widgets/Search/Widget':
for source in widget['config']['sources']:
if service.lower() in source['url'].lower(
) and 'searchFields' in source.keys():
arcpy.AddMessage(
"{} | {} | Searching on: {}".format(
app.title, source['url'],
", ".join(source['searchFields'])))
for widget in app.get_data()['widgetPool']['widgets']:
if 'uri' in widget.keys(
) and widget['uri'] == 'widgets/Search/Widget':
for source in widget['config']['sources']:
if service.lower() in source['url'].lower(
) and 'searchFields' in source.keys():
arcpy.AddMessage(
"{} | {} | Searching on: {}".format(
app.title, source['url'],
", ".join(source['searchFields'])))
except:
continue
def get_feature_service_list(
): #functions to get Feature layer list from portal
# Authentification on active portal using Python API (arcpy lib)
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
# Get content of the Portal and get url's of layers and tables
#content = gis.content
search_results = gis.content.search(query='',
item_type='Feature Service',
max_items=200)
layers_list = []
#Only layers not tables
for itm in search_results:
try:
layers_list.extend([x.url for x in itm.layers])
except TypeError:
pass
return layers_list
def __init__(self):
# discover Geoenrichment URL
portal = arcpy.GetActivePortalURL()
portal_self_url = portal + "/sharing/portals/self"
data = {"f": "json", "token": self.getToken()}
response = requests.post(portal_self_url, data)
response_json = response.json()
self.ge_url = response_json["helperServices"]["geoenrichment"]["url"]
def main():
# sign into the active portal of the ArcGIS Pro Project this script is run from
gis = GIS('pro')
arcpy.AddMessage("Logged into {} as {}".format(
arcpy.GetActivePortalURL(), gis.properties['user']['username']))
# get output file path as text, and optionally an item owner as text
output_file_path = arcpy.GetParameterAsText(0)
item_owner = arcpy.GetParameterAsText(1)
arcpy.AddMessage('Scanning items')
# create list of either all items in the active portal, or all of the items owned by one user in the active portal
if len(item_owner) > 0:
items = gis.content.search(query="owner: {}".format(item_owner),
max_items=10000)
else:
items = gis.content.search(query="".format(item_owner),
max_items=10000)
# initialise empty dictionary to store user data
item_metadata = {}
# iterate over items in item list and populate info into item_metadata dictionary
for item in items:
# filter out item types that aren't useful to report on
if item.type not in ('Code Attachment', 'Mobile Map Package',
'Layer Template', 'Mobile Application'):
item_metadata[item.id] = {
'title': item.title,
'type': item.type,
'id': item.id,
'summary': "",
'description': ""
}
# accesses item summary
if item['snippet'] is not None:
item_metadata[item.id]['summary'] = item['snippet']
# access item description
if item['description'] is not None:
item_metadata[item.id]['description'] = item['description']
arcpy.AddMessage('Writing csv')
# writes a csv using the csv module and the information gathered iterating over portal items
with open(output_file_path, mode='w', newline="",
encoding="utf-8") as item_report:
field_names = ['title', 'type', 'id', 'summary', 'description']
report_writer = csv.DictWriter(item_report, fieldnames=field_names)
report_writer.writeheader()
for k in item_metadata.keys():
report_writer.writerow(item_metadata[k])
# opens file upon completion
os.startfile(output_file_path)
def main():
# signs into the active portal of the ArcGIS Pro Project this script is run from
arcpy.AddMessage("Logged into {} as {}".format(
arcpy.GetActivePortalURL(), gis.properties['user']['username']))
# creates variable for user input services
services = arcpy.GetParameter(0)
# inputs services to web map search function
wm_search(services)
# inputs services to app search function
app_search(services)
def sign_into_portal(self):
portal_config = self.config['portal']
portal_url = arcpy.GetActivePortalURL()
if 'username' in portal_config and 'password' in portal_config:
try:
portal_info = arcpy.SignInToPortal(portal_url,
portal_config['username'],
portal_config['password'])
token = portal_info['token']
except (ValueError, KeyError):
return None
elif 'app_id' in portal_config and 'refresh_token' in portal_config:
try:
payload = {
'client_id': portal_config['app_id'],
'refresh_token': portal_config['refresh_token'],
'grant_type': 'refresh_token'
}
req = requests.post(portal_url + '/sharing/rest/oauth2/token',
data=payload,
verify=False)
req_json = req.json()
token = req_json['access_token']
except (ValueError, KeyError):
return None
elif 'app_id' in portal_config and 'app_secret' in portal_config:
try:
payload = {
'client_id': portal_config['app_id'],
'client_secret': portal_config['app_secret'],
'grant_type': 'client_credentials'
}
req = requests.post(portal_url + '/sharing/rest/oauth2/token',
data=payload,
verify=False)
req_json = req.json()
token = req_json['access_token']
except (ValueError, KeyError):
return None
else:
infos = arcpy.GetSigninToken()
if infos:
token = infos['token']
else:
return None
self.token = token
return self.token
def execute(self, parameters, messages):
"""The source code of the tool."""
arcpy.SetProgressor("default",
message="Accesing to a destinational resouse")
# Acessing outpud data
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
layer = FeatureLayer(parameters[0].valueAsText)
arcpy.SetProgressorLabel("Prepearing input data")
#Prepearing input data
feature_set = arcpy.FeatureSet(parameters[1].valueAsText)
feature_set_dict = json.loads(feature_set.JSON)
# Matching parameter
matching = parameters[2].value
# Split features by number of threads
list_of_lists = chunkIt(feature_set_dict['features'],
parameters[3].value)
# List of threads
threads = []
arcpy.SetProgressorLabel("Starting threads")
# Starting threads
for feature_list in list_of_lists:
threads.append(
Thread(target=create_and_append,
args=[
feature_list,
arcpy.GetSigninToken(), portal_url,
parameters[0].valueAsText, matching
]))
threads[-1].start()
# Joining all threads
arcpy.SetProgressorLabel("Executing appendence")
for thread in threads:
thread.join()
return
def wm_search(services):
arcpy.AddMessage('Searching' + ' webmaps in ' + arcpy.GetActivePortalURL())
web_maps = gis.content.search(query="",
item_type="Web Map",
max_items=10000)
for item in web_maps:
web_map = WebMap(item)
layers = web_map.layers
for layer in layers:
# loops through all input services
for service in services:
try:
if layer['layerType'] == "VectorTileLayer":
if service.lower() in layer.styleUrl.lower():
arcpy.AddMessage(
f"{item.title} | {layer.styleUrl}")
elif service.lower() in layer.url.lower():
arcpy.AddMessage(f"{item.title} | {layer.url}")
except:
continue
arcpy.AddMessage('Search Complete')
def __init__(self, portal_url=None, token=None, debug=False):
if portal_url is None:
self.portal_url = arcpy.GetActivePortalURL()
else:
self.portal_url = portal_url
# in the absence of information, default to HTTP
self.protocol = 'https'
self.is_arcgis_online = False
url_parts = self._parse_url(self.portal_url)
if url_parts:
if url_parts.scheme:
self.protocol = url_parts.scheme
self.host = self._normalize_host_url(url_parts)
if url_parts.netloc == 'www.arcgis.com':
self.is_arcgis_online = True
self.protocol = 'https'
else:
arcpy.AddError(NO_PORTAL_URL_MSG)
sys.exit()
self.base_url = '{}://{}/sharing/rest'.format(self.protocol, self.host)
self.secure_url = 'https://{}/sharing/rest'.format(self.host)
self.token = token
self.debug = debug
self.headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': ('ago.py -- ArcGIS portal module 0.1')
}
self.portal_name = None
self.portal_info = {}
self.username = None
self.login_method = None
self.expiration = None
self._password = None
def one_function_to_rule_them_all(common_info: CommonInfo,
specific_info: SpecificInfo):
'''Calls all the previous functions in appropriate order
Args:
common_info (CommonInfo): Info common to all layers (wfh and operator)
specific_info (SpecificInfo): Info specific to a particular layer (wfh or operator)
Raises:
NotImplementedError: If a method other than 'wfh' or 'operator' is provided
'''
print('Getting AGOL references...')
password = getpass('Enter Password: '******'Cleaning up scratch areas...')
if arcpy.Exists(str(common_info.scratch_gdb)):
print(f'Deleting existing {common_info.scratch_gdb}...')
arcpy.management.Delete(str(common_info.scratch_gdb))
print(f'Creating {common_info.scratch_gdb}...')
arcpy.management.CreateFileGDB(str(common_info.scratch_gdb.parent),
str(common_info.scratch_gdb.name))
dhrm_data = get_dhrm_dataframe(common_info.employee_data_path)
if specific_info.method == 'wfh':
get_wfh_eins(specific_info.data_source, dhrm_data,
common_info.csv_path)
elif specific_info.method == 'operator':
get_operator_eins(specific_info.data_source, dhrm_data,
common_info.csv_path)
else:
raise NotImplementedError(
f'Method {specific_info.method} not recognized...')
geocode_points(
str(common_info.csv_path),
str(common_info.geocoded_points_path),
str(common_info.locator_path),
'real_addr',
'real_zip',
)
hex_bin(str(common_info.geocoded_points_path),
str(common_info.hex_fc_path),
str(common_info.hexes_fc_path),
simple_count=specific_info.simple_summary,
within_table=str(common_info.within_table_path))
remove_single_count_hexes(str(common_info.hexes_fc_path),
str(common_info.trimmed_hex_fc_path))
sharing_layer, sharing_map = add_layer_to_map(
str(common_info.project_path), common_info.map_name,
str(common_info.trimmed_hex_fc_path))
update_agol_feature_service(sharing_map, sharing_layer, sd_item, fs_item,
specific_info)
arcpy.AddError("Must specify username, password, and org_url, missing at least one parameter.")
sys.exit(-1)
try:
arcpy.AddMessage("Getting Token...")
if args.username:
token = get_token(args.org_url, args.username, args.password)
org_url = args.org_url
else:
token_info = arcpy.GetSigninToken()
if token_info is None:
arcpy.AddError("You are not signed into a Portal or AGOL Organization.")
sys.exit(-1)
else:
token = token_info['token']
org_url = arcpy.GetActivePortalURL()
except Exception as e:
arcpy.AddError("Error Getting Token. Check username, password, and url.")
arcpy.AddError(e)
sys.exit(-1)
arcpy.AddMessage("Updating service definitions...")
if isinstance(args.item_ids,list):
for item_id in args.item_ids:
try:
errors = update_required_fields(item_id, org_url, token)
if not errors:
arcpy.AddMessage("Successfully updated required fields for item id: {}".format(item_id))
else:
arcpy.AddError("Failed to update service layers: {}".format(",".join(errors)))
except Exception as e:
def updateParameters(self, parameters):
"""Modify the values and properties of parameters before internal
validation is performed. This method is called whenever a parameter
has been changed."""
if not (parameters[0].altered and parameters[0].altered
and parameters[0].altered
): # Refresh params if tool is reopened but not reloaded
FeatureClassToFeatureLayerSyncClass.current_params = [
None, None, None
]
if parameters[
0].altered and FeatureClassToFeatureLayerSyncClass.current_params[
0] != parameters[0].valueAsText:
# Authentification on active portal using Python API (arcpy lib)
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
GIS(portal_url, token=token['token'])
# Get the layer and it's properties
layer = FeatureLayer(parameters[0].valueAsText)
properties = layer.properties
# Apply filter of feature type to input parameter
# Form a pairs os types for filter applying
list_of_types = {
'Point': "esriGeometryPoint",
'Multipoint': "esriGeometryMultipoint",
'Polyline': 'esriGeometryPolyline',
'Polygon': 'esriGeometryPolygon'
}
filter_type = [
x[0] for x in list_of_types.items()
if x[1] == properties.geometryType
]
# Define list filter
parameters[1].filter.list = filter_type
#Lists to populate fields to a matching table
system_fields = []
# System fields with info about changes + Global and Object ID. Populates automaticaly
try:
system_fields.extend(
[x[1] for x in properties.editFieldsInfo.items()])
except AttributeError:
pass
try:
system_fields.append(properties.objectIdField)
except AttributeError:
pass
try:
system_fields.append(properties.globalIdField)
except AttributeError:
pass
# Create a list of fields of utput feature class in table for creating matching schema
parameters[2].values = [[x.name, '']
for x in layer.properties.fields
if not x.name in system_fields]
# Add a filter to create a dropdown list
parameters[2].filters[0].list = [
x.name for x in layer.properties.fields
if not x.name in system_fields
]
#Add input fields to self for feaurute actions
FeatureClassToFeatureLayerSyncClass.out_fields_desc = layer.properties
#Show matching table ONLY if input and output params is set
if parameters[1].value and parameters[0].value:
parameters[2].enabled = True
else:
parameters[2].enabled = False
FeatureClassToFeatureLayerSyncClass.current_params[0] = parameters[
0].valueAsText
if parameters[
1].altered and FeatureClassToFeatureLayerSyncClass.current_params[
1] != parameters[1].valueAsText:
description = arcpy.Describe(parameters[1].valueAsText)
field_count = description.fieldInfo.count
field_list = []
for i in range(field_count):
field_list.append(description.fieldInfo.getFieldName(i))
parameters[2].filters[1].list = field_list
# Show matching table ONLY if input and output params is set
if parameters[1].value and parameters[0].value:
parameters[2].enabled = True
else:
parameters[2].enabled = False
# Save parameter value to choose state
FeatureClassToFeatureLayerSyncClass.current_params[1] = parameters[
1].valueAsText
# Save description for feaurute action
FeatureClassToFeatureLayerSyncClass.in_fields_desc = arcpy.ListFields(
parameters[1].valueAsText)
FeatureClassToFeatureLayerSyncClass.layer_desc = description
if parameters[
2].altered and FeatureClassToFeatureLayerSyncClass.current_params[
2] != parameters[2].valueAsText:
FeatureClassToFeatureLayerSyncClass.current_params[2] = parameters[
2].valueAsText
return
def main():
# logs into active portal in ArcGIS Pro
gis = GIS('pro')
arcpy.AddMessage("Logged into {} as {}".format(
arcpy.GetActivePortalURL(), gis.properties['user']['username']))
# creates list of items of all map image, feature, vector tile and image services (up to 10000 of each) in active portal
services = (
gis.content.search(query="", item_type="Map Service", max_items=10000)
+ gis.content.search(
query="", item_type="Feature Service", max_items=10000) +
gis.content.search(
query="", item_type="Vector Tile Service", max_items=10000) +
gis.content.search(
query="", item_type="Image Service", max_items=10000))
arcpy.AddMessage('Searching webmaps in {}'.format(
arcpy.GetActivePortalURL()))
# creates list of items of all webmaps in active portal
web_maps = gis.content.search(query="",
item_type="Web Map",
max_items=10000)
# loops through list of webmap items
for item in web_maps:
# creates a WebMap object from input webmap item
web_map = WebMap(item)
# accesses basemap layer(s) in WebMap object
basemaps = web_map.basemap['baseMapLayers']
# accesses layers in WebMap object
layers = web_map.layers
# loops through basemap layers
for bm in basemaps:
# tests whether the bm layer has a styleUrl(VTS) or url (everything else)
if 'styleUrl' in bm.keys():
for service in services:
if service.url in bm['styleUrl']:
services.remove(service)
elif 'url' in bm.keys():
for service in services:
if service.url in bm['url']:
services.remove(service)
# loops through layers
for layer in layers:
# tests whether the layer has a styleUrl(VTS) or url (everything else)
if hasattr(layer, 'styleUrl'):
for service in services:
if service.url in layer.styleUrl:
services.remove(service)
elif hasattr(layer, 'url'):
for service in services:
if service.url in layer.url:
services.remove(service)
arcpy.AddMessage(
'The following services are not used in any webmaps in {}'.format(
arcpy.GetActivePortalURL()))
# as we have removed all services being used in active portal, print list of remaining unused services
for service in services:
arcpy.AddMessage("{} | {}".format(
service.title,
arcpy.GetActivePortalURL() + r'home/item.html?id=' + service.id))
arcpy.AddMessage(
"There are a total of {} unused services in your portal".format(
str(len(services))))
def sharePackage2(in_package, folder, username, password, maintain, summary,
tags, credits, everyone, org, groups):
try:
active_url = arcpy.GetActivePortalURL()
except:
active_url = 'https://www.arcgis.com/'
agol_helper = ago.AGOLHelper(portal_url=active_url)
# If not app-signed in, and have user/pass, sign in the old manual way
if username and password and "Signed in through app" not in username:
agol_helper.login(username, password)
elif arcpy.GetSigninToken() is not None:
# Sign in using info from the app
agol_helper.token_login()
else:
arcpy.AddIDMessage("Error", 1561)
return
# Make sure file exists
try:
uploadSize = os.stat(in_package).st_size
except FileNotFoundError:
raise Exception("The file {0} was not found".format(in_package))
fileName, fileExt = os.path.splitext(os.path.basename(in_package))
try:
uploadType = pkgTypes[fileExt.upper()]
except KeyError:
raise Exception(
"Unknown/unsupported package type extension: {0}".format(fileExt))
portalFolders = agol_helper.list_folders()
if folder == "<root>" or None: folder = ''
folderID = ""
moveFolder = False
if folder:
if folder not in portalFolders.keys():
# Create a new folder
folderID = agol_helper.create_folder(folder)
arcpy.AddMessage("Created: {}".format(folderID))
#refresh the folder list
portalFolders = agol_helper.list_folders()
#previousPkgId = agol_helper.search(fileName, uploadType)
previousPkgId = agol_helper.search(item_type=uploadType, name=fileName)
if len(previousPkgId) == 0:
# Pkg does not exist
if maintain:
# No pkg + maintain meta == quit
raise Exception(
"Existing package not found. Check to make sure it exists or disable maintain metadata."
)
if folder:
if folder in portalFolders.keys():
folderID = portalFolders[folder]
moveFolder = True
else:
# Pkg exists
newItemID = previousPkgId[0]
itemInfo = agol_helper.item(newItemID)
# original pkg lives here.
pkgFolderID = itemInfo['ownerFolder'] if itemInfo['ownerFolder'] else ""
if folder:
if folder in portalFolders.keys():
if maintain and portalFolders[folder] != pkgFolderID:
raise Exception(
"Existing package to update not found in folder {}. Check the folder or disable maintain metadata."
.format(folder))
else:
# Existing pkg lives in supplied folder. It'll be updated.
folderID = portalFolders[folder]
if folderID != pkgFolderID:
# Package of same name exists but uploading to a different folder
moveFolder = True
# no else here - this is case where folder needs to be created, covered previously
else:
if maintain and pkgFolderID:
# pkg lives in folder, but root was specified:
raise Exception(
"Did not find package to update in <root> Does it exist in a folder?"
)
# no else here - covered previously with folderID variable initialize
# Set metadata by getting original metadata or adding new
if not maintain:
try:
# Only available in Pro 1.2 or 10.4
metaFromPkg = arcpy.GetPackageInfo(in_package)
description = metaFromPkg['description']
if not summary: summary = metaFromPkg['summary']
if not tags: tags = metaFromPkg['tags']
if not credits: credits = metaFromPkg['credits']
except AttributeError:
description = ''
pkgMetadata = (summary, description, tags, credits, '')
else:
metadataURL = "{}/content/users/{}/{}/items/{}".format(
agol_helper.base_url, agol_helper.username, folderID, newItemID)
metadata = agol_helper.url_request(metadataURL, {
'token': agol_helper.token,
'f': 'json'
})
#re-set everyone if necessary from original share options
everyone = True if metadata['sharing'][
'access'] == 'public' else everyone
org = True if everyone else True if metadata['sharing'][
'access'] == 'org' else org
groups = metadata['sharing']['groups'] if metadata['sharing'][
'groups'] else groups
snippet = metadata['item']['snippet'] if metadata['item'][
'snippet'] else ''
description = metadata['item']['description'] if metadata['item'][
'description'] else ''
tags = ','.join(metadata['item']['tags'])
accessInfo = metadata['item']['accessInformation'] if metadata['item'][
'accessInformation'] else ''
licenseInfo = metadata['item']['licenseInfo'] if metadata['item'][
'licenseInfo'] else ''
pkgMetadata = (snippet, description, tags, accessInfo, licenseInfo)
# Save original thumbnail to update with metadata
try:
thumbnailURL = "{}/content/items/{}/info/{}".format(
agol_helper.base_url, newItemID, metadata['item']['thumbnail'])
saveThumb = os.path.join(arcpy.env.scratchFolder, "thumbnail.png")
agol_helper.save_file(thumbnailURL, saveThumb)
pkgMetadata += (saveThumb, )
except:
arcpy.AddWarning("Problem getting thumbnail")
arcpy.AddMessage("Using existing metadata")
# Behavior is to always overwrite a package if it exists
extraParams = {'overwrite': 'true'}
# Upload the package
arcpy.AddMessage("Beginning file upload")
newItemIDres = agol_helper.add_item(in_package,
agol_helper.username,
folderID,
uploadType,
params=extraParams)
if 'success' in newItemIDres:
if newItemIDres['success']:
newItemID = newItemIDres['id']
else:
raise Exception("(returned msg) {}".format(newItemIDres))
# Commit the file
arcpy.AddMessage("Committing the file on the portal")
resCom = agol_helper.commit(newItemID, agol_helper.username)
status = 'processing' # partial | processing | failed | completed
while status == 'processing' or status == 'partial':
status = agol_helper.item_status(newItemID,
agol_helper.username)['status']
time.sleep(1)
if status == 'failed':
raise Exception("Failed in processing the file on the portal")
if moveFolder:
#move new package into folder
moveResp = agol_helper.move_items(folderID, [newItemID])
if not moveResp['results'][0]['success']:
arcpy.AddMessage(
"Failed to move item to folder: '{}'. Item will be created in root"
.format(folder))
folderID = ""
# Set or Update the metadata
arcpy.AddMessage("Setting metadata and sharing settings")
uresp = agol_helper.update_item(newItemID,
pkgMetadata,
folder_id=folderID,
title=fileName)
try:
if not uresp['success']:
arcpy.AddWarning("Could not set sharing properties")
except:
arcpy.AddWarning("Problem setting metadata values:")
arcpy.AddError(" {0}".format(uresp['error']))
# Clean up thumbnail
try:
os.remove(saveThumb)
except (NameError, IOError):
pass
# Set Sharing options
if not maintain:
if everyone or groups or org:
groupIDs = []
if groups:
userGroups = agol_helper.list_groups(agol_helper.username)
for group in userGroups.keys():
arcpy.AddMessage(group)
for selectedgroup in groups:
if group == selectedgroup:
groupIDs.append(userGroups[group])
gresp = agol_helper.share_items(groupIDs, everyone, org,
[newItemID])
try:
if not gresp['results'][0]['success']:
arcpy.AddWarning("Could not set sharing properties")
arcpy.AddError(" {0}".format(
gresp['results'][0]['error']['message']))
except:
arcpy.AddWarning("Problem sharing item:")
arcpy.AddError(" {0}".format(gresp))
def setup(self):
"""
Sets up the Auditor by logging into the ArcGIS org, getting all the items and folders, and reading in the
metatable(s). To be called in __init__().
In case of multiple calls (ie, for retry()), all data are re-instantiated/initialized.
"""
#: temp_dir used by fixes.metadata_fix() to hold xml of sde metadata
temp_dir = Path(arcpy.env.scratchFolder, 'auditor')
if not temp_dir.exists():
if self.verbose:
print(f'Creating temp directory {temp_dir}...')
temp_dir.mkdir()
self.log.info(f'Logging into {credentials.ORG} as {credentials.USERNAME}')
self.gis = arcgis.gis.GIS(credentials.ORG, credentials.USERNAME, credentials.PASSWORD)
#: Make sure ArcGIS Pro is properly logged in
arcpy.SignInToPortal(arcpy.GetActivePortalURL(), credentials.USERNAME, credentials.PASSWORD)
user_item = self.gis.users.me # pylint: disable=no-member
#: Build dict of folders. 'None' gives us the root folder.
if self.verbose:
print(f'Getting {self.username}\'s folders...')
folders = {None: None}
for folder in user_item.folders:
folders[folder['id']] = folder['title']
self.items_to_check = [] #: Clear this out again in case retry calls setup() multiple times.
#: Get item object and it's correspond folder for each relevant item
if self.verbose:
print('Getting item objects...')
#: User-provided list
if self.item_ids:
for item_id in self.item_ids:
item = self.gis.content.get(item_id) # pylint: disable=no-member
if not item:
raise ValueError(f'Item {item_id} not found')
self.items_to_check.append(item)
try:
self.itemid_and_folder[item.itemid] = folders[item.ownerFolder]
except KeyError:
raise ValueError(f'Folder id {item.ownerFolder} not found (wrong user?)')
#: No user-provided item ids, get all hosted feature services in every folder
else:
for _, name in folders.items():
for item in user_item.items(name, 1000):
if item.type == 'Feature Service':
self.items_to_check.append(item)
self.itemid_and_folder[item.itemid] = name
#: Read the metatable into memory as a dictionary based on itemid.
#: Getting this once so we don't have to re-read every iteration
if self.verbose:
print('Getting metatable info...')
self.metatable = Metatable()
sgid_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'Authoritative']
agol_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'CATEGORY']
self.metatable.read_metatable(self.sgid_table, sgid_fields)
self.metatable.read_metatable(self.agol_table, agol_fields)
if self.metatable.duplicate_keys:
raise RuntimeError(f'Duplicate AGOL item IDs found in metatables: {self.metatable.duplicate_keys}')
#: Get the groups
if self.verbose:
print('Getting groups...')
groups = self.gis.groups.search('title:*') # pylint: disable=no-member
self.groups_dict = {g.title: g.id for g in groups}
#%%
df.to_csv(custAcctFile,header=False, index=True)
#%% [markdown]
# [```gis.features.SpatialDataFrame()```](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.features.toc.html?highlight=spatialdataframe#arcgis.features.SpatialDataFrame.from_xy)
#%%
from arcgis.features import SpatialDataFrame
from arcgis.gis import GIS
from getpass import getpass
from IPython.display import display
#%%
sdf = SpatialDataFrame.from_xy(df,"POINT_X","POINT_Y")
gis = GIS(arcpy.GetActivePortalURL(), username=input("Enter User Name "), password=(getpass()))
#gis = GIS()
#portalDesc = arcpy.GetPortalDescription()
# search and list all items owned by connected user
#query=f'owner:{portalDesc["user"]["username"]} AND title:CW BaseMap'
#itemType="Feature Layer"
#sortField="title"
#sortOrder="asc"
# default max__items is 10
#maxItems=100
#m = gis.content.search(query,itemType,sortField,sortOrder,maxItems)
#%%
consumptionLyr = gis.content.import_data(sdf)
print(f'Using {aprx}')
arcpy.env.workspace = prjPath
arcpy.env.overwriteOutput = True
#%% [markdown]
# Now we [get the active portal](http://pro.arcgis.com/en/pro-app/arcpy/functions/getactiveportalurl.htm) and instantiate the [GIS](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.gis.toc.html#gis). The [GetPortalDescription()](http://pro.arcgis.com/en/pro-app/arcpy/functions/getportaldescription.htm) returns a dictionary containing portal information.
#%% [markdown]
# TODO switching active portal fails
#%%
response = input(
f'Do you want to use the active portal? {(arcpy.GetActivePortalURL())} [y or n]'
)
if response.upper() in "YES":
username = input("Enter User Name")
gis = GIS(arcpy.GetActivePortalURL(), username, password=(getpass()))
else:
portalURL = input("Enter Portal Address")
username = input("Enter User Name")
gis = GIS(portalURL, username, password=getpass())
portal_desc = arcpy.GetPortalDescription()
print(
f'Portal Name - {portal_desc["portalName"]}\nConnected to {portal_desc["name"]} as user {username}'
)
#%% [markdown]
# Using [gis.content.search()](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.gis.toc.html#arcgis.gis.ContentManager.search) we are going to find all Groups and [item_types](https://developers.arcgis.com/rest/users-groups-and-items/items-and-item-types.htm) (that are Feature Layers) owned by connected user.
#%%
# search for all Groups owned by connected user
sourceGroups = gis.groups.search(query=f'owner:{username}')
def process(self):
#: Set up paths and directories
feature_service_name = secrets.FEATURE_SERVICE_NAME
temp_csv_dir = os.path.join(arcpy.env.scratchFolder, 'fleet')
temp_fc_path = os.path.join(arcpy.env.scratchGDB, feature_service_name)
sddraft_path = os.path.join(arcpy.env.scratchFolder,
f'{feature_service_name}.sddraft')
sd_path = sddraft_path[:-5]
paths = [temp_csv_dir, temp_fc_path, sddraft_path, sd_path]
for item in paths:
if arcpy.Exists(item):
self.log.info(f'Deleting {item} prior to use...')
arcpy.Delete_management(item)
os.mkdir(temp_csv_dir)
if not secrets.KNOWNHOSTS or not os.path.isfile(secrets.KNOWNHOSTS):
raise FileNotFoundError(
f'known_hosts file {secrets.KNOWNHOSTS} not found. Please create with ssh-keyscan.'
)
#: Download all the files in the upload folder on sftp to temp_csv_dir
self.log.info(
f'Downloading all files from {secrets.SFTP_HOST}/upload...')
connection_opts = pysftp.CnOpts(knownhosts=secrets.KNOWNHOSTS)
with pysftp.Connection(secrets.SFTP_HOST,
username=secrets.SFTP_USERNAME,
password=secrets.SFTP_PASSWORD,
cnopts=connection_opts) as sftp:
sftp.get_d('upload', temp_csv_dir, preserve_mtime=True)
#: Get the latest file
source_path, source_date = self.get_latest_csv(temp_csv_dir,
previous_days=7)
self.log.info(
f'Converting {source_path} to feature class {temp_fc_path}...')
wgs84 = arcpy.SpatialReference(4326)
result = arcpy.management.XYTableToPoint(source_path,
temp_fc_path,
'LONGITUDE',
'LATITUDE',
coordinate_system=wgs84)
self.log.debug(result.getMessages())
try_count = 1
while True:
try:
self.log.info(f'Updating service, try {try_count} of 3...')
self.log.info(
f'Connecting to AGOL as {secrets.AGOL_USERNAME}...')
gis = arcgis.gis.GIS('https://www.arcgis.com',
secrets.AGOL_USERNAME,
secrets.AGOL_PASSWORD)
sd_item = gis.content.get(secrets.SD_ITEM_ID)
self.log.info('Getting map and layer...')
arcpy.SignInToPortal(arcpy.GetActivePortalURL(),
secrets.AGOL_USERNAME,
secrets.AGOL_PASSWORD)
layer, fleet_map = self.get_map_layer(secrets.PROJECT_PATH,
temp_fc_path)
#: draft, stage, update, publish
self.log.info('Staging and updating...')
self.update_agol_feature_service(fleet_map, layer,
feature_service_name,
sddraft_path, sd_path,
sd_item)
#: Update item description
self.log.info('Updating item description...')
feature_item = gis.content.get(secrets.FEATURES_ITEM_ID)
year = source_date[:4]
month = source_date[4:6]
day = source_date[6:]
description = f'Vehicle location data obtained from Fleet; updated on {year}-{month}-{day}'
feature_item.update(
item_properties={'description': description})
except Exception as e:
err_msg = f'Error on attempt {try_count} of 3; retrying.'
self.log.exception(err_msg)
#: Fail for good if 3x retry fails, otherwise increment, sleep,
#: and retry
if try_count > 3:
err_msg = 'Connection errors; giving up after 3 retries'
self.log.exception(err_msg)
raise e
sleep(try_count**2)
try_count += 1
continue
#: If we haven't gotten an error, break out of while True.
break
analysis = arcpy.mapping.AnalyzeForSD(newSDdraft)
if analysis['errors'] == {}:
if os.path.exists(sd):
os.remove(sd)
print("{0} already exists! Deleted! ".format(sd))
# Stage the service
arcpy.StageService_server(newSDdraft, sd)
# Upload the service.
arcpy.UploadServiceDefinition_server(in_sd_file=sd,
in_server="My Hosted Services")
print "Uploaded and overwrote service"
else:
# If the sddraft analysis contained errors, display them and quit.
print analysis['errors']
##### Main Script
if os.path.isdir(workSpace) == False:
print "Not valid path..."
else:
arcpy.SignInToPortal_server(portaluser, psd, "")
print(arcpy.GetActivePortalURL())
files = os.listdir(workSpace)
for f in files:
if f.endswith(".mxd"):
# mxdPath = os.path.join(workSpace, f)
print("publishing: " + f)
publishingHostedFeatureService(workSpace, f)
print(f + " Publishing Done !")
else:
continue
