def main(): # sign into the active portal of the ArcGIS Pro Project this script is run from gis = GIS('pro') arcpy.AddMessage("Logged into {} as {}".format(arcpy.GetActivePortalURL(), gis.properties['user']['username'])) # get output file path as text output_file_path = arcpy.GetParameterAsText(0) arcpy.AddMessage("Scanning services in {}".format(arcpy.GetActivePortalURL())) # creates list of all service items in active portal services = (gis.content.search(query="", item_type="Map Service", max_items=10000) + gis.content.search(query="", item_type="Feature Service", max_items=10000) + gis.content.search(query="", item_type="Vector Tile Service", max_items=10000) + gis.content.search(query="", item_type="Image Service", max_items=10000)) # initialise empty dictionary to store service information service_info = {} arcpy.AddMessage("Accessing service properties") # iterate over services in service list and populate info into service_info dictionary for service in services: layers = [] try: for layer in service.layers: if layer.properties['type'] != 'Group Layer': layers.append(layer.properties['name']) except: layers.append("No layers") groups = [] if len(service.shared_with['groups']) != 0: for group in service.shared_with['groups']: groups.append(group.title) else: groups.append("Not shared to groups") service_info[service.id] = {'service name': service.title, 'service type': service.type, 'service layers': ", ".join(layers), 'service url': service.url, 'sharing': service.access, 'shared groups': ", ".join(groups)} arcpy.AddMessage("Writing csv with service information") # writes a csv using the csv module and the information gathered iterating over portal services with open(output_file_path, mode='w', newline="") as serviceReport: fieldnames = ['service name', 'service type', 'service layers', 'service url', 'sharing', 'shared groups'] reportwriter = csv.DictWriter(serviceReport, fieldnames=fieldnames) reportwriter.writeheader() for k in service_info.keys(): reportwriter.writerow(service_info[k]) # opens file upon completion os.startfile(output_file_path)
def create_and_append(feature_list=None, token=None, portal_url=None, service_url=None, matching=None): token = arcpy.GetSigninToken() portal_url = arcpy.GetActivePortalURL() gis = GIS(portal_url, token=token['token']) layer = FeatureLayer(service_url) features_to_append = [] for feature in feature_list: new_feature = {'attributes': {}, 'geometry': feature['geometry']} #Find fields and it's value for field in matching: new_feature['attributes'][field[0]] = [ x[1] for x in feature['attributes'].items() if x[0] == field[1] ][0] features_to_append.append(new_feature.copy()) if len(features_to_append) > 500: result = layer.edit_features(adds=features_to_append) features_to_append = [] if features_to_append: layer.edit_features(adds=features_to_append)
def app_search(services): arcpy.AddMessage('Searching webapp configured searches in ' + arcpy.GetActivePortalURL()) web_apps = gis.content.search(query='', item_type='Web Mapping Application', max_items=10000) for app in web_apps: for service in services: try: for widget in app.get_data()['widgetOnScreen']['widgets']: if 'uri' in widget.keys( ) and widget['uri'] == 'widgets/Search/Widget': for source in widget['config']['sources']: if service.lower() in source['url'].lower( ) and 'searchFields' in source.keys(): arcpy.AddMessage( "{} | {} | Searching on: {}".format( app.title, source['url'], ", ".join(source['searchFields']))) for widget in app.get_data()['widgetPool']['widgets']: if 'uri' in widget.keys( ) and widget['uri'] == 'widgets/Search/Widget': for source in widget['config']['sources']: if service.lower() in source['url'].lower( ) and 'searchFields' in source.keys(): arcpy.AddMessage( "{} | {} | Searching on: {}".format( app.title, source['url'], ", ".join(source['searchFields']))) except: continue
def get_feature_service_list( ): #functions to get Feature layer list from portal # Authentification on active portal using Python API (arcpy lib) token = arcpy.GetSigninToken() portal_url = arcpy.GetActivePortalURL() gis = GIS(portal_url, token=token['token']) # Get content of the Portal and get url's of layers and tables #content = gis.content search_results = gis.content.search(query='', item_type='Feature Service', max_items=200) layers_list = [] #Only layers not tables for itm in search_results: try: layers_list.extend([x.url for x in itm.layers]) except TypeError: pass return layers_list
def __init__(self): # discover Geoenrichment URL portal = arcpy.GetActivePortalURL() portal_self_url = portal + "/sharing/portals/self" data = {"f": "json", "token": self.getToken()} response = requests.post(portal_self_url, data) response_json = response.json() self.ge_url = response_json["helperServices"]["geoenrichment"]["url"]
def main(): # sign into the active portal of the ArcGIS Pro Project this script is run from gis = GIS('pro') arcpy.AddMessage("Logged into {} as {}".format( arcpy.GetActivePortalURL(), gis.properties['user']['username'])) # get output file path as text, and optionally an item owner as text output_file_path = arcpy.GetParameterAsText(0) item_owner = arcpy.GetParameterAsText(1) arcpy.AddMessage('Scanning items') # create list of either all items in the active portal, or all of the items owned by one user in the active portal if len(item_owner) > 0: items = gis.content.search(query="owner: {}".format(item_owner), max_items=10000) else: items = gis.content.search(query="".format(item_owner), max_items=10000) # initialise empty dictionary to store user data item_metadata = {} # iterate over items in item list and populate info into item_metadata dictionary for item in items: # filter out item types that aren't useful to report on if item.type not in ('Code Attachment', 'Mobile Map Package', 'Layer Template', 'Mobile Application'): item_metadata[item.id] = { 'title': item.title, 'type': item.type, 'id': item.id, 'summary': "", 'description': "" } # accesses item summary if item['snippet'] is not None: item_metadata[item.id]['summary'] = item['snippet'] # access item description if item['description'] is not None: item_metadata[item.id]['description'] = item['description'] arcpy.AddMessage('Writing csv') # writes a csv using the csv module and the information gathered iterating over portal items with open(output_file_path, mode='w', newline="", encoding="utf-8") as item_report: field_names = ['title', 'type', 'id', 'summary', 'description'] report_writer = csv.DictWriter(item_report, fieldnames=field_names) report_writer.writeheader() for k in item_metadata.keys(): report_writer.writerow(item_metadata[k]) # opens file upon completion os.startfile(output_file_path)
def main(): # signs into the active portal of the ArcGIS Pro Project this script is run from arcpy.AddMessage("Logged into {} as {}".format( arcpy.GetActivePortalURL(), gis.properties['user']['username'])) # creates variable for user input services services = arcpy.GetParameter(0) # inputs services to web map search function wm_search(services) # inputs services to app search function app_search(services)
def sign_into_portal(self): portal_config = self.config['portal'] portal_url = arcpy.GetActivePortalURL() if 'username' in portal_config and 'password' in portal_config: try: portal_info = arcpy.SignInToPortal(portal_url, portal_config['username'], portal_config['password']) token = portal_info['token'] except (ValueError, KeyError): return None elif 'app_id' in portal_config and 'refresh_token' in portal_config: try: payload = { 'client_id': portal_config['app_id'], 'refresh_token': portal_config['refresh_token'], 'grant_type': 'refresh_token' } req = requests.post(portal_url + '/sharing/rest/oauth2/token', data=payload, verify=False) req_json = req.json() token = req_json['access_token'] except (ValueError, KeyError): return None elif 'app_id' in portal_config and 'app_secret' in portal_config: try: payload = { 'client_id': portal_config['app_id'], 'client_secret': portal_config['app_secret'], 'grant_type': 'client_credentials' } req = requests.post(portal_url + '/sharing/rest/oauth2/token', data=payload, verify=False) req_json = req.json() token = req_json['access_token'] except (ValueError, KeyError): return None else: infos = arcpy.GetSigninToken() if infos: token = infos['token'] else: return None self.token = token return self.token
def execute(self, parameters, messages): """The source code of the tool.""" arcpy.SetProgressor("default", message="Accesing to a destinational resouse") # Acessing outpud data token = arcpy.GetSigninToken() portal_url = arcpy.GetActivePortalURL() gis = GIS(portal_url, token=token['token']) layer = FeatureLayer(parameters[0].valueAsText) arcpy.SetProgressorLabel("Prepearing input data") #Prepearing input data feature_set = arcpy.FeatureSet(parameters[1].valueAsText) feature_set_dict = json.loads(feature_set.JSON) # Matching parameter matching = parameters[2].value # Split features by number of threads list_of_lists = chunkIt(feature_set_dict['features'], parameters[3].value) # List of threads threads = [] arcpy.SetProgressorLabel("Starting threads") # Starting threads for feature_list in list_of_lists: threads.append( Thread(target=create_and_append, args=[ feature_list, arcpy.GetSigninToken(), portal_url, parameters[0].valueAsText, matching ])) threads[-1].start() # Joining all threads arcpy.SetProgressorLabel("Executing appendence") for thread in threads: thread.join() return
def wm_search(services): arcpy.AddMessage('Searching' + ' webmaps in ' + arcpy.GetActivePortalURL()) web_maps = gis.content.search(query="", item_type="Web Map", max_items=10000) for item in web_maps: web_map = WebMap(item) layers = web_map.layers for layer in layers: # loops through all input services for service in services: try: if layer['layerType'] == "VectorTileLayer": if service.lower() in layer.styleUrl.lower(): arcpy.AddMessage( f"{item.title} | {layer.styleUrl}") elif service.lower() in layer.url.lower(): arcpy.AddMessage(f"{item.title} | {layer.url}") except: continue arcpy.AddMessage('Search Complete')
def __init__(self, portal_url=None, token=None, debug=False): if portal_url is None: self.portal_url = arcpy.GetActivePortalURL() else: self.portal_url = portal_url # in the absence of information, default to HTTP self.protocol = 'https' self.is_arcgis_online = False url_parts = self._parse_url(self.portal_url) if url_parts: if url_parts.scheme: self.protocol = url_parts.scheme self.host = self._normalize_host_url(url_parts) if url_parts.netloc == 'www.arcgis.com': self.is_arcgis_online = True self.protocol = 'https' else: arcpy.AddError(NO_PORTAL_URL_MSG) sys.exit() self.base_url = '{}://{}/sharing/rest'.format(self.protocol, self.host) self.secure_url = 'https://{}/sharing/rest'.format(self.host) self.token = token self.debug = debug self.headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'User-Agent': ('ago.py -- ArcGIS portal module 0.1') } self.portal_name = None self.portal_info = {} self.username = None self.login_method = None self.expiration = None self._password = None
def one_function_to_rule_them_all(common_info: CommonInfo, specific_info: SpecificInfo): '''Calls all the previous functions in appropriate order Args: common_info (CommonInfo): Info common to all layers (wfh and operator) specific_info (SpecificInfo): Info specific to a particular layer (wfh or operator) Raises: NotImplementedError: If a method other than 'wfh' or 'operator' is provided ''' print('Getting AGOL references...') password = getpass('Enter Password: '******'Cleaning up scratch areas...') if arcpy.Exists(str(common_info.scratch_gdb)): print(f'Deleting existing {common_info.scratch_gdb}...') arcpy.management.Delete(str(common_info.scratch_gdb)) print(f'Creating {common_info.scratch_gdb}...') arcpy.management.CreateFileGDB(str(common_info.scratch_gdb.parent), str(common_info.scratch_gdb.name)) dhrm_data = get_dhrm_dataframe(common_info.employee_data_path) if specific_info.method == 'wfh': get_wfh_eins(specific_info.data_source, dhrm_data, common_info.csv_path) elif specific_info.method == 'operator': get_operator_eins(specific_info.data_source, dhrm_data, common_info.csv_path) else: raise NotImplementedError( f'Method {specific_info.method} not recognized...') geocode_points( str(common_info.csv_path), str(common_info.geocoded_points_path), str(common_info.locator_path), 'real_addr', 'real_zip', ) hex_bin(str(common_info.geocoded_points_path), str(common_info.hex_fc_path), str(common_info.hexes_fc_path), simple_count=specific_info.simple_summary, within_table=str(common_info.within_table_path)) remove_single_count_hexes(str(common_info.hexes_fc_path), str(common_info.trimmed_hex_fc_path)) sharing_layer, sharing_map = add_layer_to_map( str(common_info.project_path), common_info.map_name, str(common_info.trimmed_hex_fc_path)) update_agol_feature_service(sharing_map, sharing_layer, sd_item, fs_item, specific_info)
arcpy.AddError("Must specify username, password, and org_url, missing at least one parameter.") sys.exit(-1) try: arcpy.AddMessage("Getting Token...") if args.username: token = get_token(args.org_url, args.username, args.password) org_url = args.org_url else: token_info = arcpy.GetSigninToken() if token_info is None: arcpy.AddError("You are not signed into a Portal or AGOL Organization.") sys.exit(-1) else: token = token_info['token'] org_url = arcpy.GetActivePortalURL() except Exception as e: arcpy.AddError("Error Getting Token. Check username, password, and url.") arcpy.AddError(e) sys.exit(-1) arcpy.AddMessage("Updating service definitions...") if isinstance(args.item_ids,list): for item_id in args.item_ids: try: errors = update_required_fields(item_id, org_url, token) if not errors: arcpy.AddMessage("Successfully updated required fields for item id: {}".format(item_id)) else: arcpy.AddError("Failed to update service layers: {}".format(",".join(errors))) except Exception as e:
def updateParameters(self, parameters): """Modify the values and properties of parameters before internal validation is performed. This method is called whenever a parameter has been changed.""" if not (parameters[0].altered and parameters[0].altered and parameters[0].altered ): # Refresh params if tool is reopened but not reloaded FeatureClassToFeatureLayerSyncClass.current_params = [ None, None, None ] if parameters[ 0].altered and FeatureClassToFeatureLayerSyncClass.current_params[ 0] != parameters[0].valueAsText: # Authentification on active portal using Python API (arcpy lib) token = arcpy.GetSigninToken() portal_url = arcpy.GetActivePortalURL() GIS(portal_url, token=token['token']) # Get the layer and it's properties layer = FeatureLayer(parameters[0].valueAsText) properties = layer.properties # Apply filter of feature type to input parameter # Form a pairs os types for filter applying list_of_types = { 'Point': "esriGeometryPoint", 'Multipoint': "esriGeometryMultipoint", 'Polyline': 'esriGeometryPolyline', 'Polygon': 'esriGeometryPolygon' } filter_type = [ x[0] for x in list_of_types.items() if x[1] == properties.geometryType ] # Define list filter parameters[1].filter.list = filter_type #Lists to populate fields to a matching table system_fields = [] # System fields with info about changes + Global and Object ID. Populates automaticaly try: system_fields.extend( [x[1] for x in properties.editFieldsInfo.items()]) except AttributeError: pass try: system_fields.append(properties.objectIdField) except AttributeError: pass try: system_fields.append(properties.globalIdField) except AttributeError: pass # Create a list of fields of utput feature class in table for creating matching schema parameters[2].values = [[x.name, ''] for x in layer.properties.fields if not x.name in system_fields] # Add a filter to create a dropdown list parameters[2].filters[0].list = [ x.name for x in layer.properties.fields if not x.name in system_fields ] #Add input fields to self for feaurute actions FeatureClassToFeatureLayerSyncClass.out_fields_desc = layer.properties #Show matching table ONLY if input and output params is set if parameters[1].value and parameters[0].value: parameters[2].enabled = True else: parameters[2].enabled = False FeatureClassToFeatureLayerSyncClass.current_params[0] = parameters[ 0].valueAsText if parameters[ 1].altered and FeatureClassToFeatureLayerSyncClass.current_params[ 1] != parameters[1].valueAsText: description = arcpy.Describe(parameters[1].valueAsText) field_count = description.fieldInfo.count field_list = [] for i in range(field_count): field_list.append(description.fieldInfo.getFieldName(i)) parameters[2].filters[1].list = field_list # Show matching table ONLY if input and output params is set if parameters[1].value and parameters[0].value: parameters[2].enabled = True else: parameters[2].enabled = False # Save parameter value to choose state FeatureClassToFeatureLayerSyncClass.current_params[1] = parameters[ 1].valueAsText # Save description for feaurute action FeatureClassToFeatureLayerSyncClass.in_fields_desc = arcpy.ListFields( parameters[1].valueAsText) FeatureClassToFeatureLayerSyncClass.layer_desc = description if parameters[ 2].altered and FeatureClassToFeatureLayerSyncClass.current_params[ 2] != parameters[2].valueAsText: FeatureClassToFeatureLayerSyncClass.current_params[2] = parameters[ 2].valueAsText return
def main(): # logs into active portal in ArcGIS Pro gis = GIS('pro') arcpy.AddMessage("Logged into {} as {}".format( arcpy.GetActivePortalURL(), gis.properties['user']['username'])) # creates list of items of all map image, feature, vector tile and image services (up to 10000 of each) in active portal services = ( gis.content.search(query="", item_type="Map Service", max_items=10000) + gis.content.search( query="", item_type="Feature Service", max_items=10000) + gis.content.search( query="", item_type="Vector Tile Service", max_items=10000) + gis.content.search( query="", item_type="Image Service", max_items=10000)) arcpy.AddMessage('Searching webmaps in {}'.format( arcpy.GetActivePortalURL())) # creates list of items of all webmaps in active portal web_maps = gis.content.search(query="", item_type="Web Map", max_items=10000) # loops through list of webmap items for item in web_maps: # creates a WebMap object from input webmap item web_map = WebMap(item) # accesses basemap layer(s) in WebMap object basemaps = web_map.basemap['baseMapLayers'] # accesses layers in WebMap object layers = web_map.layers # loops through basemap layers for bm in basemaps: # tests whether the bm layer has a styleUrl(VTS) or url (everything else) if 'styleUrl' in bm.keys(): for service in services: if service.url in bm['styleUrl']: services.remove(service) elif 'url' in bm.keys(): for service in services: if service.url in bm['url']: services.remove(service) # loops through layers for layer in layers: # tests whether the layer has a styleUrl(VTS) or url (everything else) if hasattr(layer, 'styleUrl'): for service in services: if service.url in layer.styleUrl: services.remove(service) elif hasattr(layer, 'url'): for service in services: if service.url in layer.url: services.remove(service) arcpy.AddMessage( 'The following services are not used in any webmaps in {}'.format( arcpy.GetActivePortalURL())) # as we have removed all services being used in active portal, print list of remaining unused services for service in services: arcpy.AddMessage("{} | {}".format( service.title, arcpy.GetActivePortalURL() + r'home/item.html?id=' + service.id)) arcpy.AddMessage( "There are a total of {} unused services in your portal".format( str(len(services))))
def sharePackage2(in_package, folder, username, password, maintain, summary, tags, credits, everyone, org, groups): try: active_url = arcpy.GetActivePortalURL() except: active_url = 'https://www.arcgis.com/' agol_helper = ago.AGOLHelper(portal_url=active_url) # If not app-signed in, and have user/pass, sign in the old manual way if username and password and "Signed in through app" not in username: agol_helper.login(username, password) elif arcpy.GetSigninToken() is not None: # Sign in using info from the app agol_helper.token_login() else: arcpy.AddIDMessage("Error", 1561) return # Make sure file exists try: uploadSize = os.stat(in_package).st_size except FileNotFoundError: raise Exception("The file {0} was not found".format(in_package)) fileName, fileExt = os.path.splitext(os.path.basename(in_package)) try: uploadType = pkgTypes[fileExt.upper()] except KeyError: raise Exception( "Unknown/unsupported package type extension: {0}".format(fileExt)) portalFolders = agol_helper.list_folders() if folder == "<root>" or None: folder = '' folderID = "" moveFolder = False if folder: if folder not in portalFolders.keys(): # Create a new folder folderID = agol_helper.create_folder(folder) arcpy.AddMessage("Created: {}".format(folderID)) #refresh the folder list portalFolders = agol_helper.list_folders() #previousPkgId = agol_helper.search(fileName, uploadType) previousPkgId = agol_helper.search(item_type=uploadType, name=fileName) if len(previousPkgId) == 0: # Pkg does not exist if maintain: # No pkg + maintain meta == quit raise Exception( "Existing package not found. Check to make sure it exists or disable maintain metadata." ) if folder: if folder in portalFolders.keys(): folderID = portalFolders[folder] moveFolder = True else: # Pkg exists newItemID = previousPkgId[0] itemInfo = agol_helper.item(newItemID) # original pkg lives here. pkgFolderID = itemInfo['ownerFolder'] if itemInfo['ownerFolder'] else "" if folder: if folder in portalFolders.keys(): if maintain and portalFolders[folder] != pkgFolderID: raise Exception( "Existing package to update not found in folder {}. Check the folder or disable maintain metadata." .format(folder)) else: # Existing pkg lives in supplied folder. It'll be updated. folderID = portalFolders[folder] if folderID != pkgFolderID: # Package of same name exists but uploading to a different folder moveFolder = True # no else here - this is case where folder needs to be created, covered previously else: if maintain and pkgFolderID: # pkg lives in folder, but root was specified: raise Exception( "Did not find package to update in <root> Does it exist in a folder?" ) # no else here - covered previously with folderID variable initialize # Set metadata by getting original metadata or adding new if not maintain: try: # Only available in Pro 1.2 or 10.4 metaFromPkg = arcpy.GetPackageInfo(in_package) description = metaFromPkg['description'] if not summary: summary = metaFromPkg['summary'] if not tags: tags = metaFromPkg['tags'] if not credits: credits = metaFromPkg['credits'] except AttributeError: description = '' pkgMetadata = (summary, description, tags, credits, '') else: metadataURL = "{}/content/users/{}/{}/items/{}".format( agol_helper.base_url, agol_helper.username, folderID, newItemID) metadata = agol_helper.url_request(metadataURL, { 'token': agol_helper.token, 'f': 'json' }) #re-set everyone if necessary from original share options everyone = True if metadata['sharing'][ 'access'] == 'public' else everyone org = True if everyone else True if metadata['sharing'][ 'access'] == 'org' else org groups = metadata['sharing']['groups'] if metadata['sharing'][ 'groups'] else groups snippet = metadata['item']['snippet'] if metadata['item'][ 'snippet'] else '' description = metadata['item']['description'] if metadata['item'][ 'description'] else '' tags = ','.join(metadata['item']['tags']) accessInfo = metadata['item']['accessInformation'] if metadata['item'][ 'accessInformation'] else '' licenseInfo = metadata['item']['licenseInfo'] if metadata['item'][ 'licenseInfo'] else '' pkgMetadata = (snippet, description, tags, accessInfo, licenseInfo) # Save original thumbnail to update with metadata try: thumbnailURL = "{}/content/items/{}/info/{}".format( agol_helper.base_url, newItemID, metadata['item']['thumbnail']) saveThumb = os.path.join(arcpy.env.scratchFolder, "thumbnail.png") agol_helper.save_file(thumbnailURL, saveThumb) pkgMetadata += (saveThumb, ) except: arcpy.AddWarning("Problem getting thumbnail") arcpy.AddMessage("Using existing metadata") # Behavior is to always overwrite a package if it exists extraParams = {'overwrite': 'true'} # Upload the package arcpy.AddMessage("Beginning file upload") newItemIDres = agol_helper.add_item(in_package, agol_helper.username, folderID, uploadType, params=extraParams) if 'success' in newItemIDres: if newItemIDres['success']: newItemID = newItemIDres['id'] else: raise Exception("(returned msg) {}".format(newItemIDres)) # Commit the file arcpy.AddMessage("Committing the file on the portal") resCom = agol_helper.commit(newItemID, agol_helper.username) status = 'processing' # partial | processing | failed | completed while status == 'processing' or status == 'partial': status = agol_helper.item_status(newItemID, agol_helper.username)['status'] time.sleep(1) if status == 'failed': raise Exception("Failed in processing the file on the portal") if moveFolder: #move new package into folder moveResp = agol_helper.move_items(folderID, [newItemID]) if not moveResp['results'][0]['success']: arcpy.AddMessage( "Failed to move item to folder: '{}'. Item will be created in root" .format(folder)) folderID = "" # Set or Update the metadata arcpy.AddMessage("Setting metadata and sharing settings") uresp = agol_helper.update_item(newItemID, pkgMetadata, folder_id=folderID, title=fileName) try: if not uresp['success']: arcpy.AddWarning("Could not set sharing properties") except: arcpy.AddWarning("Problem setting metadata values:") arcpy.AddError(" {0}".format(uresp['error'])) # Clean up thumbnail try: os.remove(saveThumb) except (NameError, IOError): pass # Set Sharing options if not maintain: if everyone or groups or org: groupIDs = [] if groups: userGroups = agol_helper.list_groups(agol_helper.username) for group in userGroups.keys(): arcpy.AddMessage(group) for selectedgroup in groups: if group == selectedgroup: groupIDs.append(userGroups[group]) gresp = agol_helper.share_items(groupIDs, everyone, org, [newItemID]) try: if not gresp['results'][0]['success']: arcpy.AddWarning("Could not set sharing properties") arcpy.AddError(" {0}".format( gresp['results'][0]['error']['message'])) except: arcpy.AddWarning("Problem sharing item:") arcpy.AddError(" {0}".format(gresp))
def setup(self): """ Sets up the Auditor by logging into the ArcGIS org, getting all the items and folders, and reading in the metatable(s). To be called in __init__(). In case of multiple calls (ie, for retry()), all data are re-instantiated/initialized. """ #: temp_dir used by fixes.metadata_fix() to hold xml of sde metadata temp_dir = Path(arcpy.env.scratchFolder, 'auditor') if not temp_dir.exists(): if self.verbose: print(f'Creating temp directory {temp_dir}...') temp_dir.mkdir() self.log.info(f'Logging into {credentials.ORG} as {credentials.USERNAME}') self.gis = arcgis.gis.GIS(credentials.ORG, credentials.USERNAME, credentials.PASSWORD) #: Make sure ArcGIS Pro is properly logged in arcpy.SignInToPortal(arcpy.GetActivePortalURL(), credentials.USERNAME, credentials.PASSWORD) user_item = self.gis.users.me # pylint: disable=no-member #: Build dict of folders. 'None' gives us the root folder. if self.verbose: print(f'Getting {self.username}\'s folders...') folders = {None: None} for folder in user_item.folders: folders[folder['id']] = folder['title'] self.items_to_check = [] #: Clear this out again in case retry calls setup() multiple times. #: Get item object and it's correspond folder for each relevant item if self.verbose: print('Getting item objects...') #: User-provided list if self.item_ids: for item_id in self.item_ids: item = self.gis.content.get(item_id) # pylint: disable=no-member if not item: raise ValueError(f'Item {item_id} not found') self.items_to_check.append(item) try: self.itemid_and_folder[item.itemid] = folders[item.ownerFolder] except KeyError: raise ValueError(f'Folder id {item.ownerFolder} not found (wrong user?)') #: No user-provided item ids, get all hosted feature services in every folder else: for _, name in folders.items(): for item in user_item.items(name, 1000): if item.type == 'Feature Service': self.items_to_check.append(item) self.itemid_and_folder[item.itemid] = name #: Read the metatable into memory as a dictionary based on itemid. #: Getting this once so we don't have to re-read every iteration if self.verbose: print('Getting metatable info...') self.metatable = Metatable() sgid_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'Authoritative'] agol_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'CATEGORY'] self.metatable.read_metatable(self.sgid_table, sgid_fields) self.metatable.read_metatable(self.agol_table, agol_fields) if self.metatable.duplicate_keys: raise RuntimeError(f'Duplicate AGOL item IDs found in metatables: {self.metatable.duplicate_keys}') #: Get the groups if self.verbose: print('Getting groups...') groups = self.gis.groups.search('title:*') # pylint: disable=no-member self.groups_dict = {g.title: g.id for g in groups}
#%% df.to_csv(custAcctFile,header=False, index=True) #%% [markdown] # [```gis.features.SpatialDataFrame()```](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.features.toc.html?highlight=spatialdataframe#arcgis.features.SpatialDataFrame.from_xy) #%% from arcgis.features import SpatialDataFrame from arcgis.gis import GIS from getpass import getpass from IPython.display import display #%% sdf = SpatialDataFrame.from_xy(df,"POINT_X","POINT_Y") gis = GIS(arcpy.GetActivePortalURL(), username=input("Enter User Name "), password=(getpass())) #gis = GIS() #portalDesc = arcpy.GetPortalDescription() # search and list all items owned by connected user #query=f'owner:{portalDesc["user"]["username"]} AND title:CW BaseMap' #itemType="Feature Layer" #sortField="title" #sortOrder="asc" # default max__items is 10 #maxItems=100 #m = gis.content.search(query,itemType,sortField,sortOrder,maxItems) #%% consumptionLyr = gis.content.import_data(sdf)
print(f'Using {aprx}') arcpy.env.workspace = prjPath arcpy.env.overwriteOutput = True #%% [markdown] # Now we [get the active portal](http://pro.arcgis.com/en/pro-app/arcpy/functions/getactiveportalurl.htm) and instantiate the [GIS](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.gis.toc.html#gis). The [GetPortalDescription()](http://pro.arcgis.com/en/pro-app/arcpy/functions/getportaldescription.htm) returns a dictionary containing portal information. #%% [markdown] # TODO switching active portal fails #%% response = input( f'Do you want to use the active portal? {(arcpy.GetActivePortalURL())} [y or n]' ) if response.upper() in "YES": username = input("Enter User Name") gis = GIS(arcpy.GetActivePortalURL(), username, password=(getpass())) else: portalURL = input("Enter Portal Address") username = input("Enter User Name") gis = GIS(portalURL, username, password=getpass()) portal_desc = arcpy.GetPortalDescription() print( f'Portal Name - {portal_desc["portalName"]}\nConnected to {portal_desc["name"]} as user {username}' ) #%% [markdown] # Using [gis.content.search()](https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.gis.toc.html#arcgis.gis.ContentManager.search) we are going to find all Groups and [item_types](https://developers.arcgis.com/rest/users-groups-and-items/items-and-item-types.htm) (that are Feature Layers) owned by connected user. #%% # search for all Groups owned by connected user sourceGroups = gis.groups.search(query=f'owner:{username}')
def process(self): #: Set up paths and directories feature_service_name = secrets.FEATURE_SERVICE_NAME temp_csv_dir = os.path.join(arcpy.env.scratchFolder, 'fleet') temp_fc_path = os.path.join(arcpy.env.scratchGDB, feature_service_name) sddraft_path = os.path.join(arcpy.env.scratchFolder, f'{feature_service_name}.sddraft') sd_path = sddraft_path[:-5] paths = [temp_csv_dir, temp_fc_path, sddraft_path, sd_path] for item in paths: if arcpy.Exists(item): self.log.info(f'Deleting {item} prior to use...') arcpy.Delete_management(item) os.mkdir(temp_csv_dir) if not secrets.KNOWNHOSTS or not os.path.isfile(secrets.KNOWNHOSTS): raise FileNotFoundError( f'known_hosts file {secrets.KNOWNHOSTS} not found. Please create with ssh-keyscan.' ) #: Download all the files in the upload folder on sftp to temp_csv_dir self.log.info( f'Downloading all files from {secrets.SFTP_HOST}/upload...') connection_opts = pysftp.CnOpts(knownhosts=secrets.KNOWNHOSTS) with pysftp.Connection(secrets.SFTP_HOST, username=secrets.SFTP_USERNAME, password=secrets.SFTP_PASSWORD, cnopts=connection_opts) as sftp: sftp.get_d('upload', temp_csv_dir, preserve_mtime=True) #: Get the latest file source_path, source_date = self.get_latest_csv(temp_csv_dir, previous_days=7) self.log.info( f'Converting {source_path} to feature class {temp_fc_path}...') wgs84 = arcpy.SpatialReference(4326) result = arcpy.management.XYTableToPoint(source_path, temp_fc_path, 'LONGITUDE', 'LATITUDE', coordinate_system=wgs84) self.log.debug(result.getMessages()) try_count = 1 while True: try: self.log.info(f'Updating service, try {try_count} of 3...') self.log.info( f'Connecting to AGOL as {secrets.AGOL_USERNAME}...') gis = arcgis.gis.GIS('https://www.arcgis.com', secrets.AGOL_USERNAME, secrets.AGOL_PASSWORD) sd_item = gis.content.get(secrets.SD_ITEM_ID) self.log.info('Getting map and layer...') arcpy.SignInToPortal(arcpy.GetActivePortalURL(), secrets.AGOL_USERNAME, secrets.AGOL_PASSWORD) layer, fleet_map = self.get_map_layer(secrets.PROJECT_PATH, temp_fc_path) #: draft, stage, update, publish self.log.info('Staging and updating...') self.update_agol_feature_service(fleet_map, layer, feature_service_name, sddraft_path, sd_path, sd_item) #: Update item description self.log.info('Updating item description...') feature_item = gis.content.get(secrets.FEATURES_ITEM_ID) year = source_date[:4] month = source_date[4:6] day = source_date[6:] description = f'Vehicle location data obtained from Fleet; updated on {year}-{month}-{day}' feature_item.update( item_properties={'description': description}) except Exception as e: err_msg = f'Error on attempt {try_count} of 3; retrying.' self.log.exception(err_msg) #: Fail for good if 3x retry fails, otherwise increment, sleep, #: and retry if try_count > 3: err_msg = 'Connection errors; giving up after 3 retries' self.log.exception(err_msg) raise e sleep(try_count**2) try_count += 1 continue #: If we haven't gotten an error, break out of while True. break
analysis = arcpy.mapping.AnalyzeForSD(newSDdraft) if analysis['errors'] == {}: if os.path.exists(sd): os.remove(sd) print("{0} already exists! Deleted! ".format(sd)) # Stage the service arcpy.StageService_server(newSDdraft, sd) # Upload the service. arcpy.UploadServiceDefinition_server(in_sd_file=sd, in_server="My Hosted Services") print "Uploaded and overwrote service" else: # If the sddraft analysis contained errors, display them and quit. print analysis['errors'] ##### Main Script if os.path.isdir(workSpace) == False: print "Not valid path..." else: arcpy.SignInToPortal_server(portaluser, psd, "") print(arcpy.GetActivePortalURL()) files = os.listdir(workSpace) for f in files: if f.endswith(".mxd"): # mxdPath = os.path.join(workSpace, f) print("publishing: " + f) publishingHostedFeatureService(workSpace, f) print(f + " Publishing Done !") else: continue