def main(): logger = setup_custom_logger("portal_migration") logger.info("Beginning Portal Migration Script") #Read in variables from config.py # Connect to the Source GIS logger.debug("Attempting to connect to the source portal: " + config.source_portal_url) source_gis = GIS(config.source_portal_url, config.source_portal_admin, config.source_portal_password) logger.info("Successfully connected to the source portal: " + config.source_portal_url) # Connect to the Target GIS logger.debug("Attempting to connect to the target portal: " + config.target_portal_url) target_gis = GIS(config.target_portal_url, config.target_portal_admin, config.target_portal_password) logger.info("Successfully connected to the target portal: " + config.target_portal_url) # This will eventually go into the user_migration class source_users = source_gis.users.search('!esri_ & !admin') # return the user list, source_items_by_id = {} for user in source_users: logger.info("User: "******" Role: " + user.role) num_items = 0 num_folders = 0 user_content = user for item in user_content: num_items += 1 source_items_by_id[item.itemid] = item folders = user.num_folders for folder in folders: num_folders += 1 folder_items = user.itme(folder=folder['title']) for item in folder_items: num_items += 1 source_items_by_id[item.itemid] = item # target_gis = GIST("https://target.jturco-ms.com/portal", "admin", "password") # create tables here for migration status....maybe web applciation as well (nice to have not need to have) # target_gis = GIS("https://green.jturco-ms.esri.com/portal", "admin", "password") # migrate users using a class for users # class should be able to handle what users to migrate, ALL, or Specific, levels? # # migrate groups using a class for groups # migrate content using a class for content return
def main(): # Test adding Data to gis useMyAccount = False myGis = AddInformationUsingDataFrame() if useMyAccount: gis = GIS(myGis.gis) else: gis = GIS() fc = gis.content.import_data(readWiki(), {"CountryCode":"Country"}) item_properties = { "title": "Worldwide gun ownership", "tags" : "guns,violence", "snippet": " GSR Worldwide gun ownership", "description": "test description", "text": json.dumps({"featureCollection": {"layers": [dict(fc.layer)]}}), "type": "Feature Collection", "typeKeywords": "Data, Feature Collection, Singlelayer", "extent" : "-102.5272,-41.7886,172.5967,64.984" } map1, item, gis = AddInformationUsingDataFrame.postMapUseWikipedia(item_properties = item_properties, gis = gis) # Test webMap using testWebMapScene()
def viewMap(): gis = GIS('home') callback_map = gis.map('San Diego convention center, San Diego, CA', 16) callback_map.on_click(find_addr) callback_map return
def __init__(self): my_gis = GIS() global hello_map hello_map = my_gis.map('Санкт-Петербург', zoomlevel=15) hello_map.basemap = 'osm' # ============ def find_addr(hello_map, g): # try: hello_map.draw(g) global address geocoded = geocoding.reverse_geocode(g) address = geocoded['address']['Match_addr'] # adres_read = Adres_read(address) # print(adres_read.get_points()) try: print('creating') # a = Osm_reader(adres_read.get_points()) # a.create_file() print('file created') except: print("beda") hello_map.on_click(address)
def home(request): gis = GIS() print("Logged in as " + str(gis.properties.user.username)) map_sj = gis.map("San Jose, CA") map_js = 'require(["esri/views/MapView"], function(MapView) {' \ ' });' return_view = '<h1>Home page</h1><br>'+str(map_sj) return HttpResponse(return_view) return render(request, 'deforest/home.html')
def establish_gis_connection(portal_url, user, password): if portal_url: print("[DEBUG]: Using Portal for ArcGIS") gis = GIS(portal_url, user, password) elif user and not portal_url: print("[DEBUG]: Using ArcGIS Online") gis = GIS("https://www.arcgis.com", user, password) else: print("[DEBUG]: Using anonymous access to ArcGIS Online") gis = GIS() return gis
def authenticate(user, password, portal_url=None): if portal_url: print("[DEBUG]: Using Portal for ArcGIS") gis = GIS(portal_url, user, password) elif user and not portal_url: print("[DEBUG]: Using ArcGIS Online") gis = GIS("https://www.arcgis.com", user, password) else: print("[DEBUG]: Using anonymous access to ArcGIS Online") gis = GIS() return gis
def authenticate(user, password, portal_url=None): #authenticate with portal or AGOL if portal_url: arcpy.AddMessage("[DEBUG]: Using Portal for ArcGIS") gis = GIS(portal_url, user, password) elif user and not portal_url: arcpy.AddMessage("[DEBUG]: Using ArcGIS Online") gis = GIS("https://www.arcgis.com", user, password) else: arcpy.AddMessage("[DEBUG]: Using anonymous access to ArcGIS Online") gis = GIS() return gis
def get_county_geometry_series(itm_id: [Path, str]): """ Get a Pandas Series with the county FIPS as a string index, and the values as a valid geometry column from an online feature service. :return: Pandas Series. """ # Get a layer instance cty_lyr = GIS().content.get(itm_id).layers[0] # retrieve the data necessary cty_df = cty_lyr.query(out_fields=['FIPS'], out_sr=4326, as_df=True) # set the index and slice off only the geometry column to return as a series cty_df.set_index(['FIPS'], inplace=True) cty_srs = cty_df['SHAPE'] return cty_srs
def main(event, context): # Cityworks settings global base_url base_url = event['CityworksURL'] cw_user = event['CityworksUsername'] cw_password = event['CityworksPassword'] # ArcGIS Online/Portal settings org_url = event['ArcGISURL'] username = event['ArcGISUsername'] password = event['ArcGISPassword'] try: # Connect to org/portal gis = GIS(org_url, username, password) # authenticate CW cw_status = get_cw_token(cw_user, cw_password) if "error" in cw_status: raise ValueError("Error Authenticating Cityworks") # get wkid global sr sr = get_wkid() if sr == "error": raise ValueError("Spatial reference not defined") except Exception as ex: print("error: " + str(ex)) else: for config in event['Configurations']: update(config)
def main(): ARCGIS_USER = config.get("ARCGIS", "USER") ARCGIS_PW = config.get("ARCGIS", "PW") ARCGIS_URL = config.get("ARCGIS", "URL") gis = GIS(ARCGIS_URL, ARCGIS_USER, ARCGIS_PW) content_data = f"title:{FILENAME} type:Feature Service owner:{ARCGIS_USER}" item = next((f for f in gis.content.search(content_data)), None) all = False if item is None: all = True data = fetch(all) path = TEMP_PATH if item is not None: path += "_temp" path += ".csv" if len(data) == 0: logger.info("Data not found") return keys = data[0].keys() with open(path, "w", newline="") as output_file: dict_writer = csv.DictWriter(output_file, keys) dict_writer.writeheader() dict_writer.writerows(data) upload_arcgis(gis, item, path)
def main(arguments): # Initialize logging logger = initialize_logger(arguments.log_file) # Create the GIS logger.info("Authenticating...") # First step is to get authenticate and get a valid token gis = GIS(arguments.org_url, username=arguments.username, password=arguments.password, verify_cert=not arguments.skip_ssl_verification) # Get the workforce project item = gis.content.get(arguments.project_id) project = workforce.Project(item) # Read the CVS file and loop through the workers information contained within this file logger.info("Parsing CSV...") with open(os.path.abspath(arguments.csv_file), 'r') as file: reader = csv.DictReader(file) # List of workers to add workers = [] for row in reader: # Create a worker using the required fields worker = workforce.Worker(project, name=row[arguments.name_field], status=row[arguments.status_field], user_id=row[arguments.user_id_field]) # These fields are optional, and are added separately if arguments.contact_number_field: worker.contact_number = row.get(arguments.contact_number_field) if arguments.title_field: worker.title = row.get(arguments.title_field) workers.append(worker) # Batch add workers logger.info("Adding Workers...") project.workers.batch_add(workers) logger.info("Completed")
def _get_layer(flc_id, flc_url, layer_name): """Find the layer with the specified name (or ID, if integer) among either the layers or tables of the FeatureLayerCollection with the specified ID or URL. """ # Get the feature layer collection. if flc_id: gis = GIS() flc = gis.content.get(flc_id) loc = f'content ID {flc_id}' assert flc is not None, f'Unable to find ArcGIS ID {flc_id}' elif flc_url: loc = f'flc URL {flc_url}' flc = FeatureLayerCollection(flc_url) else: raise ValueError('Either flc_id or url must be provided') # Now get the layer. layer = None if isinstance(layer_name, str): layer = _get_layer_by_name(layer_name, flc.layers, flc.tables) elif isinstance(layer_name, int): layer = _get_layer_by_id(layer_name, flc.layers, flc.tables) if layer: return layer raise ValueError(f'Unable to find layer {layer_name} in {loc}')
def overwrite_csv(username: str, password: str, new_df: DataFrame, old_csv_name: str): """ Overwrites the existing table/feature layer named `old_csv_name` using `new_df` Only works if `new_df` has the same columns as the old feature/table (Create an existing table/feature layer by manually uploading a csv to arcGIS and selecting the "Publish this file as a hosted layer" option) """ gis = GIS(url='https://www.arcgis.com', username=username, password=password) csv_file_name = f"{old_csv_name}.csv" new_df.to_csv(csv_file_name, index=False) old_item = gis.content.search(f"title: {old_csv_name}", 'Feature Layer')[0] old_feature_layer = FeatureLayerCollection.fromitem(old_item) logger.info( f"Overwriting feature layer named '{old_csv_name}'.... there will now be {len(new_df)} features." ) overwrite_response = old_feature_layer.manager.overwrite(csv_file_name) logger.info( f'Done overwriting feature layer. Response: {overwrite_response}') os.remove(csv_file_name)
def _main(): g = GIS(profile="python_playground") server = g.admin.servers.list()[0] messages = server.logs.query(datetime.now().isoformat(), level="SEVERE")["logMessages"] grouped_messages = _group_by_code(messages) _output_graph_image(grouped_messages)
def update_covid_testing_city_county_data(**kwargs): """ The actual python callable that Airflow schedules. """ # Getting data from google sheet filename = kwargs.get("filename") workbook = kwargs.get("workbook") sheet_name = kwargs.get("sheet_name") df = get_county_data(filename, workbook, sheet_name) df.to_csv("/tmp/county_city_cumulative.csv", index=False) # Updating ArcGIS arcconnection = BaseHook.get_connection("arcgis") arcuser = arcconnection.login arcpassword = arcconnection.password gis = GIS("http://lahub.maps.arcgis.com", username=arcuser, password=arcpassword) filename = "/tmp/county_city_cumulative.csv" gis_item = gis.content.get(LA_CITY_TESTS_FEATURE_ID) gis_layer_collection = arcgis.features.FeatureLayerCollection.fromitem(gis_item) gis_layer_collection.manager.overwrite(filename) # Clean up os.remove(filename)
def insertFeature(): # ArcGIS Online Authentication print("...Authenticating with ArcGIS Online") agolURL = 'agolurl' agolUsername = '******' agolPwd = 'password' gis = GIS(agolURL, agolUsername, agolPwd) print("...Connected to ArcGIS Online") # Find arcgis online item and layer itemId = config.get('ArcgisOnline', 'agolItemId') agolItem = gis.content.get(itemId) layerName = config.get('ArcgisOnline', 'agolLayerName') agolLayer = findLayerInItem(agolItem, layerName) # Create feature to insert insertFeature = { 'attributes': { 'foo': 'bar', 'key': 'value', }, 'geometry': { 'x': -8372854.089165498, 'y': 4882472.522499578 } } print(insertFeature) res = agolLayer.edit_features(adds=[insertFeature]) print(res)
def hello(): #html = urllib2.urlopen("https://www.miemssalert.com/chats/Default.aspx?hdRegion=5&hdtab=Hospitals").read() #html = urllib2.urlopen("https://www.miemssalert.com/chats/Default.aspx?hdRegion=3&hdtab=Hospitals").read() html = urllib2.urlopen("https://www.miemssalert.com/chats/Default.aspx?hdRegion=124&hdtab=Hospitals").read() soup = BeautifulSoup(html,features="lxml") table = soup.find("table", {"id": "tblHospitals"}) table_data = table.find_all("tr") table_data.pop(0) hospitals = {} i = 0 for unit in table_data: alerts = unit.find_all('td') hospitals[i] = [] j = 0 for flag in alerts: if j < 7: hospitals[i].append(flag.string) j = j + 1 i = i + 1 gis = GIS() webmap = gis.content.get('41281c51f9de45edaf1c8ed44bb10e30') return webmap
def thanks(df=None): portal_url = current_app.config['PORTAL_URL'] portal_user = current_app.config['PORTAL_USER'] portal_password = current_app.config['PORTAL_PASSWORD'] cases_url = current_app.config['COVID_CASES_URL'] ppe_url = current_app.config['PPE_INVENTORY_URL'] # Show the data that was just entered. portal = GIS(portal_url, portal_user, portal_password) if df == 'cases': # Generate new CSV files while we're at it. # In production they will be written to the "capacity" webserver # In a test environment they end up in the local folder. results_df = FeatureLayer(cases_url).query( where="editor='EMD'", order_by_fields="utc_date DESC", return_all_records=False, result_record_count=1, return_geometry=False).sdf elif df == 'ppe': results_df = FeatureLayer(ppe_url).query( where="facility='Clatsop'", order_by_fields="utc_date DESC", return_all_records=False, result_record_count=1, return_geometry=False).sdf else: results_df = pd.DataFrame() return render_template('thanks.html', df=results_df)
def connectUsernamePassword(self, username, password, verifySSL=True) -> GIS: """ Method for connecting to a portal that uses built-in users for authentication Parameters: username (string): log in name for the user. If not supplied, a prompt will be used to capture it. password (string): Password for the user. If not supplied, a prompt will be used to capture it. verifySSL (boolean): Flag for requiring a valid SSL certificate on the portal Returns: GIS: An argis.gis.GIS object """ if not username or not password: raise Exception("username and password are required") try: gis = GIS(self.portalUrl, username, password, verify_cert=verifySSL) return gis except Exception as e: raise e
def main(): # Set up the global information and variables global data_dir # Directory where csv files are located global metadata_dir # Directory where meatadata files are located global failed_series global online_username global gis_online_connection failed_series = [] # ### Create a connection to your ArcGIS Online Organization # Use the ArcGIS API for python to connect to your ArcGIS Online Organization # to publish and manage data. For more information about this python library # visit the developer resources at # [https://developers.arcgis.com/python/](https://developers.arcgis.com/python/] online_username = input('Username: '******'Password: '******'41f1252fa7ab435e8bb812523200a8b0', '1.1.1') return
def deployLogic(workGDB, itemid, original_sd_file, service_name): gis = GIS(url='https://arcgis.com', username=username, password=password) item = gis.content.get(itemid) sd_file_name = os.path.basename(original_sd_file) if sd_file_name != item.related_items("Service2Data")[0].name: raise Exception('Erroneous itemid, service name or original sd file'.format(itemid)) # Unpack original_sd_file using 7-zip path_7z = fnmatch.filter(os.environ['path'].split(';'), '*7-Zip') temp_dir = tempfile.mkdtemp() if len(path_7z): exe_7z = os.path.join(path_7z[0], '7z.exe') call_unzip = '{0} x {1} -o{2}'.format(exe_7z, original_sd_file, temp_dir) else: raise Exception('7-Zip could not be found in the PATH environment variable') subprocess.call(call_unzip) # Replace Live.gdb content liveGDB = os.path.join(temp_dir, 'p20', 'live.gdb') # os.mkdir(os.path.join(temp_dir, 'p20')) arcpy.management.CreateFileGDB(os.path.dirname(liveGDB), os.path.basename(liveGDB)) shutil.rmtree(liveGDB) os.mkdir(liveGDB) for root, dirs, files in os.walk(workGDB): files = [f for f in files if '.lock' not in f] for f in files: shutil.copy2(os.path.join(workGDB, f), os.path.join(liveGDB, f)) # Zip file os.chdir(temp_dir) updated_sd = os.path.join(temp_dir, sd_file_name) call_zip = '{0} a {1} -m1=LZMA'.format(exe_7z, updated_sd) subprocess.call(call_unzip) # Replace file manager = arcgis.features.FeatureLayerCollection.fromitem(item).manager status = manager.overwrite(updated_sd) # Return return True
def conn_portal(webgis_config): """Creates connection to an ArcGIS Enterprise Portal.""" print('Establishing connection to ArcGIS Enterprise Portal...') w_gis = None try: if cfg_webgis['profile']: w_gis = GIS(profile=webgis_config['profile']) else: w_gis = GIS(webgis_config['portal_url'], webgis_config['username'], webgis_config['password']) except Exception as e: print('Error: {}'.format(e)) print( 'Exiting script: not able to connect to ArcGIS Enterprise Portal.') exit() return w_gis
def main(portal, username, password, piper_item, piper_layer, piper_feature_id_field, piper_update_field, parks_item, parks_layer, parks_transfer_field): gis = GIS(portal, username, password) park_piper(gis, piper_item, piper_layer, piper_feature_id_field, piper_update_field, parks_item, parks_layer, parks_transfer_field)
def arcgisLogin(useRootPath=False): username, password = getCredentials(useRootPath=useRootPath) from arcgis.gis import GIS gis = GIS("https://www.arcgis.com", username=username, password=password) print("Login successful!!") return gis
def lambda_handler(event, context): s3bucket = event['Records'][0]['s3']['bucket']['name'] s3key = event['Records'][0]['s3']['object']['key'] response = s3client.get_object(Bucket=str(s3bucket), Key=str(s3key)) data = json.loads(response['Body'].read().decode('utf-8')) lst = [] lst.append(str(data['DeviceId'])) lst.append(',') lst.append(str(data['ProductionKW'])) lst.append(',') lst.append(str(data['ConsumptionKW'])) lst.append(',') lst.append(str(data['Latitude'])) lst.append(',') lst.append(str(data['Longitude'])) lst.append(',') lst.append(str(data['Address'])) csvdata = ''.join(lst) filename = '/tmp/smartmeters.csv' headers = 'DeviceId,ProductionKW,ConsumptionKW,Latitude,Longitude,Address' print(csvdata) with open(filename, 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL) writer.writerow(headers.split(",")) writer.writerow(csvdata.split(",")) username = ssmclient.get_parameter(Name='ArcGISUsername') password = ssmclient.get_parameter(Name='ArcGISPassword') gis = GIS(str('https://www.arcgis.com'), str(username['Parameter']['Value']), str(password['Parameter']['Value'])) my_content = gis.content.add({}, filename) time.sleep(2) item = gis.content.get(my_content.id) print(item.publish())
def main(): print("Starting process: Update Feature Layers for UCS AGOL.") start_time = time.time() print("Started on " + time.ctime(start_time) + ".") print("ArcGIS Python API version " + arcgis.__version__) try: #login to AGOL print("Logging in to AGOL...") conn = GIS("https://www.arcgis.com", username="******", password="******") except: print("Could not login. Bailing out.") sys.exit() try: update_covid19_cases_feature_layer2(conn = conn) update_heat_index_forecast_feature_layer(conn = conn, day=0) except Exception as inst: print(type(inst)) # the exception instance print(inst.args) # arguments stored in .args print(inst) # __str__ allows args to be printed directly, e = sys.exc_info()[1] print(e.args[0]) finally: print("Done.") print("--- %s seconds ---" % round((time.time() - start_time)))
def upload_arcgis(): ARCGIS_USER = config.get("ARCGIS", "USER") ARCGIS_PW = config.get("ARCGIS", "PW") ARCGIS_URL = config.get("ARCGIS", "URL") gis = GIS(ARCGIS_URL, ARCGIS_USER, ARCGIS_PW) content_data = f"title:{FILENAME} type:CSV owner:{ARCGIS_USER}" items = gis.content.search(content_data) item_params = dict(title=FILENAME) overwrite = False if len(items) > 0: logger.info("Updating file in Arcgis") item = items[0] item.update(item_params, data=TEMP_PATH) overwrite = True else: logger.info("Uploading file to Arcgis") item = gis.content.add(item_params, data=TEMP_PATH) item.share(everyone=True) logger.info("Publishing layer") publish_params = dict( name=FILENAME, type="csv", locationType="coordinates", latitudeFieldName="latitude", longitudeFieldName="longitude", ) item = item.publish(publish_parameters=publish_params, overwrite=overwrite) item.share(everyone=True)
def main(arguments): # initialize logger logger = initialize_logging(arguments.log_file) # Create the GIS logger.info("Authenticating...") # First step is to get authenticate and get a valid token gis = GIS(arguments.org_url, username=arguments.username, password=arguments.password, verify_cert=not arguments.skip_ssl_verification) # Get the project item = gis.content.get(arguments.project_id) project = workforce.Project(item) invalid_assignments = get_invalid_assignments(project, arguments.time_tolerance, arguments.distance_tolerance, arguments.min_accuracy, arguments.workers) with open(arguments.config_file, 'r') as f: field_mappings = json.load(f) target_fl = arcgis.features.FeatureLayer(arguments.target_fl, gis) # Check if layer exists try: x = target_fl.properties except Exception as e: logger.info(e) logger.info( "Layer could not be found based on given input. Please check your parameters again. Exiting the script" ) sys.exit(0) copy_assignments(project, invalid_assignments, target_fl, field_mappings)
def main(): time.sleep(30) # wait for 30s to avoid exceeding rate limit Bus_Stop_URL = constants.Bus_Stop_URL # get request URL/ItemID from constants module Bus_Stop_Item_ID = constants.Bus_Stop_Item_ID new_bus_stop_data = CommonModules.request_from_LTA(Bus_Stop_URL, constants.LTA_ACCOUNT_KEY, exceed_return_limit=True) # use the account key to query taxi data new_bus_stop_data = AmendBusStopCode(new_bus_stop_data) # process bus stop data to maintain 5 digits text agol_connection = GIS(constants.agol_home_url, constants.agol_username, constants.agol_password, verify_cert=False) # connect to ArcGIS Online using your credentials bus_stop_layer = agol_connection.content.get(Bus_Stop_Item_ID).layers[0] # get first layer from the bus stop feature service data_schema = list(new_bus_stop_data[0].keys()) # get the list of attribute names as schema update_result = CommonModules.RemoveAndAppendNewData(bus_stop_layer, new_bus_stop_data, data_schema, location=True, geometry="point") # remvoe existing bus stops and append new bus stops CommonModules.UpdateLastUpdatedTime(agol_connection, "Bus Stop") # update Data Updated Time print(update_result)
# Script to theme your ArcGIS Enterprise from arcgis.gis import GIS dept_url = 'https://dev005223.esri.com/portal' admin_username = '******' admin_password = '******' #create a GIS connection dept_gis = GIS(dept_url, admin_username, admin_password, verify_cert=False) print(" Connected to the city's ArcGIS Enterprise") try: # Set logo dept_gis.admin.ux.set_logo('style_files/logo.png') print(" Set Enterprise logo") # set background dept_gis.admin.ux.set_background("style_files/background.png") print(" Set Enterprise background") # set banner dept_gis.admin.ux.set_banner("style_files/banner.png") print(" Set Enterprise banner") # set name dept_gis.admin.ux.set_name_description("Pasadena City Fire Dept. GIS", "Official WebGIS of Fire Department of the city of Pasadena") print(" Set Enterprise name") # set extent
city_list = ['pasadena', 'santa_monica'] enterpise_list = ['https://dev005220.esri.com/portal', 'https://dev005223.esri.com/portal'] admin_username = '******' admin_password = '******' download_path = '/Users/atma6951/Documents/temp/e_configs' # Loop through each city and create a group meant for collaboration for city_id in list(range(len(city_list))): city_name = city_list[city_id] city_url = enterpise_list[city_id] print("Processing " + city_name.replace("_", " ")) #create a GIS connection city_gis = GIS(city_url, admin_username, admin_password, verify_cert=False) print(" Connected to the city's ArcGIS Enterprise") #download the config archive from group that participates in collaboration gsr = city_gis.groups.search("title: LA county GIS collaboration", max_groups=1) city_collab_group = gsr[0] print(" Searching the collaboration group for content") config_item = [i for i in city_collab_group.content() if i.title.endswith("config")][0] config_file = config_item.download(download_path) print(" Downloaded the config archive") #extract the zip file zf = ZipFile(config_file) zf.extractall(download_path) config_str_path = [afile for afile in zf.filelist if afile.filename.endswith('/config.json')][0]
from arcgis.gis import GIS while 1: my_gis = GIS() my_gis.map()