def setUpClass(self): # pylint: disable=bad-classmethod-argument
self.maxDiff = None
tbx_path = os.path.join(os.path.dirname(CWD), "LargeNetworkAnalysisTools.pyt")
arcpy.ImportToolbox(tbx_path)
self.input_data_folder = os.path.join(CWD, "TestInput")
sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb")
self.origins = os.path.join(sf_gdb, "Analysis", "TractCentroids")
self.destinations = os.path.join(sf_gdb, "Analysis", "Hospitals")
self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND")
tms = arcpy.nax.GetTravelModes(self.local_nd)
self.local_tm_time = tms["Driving Time"]
self.local_tm_dist = tms["Driving Distance"]
self.portal_nd = portal_credentials.PORTAL_URL # Must be arcgis.com for test to work
self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE
arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD)
# Create a unique output directory and gdb for this test
self.output_folder = os.path.join(
CWD, "TestOutput", "Output_Tool_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
os.makedirs(self.output_folder)
self.output_gdb = os.path.join(self.output_folder, "outputs.gdb")
arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb))
def setUpClass(self): # pylint: disable=bad-classmethod-argument
self.maxDiff = None
self.input_data_folder = os.path.join(CWD, "TestInput")
self.sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb")
self.origins = os.path.join(self.sf_gdb, "Analysis", "TractCentroids")
self.destinations = os.path.join(self.sf_gdb, "Analysis", "Hospitals")
self.local_nd = os.path.join(self.sf_gdb, "Transportation",
"Streets_ND")
self.local_tm_time = "Driving Time"
self.local_tm_dist = "Driving Distance"
self.portal_nd = portal_credentials.PORTAL_URL
self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE
arcpy.SignInToPortal(self.portal_nd,
portal_credentials.PORTAL_USERNAME,
portal_credentials.PORTAL_PASSWORD)
# Create a unique output directory and gdb for this test
self.output_folder = os.path.join(
CWD, "TestOutput", "Output_SolveLargeODCM_" +
datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
os.makedirs(self.output_folder)
self.output_gdb = os.path.join(self.output_folder, "outputs.gdb")
arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb),
os.path.basename(self.output_gdb))
# Copy some data to the output gdb to serve as barriers. Do not use tutorial data directly as input because
# the tests will write network location fields to it, and we don't want to modify the user's original data.
self.barriers = os.path.join(self.output_gdb, "Barriers")
arcpy.management.Copy(
os.path.join(self.sf_gdb, "Analysis", "CentralDepots"),
self.barriers)
self.od_args = {
"origins": self.origins,
"destinations": self.destinations,
"network_data_source": self.local_nd,
"travel_mode": self.local_tm_dist,
"output_od_lines": os.path.join(self.output_gdb, "OutODLines"),
"output_origins": os.path.join(self.output_gdb, "OutOrigins"),
"output_destinations": os.path.join(self.output_gdb,
"OutDestinations"),
"chunk_size": 20,
"max_processes": 4,
"time_units": "Minutes",
"distance_units": "Miles",
"cutoff": 2,
"num_destinations": 1,
"precalculate_network_locations": True,
"barriers": [self.barriers]
}
def sign_into_portal(self):
portal_config = self.config['portal']
portal_url = arcpy.GetActivePortalURL()
if 'username' in portal_config and 'password' in portal_config:
try:
portal_info = arcpy.SignInToPortal(portal_url,
portal_config['username'],
portal_config['password'])
token = portal_info['token']
except (ValueError, KeyError):
return None
elif 'app_id' in portal_config and 'refresh_token' in portal_config:
try:
payload = {
'client_id': portal_config['app_id'],
'refresh_token': portal_config['refresh_token'],
'grant_type': 'refresh_token'
}
req = requests.post(portal_url + '/sharing/rest/oauth2/token',
data=payload,
verify=False)
req_json = req.json()
token = req_json['access_token']
except (ValueError, KeyError):
return None
elif 'app_id' in portal_config and 'app_secret' in portal_config:
try:
payload = {
'client_id': portal_config['app_id'],
'client_secret': portal_config['app_secret'],
'grant_type': 'client_credentials'
}
req = requests.post(portal_url + '/sharing/rest/oauth2/token',
data=payload,
verify=False)
req_json = req.json()
token = req_json['access_token']
except (ValueError, KeyError):
return None
else:
infos = arcpy.GetSigninToken()
if infos:
token = infos['token']
else:
return None
self.token = token
return self.token
def create_sd_files_from_map(self,
map_name=None,
pro_prjx=None,
service_id=""):
sds = {}
try:
the_prj = pro_prjx if pro_prjx else self._config.baseprjx
aprx = arcpy.mp.ArcGISProject(the_prj)
if map_name in [m.name for m in aprx.listMaps()]:
service_object = getattr(self._config, map_name.lower())
service_name = service_object["servicename"]
try:
service_name = service_name % service_id
except Exception as e:
self.errorlog(str(e))
self.log("This is an expected error. ;-)")
self.log(f"Service name:{service_name}")
sddraft_file = TempFileName.generate_temporary_file_name(
suffix=".sddraft")
sdfile = TempFileName.generate_temporary_file_name(
suffix=".sd")
self.log(f"Creating:{sddraft_file},{sdfile}")
m = aprx.listMaps(map_name)[0]
arcpy.SignInToPortal(self._config.portal, self._config.user,
self._config.password)
#sharing_draft = m.getWebLayerSharingDraft("HOSTING_SERVER", "FEATURE", service_name)
# change for JXT
sharing_draft = m.getWebLayerSharingDraft(
"FEDERATED_SERVER", "MAP_IMAGE", service_name)
sharing_draft.federatedServerUrl = self._config.fed_server
sharing_draft.copyDataToServer = True
# change to Map Image server
sharing_draft.summary = service_object["summary"]
sharing_draft.tags = service_object["tags"]
sharing_draft.description = service_object[
"description"] + f'<br/>Item updated on {datetime.datetime.now().strftime("%d %B %Y %H:%M:%S")} by automated script.'
sharing_draft.credits = ""
sharing_draft.exportToSDDraft(sddraft_file)
arcpy.StageService_server(sddraft_file, sdfile)
sds[service_name] = sdfile
except Exception as e:
self.errorlog(e)
return sds
def create_report(self):
"""
Returns a list of dicts whose keys are column headings and values are the column values:
[{itemid: 'some_uuid', title: 'AGOL title', ...} ...]
"""
self.logger.info('Creating AGOL Usage Report...')
item_info_dicts = []
try:
arcpy.SignInToPortal(credentials.ORG, credentials.USERNAME,
credentials.PASSWORD)
except NameError as ex: #: Travis CI throws a NameError on arcpy calls.
self.logger.info(ex)
org = tools.Organization(self.logger, credentials.ORG,
credentials.USERNAME, credentials.PASSWORD)
folders = org.get_users_folders()
items = org.get_feature_services_in_folders(folders)
open_data_groups = org.get_open_data_groups()
metatable = tools.Metatable(self.logger)
sgid_fields = [
'TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'Authoritative'
]
agol_fields = [
'TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'CATEGORY'
]
metatable.read_metatable(credentials.SGID_METATABLE, sgid_fields)
metatable.read_metatable(credentials.AGOL_METATABLE, agol_fields)
for item_tuple in items:
item, folder = item_tuple
metatable_category = None
if item.itemid in metatable.metatable_dict:
metatable_category = metatable.metatable_dict[
item.itemid].category
item_info_dicts.append(
org.get_item_info(item, open_data_groups, folder,
metatable_category))
return item_info_dicts
def get_token(base_url: str) -> Optional[str]:
"""get token"""
token: Optional[str] = None
try:
hosted_feature_service: bool = arcpy.GetParameter(0)
ags_service: bool = arcpy.GetParameter(1)
portal_url: str = arcpy.GetParameterAsText(2)
username: str = arcpy.GetParameterAsText(7)
password: str = arcpy.GetParameterAsText(8)
if not (is_blank(username) or is_blank(password)):
arcpy.AddMessage('\nGenerating Token\n')
# Generate token for hosted feature service
if hosted_feature_service:
arcpy.SignInToPortal(portal_url, username, password)
token = arcpy.GetSigninToken()['token']
# Generate token for AGS feature service
elif ags_service:
token = generate_token(base_url, username, password)
if token is None:
raise ValueError('Error generate token')
except:
raise Exception('Error generate token')
return token
def UploadGIS_Service():
try:
print('-' * 50)
print("Sign into ArcGIS portal ...")
callbackUserData = arcpy.SignInToPortal(**args.userData)
args.userData.update(callbackUserData)
print("Uploading GIS layers ...")
sharingDraft = map3D.getWebLayerSharingDraft(args.serverType,
args.serviceType,
args.service)
sharingDraft.summary = args.sharingDraftAttr['summary']
sharingDraft.tags = args.sharingDraftAttr['tags']
sharingDraft.description = args.sharingDraftAttr['description']
sharingDraft.overwriteExistingService = args.sharingDraftAttr[
'overwriteExistingService']
sharingDraft.exportToSDDraft(args.sddraftOutputFilename)
arcpy.server.StageService(args.sddraftOutputFilename,
args.sdOutputFilename)
outServiceDefinitions = arcpy.server.UploadServiceDefinition(
**args.uploadAttr)
for i in range(outServiceDefinitions.outputCount):
args.serviceDefinitions[args.serviceDefinitionFeatures[
i]] = outServiceDefinitions.getOutput(i)
print('Successfully uploaded GIS service!')
print('Service definitions:')
pprint(args.serviceDefinitions)
with open(args.serviceDefinitionsJSON, 'w', encoding='utf-8') as f:
json.dump(args.serviceDefinitions, f, ensure_ascii=False)
except Exception as e:
print(f'\n----------\nError in UploadGIS_Service:\n {e}\n')
print('-' * 50)
def publish_results():
# Sign in to portal
arcpy.SignInToPortal("https://esriau.maps.arcgis.com/","","")
# Set output file names
outdir = os.path.join(os.path.dirname(__file__), 'sd')
service = "FS_WazeEllipses_KL"
sddraft_filename = service + ".sddraft"
sddraft_output_filename = os.path.join(outdir, sddraft_filename)
# Reference map to publish
aprx = arcpy.mp.ArcGISProject(os.path.join(os.path.dirname(__file__), 'WazeForPublishing.aprx'))
m = aprx.listMaps("Waze For Publishing")[0]
# Create FeatureSharingDraft and set service properties
sharing_draft = m.getWebLayerSharingDraft("HOSTING_SERVER", "FEATURE", service)
sharing_draft.overwriteExistingService = "True"
sharing_draft.portalFolder = "Waze"
sharing_draft.summary = "Waze Ellipses created from a python script that pulls latest Waze data for Kuala Lumpur"
sharing_draft.tags = "Waze, BGT, Kuala Lumpur"
sharing_draft.description = "Latest Waze Jam incidents downloaded and a density clustering performed. Ellipses generated around the main clusters"
sharing_draft.credits = "Waze CCP"
sharing_draft.useLimitations = "Demo Purposes Only"
# Create Service Definition Draft file
sharing_draft.exportToSDDraft(sddraft_output_filename)
# Stage Service
sd_filename = service + ".sd"
sd_output_filename = os.path.join(outdir, sd_filename)
arcpy.StageService_server(sddraft_output_filename, sd_output_filename)
# Share to portal
print("Uploading Service Definition for publishing")
arcpy.UploadServiceDefinition_server(sd_output_filename, "My Hosted Services", in_override="OVERRIDE_DEFINITION", in_public="PUBLIC")
print("Successfully Uploaded & Published.")
import arcpy
import config
import os
import sys
arcpy.CheckOutExtension("Spatial")
arcpy.env.overwriteOutput = True
# Set workspace and scratch folder
ws = arcpy.env.workspace = config.workspace
sc = arcpy.env.scratchworkspace = config.scratchworkspace
# Sign in Portal
try:
arcpy.SignInToPortal('https://{}/portal/'.format(config.portalname),
config.username, config.password)
print('Signed in to portal')
except:
print("Signin error:", sys.exc_info()[0])
# Set input and output data then run the model
intbx = 'hotspot.tbx'
inputfc = os.path.join('HotSpotData.gdb', 'calls')
try:
inputRaster = arcpy.management.MakeRasterLayer('stowe_elev.tif',
'outRlayer1')
distance_method = 'Manhattan'
arcpy.ImportToolbox(intbx)
result_item = arcpy.hotspotscript_hotspot(inputfc, inputRaster,
distance_method)
print("Tool runs successfully.")
def one_function_to_rule_them_all(common_info: CommonInfo,
specific_info: SpecificInfo):
'''Calls all the previous functions in appropriate order
Args:
common_info (CommonInfo): Info common to all layers (wfh and operator)
specific_info (SpecificInfo): Info specific to a particular layer (wfh or operator)
Raises:
NotImplementedError: If a method other than 'wfh' or 'operator' is provided
'''
print('Getting AGOL references...')
password = getpass('Enter Password: '******'Cleaning up scratch areas...')
if arcpy.Exists(str(common_info.scratch_gdb)):
print(f'Deleting existing {common_info.scratch_gdb}...')
arcpy.management.Delete(str(common_info.scratch_gdb))
print(f'Creating {common_info.scratch_gdb}...')
arcpy.management.CreateFileGDB(str(common_info.scratch_gdb.parent),
str(common_info.scratch_gdb.name))
dhrm_data = get_dhrm_dataframe(common_info.employee_data_path)
if specific_info.method == 'wfh':
get_wfh_eins(specific_info.data_source, dhrm_data,
common_info.csv_path)
elif specific_info.method == 'operator':
get_operator_eins(specific_info.data_source, dhrm_data,
common_info.csv_path)
else:
raise NotImplementedError(
f'Method {specific_info.method} not recognized...')
geocode_points(
str(common_info.csv_path),
str(common_info.geocoded_points_path),
str(common_info.locator_path),
'real_addr',
'real_zip',
)
hex_bin(str(common_info.geocoded_points_path),
str(common_info.hex_fc_path),
str(common_info.hexes_fc_path),
simple_count=specific_info.simple_summary,
within_table=str(common_info.within_table_path))
remove_single_count_hexes(str(common_info.hexes_fc_path),
str(common_info.trimmed_hex_fc_path))
sharing_layer, sharing_map = add_layer_to_map(
str(common_info.project_path), common_info.map_name,
str(common_info.trimmed_hex_fc_path))
update_agol_feature_service(sharing_map, sharing_layer, sd_item, fs_item,
specific_info)
print(f'Current date and time: {dt.datetime.now()}')
# Start timer and print start time in UTC
start_time = time.time()
readable_start = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
print("The script start time is {}".format(readable_start))
# Set variables, get AGOL username and password
portal_url = arcpy.GetActivePortalURL()
print(portal_url)
user = getpass.getpass(prompt=' Enter arcgis.com username:\n')
pw = getpass.getpass(prompt=' Enter arcgis.com password:\n')
arcpy.SignInToPortal(portal_url, user, pw)
del pw
# Updated count numbers are copied from table at 'https://coronavirus.utah.gov/case-counts/'
# CSV file with updates should be named 'COVID_Case_Counts_latest.csv'
# Update this 'work_dir' variable with the folder you store the updated CSV in
work_dir = r'C:\COVID19'
updates = pd.read_csv(os.path.join(work_dir, 'COVID_Case_Counts_latest.csv'))
updates.sort_values('Jurisdiction', inplace=True)
# TEST layer
# counts_service = r'https://services1.arcgis.com/99lidPhWCzftIe9K/ArcGIS/rest/services/EMN_Cases_by_LHD_TEST_v3/FeatureServer/0'
# TEST table
# counts_by_day = r'https://services1.arcgis.com/99lidPhWCzftIe9K/arcgis/rest/services/EMN_Cases_by_LHD_by_Day_TEST_v3/FeatureServer/0'
def setup(self):
"""
Sets up the Auditor by logging into the ArcGIS org, getting all the items and folders, and reading in the
metatable(s). To be called in __init__().
In case of multiple calls (ie, for retry()), all data are re-instantiated/initialized.
"""
#: temp_dir used by fixes.metadata_fix() to hold xml of sde metadata
temp_dir = Path(arcpy.env.scratchFolder, 'auditor')
if not temp_dir.exists():
if self.verbose:
print(f'Creating temp directory {temp_dir}...')
temp_dir.mkdir()
self.log.info(f'Logging into {credentials.ORG} as {credentials.USERNAME}')
self.gis = arcgis.gis.GIS(credentials.ORG, credentials.USERNAME, credentials.PASSWORD)
#: Make sure ArcGIS Pro is properly logged in
arcpy.SignInToPortal(arcpy.GetActivePortalURL(), credentials.USERNAME, credentials.PASSWORD)
user_item = self.gis.users.me # pylint: disable=no-member
#: Build dict of folders. 'None' gives us the root folder.
if self.verbose:
print(f'Getting {self.username}\'s folders...')
folders = {None: None}
for folder in user_item.folders:
folders[folder['id']] = folder['title']
self.items_to_check = [] #: Clear this out again in case retry calls setup() multiple times.
#: Get item object and it's correspond folder for each relevant item
if self.verbose:
print('Getting item objects...')
#: User-provided list
if self.item_ids:
for item_id in self.item_ids:
item = self.gis.content.get(item_id) # pylint: disable=no-member
if not item:
raise ValueError(f'Item {item_id} not found')
self.items_to_check.append(item)
try:
self.itemid_and_folder[item.itemid] = folders[item.ownerFolder]
except KeyError:
raise ValueError(f'Folder id {item.ownerFolder} not found (wrong user?)')
#: No user-provided item ids, get all hosted feature services in every folder
else:
for _, name in folders.items():
for item in user_item.items(name, 1000):
if item.type == 'Feature Service':
self.items_to_check.append(item)
self.itemid_and_folder[item.itemid] = name
#: Read the metatable into memory as a dictionary based on itemid.
#: Getting this once so we don't have to re-read every iteration
if self.verbose:
print('Getting metatable info...')
self.metatable = Metatable()
sgid_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'Authoritative']
agol_fields = ['TABLENAME', 'AGOL_ITEM_ID', 'AGOL_PUBLISHED_NAME', 'CATEGORY']
self.metatable.read_metatable(self.sgid_table, sgid_fields)
self.metatable.read_metatable(self.agol_table, agol_fields)
if self.metatable.duplicate_keys:
raise RuntimeError(f'Duplicate AGOL item IDs found in metatables: {self.metatable.duplicate_keys}')
#: Get the groups
if self.verbose:
print('Getting groups...')
groups = self.gis.groups.search('title:*') # pylint: disable=no-member
self.groups_dict = {g.title: g.id for g in groups}
### Title: SEA1_AttachmentViewer_AutoPublisher.py
### Description: This Python script publishes a refreshed hosted feature layer to ArcGIS Online, which is then
### consumed in Attachment Viewer. This script is intended to be triggered on a nightly basis in
### order for new data to be visible from the previous day.
### Company: Foresite Group
### Author: Darren Foster
### MUST RUN IN PYTHON 3.6 ###
import arcpy
import os
from arcgis.gis import GIS
from datetime import datetime
arcpy.env.overwriteOutput = True
### Sign in to portal (In this case, sign into ArcGIS Online) ###
arcpy.SignInToPortal('https://www.arcgis.com', '$$$$$', '$$$$$')
### Set constants ###
OUT_DIR = r'\\fginc-file\GIS\Foster\AutoPublishing\SEA1_AttachmentViewer'
SERVICE = 'SEA1_AttViewer_HFS'
SD_DRAFT = SERVICE + '.sddraft'
SD_FILENAME = SERVICE + '.sd'
LOG = r'\\fginc-file\GIS\Foster\AutoPublishing\SEA1_AttachmentViewer\Logger.txt'
### Construct a GIS object ###
gis = GIS('https://www.arcgis.com', '$$$$$', '$$$$$')
### Paths to the Service Definition and the Draft ###
sdDraftOutput = os.path.join(OUT_DIR, SD_DRAFT)
sdPath = os.path.join(OUT_DIR, SD_FILENAME)
# changes the os directory using the full path name
os.chdir(path)
# store current working directory
folder_dir = os.getcwd()
# find and store all files with .shp ending
shps = glob.glob(folder_dir + '\*.shp')
for shp_path in shps:
m.addDataFromPath(shp_path)
aprx.save()
arcpy.SignInToPortal('https://www.arcgis.com', 'REPLACE THIS: USER_NAME', 'REPLACE THIS: USER_PASSWORD')
# Set output file names
# where to store the ssd files
outdir = r"REPLACE THIS: LOCATION TO STORE THE .SSD FILES"
# what to name it once published
service = "REPLACE THIS: NAME OF FEATURE LAYER"
sddraft_filename = service + ".sddraft"
sddraft_output_filename = os.path.join(outdir, sddraft_filename)
# Reference map to publish
# get the current map document
aprx = arcpy.mp.ArcGISProject("CURRENT")
def CreateSDFile(configFile):
try:
with open(configFile) as json_config:
config = json.load(json_config)
except Exception as e:
arcpy.AddMessage("Failed to load config.")
return
try:
toolbox = config["toolbox"]
alias = config["alias"]
toolName = config["toolName"]
toolArgs = config["toolArgs"]
sddraft = config["sdDraft"]
sd = config["sd"]
portalurl = config["portalURL"]
portalusername = config["portalUsername"]
portalpassword = config["portalPassword"]
serverURL = config["serverURL"]
serviceName = config["serviceName"]
serverType = config["serverType"]
connectionFilePath = config["connectionFilePath"]
copyDataToServer = config["copyDataToServer"]
folderName = config["folderName"]
summary = config["summary"]
tags = config["tags"]
executionType = config["executionType"]
resultMapServer = config["resultMapServer"]
showMessages = config["showMessages"]
maximumRecords = config["maximumRecords"]
minInstances = config["minInstances"]
maxInstances = config["maxInstances"]
maxUsageTime = config["maxUsageTime"]
maxWaitTime = config["maxWaitTime"]
maxIdleTime = config["maxIdleTime"]
constantValues = config["constantValues"]
except KeyError as keyErr:
arcpy.AddMessage(f"Config file missing value: {keyErr}")
return
except Exception as e:
arcpy.AddMessage(f"Error occured in retreiving config values: {e}")
return
arcpy.AddMessage("Successfuly read all configuration values.")
arcpy.ImportToolbox(toolbox, alias)
arcpy.AddMessage(arcpy.GetMessages(0))
customToolMethod = getattr(arcpy, f"{toolName}_{alias}")
result = customToolMethod(*toolArgs)
arcpy.AddMessage(arcpy.GetMessages(0))
arcpy.SignInToPortal(portalurl, portalusername, portalpassword)
arcpy.AddMessage(arcpy.GetMessages(0))
analyzeMessages = arcpy.CreateGPSDDraft(
result,
sddraft,
serviceName,
server_type=serverType,
connection_file_path=connectionFilePath,
copy_data_to_server=copyDataToServer,
folder_name=folderName,
summary=summary,
tags=tags,
executionType=executionType,
resultMapServer=resultMapServer,
showMessages=showMessages,
maximumRecords=maximumRecords,
minInstances=minInstances,
maxInstances=maxInstances,
maxUsageTime=maxUsageTime,
maxWaitTime=maxWaitTime,
maxIdleTime=maxIdleTime,
constantValues=constantValues)
arcpy.AddMessage(arcpy.GetMessages(0))
# Stage and upload the service if the sddraft analysis did not
# contain errors
if analyzeMessages['errors'] == {}:
# Execute StageService
arcpy.StageService_server(sddraft, sd)
arcpy.AddMessage(arcpy.GetMessages(0))
# Execute UploadServiceDefinition
# Use URL to a federated server
arcpy.UploadServiceDefinition_server(sd, serverURL)
arcpy.AddMessage(arcpy.GetMessages(0))
else:
# If the sddraft analysis contained errors, display them
print(analyzeMessages['errors'])
gis = GIS(Config.PORTAL_URL, Config.PORTAL_USER, Config.PORTAL_PASSWORD)
if update_service_definition(gis, sdname, service_name):
print("Service definition has been updated.")
else:
print("Uploading definition using \"%s\" %s" % (ags_file, folder))
# Upload the service definition to SERVER
# In theory everything needed to publish the service is already in the SD file.
# https://pro.arcgis.com/en/pro-app/latest/tool-reference/server/upload-service-definition.htm
# You can override permissions, ownership, groups here too.
try:
# In theory ags_file could be Config.SERVER_URL but then where does it get authenticated?
# arcpy.server.UploadServiceDefinition(sdname, ags_file, in_startupType="STARTED")
# in_startupType HAS TO BE "STARTED" else no service is started on the SERVER.
rval = arcpy.SignInToPortal(Config.PORTAL_URL, Config.PORTAL_USER,
Config.PORTAL_PASSWORD)
rval = arcpy.server.UploadServiceDefinition(
sdname, Config.SERVER_URL, in_startupType="STARTED")
except Exception as e:
print("Upload failed.", e)
service = Config.SERVER_URL + '/rest/services/'
if folder:
service += folder + "/"
service += service_name + '/MapServer'
print("Map Image published successfully - ", service)
fetch_tile(service, "C:/temp/output.png")
SERVER = "1.2.3.4"
PORT = "25"
FROM = "*****@*****.**"
MAILDOMAIN = '@ME.COM'
# Data Steward getting the email. Needs to be their email address...without @nps.gov at the end
userList=["BIG_FOOT"]
# get a list of usernames from the list of named tuples returned from ListUsers
userNames = [u for u in userList]
# take the userNames list and make email addresses by appending the appropriate suffix.
emailList = [name + MAILDOMAIN for name in userNames]
TO = emailList
# Grab date for the email
DATE = d
# Sign in to portal
arcpy.SignInToPortal('https://www.arcgis.com', 'USER', 'PASSWORD')
# Set output file names
outdir = r"C:\PRODUCTION\GRSM_EXOTICS"
sd_filename = service + ".sd"
sd_output_filename = os.path.join(outdir, sd_filename)
sddraft_filename = service + ".sddraft"
sddraft_output_filename = os.path.join(outdir, sddraft_filename)
#Delete any left over SD files from failed previous run
try:
os.remove(sd_output_filename)
print("Successfully deleted ", sd_output_filename)
except:
print("Error while deleting file ", sd_output_filename, ", perhaps it doesn't exist")
try:
os.remove(sddraft_output_filename)
sdPath = r"C:\temp\UC2018\PublishingSamples\Output\%s.sd" % (serviceName)
restEndPoint = "https://services.arcgis.com/EguFTd9xPXEoDtC7/arcgis/rest/services/"
queryCriteria = "where=&objectIds=1&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=&returnGeometry=true&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&returnIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnDistinctValues=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&quantizationParameters=&sqlFormat=none"
baseJSONFile = r'baselines\base_' + serviceName + '_queryResult_Id1.json'
# list the AGO or enterprise url and credentials here
portalURL = r'https://www.arcgis.com'
cred_detail = []
with open("secure/AGO_pass.txt") as f:
for line in f:
cred_detail.append(line.splitlines())
username = cred_detail[0][0]
password = cred_detail[1][0]
# Sign into AGO and set as active portal
arcpy.SignInToPortal(portalURL, username, password)
# Maintain a reference of an ArcGISProject object pointing to your project
aprx = arcpy.mp.ArcGISProject(aprxPath)
# Maintain a reference of a Map object pointing to your desired map
m = aprx.listMaps('Map1')[0]
''' the first step to automate the publishing of a map, layer, or list of layers to a hosted web layer using ArcPy, in new object-oriented approach.
Use "getWebLayerSharingDraft" method to create a FeatureSharingDraft object (reference: http://pro.arcgis.com/en/pro-app/arcpy/sharing/featuresharingdraft-class.htm)
Syntax = getWebLayerSharingDraft (server_type, service_type, service_name, {...})
Then to a Service Definition Draft (.sddraft) file with "exportToSDDraft" method.
Syntax = exportToSDDraft (out_sddraft)
'''
def process(self):
#: Set up paths and directories
feature_service_name = secrets.FEATURE_SERVICE_NAME
temp_csv_dir = os.path.join(arcpy.env.scratchFolder, 'fleet')
temp_fc_path = os.path.join(arcpy.env.scratchGDB, feature_service_name)
sddraft_path = os.path.join(arcpy.env.scratchFolder,
f'{feature_service_name}.sddraft')
sd_path = sddraft_path[:-5]
paths = [temp_csv_dir, temp_fc_path, sddraft_path, sd_path]
for item in paths:
if arcpy.Exists(item):
self.log.info(f'Deleting {item} prior to use...')
arcpy.Delete_management(item)
os.mkdir(temp_csv_dir)
if not secrets.KNOWNHOSTS or not os.path.isfile(secrets.KNOWNHOSTS):
raise FileNotFoundError(
f'known_hosts file {secrets.KNOWNHOSTS} not found. Please create with ssh-keyscan.'
)
#: Download all the files in the upload folder on sftp to temp_csv_dir
self.log.info(
f'Downloading all files from {secrets.SFTP_HOST}/upload...')
connection_opts = pysftp.CnOpts(knownhosts=secrets.KNOWNHOSTS)
with pysftp.Connection(secrets.SFTP_HOST,
username=secrets.SFTP_USERNAME,
password=secrets.SFTP_PASSWORD,
cnopts=connection_opts) as sftp:
sftp.get_d('upload', temp_csv_dir, preserve_mtime=True)
#: Get the latest file
source_path, source_date = self.get_latest_csv(temp_csv_dir,
previous_days=7)
self.log.info(
f'Converting {source_path} to feature class {temp_fc_path}...')
wgs84 = arcpy.SpatialReference(4326)
result = arcpy.management.XYTableToPoint(source_path,
temp_fc_path,
'LONGITUDE',
'LATITUDE',
coordinate_system=wgs84)
self.log.debug(result.getMessages())
try_count = 1
while True:
try:
self.log.info(f'Updating service, try {try_count} of 3...')
self.log.info(
f'Connecting to AGOL as {secrets.AGOL_USERNAME}...')
gis = arcgis.gis.GIS('https://www.arcgis.com',
secrets.AGOL_USERNAME,
secrets.AGOL_PASSWORD)
sd_item = gis.content.get(secrets.SD_ITEM_ID)
self.log.info('Getting map and layer...')
arcpy.SignInToPortal(arcpy.GetActivePortalURL(),
secrets.AGOL_USERNAME,
secrets.AGOL_PASSWORD)
layer, fleet_map = self.get_map_layer(secrets.PROJECT_PATH,
temp_fc_path)
#: draft, stage, update, publish
self.log.info('Staging and updating...')
self.update_agol_feature_service(fleet_map, layer,
feature_service_name,
sddraft_path, sd_path,
sd_item)
#: Update item description
self.log.info('Updating item description...')
feature_item = gis.content.get(secrets.FEATURES_ITEM_ID)
year = source_date[:4]
month = source_date[4:6]
day = source_date[6:]
description = f'Vehicle location data obtained from Fleet; updated on {year}-{month}-{day}'
feature_item.update(
item_properties={'description': description})
except Exception as e:
err_msg = f'Error on attempt {try_count} of 3; retrying.'
self.log.exception(err_msg)
#: Fail for good if 3x retry fails, otherwise increment, sleep,
#: and retry
if try_count > 3:
err_msg = 'Connection errors; giving up after 3 retries'
self.log.exception(err_msg)
raise e
sleep(try_count**2)
try_count += 1
continue
#: If we haven't gotten an error, break out of while True.
break
password = config.get('ago', 'password')
proxy = config.get('ago', 'proxy')
# Set arcpy options
arcpy.SetLogHistory(False)
arcpy.env.workspace = os.getcwd()
arcpy.env.overwriteOutput = True
# Define where the .aprx files are located
prjPath = config.get('local', 'prjPath')
# Get a list of the .aprx files to stage the services
aprx_files = glob.glob(prjPath)
# For some reason SDDraft tool needs an explicit arcpy AGO login now
arcpy.SignInToPortal(portal, user, password)
# Define the AGO organization to connect to
gis = GIS(portal, user, password, proxy_host=proxy, proxy_port=8080)
# Files have to be created sequentially because ArcGIS Pro doesn't like multiprocessing
for aprx in aprx_files:
prj = arcpy.mp.ArcGISProject(aprx)
prj_mp = prj.listMaps()[0]
try:
if checks(prj_mp):
print(prj_mp.name)
create_sd(prj_mp.name)
logger.info('{} service definition successfully created.'.format(
prj_mp.name))
else:
"\"Viernes\",\"Sabado\",\"Domingo\")\n day = inicio.weekday()\n daystring = weekDay"
+
"s[day]\n label = \'El \' + daystring + \' lloverá \' + str(mm) + \'mm en esta región\'\n return(l"
+ "abel)")
#import arcgis project aprx
aprx = arcpy.mp.ArcGISProject(project_folder +
r'\pronosticos\pronosticos.aprx')
#save project
print('saving project')
aprx.save()
print('project saved')
#connect to arcgis portal
arcpy.SignInToPortal("https://sig.icafe.cr/portal", "adminicafe", "icafesig")
#Create sddraft
print("Creating sddraft")
m = aprx.listMaps('Map')[0]
service = "pronosticos_image"
sddraft_filename = service + ".sddraft"
sddraft_output_filename = os.path.join(project_folder, sddraft_filename)
sharing_draft = m.getWebLayerSharingDraft("FEDERATED_SERVER", "MAP_IMAGE",
service)
sharing_draft.federatedServerUrl = 'https://sig.icafe.cr/server'
sharing_draft.summary = "Pronosticos de precipitación"
sharing_draft.tags = "Pronosticos Precipitación"
sharing_draft.overwriteExistingService = True
sharing_draft.portalFolder = 'Pronosticos'
import arcpy, os
from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection
print("INITIALIZING SCRIPT EXECUTION")
arcpy.env.overwriteOutput=1
#port = 'https://www.arcgis.com'
port = arcpy.GetParameterAsText(0)
#user = '******'
user = arcpy.GetParameterAsText(1)
#passw = '#'
passw = arcpy.GetParameterAsText(2)
arcpy.SignInToPortal(port, user, passw)
gis = GIS(port, user, passw)
open_data = gis.groups.search('title:Apex Open Data')
group = open_data[0]
direct = arcpy.GetParameterAsText(3)
#direct = r"C:\Users\Jlong\Documents\ArcGIS\Projects\PublishAGOLdata"
mapdoc = arcpy.GetParameterAsText(4)
#mapdoc = r"C:\Users\Jlong\Documents\ArcGIS\Projects/PublishAGOLdata/PublishAGOLdata.aprx"
agol_serv_con = 'My Hosted Services'
aprx = arcpy.mp.ArcGISProject(mapdoc)
'''
def main():
# -----------------------------------------------------------------------------
# Constants
# -----------------------------------------------------------------------------
DEFAULT_AGO_PORTAL_URL = "https://governmentofbc.maps.arcgis.com" # Can also reference a local portal
DEFAULT_AGO_PORTAL_USER = "******"
DEFAULT_AGO_FOLDER = "ago_feature_layers_hosted_active"
# -----------------------------------------------------------------------------
# Parameters
# -----------------------------------------------------------------------------
argParser = argparse.ArgumentParser(
description=
"Publishes Feature Classes from a ArcGIS Pro project and updates or creates new Feature Layer AGO Items"
)
argParser.add_argument(
'-user',
dest='user',
action='store',
default=DEFAULT_AGO_PORTAL_USER,
required=False,
help='the ArcGIS Online User to publish AGO Items with')
argParser.add_argument('-pwd',
dest='password',
action='store',
default=None,
required=True,
help='the ArcGIS Online User password')
argParser.add_argument(
'-path',
dest='prjPath',
action='store',
default=None,
required=True,
help='the full path and name to the ArcGIS Pro Project')
argParser.add_argument('-portal',
dest='portal',
action='store',
default=DEFAULT_AGO_PORTAL_URL,
required=False,
help='AGO portal URL')
argParser.add_argument(
'-folder',
dest='shrFolder',
action='store',
default=DEFAULT_AGO_FOLDER,
required=False,
help=
'The ID of the folder that contains all hosted items managed by this script'
)
argParser.add_argument(
'-fcs',
dest='fcs',
action='store',
default=None,
required=True,
help=
'The text file that contains a list of feature classes and update times i.e. E:/apps_data/userdata/scripts/public_2_ago/layers_for_ago2.txt'
)
try:
args = argParser.parse_args()
except argparse.ArgumentError as e:
argParser.print_help()
sys.exit(1)
start_time = time.time()
now = time.ctime(int(start_time))
print('Start Time: ' + now)
print('Connecting to {}'.format(args.portal))
arcpy.SignInToPortal(args.portal,
username=args.user,
password=args.password)
gis = GIS(url=args.portal,
username=args.user,
password=args.password,
verify_cert=False)
prjPath = args.prjPath.replace('/', '\\')
print('Open ArcPro Project ' + prjPath)
prj = arcpy.mp.ArcGISProject(prjPath)
print("Logged in as " + str(gis.properties.user.username))
query_dict = {
'f': 'json',
'username': args.user,
'password': args.password,
'referer': args.portal
}
url = args.portal + "/sharing/rest/generateToken"
r = requests.post(url + "?f=json", params=query_dict)
t = json.loads(r.text)
token = t['token']
print('token: ' + token)
if "token" not in r.text:
print('cound not get token, error')
sys.exit(1)
fn = args.fcs
with open(fn, 'r') as f:
for line in f:
print('')
print('***********************************************')
x = line.split(";")
cron = x[0]
print('cron expression - ' + cron)
stalemin = int(x[1])
print('Stale Minutes - ' + str(stalemin))
title = x[2]
print('Title - ' + title)
metadataurl = x[3]
print('Metadata Url - ' + metadataurl)
agourl = x[4]
print('AGO Item URL - ' + agourl)
#find GUID of ITEM
guid = agourl.split('id=')[1]
#check to see the modified date of the AGO item
#agoitem = gis.content.get(guid)
print("search for " + title + ' owner - ' + args.user)
#agoitemlist = gis.content.search(query="title:"+ title +" AND owner:" + args.user, item_type='Feature Layer')
agoitemlist = gis.content.search(query="id:" + guid +
" AND owner:" + args.user,
item_type='Feature Layer')
if agoitemlist:
agoitem = agoitemlist[0]
print('found ' + agoitem.url)
else:
print('not found ' + title)
if agoitem:
#print (agoitem)
#print (agoitem.modified)
#print (agoitem.url)
# 2: request the json data for the feature
query_dict = {"f": "json", "token": token}
jsonResponse = requests.get(agoitem.url + "/0",
params=query_dict)
# lastEditDate is in the editingInfo section of the json response
# to access other sections, change "editingInfo" to the section name ("types" for example)
# using OrderedDict keeps the file ordered as sent by server, but is not necessary
#print(jsonResponse.text)
resptxt = jsonResponse.text
if resptxt.find("editingInfo") > 0:
t = json.loads(jsonResponse.text)
lastEditDate = t['editingInfo']['lastEditDate']
else:
jsonResponse = requests.get(agoitem.url + "/1",
params=query_dict)
#print(jsonResponse.text)
resptxt = jsonResponse.text
if resptxt.find("editingInfo") > 0:
t = json.loads(jsonResponse.text)
lastEditDate = t['editingInfo']['lastEditDate']
else:
jsonResponse = requests.get(agoitem.url + "/2",
params=query_dict)
resptxt = jsonResponse.text
if resptxt.find("editingInfo") > 0:
t = json.loads(jsonResponse.text)
lastEditDate = t['editingInfo']['lastEditDate']
else:
jsonResponse = requests.get(agoitem.url + "/3",
params=query_dict)
resptxt = jsonResponse.text
if resptxt.find("editingInfo") > 0:
t = json.loads(jsonResponse.text)
lastEditDate = t['editingInfo']['lastEditDate']
print(lastEditDate)
if not lastEditDate:
lastEditDate = agoitem.modified
print(
'using AGO ITEM modified date as feature service edit date is null'
)
editTime = lastEditDate / 1000
print("Last Edited: " +
time.strftime('%c', time.localtime(editTime)))
converted_d1 = datetime.datetime.fromtimestamp(
round(lastEditDate / 1000))
current_time = datetime.datetime.now()
#print (current_time)
#print (converted_d1)
dateage = current_time - converted_d1
print(dateage)
age = int(dateage.total_seconds() / 60)
print(age)
diff = stalemin - age
if diff < 0:
print("Older than stale minutes so running update " +
str(diff))
# run PRO->SD->feature layer steps
publish2ago(DEFAULT_AGO_FOLDER, agoitem, gis, metadataurl,
prjPath, prj, args.user, guid)
else:
print("Younger than stale minutes so skipping update " +
str(diff))
else:
print(guid + " Not found in AGO - " + title)
import arcpy
# ArcGIS Online にサイン イン (自身のArcGIS Online のユーザー名、パスワードを入力)
arcpy.SignInToPortal("https://arcgis.com", "ユーザー名", "パスワード")
# プロジェクト ファイルのオブジェクト取得
aprx = arcpy.mp.ArcGISProject("CURRENT")
# マップとレイヤーを取得
map = aprx.listMaps()[0]
lyrs = map.listLayers()
# 作成するサービス定義ドラフトファイル、サービス定義ファイルのファイル名を設定
sddraft_output = r"C:\data\output\sample.sddraft"
sd_output = r"C:\data\output\sample.sd"
# サービス定義ドラフトの作成
sharing_draft = map.getWebLayerSharingDraft("HOSTING_SERVER", "FEATURE",
"Sample", lyrs)
sharing_draft.exportToSDDraft(sddraft_output)
# サービス定義ファイルの作成（サービスのステージング ツールの使用）
arcpy.StageService_server(sddraft_output, sd_output)
# ArcGIS Online にアップロード（サービス定義のアップロード ツールの使用）
arcpy.UploadServiceDefinition_server(sd_output, "HOSTING_SERVER")
with arcpy.EnvManager(
scratchWorkspace=
r"C:\Users\ASF\Documents\COVID19\Disasters\Watermaps\Watermaps.gdb",
workspace=
r"C:\Users\ASF\Documents\COVID19\Disasters\Watermaps\Watermaps.gdb"):
try:
arcpy.AID.AIDISDP(md_rtc, aid_rtc, None)
except:
print("AID errors generated and ignored.")
pass
print('RTC AID package complete.')
# update image services
import keyring
pw = keyring.get_password("portal_creds", "hkristenson_ASF")
arcpy.SignInToPortal(r'https://asf-daac.maps.arcgis.com/', 'hkristenson_ASF',
pw)
# print('Updating Watermap Extent Image Service...')
# arcpy.AID.MAIDIS("asf-daac", "Update Service", "test", "None", "HKH_Watermaps_Clipped", None, aid_wm, "Dedicated Instance", "Watermap Extent products generated from Sentinel-1 SAR imagery over flood-prone regions in the Hindu Kush Himalayan (HKH) region for the 2022 monsoon season, generated by ASF.", "Imagery products processed by ASF DAAC HyP3 2022 using GAMMA software. Contains modified Copernicus Sentinel data 2022, processed by ESA.", '', False, False, True, None, None, None, None)
# print('Watermap Extent Image Service updated.')
print('Updating Watermap Extent Image Service...')
arcpy.AID.MAIDIS(
"asf-daac", "Update Service", "HKH", "None", "HKH_Watermap_Extent", None,
aid_wm, "Dedicated Instance",
"Watermap Extent products generated from Sentinel-1 SAR imagery over flood-prone regions in the Hindu Kush Himalayan (HKH) region for the 2022 monsoon season, generated by ASF.",
"Imagery products processed by ASF DAAC HyP3 2022 using GAMMA software. Contains modified Copernicus Sentinel data 2022, processed by ESA.",
'', False, False, True, None, None, None, None)
print('Watermap Extent Image Service updated.')
print('Updating RGB Image Service...')
arcpy.AID.MAIDIS(
"asf-daac", "Update Service", "HKH", "None", "HKH_RGB", None, aid_rgb,
"\n Information Technology " \
" \n GIS Administrator " \
"\n P: 241-3409-7515 "
message = """From: {0}\r\nTo: {1}\r\nSubject: {2}\r\n\
{3}\r\n\n\n {4}\r\n\n\n {5}""".format(FROM, ", ".join(TO), subject,
header_mesg, text, footer)
server = smtplib.SMTP(SERVER)
server.sendmail(FROM, TO, message)
server.quit()
arcpy.env.overwriteOutput = 1
arcpy.SignInToPortal('https://www.arcgis.com', 'jlong_apexnc', '#')
projdoc = r"C:\Users\Jlong\Documents\ArcGIS\Projects\RecollectPublishServicesLGDB\RecollectPublishServicesLGDB.aprx"
direct = r"C:\Users\Jlong\Documents\ArcGIS\Projects\RecollectPublishServicesLGDB"
def email(x):
auto_email(["*****@*****.**"],
subject="ERROR DETECTED: Recollect Data "
"Import/Ovewrite/Dataupload"
"Script",
text=str(x))
def create_service_definition(map_proj, sname, mpname, proj_dir, weblyrname):
agol_serv_con = 'My Hosted Services'
