def generate_data_stores_inventory(included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
config_dir=default_config_dir):
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
if len(env_names) == 0:
raise RuntimeError('No environments specified!')
for env_name in env_names:
env = user_config['environments'][env_name]
ags_instances = superfilter(env['ags_instances'].keys(),
included_instances, excluded_instances)
log.info('Listing data stores on ArcGIS Server instances {}'.format(
', '.join(ags_instances)))
for ags_instance in ags_instances:
ags_instance_props = env['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
data_stores = list_data_stores(server_url,
token,
session=session)
for data_store in data_stores:
yield dict(env_name=env_name,
ags_instance=ags_instance,
**data_store)
def cleanup_instance(
ags_instance,
env_name,
config,
user_config
):
configured_services = config['services']
service_folder = config.get('service_folder')
log.info(
'Cleaning up unused services on environment {}, ArcGIS Server instance {}, service folder {}'
.format(env_name, ags_instance, service_folder)
)
ags_instance_props = user_config['environments'][env_name]['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get('proxies')
with create_session(server_url, proxies=proxies) as session:
existing_services = list_services(server_url, token, service_folder, session=session)
services_to_remove = [service for service in existing_services if service['serviceName'] not in configured_services]
log.info(
'Removing {} services: {}'
.format(
len(services_to_remove),
', '.join((service['serviceName'] for service in services_to_remove))
)
)
for service in services_to_remove:
delete_service(server_url, token, service['serviceName'], service_folder, service['type'], session=session)
def test_services(included_services=asterisk_tuple,
excluded_services=empty_tuple,
included_service_folders=asterisk_tuple,
excluded_service_folders=empty_tuple,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
warn_on_errors=False,
config_dir=default_config_dir):
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
if len(env_names) == 0:
raise RuntimeError('No environments specified!')
for env_name in env_names:
env = user_config['environments'][env_name]
ags_instances = superfilter(env['ags_instances'].keys(),
included_instances, excluded_instances)
log.info('Testing services on ArcGIS Server instances {}'.format(
', '.join(ags_instances)))
for ags_instance in ags_instances:
ags_instance_props = env['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
service_folders = list_service_folders(server_url,
token,
session=session)
for service_folder in superfilter(service_folders,
included_service_folders,
excluded_service_folders):
for service in list_services(server_url,
token,
service_folder,
session=session):
service_name = service['serviceName']
service_type = service['type']
if superfilter((service_name, ), included_services,
excluded_services):
test_data = test_service(server_url,
token,
service_name,
service_folder,
service_type,
warn_on_errors,
session=session)
yield dict(env_name=env_name,
ags_instance=ags_instance,
service_folder=service_folder,
service_name=service_name,
service_type=service_type,
**test_data)
def make_sites_editable(ags_instances, env_name, user_config, initial_site_modes):
for ags_instance in ags_instances:
ags_instance_props = user_config['environments'][env_name]['ags_instances'][ags_instance]
site_mode = ags_instance_props.get('site_mode')
if site_mode:
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get('proxies')
if initial_site_modes[ags_instance] != 'EDITABLE':
with create_session(server_url, proxies=proxies) as session:
set_site_mode(server_url, token, 'EDITABLE', session=session)
def get_site_modes(ags_instances, env_name, user_config):
result = {}
for ags_instance in ags_instances:
ags_instance_props = user_config['environments'][env_name]['ags_instances'][ags_instance]
site_mode = ags_instance_props.get('site_mode')
if site_mode:
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get('proxies')
with create_session(server_url, proxies=proxies) as session:
current_site_mode = get_site_mode(server_url, token, session=session)
result[ags_instance] = current_site_mode
return result
def set_publishing_summary(
user_config,
env_name,
ags_instance,
service_name,
service_folder,
service_type,
timestamp
):
try:
ags_instance_props = user_config['environments'][env_name]['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get('proxies')
with create_session(server_url, proxies=proxies) as session:
item_info = get_service_item_info(
server_url,
token,
service_name,
service_folder,
service_type,
session=session
)
item_info['summary'] = 'Last published by {} on {:%#m/%#d/%y at %#I:%M:%S %p}'.format(
getpass.getuser(),
timestamp
)
set_service_item_info(
server_url,
token,
item_info,
service_name,
service_folder,
service_type,
session=session
)
except StandardError:
log.warning(
'An error occurred while updating timestamp for service {}/{} to ArcGIS Server instance {}'
.format(
service_folder,
service_name,
ags_instance
),
exc_info=True
)
def generate_tokens(self,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
username=None,
password=None,
reuse_credentials=False,
expiration=15):
user_config = get_config('userconfig', self.config_dir)
env_names = superfilter(user_config['environments'].keys(),
included_envs, excluded_envs)
if len(env_names) == 0:
raise RuntimeError('No environments specified!')
if reuse_credentials:
username, password = prompt_for_credentials(username, password)
needs_save = False
for env_name in env_names:
env = user_config['environments'][env_name]
ags_instances = superfilter(env['ags_instances'].keys(),
included_instances, excluded_instances)
log.info(
'Refreshing tokens for ArcGIS Server instances: {}'.format(
', '.join(ags_instances)))
for ags_instance in ags_instances:
ags_instance_props = env['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
new_token = generate_token(server_url,
username,
password,
expiration,
ags_instance,
session=session)
if new_token:
ags_instance_props['token'] = new_token
if not needs_save:
needs_save = True
if needs_save:
set_config(user_config, 'userconfig', self.config_dir)
def restore_site_modes(ags_instances, env_name, user_config, initial_site_modes):
for ags_instance in ags_instances:
ags_instance_props = user_config['environments'][env_name]['ags_instances'][ags_instance]
site_mode = ags_instance_props.get('site_mode')
if site_mode:
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get('proxies')
with create_session(server_url, proxies=proxies) as session:
current_site_mode = get_site_mode(server_url, token, session=session)
if site_mode.upper() == 'INITIAL':
if current_site_mode != initial_site_modes[ags_instance]:
set_site_mode(server_url, token, initial_site_modes[ags_instance], session=session)
elif site_mode.upper() == 'READ_ONLY':
if current_site_mode != 'READ_ONLY':
set_site_mode(server_url, token, 'READ_ONLY', session=session)
elif site_mode.upper() == 'EDITABLE':
if current_site_mode != 'EDITABLE':
set_site_mode(server_url, token, 'EDITABLE', session=session)
else:
log.warn('Unrecognized site mode {}'.format(site_mode))
def restart_services(included_services=asterisk_tuple,
excluded_services=empty_tuple,
included_service_folders=asterisk_tuple,
excluded_service_folders=empty_tuple,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
include_running_services=True,
delay=30,
max_retries=3,
test_after_restart=True,
config_dir=default_config_dir):
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
if len(env_names) == 0:
raise RuntimeError('No environments specified!')
for env_name in env_names:
env = user_config['environments'][env_name]
ags_instances = superfilter(env['ags_instances'].keys(),
included_instances, excluded_instances)
log.info('Restarting services on ArcGIS Server instances {}'.format(
', '.join(ags_instances)))
for ags_instance in ags_instances:
ags_instance_props = env['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
service_folders = list_service_folders(server_url,
token,
session=session)
for service_folder in superfilter(service_folders,
included_service_folders,
excluded_service_folders):
for service in list_services(server_url,
token,
service_folder,
session=session):
service_name = service['serviceName']
service_type = service['type']
if superfilter((service_name, ), included_services,
excluded_services):
if not include_running_services:
status = get_service_status(server_url,
token,
service_name,
service_folder,
service_type,
session=session)
configured_state = status.get(
'configuredState')
if configured_state == 'STARTED':
log.debug(
'Skipping restart of service {}/{} ({}) because its configured state is {} and include_running_services is {}'
.format(service_folder, service_name,
service_type, configured_state,
include_running_services))
continue
restart_service(server_url,
token,
service_name,
service_folder,
service_type,
delay,
max_retries,
test_after_restart,
session=session)
restart_service(server_url,
token,
service_name,
service_folder,
service_type,
delay,
max_retries,
test_after_restart,
session=session)
def find_service_dataset_usages(included_datasets=asterisk_tuple,
excluded_datasets=empty_tuple,
included_users=asterisk_tuple,
excluded_users=empty_tuple,
included_databases=asterisk_tuple,
excluded_databases=empty_tuple,
included_versions=asterisk_tuple,
excluded_versions=empty_tuple,
included_services=asterisk_tuple,
excluded_services=empty_tuple,
included_service_folders=asterisk_tuple,
excluded_service_folders=empty_tuple,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
config_dir=default_config_dir):
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
if len(env_names) == 0:
raise RuntimeError('No environments specified!')
for env_name in env_names:
env = user_config['environments'][env_name]
ags_instances = superfilter(env['ags_instances'].keys(),
included_instances, excluded_instances)
log.info(
'Finding service dataset usages on ArcGIS Server instances {}'.
format(', '.join(ags_instances)))
for ags_instance in ags_instances:
ags_instance_props = env['ags_instances'][ags_instance]
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
service_folders = list_service_folders(server_url,
token,
session=session)
for service_folder in superfilter(service_folders,
included_service_folders,
excluded_service_folders):
for service in list_services(server_url,
token,
service_folder,
session=session):
service_name = service['serviceName']
service_type = service['type']
service_props = dict(env_name=env_name,
ags_instance=ags_instance,
service_folder=service_folder,
service_name=service_name,
service_type=service_type)
if superfilter((service_name, ), included_services,
excluded_services):
for dataset_props in list_service_workspaces(
server_url,
token,
service_name,
service_folder,
service_type,
session=session):
if (superfilter(
(dataset_props['dataset_name'], ),
included_datasets, excluded_datasets)
and superfilter(
(dataset_props['user'], ),
included_users, excluded_users)
and superfilter(
(dataset_props['database'], ),
included_databases,
excluded_databases)
and superfilter(
(dataset_props['version'], ),
included_versions,
excluded_versions)):
yield dict(
chain(service_props.iteritems(),
dataset_props.iteritems()))
def list_service_layer_fields(included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
included_service_folders=asterisk_tuple,
excluded_service_folders=empty_tuple,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_services=asterisk_tuple,
excluded_services=empty_tuple,
warn_on_errors=False,
config_dir=default_config_dir):
import arcpy
arcpy.env.overwriteOutput = True
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
for env_name in env_names:
log.debug(
'Listing service layers and fields for environment {}'.format(
env_name))
env = user_config['environments'][env_name]
for ags_instance in superfilter(env['ags_instances'],
included_instances,
excluded_instances):
ags_instance_props = user_config['environments'][env_name][
'ags_instances'][ags_instance]
ags_connection = ags_instance_props['ags_connection']
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
service_folders = list_service_folders(server_url,
token,
session=session)
for service_folder in superfilter(service_folders,
included_service_folders,
excluded_service_folders):
for service in list_services(server_url,
token,
service_folder,
session=session):
service_name = service['serviceName']
service_type = service['type']
if (service_type == 'MapServer' and superfilter(
(service_name, ), included_services,
excluded_services)):
service_props = dict(env_name=env_name,
ags_instance=ags_instance,
service_folder=service_folder,
service_name=service_name,
service_type=service_type,
ags_connection=ags_connection)
try:
service_manifest = get_service_manifest(
server_url,
token,
service_name,
service_folder,
service_type,
session=session)
service_props[
'mxd_path'] = mxd_path = service_manifest[
'resources'][0]['onPremisePath']
log.info(
'Listing layers and fields for {service_type} service {service_folder}/{service_name} '
'on ArcGIS Server instance {ags_instance} '
'(Connection File: {ags_connection}, MXD Path: {mxd_path})'
.format(**service_props))
if not arcpy.Exists(mxd_path):
raise RuntimeError(
'MXD {} does not exist!'.format(
mxd_path))
mxd = open_mxd(mxd_path)
for layer in list_layers_in_mxd(mxd):
if not ((hasattr(layer, 'isGroupLayer')
and layer.isGroupLayer) or
(hasattr(layer, 'isRasterLayer')
and layer.isRasterLayer)):
layer_name = getattr(
layer, 'longName', layer.name)
try:
layer_props = get_layer_properties(
layer)
except StandardError as e:
log.exception(
'An error occurred while retrieving properties for layer {} in MXD {}'
.format(layer_name, mxd_path))
if not warn_on_errors:
raise
else:
yield dict(
error=
'Error retrieving layer properties: {}'
.format(e.message),
layer_name=layer_name,
**service_props)
continue
try:
if layer_props['is_broken']:
raise RuntimeError(
'Layer\'s data source is broken (Layer: {}, Data Source: {})'
.format(
layer_name,
getattr(
layer,
'dataSource',
'n/a')))
for field_props in get_layer_fields(
layer):
field_props['needs_index'] = not field_props[
'has_index'] and (
field_props[
'in_definition_query']
or field_props[
'in_label_class_expression']
or field_props[
'in_label_class_sql_query']
or field_props[
'field_name']
== layer_props[
'symbology_field']
or field_props[
'field_type']
== 'Geometry')
yield dict(
chain(
service_props.
iteritems(),
layer_props.iteritems(
),
field_props.iteritems(
)))
except StandardError as e:
log.exception(
'An error occurred while listing fields for layer {} in MXD {}'
.format(layer_name, mxd_path))
if not warn_on_errors:
raise
else:
yield dict(
chain(
service_props.
iteritems(),
layer_props.iteritems(
)),
error=
'Error retrieving layer fields: {}'
.format(e.message))
except StandardError as e:
log.exception(
'An error occurred while listing layers and fields for '
'{service_type} service {service_folder}/{service_name} on '
'ArcGIS Server instance {ags_instance} (Connection File: {ags_connection})'
.format(**service_props))
if not warn_on_errors:
raise
else:
yield dict(error=e.message,
**service_props)
def analyze_services(included_envs=asterisk_tuple,
excluded_envs=empty_tuple,
included_service_folders=asterisk_tuple,
excluded_service_folders=empty_tuple,
included_instances=asterisk_tuple,
excluded_instances=empty_tuple,
included_services=asterisk_tuple,
excluded_services=empty_tuple,
warn_on_errors=True,
config_dir=default_config_dir):
import arcpy
arcpy.env.overwriteOutput = True
user_config = get_config('userconfig', config_dir)
env_names = superfilter(user_config['environments'].keys(), included_envs,
excluded_envs)
for env_name in env_names:
log.debug('Analyzing services for environment {}'.format(env_name))
env = user_config['environments'][env_name]
for ags_instance in superfilter(env['ags_instances'],
included_instances,
excluded_instances):
ags_instance_props = user_config['environments'][env_name][
'ags_instances'][ags_instance]
ags_connection = ags_instance_props['ags_connection']
server_url = ags_instance_props['url']
token = ags_instance_props['token']
proxies = ags_instance_props.get('proxies') or user_config.get(
'proxies')
with create_session(server_url, proxies=proxies) as session:
service_folders = list_service_folders(server_url,
token,
session=session)
for service_folder in superfilter(service_folders,
included_service_folders,
excluded_service_folders):
for service in list_services(server_url,
token,
service_folder,
session=session):
service_name = service['serviceName']
service_type = service['type']
if (service_type in ('MapServer', 'GeocodeServer')
and superfilter(
(service_name, ), included_services,
excluded_services)):
service_props = dict(env_name=env_name,
ags_instance=ags_instance,
service_folder=service_folder,
service_name=service_name,
service_type=service_type)
try:
service_manifest = get_service_manifest(
server_url,
token,
service_name,
service_folder,
service_type,
session=session)
service_props[
'file_path'] = file_path = service_manifest[
'resources'][0]['onPremisePath']
file_type = {
'MapServer': 'MXD',
'GeocodeServer': 'Locator'
}[service_type]
log.info(
'Analyzing {} service {}/{} on ArcGIS Server instance {} (Connection File: {}, {} Path: {})'
.format(service_type, service_folder,
service_name, ags_instance,
ags_connection, file_type,
file_path))
if not arcpy.Exists(file_path):
raise RuntimeError(
'{} {} does not exist!'.format(
file_type, file_path))
try:
tempdir = tempfile.mkdtemp()
log.debug(
'Temporary directory created: {}'.
format(tempdir))
sddraft = os.path.join(
tempdir, service_name + '.sddraft')
log.debug(
'Creating SDDraft file: {}'.format(
sddraft))
if service_type == 'MapServer':
mxd = open_mxd(file_path)
analysis = arcpy.mapping.CreateMapSDDraft(
mxd, sddraft, service_name,
'FROM_CONNECTION_FILE',
ags_connection, False,
service_folder)
elif service_type == 'GeocodeServer':
locator_path = file_path
analysis = arcpy.CreateGeocodeSDDraft(
locator_path, sddraft,
service_name,
'FROM_CONNECTION_FILE',
ags_connection, False,
service_folder)
else:
raise RuntimeError(
'Unsupported service type {}!'.
format(service_type))
for key, log_method in (('messages',
log.info),
('warnings',
log.warn),
('errors',
log.error)):
items = analysis[key]
severity = key[:-1].title()
if items:
log.info('----' + key.upper() +
'---')
for ((message, code), layerlist
) in items.iteritems():
code = '{:05d}'.format(code)
log_method(
' {} (CODE {})'.format(
message, code))
code = '="{}"'.format(code)
issue_props = dict(
severity=severity,
code=code,
message=message)
if not layerlist:
yield dict(
chain(
service_props.
iteritems(),
issue_props.
iteritems()))
else:
log_method(
' applies to:')
for layer in layerlist:
layer_name = layer.longName if hasattr(
layer, 'longName'
) else layer.name
layer_props = dict(
dataset_name=layer.
datasetName,
workspace_path=layer
.workspacePath,
layer_name=
layer_name)
log_method(
' {}'.
format(layer_name))
yield dict(
chain(
service_props.
iteritems(),
issue_props.
iteritems(),
layer_props.
iteritems()))
log_method('')
if analysis['errors']:
error_message = 'Analysis failed for service {}/{} at {:%#m/%#d/%y %#I:%M:%S %p}' \
.format(service_folder, service_name, datetime.datetime.now())
log.error(error_message)
raise RuntimeError(
error_message, analysis['errors'])
finally:
log.debug(
'Cleaning up temporary directory: {}'.
format(tempdir))
rmtree(tempdir, ignore_errors=True)
except StandardError as e:
log.exception(
'An error occurred while analyzing {} service {}/{} on ArcGIS Server instance {}'
.format(service_type, service_folder,
service_name, ags_instance))
if not warn_on_errors:
raise
else:
yield dict(severity='Error',
message=e.message,
**service_props)