def __init__(self, username, password, config, hostname=None):
self.config_parser = ConfigParser()
self.config = self.config_parser.load_config(config) if isinstance(
config, basestring) else config
# Allow the user to specify a host as an argument, in case it's set dynamically
if hostname:
self.config['agsUrl'] = self.config_parser.update_hostname(
self.config['agsUrl'], hostname)
self.api = Api(ags_url=self.config['agsUrl'],
token_url=self.config['tokenUrl']
if 'tokenUrl' in self.config else None,
portal_url=self.config['portalUrl']
if 'portalUrl' in self.config else None,
username=username,
password=password,
verify_certs=self.config['verifyCerts']
if 'verifyCerts' in self.config else False)
# This is a S-L-O-W import, so defer as long as possible
from slap.esri import ArcpyHelper
self.arcpy_helper = ArcpyHelper(username=username,
password=password,
ags_admin_url=self.config['agsUrl'])
def test_load_config_from_file(self):
config_data = '{"foo": "bar"}'
expected = json.loads(config_data)
with patch('__builtin__.open', mock_open(read_data=config_data)):
config_parser = ConfigParser()
actual = config_parser._load_config_from_file('/path/to/config')
self.assertEqual(expected, actual)
def test_load_config_from_file(self):
config_data = '{"foo": "bar"}'
expected = json.loads(config_data)
with patch('__builtin__.open', mock_open(read_data=config_data)):
config_parser = ConfigParser()
actual = config_parser._load_config_from_file('/path/to/config')
self.assertEqual(expected, actual)
def __init__(self, username, password, config, hostname=None):
self.config_parser = ConfigParser()
self.config = self.config_parser.load_config(config) if isinstance(config, basestring) else config
# Allow the user to specify a host as an argument, in case it's set dynamically
if hostname:
self.config['agsUrl'] = self.config_parser.update_hostname(self.config['agsUrl'], hostname)
self.api = Api(
ags_url=self.config['agsUrl'],
token_url=self.config['tokenUrl'] if 'tokenUrl' in self.config else None,
portal_url=self.config['portalUrl'] if 'portalUrl' in self.config else None,
username=username,
password=password,
verify_certs=self.config['verifyCerts'] if 'verifyCerts' in self.config else False
)
# This is a S-L-O-W import, so defer as long as possible
from slap.esri import ArcpyHelper
self.arcpy_helper = ArcpyHelper(
username=username,
password=password,
ags_admin_url=self.config['agsUrl']
)
def setUp(self):
self.config_parser = ConfigParser()
self.config_parser.map_service_default_json = {}
self.config_parser.image_service_default_json = {}
self.config_parser.gp_service_default_json = {}
class TestConfigParser(TestCase):
def setUp(self):
self.config_parser = ConfigParser()
self.config_parser.map_service_default_json = {}
self.config_parser.image_service_default_json = {}
self.config_parser.gp_service_default_json = {}
def test_load_config_from_file(self):
config_data = '{"foo": "bar"}'
expected = json.loads(config_data)
with patch('__builtin__.open', mock_open(read_data=config_data)):
config_parser = ConfigParser()
actual = config_parser._load_config_from_file('/path/to/config')
self.assertEqual(expected, actual)
def test_load_config(self):
file_path = '/path/to/config'
expected_config = dict(foo='bar')
with patch('slap.config.ConfigParser._load_config_from_file'
) as mock_load_file:
mock_load_file.return_value = expected_config
with patch('slap.config.ConfigParser.parse_config') as mock_parse:
self.config_parser.load_config(file_path)
mock_load_file.assert_called_once_with(file_path)
mock_parse.assert_called_once_with(expected_config)
def test_empty_config(self):
self.assertEqual(
self.config_parser.parse_config({}), {
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_root_only_config(self):
self.assertEqual(
self.config_parser.parse_config({'serverUrl':
'https://my/server'}),
{
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_no_map_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': [{
'input': 'image',
'json': {}
}]
}
}
self.assertEqual(
self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'image',
'json': {}
}]
}
})
def test_no_gp_services(self):
config = {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'input': 'map',
'json': {}
}]
},
'imageServices': {
'services': [{
'input': 'image',
'json': {}
}]
}
}
self.assertEqual(
self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'map',
'json': {}
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'image',
'json': {}
}]
}
})
def test_no_image_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{
'input': 'gp',
'json': {}
}]
},
'mapServices': {
'services': [{
'input': 'map',
'json': {}
}]
}
}
self.assertEqual(
self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'map',
'json': {}
}]
},
'gpServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': []
}
})
def test_get_root_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.config_parser.get_root_keys(config),
{'root_level': 'root'})
def test_get_type_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(
self.config_parser.get_type_keys(config, 'mapServices'),
{'type_level': 'type'})
def test_flattening_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
expected = {'input': 'map', 'root_level': 'root', 'type_level': 'type'}
self.assertEqual(
self.config_parser.parse_config(config)['mapServices']['services']
[0], expected)
def test_flattening_nested_keys(self):
config = {
'root_level': 'root',
'properties': {
'myRootProp': 'someValue'
},
'mapServices': {
'type_level':
'type',
'properties': {
'myTypeProp': 'someOtherValue'
},
'services': [{
'input': 'map',
'properties': {
'myServiceProp': 'someThirdValue'
}
}]
}
}
self.assertEqual(
self.config_parser.parse_config(config)['mapServices']['services']
[0], {
'input': 'map',
'root_level': 'root',
'type_level': 'type',
'properties': {
'myRootProp': 'someValue',
'myTypeProp': 'someOtherValue',
'myServiceProp': 'someThirdValue'
}
})
def check_missing_key(self, config):
self.config_parser.config = config
with self.assertRaises(KeyError):
self.config_parser.check_required_keys(config)
def test_raises_for_missing_server_url(self):
self.check_missing_key({
'mapServices': {
'services': [{
'input': 'foo'
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_raises_for_missing_input(self):
self.check_missing_key({
'mapServices': {
'services': [{
'serverUrl': 'foo'
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_merge_json_dict(self):
default_json = {
"type": "MapServer",
"capabilities": "Map,Query,Data",
"properties": {
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
},
}
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(
expected, self.config_parser.merge_json(default_json, config_json))
def test_merge_json_string(self):
default_json_string = '{"type": "MapServer","capabilities": "Map,Query,Data",' \
'"properties": {"outputDir": "c:\\\\arcgis\\\\arcgisoutput","virtualOutputDir": ' \
'"/rest/directories/arcgisoutput"}}'
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(
self.config_parser.merge_json(default_json_string, config_json),
expected)
class Publisher:
config = None
connection_file_path = None
config_parser = ConfigParser()
api = None
def __init__(self):
pass
def load_config(self, path_to_config):
self.config = self.config_parser.load_config(path_to_config)
def create_server_connection_file(self, username, password):
connection_file_name = 'temp.ags'
output_path = self.config_parser.get_full_path('./')
self.connection_file_path = os.path.join(output_path,
connection_file_name)
arcpy.mapping.CreateGISServerConnectionFile(
connection_type='PUBLISH_GIS_SERVICES',
out_folder_path=output_path,
out_name=connection_file_name,
server_url=self.config['agsUrl'],
server_type='ARCGIS_SERVER',
use_arcgis_desktop_staging_folder=False,
staging_folder_path=output_path,
username=username,
password=password,
save_username_password=True)
def init_api(self, ags_url, token_url, portal_url, certs, username,
password):
self.api = Api(ags_url=ags_url,
token_url=token_url,
portal_url=portal_url,
certs=certs,
username=username,
password=password)
def publish_gp(self, config_entry, filename, sddraft):
if "result" in config_entry:
result = self.config_parser.get_full_path(config_entry["result"])
else:
raise Exception(
"Result must be included in config for publishing a GP tool")
self.message("Generating service definition draft for gp tool...")
arcpy.CreateGPSDDraft(
result=result,
out_sddraft=sddraft,
service_name=config_entry["serviceName"]
if "serviceName" in config_entry else filename,
server_type=config_entry["serverType"]
if "serverType" in config_entry else 'ARCGIS_SERVER',
connection_file_path=self.connection_file_path,
copy_data_to_server=config_entry["copyDataToServer"]
if "copyDataToServer" in config_entry else False,
folder_name=config_entry["folderName"]
if "folderName" in config_entry else None,
summary=config_entry["summary"]
if "summary" in config_entry else None,
tags=config_entry["tags"] if "tags" in config_entry else None,
executionType=config_entry["executionType"]
if "executionType" in config_entry else 'Asynchronous',
resultMapServer=False,
showMessages="INFO",
maximumRecords=5000,
minInstances=2,
maxInstances=3,
maxUsageTime=100,
maxWaitTime=10,
maxIdleTime=180)
return arcpy.mapping.AnalyzeForSD(sddraft)
def publish_mxd(self, config_entry, filename, sddraft):
if "workspaces" in config_entry:
self.set_workspaces(config_entry["input"],
config_entry["workspaces"])
mxd = arcpy.mapping.MapDocument(
self.config_parser.get_full_path(config_entry["input"]))
self.message("Generating service definition draft for mxd...")
arcpy.mapping.CreateMapSDDraft(
map_document=mxd,
out_sddraft=sddraft,
service_name=config_entry["serviceName"]
if "serviceName" in config_entry else filename,
server_type=config_entry["serverType"]
if "serverType" in config_entry else 'ARCGIS_SERVER',
connection_file_path=self.connection_file_path,
copy_data_to_server=config_entry["copyDataToServer"]
if "copyDataToServer" in config_entry else False,
folder_name=config_entry["folderName"]
if "folderName" in config_entry else None,
summary=config_entry["summary"]
if "summary" in config_entry else None,
tags=config_entry["tags"] if "tags" in config_entry else None)
return arcpy.mapping.AnalyzeForSD(sddraft)
def publish_image_service(self, config_entry, filename, sddraft):
self.message(
"Generating service definition draft for image service...")
arcpy.CreateImageSDDraft(
raster_or_mosaic_layer=config_entry["input"],
out_sddraft=sddraft,
service_name=config_entry["serviceName"]
if "serviceName" in config_entry else filename,
connection_file_path=self.connection_file_path,
server_type=config_entry["serverType"]
if "serverType" in config_entry else 'ARCGIS_SERVER',
copy_data_to_server=config_entry["copyDataToServer"]
if "copyDataToServer" in config_entry else False,
folder_name=config_entry["folderName"]
if "folderName" in config_entry else None,
summary=config_entry["summary"]
if "summary" in config_entry else None,
tags=config_entry["tags"] if "tags" in config_entry else None)
return arcpy.mapping.AnalyzeForSD(sddraft)
def get_output_directory(self, config_entry):
return self.config_parser.get_full_path(
config_entry["output"]
) if "output" in config_entry else self.config_parser.get_full_path(
'output')
def set_workspaces(self, path_to_mxd, workspaces):
full_mxd_path = self.config_parser.get_full_path(path_to_mxd)
mxd = arcpy.mapping.MapDocument(full_mxd_path)
for workspace in workspaces:
self.message("Replacing workspace " + workspace["old"]["path"] +
" => " + workspace["new"]["path"])
mxd.replaceWorkspaces(
old_workspace_path=workspace["old"]["path"],
old_workspace_type=workspace["old"]["type"]
if "type" in workspace["old"] else "SDE_WORKSPACE",
new_workspace_path=workspace["new"]["path"],
new_workspace_type=workspace["new"]["type"]
if "type" in workspace["new"] else "SDE_WORKSPACE",
validate=False)
mxd.relativePaths = True
mxd.save()
del mxd
@staticmethod
def analysis_successful(analysis_errors):
if analysis_errors == {}:
return True
else:
raise RuntimeError('Analysis contained errors: ', analysis_errors)
def get_sddraft_output(self, original_name, output_path):
return self._get_output_filename(original_name, output_path, 'sddraft')
def get_sd_output(self, original_name, output_path):
return self._get_output_filename(original_name, output_path, 'sd')
@staticmethod
def _get_output_filename(original_name, output_path, extension):
return os.path.join(output_path,
'{}.' + extension).format(original_name)
def publish_input(self, input_value):
input_was_published = False
for service_type in self.config_parser.service_types:
if not input_was_published:
input_was_published = self.check_service_type(
service_type, input_value)
if not input_was_published:
raise ValueError('Input ' + input_value +
' was not found in config.')
def check_service_type(self, service_type, value):
ret = False
if service_type in self.config:
for config in self.config[service_type]['services']:
if config["input"] == value:
self.publish_service(service_type, config)
ret = True
break
return ret
def publish_all(self):
for type in self.config_parser.service_types:
self.publish_services(type)
def _get_method_by_type(self, type):
if type == 'mapServices':
return self.publish_mxd
if type == 'imageServices':
return self.publish_image_service
if type == 'gpServices':
return self.publish_gp
raise ValueError('Invalid type: ' + type)
def publish_services(self, type):
for config_entry in self.config[type]['services']:
self.publish_service(type, config_entry)
def publish_service(self, service_type, config_entry):
filename = os.path.splitext(os.path.split(config_entry["input"])[1])[0]
config_entry['json']['serviceName'] = config_entry[
"serviceName"] if "serviceName" in config_entry else filename
output_directory = self.get_output_directory(config_entry)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
sddraft = self.get_sddraft_output(filename, output_directory)
sd = self.get_sd_output(filename, output_directory)
self.message("Publishing " + config_entry["input"])
analysis = self._get_method_by_type(service_type)(config_entry,
filename, sddraft)
if self.analysis_successful(analysis['errors']):
self.publish_draft(sddraft, sd, config_entry)
self.message(config_entry["input"] + " published successfully")
else:
self.message("Error publishing " + config_entry['input'] +
analysis)
def publish_draft(self, sddraft, sd, config):
self.message("Staging service definition...")
arcpy.StageService_server(sddraft, sd)
self.delete_service(config)
self.message("Uploading service definition...")
arcpy.UploadServiceDefinition_server(sd, self.connection_file_path)
self.update_service(config)
def delete_service(self, config):
service_exists = self.api.service_exists(
config['json']['serviceName'],
config["folderName"] if "folderName" in config else None)
if service_exists['exists']:
self.message("Deleting old service...")
self.api.delete_service(
config['json']['serviceName'],
config["folderName"] if "folderName" in config else None)
def update_service(self, config):
if 'json' in config:
default_json = self.api.get_service_params(
service_name=config['json']['serviceName'],
folder=config["folderName"]
if "folderName" in config else None)
json = self.config_parser.merge_json(default_json, config['json'])
self.api.edit_service(service_name=config['json']['serviceName'],
folder=config["folderName"]
if "folderName" in config else None,
params=json)
def message(self, message):
print message
def setUp(self):
self.config_parser = ConfigParser()
self.config_parser.map_service_default_json = {}
self.config_parser.image_service_default_json = {}
self.config_parser.gp_service_default_json = {}
class TestConfigParser(TestCase):
def setUp(self):
self.config_parser = ConfigParser()
self.config_parser.map_service_default_json = {}
self.config_parser.image_service_default_json = {}
self.config_parser.gp_service_default_json = {}
def test_load_config_from_file(self):
config_data = '{"foo": "bar"}'
expected = json.loads(config_data)
with patch('__builtin__.open', mock_open(read_data=config_data)):
config_parser = ConfigParser()
actual = config_parser._load_config_from_file('/path/to/config')
self.assertEqual(expected, actual)
def test_load_config(self):
file_path = '/path/to/config'
expected_config = dict(foo='bar')
with patch('slap.config.ConfigParser._load_config_from_file') as mock_load_file:
mock_load_file.return_value = expected_config
with patch('slap.config.ConfigParser.parse_config') as mock_parse:
self.config_parser.load_config(file_path)
mock_load_file.assert_called_once_with(file_path)
mock_parse.assert_called_once_with(expected_config)
def test_empty_config(self):
self.assertEqual(self.config_parser.parse_config({}), {
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_root_only_config(self):
self.assertEqual(self.config_parser.parse_config({'serverUrl': 'https://my/server'}), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_no_map_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': [{'input': 'image', 'json': {}}]
}
}
self.assertEqual(self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'image', 'json': {}}]
}
})
def test_no_gp_services(self):
config = {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'input': 'map', 'json': {}}]
},
'imageServices': {
'services': [{'input': 'image', 'json': {}}]
}
}
self.assertEqual(self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'map', 'json': {}}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'image', 'json': {}}]
}
})
def test_no_image_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{'input': 'gp', 'json': {}}]
},
'mapServices': {
'services': [{'input': 'map', 'json': {}}]
}
}
self.assertEqual(self.config_parser.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'map', 'json': {}}]
},
'gpServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': []
}
})
def test_get_root_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.config_parser.get_root_keys(config), {'root_level': 'root'})
def test_get_type_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.config_parser.get_type_keys(config, 'mapServices'), {'type_level': 'type'})
def test_flattening_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
expected = {
'input': 'map',
'root_level': 'root',
'type_level': 'type'
}
self.assertEqual(self.config_parser.parse_config(config)['mapServices']['services'][0], expected)
def test_flattening_nested_keys(self):
config = {
'root_level': 'root',
'properties': {
'myRootProp': 'someValue'
},
'mapServices': {
'type_level': 'type',
'properties': {
'myTypeProp': 'someOtherValue'
},
'services': [{
'input': 'map',
'properties': {
'myServiceProp': 'someThirdValue'
}
}]
}
}
self.assertEqual(self.config_parser.parse_config(config)['mapServices']['services'][0], {
'input': 'map',
'root_level': 'root',
'type_level': 'type',
'properties': {
'myRootProp': 'someValue',
'myTypeProp': 'someOtherValue',
'myServiceProp': 'someThirdValue'
}
})
def check_missing_key(self, config):
self.config_parser.config = config
with self.assertRaises(KeyError):
self.config_parser.check_required_keys(config)
def test_raises_for_missing_server_url(self):
self.check_missing_key( {
'mapServices': {
'services': [{'input': 'foo'}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_raises_for_missing_input(self):
self.check_missing_key( {
'mapServices': {
'services': [{'serverUrl': 'foo'}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_merge_json_dict(self):
default_json = {
"type": "MapServer",
"capabilities": "Map,Query,Data",
"properties": {
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
},
}
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(expected, self.config_parser.merge_json(default_json, config_json))
def test_merge_json_string(self):
default_json_string = '{"type": "MapServer","capabilities": "Map,Query,Data",' \
'"properties": {"outputDir": "c:\\\\arcgis\\\\arcgisoutput","virtualOutputDir": ' \
'"/rest/directories/arcgisoutput"}}'
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(self.config_parser.merge_json(default_json_string, config_json), expected)
def setUp(self):
self.m = ConfigParser()
self.m.map_service_default_json = {}
self.m.image_service_default_json = {}
self.m.gp_service_default_json = {}
class TestConfigParser(TestCase):
m = None
def setUp(self):
self.m = ConfigParser()
self.m.map_service_default_json = {}
self.m.image_service_default_json = {}
self.m.gp_service_default_json = {}
def test_get_full_path(self):
self.assertEqual(os.path.join(os.getcwd(), 'foo'), self.m.get_full_path('foo'))
def test_empty_config(self):
self.assertEqual(self.m.parse_config({}), {
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_root_only_config(self):
self.assertEqual(self.m.parse_config({'serverUrl': 'https://my/server'}), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_no_map_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': [{'input': 'image', 'json': {}}]
}
}
self.assertEqual(self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'image', 'json': {}}]
}
})
def test_no_gp_services(self):
config = {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'input': 'map', 'json': {}}]
},
'imageServices': {
'services': [{'input': 'image', 'json': {}}]
}
}
self.assertEqual(self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'map', 'json': {}}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'image', 'json': {}}]
}
})
def test_no_image_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{'input': 'gp', 'json': {}}]
},
'mapServices': {
'services': [{'input': 'map', 'json': {}}]
}
}
self.assertEqual(self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'map', 'json': {}}]
},
'gpServices': {
'services': [{'serverUrl': 'https://my/server', 'input': 'gp', 'json': {}}]
},
'imageServices': {
'services': []
}
})
def test_get_root_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.m.get_root_keys(config), {'root_level': 'root'})
def test_get_type_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.m.get_type_keys(config, 'mapServices'), {'type_level': 'type'})
def test_flattening_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
expected = {
'input': 'map',
'root_level': 'root',
'type_level': 'type'
}
self.assertEqual(self.m.parse_config(config)['mapServices']['services'][0], expected)
def test_flattening_nested_keys(self):
config = {
'root_level': 'root',
'properties': {
'myRootProp': 'someValue'
},
'mapServices': {
'type_level': 'type',
'properties': {
'myTypeProp': 'someOtherValue'
},
'services': [{
'input': 'map',
'properties': {
'myServiceProp': 'someThirdValue'
}
}]
}
}
self.assertEqual(self.m.parse_config(config)['mapServices']['services'][0], {
'input': 'map',
'root_level': 'root',
'type_level': 'type',
'properties': {
'myRootProp': 'someValue',
'myTypeProp': 'someOtherValue',
'myServiceProp': 'someThirdValue'
}
})
def check_missing_key(self, config):
self.m.config = config
with self.assertRaises(KeyError):
self.m.check_required_keys()
def test_raises_for_missing_server_url(self):
self.check_missing_key( {
'mapServices': {
'services': [{'input': 'foo'}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_raises_for_missing_input(self):
self.check_missing_key( {
'mapServices': {
'services': [{'serverUrl': 'foo'}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_merge_json_dict(self):
default_json = {
"type": "MapServer",
"capabilities": "Map,Query,Data",
"properties": {
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
},
}
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(expected, self.m.merge_json(default_json, config_json))
def test_merge_json_string(self):
default_json_string = '{"type": "MapServer","capabilities": "Map,Query,Data",' \
'"properties": {"outputDir": "c:\\\\arcgis\\\\arcgisoutput","virtualOutputDir": ' \
'"/rest/directories/arcgisoutput"}}'
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(self.m.merge_json(default_json_string, config_json), expected)
class Publisher:
def __init__(self, username, password, config, hostname=None):
self.config_parser = ConfigParser()
self.config = self.config_parser.load_config(config) if isinstance(config, basestring) else config
# Allow the user to specify a host as an argument, in case it's set dynamically
if hostname:
self.config['agsUrl'] = self.config_parser.update_hostname(self.config['agsUrl'], hostname)
self.api = Api(
ags_url=self.config['agsUrl'],
token_url=self.config['tokenUrl'] if 'tokenUrl' in self.config else None,
portal_url=self.config['portalUrl'] if 'portalUrl' in self.config else None,
username=username,
password=password,
verify_certs=self.config['verifyCerts'] if 'verifyCerts' in self.config else False
)
# This is a S-L-O-W import, so defer as long as possible
from slap.esri import ArcpyHelper
self.arcpy_helper = ArcpyHelper(
username=username,
password=password,
ags_admin_url=self.config['agsUrl']
)
@staticmethod
def analysis_successful(analysis_errors):
if analysis_errors == {}:
return True
else:
raise RuntimeError('Analysis contained errors: ', analysis_errors)
def publish_input(self, input_value):
input_was_published = False
for service_type in self.config_parser.service_types:
if not input_was_published:
input_was_published = self._check_service_type(service_type, input_value)
if not input_was_published:
raise ValueError('Input ' + input_value + ' was not found in config.')
def _check_service_type(self, service_type, value):
ret = False
if service_type in self.config:
for config in self.config[service_type]['services']:
if config["input"] == value:
self.publish_service(service_type, config)
ret = True
break
return ret
def publish_all(self):
for service_type in self.config_parser.service_types:
self.publish_services(service_type)
def publish_services(self, service_type):
for config_entry in self.config[service_type]['services']:
self.publish_service(service_type, config_entry)
def publish_service(self, service_type, config_entry):
input_path, output_path, service_name, folder_name, json, initial_state = \
self._get_publishing_params_from_config(config_entry)
filename, sddraft, sd = self._get_service_definition_paths(input_path, output_path)
self.message("Publishing " + input_path)
analysis = self._get_method_by_service_type(service_type)(config_entry, filename, sddraft)
if self.analysis_successful(analysis['errors']): # This may throw an exception
self.publish_sd_draft(sddraft, sd, service_name, folder_name, initial_state, json)
self.message(input_path + " published successfully")
def _get_publishing_params_from_config(self, config_entry):
input_path = config_entry['input']
output_path = config_entry['output'] if 'output' in config_entry else 'output'
service_name = self._get_service_name_from_config(config_entry)
folder_name = config_entry["folderName"] if "folderName" in config_entry else None
json = config_entry['json'] if 'json' in config_entry else {}
initial_state = config_entry["initialState"] if "initialState" in config_entry else "STARTED"
return input_path, output_path, service_name, folder_name, json, initial_state
@staticmethod
def _get_service_name_from_config(config_entry):
if "serviceName" in config_entry:
return config_entry['serviceName']
elif 'json' in config_entry and 'serviceName' in config_entry['json']:
return config_entry['json']['serviceName']
else:
return os.path.splitext(os.path.split(config_entry["input"])[1])[0]
def _get_service_definition_paths(self, input_path, output_path):
filename = os.path.splitext(os.path.split(input_path)[1])[0]
output_directory = self._create_output_directory(output_path)
sddraft = os.path.join(output_directory, '{}.' + "sddraft").format(filename)
sd = os.path.join(output_directory, '{}.' + "sd").format(filename)
return filename, sddraft, sd
def _create_output_directory(self, output_path):
output_directory = self.arcpy_helper.get_full_path(output_path)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
return output_directory
def _get_method_by_service_type(self, service_type):
if service_type == 'mapServices':
return self.arcpy_helper.publish_mxd
if service_type == 'imageServices':
return self.arcpy_helper.publish_image_service
if service_type == 'gpServices':
return self.arcpy_helper.publish_gp
raise ValueError('Invalid type: ' + service_type)
def publish_sd_draft(self, path_to_sddraft, path_to_sd, service_name, folder_name=None, initial_state='STARTED', json=None):
self.arcpy_helper.stage_service_definition(sddraft=path_to_sddraft, sd=path_to_sd)
self.delete_service(service_name=service_name, folder_name=folder_name)
self.arcpy_helper.upload_service_definition(sd=path_to_sd, initial_state=initial_state)
if json:
self.update_service(service_name=service_name, json=json, folder_name=folder_name)
def delete_service(self, service_name, folder_name=None):
service_exists = self.api.service_exists(service_name=service_name, folder=folder_name)
if service_exists['exists']:
self.message("Deleting old service...")
self.api.delete_service(service_name=service_name, folder=folder_name)
def update_service(self, service_name, folder_name=None, json=None):
default_json = self.api.get_service_params(service_name=service_name, folder=folder_name)
json = self.config_parser.merge_json(default_json, json if json else {})
self.api.edit_service(service_name=service_name, folder=folder_name, params=json)
def register_data_sources(self):
if "dataSources" in self.config:
self.arcpy_helper.register_data_sources(self.config["dataSources"])
@staticmethod
def message(message):
print message
class Publisher:
def __init__(self, username, password, config, hostname=None):
self.config_parser = ConfigParser()
self.config = self.config_parser.load_config(config) if isinstance(
config, basestring) else config
# Allow the user to specify a host as an argument, in case it's set dynamically
if hostname:
self.config['agsUrl'] = self.config_parser.update_hostname(
self.config['agsUrl'], hostname)
self.api = Api(ags_url=self.config['agsUrl'],
token_url=self.config['tokenUrl']
if 'tokenUrl' in self.config else None,
portal_url=self.config['portalUrl']
if 'portalUrl' in self.config else None,
username=username,
password=password,
verify_certs=self.config['verifyCerts']
if 'verifyCerts' in self.config else False)
# This is a S-L-O-W import, so defer as long as possible
from slap.esri import ArcpyHelper
self.arcpy_helper = ArcpyHelper(username=username,
password=password,
ags_admin_url=self.config['agsUrl'])
@staticmethod
def analysis_successful(analysis_errors):
if analysis_errors == {}:
return True
else:
raise RuntimeError('Analysis contained errors: ', analysis_errors)
def publish_input(self, input_value):
input_was_published = False
for service_type in self.config_parser.service_types:
if not input_was_published:
input_was_published = self._check_service_type(
service_type, input_value)
if not input_was_published:
raise ValueError('Input ' + input_value +
' was not found in config.')
def _check_service_type(self, service_type, value):
ret = False
if service_type in self.config:
for config in self.config[service_type]['services']:
if config["input"] == value:
self.publish_service(service_type, config)
ret = True
break
return ret
def publish_all(self):
for service_type in self.config_parser.service_types:
self.publish_services(service_type)
def publish_services(self, service_type):
for config_entry in self.config[service_type]['services']:
self.publish_service(service_type, config_entry)
def publish_service(self, service_type, config_entry):
input_path, output_path, service_name, folder_name, json, initial_state = \
self._get_publishing_params_from_config(config_entry)
filename, sddraft, sd = self._get_service_definition_paths(
input_path, output_path)
self.message("Publishing " + input_path)
analysis = self._get_method_by_service_type(service_type)(config_entry,
filename,
sddraft)
if self.analysis_successful(
analysis['errors']): # This may throw an exception
self.publish_sd_draft(sddraft, sd, service_name, folder_name,
initial_state, json)
self.message(input_path + " published successfully")
def _get_publishing_params_from_config(self, config_entry):
input_path = config_entry['input']
output_path = config_entry[
'output'] if 'output' in config_entry else 'output'
service_name = self._get_service_name_from_config(config_entry)
folder_name = config_entry[
"folderName"] if "folderName" in config_entry else None
json = config_entry['json'] if 'json' in config_entry else {}
initial_state = config_entry[
"initialState"] if "initialState" in config_entry else "STARTED"
return input_path, output_path, service_name, folder_name, json, initial_state
@staticmethod
def _get_service_name_from_config(config_entry):
if "serviceName" in config_entry:
return config_entry['serviceName']
elif 'json' in config_entry and 'serviceName' in config_entry['json']:
return config_entry['json']['serviceName']
else:
return os.path.splitext(os.path.split(config_entry["input"])[1])[0]
def _get_service_definition_paths(self, input_path, output_path):
filename = os.path.splitext(os.path.split(input_path)[1])[0]
output_directory = self._create_output_directory(output_path)
sddraft = os.path.join(output_directory,
'{}.' + "sddraft").format(filename)
sd = os.path.join(output_directory, '{}.' + "sd").format(filename)
return filename, sddraft, sd
def _create_output_directory(self, output_path):
output_directory = self.arcpy_helper.get_full_path(output_path)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
return output_directory
def _get_method_by_service_type(self, service_type):
if service_type == 'mapServices':
return self.arcpy_helper.publish_mxd
if service_type == 'imageServices':
return self.arcpy_helper.publish_image_service
if service_type == 'gpServices':
return self.arcpy_helper.publish_gp
raise ValueError('Invalid type: ' + service_type)
def publish_sd_draft(self,
path_to_sddraft,
path_to_sd,
service_name,
folder_name=None,
initial_state='STARTED',
json=None):
self.arcpy_helper.stage_service_definition(sddraft=path_to_sddraft,
sd=path_to_sd)
self.delete_service(service_name=service_name, folder_name=folder_name)
self.arcpy_helper.upload_service_definition(
sd=path_to_sd, initial_state=initial_state)
if json:
self.update_service(service_name=service_name,
json=json,
folder_name=folder_name)
def delete_service(self, service_name, folder_name=None):
service_exists = self.api.service_exists(service_name=service_name,
folder=folder_name)
if service_exists['exists']:
self.message("Deleting old service...")
self.api.delete_service(service_name=service_name,
folder=folder_name)
def update_service(self, service_name, folder_name=None, json=None):
default_json = self.api.get_service_params(service_name=service_name,
folder=folder_name)
json = self.config_parser.merge_json(default_json,
json if json else {})
self.api.edit_service(service_name=service_name,
folder=folder_name,
params=json)
def register_data_sources(self):
if "dataSources" in self.config:
self.arcpy_helper.register_data_sources(self.config["dataSources"])
@staticmethod
def message(message):
print message
def setUp(self):
self.m = ConfigParser()
self.m.map_service_default_json = {}
self.m.image_service_default_json = {}
self.m.gp_service_default_json = {}
class TestConfigParser(TestCase):
m = None
def setUp(self):
self.m = ConfigParser()
self.m.map_service_default_json = {}
self.m.image_service_default_json = {}
self.m.gp_service_default_json = {}
def test_get_full_path(self):
self.assertEqual(os.path.join(os.getcwd(), 'foo'),
self.m.get_full_path('foo'))
def test_empty_config(self):
self.assertEqual(
self.m.parse_config({}), {
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_root_only_config(self):
self.assertEqual(
self.m.parse_config({'serverUrl': 'https://my/server'}), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_no_map_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': [{
'input': 'image',
'json': {}
}]
}
}
self.assertEqual(
self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': []
},
'gpServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'image',
'json': {}
}]
}
})
def test_no_gp_services(self):
config = {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'input': 'map',
'json': {}
}]
},
'imageServices': {
'services': [{
'input': 'image',
'json': {}
}]
}
}
self.assertEqual(
self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'map',
'json': {}
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'image',
'json': {}
}]
}
})
def test_no_image_services(self):
config = {
'serverUrl': 'https://my/server',
'gpServices': {
'services': [{
'input': 'gp',
'json': {}
}]
},
'mapServices': {
'services': [{
'input': 'map',
'json': {}
}]
}
}
self.assertEqual(
self.m.parse_config(config), {
'serverUrl': 'https://my/server',
'mapServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'map',
'json': {}
}]
},
'gpServices': {
'services': [{
'serverUrl': 'https://my/server',
'input': 'gp',
'json': {}
}]
},
'imageServices': {
'services': []
}
})
def test_get_root_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.m.get_root_keys(config), {'root_level': 'root'})
def test_get_type_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
self.assertEqual(self.m.get_type_keys(config, 'mapServices'),
{'type_level': 'type'})
def test_flattening_keys(self):
config = {
'root_level': 'root',
'mapServices': {
'type_level': 'type',
'services': [{
'input': 'map'
}]
}
}
expected = {'input': 'map', 'root_level': 'root', 'type_level': 'type'}
self.assertEqual(
self.m.parse_config(config)['mapServices']['services'][0],
expected)
def test_flattening_nested_keys(self):
config = {
'root_level': 'root',
'properties': {
'myRootProp': 'someValue'
},
'mapServices': {
'type_level':
'type',
'properties': {
'myTypeProp': 'someOtherValue'
},
'services': [{
'input': 'map',
'properties': {
'myServiceProp': 'someThirdValue'
}
}]
}
}
self.assertEqual(
self.m.parse_config(config)['mapServices']['services'][0], {
'input': 'map',
'root_level': 'root',
'type_level': 'type',
'properties': {
'myRootProp': 'someValue',
'myTypeProp': 'someOtherValue',
'myServiceProp': 'someThirdValue'
}
})
def check_missing_key(self, config):
self.m.config = config
with self.assertRaises(KeyError):
self.m.check_required_keys()
def test_raises_for_missing_server_url(self):
self.check_missing_key({
'mapServices': {
'services': [{
'input': 'foo'
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_raises_for_missing_input(self):
self.check_missing_key({
'mapServices': {
'services': [{
'serverUrl': 'foo'
}]
},
'gpServices': {
'services': []
},
'imageServices': {
'services': []
}
})
def test_merge_json_dict(self):
default_json = {
"type": "MapServer",
"capabilities": "Map,Query,Data",
"properties": {
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
},
}
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(expected, self.m.merge_json(default_json,
config_json))
def test_merge_json_string(self):
default_json_string = '{"type": "MapServer","capabilities": "Map,Query,Data",' \
'"properties": {"outputDir": "c:\\\\arcgis\\\\arcgisoutput","virtualOutputDir": ' \
'"/rest/directories/arcgisoutput"}}'
config_json = {
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False
}
}
expected = {
"type": "MapServer",
"capabilities": "Map,Query",
"properties": {
"schemaLockingEnabled": False,
"outputDir": "c:\\arcgis\\arcgisoutput",
"virtualOutputDir": "/rest/directories/arcgisoutput"
}
}
self.assertEqual(self.m.merge_json(default_json_string, config_json),
expected)