def serviceStatus(serviceFullName,smallKey, smallKeyFolder, server, port,geocatUrl, geocatUsername, geocatPassword,logs):
"""Check the status of a pubilshed service.
Args:
smallKey: Small key of current payload.
SmallKeyFolder: folder of current payload
serviceFullName: Name of the service.
server: Domain of server to connect to.
port: Port of server to connect to.
geocatUrl: geocat Url
geocatUsername: geocat user name
geocatPassword: geocat password
logs: log list holds all log items for current publication
Returns:
A string - 'ERROR' or 'SUCCESS'.
"""
status = 'SUCCESS'
baseUrl = "http://{}:{}/arcgis/rest/services".format(server, port)
response = json.load(urllib2.urlopen(baseUrl + '/' + serviceFullName + '/' + 'MapServer' + "?f=json"))
if "error" in response:
status = 'ERROR'
else:
#check further if there is any records returned
queryUrl = baseUrl + '/' + serviceFullName + '/' + 'MapServer'
queryUrl= queryUrl + "/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=&returnIdsOnly=false&returnCountOnly=true&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=json"
response= json.load(urllib2.urlopen(queryUrl))
if "error" in response:
status ="ERROR"
checkError.printLog(logs,"Service " + smallKey + " returns error.")
onlineResources.updateErrorStatus(smallKey, smallKeyFolder, RESTENDPOINTNOTFOUND['code'], geocatUrl, geocatUsername, geocatPassword)
return status
def sendEmail(server, fromaddr, toaddrs, smallKey, workspace, logs):
"""send email
Args:
server: smtp server
fromaddr: sender email address
toaddrs: receriver email addresses
smallKey: smallKey
worksapce: playload folder
logs: log list holds all log items for current publication
"""
try:
#Get the UUID from the supplied JSON file
try:
jsonPath = os.path.join(workspace, smallKey + '.json')
jsonData = open(jsonPath)
jsonObj = json.load(jsonData)
jsonData.close()
except IOError:
checkError.printLog(logs, "Json file is not found")
raise
uuid = jsonObj['config']['UUID']
errorInfo = '\n'.join([str(i) for i in logs])
subject = "Error publishing data for UUID [{0}]".format(uuid)
body = """Hello,
This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention.
{0}]
Please get in touch with the BASD development team if you believe this error requires their attention.
""".format(errorInfo)
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = fromaddr
msg['To'] = toaddrs
# sending
toaddrs = toaddrs.split(',')
session = smtplib.SMTP(server)
try:
session.sendmail(fromaddr, toaddrs, msg.as_string())
finally:
session.quit()
except (socket.gaierror, socket.error, socket.herror,
smtplib.SMTPException), e:
checkError.printLog(logs, str(e.errno) + ":" + e.strerror)
raise
def sendEmail(server, fromaddr, toaddrs, smallKey, workspace, logs):
"""send email
Args:
server: smtp server
fromaddr: sender email address
toaddrs: receriver email addresses
smallKey: smallKey
worksapce: playload folder
logs: log list holds all log items for current publication
"""
try:
#Get the UUID from the supplied JSON file
try:
jsonPath = os.path.join(workspace, smallKey + '.json')
jsonData = open(jsonPath)
jsonObj = json.load(jsonData)
jsonData.close()
except IOError:
checkError.printLog(logs,"Json file is not found")
raise
uuid = jsonObj['config']['UUID']
errorInfo = '\n'.join([str(i) for i in logs])
subject ="Error publishing data for UUID [{0}]".format(uuid)
body= """Hello,
This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention.
{0}]
Please get in touch with the BASD development team if you believe this error requires their attention.
""".format(errorInfo)
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = fromaddr
msg['To'] = toaddrs
# sending
toaddrs= toaddrs.split(',')
session = smtplib.SMTP(server)
try:
session.sendmail(fromaddr, toaddrs, msg.as_string())
finally:
session.quit()
except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e:
checkError.printLog(logs,str(e.errno) + ":"+ e.strerror)
raise
def makeDescriptor(smallkey, baseUrl,logs):
"""Use the makeDescriptor service to create a JSON descriptor file.
Assumption: The JSON file exists in a folder.
Args:
smallKey: Small key of current payload.
baseUrl: Base URL of makeDescriptor service.
logs: log list holds all log items for current publication
"""
makeDescriptorUrl = baseUrl + '/' + smallkey
print "make descriptorUrl:"+ makeDescriptorUrl
response = json.load(urllib2.urlopen(makeDescriptorUrl + "?f=json"))
if 'Error' in response:
checkError.printLog(logs,response['Error'])
else:
checkError.printLog(logs,response['msg'])
def cleanUp(workspace, smallKey,logs):
"""Clean up the workspace by removing smallkey folder and zip file.
Args:
workspace: ArcPy workspace environment setting (folder path).
smallKey: ID of shapefile.
logs: log list holds all log items for current publication
"""
try:
for root, dirs, files in os.walk(workspace, topdown=False):
for f in files:
print f
os.unlink(os.path.join(root, f))
for d in dirs:
print d
shutil.rmtree(os.path.join(root, d),ignore_errors=True)
checkError.printLog(logs,workspace + " workspace cleaned up")
except OSError as e:
checkError.printLog(logs,"Folder clean up Failed with: "+ e.strerror)
def cleanUp(workspace, smallKey, logs):
"""Clean up the workspace by removing smallkey folder and zip file.
Args:
workspace: ArcPy workspace environment setting (folder path).
smallKey: ID of shapefile.
logs: log list holds all log items for current publication
"""
try:
for root, dirs, files in os.walk(workspace, topdown=False):
for f in files:
print f
os.unlink(os.path.join(root, f))
for d in dirs:
print d
shutil.rmtree(os.path.join(root, d), ignore_errors=True)
checkError.printLog(logs, workspace + " workspace cleaned up")
except OSError as e:
checkError.printLog(logs, "Folder clean up Failed with: " + e.strerror)
def createMXD(inFolder, template,logs):
"""Create MXD from the layers in the folder.
Args:
inFolder: Path of folder to work from.
template: Template MXD file.
logs: log list holds all log items for current publication
Returns:
A new MXD file.
"""
checkError.printLog(logs,"Creating MXD...")
arcpy.env.workspace = inFolder
#Open the template
mxd = arcpy.mapping.MapDocument(template)
#Save the template to a new MXD, specific for this data
mxd.saveACopy(inFolder + "\\" + "publishMXD.mxd")
#Reopen the new file
mxd = None
mxd = arcpy.mapping.MapDocument(inFolder + "\\" + "publishMXD.mxd")
#Add layer
#http://help.arcgis.com/en/arcgisdesktop/10.0/help/index.html#/AddLayer/00s300000025000000/
#http://gis.stackexchange.com/questions/4882/how-do-i-add-a-shapefile-in-arcgis-via-python
shps = arcpy.ListFeatureClasses("*.shp", "")
dataFrame = arcpy.mapping.ListDataFrames(mxd, "*")[0]
if shps:
for shp in shps:
newLayer = arcpy.mapping.Layer(inFolder + "\\" + shp)
arcpy.mapping.AddLayer(dataFrame, newLayer, "BOTTOM")
applyRandomSymbology(mxd, dataFrame, 0,logs)
del newLayer
mxd.save()
checkError.printLog(logs,"Publishing MXD created")
else: #If there's no shapefile
checkError.printLog(logs,"No shapefile. Check payload folder")
return mxd
def main():
#Parameters retrieved by FileSystemWatcher from
#filesystemwatcher_config.json
#logs to hold all log information for current smallkey publication
logs =[]
#Parameters retrieved by FileSystemWatcher from
#filesystemwatcher_config.json
smallKey = arcpy.GetParameterAsText(0)
smallKeyFolder = arcpy.GetParameterAsText(1)
server = arcpy.GetParameterAsText(2)
port = arcpy.GetParameterAsText(3)
pubTemplate = arcpy.GetParameterAsText(4)
connPath = arcpy.GetParameterAsText(5)
publishStatus = arcpy.GetParameterAsText(6)
folder = arcpy.GetParameterAsText(7)
geocatUrl = arcpy.GetParameterAsText(8)
geocatUsername = arcpy.GetParameterAsText(9)
geocatPassword = arcpy.GetParameterAsText(10)
agsUser = arcpy.GetParameterAsText(11)
agsPassword = arcpy.GetParameterAsText(12)
smtpserver = arcpy.GetParameterAsText(13)
fromaddr = arcpy.GetParameterAsText(14)
toaddrs = arcpy.GetParameterAsText(15)
metaDataUrl = arcpy.GetParameterAsText(16)
webAdaptorName = arcpy.GetParameterAsText(17)
## print("smallKey ="+ smallKey)
## print("smallKeyFolder="+ smallKeyFolder)
## print("server ="+ server)
## print("port ="+ port)
## print("pubTemplate ="+ pubTemplate)
## print("connPath ="+ connPath)
## print("publishStatus="+ publishStatus)
## print("folder ="+folder)
## print("geocatUrl ="+ geocatUrl)
## print("geocatUsername ="******"geocatPassword ="******"agsUser ="******"agsPassword ="******"smtpserver ="+smtpserver)
## print("fromaddr="+ fromaddr)
## print("toaddrs ="+ toaddrs)
## print("metaDataUrl ="+metaDataUrl)
try:
serviceName = smallKey
mapServiceFullName = folder + "/" + serviceName
serviceNameDelete = serviceName + ".MapServer"
#Folder to move payload zip files to after they are published
(payloadFolder, sk) = os.path.split(smallKeyFolder)
payloadZip = os.path.join(payloadFolder, smallKey + ".zip")
publishedFolderName = "Published"
publishedFolderPath = os.path.join(payloadFolder, publishedFolderName)
badLoadsFolderName = "Bad_Payloads"
badLoadsFolderPath = os.path.join(payloadFolder, badLoadsFolderName)
#check error
errReturns= checkError.errorValidation(smallKey,smallKeyFolder,publishStatus,geocatUrl, geocatUsername,geocatPassword,metaDataUrl,logs)
if errReturns == 1: #fatal error
sys.exit(1)
serviceExists = False
#Get the list of existing map service
agsServiceList = publishService.getCatalog(server, port,logs)
#Check if the map service already exists
if mapServiceFullName in agsServiceList:
serviceExists = True
if publishStatus in ("1", "2"): #NEW or UPDATE
if publishStatus == "1": #NEW
if serviceExists:
checkError.printLog(logs,"")
checkError.printLog(logs,mapServiceFullName + " already exists. System exit.")
moveFileToFolder(payloadZip, badLoadsFolderPath,logs)
sys.exit(0)
else: #UPDATE
checkError.printLog(logs,"")
checkError.printLog(logs,"Attempting to update the service: " + mapServiceFullName)
if not serviceExists:
checkError.printLog(logs,"Service does not exist. Publishing as new service.")
checkError.printLog(logs,"")
else:
deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port)
#Publish the new service
shpFolder = os.path.join(smallKeyFolder, smallKey)
pMXD = publishService.createMXD(shpFolder, pubTemplate,logs)
try:
publishService.publishMXD(shpFolder, pMXD, connPath, serviceName, folder,logs)
finally:
del pMXD
#Check publishing status
status = publishService.serviceStatus(mapServiceFullName, smallKey, smallKeyFolder,server, port,geocatUrl, geocatUsername, geocatPassword,logs)
#If the service is published successfully, make the
#descriptor file, otherwise exit
if status == 'SUCCESS':
publishService.addFileSizeToJson(smallKey, smallKeyFolder, shpFolder)
moveFileToFolder(payloadZip, publishedFolderPath,logs)
if onlineResources.updateOnlineResources(smallKey, smallKeyFolder, webAdaptorName, folder, geocatUrl, geocatUsername, geocatPassword,logs)==1:
sys.exit(1)
elif status == 'ERROR':
sys.exit(1)
##cleanUp(smallKeyFolder, smallKey,logs)
elif publishStatus == "0": #NOTHING
checkError.printLog(logs,"")
checkError.printLog(logs,"Status code 0 ignored.")
elif publishStatus == "3": #DELETE
checkError.printLog(logs,"")
checkError.printLog(logs,"Attempting to delete the service: " + mapServiceFullName)
if serviceExists:
deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port)
checkError.printLog(logs,mapServiceFullName + " map service has been deleted")
publishedZipPath = os.path.join(payloadFolder, publishedFolderName, smallKey + ".zip")
checkError.printLog(logs,"Deleted: " + publishedZipPath)
if os.path.isfile(publishedZipPath):
os.remove(publishedZipPath)
checkError.printLog(logs,"Deleted: " + payloadZip)
if os.path.isfile(payloadZip):
os.remove(payloadZip)
else:
checkError.printLog(logs,"Service does not exist. Exiting.")
sys.exit(0)
else:
checkError.printLog(logs,"Unknown publish status: " + publishStatus)
if errReturns == 2: #warning error #add at last to avoid duplicated emails
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "\n\nERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type) + ": " + str(sys.exc_value) + "\n"
if hasattr(sys.exc_value, 'code'):
if sys.exc_value.code !=0: #only for un-normal exit
moveFileToFolder(payloadZip, badLoadsFolderPath,logs)
checkError.printLog(logs,pymsg)
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs)
else:
moveFileToFolder(payloadZip, badLoadsFolderPath,logs)
checkError.printLog(logs,pymsg)
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs)
def updateOnlineResources(smallKey, workspace, server, serviceFolder, geocatUrl, geocatUsername, geocatPassword,logs):
"""Update OnlineResources in the Data Catalogue with the appropriate endpoint URLs.
Args:
smallKey: ID of shapefile.
workspace: Folder in which data is unzipped to.
server: Domain of server to connect to.
serviceFolder: Folder containing map services.
geocatUrl: URL of GeoCat's metadata.addmapresources call.
geocatUsername: GeoCat username with administrator privileges.
geocatPassword: Password for matching GeoCat admin account.
returns:
0: no error
1: fatal error
"""
result=0
servicesPath = 'arcgis/services'
restServicesPath = 'arcgis/rest/services'
#URLs of all services to add
wms = 'http://%s/%s/%s/%s/MapServer/WMSServer?request=GetCapabilities&service=WMS' % (server, servicesPath, serviceFolder, smallKey)
wfs = 'http://%s/%s/%s/%s/MapServer/WFSServer?request=GetCapabilities&service=WFS' % (server, servicesPath, serviceFolder, smallKey)
rest = 'http://%s/%s/%s/%s/MapServer/0' % (server, restServicesPath, serviceFolder, smallKey)
geoJson = 'http://%s/%s/%s/%s/MapServer/exts/GeoJSONServer/GeoJSON?query=true&layer=0&f=pjson' % (server, restServicesPath, serviceFolder, smallKey)
kml = 'http://%s/%s/%s/%s/MapServer/generateKml?docName=Output&l:0=on&layers=0&layerOptions=nonComposite' % (server, restServicesPath, serviceFolder, smallKey)
#Get the UUID from the supplied JSON file
jsonPath = os.path.join(workspace, smallKey + '.json')
jsonData = open(jsonPath)
jsonObj = json.load(jsonData)
jsonData.close()
uuid = jsonObj['config']['UUID']
#SOAP request body
#CDATA used to send raw URLs without escaping characters (required for GeoJSON query)
soapBody = """<request>
<uuid>%s</uuid>
<smallKey>%s</smallKey>
<statusCode>0</statusCode>
<endPoints>
<endPoint protocol="REST"><![CDATA[%s]]></endPoint>
<endPoint protocol="WMS"><![CDATA[%s]]></endPoint>
<endPoint protocol="WFS"><![CDATA[%s]]></endPoint>
<endPoint protocol="KML"><![CDATA[%s]]></endPoint>
<endPoint protocol="GeoJSON"><![CDATA[%s]]></endPoint>
</endPoints>
</request>""" % (uuid, smallKey, rest, wms, wfs, kml, geoJson)
base64string = base64.b64encode('%s:%s' % (geocatUsername, geocatPassword)).replace('\n', '')
headers = {'Content-Type': 'application/soap+xml+x-www-form-urlencoded; charset=utf-8',
'Authorization': 'Basic %s' % base64string}
#Build our Request object (POST)
request = urllib2.Request(geocatUrl, soapBody, headers)
try:
response = urllib2.urlopen(request)
except IOError, e:
if hasattr(e, 'reason'):
checkError.printLog(logs,'OnlineResources: We failed to reach a server.')
checkError.printLog(logs,'Reason: %s' % e.reason)
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif hasattr(e, 'code'):
#Contains HTTP error codes and responses
#Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes
responses = BaseHTTPRequestHandler.responses
checkError.printLog(logs,'OnlineResources: The server couldn\'t fulfill the request.')
checkError.printLog(logs,'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1]))
if e.code ==404:
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif e.code ==401:
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif e.code==500:
checkError.updateErrorStatus(smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword,logs)
result =1
return result
#Build our Request object (POST)
request = urllib2.Request(geocatUrl, soapBody, headers)
try:
response = urllib2.urlopen(request)
except IOError, e:
if hasattr(e, 'reason'):
checkError.printLog(logs,'OnlineResources: We failed to reach a server.')
checkError.printLog(logs,'Reason: %s' % e.reason)
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif hasattr(e, 'code'):
#Contains HTTP error codes and responses
#Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes
responses = BaseHTTPRequestHandler.responses
checkError.printLog(logs,'OnlineResources: The server couldn\'t fulfill the request.')
checkError.printLog(logs,'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1]))
if e.code ==404:
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif e.code ==401:
checkError.updateErrorStatus(smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword,logs)
elif e.code==500:
checkError.updateErrorStatus(smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword,logs)
result =1
return result
checkError.printLog(logs,'OnlineResources update successful')
return result
def publishMXD(inFolder, mxd, connPath, serviceName, folder, logs, summary=None, tags=None):
"""Publish the service.
Args:
inFolder: Absolute path of workspace folder.
mxd: MXD file to publish.
connPath: Path to connection file that is used to connect to a GIS Server.
serviceName: Name of the service.
folder: Name of the folder to publish in.
logs: log list holds all log items for current publication
summary (optional): A string that represents the Item Description Summary (default=None).
tags (optional): A string that represents the Item Description Tags (default=None).
"""
workspace = inFolder
checkError.printLog(logs,"Publishing MXD in: " + workspace)
# Provide other service details
service = serviceName
sddraft = workspace + "/" + service + '.sddraft'
sd = workspace + "/" + service + '.sd'
folderName = folder
checkError.printLog(logs, " Service: " + service)
checkError.printLog(logs, " Folder: " + folderName)
checkError.printLog(logs, " SD Draft: " + sddraft)
checkError.printLog(logs, " SD path: " + sd)
# make sure the folder is registered with the server, if not, add it to the
# datastore
#if workspace not in [i[2] for i in arcpy.ListDataStoreItems(connPath, 'FOLDER')]:
# # both the client and server paths are the same
# dsStatus = arcpy.AddDataStoreItem(connPath, "FOLDER", "Workspace for " + service, workspace, workspace)
# print "Data store: " + str(dsStatus)
# Create service definition draft
# Data will be copied to server
# Syntax: CreateMapSDDraft(map_document, out_sddraft, service_name, {server_type}, {connection_file_path}, {copy_data_to_server}, {folder_name}, {summary}, {tags})
arcpy.mapping.CreateMapSDDraft(mxd, sddraft, service, 'ARCGIS_SERVER', connPath, True, folderName, summary, tags)
#Modify the sd to enable wms, wfs, and then wcs capabilities on the service
soeType = ['WMSServer', 'WFSServer', 'GeoJSONServer']
ogcSDDraft = enableCapabilities(soeType, sddraft, service, workspace, logs) # THIS IS THE LINE CAUSING PROBLEMS!!!!
# Analyze the service definition draft
analysis = arcpy.mapping.AnalyzeForSD(ogcSDDraft)
# Print errors, warnings, and messages returned from the analysis
checkError.printLog(logs,"The following information was returned during analysis of the MXD:")
for key in ('messages', 'warnings', 'errors'):
checkError.printLog(logs,'----' + key.upper() + '---')
vars = analysis[key]
errorList =""
if not vars:
checkError.printLog(logs,' None')
else:
for ((message, code), layerlist) in vars.iteritems():
errorList= ' '+ message+ ' CODE %i' % code
errorList = errorList+ ' applies to:'
for layer in layerlist:
errorList= errorList+ layer.name,
checkError.printLog(logs,errorList)
# Stage and upload the service if the sddraft analysis did not contain
# errors
if analysis['errors'] == {}:
# Execute StageService. This creates the service definition.
arcpy.StageService_server(ogcSDDraft, sd)
# Execute UploadServiceDefinition. This uploads the service definition
# and publishes the service.
arcpy.UploadServiceDefinition_server(sd, connPath)
checkError.printLog(logs, "Service successfully published")
del ogcSDDraft
else:
checkError.printLog(logs,analysis['errors'])
checkError.printLog(logs,"Service could not be published because errors were found during analysis.")
def enableCapabilities(soeType, sddraft, smallKey, workspace, logs):
"""Enable capabilities for the service and set maxRecordCount.
Args:
soeType: List of capabilities.
sddraft: Path to Service Definition Draft file.
smallKey: Small key of current payload.
workspace: Absolute path of workspace folder.
logs: log list holds all log items for current publication
Returns:
Path to output .sddraft file.
"""
#Properties dictionary for WMS/WFS Service
propList = getMetadata(workspace, smallKey, logs) #THIS IS THE LINE CAUSING PROBLEMS
propList = escapeSpecialCharacters(propList)
#New maxRecordCount to set for publishing services (default: 1000)
maxRecordCount = 10000
#New maxInstances to set for publishing services (default: 2)
maxInstances = 1
#Read the sddraft xml.
doc = DOM.parse(sddraft)
#Find all elements named TypeName. This is where the server object
#extension (SOE) names are defined.
typeNames = doc.getElementsByTagName('TypeName')
for typeName in typeNames:
#Get the TypeName whose properties we want to modify.
if typeName.firstChild.data in soeType:
extension = typeName.parentNode
for extElement in extension.childNodes:
#Enabled SOE.
if extElement.tagName == 'Enabled':
extElement.firstChild.data = 'true'
#Set WMS/WFS service properties
if typeName.firstChild.data == "WMSServer" or typeName.firstChild.data == "WFSServer":
svcExtension = typeName.parentNode
for extElement in svcExtension.childNodes:
if extElement.tagName == "Props":
for propArray in extElement.childNodes:
for propSetProperty in propArray.childNodes:
for prop in propSetProperty.childNodes:
if prop.nodeType == 1 and prop.tagName == "Key":
setServiceProperties(prop, doc, propList)
#Set maxRecordCount for MapServer services
elif typeName.firstChild.data == "MapServer":
svcConfiguration = typeName.parentNode
for svcConfigElement in svcConfiguration.childNodes:
if svcConfigElement.tagName == "Definition":
for definitionElement in svcConfigElement.childNodes:
if definitionElement.tagName == "ConfigurationProperties":
for propArray in definitionElement.childNodes:
for propSet in propArray.childNodes:
for prop in propSet.childNodes:
if prop.tagName == "Key":
if prop.firstChild.data == "maxRecordCount":
prop.nextSibling.firstChild.data = maxRecordCount
print "maxRecordCount set to: %s" % str(maxRecordCount)
if definitionElement.tagName == "Props":
for propArray in definitionElement.childNodes:
for propSet in propArray.childNodes:
for prop in propSet.childNodes:
if prop.tagName == "Key":
if prop.firstChild.data == "MaxInstances":
prop.nextSibling.firstChild.data = maxInstances
print "maxInstances set to: %s" % str(maxInstances)
print "WMS/WFS service properties set"
#Output to a new sddraft
outXML = os.path.join(workspace, "ogcEnabledSDDraft.sddraft")
if os.path.exists(outXML):
os.remove(outXML)
f = open(outXML, 'w')
f.write(doc.toxml(encoding="utf-8"))
f.close()
checkError.printLog(logs,"Service definition created with %s enabled" % ", ".join(map(str, soeType)))
checkError.printLog(logs,"")
del f, doc
return outXML
def updateOnlineResources(smallKey, workspace, server, serviceFolder,
geocatUrl, geocatUsername, geocatPassword, logs):
"""Update OnlineResources in the Data Catalogue with the appropriate endpoint URLs.
Args:
smallKey: ID of shapefile.
workspace: Folder in which data is unzipped to.
server: Domain of server to connect to.
serviceFolder: Folder containing map services.
geocatUrl: URL of GeoCat's metadata.addmapresources call.
geocatUsername: GeoCat username with administrator privileges.
geocatPassword: Password for matching GeoCat admin account.
returns:
0: no error
1: fatal error
"""
result = 0
servicesPath = 'arcgis/services'
restServicesPath = 'arcgis/rest/services'
#URLs of all services to add
wms = 'http://%s/%s/%s/%s/MapServer/WMSServer?request=GetCapabilities&service=WMS' % (
server, servicesPath, serviceFolder, smallKey)
wfs = 'http://%s/%s/%s/%s/MapServer/WFSServer?request=GetCapabilities&service=WFS' % (
server, servicesPath, serviceFolder, smallKey)
rest = 'http://%s/%s/%s/%s/MapServer/0' % (server, restServicesPath,
serviceFolder, smallKey)
geoJson = 'http://%s/%s/%s/%s/MapServer/exts/GeoJSONServer/GeoJSON?query=true&layer=0&f=pjson' % (
server, restServicesPath, serviceFolder, smallKey)
kml = 'http://%s/%s/%s/%s/MapServer/generateKml?docName=Output&l:0=on&layers=0&layerOptions=nonComposite' % (
server, restServicesPath, serviceFolder, smallKey)
#Get the UUID from the supplied JSON file
jsonPath = os.path.join(workspace, smallKey + '.json')
jsonData = open(jsonPath)
jsonObj = json.load(jsonData)
jsonData.close()
uuid = jsonObj['config']['UUID']
#SOAP request body
#CDATA used to send raw URLs without escaping characters (required for GeoJSON query)
soapBody = """<request>
<uuid>%s</uuid>
<smallKey>%s</smallKey>
<statusCode>0</statusCode>
<endPoints>
<endPoint protocol="REST"><![CDATA[%s]]></endPoint>
<endPoint protocol="WMS"><![CDATA[%s]]></endPoint>
<endPoint protocol="WFS"><![CDATA[%s]]></endPoint>
<endPoint protocol="KML"><![CDATA[%s]]></endPoint>
<endPoint protocol="GeoJSON"><![CDATA[%s]]></endPoint>
</endPoints>
</request>""" % (uuid, smallKey, rest, wms, wfs, kml, geoJson)
base64string = base64.b64encode(
'%s:%s' % (geocatUsername, geocatPassword)).replace('\n', '')
headers = {
'Content-Type':
'application/soap+xml+x-www-form-urlencoded; charset=utf-8',
'Authorization': 'Basic %s' % base64string
}
#Build our Request object (POST)
request = urllib2.Request(geocatUrl, soapBody, headers)
try:
response = urllib2.urlopen(request)
except IOError, e:
if hasattr(e, 'reason'):
checkError.printLog(
logs, 'OnlineResources: We failed to reach a server.')
checkError.printLog(logs, 'Reason: %s' % e.reason)
checkError.updateErrorStatus(
smallKey, workspace, checkError.METADATADOESNOTEXIST['code'],
geocatUrl, geocatUsername, geocatPassword, logs)
elif hasattr(e, 'code'):
#Contains HTTP error codes and responses
#Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes
responses = BaseHTTPRequestHandler.responses
checkError.printLog(
logs,
'OnlineResources: The server couldn\'t fulfill the request.')
checkError.printLog(
logs, 'Error code: %s - %s: %s' %
(e.code, responses[e.code][0], responses[e.code][1]))
if e.code == 404:
checkError.updateErrorStatus(
smallKey, workspace,
checkError.METADATADOESNOTEXIST['code'], geocatUrl,
geocatUsername, geocatPassword, logs)
elif e.code == 401:
checkError.updateErrorStatus(
smallKey, workspace,
checkError.METADATAUSERNORIGHT['code'], geocatUrl,
geocatUsername, geocatPassword, logs)
elif e.code == 500:
checkError.updateErrorStatus(
smallKey, workspace, checkError.OTHERGENERICERROR['code'],
geocatUrl, geocatUsername, geocatPassword, logs)
result = 1
return result
smallKey, workspace, checkError.METADATADOESNOTEXIST['code'],
geocatUrl, geocatUsername, geocatPassword, logs)
elif hasattr(e, 'code'):
#Contains HTTP error codes and responses
#Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes
responses = BaseHTTPRequestHandler.responses
checkError.printLog(
logs,
'OnlineResources: The server couldn\'t fulfill the request.')
checkError.printLog(
logs, 'Error code: %s - %s: %s' %
(e.code, responses[e.code][0], responses[e.code][1]))
if e.code == 404:
checkError.updateErrorStatus(
smallKey, workspace,
checkError.METADATADOESNOTEXIST['code'], geocatUrl,
geocatUsername, geocatPassword, logs)
elif e.code == 401:
checkError.updateErrorStatus(
smallKey, workspace,
checkError.METADATAUSERNORIGHT['code'], geocatUrl,
geocatUsername, geocatPassword, logs)
elif e.code == 500:
checkError.updateErrorStatus(
smallKey, workspace, checkError.OTHERGENERICERROR['code'],
geocatUrl, geocatUsername, geocatPassword, logs)
result = 1
return result
checkError.printLog(logs, 'OnlineResources update successful')
return result
body= """Hello,
This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention.
{0}]
Please get in touch with the BASD development team if you believe this error requires their attention.
""".format(errorInfo)
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = fromaddr
msg['To'] = toaddrs
# sending
toaddrs= toaddrs.split(',')
session = smtplib.SMTP(server)
try:
session.sendmail(fromaddr, toaddrs, msg.as_string())
finally:
session.quit()
except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e:
checkError.printLog(logs,str(e.errno) + ":"+ e.strerror)
raise
else:
checkError.printLog(logs,"Mail successfully sent.")
def main():
#Parameters retrieved by FileSystemWatcher from
#filesystemwatcher_config.json
#logs to hold all log information for current smallkey publication
logs = []
#Parameters retrieved by FileSystemWatcher from
#filesystemwatcher_config.json
smallKey = arcpy.GetParameterAsText(0)
smallKeyFolder = arcpy.GetParameterAsText(1)
server = arcpy.GetParameterAsText(2)
port = arcpy.GetParameterAsText(3)
pubTemplate = arcpy.GetParameterAsText(4)
connPath = arcpy.GetParameterAsText(5)
publishStatus = arcpy.GetParameterAsText(6)
folder = arcpy.GetParameterAsText(7)
geocatUrl = arcpy.GetParameterAsText(8)
geocatUsername = arcpy.GetParameterAsText(9)
geocatPassword = arcpy.GetParameterAsText(10)
agsUser = arcpy.GetParameterAsText(11)
agsPassword = arcpy.GetParameterAsText(12)
smtpserver = arcpy.GetParameterAsText(13)
fromaddr = arcpy.GetParameterAsText(14)
toaddrs = arcpy.GetParameterAsText(15)
metaDataUrl = arcpy.GetParameterAsText(16)
webAdaptorName = arcpy.GetParameterAsText(17)
## print("smallKey ="+ smallKey)
## print("smallKeyFolder="+ smallKeyFolder)
## print("server ="+ server)
## print("port ="+ port)
## print("pubTemplate ="+ pubTemplate)
## print("connPath ="+ connPath)
## print("publishStatus="+ publishStatus)
## print("folder ="+folder)
## print("geocatUrl ="+ geocatUrl)
## print("geocatUsername ="******"geocatPassword ="******"agsUser ="******"agsPassword ="******"smtpserver ="+smtpserver)
## print("fromaddr="+ fromaddr)
## print("toaddrs ="+ toaddrs)
## print("metaDataUrl ="+metaDataUrl)
try:
serviceName = smallKey
mapServiceFullName = folder + "/" + serviceName
serviceNameDelete = serviceName + ".MapServer"
#Folder to move payload zip files to after they are published
(payloadFolder, sk) = os.path.split(smallKeyFolder)
payloadZip = os.path.join(payloadFolder, smallKey + ".zip")
publishedFolderName = "Published"
publishedFolderPath = os.path.join(payloadFolder, publishedFolderName)
badLoadsFolderName = "Bad_Payloads"
badLoadsFolderPath = os.path.join(payloadFolder, badLoadsFolderName)
#check error
errReturns = checkError.errorValidation(smallKey, smallKeyFolder,
publishStatus, geocatUrl,
geocatUsername, geocatPassword,
metaDataUrl, logs)
if errReturns == 1: #fatal error
sys.exit(1)
serviceExists = False
#Get the list of existing map service
agsServiceList = publishService.getCatalog(server, port, logs)
#Check if the map service already exists
if mapServiceFullName in agsServiceList:
serviceExists = True
if publishStatus in ("1", "2"): #NEW or UPDATE
if publishStatus == "1": #NEW
if serviceExists:
checkError.printLog(logs, "")
checkError.printLog(
logs,
mapServiceFullName + " already exists. System exit.")
moveFileToFolder(payloadZip, badLoadsFolderPath, logs)
sys.exit(0)
else: #UPDATE
checkError.printLog(logs, "")
checkError.printLog(
logs,
"Attempting to update the service: " + mapServiceFullName)
if not serviceExists:
checkError.printLog(
logs,
"Service does not exist. Publishing as new service.")
checkError.printLog(logs, "")
else:
deleteService.deleteService(server, serviceNameDelete,
agsUser, agsPassword, folder,
port)
#Publish the new service
shpFolder = os.path.join(smallKeyFolder, smallKey)
pMXD = publishService.createMXD(shpFolder, pubTemplate, logs)
try:
publishService.publishMXD(shpFolder, pMXD, connPath,
serviceName, folder, logs)
finally:
del pMXD
#Check publishing status
status = publishService.serviceStatus(mapServiceFullName, smallKey,
smallKeyFolder, server, port,
geocatUrl, geocatUsername,
geocatPassword, logs)
#If the service is published successfully, make the
#descriptor file, otherwise exit
if status == 'SUCCESS':
publishService.addFileSizeToJson(smallKey, smallKeyFolder,
shpFolder)
moveFileToFolder(payloadZip, publishedFolderPath, logs)
if onlineResources.updateOnlineResources(
smallKey, smallKeyFolder, webAdaptorName, folder,
geocatUrl, geocatUsername, geocatPassword, logs) == 1:
sys.exit(1)
elif status == 'ERROR':
sys.exit(1)
##cleanUp(smallKeyFolder, smallKey,logs)
elif publishStatus == "0": #NOTHING
checkError.printLog(logs, "")
checkError.printLog(logs, "Status code 0 ignored.")
elif publishStatus == "3": #DELETE
checkError.printLog(logs, "")
checkError.printLog(
logs,
"Attempting to delete the service: " + mapServiceFullName)
if serviceExists:
deleteService.deleteService(server, serviceNameDelete, agsUser,
agsPassword, folder, port)
checkError.printLog(
logs, mapServiceFullName + " map service has been deleted")
publishedZipPath = os.path.join(payloadFolder,
publishedFolderName,
smallKey + ".zip")
checkError.printLog(logs, "Deleted: " + publishedZipPath)
if os.path.isfile(publishedZipPath):
os.remove(publishedZipPath)
checkError.printLog(logs, "Deleted: " + payloadZip)
if os.path.isfile(payloadZip):
os.remove(payloadZip)
else:
checkError.printLog(logs, "Service does not exist. Exiting.")
sys.exit(0)
else:
checkError.printLog(logs,
"Unknown publish status: " + publishStatus)
if errReturns == 2: #warning error #add at last to avoid duplicated emails
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey,
smallKeyFolder, logs)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "\n\nERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type) + ": " + str(sys.exc_value) + "\n"
if hasattr(sys.exc_value, 'code'):
if sys.exc_value.code != 0: #only for un-normal exit
moveFileToFolder(payloadZip, badLoadsFolderPath, logs)
checkError.printLog(logs, pymsg)
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey,
smallKeyFolder, logs)
else:
moveFileToFolder(payloadZip, badLoadsFolderPath, logs)
checkError.printLog(logs, pymsg)
sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey,
smallKeyFolder, logs)
body = """Hello,
This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention.
{0}]
Please get in touch with the BASD development team if you believe this error requires their attention.
""".format(errorInfo)
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = fromaddr
msg['To'] = toaddrs
# sending
toaddrs = toaddrs.split(',')
session = smtplib.SMTP(server)
try:
session.sendmail(fromaddr, toaddrs, msg.as_string())
finally:
session.quit()
except (socket.gaierror, socket.error, socket.herror,
smtplib.SMTPException), e:
checkError.printLog(logs, str(e.errno) + ":" + e.strerror)
raise
else:
checkError.printLog(logs, "Mail successfully sent.")
