def serviceStatus(serviceFullName,smallKey, smallKeyFolder, server, port,geocatUrl, geocatUsername, geocatPassword,logs): """Check the status of a pubilshed service. Args: smallKey: Small key of current payload. SmallKeyFolder: folder of current payload serviceFullName: Name of the service. server: Domain of server to connect to. port: Port of server to connect to. geocatUrl: geocat Url geocatUsername: geocat user name geocatPassword: geocat password logs: log list holds all log items for current publication Returns: A string - 'ERROR' or 'SUCCESS'. """ status = 'SUCCESS' baseUrl = "http://{}:{}/arcgis/rest/services".format(server, port) response = json.load(urllib2.urlopen(baseUrl + '/' + serviceFullName + '/' + 'MapServer' + "?f=json")) if "error" in response: status = 'ERROR' else: #check further if there is any records returned queryUrl = baseUrl + '/' + serviceFullName + '/' + 'MapServer' queryUrl= queryUrl + "/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=&returnIdsOnly=false&returnCountOnly=true&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=json" response= json.load(urllib2.urlopen(queryUrl)) if "error" in response: status ="ERROR" checkError.printLog(logs,"Service " + smallKey + " returns error.") onlineResources.updateErrorStatus(smallKey, smallKeyFolder, RESTENDPOINTNOTFOUND['code'], geocatUrl, geocatUsername, geocatPassword) return status
def sendEmail(server, fromaddr, toaddrs, smallKey, workspace, logs): """send email Args: server: smtp server fromaddr: sender email address toaddrs: receriver email addresses smallKey: smallKey worksapce: playload folder logs: log list holds all log items for current publication """ try: #Get the UUID from the supplied JSON file try: jsonPath = os.path.join(workspace, smallKey + '.json') jsonData = open(jsonPath) jsonObj = json.load(jsonData) jsonData.close() except IOError: checkError.printLog(logs, "Json file is not found") raise uuid = jsonObj['config']['UUID'] errorInfo = '\n'.join([str(i) for i in logs]) subject = "Error publishing data for UUID [{0}]".format(uuid) body = """Hello, This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention. {0}] Please get in touch with the BASD development team if you believe this error requires their attention. """.format(errorInfo) msg = MIMEText(body) msg['Subject'] = subject msg['From'] = fromaddr msg['To'] = toaddrs # sending toaddrs = toaddrs.split(',') session = smtplib.SMTP(server) try: session.sendmail(fromaddr, toaddrs, msg.as_string()) finally: session.quit() except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e: checkError.printLog(logs, str(e.errno) + ":" + e.strerror) raise
def sendEmail(server, fromaddr, toaddrs, smallKey, workspace, logs): """send email Args: server: smtp server fromaddr: sender email address toaddrs: receriver email addresses smallKey: smallKey worksapce: playload folder logs: log list holds all log items for current publication """ try: #Get the UUID from the supplied JSON file try: jsonPath = os.path.join(workspace, smallKey + '.json') jsonData = open(jsonPath) jsonObj = json.load(jsonData) jsonData.close() except IOError: checkError.printLog(logs,"Json file is not found") raise uuid = jsonObj['config']['UUID'] errorInfo = '\n'.join([str(i) for i in logs]) subject ="Error publishing data for UUID [{0}]".format(uuid) body= """Hello, This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention. {0}] Please get in touch with the BASD development team if you believe this error requires their attention. """.format(errorInfo) msg = MIMEText(body) msg['Subject'] = subject msg['From'] = fromaddr msg['To'] = toaddrs # sending toaddrs= toaddrs.split(',') session = smtplib.SMTP(server) try: session.sendmail(fromaddr, toaddrs, msg.as_string()) finally: session.quit() except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e: checkError.printLog(logs,str(e.errno) + ":"+ e.strerror) raise
def makeDescriptor(smallkey, baseUrl,logs): """Use the makeDescriptor service to create a JSON descriptor file. Assumption: The JSON file exists in a folder. Args: smallKey: Small key of current payload. baseUrl: Base URL of makeDescriptor service. logs: log list holds all log items for current publication """ makeDescriptorUrl = baseUrl + '/' + smallkey print "make descriptorUrl:"+ makeDescriptorUrl response = json.load(urllib2.urlopen(makeDescriptorUrl + "?f=json")) if 'Error' in response: checkError.printLog(logs,response['Error']) else: checkError.printLog(logs,response['msg'])
def cleanUp(workspace, smallKey,logs): """Clean up the workspace by removing smallkey folder and zip file. Args: workspace: ArcPy workspace environment setting (folder path). smallKey: ID of shapefile. logs: log list holds all log items for current publication """ try: for root, dirs, files in os.walk(workspace, topdown=False): for f in files: print f os.unlink(os.path.join(root, f)) for d in dirs: print d shutil.rmtree(os.path.join(root, d),ignore_errors=True) checkError.printLog(logs,workspace + " workspace cleaned up") except OSError as e: checkError.printLog(logs,"Folder clean up Failed with: "+ e.strerror)
def cleanUp(workspace, smallKey, logs): """Clean up the workspace by removing smallkey folder and zip file. Args: workspace: ArcPy workspace environment setting (folder path). smallKey: ID of shapefile. logs: log list holds all log items for current publication """ try: for root, dirs, files in os.walk(workspace, topdown=False): for f in files: print f os.unlink(os.path.join(root, f)) for d in dirs: print d shutil.rmtree(os.path.join(root, d), ignore_errors=True) checkError.printLog(logs, workspace + " workspace cleaned up") except OSError as e: checkError.printLog(logs, "Folder clean up Failed with: " + e.strerror)
def createMXD(inFolder, template,logs): """Create MXD from the layers in the folder. Args: inFolder: Path of folder to work from. template: Template MXD file. logs: log list holds all log items for current publication Returns: A new MXD file. """ checkError.printLog(logs,"Creating MXD...") arcpy.env.workspace = inFolder #Open the template mxd = arcpy.mapping.MapDocument(template) #Save the template to a new MXD, specific for this data mxd.saveACopy(inFolder + "\\" + "publishMXD.mxd") #Reopen the new file mxd = None mxd = arcpy.mapping.MapDocument(inFolder + "\\" + "publishMXD.mxd") #Add layer #http://help.arcgis.com/en/arcgisdesktop/10.0/help/index.html#/AddLayer/00s300000025000000/ #http://gis.stackexchange.com/questions/4882/how-do-i-add-a-shapefile-in-arcgis-via-python shps = arcpy.ListFeatureClasses("*.shp", "") dataFrame = arcpy.mapping.ListDataFrames(mxd, "*")[0] if shps: for shp in shps: newLayer = arcpy.mapping.Layer(inFolder + "\\" + shp) arcpy.mapping.AddLayer(dataFrame, newLayer, "BOTTOM") applyRandomSymbology(mxd, dataFrame, 0,logs) del newLayer mxd.save() checkError.printLog(logs,"Publishing MXD created") else: #If there's no shapefile checkError.printLog(logs,"No shapefile. Check payload folder") return mxd
def main(): #Parameters retrieved by FileSystemWatcher from #filesystemwatcher_config.json #logs to hold all log information for current smallkey publication logs =[] #Parameters retrieved by FileSystemWatcher from #filesystemwatcher_config.json smallKey = arcpy.GetParameterAsText(0) smallKeyFolder = arcpy.GetParameterAsText(1) server = arcpy.GetParameterAsText(2) port = arcpy.GetParameterAsText(3) pubTemplate = arcpy.GetParameterAsText(4) connPath = arcpy.GetParameterAsText(5) publishStatus = arcpy.GetParameterAsText(6) folder = arcpy.GetParameterAsText(7) geocatUrl = arcpy.GetParameterAsText(8) geocatUsername = arcpy.GetParameterAsText(9) geocatPassword = arcpy.GetParameterAsText(10) agsUser = arcpy.GetParameterAsText(11) agsPassword = arcpy.GetParameterAsText(12) smtpserver = arcpy.GetParameterAsText(13) fromaddr = arcpy.GetParameterAsText(14) toaddrs = arcpy.GetParameterAsText(15) metaDataUrl = arcpy.GetParameterAsText(16) webAdaptorName = arcpy.GetParameterAsText(17) ## print("smallKey ="+ smallKey) ## print("smallKeyFolder="+ smallKeyFolder) ## print("server ="+ server) ## print("port ="+ port) ## print("pubTemplate ="+ pubTemplate) ## print("connPath ="+ connPath) ## print("publishStatus="+ publishStatus) ## print("folder ="+folder) ## print("geocatUrl ="+ geocatUrl) ## print("geocatUsername ="******"geocatPassword ="******"agsUser ="******"agsPassword ="******"smtpserver ="+smtpserver) ## print("fromaddr="+ fromaddr) ## print("toaddrs ="+ toaddrs) ## print("metaDataUrl ="+metaDataUrl) try: serviceName = smallKey mapServiceFullName = folder + "/" + serviceName serviceNameDelete = serviceName + ".MapServer" #Folder to move payload zip files to after they are published (payloadFolder, sk) = os.path.split(smallKeyFolder) payloadZip = os.path.join(payloadFolder, smallKey + ".zip") publishedFolderName = "Published" publishedFolderPath = os.path.join(payloadFolder, publishedFolderName) badLoadsFolderName = "Bad_Payloads" badLoadsFolderPath = os.path.join(payloadFolder, badLoadsFolderName) #check error errReturns= checkError.errorValidation(smallKey,smallKeyFolder,publishStatus,geocatUrl, geocatUsername,geocatPassword,metaDataUrl,logs) if errReturns == 1: #fatal error sys.exit(1) serviceExists = False #Get the list of existing map service agsServiceList = publishService.getCatalog(server, port,logs) #Check if the map service already exists if mapServiceFullName in agsServiceList: serviceExists = True if publishStatus in ("1", "2"): #NEW or UPDATE if publishStatus == "1": #NEW if serviceExists: checkError.printLog(logs,"") checkError.printLog(logs,mapServiceFullName + " already exists. System exit.") moveFileToFolder(payloadZip, badLoadsFolderPath,logs) sys.exit(0) else: #UPDATE checkError.printLog(logs,"") checkError.printLog(logs,"Attempting to update the service: " + mapServiceFullName) if not serviceExists: checkError.printLog(logs,"Service does not exist. Publishing as new service.") checkError.printLog(logs,"") else: deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port) #Publish the new service shpFolder = os.path.join(smallKeyFolder, smallKey) pMXD = publishService.createMXD(shpFolder, pubTemplate,logs) try: publishService.publishMXD(shpFolder, pMXD, connPath, serviceName, folder,logs) finally: del pMXD #Check publishing status status = publishService.serviceStatus(mapServiceFullName, smallKey, smallKeyFolder,server, port,geocatUrl, geocatUsername, geocatPassword,logs) #If the service is published successfully, make the #descriptor file, otherwise exit if status == 'SUCCESS': publishService.addFileSizeToJson(smallKey, smallKeyFolder, shpFolder) moveFileToFolder(payloadZip, publishedFolderPath,logs) if onlineResources.updateOnlineResources(smallKey, smallKeyFolder, webAdaptorName, folder, geocatUrl, geocatUsername, geocatPassword,logs)==1: sys.exit(1) elif status == 'ERROR': sys.exit(1) ##cleanUp(smallKeyFolder, smallKey,logs) elif publishStatus == "0": #NOTHING checkError.printLog(logs,"") checkError.printLog(logs,"Status code 0 ignored.") elif publishStatus == "3": #DELETE checkError.printLog(logs,"") checkError.printLog(logs,"Attempting to delete the service: " + mapServiceFullName) if serviceExists: deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port) checkError.printLog(logs,mapServiceFullName + " map service has been deleted") publishedZipPath = os.path.join(payloadFolder, publishedFolderName, smallKey + ".zip") checkError.printLog(logs,"Deleted: " + publishedZipPath) if os.path.isfile(publishedZipPath): os.remove(publishedZipPath) checkError.printLog(logs,"Deleted: " + payloadZip) if os.path.isfile(payloadZip): os.remove(payloadZip) else: checkError.printLog(logs,"Service does not exist. Exiting.") sys.exit(0) else: checkError.printLog(logs,"Unknown publish status: " + publishStatus) if errReturns == 2: #warning error #add at last to avoid duplicated emails sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs) except: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "\n\nERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \ str(sys.exc_type) + ": " + str(sys.exc_value) + "\n" if hasattr(sys.exc_value, 'code'): if sys.exc_value.code !=0: #only for un-normal exit moveFileToFolder(payloadZip, badLoadsFolderPath,logs) checkError.printLog(logs,pymsg) sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs) else: moveFileToFolder(payloadZip, badLoadsFolderPath,logs) checkError.printLog(logs,pymsg) sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs)
def updateOnlineResources(smallKey, workspace, server, serviceFolder, geocatUrl, geocatUsername, geocatPassword,logs): """Update OnlineResources in the Data Catalogue with the appropriate endpoint URLs. Args: smallKey: ID of shapefile. workspace: Folder in which data is unzipped to. server: Domain of server to connect to. serviceFolder: Folder containing map services. geocatUrl: URL of GeoCat's metadata.addmapresources call. geocatUsername: GeoCat username with administrator privileges. geocatPassword: Password for matching GeoCat admin account. returns: 0: no error 1: fatal error """ result=0 servicesPath = 'arcgis/services' restServicesPath = 'arcgis/rest/services' #URLs of all services to add wms = 'http://%s/%s/%s/%s/MapServer/WMSServer?request=GetCapabilities&service=WMS' % (server, servicesPath, serviceFolder, smallKey) wfs = 'http://%s/%s/%s/%s/MapServer/WFSServer?request=GetCapabilities&service=WFS' % (server, servicesPath, serviceFolder, smallKey) rest = 'http://%s/%s/%s/%s/MapServer/0' % (server, restServicesPath, serviceFolder, smallKey) geoJson = 'http://%s/%s/%s/%s/MapServer/exts/GeoJSONServer/GeoJSON?query=true&layer=0&f=pjson' % (server, restServicesPath, serviceFolder, smallKey) kml = 'http://%s/%s/%s/%s/MapServer/generateKml?docName=Output&l:0=on&layers=0&layerOptions=nonComposite' % (server, restServicesPath, serviceFolder, smallKey) #Get the UUID from the supplied JSON file jsonPath = os.path.join(workspace, smallKey + '.json') jsonData = open(jsonPath) jsonObj = json.load(jsonData) jsonData.close() uuid = jsonObj['config']['UUID'] #SOAP request body #CDATA used to send raw URLs without escaping characters (required for GeoJSON query) soapBody = """<request> <uuid>%s</uuid> <smallKey>%s</smallKey> <statusCode>0</statusCode> <endPoints> <endPoint protocol="REST"><![CDATA[%s]]></endPoint> <endPoint protocol="WMS"><![CDATA[%s]]></endPoint> <endPoint protocol="WFS"><![CDATA[%s]]></endPoint> <endPoint protocol="KML"><![CDATA[%s]]></endPoint> <endPoint protocol="GeoJSON"><![CDATA[%s]]></endPoint> </endPoints> </request>""" % (uuid, smallKey, rest, wms, wfs, kml, geoJson) base64string = base64.b64encode('%s:%s' % (geocatUsername, geocatPassword)).replace('\n', '') headers = {'Content-Type': 'application/soap+xml+x-www-form-urlencoded; charset=utf-8', 'Authorization': 'Basic %s' % base64string} #Build our Request object (POST) request = urllib2.Request(geocatUrl, soapBody, headers) try: response = urllib2.urlopen(request) except IOError, e: if hasattr(e, 'reason'): checkError.printLog(logs,'OnlineResources: We failed to reach a server.') checkError.printLog(logs,'Reason: %s' % e.reason) checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif hasattr(e, 'code'): #Contains HTTP error codes and responses #Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes responses = BaseHTTPRequestHandler.responses checkError.printLog(logs,'OnlineResources: The server couldn\'t fulfill the request.') checkError.printLog(logs,'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1])) if e.code ==404: checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif e.code ==401: checkError.updateErrorStatus(smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif e.code==500: checkError.updateErrorStatus(smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword,logs) result =1 return result
#Build our Request object (POST) request = urllib2.Request(geocatUrl, soapBody, headers) try: response = urllib2.urlopen(request) except IOError, e: if hasattr(e, 'reason'): checkError.printLog(logs,'OnlineResources: We failed to reach a server.') checkError.printLog(logs,'Reason: %s' % e.reason) checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif hasattr(e, 'code'): #Contains HTTP error codes and responses #Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes responses = BaseHTTPRequestHandler.responses checkError.printLog(logs,'OnlineResources: The server couldn\'t fulfill the request.') checkError.printLog(logs,'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1])) if e.code ==404: checkError.updateErrorStatus(smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif e.code ==401: checkError.updateErrorStatus(smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword,logs) elif e.code==500: checkError.updateErrorStatus(smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword,logs) result =1 return result checkError.printLog(logs,'OnlineResources update successful') return result
def publishMXD(inFolder, mxd, connPath, serviceName, folder, logs, summary=None, tags=None): """Publish the service. Args: inFolder: Absolute path of workspace folder. mxd: MXD file to publish. connPath: Path to connection file that is used to connect to a GIS Server. serviceName: Name of the service. folder: Name of the folder to publish in. logs: log list holds all log items for current publication summary (optional): A string that represents the Item Description Summary (default=None). tags (optional): A string that represents the Item Description Tags (default=None). """ workspace = inFolder checkError.printLog(logs,"Publishing MXD in: " + workspace) # Provide other service details service = serviceName sddraft = workspace + "/" + service + '.sddraft' sd = workspace + "/" + service + '.sd' folderName = folder checkError.printLog(logs, " Service: " + service) checkError.printLog(logs, " Folder: " + folderName) checkError.printLog(logs, " SD Draft: " + sddraft) checkError.printLog(logs, " SD path: " + sd) # make sure the folder is registered with the server, if not, add it to the # datastore #if workspace not in [i[2] for i in arcpy.ListDataStoreItems(connPath, 'FOLDER')]: # # both the client and server paths are the same # dsStatus = arcpy.AddDataStoreItem(connPath, "FOLDER", "Workspace for " + service, workspace, workspace) # print "Data store: " + str(dsStatus) # Create service definition draft # Data will be copied to server # Syntax: CreateMapSDDraft(map_document, out_sddraft, service_name, {server_type}, {connection_file_path}, {copy_data_to_server}, {folder_name}, {summary}, {tags}) arcpy.mapping.CreateMapSDDraft(mxd, sddraft, service, 'ARCGIS_SERVER', connPath, True, folderName, summary, tags) #Modify the sd to enable wms, wfs, and then wcs capabilities on the service soeType = ['WMSServer', 'WFSServer', 'GeoJSONServer'] ogcSDDraft = enableCapabilities(soeType, sddraft, service, workspace, logs) # THIS IS THE LINE CAUSING PROBLEMS!!!! # Analyze the service definition draft analysis = arcpy.mapping.AnalyzeForSD(ogcSDDraft) # Print errors, warnings, and messages returned from the analysis checkError.printLog(logs,"The following information was returned during analysis of the MXD:") for key in ('messages', 'warnings', 'errors'): checkError.printLog(logs,'----' + key.upper() + '---') vars = analysis[key] errorList ="" if not vars: checkError.printLog(logs,' None') else: for ((message, code), layerlist) in vars.iteritems(): errorList= ' '+ message+ ' CODE %i' % code errorList = errorList+ ' applies to:' for layer in layerlist: errorList= errorList+ layer.name, checkError.printLog(logs,errorList) # Stage and upload the service if the sddraft analysis did not contain # errors if analysis['errors'] == {}: # Execute StageService. This creates the service definition. arcpy.StageService_server(ogcSDDraft, sd) # Execute UploadServiceDefinition. This uploads the service definition # and publishes the service. arcpy.UploadServiceDefinition_server(sd, connPath) checkError.printLog(logs, "Service successfully published") del ogcSDDraft else: checkError.printLog(logs,analysis['errors']) checkError.printLog(logs,"Service could not be published because errors were found during analysis.")
def enableCapabilities(soeType, sddraft, smallKey, workspace, logs): """Enable capabilities for the service and set maxRecordCount. Args: soeType: List of capabilities. sddraft: Path to Service Definition Draft file. smallKey: Small key of current payload. workspace: Absolute path of workspace folder. logs: log list holds all log items for current publication Returns: Path to output .sddraft file. """ #Properties dictionary for WMS/WFS Service propList = getMetadata(workspace, smallKey, logs) #THIS IS THE LINE CAUSING PROBLEMS propList = escapeSpecialCharacters(propList) #New maxRecordCount to set for publishing services (default: 1000) maxRecordCount = 10000 #New maxInstances to set for publishing services (default: 2) maxInstances = 1 #Read the sddraft xml. doc = DOM.parse(sddraft) #Find all elements named TypeName. This is where the server object #extension (SOE) names are defined. typeNames = doc.getElementsByTagName('TypeName') for typeName in typeNames: #Get the TypeName whose properties we want to modify. if typeName.firstChild.data in soeType: extension = typeName.parentNode for extElement in extension.childNodes: #Enabled SOE. if extElement.tagName == 'Enabled': extElement.firstChild.data = 'true' #Set WMS/WFS service properties if typeName.firstChild.data == "WMSServer" or typeName.firstChild.data == "WFSServer": svcExtension = typeName.parentNode for extElement in svcExtension.childNodes: if extElement.tagName == "Props": for propArray in extElement.childNodes: for propSetProperty in propArray.childNodes: for prop in propSetProperty.childNodes: if prop.nodeType == 1 and prop.tagName == "Key": setServiceProperties(prop, doc, propList) #Set maxRecordCount for MapServer services elif typeName.firstChild.data == "MapServer": svcConfiguration = typeName.parentNode for svcConfigElement in svcConfiguration.childNodes: if svcConfigElement.tagName == "Definition": for definitionElement in svcConfigElement.childNodes: if definitionElement.tagName == "ConfigurationProperties": for propArray in definitionElement.childNodes: for propSet in propArray.childNodes: for prop in propSet.childNodes: if prop.tagName == "Key": if prop.firstChild.data == "maxRecordCount": prop.nextSibling.firstChild.data = maxRecordCount print "maxRecordCount set to: %s" % str(maxRecordCount) if definitionElement.tagName == "Props": for propArray in definitionElement.childNodes: for propSet in propArray.childNodes: for prop in propSet.childNodes: if prop.tagName == "Key": if prop.firstChild.data == "MaxInstances": prop.nextSibling.firstChild.data = maxInstances print "maxInstances set to: %s" % str(maxInstances) print "WMS/WFS service properties set" #Output to a new sddraft outXML = os.path.join(workspace, "ogcEnabledSDDraft.sddraft") if os.path.exists(outXML): os.remove(outXML) f = open(outXML, 'w') f.write(doc.toxml(encoding="utf-8")) f.close() checkError.printLog(logs,"Service definition created with %s enabled" % ", ".join(map(str, soeType))) checkError.printLog(logs,"") del f, doc return outXML
def updateOnlineResources(smallKey, workspace, server, serviceFolder, geocatUrl, geocatUsername, geocatPassword, logs): """Update OnlineResources in the Data Catalogue with the appropriate endpoint URLs. Args: smallKey: ID of shapefile. workspace: Folder in which data is unzipped to. server: Domain of server to connect to. serviceFolder: Folder containing map services. geocatUrl: URL of GeoCat's metadata.addmapresources call. geocatUsername: GeoCat username with administrator privileges. geocatPassword: Password for matching GeoCat admin account. returns: 0: no error 1: fatal error """ result = 0 servicesPath = 'arcgis/services' restServicesPath = 'arcgis/rest/services' #URLs of all services to add wms = 'http://%s/%s/%s/%s/MapServer/WMSServer?request=GetCapabilities&service=WMS' % ( server, servicesPath, serviceFolder, smallKey) wfs = 'http://%s/%s/%s/%s/MapServer/WFSServer?request=GetCapabilities&service=WFS' % ( server, servicesPath, serviceFolder, smallKey) rest = 'http://%s/%s/%s/%s/MapServer/0' % (server, restServicesPath, serviceFolder, smallKey) geoJson = 'http://%s/%s/%s/%s/MapServer/exts/GeoJSONServer/GeoJSON?query=true&layer=0&f=pjson' % ( server, restServicesPath, serviceFolder, smallKey) kml = 'http://%s/%s/%s/%s/MapServer/generateKml?docName=Output&l:0=on&layers=0&layerOptions=nonComposite' % ( server, restServicesPath, serviceFolder, smallKey) #Get the UUID from the supplied JSON file jsonPath = os.path.join(workspace, smallKey + '.json') jsonData = open(jsonPath) jsonObj = json.load(jsonData) jsonData.close() uuid = jsonObj['config']['UUID'] #SOAP request body #CDATA used to send raw URLs without escaping characters (required for GeoJSON query) soapBody = """<request> <uuid>%s</uuid> <smallKey>%s</smallKey> <statusCode>0</statusCode> <endPoints> <endPoint protocol="REST"><![CDATA[%s]]></endPoint> <endPoint protocol="WMS"><![CDATA[%s]]></endPoint> <endPoint protocol="WFS"><![CDATA[%s]]></endPoint> <endPoint protocol="KML"><![CDATA[%s]]></endPoint> <endPoint protocol="GeoJSON"><![CDATA[%s]]></endPoint> </endPoints> </request>""" % (uuid, smallKey, rest, wms, wfs, kml, geoJson) base64string = base64.b64encode( '%s:%s' % (geocatUsername, geocatPassword)).replace('\n', '') headers = { 'Content-Type': 'application/soap+xml+x-www-form-urlencoded; charset=utf-8', 'Authorization': 'Basic %s' % base64string } #Build our Request object (POST) request = urllib2.Request(geocatUrl, soapBody, headers) try: response = urllib2.urlopen(request) except IOError, e: if hasattr(e, 'reason'): checkError.printLog( logs, 'OnlineResources: We failed to reach a server.') checkError.printLog(logs, 'Reason: %s' % e.reason) checkError.updateErrorStatus( smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif hasattr(e, 'code'): #Contains HTTP error codes and responses #Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes responses = BaseHTTPRequestHandler.responses checkError.printLog( logs, 'OnlineResources: The server couldn\'t fulfill the request.') checkError.printLog( logs, 'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1])) if e.code == 404: checkError.updateErrorStatus( smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif e.code == 401: checkError.updateErrorStatus( smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif e.code == 500: checkError.updateErrorStatus( smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword, logs) result = 1 return result
smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif hasattr(e, 'code'): #Contains HTTP error codes and responses #Error code list: http://www.voidspace.org.uk/python/articles/urllib2.shtml#error-codes responses = BaseHTTPRequestHandler.responses checkError.printLog( logs, 'OnlineResources: The server couldn\'t fulfill the request.') checkError.printLog( logs, 'Error code: %s - %s: %s' % (e.code, responses[e.code][0], responses[e.code][1])) if e.code == 404: checkError.updateErrorStatus( smallKey, workspace, checkError.METADATADOESNOTEXIST['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif e.code == 401: checkError.updateErrorStatus( smallKey, workspace, checkError.METADATAUSERNORIGHT['code'], geocatUrl, geocatUsername, geocatPassword, logs) elif e.code == 500: checkError.updateErrorStatus( smallKey, workspace, checkError.OTHERGENERICERROR['code'], geocatUrl, geocatUsername, geocatPassword, logs) result = 1 return result checkError.printLog(logs, 'OnlineResources update successful') return result
body= """Hello, This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention. {0}] Please get in touch with the BASD development team if you believe this error requires their attention. """.format(errorInfo) msg = MIMEText(body) msg['Subject'] = subject msg['From'] = fromaddr msg['To'] = toaddrs # sending toaddrs= toaddrs.split(',') session = smtplib.SMTP(server) try: session.sendmail(fromaddr, toaddrs, msg.as_string()) finally: session.quit() except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e: checkError.printLog(logs,str(e.errno) + ":"+ e.strerror) raise else: checkError.printLog(logs,"Mail successfully sent.")
def main(): #Parameters retrieved by FileSystemWatcher from #filesystemwatcher_config.json #logs to hold all log information for current smallkey publication logs = [] #Parameters retrieved by FileSystemWatcher from #filesystemwatcher_config.json smallKey = arcpy.GetParameterAsText(0) smallKeyFolder = arcpy.GetParameterAsText(1) server = arcpy.GetParameterAsText(2) port = arcpy.GetParameterAsText(3) pubTemplate = arcpy.GetParameterAsText(4) connPath = arcpy.GetParameterAsText(5) publishStatus = arcpy.GetParameterAsText(6) folder = arcpy.GetParameterAsText(7) geocatUrl = arcpy.GetParameterAsText(8) geocatUsername = arcpy.GetParameterAsText(9) geocatPassword = arcpy.GetParameterAsText(10) agsUser = arcpy.GetParameterAsText(11) agsPassword = arcpy.GetParameterAsText(12) smtpserver = arcpy.GetParameterAsText(13) fromaddr = arcpy.GetParameterAsText(14) toaddrs = arcpy.GetParameterAsText(15) metaDataUrl = arcpy.GetParameterAsText(16) webAdaptorName = arcpy.GetParameterAsText(17) ## print("smallKey ="+ smallKey) ## print("smallKeyFolder="+ smallKeyFolder) ## print("server ="+ server) ## print("port ="+ port) ## print("pubTemplate ="+ pubTemplate) ## print("connPath ="+ connPath) ## print("publishStatus="+ publishStatus) ## print("folder ="+folder) ## print("geocatUrl ="+ geocatUrl) ## print("geocatUsername ="******"geocatPassword ="******"agsUser ="******"agsPassword ="******"smtpserver ="+smtpserver) ## print("fromaddr="+ fromaddr) ## print("toaddrs ="+ toaddrs) ## print("metaDataUrl ="+metaDataUrl) try: serviceName = smallKey mapServiceFullName = folder + "/" + serviceName serviceNameDelete = serviceName + ".MapServer" #Folder to move payload zip files to after they are published (payloadFolder, sk) = os.path.split(smallKeyFolder) payloadZip = os.path.join(payloadFolder, smallKey + ".zip") publishedFolderName = "Published" publishedFolderPath = os.path.join(payloadFolder, publishedFolderName) badLoadsFolderName = "Bad_Payloads" badLoadsFolderPath = os.path.join(payloadFolder, badLoadsFolderName) #check error errReturns = checkError.errorValidation(smallKey, smallKeyFolder, publishStatus, geocatUrl, geocatUsername, geocatPassword, metaDataUrl, logs) if errReturns == 1: #fatal error sys.exit(1) serviceExists = False #Get the list of existing map service agsServiceList = publishService.getCatalog(server, port, logs) #Check if the map service already exists if mapServiceFullName in agsServiceList: serviceExists = True if publishStatus in ("1", "2"): #NEW or UPDATE if publishStatus == "1": #NEW if serviceExists: checkError.printLog(logs, "") checkError.printLog( logs, mapServiceFullName + " already exists. System exit.") moveFileToFolder(payloadZip, badLoadsFolderPath, logs) sys.exit(0) else: #UPDATE checkError.printLog(logs, "") checkError.printLog( logs, "Attempting to update the service: " + mapServiceFullName) if not serviceExists: checkError.printLog( logs, "Service does not exist. Publishing as new service.") checkError.printLog(logs, "") else: deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port) #Publish the new service shpFolder = os.path.join(smallKeyFolder, smallKey) pMXD = publishService.createMXD(shpFolder, pubTemplate, logs) try: publishService.publishMXD(shpFolder, pMXD, connPath, serviceName, folder, logs) finally: del pMXD #Check publishing status status = publishService.serviceStatus(mapServiceFullName, smallKey, smallKeyFolder, server, port, geocatUrl, geocatUsername, geocatPassword, logs) #If the service is published successfully, make the #descriptor file, otherwise exit if status == 'SUCCESS': publishService.addFileSizeToJson(smallKey, smallKeyFolder, shpFolder) moveFileToFolder(payloadZip, publishedFolderPath, logs) if onlineResources.updateOnlineResources( smallKey, smallKeyFolder, webAdaptorName, folder, geocatUrl, geocatUsername, geocatPassword, logs) == 1: sys.exit(1) elif status == 'ERROR': sys.exit(1) ##cleanUp(smallKeyFolder, smallKey,logs) elif publishStatus == "0": #NOTHING checkError.printLog(logs, "") checkError.printLog(logs, "Status code 0 ignored.") elif publishStatus == "3": #DELETE checkError.printLog(logs, "") checkError.printLog( logs, "Attempting to delete the service: " + mapServiceFullName) if serviceExists: deleteService.deleteService(server, serviceNameDelete, agsUser, agsPassword, folder, port) checkError.printLog( logs, mapServiceFullName + " map service has been deleted") publishedZipPath = os.path.join(payloadFolder, publishedFolderName, smallKey + ".zip") checkError.printLog(logs, "Deleted: " + publishedZipPath) if os.path.isfile(publishedZipPath): os.remove(publishedZipPath) checkError.printLog(logs, "Deleted: " + payloadZip) if os.path.isfile(payloadZip): os.remove(payloadZip) else: checkError.printLog(logs, "Service does not exist. Exiting.") sys.exit(0) else: checkError.printLog(logs, "Unknown publish status: " + publishStatus) if errReturns == 2: #warning error #add at last to avoid duplicated emails sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs) except: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "\n\nERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \ str(sys.exc_type) + ": " + str(sys.exc_value) + "\n" if hasattr(sys.exc_value, 'code'): if sys.exc_value.code != 0: #only for un-normal exit moveFileToFolder(payloadZip, badLoadsFolderPath, logs) checkError.printLog(logs, pymsg) sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs) else: moveFileToFolder(payloadZip, badLoadsFolderPath, logs) checkError.printLog(logs, pymsg) sendEmail.sendEmail(smtpserver, fromaddr, toaddrs, smallKey, smallKeyFolder, logs)
body = """Hello, This email is to inform you of a potential problem in the PAGER (Publication to ArcGIS Environments and RAMP) workflow that has returned a status code requiring your attention. {0}] Please get in touch with the BASD development team if you believe this error requires their attention. """.format(errorInfo) msg = MIMEText(body) msg['Subject'] = subject msg['From'] = fromaddr msg['To'] = toaddrs # sending toaddrs = toaddrs.split(',') session = smtplib.SMTP(server) try: session.sendmail(fromaddr, toaddrs, msg.as_string()) finally: session.quit() except (socket.gaierror, socket.error, socket.herror, smtplib.SMTPException), e: checkError.printLog(logs, str(e.errno) + ":" + e.strerror) raise else: checkError.printLog(logs, "Mail successfully sent.")