def __init__(self,configFilePath=""):
 
     # This checks to see whether this Data Prep is ran as a standalone or as a sub process in another application
     if configFilePath and configFilePath != "":
         if type(configFilePath) is dict:
             configParams = configFilePath
             self.calledFromApp = True
         else:
             configParams = Common.init_config_json(config_file=configFilePath)
             self.calledFromApp = False
     
         if "Databases" in configParams:
             self.databases = configParams["Databases"]
             
         return None
     else:
         print "Error, no config file path specified."
         return False
Example #2
0
    def __init__(self, configFilePath=""):

        # This checks to see whether this Data Prep is ran as a standalone or as a sub process in another application
        if configFilePath and configFilePath != "":
            if type(configFilePath) is dict:
                configParams = configFilePath
                self.calledFromApp = True
            else:
                configParams = Common.init_config_json(
                    config_file=configFilePath)
                self.calledFromApp = False

            if "Databases" in configParams:
                self.databases = configParams["Databases"]

            return None
        else:
            print "Error, no config file path specified."
            return False
Example #3
0
 def createRoles(self,configFiles,dateTimeFormat=None):
    
     
     if dateTimeFormat is None:
         dateTimeFormat = '%Y-%m-%d %H:%M'
     
     scriptStartTime = datetime.datetime.now()
     try:
 
         print "********************Create Roles********************"
 
         print "Script started at %s" % scriptStartTime.strftime(dateTimeFormat)
      
         if self.securityhandler.valid == False:
             print "Login required"
         else:
             orgTools = orgtools.orgtools(securityinfo=self)
 
             if orgTools is None:
                 print "Error creating orgtools"
             else:
 
                 for configFile in configFiles:
 
                     config = common.init_config_json(config_file=configFile)
                     if config is not None:
                        
                         startTime = datetime.datetime.now()
                         print "Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))
 
                         roleInfos = config['Roles']
                         for roleInfo in roleInfos:
                             createRoleResults = orgTools.createRole(roleInfo['Name'],roleInfo['Description'],roleInfo['Privileges'])
                           
                     else:
                         print "Config %s not found" % configFile
 
 
     except(TypeError,ValueError,AttributeError),e:
         print e
Example #4
0
    def createRoles(self, configFiles, dateTimeFormat=None):

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            print("********************Create Roles********************")

            print("Script started at %s" %
                  scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print("Error creating orgtools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(
                            config_file=configFile)
                        if config is not None:

                            startTime = datetime.datetime.now()
                            print("Processing config %s, starting at: %s" %
                                  (configFile,
                                   startTime.strftime(dateTimeFormat)))

                            roleInfos = config['Roles']
                            for roleInfo in roleInfos:
                                createRoleResults = orgTools.createRole(
                                    roleInfo['Name'], roleInfo['Description'],
                                    roleInfo['Privileges'])

                        else:
                            print("Config %s not found" % configFile)

        except (TypeError, ValueError, AttributeError) as e:
            print(e)
        except (common.ArcRestHelperError) as e:
            print("error in function: %s" % e[0]['function'])
            print("error on line: %s" % e[0]['line'])
            print("error in file name: %s" % e[0]['filename'])
            print("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(
                        e,
                    (ReportTools.ReportToolsError, DataPrep.DataPrepError)):
                    print("error in function: %s" % e[0]['function'])
                    print("error on line: %s" % e[0]['line'])
                    print("error in file name: %s" % e[0]['filename'])
                    print("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print("error on line: %s" % line)
                    print("error in file name: %s" % filename)
                    print("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print("error on line: %s" % line)
                print("error in file name: %s" % filename)
                print("with error message: %s" % synerror)
        finally:
            print("Script complete, time to complete: %s" %
                  str(datetime.datetime.now() - scriptStartTime))
            print("###############Create Groups Completed#################")
            print("")

            #if orgTools is not None:
            #orgTools.dispose()
            groupInfo = None
            groupFile = None
            iconPath = None
            startTime = None
            thumbnail = None
            result = None
            config = None
            sciptPath = None
            orgTools = None
            del groupInfo
            del groupFile
            del iconPath
            del startTime
            del thumbnail
            del result
            del config
            del sciptPath
            del orgTools

            gc.collect()
Example #5
0
    def stageContent(self,configFiles,dateTimeFormat=None):

        results = None
        groups = None
        items = None
        group = None
        content = None
        contentInfo = None
        startTime = None
        orgTools = None


        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'


        scriptStartTime = datetime.datetime.now()
        try:

            print ("********************Stage Content Started********************")

            print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print ("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print ("Error creating org tools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(config_file=configFile)
                        if config is not None:
                            if 'ContentItems' in config:
                                startTime = datetime.datetime.now()
                                print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat)))
                                contentInfo = config['ContentItems']
                                for cont in contentInfo:
                                    content = cont['Content']
                                    group = cont['ShareToGroup']

                                    print ("Sharing content to: %s" % group)
                                    if os.path.isfile(content):
                                        with open(content, 'rb') as csvfile:
                                            items = []
                                            groups = []
                                            for row in csv.DictReader(csvfile,dialect='excel'):
                                                if cont['Type'] == "Group":
                                                    groups.append(row['id'])
                                                elif cont['Type'] == "Items":
                                                    items.append(row['id'])
                                            results = orgTools.shareItemsToGroup(shareToGroupName=group,items=items,groups=groups)

                                print ("Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime)))
                            else:
                                print ("Config file missing ContentItems section")
                        else:
                            print ("Config %s not found" % configFile)


        except(TypeError,ValueError,AttributeError) as e:
            print (e)
        except (common.ArcRestHelperError) as e:
            print ("error in function: %s" % e[0]['function'])
            print ("error on line: %s" % e[0]['line'])
            print ("error in file name: %s" % e[0]['filename'])
            print ("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print ("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)):
                    print ("error in function: %s" % e[0]['function'])
                    print ("error on line: %s" % e[0]['line'])
                    print ("error in file name: %s" % e[0]['filename'])
                    print ("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print ("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print ("error on line: %s" % line)
                    print ("error in file name: %s" % filename)
                    print ("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print ("error on line: %s" % line)
                print ("error in file name: %s" % filename)
                print ("with error message: %s" % synerror)
        finally:
            print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime))
            print ("###############Stage Content Completed#################")
            print ("")
            #if orgTools is not None:
                #orgTools.dispose()

            results = None
            groups = None
            items = None
            group = None
            content = None
            contentInfo = None
            startTime = None
            orgTools = None

            del results
            del groups
            del items
            del group
            del content
            del contentInfo
            del startTime
            del orgTools

            gc.collect()
Example #6
0
    def publishfromconfig(self,configFiles,combinedApp=None,dateTimeFormat=None):
        publishTools = None
        webmaps = None
        config = None
        resultsItems = None
        resultFS = None
        resultMaps = None
        resultApps = None
        combinedResults = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            webmaps = []
            print ("********************Script Started********************")

            print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat))

            # start report processing (moved out from under ArcREST logic. no AGO crednetials needed to run reports)
            for configFile in configFiles:
                config = common.init_config_json(config_file=configFile)
                if config is not None:
                    if 'ReportDetails' in config:
                        if reportToolsInstalled == False:
                            print ("Report section is included in the config file but the solutionreporttools cannot be located")
                        else:
                            reportConfig = config['ReportDetails']
                            # This code checks to see if you want to export the data from SDE to a local GDB. The parameter is set in config file.
                            # Could be performance gain to run locally. If you choose this option, both the report and the data prep in memory config
                            # are modified so they can point to the local temp location.

                            if 'RunReport' in reportConfig and (str(reportConfig['RunReport']).upper() =="TRUE" or str(reportConfig['RunReport']).upper() =="YES"):
                                reportConfig = ReportTools.reportDataPrep(reportConfig)

                                print ("-----Report Section Starting-----")
                                startTime = datetime.datetime.now()
                                print ("Processing reports in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat)))
                                ReportTools.create_report_layers_using_config(config=reportConfig)
                                print ("Reports in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime)))

                                print ("-----Report Section Complete-----")
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print ("PublishingDetails is missing the PublishData parameter:  type string, values, True or False")
                            publishData = 'TRUE'
                        if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"):

                            print (" ")
                            print ("-----Publishing Section Starting-----")
                            startTime = datetime.datetime.now()
                            print ("Processing publishing in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat)))


                            publishTools = publishingtools.publishingtools(securityinfo=self)
                            if publishTools.valid == False :
                                print ("Error creating publishing tools: %s" % publishTools.message)
                            else:
                                print ("Publishing tools created: %s" % publishTools.message)
                                resultFS = []
                                if 'Items' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Creating Items: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultsItems = publishTools.publishItems(items_info=publishingConfig['Items'])
                                    print ("Items created, time to complete: %s" % str(datetime.datetime.now() - startSectTime))

                                if 'FeatureCollections' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Creating Feature Collection: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultFS = publishTools.publishFeatureCollections(configs=publishingConfig['FeatureCollections'])
                                    print ("Feature Collection published, time to complete: %s" % str(datetime.datetime.now() - startSectTime))
                                if 'FeatureServices' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Creating Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultFS = resultFS + publishTools.publishFsFromMXD(fs_config=publishingConfig['FeatureServices'])
                                    if len(resultFS) == 0:
                                        print ("Exiting, error creating feature services")
                                        return
                                    print ("Feature Services published, time to complete: %s" % str(datetime.datetime.now() - startSectTime))
                                if 'ExistingServices' in publishingConfig:

                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Updating Existing Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultES = publishTools.updateFeatureService(efs_config=publishingConfig['ExistingServices'])
                                    print ("Updating Existing Feature Services completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime))
                                if 'MapDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Creating maps: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultMaps = publishTools.publishMap(maps_info=publishingConfig['MapDetails'],fsInfo=resultFS,itInfo=resultsItems)
                                    for maps in resultMaps:
                                        if 'MapInfo' in maps:
                                            if 'Results' in maps['MapInfo']:
                                                if 'itemId' in maps['MapInfo']['Results']:
                                                    webmaps.append(maps['MapInfo']['Results']['itemId'])
                                    print ("Creating maps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime))
                                if 'AppDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print (" ")
                                    print ("Creating apps: %s" % str(startSectTime.strftime(dateTimeFormat)))
                                    resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=resultMaps,fsInfo=resultFS)
                                    print ("Creating apps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime))


                            print ("Publishing complete in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime)))

                            print ("-----Publishing Section Complete-----")

                else:
                    print ("Config %s not found" % configFile)

            if combinedApp:
                if os.path.exists(combinedApp):
                    print (" ")
                    startSectTime = datetime.datetime.now()
                    print ("Creating combined result: %s" % str(startSectTime.strftime(dateTimeFormat)))

                    config = common.init_config_json(config_file=combinedApp)
                    combinedResults = publishTools.publishCombinedWebMap(maps_info=config['PublishingDetails']['MapDetails'],webmaps=webmaps)
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print ("PublishingDetails is missing the PublishData parameter:  type string, values, True or False")
                            publishData = 'TRUE'

                        if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"):

                            if 'AppDetails' in publishingConfig:
                                resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=combinedResults)
                            print ("Creating combind result completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime))
        except(TypeError,ValueError,AttributeError) as e:
            print (e)
        except (common.ArcRestHelperError) as e:
            print ("error in function: %s" % e[0]['function'])
            print ("error on line: %s" % e[0]['line'])
            print ("error in file name: %s" % e[0]['filename'])
            print ("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print ("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)):
                    print ("error in function: %s" % e[0]['function'])
                    print ("error on line: %s" % e[0]['line'])
                    print ("error in file name: %s" % e[0]['filename'])
                    print ("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print ("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print ("error on line: %s" % line)
                    print ("error in file name: %s" % filename)
                    print ("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print ("error on line: %s" % line)
                print ("error in file name: %s" % filename)
                print ("with error message: %s" % synerror)

        finally:
            print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime))
            print ("###############Script Completed#################")
            print ("")
            if publishTools is not None:
                publishTools.dispose()

            publishTools = None
            webmaps = None
            config = None
            resultFS = None
            resultsItems = None
            resultMaps = None
            resultApps = None
            combinedResults = None

            del publishTools
            del webmaps
            del config
            del resultFS
            del resultMaps
            del resultApps
            del combinedResults
            del resultsItems
            gc.collect()
Example #7
0
    def createGroups(self,configFiles,dateTimeFormat=None):
        groupInfo = None
        groupFile = None
        iconPath = None
        startTime = None
        thumbnail = None
        result = None
        config = None
        sciptPath = None
        orgTools = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            print ("********************Create Groups********************")

            print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print ("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print ("Error creating orgtools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(config_file=configFile)
                        if config is not None:

                            startTime = datetime.datetime.now()
                            print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat)))

                            groupInfo = config['Groups']
                            groupFile = groupInfo['GroupInfo']
                            iconPath = groupInfo['IconPath']

                            if os.path.isfile(groupFile):
                                with open(groupFile, 'rb') as csvfile:

                                    for row in csv.DictReader(csvfile,dialect='excel'):
                                        if os.path.isfile(os.path.join(iconPath,row['thumbnail'])):
                                            thumbnail = os.path.join(iconPath,row['thumbnail'])
                                            if not os.path.isabs(thumbnail):

                                                sciptPath = os.getcwd()
                                                thumbnail = os.path.join(sciptPath,thumbnail)

                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'],thumbnail=thumbnail)

                                        else:
                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'])

                                        if result is None:
                                            pass
                                        else:
                                            print ("Group created: " + result.title)



                            print ("Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime)))

                        else:
                            print ("Config %s not found" % configFile)


        except(TypeError,ValueError,AttributeError) as e:
            print (e)
        except (common.ArcRestHelperError) as e:
            print ("error in function: %s" % e[0]['function'])
            print ("error on line: %s" % e[0]['line'])
            print ("error in file name: %s" % e[0]['filename'])
            print ("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print ("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)):
                    print ("error in function: %s" % e[0]['function'])
                    print ("error on line: %s" % e[0]['line'])
                    print ("error in file name: %s" % e[0]['filename'])
                    print ("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print ("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print ("error on line: %s" % line)
                    print ("error in file name: %s" % filename)
                    print ("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print ("error on line: %s" % line)
                print ("error in file name: %s" % filename)
                print ("with error message: %s" % synerror)
        finally:
            print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime))
            print ("###############Create Groups Completed#################")
            print ("")

            #if orgTools is not None:
                #orgTools.dispose()
            groupInfo = None
            groupFile = None
            iconPath = None
            startTime = None
            thumbnail = None
            result = None
            config = None
            sciptPath = None
            orgTools = None
            del groupInfo
            del groupFile
            del iconPath
            del startTime
            del thumbnail
            del result
            del config
            del sciptPath
            del orgTools

            gc.collect()
Example #8
0
    def createRoles(self,configFiles,dateTimeFormat=None):


        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            print ("********************Create Roles********************")

            print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print ("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print ("Error creating orgtools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(config_file=configFile)
                        if config is not None:

                            startTime = datetime.datetime.now()
                            print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat)))

                            roleInfos = config['Roles']
                            for roleInfo in roleInfos:
                                createRoleResults = orgTools.createRole(roleInfo['Name'],roleInfo['Description'],roleInfo['Privileges'])

                        else:
                            print ("Config %s not found" % configFile)


        except(TypeError,ValueError,AttributeError) as e:
            print (e)
        except (common.ArcRestHelperError) as e:
            print ("error in function: %s" % e[0]['function'])
            print ("error on line: %s" % e[0]['line'])
            print ("error in file name: %s" % e[0]['filename'])
            print ("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print ("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)):
                    print ("error in function: %s" % e[0]['function'])
                    print ("error on line: %s" % e[0]['line'])
                    print ("error in file name: %s" % e[0]['filename'])
                    print ("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print ("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print ("error on line: %s" % line)
                    print ("error in file name: %s" % filename)
                    print ("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print ("error on line: %s" % line)
                print ("error in file name: %s" % filename)
                print ("with error message: %s" % synerror)
        finally:
            print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime))
            print ("###############Create Groups Completed#################")
            print ("")

            #if orgTools is not None:
                #orgTools.dispose()
            groupInfo = None
            groupFile = None
            iconPath = None
            startTime = None
            thumbnail = None
            result = None
            config = None
            sciptPath = None
            orgTools = None
            del groupInfo
            del groupFile
            del iconPath
            del startTime
            del thumbnail
            del result
            del config
            del sciptPath
            del orgTools

            gc.collect()
Example #9
0
    def __init__(self, securityinfo):

        """Constructor
            A number of different security handlers are suppoted by ArcGIS, this
            function simplifies the creation of the handlers.  A dict with the
            type of handler and information to created it is required to generate
            the handlers

           Inputs:
              securityinfo - The dict with details to create the handler.  The
                        valid parameters are:
                            - security_type: Required, valid values are Portal,
                                NTLM, PKI, OAUTH, LDAP
                            - proxy_url: Optional, url for proxy server
                            - proxy_port: Optional, port for proxy server
                            - referer_url: Optional, url for referer
                            - token_url: Optional, url for token server
                            - username: Optional, Username for log in, not
                                required for all security types
                            - password: Optional, Password for log in, not
                                required for all security types
                            - certificatefile: Only required for PKI
                            - keyfile: Only required for PKI
                            - client_id: Only required for OAuth
                            - secret_id: Only required for OAuth
        """

        if not securityinfo is None:
            if isinstance(securityinfo, securityhandlerhelper):

                self._securityHandler = securityinfo.securityhandler
                self._username = securityinfo._username
                self._password = securityinfo._password
                self._proxy_url = securityinfo._proxy_url
                self._proxy_port = securityinfo._proxy_port
                self._token_url = securityinfo._token_url
                self._security_type = securityinfo._security_type
                self._featureServiceFieldCase = securityinfo._featureServiceFieldCase
                self._keyfile = securityinfo._keyfile
                self._certificatefile = securityinfo._certificatefile
                self._referer_url = securityinfo._referer_url
                self._client_id = securityinfo._client_id
                self._secret_id = securityinfo._secret_id
                self._is_portal = securityinfo._is_portal
                self._message = securityinfo._message
                self._valid = securityinfo._valid

                # self._securityHandler = securityinfo
                return
            else:
                pass
            if isinstance(securityinfo, str) and os.path.isfile(securityinfo):
                securityinfo = common.init_config_json(config_file=securityinfo)
                if "Credentials" in securityinfo:
                    securityinfo = securityinfo["Credentials"]

            if "security_type" in securityinfo:
                self._security_type = securityinfo["security_type"]
            else:
                self._message = "Security type not specified"
                self._valid = False
                return

            if "proxy_url" in securityinfo:
                self._proxy_url = securityinfo["proxy_url"]

            if "proxy_port" in securityinfo:
                self._proxy_port = securityinfo["proxy_port"]

            if "referer_url" in securityinfo:
                self._referer_url = securityinfo["referer_url"]

            if "token_url" in securityinfo and securityinfo["token_url"] is not None:
                self._token_url = securityinfo["token_url"]
                if not self._token_url.startswith("http://") and not self._token_url.startswith("https://"):
                    self._token_url = "https://" + self._token_url

            if "org_url" in securityinfo and securityinfo["org_url"] is not None:
                self._org_url = securityinfo["org_url"]
                if not self._org_url.startswith("http://") and not self._org_url.startswith("https://"):
                    self._org_url = "http://" + self._org_url

            if "username" in securityinfo:
                self._username = securityinfo["username"]
            if "password" in securityinfo:
                self._password = securityinfo["password"]

            if "certificatefile" in securityinfo:
                self._certificatefile = securityinfo["certificatefile"]
            if "keyfile" in securityinfo:
                self._keyfile = securityinfo["keyfile"]

            if "client_id" in securityinfo:
                self._client_id = securityinfo["client_id"]
            if "secret_id" in securityinfo:
                self._secret_id = securityinfo["secret_id"]

            if str(securityinfo["security_type"]).upper() == "ArcGIS".upper():

                self._securityHandler = security.ArcGISTokenSecurityHandler(
                    proxy_url=self._proxy_url, proxy_port=self._proxy_port
                )

                self._org_url = self._securityHandler.org_url
                self._username = self._securityHandler.username
                self._valid = True
                self._message = "ArcGIS security handler created"
            elif (
                str(securityinfo["security_type"]).upper() == "Portal".upper()
                or str(securityinfo["security_type"]).upper() == "AGOL".upper()
            ):
                if self._org_url is None or self._org_url == "":
                    self._org_url = "http://www.arcgis.com"
                if self._username == "" or self._password == "":
                    self._message = "No username or password, no security handler generated"
                    self._valid = True
                else:
                    if self._org_url is None or ".arcgis.com" in self._org_url:
                        self._securityHandler = security.AGOLTokenSecurityHandler(
                            username=self._username,
                            password=self._password,
                            org_url=self._org_url,
                            token_url=self._token_url,
                            proxy_url=self._proxy_url,
                            proxy_port=self._proxy_port,
                        )
                        self._message = "ArcGIS Online security handler created"
                    else:
                        self._securityHandler = security.PortalTokenSecurityHandler(
                            username=self._username,
                            password=self._password,
                            org_url=self._org_url,
                            proxy_url=self._proxy_url,
                            proxy_port=self._proxy_port,
                        )
                        self._message = "Portal security handler created"

            elif str(securityinfo["security_type"]).upper() == "NTLM".upper():
                if self._username == "" or self._password == "":
                    self._message = "Username and password required for NTLM"
                    self._valid = False
                self._securityHandler = security.NTLMSecurityHandler(
                    username=self._username,
                    password=self._password,
                    org_url=self._org_url,
                    proxy_url=self._proxy_url,
                    proxy_port=self._proxy_port,
                    referer_url=self._referer_url,
                )
                self._message = "NTLM security handler created"
            elif str(securityinfo["security_type"]).upper() == "LDAP".upper():
                if self._username == "" or self._password == "":
                    self._message = "Username and password required for LDAP"
                    self._valid = False
                self._securityHandler = security.LDAPSecurityHandler(
                    username=self._username,
                    password=self._password,
                    org_url=self._org_url,
                    proxy_url=self._proxy_url,
                    proxy_port=self._proxy_port,
                    referer_url=self._referer_url,
                )
                self._message = "LDAP security handler created"
            elif str(securityinfo["security_type"]).upper() == "PKI".upper():
                if self._keyfile == "" or self._certificatefile == "":
                    self._message = "key file and certification file required for PKI"
                    self._valid = False
                self._securityHandler = security.PKISecurityHandler(
                    keyfile=self._keyfile,
                    certificatefile=self._certificatefile,
                    org_url=self._org_url,
                    proxy_url=self._proxy_url,
                    proxy_port=self._proxy_port,
                    referer_url=self._referer_url,
                )
                self._message = "PKI security handler created"
            elif str(securityinfo["security_type"]).upper() == "OAUTH".upper():
                if self._secret_id == "" or self._client_id == "":
                    self._message = "client_id and secret_id required for OAUTH"
                    self._valid = False
                self._securityHandler = security.OAuthSecurityHandler(
                    client_id=self._client_id,
                    secret_id=self._secret_id,
                    org_url=self._org_url,
                    proxy_url=self._proxy_url,
                    proxy_port=self._proxy_port,
                )

                self._message = "OAuth security handler created"
            else:
                print "No valid security type set"
                self._message = "No valid security type set"

            admin = Administration(url=self._org_url, securityHandler=self._securityHandler)

            try:
                portal = admin.portals.portalSelf
                for hostingServer in portal.featureServers:
                    if isinstance(hostingServer, AGSAdministration):
                        try:
                            serData = hostingServer.data

                            dataItems = serData.rootDataItems
                            if "rootItems" in dataItems:
                                for rootItem in dataItems["rootItems"]:
                                    if rootItem == "/enterpriseDatabases":
                                        rootItems = serData.findDataItems(ancestorPath=rootItem, type="fgdb,egdb")
                                        if not rootItems is None and "items" in rootItems:
                                            for item in rootItems["items"]:
                                                if "info" in item:
                                                    if (
                                                        "isManaged" in item["info"]
                                                        and item["info"]["isManaged"] == True
                                                    ):
                                                        conStrDic = {}
                                                        conStr = item["info"]["connectionString"].split(";")
                                                        for conStrValue in conStr:
                                                            spltval = conStrValue.split("=")
                                                            conStrDic[spltval[0]] = spltval[1]
                                                        if "DBCLIENT" in conStrDic:
                                                            if (
                                                                str(conStrDic["DBCLIENT"]).upper()
                                                                == "postgresql".upper()
                                                            ):
                                                                self._featureServiceFieldCase = "lower"
                        except urllib2.HTTPError, err:
                            print err
                        except Exception, e:
                            print e
                        except Exception, e:
                            print e
Example #10
0
    def stageContent(self, configFiles, dateTimeFormat=None):

        results = None
        groups = None
        items = None
        group = None
        content = None
        contentInfo = None
        startTime = None
        orgTools = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            print(
                "********************Stage Content Started********************"
            )

            print("Script started at %s" %
                  scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print("Error creating org tools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(
                            config_file=configFile)
                        if config is not None:
                            if 'ContentItems' in config:
                                startTime = datetime.datetime.now()
                                print("Processing config %s, starting at: %s" %
                                      (configFile,
                                       startTime.strftime(dateTimeFormat)))
                                contentInfo = config['ContentItems']
                                for cont in contentInfo:
                                    content = cont['Content']
                                    group = cont['ShareToGroup']

                                    print("Sharing content to: %s" % group)
                                    if os.path.isfile(content):
                                        with open(content, 'rb') as csvfile:
                                            items = []
                                            groups = []
                                            for row in csv.DictReader(
                                                    csvfile, dialect='excel'):
                                                if cont['Type'] == "Group":
                                                    groups.append(row['id'])
                                                elif cont['Type'] == "Items":
                                                    items.append(row['id'])
                                            results = orgTools.shareItemsToGroup(
                                                shareToGroupName=group,
                                                items=items,
                                                groups=groups)

                                print(
                                    "Config %s completed, time to complete: %s"
                                    %
                                    (configFile,
                                     str(datetime.datetime.now() - startTime)))
                            else:
                                print(
                                    "Config file missing ContentItems section")
                        else:
                            print("Config %s not found" % configFile)

        except (TypeError, ValueError, AttributeError) as e:
            print(e)
        except (common.ArcRestHelperError) as e:
            print("error in function: %s" % e[0]['function'])
            print("error on line: %s" % e[0]['line'])
            print("error in file name: %s" % e[0]['filename'])
            print("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(
                        e,
                    (ReportTools.ReportToolsError, DataPrep.DataPrepError)):
                    print("error in function: %s" % e[0]['function'])
                    print("error on line: %s" % e[0]['line'])
                    print("error in file name: %s" % e[0]['filename'])
                    print("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print("error on line: %s" % line)
                    print("error in file name: %s" % filename)
                    print("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print("error on line: %s" % line)
                print("error in file name: %s" % filename)
                print("with error message: %s" % synerror)
        finally:
            print("Script complete, time to complete: %s" %
                  str(datetime.datetime.now() - scriptStartTime))
            print("###############Stage Content Completed#################")
            print("")
            #if orgTools is not None:
            #orgTools.dispose()

            results = None
            groups = None
            items = None
            group = None
            content = None
            contentInfo = None
            startTime = None
            orgTools = None

            del results
            del groups
            del items
            del group
            del content
            del contentInfo
            del startTime
            del orgTools

            gc.collect()
Example #11
0
    def publishfromconfig(self,
                          configFiles,
                          combinedApp=None,
                          dateTimeFormat=None):
        publishTools = None
        webmaps = None
        config = None
        resultsItems = None
        resultFS = None
        resultMaps = None
        resultApps = None
        combinedResults = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'
        env.overwriteOutput = True

        scriptStartTime = datetime.datetime.now()
        try:

            webmaps = []
            print "********************Script Started********************"

            print "Script started at %s" % scriptStartTime.strftime(
                dateTimeFormat)

            # start report processing (moved out from under ArcREST logic. no AGO crednetials needed to run reports)
            for configFile in configFiles:
                config = common.init_config_json(config_file=configFile)
                if config is not None:
                    if 'ReportDetails' in config:
                        if reportToolsInstalled == False:
                            print "Report section is included in the config file but the solutionreporttools cannot be located"
                        else:
                            reportConfig = config['ReportDetails']
                            # This code checks to see if you want to export the data from SDE to a local GDB. The parameter is set in config file.
                            # Could be performance gain to run locally. If you choose this option, both the report and the data prep in memory config
                            # are modified so they can point to the local temp location.

                            if 'RunReport' in reportConfig and (
                                    str(reportConfig['RunReport']).upper()
                                    == "TRUE"
                                    or str(reportConfig['RunReport']).upper()
                                    == "YES"):
                                reportConfig = ReportTools.reportDataPrep(
                                    reportConfig)

                                print "-----Report Section Starting-----"
                                startTime = datetime.datetime.now()
                                print "Processing reports in config %s, starting at: %s" % (
                                    configFile,
                                    startTime.strftime(dateTimeFormat))
                                ReportTools.create_report_layers_using_config(
                                    config=reportConfig)
                                print "Reports in config %s completed, time to complete: %s" % (
                                    configFile,
                                    str(datetime.datetime.now() - startTime))

                                print "-----Report Section Complete-----"
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                            publishData = 'TRUE'
                        if (str(publishData).upper() == "TRUE"
                                or str(publishData).upper() == "YES"):

                            print " "
                            print "-----Publishing Section Starting-----"
                            startTime = datetime.datetime.now()
                            print "Processing publishing in config %s, starting at: %s" % (
                                configFile, startTime.strftime(dateTimeFormat))

                            publishTools = publishingtools.publishingtools(
                                securityinfo=self)
                            if publishTools.valid == False:
                                print "Error creating publishing tools: %s" % publishTools.message
                            else:
                                print "Publishing tools created: %s" % publishTools.message
                                resultFS = []
                                if 'Items' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Creating Items: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultsItems = publishTools.publishItems(
                                        items_info=publishingConfig['Items'])
                                    print "Items created, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)

                                if 'FeatureCollections' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Creating Feature Collection: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultFS = publishTools.publishFeatureCollections(
                                        configs=publishingConfig[
                                            'FeatureCollections'])
                                    print "Feature Collection published, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)
                                if 'FeatureServices' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Creating Feature Services: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultFS = resultFS + publishTools.publishFsFromMXD(
                                        fs_config=publishingConfig[
                                            'FeatureServices'])
                                    print "Feature Services published, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)
                                if 'ExistingServices' in publishingConfig:

                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Updating Existing Feature Services: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultES = publishTools.updateFeatureService(
                                        efs_config=publishingConfig[
                                            'ExistingServices'])
                                    print "Updating Existing Feature Services completed, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)
                                if 'MapDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Creating maps: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultMaps = publishTools.publishMap(
                                        maps_info=publishingConfig[
                                            'MapDetails'],
                                        fsInfo=resultFS,
                                        itInfo=resultsItems)
                                    for maps in resultMaps:
                                        if 'MapInfo' in maps:
                                            if 'Results' in maps['MapInfo']:
                                                if 'itemId' in maps['MapInfo'][
                                                        'Results']:
                                                    webmaps.append(
                                                        maps['MapInfo']
                                                        ['Results']['itemId'])
                                    print "Creating maps completed, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)
                                if 'AppDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print " "
                                    print "Creating apps: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                    resultApps = publishTools.publishApp(
                                        app_info=publishingConfig[
                                            'AppDetails'],
                                        map_info=resultMaps)
                                    print "Creating apps completed, time to complete: %s" % str(
                                        datetime.datetime.now() -
                                        startSectTime)

                            print "Publishing complete in config %s completed, time to complete: %s" % (
                                configFile,
                                str(datetime.datetime.now() - startTime))

                            print "-----Publishing Section Complete-----"

                else:
                    print "Config %s not found" % configFile

            if combinedApp:
                if os.path.exists(combinedApp):
                    print " "
                    startSectTime = datetime.datetime.now()
                    print "Creating combined result: %s" % str(
                        startSectTime.strftime(dateTimeFormat))

                    config = common.init_config_json(config_file=combinedApp)
                    combinedResults = publishTools.publishCombinedWebMap(
                        maps_info=config['PublishingDetails']['MapDetails'],
                        webmaps=webmaps)
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                            publishData = 'TRUE'

                        if (str(publishData).upper() == "TRUE"
                                or str(publishData).upper() == "YES"):

                            if 'AppDetails' in publishingConfig:
                                resultApps = publishTools.publishApp(
                                    app_info=publishingConfig['AppDetails'],
                                    map_info=combinedResults)
                            print "Creating combind result completed, time to complete: %s" % str(
                                datetime.datetime.now() - startSectTime)
        except (TypeError, ValueError, AttributeError), e:
            print e
Example #12
0
    def createGroups(self, configFiles, dateTimeFormat=None):
        groupInfo = None
        groupFile = None
        iconPath = None
        startTime = None
        thumbnail = None
        result = None
        config = None
        sciptPath = None
        orgTools = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'
        env.overwriteOutput = True

        scriptStartTime = datetime.datetime.now()
        try:

            print "********************Create Groups********************"

            print "Script started at %s" % scriptStartTime.strftime(
                dateTimeFormat)

            if self.securityhandler.valid == False:
                print "Login required"
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print "Error creating orgtools"
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(
                            config_file=configFile)
                        if config is not None:

                            startTime = datetime.datetime.now()
                            print "Processing config %s, starting at: %s" % (
                                configFile, startTime.strftime(dateTimeFormat))

                            groupInfo = config['Groups']
                            groupFile = groupInfo['GroupInfo']
                            iconPath = groupInfo['IconPath']

                            if os.path.isfile(groupFile):
                                with open(groupFile, 'rb') as csvfile:

                                    for row in csv.DictReader(csvfile,
                                                              dialect='excel'):
                                        if os.path.isfile(
                                                os.path.join(
                                                    iconPath,
                                                    row['thumbnail'])):
                                            thumbnail = os.path.join(
                                                iconPath, row['thumbnail'])
                                            if not os.path.isabs(thumbnail):

                                                sciptPath = os.getcwd()
                                                thumbnail = os.path.join(
                                                    sciptPath, thumbnail)

                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'],thumbnail=thumbnail)

                                        else:
                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'])

                                        if result is None:
                                            pass
                                        else:
                                            print "Group created: " + result.title

                            print "Config %s completed, time to complete: %s" % (
                                configFile,
                                str(datetime.datetime.now() - startTime))

                        else:
                            print "Config %s not found" % configFile

        except (TypeError, ValueError, AttributeError), e:
            print e
Example #13
0
 def stageContent(self,configFiles,dateTimeFormat=None):
   
     results = None    
     groups = None    
     items = None    
     group = None
     content = None
     contentInfo = None
     startTime = None
     orgTools = None
     
     
     if dateTimeFormat is None:
         dateTimeFormat = '%Y-%m-%d %H:%M'
     env.overwriteOutput = True    
     
    
     scriptStartTime = datetime.datetime.now()
     try:
 
         print "********************Stage Content Started********************"
 
         print "Script started at %s" % scriptStartTime.strftime(dateTimeFormat)
               
         if self.securityhandler.valid == False:
             print "Login required"
         else:
             orgTools = orgtools.orgtools(securityinfo=self)
 
             if orgTools is None:
                 print "Error creating org tools"
             else:
              
                 for configFile in configFiles:
 
                     config = common.init_config_json(config_file=configFile)
                     if config is not None:
                         if 'ContentItems' in config:
                             startTime = datetime.datetime.now()
                             print "Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))
                             contentInfo = config['ContentItems']
                             for cont in contentInfo:
                                 content = cont['Content']
                                 group = cont['ShareToGroup']
                     
                                 print "Sharing content to: %s" % group
                                 if os.path.isfile(content):
                                     with open(content, 'rb') as csvfile:
                                         items = []
                                         groups = []
                                         for row in csv.DictReader(csvfile,dialect='excel'):
                                             if cont['Type'] == "Group":
                                                 groups.append(row['id'])
                                             elif cont['Type'] == "Items":
                                                 items.append(row['id'])
                                         results = orgTools.shareItemsToGroup(shareToGroupName=group,items=items,groups=groups)
     
                             print "Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))
                         else:
                             print "Config file missing ContentItems section"    
                     else:
                         print "Config %s not found" % configFile
 
 
     except(TypeError,ValueError,AttributeError),e:
         print e
Example #14
0
 def publishfromconfig(self,configFiles,combinedApp=None,dateTimeFormat=None):
     publishTools = None
     webmaps = None
     config = None
     resultFS = None
     resultMaps = None
     resultApps = None
     combinedResults = None
 
     if dateTimeFormat is None:
         dateTimeFormat = '%Y-%m-%d %H:%M'
     env.overwriteOutput = True
 
     scriptStartTime = datetime.datetime.now()
     try:
         
         webmaps = []
         print "********************Script Started********************"
       
         print "Script started at %s" % scriptStartTime.strftime(dateTimeFormat)
       
         # start report processing (moved out from under ArcREST logic. no AGO crednetials needed to run reports)
         for configFile in configFiles:
             config = common.init_config_json(config_file=configFile)
             if config is not None:
                 if 'ReportDetails' in config:
                     if reportToolsInstalled == False:
                         print "Report section is included in the config file but the solutionreporttools cannot be located"
                     else:
                         reportConfig = config['ReportDetails']
                         # This code checks to see if you want to export the data from SDE to a local GDB. The parameter is set in config file.
                         # Could be performance gain to run locally. If you choose this option, both the report and the data prep in memory config
                         # are modified so they can point to the local temp location.
 
                         if 'RunReport' in reportConfig and (str(reportConfig['RunReport']).upper() =="TRUE" or str(reportConfig['RunReport']).upper() =="YES"):
                             reportConfig = ReportTools.reportDataPrep(reportConfig)
 
                             print "-----Report Section Starting-----"
                             startTime = datetime.datetime.now()
                             print "Processing reports in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))
                             ReportTools.create_report_layers_using_config(config=reportConfig)
                             print "Reports in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))
 
                             print "-----Report Section Complete-----"
                 if 'PublishingDetails' in config:
                     publishingConfig = config['PublishingDetails']
 
                     if 'PublishData' in publishingConfig:
                         publishData = publishingConfig['PublishData']
                     else:
                         print "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                         publishData = 'TRUE'
                     if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"):
 
                         print " "
                         print "-----Publishing Section Starting-----"
                         startTime = datetime.datetime.now()
                         print "Processing publishing in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))
 
 
                         publishTools = publishingtools.publishingtools(securityinfo=self)
                         if publishTools.valid == False :
                             print "Error creating publishing tools: %s" % publishTools.message
                         else:
                             print "Publishing tools created: %s" % publishTools.message
 
                             if 'FeatureServices' in publishingConfig:
                                 startSectTime = datetime.datetime.now()
                                 print " "
                                 print "Creating Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat))
                                 resultFS = publishTools.publishFsFromMXD(fs_config=publishingConfig['FeatureServices'])
                                 print "Feature Services published, time to complete: %s" % str(datetime.datetime.now() - startSectTime)
                             if 'ExistingServices' in publishingConfig:
 
                                 startSectTime = datetime.datetime.now()
                                 print " "
                                 print "Updating Existing Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat))
                                 resultES = publishTools.updateFeatureService(efs_config=publishingConfig['ExistingServices'])
                                 print "Updating Existing Feature Services completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)
                             if 'MapDetails' in publishingConfig:
                                 startSectTime = datetime.datetime.now()
                                 print " "
                                 print "Creating maps: %s" % str(startSectTime.strftime(dateTimeFormat))
                                 resultMaps = publishTools.publishMap(maps_info=publishingConfig['MapDetails'],fsInfo=resultFS)
                                 for maps in resultMaps:
                                     if 'MapInfo' in maps:
                                         if 'Results' in maps['MapInfo']:
                                             if 'id' in maps['MapInfo']['Results']:
                                                 webmaps.append(maps['MapInfo']['Results']['id'])
                                 print "Creating maps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)
                             if 'AppDetails' in publishingConfig:
                                 startSectTime = datetime.datetime.now()
                                 print " "
                                 print "Creating apps: %s" % str(startSectTime.strftime(dateTimeFormat))
                                 resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=resultMaps)
                                 print "Creating apps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)
 
 
                         print "Publishing complete in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))
 
                         print "-----Publishing Section Complete-----"
 
             else:
                 print "Config %s not found" % configFile
 
         if combinedApp:
             if os.path.exists(combinedApp):
                 print " "
                 startSectTime = datetime.datetime.now()
                 print "Creating combind result: %s" % str(startSectTime.strftime(dateTimeFormat))
 
                 config = common.init_config_json(config_file=combinedApp)
                 combinedResults = publishTools.publishCombinedWebMap(maps_info=config['PublishingDetails']['MapDetails'],webmaps=webmaps)
                 if 'PublishingDetails' in config:
                     publishingConfig = config['PublishingDetails']
 
                     if 'PublishData' in publishingConfig:
                         publishData = publishingConfig['PublishData']
                     else:
                         print "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                         publishData = 'TRUE'
 
                     if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"):
 
                         if 'AppDetails' in publishingConfig:
                             resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=combinedResults)
                         print "Creating combind result completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)
     except(TypeError,ValueError,AttributeError),e:
         print e
Example #15
0
 def createGroups(self,configFiles,dateTimeFormat=None):
     groupInfo = None    
     groupFile = None    
     iconPath = None    
     startTime = None
     thumbnail = None
     result = None
     config = None
     sciptPath = None
     orgTools = None
     
     if dateTimeFormat is None:
         dateTimeFormat = '%Y-%m-%d %H:%M'
     env.overwriteOutput = True    
     
     scriptStartTime = datetime.datetime.now()
     try:
 
         print "********************Create Groups********************"
 
         print "Script started at %s" % scriptStartTime.strftime(dateTimeFormat)
      
         if self.securityhandler.valid == False:
             print "Login required"
         else:
             orgTools = orgtools.orgtools(securityinfo=self)
 
             if orgTools is None:
                 print "Error creating orgtools"
             else:
 
                 for configFile in configFiles:
 
                     config = common.init_config_json(config_file=configFile)
                     if config is not None:
                        
                         startTime = datetime.datetime.now()
                         print "Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))
 
                         groupInfo = config['Groups']
                         groupFile = groupInfo['GroupInfo']
                         iconPath = groupInfo['IconPath']
 
                         if os.path.isfile(groupFile):
                             with open(groupFile, 'rb') as csvfile:
 
                                 for row in csv.DictReader(csvfile,dialect='excel'):
                                     if os.path.isfile(os.path.join(iconPath,row['thumbnail'])):
                                         thumbnail = os.path.join(iconPath,row['thumbnail'])
                                         if not os.path.isabs(thumbnail):
 
                                             sciptPath = os.getcwd()
                                             thumbnail = os.path.join(sciptPath,thumbnail)
 
                                         result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                          isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'],thumbnail=thumbnail)
 
                                     else:
                                         result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                          isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'])
 
                                     if 'error' in result:
                                         print "Error Creating Group %s" % result['error']
                                     else:
                                         print "Group created: " + row['title']
 
 
 
                         print "Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))
 
                     else:
                         print "Config %s not found" % configFile
 
 
     except(TypeError,ValueError,AttributeError),e:
         print e
Example #16
0
    def createGroups(self, configFiles, dateTimeFormat=None):
        groupInfo = None
        groupFile = None
        iconPath = None
        startTime = None
        thumbnail = None
        result = None
        config = None
        sciptPath = None
        orgTools = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            print("********************Create Groups********************")

            print("Script started at %s" %
                  scriptStartTime.strftime(dateTimeFormat))

            if self.securityhandler.valid == False:
                print("Login required")
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print("Error creating orgtools")
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(
                            config_file=configFile)
                        if config is not None:

                            startTime = datetime.datetime.now()
                            print("Processing config %s, starting at: %s" %
                                  (configFile,
                                   startTime.strftime(dateTimeFormat)))

                            groupInfo = config['Groups']
                            groupFile = groupInfo['GroupInfo']
                            iconPath = groupInfo['IconPath']

                            if os.path.isfile(groupFile):
                                with open(groupFile, 'rb') as csvfile:

                                    for row in csv.DictReader(csvfile,
                                                              dialect='excel'):
                                        if os.path.isfile(
                                                os.path.join(
                                                    iconPath,
                                                    row['thumbnail'])):
                                            thumbnail = os.path.join(
                                                iconPath, row['thumbnail'])
                                            if not os.path.isabs(thumbnail):

                                                sciptPath = os.getcwd()
                                                thumbnail = os.path.join(
                                                    sciptPath, thumbnail)

                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'],thumbnail=thumbnail)

                                        else:
                                            result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \
                                                             isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'])

                                        if result is None:
                                            pass
                                        else:
                                            print("Group created: " +
                                                  result.title)

                            print("Config %s completed, time to complete: %s" %
                                  (configFile,
                                   str(datetime.datetime.now() - startTime)))

                        else:
                            print("Config %s not found" % configFile)

        except (TypeError, ValueError, AttributeError) as e:
            print(e)
        except (common.ArcRestHelperError) as e:
            print("error in function: %s" % e[0]['function'])
            print("error on line: %s" % e[0]['line'])
            print("error in file name: %s" % e[0]['filename'])
            print("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(
                        e,
                    (ReportTools.ReportToolsError, DataPrep.DataPrepError)):
                    print("error in function: %s" % e[0]['function'])
                    print("error on line: %s" % e[0]['line'])
                    print("error in file name: %s" % e[0]['filename'])
                    print("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print("error on line: %s" % line)
                    print("error in file name: %s" % filename)
                    print("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print("error on line: %s" % line)
                print("error in file name: %s" % filename)
                print("with error message: %s" % synerror)
        finally:
            print("Script complete, time to complete: %s" %
                  str(datetime.datetime.now() - scriptStartTime))
            print("###############Create Groups Completed#################")
            print("")

            #if orgTools is not None:
            #orgTools.dispose()
            groupInfo = None
            groupFile = None
            iconPath = None
            startTime = None
            thumbnail = None
            result = None
            config = None
            sciptPath = None
            orgTools = None
            del groupInfo
            del groupFile
            del iconPath
            del startTime
            del thumbnail
            del result
            del config
            del sciptPath
            del orgTools

            gc.collect()
Example #17
0
    def publishfromconfig(self,
                          configFiles,
                          combinedApp=None,
                          dateTimeFormat=None):
        publishTools = None
        webmaps = None
        config = None
        resultsItems = None
        resultFS = None
        resultMaps = None
        resultApps = None
        combinedResults = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'

        scriptStartTime = datetime.datetime.now()
        try:

            webmaps = []
            print("********************Script Started********************")

            print("Script started at %s" %
                  scriptStartTime.strftime(dateTimeFormat))

            # start report processing (moved out from under ArcREST logic. no AGO crednetials needed to run reports)
            for configFile in configFiles:
                config = common.init_config_json(config_file=configFile)
                if config is not None:
                    if 'ReportDetails' in config:
                        if reportToolsInstalled == False:
                            print(
                                "Report section is included in the config file but the solutionreporttools cannot be located"
                            )
                        else:
                            reportConfig = config['ReportDetails']
                            # This code checks to see if you want to export the data from SDE to a local GDB. The parameter is set in config file.
                            # Could be performance gain to run locally. If you choose this option, both the report and the data prep in memory config
                            # are modified so they can point to the local temp location.

                            if 'RunReport' in reportConfig and (
                                    str(reportConfig['RunReport']).upper()
                                    == "TRUE"
                                    or str(reportConfig['RunReport']).upper()
                                    == "YES"):
                                reportConfig = ReportTools.reportDataPrep(
                                    reportConfig)

                                print("-----Report Section Starting-----")
                                startTime = datetime.datetime.now()
                                print(
                                    "Processing reports in config %s, starting at: %s"
                                    % (configFile,
                                       startTime.strftime(dateTimeFormat)))
                                ReportTools.create_report_layers_using_config(
                                    config=reportConfig)
                                print(
                                    "Reports in config %s completed, time to complete: %s"
                                    %
                                    (configFile,
                                     str(datetime.datetime.now() - startTime)))

                                print("-----Report Section Complete-----")
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print(
                                "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                            )
                            publishData = 'TRUE'
                        if (str(publishData).upper() == "TRUE"
                                or str(publishData).upper() == "YES"):

                            print(" ")
                            print("-----Publishing Section Starting-----")
                            startTime = datetime.datetime.now()
                            print(
                                "Processing publishing in config %s, starting at: %s"
                                % (configFile,
                                   startTime.strftime(dateTimeFormat)))

                            publishTools = publishingtools.publishingtools(
                                securityinfo=self)
                            if publishTools.valid == False:
                                print("Error creating publishing tools: %s" %
                                      publishTools.message)
                            else:
                                print("Publishing tools created: %s" %
                                      publishTools.message)
                                resultFS = []
                                if 'Items' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print("Creating Items: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                          )
                                    resultsItems = publishTools.publishItems(
                                        items_info=publishingConfig['Items'])
                                    print(
                                        "Items created, time to complete: %s" %
                                        str(datetime.datetime.now() -
                                            startSectTime))

                                if 'FeatureCollections' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print("Creating Feature Collection: %s" %
                                          str(
                                              startSectTime.strftime(
                                                  dateTimeFormat)))
                                    resultFS = publishTools.publishFeatureCollections(
                                        configs=publishingConfig[
                                            'FeatureCollections'])
                                    print(
                                        "Feature Collection published, time to complete: %s"
                                        % str(datetime.datetime.now() -
                                              startSectTime))
                                if 'FeatureServices' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print("Creating Feature Services: %s" %
                                          str(
                                              startSectTime.strftime(
                                                  dateTimeFormat)))
                                    resultFS = resultFS + publishTools.publishFsFromMXD(
                                        fs_config=publishingConfig[
                                            'FeatureServices'])
                                    if len(resultFS) == 0:
                                        print(
                                            "Exiting, error creating feature services"
                                        )
                                        return
                                    print(
                                        "Feature Services published, time to complete: %s"
                                        % str(datetime.datetime.now() -
                                              startSectTime))
                                if 'ExistingServices' in publishingConfig:

                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print(
                                        "Updating Existing Feature Services: %s"
                                        % str(
                                            startSectTime.strftime(
                                                dateTimeFormat)))
                                    resultES = publishTools.updateFeatureService(
                                        efs_config=publishingConfig[
                                            'ExistingServices'])
                                    print(
                                        "Updating Existing Feature Services completed, time to complete: %s"
                                        % str(datetime.datetime.now() -
                                              startSectTime))
                                if 'MapDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print("Creating maps: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                          )
                                    resultMaps = publishTools.publishMap(
                                        maps_info=publishingConfig[
                                            'MapDetails'],
                                        fsInfo=resultFS,
                                        itInfo=resultsItems)
                                    for maps in resultMaps:
                                        if 'MapInfo' in maps:
                                            if 'Results' in maps['MapInfo']:
                                                if 'itemId' in maps['MapInfo'][
                                                        'Results']:
                                                    webmaps.append(
                                                        maps['MapInfo']
                                                        ['Results']['itemId'])
                                    print(
                                        "Creating maps completed, time to complete: %s"
                                        % str(datetime.datetime.now() -
                                              startSectTime))
                                if 'AppDetails' in publishingConfig:
                                    startSectTime = datetime.datetime.now()
                                    print(" ")
                                    print("Creating apps: %s" % str(
                                        startSectTime.strftime(dateTimeFormat))
                                          )
                                    resultApps = publishTools.publishApp(
                                        app_info=publishingConfig[
                                            'AppDetails'],
                                        map_info=resultMaps,
                                        fsInfo=resultFS)
                                    print(
                                        "Creating apps completed, time to complete: %s"
                                        % str(datetime.datetime.now() -
                                              startSectTime))

                            print(
                                "Publishing complete in config %s completed, time to complete: %s"
                                % (configFile,
                                   str(datetime.datetime.now() - startTime)))

                            print("-----Publishing Section Complete-----")

                else:
                    print("Config %s not found" % configFile)

            if combinedApp:
                if os.path.exists(combinedApp):
                    print(" ")
                    startSectTime = datetime.datetime.now()
                    print("Creating combined result: %s" %
                          str(startSectTime.strftime(dateTimeFormat)))

                    config = common.init_config_json(config_file=combinedApp)
                    combinedResults = publishTools.publishCombinedWebMap(
                        maps_info=config['PublishingDetails']['MapDetails'],
                        webmaps=webmaps)
                    if 'PublishingDetails' in config:
                        publishingConfig = config['PublishingDetails']

                        if 'PublishData' in publishingConfig:
                            publishData = publishingConfig['PublishData']
                        else:
                            print(
                                "PublishingDetails is missing the PublishData parameter:  type string, values, True or False"
                            )
                            publishData = 'TRUE'

                        if (str(publishData).upper() == "TRUE"
                                or str(publishData).upper() == "YES"):

                            if 'AppDetails' in publishingConfig:
                                resultApps = publishTools.publishApp(
                                    app_info=publishingConfig['AppDetails'],
                                    map_info=combinedResults)
                            print(
                                "Creating combind result completed, time to complete: %s"
                                % str(datetime.datetime.now() - startSectTime))
        except (TypeError, ValueError, AttributeError) as e:
            print(e)
        except (common.ArcRestHelperError) as e:
            print("error in function: %s" % e[0]['function'])
            print("error on line: %s" % e[0]['line'])
            print("error in file name: %s" % e[0]['filename'])
            print("with error message: %s" % e[0]['synerror'])
            if 'arcpyError' in e[0]:
                print("with arcpy message: %s" % e[0]['arcpyError'])

        except Exception as e:
            if (reportToolsInstalled):
                if isinstance(
                        e,
                    (ReportTools.ReportToolsError, DataPrep.DataPrepError)):
                    print("error in function: %s" % e[0]['function'])
                    print("error on line: %s" % e[0]['line'])
                    print("error in file name: %s" % e[0]['filename'])
                    print("with error message: %s" % e[0]['synerror'])
                    if 'arcpyError' in e[0]:
                        print("with arcpy message: %s" % e[0]['arcpyError'])
                else:
                    line, filename, synerror = trace()
                    print("error on line: %s" % line)
                    print("error in file name: %s" % filename)
                    print("with error message: %s" % synerror)
            else:
                line, filename, synerror = trace()
                print("error on line: %s" % line)
                print("error in file name: %s" % filename)
                print("with error message: %s" % synerror)

        finally:
            print("Script complete, time to complete: %s" %
                  str(datetime.datetime.now() - scriptStartTime))
            print("###############Script Completed#################")
            print("")
            if publishTools is not None:
                publishTools.dispose()

            publishTools = None
            webmaps = None
            config = None
            resultFS = None
            resultsItems = None
            resultMaps = None
            resultApps = None
            combinedResults = None

            del publishTools
            del webmaps
            del config
            del resultFS
            del resultMaps
            del resultApps
            del combinedResults
            del resultsItems
            gc.collect()
Example #18
0
    def stageContent(self, configFiles, dateTimeFormat=None):

        results = None
        groups = None
        items = None
        group = None
        content = None
        contentInfo = None
        startTime = None
        orgTools = None

        if dateTimeFormat is None:
            dateTimeFormat = '%Y-%m-%d %H:%M'
        env.overwriteOutput = True

        scriptStartTime = datetime.datetime.now()
        try:

            print "********************Stage Content Started********************"

            print "Script started at %s" % scriptStartTime.strftime(
                dateTimeFormat)

            if self.securityhandler.valid == False:
                print "Login required"
            else:
                orgTools = orgtools.orgtools(securityinfo=self)

                if orgTools is None:
                    print "Error creating org tools"
                else:

                    for configFile in configFiles:

                        config = common.init_config_json(
                            config_file=configFile)
                        if config is not None:
                            if 'ContentItems' in config:
                                startTime = datetime.datetime.now()
                                print "Processing config %s, starting at: %s" % (
                                    configFile,
                                    startTime.strftime(dateTimeFormat))
                                contentInfo = config['ContentItems']
                                for cont in contentInfo:
                                    content = cont['Content']
                                    group = cont['ShareToGroup']

                                    print "Sharing content to: %s" % group
                                    if os.path.isfile(content):
                                        with open(content, 'rb') as csvfile:
                                            items = []
                                            groups = []
                                            for row in csv.DictReader(
                                                    csvfile, dialect='excel'):
                                                if cont['Type'] == "Group":
                                                    groups.append(row['id'])
                                                elif cont['Type'] == "Items":
                                                    items.append(row['id'])
                                            results = orgTools.shareItemsToGroup(
                                                shareToGroupName=group,
                                                items=items,
                                                groups=groups)

                                print "Config %s completed, time to complete: %s" % (
                                    configFile,
                                    str(datetime.datetime.now() - startTime))
                            else:
                                print "Config file missing ContentItems section"
                        else:
                            print "Config %s not found" % configFile

        except (TypeError, ValueError, AttributeError), e:
            print e
Example #19
0
    def __init__(self, securityinfo):

        """Constructor
            A number of different security handlers are suppoted by ArcGIS, this
            function simplifies the creation of the handlers.  A dict with the
            type of handler and information to created it is required to generate
            the handlers

           Inputs:
              securityinfo - The dict with details to create the handler.  The
                        valid parameters are:
                            - security_type: Required, valid values are Portal,
                                NTLM, PKI, OAUTH, LDAP
                            - proxy_url: Optional, url for proxy server
                            - proxy_port: Optional, port for proxy server
                            - referer_url: Optional, url for referer
                            - token_url: Optional, url for token server
                            - username: Optional, Username for log in, not
                                required for all security types
                            - password: Optional, Password for log in, not
                                required for all security types
                            - certificatefile: Only required for PKI
                            - keyfile: Only required for PKI
                            - client_id: Only required for OAuth
                            - secret_id: Only required for OAuth
        """

        if not securityinfo is None:
            if isinstance(securityinfo,securityhandlerhelper):

                self._securityHandler = securityinfo.securityhandler
                self._username = securityinfo._username
                self._password = securityinfo._password
                self._proxy_url = securityinfo._proxy_url
                self._proxy_port = securityinfo._proxy_port
                self._token_url = securityinfo._token_url
                self._security_type = securityinfo._security_type
                self._featureServiceFieldCase = securityinfo._featureServiceFieldCase
                self._keyfile = securityinfo._keyfile
                self._certificatefile = securityinfo._certificatefile
                self._referer_url = securityinfo._referer_url
                self._client_id = securityinfo._client_id
                self._secret_id = securityinfo._secret_id
                self._is_portal = securityinfo._is_portal
                self._message = securityinfo._message
                self._valid = securityinfo._valid

                #self._securityHandler = securityinfo
                return
            else:
                pass
            if isinstance(securityinfo,str) and os.path.isfile(securityinfo):
                securityinfo = common.init_config_json(config_file=securityinfo)
                if 'Credentials' in securityinfo:
                    securityinfo = securityinfo['Credentials']

            if 'security_type' in securityinfo:
                self._security_type = securityinfo['security_type']
            else:
                self._message = 'Security type not specified'
                self._valid = False
                return

            if 'proxy_url' in securityinfo:
                self._proxy_url = securityinfo['proxy_url']

            if 'proxy_port' in securityinfo:
                self._proxy_port = securityinfo['proxy_port']

            if 'referer_url' in securityinfo:
                self._referer_url = securityinfo['referer_url']

            if 'token_url' in securityinfo and securityinfo['token_url'] is not None:
                self._token_url = securityinfo['token_url']
                if not self._token_url.startswith('http://') and \
                   not self._token_url.startswith('https://'):
                    self._token_url = 'https://' + self._token_url

            if 'org_url' in securityinfo and securityinfo['org_url'] is not None:
                self._org_url = securityinfo['org_url']
                if not self._org_url.startswith('http://') and not self._org_url.startswith('https://'):
                    self._org_url = 'http://' + self._org_url

            if 'username' in securityinfo:
                self._username = securityinfo['username']
            if 'password' in securityinfo:
                self._password = securityinfo['password']

            if 'certificatefile' in securityinfo:
                self._certificatefile = securityinfo['certificatefile']
            if 'keyfile' in securityinfo:
                self._keyfile = securityinfo['keyfile']

            if 'client_id' in securityinfo:
                self._client_id = securityinfo['client_id']
            if 'secret_id' in securityinfo:
                self._secret_id = securityinfo['secret_id']

            if str(securityinfo['security_type']).upper() == 'ArcGIS'.upper():

                self._securityHandler = security.ArcGISTokenSecurityHandler(proxy_url=self._proxy_url,
                                            proxy_port=self._proxy_port)

                self._org_url = self._securityHandler.org_url
                self._username = self._securityHandler.username
                self._valid = True
                self._message = "ArcGIS security handler created"
            elif str(securityinfo['security_type']).upper() == 'Portal'.upper() or \
                 str(securityinfo['security_type']).upper() == 'AGOL'.upper():
                if self._org_url is None or self._org_url == '':
                    self._org_url = 'http://www.arcgis.com'
                if self._username is None or self._username == '' or \
                    self._password is None or self._password == '':
                    self._message = "No username or password, no security handler generated"
                    self._valid = True
                else:
                    if self._org_url is None or '.arcgis.com' in self._org_url:
                        self._securityHandler = security.AGOLTokenSecurityHandler(username=self._username,
                                                                                 password=self._password,
                                                                                 org_url=self._org_url,
                                                                                 token_url=self._token_url,
                                                                                 proxy_url=self._proxy_url,
                                                                                 proxy_port=self._proxy_port)
                        self._message = "ArcGIS Online security handler created"
                    else:
                        self._securityHandler = security.PortalTokenSecurityHandler(username=self._username,
                                                                                   password=self._password,
                                                                                   org_url=self._org_url,
                                                                                   proxy_url=self._proxy_url,
                                                                                   proxy_port=self._proxy_port)
                        self._message = "Portal security handler created"

            elif str(securityinfo['security_type']).upper() == 'NTLM'.upper():
                if self._username is None or self._username == '' or \
                    self._password is None or self._password == '':
                    self._message = "Username and password required for NTLM"
                    self._valid = False
                else:
                    self._securityHandler = security.NTLMSecurityHandler(username=self._username,
                                                                        password=self._password,
                                                                        org_url=self._org_url,
                                                                        proxy_url=self._proxy_url,
                                                                        proxy_port=self._proxy_port,
                                                                        referer_url=self._referer_url)
                    self._message = "NTLM security handler created"
            elif str(securityinfo['security_type']).upper() == 'LDAP'.upper():
                if self._username is None or self._username == '' or \
                    self._password is None or self._password == '':
                    self._message = "Username and password required for LDAP"
                    self._valid = False
                else:
                    self._securityHandler = security.LDAPSecurityHandler(username=self._username,
                                                                        password=self._password,
                                                                        org_url=self._org_url,
                                                                        proxy_url=self._proxy_url,
                                                                        proxy_port=self._proxy_port,
                                                                        referer_url=self._referer_url)
                    self._message = "LDAP security handler created"
            elif str(securityinfo['security_type']).upper() == 'PKI'.upper():
                if self._keyfile is None or self._keyfile == '' or \
                    self._certificatefile is None or self._certificatefile == '':
                    self._message = "Key file and certification file required for PKI"
                    self._valid = False
                else:
                    self._securityHandler = security.PKISecurityHandler(keyfile = self._keyfile,
                                                                        certificatefile = self._certificatefile,
                                                                        org_url=self._org_url,
                                                                        proxy_url=self._proxy_url,
                                                                        proxy_port=self._proxy_port,
                                                                        referer_url=self._referer_url)
                    self._message = "PKI security handler created"
            elif str(securityinfo['security_type']).upper() == 'OAUTH'.upper():
                if self._secret_id is None or self._secret_id == '' or \
                    self._client_id is None or self._client_id == '':
                    self._message = "client_id and secret_id required for OAUTH"
                    self._valid = False
                else:
                    self._securityHandler = security.OAuthSecurityHandler(client_id=self._client_id,
                                                                         secret_id = self._secret_id,
                                                                         org_url=self._org_url,
                                                                         proxy_url=self._proxy_url,
                                                                         proxy_port=self._proxy_port)

                    self._message = "OAuth security handler created"
            else:
                print ("No valid security type set")
                self._message = "No valid security type set"
            if self._securityHandler is not None:

                admin = Administration(url=self._org_url,
                                       securityHandler=self._securityHandler)

                try:
                    portal = admin.portals.portalSelf
                    for hostingServer in portal.featureServers:
                        if isinstance(hostingServer, AGSAdministration):
                            try:
                                serData = hostingServer.data

                                dataItems = serData.rootDataItems
                                if 'rootItems' in dataItems:
                                    for rootItem in dataItems['rootItems']:
                                        if rootItem == '/enterpriseDatabases':
                                            rootItems = serData.findDataItems(ancestorPath=rootItem,type='fgdb,egdb')
                                            if not rootItems is None and 'items' in rootItems:
                                                for item in rootItems['items']:
                                                    if 'info' in item:
                                                        if 'isManaged' in item['info'] and item['info']['isManaged'] == True:
                                                            conStrDic = {}
                                                            conStr = item['info']['connectionString'].split(";")
                                                            for conStrValue in conStr:
                                                                spltval = conStrValue.split("=")
                                                                conStrDic[spltval[0]] = spltval[1]
                                                            if 'DBCLIENT' in conStrDic:
                                                                if str(conStrDic['DBCLIENT']).upper() == 'postgresql'.upper():
                                                                    self._featureServiceFieldCase = 'lower'
                            except HTTPError as err:
                                if err.code == 403:
                                    print ("Admistrative access denied, unable to check if hosting servers")
                                else:
                                    print (err)
                            except Exception as e:
                                print (e)

                except HTTPError as err:
                    if err.code == 403:
                        print ("Admistrative access denied, unable to check if hosting servers")
                    else:
                        print (err)
                except Exception as e:
                    print (e)

                if 'error' in self._securityHandler.message:
                    self._message = self._securityHandler.message
                    self._valid = False

                else:
                    if self._securityHandler.message is not None:
                        self._message = self._securityHandler.message
                    self._valid = True
        else:
            self._message = 'Security info not set'
            self._valid = True
Example #20
0
    def __init__(self, securityinfo):

        """Constructor
            A number of different security handlers are suppoted by ArcGIS, this
            function simplifies the creation of the handlers.  A dict with the
            type of handler and information to created it is required to generate
            the handlers

           Inputs:
              securityinfo - The dict with details to create the handler.  The
                        valid parameters are:
                            - security_type: Required, valid values are Portal,
                                NTLM, PKI, OAUTH, LDAP
                            - proxy_url: Optional, url for proxy server
                            - proxy_port: Optional, port for proxy server
                            - referer_url: Optional, url for referer
                            - token_url: Optional, url for token server
                            - username: Optional, Username for log in, not
                                required for all security types
                            - password: Optional, Password for log in, not
                                required for all security types
                            - certificatefile: Only required for PKI
                            - keyfile: Only required for PKI
                            - client_id: Only required for OAuth
                            - secret_id: Only required for OAuth
        """

        if not securityinfo is None:
            if isinstance(securityinfo,securityhandlerhelper):

                self._securityHandler = securityinfo.securityhandler
                self._username = securityinfo._username
                self._password = securityinfo._password
                self._proxy_url = securityinfo._proxy_url
                self._proxy_port = securityinfo._proxy_port
                self._token_url = securityinfo._token_url
                self._security_type = securityinfo._security_type
                self._featureServiceFieldCase = securityinfo._featureServiceFieldCase
                self._keyfile = securityinfo._keyfile
                self._certificatefile = securityinfo._certificatefile
                self._referer_url = securityinfo._referer_url
                self._client_id = securityinfo._client_id
                self._secret_id = securityinfo._secret_id
                self._is_portal = securityinfo._is_portal
                self._message = securityinfo._message
                self._valid = securityinfo._valid

                #self._securityHandler = securityinfo
                return
            else:
                pass
            if isinstance(securityinfo,str) and os.path.isfile(securityinfo):
                securityinfo = common.init_config_json(config_file=securityinfo)
                if 'Credentials' in securityinfo:
                    securityinfo = securityinfo['Credentials']

            if 'security_type' in securityinfo:
                self._security_type = securityinfo['security_type']
            else:
                self._message = 'Security type not specified'
                self._valid = False
                return

            if 'proxy_url' in securityinfo:
                self._proxy_url = securityinfo['proxy_url']

            if 'proxy_port' in securityinfo:
                self._proxy_port = securityinfo['proxy_port']

            if 'referer_url' in securityinfo:
                self._referer_url = securityinfo['referer_url']

            if 'token_url' in securityinfo and securityinfo['token_url'] is not None:
                self._token_url = securityinfo['token_url']
                if not self._token_url.startswith('http://') and \
                   not self._token_url.startswith('https://'):
                    self._token_url = 'https://' + self._token_url

            if 'org_url' in securityinfo and securityinfo['org_url'] is not None:
                self._org_url = securityinfo['org_url']
                if not self._org_url.startswith('http://') and not self._org_url.startswith('https://'):
                    self._org_url = 'http://' + self._org_url

            if 'username' in securityinfo:
                self._username = securityinfo['username']
            if 'password' in securityinfo:
                self._password = securityinfo['password']

            if 'certificatefile' in securityinfo:
                self._certificatefile = securityinfo['certificatefile']
            if 'keyfile' in securityinfo:
                self._keyfile = securityinfo['keyfile']

            if 'client_id' in securityinfo:
                self._client_id = securityinfo['client_id']
            if 'secret_id' in securityinfo:
                self._secret_id = securityinfo['secret_id']

            if str(securityinfo['security_type']).upper() == 'ArcGIS'.upper():

                self._securityHandler = security.ArcGISTokenSecurityHandler(proxy_url=self._proxy_url,
                                            proxy_port=self._proxy_port)

                self._org_url = self._securityHandler.org_url
                self._username = self._securityHandler.username
                self._valid = True
                self._message = "ArcGIS security handler created"
            elif str(securityinfo['security_type']).upper() == 'Portal'.upper() or \
                 str(securityinfo['security_type']).upper() == 'AGOL'.upper():
                if self._org_url is None or self._org_url =='':
                    self._org_url = 'http://www.arcgis.com'
                if self._username == "" or self._password == "":
                    self._message = "No username or password, no security handler generated"
                    self._valid = True
                else:
                    if self._org_url is None or '.arcgis.com' in self._org_url:
                        self._securityHandler = security.AGOLTokenSecurityHandler(username=self._username,
                                                                                 password=self._password,
                                                                                 org_url=self._org_url,
                                                                                 token_url=self._token_url,
                                                                                 proxy_url=self._proxy_url,
                                                                                 proxy_port=self._proxy_port)
                        self._message = "ArcGIS Online security handler created"
                    else:
                        self._securityHandler = security.PortalTokenSecurityHandler(username=self._username,
                                                                                   password=self._password,
                                                                                   org_url=self._org_url,
                                                                                   proxy_url=self._proxy_url,
                                                                                   proxy_port=self._proxy_port)
                        self._message = "Portal security handler created"

            elif str(securityinfo['security_type']).upper() == 'NTLM'.upper():
                if self._username == "" or self._password == "":
                    self._message = "Username and password required for NTLM"
                    self._valid = False
                self._securityHandler = security.NTLMSecurityHandler(username=self._username,
                                                    password=self._password,
                                                    org_url=self._org_url,
                                                    proxy_url=self._proxy_url,
                                                    proxy_port=self._proxy_port,
                                                    referer_url=self._referer_url
                                                     )
                self._message = "NTLM security handler created"
            elif str(securityinfo['security_type']).upper() == 'LDAP'.upper():
                if self._username == "" or self._password == "":
                    self._message = "Username and password required for LDAP"
                    self._valid = False
                self._securityHandler = security.LDAPSecurityHandler(username=self._username,
                                                    password=self._password,
                                                    org_url=self._org_url,
                                                    proxy_url=self._proxy_url,
                                                    proxy_port=self._proxy_port,
                                                    referer_url=self._referer_url
                                                    )
                self._message = "LDAP security handler created"
            elif str(securityinfo['security_type']).upper() == 'PKI'.upper():
                if self._keyfile == "" or self._certificatefile == "":
                    self._message = "key file and certification file required for PKI"
                    self._valid = False
                self._securityHandler = security.PKISecurityHandler(keyfile = self._keyfile,
                                                    certificatefile = self._certificatefile,
                                                    org_url=self._org_url,
                                                    proxy_url=self._proxy_url,
                                                    proxy_port=self._proxy_port,
                                                    referer_url=self._referer_url
                                                    )
                self._message = "PKI security handler created"
            elif str(securityinfo['security_type']).upper() == 'OAUTH'.upper():
                if self._secret_id == "" or self._client_id == "":
                    self._message = "client_id and secret_id required for OAUTH"
                    self._valid = False
                self._securityHandler = security.OAuthSecurityHandler(client_id=self._client_id,
                                                    secret_id = self._secret_id,
                                                    org_url=self._org_url,
                                                    proxy_url=self._proxy_url,
                                                    proxy_port=self._proxy_port
                                                    )

                self._message = "OAuth security handler created"
            else:
                print "No valid security type set"
                self._message = "No valid security type set"

            admin = Administration(url=self._org_url,
                                   securityHandler=self._securityHandler)

            try:
                portal = admin.portals.portalSelf
                for hostingServer in portal.featureServers:
                    if isinstance(hostingServer, AGSAdministration):
                        try:
                            serData = hostingServer.data

                            dataItems = serData.rootDataItems
                            if 'rootItems' in dataItems:
                                for rootItem in dataItems['rootItems']:
                                    if rootItem == '/enterpriseDatabases':
                                        rootItems = serData.findDataItems(ancestorPath=rootItem,type='fgdb,egdb')
                                        if not rootItems is None and 'items' in rootItems:
                                            for item in rootItems['items']:
                                                if 'info' in item:
                                                    if 'isManaged' in item['info'] and item['info']['isManaged'] == True:
                                                        conStrDic = {}
                                                        conStr = item['info']['connectionString'].split(";")
                                                        for conStrValue in conStr:
                                                            spltval = conStrValue.split("=")
                                                            conStrDic[spltval[0]] = spltval[1]
                                                        if 'DBCLIENT' in conStrDic:
                                                            if str(conStrDic['DBCLIENT']).upper() == 'postgresql'.upper():
                                                                self._featureServiceFieldCase = 'lower'
                        except urllib2.HTTPError, err:
                            print err
                        except Exception, e:
                            print e
                        except Exception, e:
                            print e