Пример #1
0
def deadvertizeAllGlideinClientMonitoring(factory_name,
                                          glidein_name,
                                          entry_name,
                                          factory_collector=DEFAULT_VAL):
    """
    Deadvertize  monitoring classads for the given entry.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
            fd.write(
                'Requirements = (ReqGlidein == "%s@%s@%s")&&(GlideinMyType == "%s")\n'
                % (entry_name, glidein_name, factory_name,
                   factoryConfig.factoryclient_id))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam,
                             "INVALIDATE_LICENSE_ADS",
                             factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
Пример #2
0
    def writeToMultiClassadFile(self, filename=None, append=True):
        # filename: Name of the file to write classads to
        # append: Wether the classads need to be appended to the file
        #         If we create file append is in a way ignored

        if filename is None:
            filename = classadSupport.generate_classad_filename(
                prefix='gfi_adm_gfc')
            append = False

        for el in self.client_data:
            createGlideinClientMonitoringFile(filename,
                                              self.factory_name,
                                              self.glidein_name,
                                              self.entry_name,
                                              el['client_name'],
                                              el['client_int_name'],
                                              el['client_int_req'],
                                              self.glidein_attrs,
                                              el['client_params'],
                                              el['client_monitors'],
                                              el['limits_triggered'],
                                              do_append=append)
            # Append from here on anyways
            append = True

        return filename
Пример #3
0
def advertizeGlideinClientMonitoring(
    factory_name,
    glidein_name,
    entry_name,
    client_name,
    client_int_name,
    client_int_req,
    glidein_attrs={},
    client_params={},
    client_monitors={},
    factory_collector=DEFAULT_VAL,
):
    tmpnam = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")

    createGlideinClientMonitoringFile(
        tmpnam,
        factory_name,
        glidein_name,
        entry_name,
        client_name,
        client_int_name,
        client_int_req,
        glidein_attrs,
        client_params,
        client_monitors,
    )
    advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True, factory_collector=factory_collector)
Пример #4
0
def deadvertizeGlidein(factory_name,
                       glidein_name,
                       entry_name,
                       factory_collector=DEFAULT_VAL):
    """
    Removes the glidefactory classad advertising the entry from the WMS Collector.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gf')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
            fd.write(
                'Requirements = (Name == "%s@%s@%s")&&(GlideinMyType == "%s")\n'
                % (entry_name, glidein_name, factory_name,
                   factoryConfig.factory_id))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam,
                             "INVALIDATE_ADS_GENERIC",
                             factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
Пример #5
0
def advertizeGlobal(factory_name, glidein_name, supported_signtypes,
                    pub_key_obj, stats_dict={}, factory_collector=DEFAULT_VAL):

    """
    Creates the glidefactoryglobal classad and advertises.

    @type factory_name: string
    @param factory_name: the name of the factory
    @type glidein_name: string
    @param glidein_name: name of the glidein
    @type supported_signtypes: string
    @param supported_signtypes: suppported sign types, i.e. sha1
    @type pub_key_obj: GlideinKey
    @param pub_key_obj: for the frontend to use in encryption
    @type stats_dict: dict
    @param stats_dict: completed jobs statistics
    @type factory_collector: string or None
    @param factory_collector: the collector to query, special value 'default' will get it from the global config

    @todo add factory downtime?
    """

    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfg')

    gfg_classad = FactoryGlobalClassad(factory_name, glidein_name,
                                       supported_signtypes, pub_key_obj)

    try:
        gfg_classad.writeToFile(tmpnam, append=False)
        exe_condor_advertise(tmpnam, gfg_classad.adAdvertiseCmd,
                             factory_collector=factory_collector)
    finally:
        # Unable to write classad
        _remove_if_there(tmpnam)
Пример #6
0
    def writeToMultiClassadFile(self, filename=None, append=True):
        # filename: Name of the file to write classads to
        # append: Wether the classads need to be appended to the file
        #         If we create file append is in a way ignored

        if filename is None:
            filename = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")
            append = False

        for el in self.client_data:
            createGlideinClientMonitoringFile(
                filename,
                self.factory_name,
                self.glidein_name,
                self.entry_name,
                el["client_name"],
                el["client_int_name"],
                el["client_int_req"],
                self.glidein_attrs,
                el["client_params"],
                el["client_monitors"],
                el["limits_triggered"],
                do_append=append,
            )
            # Append from here on anyways
            append = True

        return filename
Пример #7
0
    def do_advertize_multi(self):
        tmpnam = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")

        ap = False
        for el in self.client_data:
            createGlideinClientMonitoringFile(
                tmpnam,
                self.factory_name,
                self.glidein_name,
                self.entry_name,
                el["client_name"],
                el["client_int_name"],
                el["client_int_req"],
                self.glidein_attrs,
                el["client_params"],
                el["client_monitors"],
                do_append=ap,
            )
            ap = True  # Append from here on

        if ap:
            error_arr = []
            try:
                advertizeGlideinClientMonitoringFromFile(
                    tmpnam, remove_file=True, is_multi=True, factory_collector=self.factory_collector
                )
            except condorExe.ExeError, e:
                error_arr.append(e)

            if len(error_arr) > 0:
                raise MultiExeError(error_arr)
Пример #8
0
    def do_advertize_multi(self):
        tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')

        ap = False
        for el in self.client_data:
            createGlideinClientMonitoringFile(tmpnam,
                                              self.factory_name,
                                              self.glidein_name,
                                              self.entry_name,
                                              el['client_name'],
                                              el['client_int_name'],
                                              el['client_int_req'],
                                              self.glidein_attrs,
                                              el['client_params'],
                                              el['client_monitors'],
                                              do_append=ap)
            ap = True  # Append from here on

        if ap:
            error_arr = []
            try:
                advertizeGlideinClientMonitoringFromFile(
                    tmpnam,
                    remove_file=True,
                    is_multi=True,
                    factory_collector=self.factory_collector)
            except condorExe.ExeError as e:
                error_arr.append(e)

            if len(error_arr) > 0:
                raise MultiExeError(error_arr)
def advertizeGlideinClientMonitoring(factory_name, glidein_name, entry_name,
                                     client_name, client_int_name, client_int_req,
                                     glidein_attrs={}, client_params={}, client_monitors={}):
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')

    createGlideinClientMonitoringFile(tmpnam, factory_name, glidein_name, entry_name,
                                      client_name, client_int_name, client_int_req,
                                      glidein_attrs, client_params, client_monitors)
    advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True)
Пример #10
0
def advertizeGlideinClientMonitoring(factory_name, glidein_name, entry_name,
                                     client_name, client_int_name, client_int_req,
                                     glidein_attrs={}, client_params={}, client_monitors={},
                                     factory_collector=DEFAULT_VAL):
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')

    createGlideinClientMonitoringFile(tmpnam, factory_name, glidein_name, entry_name,
                                      client_name, client_int_name, client_int_req,
                                      glidein_attrs, client_params, client_monitors)
    advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True, factory_collector=factory_collector)
Пример #11
0
def deadvertizeGlobal(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Removes the glidefactoryglobal classad advertising the factory globals from the WMS Collector.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfg')
    # TODO: use tempfile
    try:
        with open(tmpnam, "w") as fd:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factory_global)
            fd.write('Requirements = (Name == "%s@%s")&&(GlideinMyType == "%s")\n' % (glidein_name, factory_name, factoryConfig.factory_id))
        exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
    finally:
        _remove_if_there(tmpnam)
Пример #12
0
def deadvertizeFactory(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Deadvertize all entry and global classads for this factory.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_fact')
    # TODO: use tempfile
    try:
        with open(tmpnam, "w") as fd:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
            fd.write('Requirements = (FactoryName =?= "%s")&&(GlideinName =?= "%s")\n' % (factory_name, glidein_name))
        exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
    finally:
        _remove_if_there(tmpnam)
    def do_advertize_iterate(self):
        error_arr = []

        tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfc')

        for el in self.client_data:
            createGlideinClientMonitoringFile(
                tmpnam, self.factory_name, self.glidein_name, self.entry_name,
                el['client_name'], el['client_int_name'], el['client_int_req'],
                self.glidein_attrs, el['client_params'], el['client_monitors'])
            try:
                advertizeGlideinClientMonitoringFromFile(tmpnam,
                                                         remove_file=True)
            except condorExe.ExeError, e:
                error_arr.append(e)
Пример #14
0
def advertizeGlobal(factory_name, glidein_name, supported_signtypes,
                    pub_key_obj, factory_collector=DEFAULT_VAL):
    
    """
    Creates the glidefactoryglobal classad and advertises.
    
    @type factory_name: string
    @param factory_name: the name of the factory
    @type glidein_name: string
    @param glidein_name: name of the glidein
    @type supported_signtypes: string
    @param supported_signtypes: suppported sign types, i.e. sha1
    @type pub_key_obj: GlideinKey
    @param pub_key_obj: for the frontend to use in encryption
    @type factory_collector: string or None
    @param factory_collector: the collector to query, special value 'default' will get it from the global config
    
    @todo add factory downtime?
    """
    
    global factoryConfig
    global advertizeGlobalCounter

    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfg')
    fd = file(tmpnam, "w")

    try:
        try:
            fd.write('MyType = "%s"\n' % factoryConfig.factory_global)
            fd.write('GlideinMyType = "%s"\n' % factoryConfig.factory_global)
            fd.write('GlideinWMSVersion = "%s"\n' % factoryConfig.glideinwms_version)
            fd.write('Name = "%s@%s"\n' % (glidein_name, factory_name))
            fd.write('FactoryName = "%s"\n' % factory_name)
            fd.write('GlideinName = "%s"\n' % glidein_name)
            fd.write('%s = "%s"\n' % (factoryConfig.factory_signtype_id, string.join(supported_signtypes, ',')))
            fd.write('PubKeyID = "%s"\n' % pub_key_obj.get_pub_key_id())
            fd.write('PubKeyType = "%s"\n' % pub_key_obj.get_pub_key_type())
            fd.write('PubKeyValue = "%s"\n' % string.replace(pub_key_obj.get_pub_key_value(), '\n', '\\n'))
            fd.write('DaemonStartTime = %li\n' % start_time)
            fd.write('UpdateSequenceNumber = %i\n' % advertizeGlobalCounter)
            advertizeGlobalCounter += 1
        finally:
            fd.close()
            
        exe_condor_advertise(tmpnam, "UPDATE_MASTER_AD", factory_collector=factory_collector)
               
    finally:
        os.remove(tmpnam)
Пример #15
0
def deadvertizeFactoryClientMonitoring(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Deadvertize all monitoring classads for this factory.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
    # TODO: use tempfile
    try:
        with open(tmpnam, "w") as fd:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
            fd.write('Requirements = (ReqFactoryName=?="%s")&&(ReqGlideinName=?="%s")&&(GlideinMyType == "%s")' %
                     (factory_name, glidein_name, factoryConfig.factoryclient_id))

        exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS", factory_collector=factory_collector)
    finally:
        _remove_if_there(tmpnam)
    def do_advertize(self):
        """
        Do the actual advertizing
        """

        idx = 0
        for factory_pool in self.factory_queue.keys():
            idx = idx + 1
            self.unique_id = 1
            self.adname = classadSupport.generate_classad_filename(prefix="gfi_ad_gc_%li" % (idx))

            # this should be done in parallel, but keep it serial for now
            filename_arr = []
            if frontendConfig.advertise_use_multi == True:
                filename_arr.append(self.adname)
            for el in self.factory_queue[factory_pool]:
                params_obj, key_obj = el
                try:
                    filename_arr_el = self.createAdvertizeWorkFile(factory_pool, params_obj, key_obj)
                    for f in filename_arr_el:
                        if f not in filename_arr:
                            filename_arr.append(f)
                except NoCredentialException:
                    filename_arr = []  # don't try to advertise
                    logSupport.log.warning(
                        "No security credentials match for factory pool %s, not advertising request" % factory_pool
                    )
                except condorExe.ExeError:
                    filename_arr = []  # don't try to advertise
                    logSupport.log.exception(
                        "Error creating request files for factory pool %s, unable to advertise: " % factory_pool
                    )
                    logSupport.log.error(
                        "Error creating request files for factory pool %s, unable to advertise" % factory_pool
                    )

            # Advertize all the files (if multi, should only be one)
            for filename in filename_arr:
                try:
                    advertizeWorkFromFile(
                        factory_pool, filename, remove_file=True, is_multi=frontendConfig.advertise_use_multi
                    )
                except condorExe.ExeError:
                    logSupport.log.exception("Advertising request failed for factory pool %s: " % factory_pool)

        self.factory_queue = {}  # clean queue
def deadvertizeAllGlideinClientMonitoring(factory_name, glidein_name, entry_name):
    """
    Deadvertize  monitoring classads for the given entry.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
            fd.write('Requirements = (ReqGlidein == "%s@%s@%s")&&(GlideinMyType == "%s")\n' % (entry_name, glidein_name, factory_name, factoryConfig.factoryclient_id))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS")
    finally:
        os.remove(tmpnam)
Пример #18
0
def deadvertizeGlobal(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Removes the glidefactoryglobal classad advertising the factory globals from the WMS Collector.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfg')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factory_global)
            fd.write('Requirements = (Name == "%s@%s")&&(GlideinMyType == "%s")\n' % (glidein_name, factory_name, factoryConfig.factory_id))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "INVALIDATE_MASTER_ADS", factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
Пример #19
0
def deadvertizeFactoryClientMonitoring(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Deadvertize all monitoring classads for this factory.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
            fd.write('Requirements = (ReqFactoryName=?="%s")&&(ReqGlideinName=?="%s")&&(GlideinMyType == "%s")' % (factory_name, glidein_name, factoryConfig.factoryclient_id))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS", factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
Пример #20
0
def deadvertizeFactory(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
    """
    Deadvertize all entry and global classads for this factory.
    """
    tmpnam = classadSupport.generate_classad_filename(prefix="gfi_de_fact")
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
            fd.write('Requirements = (FactoryName =?= "%s")&&(GlideinName =?= "%s")\n' % (factory_name, glidein_name))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
def deadvertizeAllWork(factory_pool, my_name):
    """
    Removes all work requests for the client in the factory.
    """
    global frontendConfig

    tmpnam = classadSupport.generate_classad_filename(prefix="gfi_de_gc")
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "Query"\n')
            fd.write('TargetType = "%s"\n' % frontendConfig.client_id)
            fd.write(
                'Requirements = (ClientName == "%s") && (GlideinMyType == "%s")\n' % (my_name, frontendConfig.client_id)
            )
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "INVALIDATE_MASTER_ADS", factory_pool)
    finally:
        os.remove(tmpnam)
Пример #22
0
def advertizeGlobal(
    factory_name, glidein_name, supported_signtypes, pub_key_obj, stats_dict={}, factory_collector=DEFAULT_VAL
):

    """
    Creates the glidefactoryglobal classad and advertises.

    @type factory_name: string
    @param factory_name: the name of the factory
    @type glidein_name: string
    @param glidein_name: name of the glidein
    @type supported_signtypes: string
    @param supported_signtypes: suppported sign types, i.e. sha1
    @type pub_key_obj: GlideinKey
    @param pub_key_obj: for the frontend to use in encryption
    @type stats_dict: dict
    @param stats_dict: completed jobs statistics
    @type factory_collector: string or None
    @param factory_collector: the collector to query, special value 'default' will get it from the global config

    @todo add factory downtime?
    """

    tmpnam = classadSupport.generate_classad_filename(prefix="gfi_ad_gfg")

    gfg_classad = FactoryGlobalClassad(factory_name, glidein_name, supported_signtypes, pub_key_obj)

    try:
        gfg_classad.writeToFile(tmpnam, append=False)
        exe_condor_advertise(tmpnam, gfg_classad.adAdvertiseCmd, factory_collector=factory_collector)
    finally:
        # Unable to write classad
        try:
            os.remove(tmpnam)
        except:
            # Do the possible to remove the file if there
            pass
Пример #23
0
def advertizeGlidein(factory_name, glidein_name, entry_name, trust_domain,
                     auth_method, supported_signtypes, pub_key_obj,
                     glidein_attrs={}, glidein_params={}, glidein_monitors={},
                     factory_collector=DEFAULT_VAL):
    
    """
    Creates the glideclient classad and advertises.
    
    @type factory_name: string
    @param factory_name: the name of the factory
    @type glidein_name: string
    @param glidein_name: name of the glidein
    @type entry_name: string
    @param entry_name: name of the entry
    @type trust_domain: string
    @param trust_domain: trust domain for this entry
    @type auth_method: string
    @param auth_method: the authentication methods this entry supports in glidein submission, i.e. grid_proxy
    @type supported_signtypes: string
    @param supported_signtypes: suppported sign types, i.e. sha1
    @type glidein_attrs: dict 
    @param glidein_attrs: glidein attrs to be published, not be overwritten by Frontends
    @type glidein_params: dict 
    @param glidein_params: params to be published, can be overwritten by Frontends
    @type glidein_monitors: dict 
    @param glidein_monitors: monitor attrs to be published
    @type pub_key_obj: GlideinKey
    @param pub_key_obj: for the frontend to use in encryption
    @type factory_collector: string or None
    @param factory_collector: the collector to query, special value 'default' will get it from the global config
    """
    global factoryConfig, advertizeGlideinCounter

    tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gf')
    fd = file(tmpnam, "w")
    try:
        try:
            fd.write('MyType = "%s"\n' % factoryConfig.factory_id)
            fd.write('GlideinMyType = "%s"\n' % factoryConfig.factory_id)
            fd.write('GlideinWMSVersion = "%s"\n' % factoryConfig.glideinwms_version)
            fd.write('Name = "%s@%s@%s"\n' % (entry_name, glidein_name, factory_name))
            fd.write('FactoryName = "%s"\n' % factory_name)
            fd.write('GlideinName = "%s"\n' % glidein_name)
            fd.write('EntryName = "%s"\n' % entry_name)
            fd.write('%s = "%s"\n' % (factoryConfig.factory_signtype_id, string.join(supported_signtypes, ',')))
            # Must have a key to communicate
            fd.write('PubKeyID = "%s"\n' % pub_key_obj.get_pub_key_id())
            fd.write('PubKeyType = "%s"\n' % pub_key_obj.get_pub_key_type())
            fd.write('PubKeyValue = "%s"\n' % string.replace(pub_key_obj.get_pub_key_value(), '\n', '\\n'))
            if 'grid_proxy' in auth_method:
                fd.write('GlideinAllowx509_Proxy = %s\n' % True)
                fd.write('GlideinRequirex509_Proxy = %s\n' % True)
                fd.write('GlideinRequireGlideinProxy = %s\n' % False)
            else:
                fd.write('GlideinAllowx509_Proxy = %s\n' % False)
                fd.write('GlideinRequirex509_Proxy = %s\n' % False)
                fd.write('GlideinRequireGlideinProxy = %s\n' % True)
            fd.write('DaemonStartTime = %li\n' % start_time)
            fd.write('UpdateSequenceNumber = %i\n' % advertizeGlideinCounter)
            advertizeGlideinCounter += 1

            # write out both the attributes, params and monitors
            for (prefix, data) in ((factoryConfig.glidein_attr_prefix, glidein_attrs),
                                  (factoryConfig.glidein_param_prefix, glidein_params),
                                  (factoryConfig.glidein_monitor_prefix, glidein_monitors)):
                for attr in data.keys():
                    el = data[attr]
                    if type(el) == type(1):
                        # don't quote ints
                        fd.write('%s%s = %s\n' % (prefix, attr, el))
                    else:
                        escaped_el = string.replace(string.replace(str(el), '"', '\\"'), '\n', '\\n')
                        fd.write('%s%s = "%s"\n' % (prefix, attr, escaped_el))
        finally:
            fd.close()

        exe_condor_advertise(tmpnam, "UPDATE_MASTER_AD", factory_collector=factory_collector)
    finally:
        os.remove(tmpnam)
Пример #24
0
def iterate_one(do_advertize, factory_in_downtime, glideinDescript,
                frontendDescript, group_name, my_entries):
    """
    One iteration of the entry group

    @type do_advertize: boolean
    @param do_advertize: True if glidefactory classads should be advertised

    @type factory_in_downtime: boolean
    @param factory_in_downtime: True if factory is in downtime

    @type glideinDescript: dict
    @param glideinDescript: Factory glidein config values

    @type frontendDescript: dict
    @param frontendDescript: Security mappings for frontend identities, security classes, and usernames

    @type group_name: string
    @param group_name: Name of the group

    @type my_entries: dict
    @param my_entries: Dictionary of entry objects (glideFactoryEntry.Entry) keyed on entry name
    """

    groupwork_done = {}
    done_something = 0

    for entry in my_entries.values():
        entry.initIteration(factory_in_downtime)

    try:
        groupwork_done = find_and_perform_work(do_advertize,
                                               factory_in_downtime,
                                               glideinDescript,
                                               frontendDescript, group_name,
                                               my_entries)
    except:
        logSupport.log.warning(
            "Error occurred while trying to find and do work.")
        logSupport.log.exception("Exception: ")

    logSupport.log.debug("Group Work done: %s" % groupwork_done)

    # Classad files to use
    gf_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gf')
    gfc_filename = classadSupport.generate_classad_filename(
        prefix='gfi_adm_gfc')

    logSupport.log.info(
        "Generating glidefactory (%s) and glidefactoryclient (%s) classads as needed"
        % (gf_filename, gfc_filename))

    entries_to_advertise = []
    for entry in my_entries.values():
        # Write classads to file if work was done or if advertise flag is set
        # Actual advertise is done using multi classad advertisement
        entrywork_done = 0
        if ((entry.name in groupwork_done)
                and ('work_done' in groupwork_done[entry.name])):
            entrywork_done = groupwork_done[entry.name]['work_done']
            done_something += entrywork_done

        if ((do_advertize) or (entrywork_done > 0)):
            entries_to_advertise.append(entry.name)
            entry.writeClassadsToFile(factory_in_downtime, gf_filename,
                                      gfc_filename)

        entry.unsetInDowntime()

    if ((do_advertize) or (done_something > 0)):
        logSupport.log.debug(
            "Generated glidefactory and glidefactoryclient classads for entries: %s"
            % string.join(entries_to_advertise, ', '))
        # ADVERTISE: glidefactory classads
        gfi.advertizeGlideinFromFile(gf_filename,
                                     remove_file=True,
                                     is_multi=True)
        # ADVERTISE: glidefactoryclient classads
        gfi.advertizeGlideinClientMonitoringFromFile(gfc_filename,
                                                     remove_file=True,
                                                     is_multi=True)
    else:
        logSupport.log.info(
            "Not advertising glidefactory and glidefactoryclient classads this round"
        )

    return done_something
Пример #25
0
def iterate_one(do_advertize, factory_in_downtime, glideinDescript,
                frontendDescript, group_name, my_entries):
    
    """
    One iteration of the entry group

    @type do_advertize: boolean
    @param do_advertize: True if glidefactory classads should be advertised

    @type factory_in_downtime: boolean
    @param factory_in_downtime: True if factory is in downtime

    @type glideinDescript: dict
    @param glideinDescript: Factory glidein config values

    @type frontendDescript: dict
    @param frontendDescript: Security mappings for frontend identities, security classes, and usernames for privsep

    @type group_name: string
    @param group_name: Name of the group

    @type my_entries: dict
    @param my_entries: Dictionary of entry objects keyed on entry name
    """

    groupwork_done = {}
    done_something = 0

    for entry in my_entries.values():
        entry.initIteration(factory_in_downtime)

    try:
        groupwork_done = find_and_perform_work(factory_in_downtime,
                                               glideinDescript,
                                               frontendDescript,
                                               group_name, my_entries)
    except:
        logSupport.log.warning("Error occurred while trying to find and do work.")
        logSupport.log.exception("Exception: ")

    logSupport.log.debug("Group Work done: %s" % groupwork_done)

    # Classad files to use
    gf_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gf')
    gfc_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')

    logSupport.log.info("Generating glidefactory (%s) and glidefactoryclient (%s) classads as needed" % (gf_filename, gfc_filename))

    entries_to_advertise = []
    for entry in my_entries.values():
        # Write classads to file if work was done or if advertise flag is set
        # Actual advertise is done using multi classad advertisement
        entrywork_done = 0
        if ( (entry.name in groupwork_done) and 
             ('work_done' in groupwork_done[entry.name]) ):
            entrywork_done = groupwork_done[entry.name]['work_done']
            done_something += entrywork_done

        if ( (do_advertize) or (entrywork_done > 0) ):
            entries_to_advertise.append(entry.name)
            entry.writeClassadsToFile(factory_in_downtime, gf_filename,
                                      gfc_filename)

        entry.unsetInDowntime()

    if ((do_advertize) or (done_something > 0)):
        logSupport.log.debug("Generated glidefactory and glidefactoryclient classads for entries: %s" % string.join(entries_to_advertise, ', '))
        # ADVERTISE: glidefactory classads
        gfi.advertizeGlideinFromFile(gf_filename,
                                     remove_file=True,
                                     is_multi=True)
        # ADVERTISE: glidefactoryclient classads
        gfi.advertizeGlideinClientMonitoringFromFile(gfc_filename,
                                                     remove_file=True,
                                                     is_multi=True)
    else:
        logSupport.log.info("Not advertising glidefactory and glidefactoryclient classads this round")

    return done_something
    def do_global_advertize(self):
        """
        Advertize globals with credentials
        """
        global advertizeGCGounter

        for factory_pool in self.global_pool:
            tmpname = classadSupport.generate_classad_filename(prefix="gfi_ad_gcg")
            glidein_params_to_encrypt = {}
            fd = file(tmpname, "w")
            x509_proxies_data = []
            if self.descript_obj.x509_proxies_plugin is not None:
                x509_proxies_data = self.descript_obj.x509_proxies_plugin.get_credentials()
                nr_credentials = len(x509_proxies_data)
                glidein_params_to_encrypt["NumberOfCredentials"] = "%s" % nr_credentials
            else:
                nr_credentials = 0
            request_name = "Global"
            if factory_pool in self.global_params:
                request_name, security_name = self.global_params[factory_pool]
                glidein_params_to_encrypt["SecurityName"] = security_name
            classad_name = "%s@%s" % (request_name, self.descript_obj.my_name)
            fd.write('MyType = "%s"\n' % frontendConfig.client_global)
            fd.write('GlideinMyType = "%s"\n' % frontendConfig.client_global)
            fd.write('GlideinWMSVersion = "%s"\n' % frontendConfig.glideinwms_version)
            fd.write('Name = "%s"\n' % classad_name)
            fd.write('FrontendName = "%s"\n' % self.descript_obj.frontend_name)
            fd.write('GroupName = "%s"\n' % self.descript_obj.group_name)
            fd.write('ClientName = "%s"\n' % self.descript_obj.my_name)
            for i in range(nr_credentials):
                cred_el = x509_proxies_data[i]
                cred_el.renew()
                cred_el.advertize = True
                if hasattr(cred_el, "filename"):
                    try:
                        if (not os.path.exists(cred_el.filename)) and (cred_el.creation_script is not None):
                            logSupport.log.debug(
                                "Proxy %s didn not exist, calling creation_script %s"
                                % (cred_el.filename, cred_el.creation_script)
                            )
                            condorExe.iexe_cmd(cred_el.creation_script)
                        data_fd = open(cred_el.filename)
                        cred_data = data_fd.read()
                        data_fd.close()
                    except:
                        cred_el.advertize = False
                        logSupport.log.exception("Advertising global credential %s failed" % cred_el.filename)
                        continue
                    glidein_params_to_encrypt[cred_el.file_id(cred_el.filename)] = cred_data
                    if hasattr(cred_el, "security_class"):
                        # Convert the sec class to a string so the Factory can interpret the value correctly
                        glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.filename)] = str(
                            cred_el.security_class
                        )
                if hasattr(cred_el, "key_fname"):
                    try:
                        data_fd = open(cred_el.key_fname)
                        cred_data = data_fd.read()
                        data_fd.close()
                    except:
                        cred_el.advertize = False
                        logSupport.log.exception("Advertising global credential %s failed: " % cred_el.filename)
                        continue

                    glidein_params_to_encrypt[cred_el.file_id(cred_el.key_fname)] = cred_data
                    if hasattr(cred_el, "security_class"):
                        # Convert the sec class to a string so the Factory can interpret the value correctly
                        glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.key_fname)] = str(
                            cred_el.security_class
                        )
                if hasattr(cred_el, "pilot_fname"):
                    try:
                        data_fd = open(cred_el.pilot_fname)
                        cred_data = data_fd.read()
                        data_fd.close()
                    except:
                        cred_el.advertize = False
                        logSupport.log.exception("Advertising global credential %s failed: " % cred_el.filename)
                        continue
                    glidein_params_to_encrypt[cred_el.file_id(cred_el.pilot_fname)] = cred_data
                    if hasattr(cred_el, "security_class"):
                        # Convert the sec class to a string so the Factory can interpret the value correctly
                        glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.pilot_fname)] = str(
                            cred_el.security_class
                        )
            if factory_pool in self.global_key:
                key_obj = self.global_key[factory_pool]
            if key_obj is not None:
                fd.write(string.join(key_obj.get_key_attrs(), "\n") + "\n")
                for attr in glidein_params_to_encrypt.keys():
                    # logSupport.log.debug("Encrypting (%s,%s)"%(attr,glidein_params_to_encrypt[attr]))
                    el = key_obj.encrypt_hex(glidein_params_to_encrypt[attr])
                    escaped_el = string.replace(string.replace(str(el), '"', '\\"'), "\n", "\\n")
                    fd.write('%s%s = "%s"\n' % (frontendConfig.encrypted_param_prefix, attr, escaped_el))

            # Update Sequence number information
            if advertizeGCGounter.has_key(classad_name):
                advertizeGCGounter[classad_name] += 1
            else:
                advertizeGCGounter[classad_name] = 0
            fd.write("UpdateSequenceNumber = %s\n" % advertizeGCGounter[classad_name])

            fd.close()

            try:
                advertizeWorkFromFile(factory_pool, tmpname, remove_file=True)
            except condorExe.ExeError:
                logSupport.log.exception("Advertising globals failed for factory pool %s: " % factory_pool)