def setUp(self):
conf = parse(XML)
self.submit_dir = conf.get_submit_dir()
self.stage_dir = conf.get_stage_dir()
self.monitor_dir = conf.get_monitor_dir()
self.log_dir = conf.get_log_dir()
self.client_log_dirs = conf.get_client_log_dirs()
self.client_proxy_dirs = conf.get_client_proxy_dirs()
self.conf = conf
self.md = glideinMainDicts(self.submit_dir,
self.stage_dir,
'another-work-dir',
self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
self.entries = conf.get_entries()
self.edicts = {}
for entry in self.entries:
nm = entry.getName()
dir_nm = 'entry_' + nm
ed = glideinEntryDicts(self.submit_dir,
self.stage_dir,
nm,
self.md.get_summary_signature(),
dir_nm,
self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
self.assertTrue(isinstance(ed, glideinEntryDicts))
self.edicts[nm] = ed
def setUp(self):
conf = parse(XML)
self.submit_dir = conf.get_submit_dir()
self.stage_dir = conf.get_stage_dir()
self.monitor_dir = conf.get_monitor_dir()
self.log_dir = conf.get_log_dir()
self.client_log_dirs = conf.get_client_log_dirs()
self.client_proxy_dirs = conf.get_client_proxy_dirs()
self.conf = conf
self.md = glideinMainDicts(self.submit_dir,
self.stage_dir,
'another-work-dir',
self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
self.entries = conf.get_entries()
self.edicts = {}
for entry in self.entries:
nm = entry.getName()
dir_nm = 'entry_' + nm
ed = glideinEntryDicts(self.submit_dir,
self.stage_dir,
nm,
self.md.get_summary_signature(),
dir_nm,
self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
self.assertTrue(isinstance(ed, glideinEntryDicts))
self.edicts[nm] = ed
def setUp(self):
conf = parse(XML)
self.conf = conf
self.gd = glideinDicts(conf.get_submit_dir(), conf.get_stage_dir(),
conf.get_log_dir(), conf.get_client_log_dirs(),
conf.get_client_proxy_dirs(),
[e.getName() for e in conf.get_entries()])
def setUp(self):
conf = parse(XML)
self.conf = conf
self.gd = glideinDicts(conf.get_submit_dir(),
conf.get_stage_dir(),
conf.get_log_dir(),
conf.get_client_log_dirs(),
conf.get_client_proxy_dirs(),
[e.getName() for e in conf.get_entries()])
def setUp(self):
self.conf = parse(XML)
self.entry_name = 'TEST_SITE_1'
self.entry = None
for entr in self.conf.get_entries():
if self.entry_name == entr.getName():
self.entry = entr
self.gsdf = GlideinSubmitDictFile('fixtures/factory/work-dir',
self.entry_name)
def setUp(self):
conf = parse(XML)
self.submit_dir = conf.get_submit_dir()
self.stage_dir = conf.get_stage_dir()
self.monitor_dir = conf.get_monitor_dir()
self.log_dir = conf.get_log_dir()
self.client_log_dirs = conf.get_client_log_dirs()
self.client_proxy_dirs = conf.get_client_proxy_dirs()
self.conf = conf
self.gmdicts = glideinMainDicts(self.submit_dir, self.stage_dir,
'work_dir', self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
def setUp(self):
conf = parse(XML)
self.submit_dir = conf.get_submit_dir()
self.stage_dir = conf.get_stage_dir()
self.monitor_dir = conf.get_monitor_dir()
self.log_dir = conf.get_log_dir()
self.client_log_dirs = conf.get_client_log_dirs()
self.client_proxy_dirs = conf.get_client_proxy_dirs()
self.conf = conf
self.gmdicts = glideinMainDicts(self.submit_dir,
self.stage_dir,
'work_dir',
self.log_dir,
self.client_log_dirs,
self.client_proxy_dirs)
def setUp(self):
self.conf = parse(XML)
self.attr_el_list = self.conf.get_child_list(u'attrs')
def test_parse(self):
parse(XML)
try:
parse(XML_ENTRY)
except RuntimeError:
pass
def setUp(self):
self.config = parse(XML)
def setUp(self):
self.conf = parse(XML)
self.esl = self.conf.get_child_list('entry_sets')
self.el = self.conf.get_child_list('entries')
self.assertTrue(len(self.esl) > 0)
def setUp(self):
self.conf = factoryXmlConfig.parse(XML)
self.cgpd = cgWParamDict.glideinDicts(self.conf)
self.cgpd.populate()
def setUp(self):
self.conf = parse(XML)
self.sec = self.conf.get_child(u'security')
self.frontends = self.sec.get_child_list(u'frontends')
egi_url = 'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&ongoing_only=yes'
#dt_xml = open("egi_down.xml")
dt_xml = urllib2.urlopen(egi_url, context=ssl._create_unverified_context())
xmlparser = xml.parsers.expat.ParserCreate()
xmlparser.StartElementHandler = egi_start_element
xmlparser.EndElementHandler = egi_end_element
xmlparser.CharacterDataHandler = egi_char_data
xmlparser.ParseFile(dt_xml)
dt_xml.close()
conf_path = "/etc/gwms-factory/glideinWMS.xml"
conf = factoryXmlConfig.parse(conf_path)
for entry in conf.get_child_list('entries'):
if entry['enabled'] == 'True':
if entry['gridtype'] == 'gt2' or entry['gridtype'] == 'gt5' or entry[
'gridtype'] == 'cream':
hostname = entry['gatekeeper'].split(':')[0]
# works for nordugrid and condor-ce
else:
hostname = entry['gatekeeper'].split()[0]
if hostname in downtimes:
entry_downtimes[entry['name']] = downtimes[hostname]
#for dt in downtimes[hostname]:
# print "%s %s %s All All # _ad_ %s" % (get_dt_format(dt['start']), get_dt_format(dt['end']), attrs['name'], ";".join(dt['desc'].split('\n')))
egi_url = 'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&ongoing_only=yes'
#dt_xml = open("egi_down.xml")
dt_xml = urllib2.urlopen(egi_url)
xmlparser = xml.parsers.expat.ParserCreate()
xmlparser.StartElementHandler = egi_start_element
xmlparser.EndElementHandler = egi_end_element
xmlparser.CharacterDataHandler = egi_char_data
xmlparser.ParseFile(dt_xml)
dt_xml.close()
conf_path = "/etc/gwms-factory/glideinWMS.xml"
conf = factoryXmlConfig.parse(conf_path)
for entry in conf.get_child_list('entries'):
if entry['enabled'] == 'True':
if entry['gridtype'] == 'gt2' or entry['gridtype'] == 'gt5' or entry['gridtype'] == 'cream':
hostname = entry['gatekeeper'].split(':')[0]
# works for nordugrid and condor-ce
else:
hostname = entry['gatekeeper'].split()[0]
if hostname in downtimes:
entry_downtimes[entry['name']] = downtimes[hostname]
#for dt in downtimes[hostname]:
# print "%s %s %s All All # _ad_ %s" % (get_dt_format(dt['start']), get_dt_format(dt['end']), attrs['name'], ";".join(dt['desc'].split('\n')))
dt_file = open(os.path.join(gfactory_dir, "glideinWMS.downtimes"))
def setUp(self):
self.conf = parse(XML)
self.files = self.conf.get_child_list(u'files')
def setUp(self):
self.conf = parse(XML)
self.ctl = self.conf.get_child_list(u'condor_tarballs')
def setUp(self):
self.conf = factoryXmlConfig.parse(XML)
self.cgpd = cgWParamDict.glideinDicts(self.conf)
self.cgpd.populate()
def setUp(self):
self.conf = parse(XML)
self.eel = self.conf.get_child_list('entries')
def main():
""" The main module
"""
# Move to the working directory
try:
if "GLIDEIN_FACTORY_DIR" in os.environ:
os.chdir(os.environ["GLIDEIN_FACTORY_DIR"])
else:
os.chdir("/var/lib/gwms-factory/work-dir/")
except OSError as ose:
logging.error("Cannot chdir to /var/lib/gwms-factory/work-dir/: %s",
ose)
return 1
# Parse the configuration
conf = parse("/etc/gwms-factory/glideinWMS.xml")
# Parse command line options
options = parse_opts()
entry_name = options.entry_name
wms_collector = options.wms_collector
# Set some variables needed later on
params = {}
status_sf = {}
nr_glideins = 1
idle_lifetime = 3600 * 24
factory_config = FactoryConfig()
glidein_descript = gfc.GlideinDescript()
frontend_descript = gfc.FrontendDescript()
collector = htcondor.Collector(wms_collector)
req_name = get_reqname(collector, options.fe_name, entry_name)
logging.debug("Using reques name %s" % req_name)
factory_config.submit_dir = conf.get_submit_dir()
constraint_gc = '(MyType=="glideclient") && (Name=="%s")' % (req_name)
ads_gc = collector.query(htcondor.AdTypes.Any, constraint_gc)
if not ads_gc:
logging.error("Cannot find glideclient classad using constraint %s",
constraint_gc)
return 1
else:
ad_gc = ads_gc[0]
log_debug(ad_gc, header='glideclient classad')
# Load factory config and get some info that will go in the pilot classad
glidein_descript.load_pub_key()
sym_key_obj, frontend_sec_name = validate_frontend(
ad_gc, frontend_descript, glidein_descript.data['PubKeyObj'])
security_class = sym_key_obj.decrypt_hex(
ad_gc['GlideinEncParamSecurityClass']) # GlideinSecurityClass
proxyid = sym_key_obj.decrypt_hex(ad_gc['GlideinEncParamSubmitProxy'])
user_name = frontend_descript.get_username(frontend_sec_name,
security_class)
# Prepare some values that ends up in the Arguments classad
# of the pilot, i.e., the ClientWeb instance
client_web_url = ad_gc['WebURL'] # -clientweb
client_signtype = ad_gc['WebSignType'] # -signtype
client_descript = ad_gc['WebDescriptFile'] # -clientdescript
client_sign = ad_gc['WebDescriptSign'] # -clientsign
client_group = ad_gc['GroupName'] # -clientgroup
client_group_web_url = ad_gc['WebGroupURL'] # -clientwebgroup
# -clientdescriptgroup
client_group_descript = ad_gc['WebGroupDescriptFile']
client_group_sign = ad_gc['WebGroupDescriptSign'] # -clientsigngroup
client_web = ClientWeb(client_web_url, client_signtype,
client_descript, client_sign, client_group,
client_group_web_url, client_group_descript,
client_group_sign)
# Create the submit_credentials object
credentials = SubmitCredentials(user_name, security_class)
credentials.id = proxyid
credentials.cred_dir = conf.get_client_proxy_dirs()[user_name]
credfname = '%s_%s' % (ad_gc['ClientName'], proxyid)
if not credentials.add_security_credential('SubmitProxy', credfname):
fname = os.path.join(credentials.cred_dir,
'credential_%s' % credfname)
logging.info((
"Problems getting credential file using credentials.add_security_credential."
" Check file %s permissions"), fname)
# Set the arguments
# I was using escapeParam for GLIDECLIENT_ReqNode and GLIDECLIENT_Collector but turned out it's not necessary
params['CONDOR_VERSION'] = 'default'
params['CONDOR_OS'] = 'default'
params['CONDOR_ARCH'] = 'default'
params['GLIDECLIENT_ReqNode'] = ad_gc[
'GlideinParamGLIDECLIENT_ReqNode']
params['GLIDECLIENT_Rank'] = ad_gc.get('GlideinParamGLIDECLIENT_Rank',
"1")
params['GLIDEIN_Collector'] = ad_gc['GlideinParamGLIDEIN_Collector']
params['USE_MATCH_AUTH'] = ad_gc['GlideinParamUSE_MATCH_AUTH']
params['Report_Failed'] = 'NEVER'
# Now that we have everything submit the pilot!
logging.getLogger().setLevel(logging.DEBUG)
submitGlideins(entry_name,
"test.test",
int(nr_glideins),
idle_lifetime,
"test:test",
credentials,
client_web,
params,
status_sf,
log=logging.getLogger(),
factoryConfig=factory_config)
return 0
