def main():
""" The main module
"""
# Move to the working directory
try:
if "GLIDEIN_FACTORY_DIR" in os.environ:
os.chdir(os.environ["GLIDEIN_FACTORY_DIR"])
else:
os.chdir("/var/lib/gwms-factory/work-dir/")
except OSError as ose:
logging.error("Cannot chdir to /var/lib/gwms-factory/work-dir/: %s",
ose)
return 1
# Parse command line options
options = parse_opts()
entry_name = options.entry_name
wms_collector = options.wms_collector
# Set some variables needed later on
params = {}
status_sf = {}
nr_glideins = 1
idle_lifetime = 3600 * 24
factory_config = FactoryConfig()
glidein_descript = gfc.GlideinDescript()
frontend_descript = gfc.FrontendDescript()
collector = htcondor.Collector(wms_collector)
req_name = get_reqname(collector, options.fe_name, entry_name)
logging.debug("Using reques name %s" % req_name)
factory_config.submit_dir = '/var/lib/gwms-factory/work-dir'
constraint_gc = '(MyType=="glideclient") && (Name=="%s")' % (req_name)
ads_gc = collector.query(htcondor.AdTypes.Any, constraint_gc)
if not ads_gc:
logging.error("Cannot find glideclient classad using constraint %s",
constraint_gc)
return 1
else:
ad_gc = ads_gc[0]
log_debug(ad_gc, header='glideclient classad')
# Load factory config and get some info that will go in the pilot classad
glidein_descript.load_pub_key()
sym_key_obj, frontend_sec_name = validate_frontend(
ad_gc, frontend_descript, glidein_descript.data['PubKeyObj'])
security_class = sym_key_obj.decrypt_hex(
ad_gc['GlideinEncParamSecurityClass']) # GlideinSecurityClass
proxyid = sym_key_obj.decrypt_hex(ad_gc['GlideinEncParamSubmitProxy'])
user_name = frontend_descript.get_username(frontend_sec_name,
security_class)
# Prepare some values that ends up in the Arguments classad
# of the pilot, i.e., the ClientWeb instance
client_web_url = ad_gc['WebURL'] # -clientweb
client_signtype = ad_gc['WebSignType'] # -signtype
client_descript = ad_gc['WebDescriptFile'] # -clientdescript
client_sign = ad_gc['WebDescriptSign'] # -clientsign
client_group = ad_gc['GroupName'] # -clientgroup
client_group_web_url = ad_gc['WebGroupURL'] # -clientwebgroup
# -clientdescriptgroup
client_group_descript = ad_gc['WebGroupDescriptFile']
client_group_sign = ad_gc['WebGroupDescriptSign'] # -clientsigngroup
client_web = ClientWeb(client_web_url, client_signtype,
client_descript, client_sign, client_group,
client_group_web_url, client_group_descript,
client_group_sign)
# Create the submit_credentials object
credentials = SubmitCredentials(user_name, security_class)
credentials.id = proxyid
credentials.cred_dir = '/var/lib/gwms-factory/client-proxies/user_%s/glidein_gfactory_instance' % user_name
credfname = '%s_%s' % (ad_gc['ClientName'], proxyid)
if not credentials.add_security_credential('SubmitProxy', credfname):
fname = os.path.join(credentials.cred_dir,
'credential_%s' % credfname)
logging.info((
"Problems getting credential file using credentials.add_security_credential."
" Check file %s permissions"), fname)
# Set the arguments
# I was using escapeParam for GLIDECLIENT_ReqNode and GLIDECLIENT_Collector but turned out it's not necessary
params['CONDOR_VERSION'] = 'default'
params['CONDOR_OS'] = 'default'
params['CONDOR_ARCH'] = 'default'
params['GLIDECLIENT_ReqNode'] = ad_gc[
'GlideinParamGLIDECLIENT_ReqNode']
params['GLIDECLIENT_Rank'] = ad_gc.get('GlideinParamGLIDECLIENT_Rank',
"1")
params['GLIDEIN_Collector'] = ad_gc['GlideinParamGLIDEIN_Collector']
params['USE_MATCH_AUTH'] = ad_gc['GlideinParamUSE_MATCH_AUTH']
params['Report_Failed'] = 'NEVER'
# Now that we have everything submit the pilot!
logging.getLogger().setLevel(logging.DEBUG)
submitGlideins(entry_name,
"test.test",
int(nr_glideins),
idle_lifetime,
"test:test",
credentials,
client_web,
params,
status_sf,
log=logging.getLogger(),
factoryConfig=factory_config)
return 0
def setUp(self):
self.cwd = os.getcwd()
os.chdir('fixtures/factory/work-dir')
self.cnf = FactoryConfig()
self.gf_cnf = glideinwms.factory.glideFactoryConfig.FactoryConfig()
os.chdir(self.cwd)
def setUp(self):
self.cwd = os.getcwd()
os.chdir('fixtures/factory/work-dir')
self.cnf = FactoryConfig()
self.gf_cnf = glideinwms.factory.glideFactoryConfig.FactoryConfig()
os.chdir(self.cwd)
class TestFactoryConfig(unittest.TestCase):
def setUp(self):
self.cwd = os.getcwd()
os.chdir('fixtures/factory/work-dir')
self.cnf = FactoryConfig()
self.gf_cnf = glideinwms.factory.glideFactoryConfig.FactoryConfig()
os.chdir(self.cwd)
def test__init__(self):
self.assertTrue(isinstance(self.cnf, FactoryConfig))
def test_config_whoamI_(self):
self.cnf.config_whoamI('my_factory', 'my_glidein')
self.assertEqual(self.cnf.factory_name, 'my_factory')
self.assertEqual(self.cnf.glidein_name, 'my_glidein')
@unittest.skip('for now')
def test_get_condor_q_credential_list(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
crdl = getCondorQCredentialList(self.cnf)
self.assertEqual([], crdl)
def test_config_dirs(self):
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
self.assertEqual(self.cnf.submit_dir, 'submit_dir')
def test_config_remove_freq(self):
sleepBetweenRemoves = 10
maxRemovesXCycle = 10
self.cnf.config_remove_freq(sleepBetweenRemoves, maxRemovesXCycle)
self.assertEqual(self.cnf.remove_sleep, 10)
self.assertEqual(self.cnf.max_removes, 10)
def test_config_submit_freq(self):
sleepBetweenSubmits = 10
maxSubmitsXCycle = 10
self.cnf.config_submit_freq(sleepBetweenSubmits, maxSubmitsXCycle)
self.assertEqual(self.cnf.submit_sleep, 10)
self.assertEqual(self.cnf.max_submits, 10)
def test_config_whoamI(self):
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
self.assertEqual(self.cnf.factory_name, factory_name)
self.assertEqual(self.cnf.glidein_name, glidein_name)
def test_get_client_log_dir(self):
entry_name = 'entry_name'
username = '******'
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
cldr = self.cnf.get_client_log_dir(entry_name, username)
expected = 'client_log_base_dir/user_username/'
expected += 'glidein_glidein_name/entry_entry_name'
self.assertEqual(expected, cldr)
def test_get_client_proxies_dir(self):
entry_name = 'entry_name'
username = '******'
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
cldr = self.cnf.get_client_proxies_dir(username)
expected = 'client_proxies_base_dir/user_username/glidein_glidein_name'
self.assertEqual(expected, cldr)
def test_sec_class2_name(self):
self.assertEqual('foo_bar', secClass2Name('foo', 'bar'))
def test_get_condor_q_data(self):
entry_name = 'entry_name'
client_name = 'client_name'
schedd_name = 'sched_name'
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.CondorQ = mock.Mock()
cd = getCondorQData(entry_name, client_name, schedd_name, self.cnf)
self.assertEqual(cd.factory_name, self.cnf.factory_name)
self.assertEqual(cd.glidein_name, self.cnf.glidein_name)
self.assertEqual(cd.client_name, client_name)
self.assertEqual(cd.entry_name, entry_name)
def test_get_q_credentials(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.SubQuery = mock.Mock()
condorq = mock.Mock()
schedd_name = 'schedd_name'
condorq.schedd_name = schedd_name
factory_name = 'factory_name'
condorq.factory_name = factory_name
glidein_name = 'glidein_name'
condorq.glidein_name = glidein_name
entry_name = 'entry_name'
condorq.entry_name = entry_name
client_name = 'client_name'
condorq.client_name = client_name
creds = mock.Mock()
client_sa = 'fake'
cred_secclass_sa = 'fake'
cred_id_sa = 'fake'
crd = getQCredentials(condorq, client_name,
creds, client_sa, cred_secclass_sa, cred_id_sa)
self.assertEqual(crd.schedd_name, condorq.schedd_name)
self.assertEqual(crd.factory_name, condorq.factory_name)
self.assertEqual(crd.glidein_name, condorq.glidein_name)
self.assertEqual(crd.entry_name, condorq.entry_name)
self.assertEqual(crd.client_name, condorq.client_name)
def test_get_q_prox_sec_class(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.SubQuery = mock.Mock()
condorq = mock.Mock()
schedd_name = 'schedd_name'
condorq.schedd_name = schedd_name
factory_name = 'factory_name'
condorq.factory_name = factory_name
glidein_name = 'glidein_name'
condorq.glidein_name = glidein_name
entry_name = 'entry_name'
condorq.entry_name = entry_name
client_name = 'client_name'
condorq.client_name = client_name
proxy_security_class = 'fake'
credential_secclass_schedd_attribute = 'fake'
client_schedd_attribute = 'fake'
crd = getQProxSecClass(condorq, client_name, proxy_security_class,
client_schedd_attribute,
credential_secclass_schedd_attribute, self.cnf)
self.assertEqual(crd.schedd_name, condorq.schedd_name)
self.assertEqual(crd.factory_name, condorq.factory_name)
self.assertEqual(crd.glidein_name, condorq.glidein_name)
self.assertEqual(crd.entry_name, condorq.entry_name)
def test_get_q_status_s_f(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
condorq = mock.Mock()
condorq.stored_data = {}
self.assertEqual({}, getQStatusSF(condorq))
def test_get_q_status(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
condorq = mock.Mock()
condorq.stored_data = {}
qs = getQStatus(condorq)
def test_get_q_status_stale(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
condorq = mock.Mock()
condorq.stored_data = {}
qs = getQStatusStale(condorq)
def test_get_condor_status_data(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
condorq = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.CondorStatus = condorq
entry_name = 'entry_name'
client_name = 'client_name'
crd = getCondorStatusData(entry_name, client_name)
self.assertEqual(crd.factory_name, self.cnf.factory_name)
self.assertEqual(crd.glidein_name, self.cnf.glidein_name)
self.assertEqual(crd.entry_name, entry_name)
self.assertEqual(crd.client_name, client_name)
def test_is_str_safe(self):
s1 = '//\\'
self.assertFalse(is_str_safe(s1))
s2 = 'lalalala'
self.assertTrue(is_str_safe(s2))
def test_env_list2dict(self):
env = ['a=b', 'c=d']
expected = {'a': 'b', 'c': 'd'}
self.assertEqual(expected, env_list2dict(env))
@unittest.skip('for now')
def test_update_x509_proxy_file(self):
# glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
# glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
# update_x509_proxy_file(
# entry_name,
# username,
# client_id,
# proxy_data,
# self.cnf)
assert False
class TestFactoryConfig(unittest.TestCase):
def setUp(self):
self.cwd = os.getcwd()
os.chdir('fixtures/factory/work-dir')
self.cnf = FactoryConfig()
self.gf_cnf = glideinwms.factory.glideFactoryConfig.FactoryConfig()
os.chdir(self.cwd)
def test__init__(self):
self.assertTrue(isinstance(self.cnf, FactoryConfig))
def test_config_whoamI_(self):
self.cnf.config_whoamI('my_factory', 'my_glidein')
self.assertEqual(self.cnf.factory_name, 'my_factory')
self.assertEqual(self.cnf.glidein_name, 'my_glidein')
@unittest.skip('for now')
def test_get_condor_q_credential_list(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
crdl = getCondorQCredentialList(self.cnf)
self.assertEqual([], crdl)
def test_config_dirs(self):
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
self.assertEqual(self.cnf.submit_dir, 'submit_dir')
def test_config_remove_freq(self):
sleepBetweenRemoves = 10
maxRemovesXCycle = 10
self.cnf.config_remove_freq(sleepBetweenRemoves, maxRemovesXCycle)
self.assertEqual(self.cnf.remove_sleep, 10)
self.assertEqual(self.cnf.max_removes, 10)
def test_config_submit_freq(self):
sleepBetweenSubmits = 10
maxSubmitsXCycle = 10
self.cnf.config_submit_freq(sleepBetweenSubmits, maxSubmitsXCycle)
self.assertEqual(self.cnf.submit_sleep, 10)
self.assertEqual(self.cnf.max_submits, 10)
def test_config_whoamI(self):
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
self.assertEqual(self.cnf.factory_name, factory_name)
self.assertEqual(self.cnf.glidein_name, glidein_name)
def test_get_client_log_dir(self):
entry_name = 'entry_name'
username = '******'
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
cldr = self.cnf.get_client_log_dir(entry_name, username)
expected = 'client_log_base_dir/user_username/'
expected += 'glidein_glidein_name/entry_entry_name'
self.assertEqual(expected, cldr)
def test_get_client_proxies_dir(self):
entry_name = 'entry_name'
username = '******'
submit_dir = 'submit_dir'
log_base_dir = 'log_base_dir'
client_log_base_dir = 'client_log_base_dir'
client_proxies_base_dir = 'client_proxies_base_dir'
self.cnf.config_dirs(submit_dir, log_base_dir,
client_log_base_dir, client_proxies_base_dir)
factory_name = 'factory_name'
glidein_name = 'glidein_name'
self.cnf.config_whoamI(factory_name, glidein_name)
cldr = self.cnf.get_client_proxies_dir(username)
expected = 'client_proxies_base_dir/user_username/glidein_glidein_name'
self.assertEqual(expected, cldr)
def test_sec_class2_name(self):
self.assertEqual('foo_bar', secClass2Name('foo', 'bar'))
def test_get_condor_q_data(self):
entry_name = 'entry_name'
client_name = 'client_name'
schedd_name = 'sched_name'
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.CondorQ = mock.Mock()
cd = getCondorQData(entry_name, client_name, schedd_name, self.cnf)
self.assertEqual(cd.factory_name, self.cnf.factory_name)
self.assertEqual(cd.glidein_name, self.cnf.glidein_name)
self.assertEqual(cd.client_name, client_name)
self.assertEqual(cd.entry_name, entry_name)
def test_get_q_credentials(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.SubQuery = mock.Mock()
condorq = mock.Mock()
schedd_name = 'schedd_name'
condorq.schedd_name = schedd_name
factory_name = 'factory_name'
condorq.factory_name = factory_name
glidein_name = 'glidein_name'
condorq.glidein_name = glidein_name
entry_name = 'entry_name'
condorq.entry_name = entry_name
client_name = 'client_name'
condorq.client_name = client_name
creds = mock.Mock()
client_sa = 'fake'
cred_secclass_sa = 'fake'
cred_id_sa = 'fake'
crd = getQCredentials(condorq, client_name,
creds, client_sa, cred_secclass_sa, cred_id_sa)
self.assertEqual(crd.schedd_name, condorq.schedd_name)
self.assertEqual(crd.factory_name, condorq.factory_name)
self.assertEqual(crd.glidein_name, condorq.glidein_name)
self.assertEqual(crd.entry_name, condorq.entry_name)
self.assertEqual(crd.client_name, condorq.client_name)
def test_get_q_prox_sec_class(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.SubQuery = mock.Mock()
condorq = mock.Mock()
schedd_name = 'schedd_name'
condorq.schedd_name = schedd_name
factory_name = 'factory_name'
condorq.factory_name = factory_name
glidein_name = 'glidein_name'
condorq.glidein_name = glidein_name
entry_name = 'entry_name'
condorq.entry_name = entry_name
client_name = 'client_name'
condorq.client_name = client_name
proxy_security_class = 'fake'
credential_secclass_schedd_attribute = 'fake'
client_schedd_attribute = 'fake'
crd = getQProxSecClass(condorq, client_name, proxy_security_class,
client_schedd_attribute,
credential_secclass_schedd_attribute, self.cnf)
self.assertEqual(crd.schedd_name, condorq.schedd_name)
self.assertEqual(crd.factory_name, condorq.factory_name)
self.assertEqual(crd.glidein_name, condorq.glidein_name)
self.assertEqual(crd.entry_name, condorq.entry_name)
def test_get_q_status_s_f(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
condorq = mock.Mock()
condorq.stored_data = {}
self.assertEqual({}, getQStatusSF(condorq))
def test_get_q_status(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
condorq = mock.Mock()
condorq.stored_data = {}
qs = getQStatus(condorq)
def test_get_q_status_stale(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
condorq = mock.Mock()
condorq.stored_data = {}
qs = getQStatusStale(condorq)
def test_get_condor_status_data(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
condorq = mock.Mock()
glideinwms.factory.glideFactoryLib.condorMonitor.CondorStatus = condorq
entry_name = 'entry_name'
client_name = 'client_name'
crd = getCondorStatusData(entry_name, client_name)
self.assertEqual(crd.factory_name, self.cnf.factory_name)
self.assertEqual(crd.glidein_name, self.cnf.glidein_name)
self.assertEqual(crd.entry_name, entry_name)
self.assertEqual(crd.client_name, client_name)
def test_is_str_safe(self):
s1 = '//\\'
self.assertFalse(is_str_safe(s1))
s2 = 'lalalala'
self.assertTrue(is_str_safe(s2))
def test_env_list2dict(self):
env = ['a=b', 'c=d']
expected = {'a': 'b', 'c': 'd'}
self.assertEqual(expected, env_list2dict(env))
@unittest.skip('for now')
def test_update_x509_proxy_file(self):
# glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
# glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
# update_x509_proxy_file(
# entry_name,
# username,
# client_id,
# proxy_data,
# self.cnf)
assert False