def Restart(self):
status = ClusterStatus()
# Get the factory id
factoryID = status.GetFactoryID()
# Hold then release the factory in the queue
(stderr, stdout) = RunExternal("condor_hold %s" % factoryID)
print "Stderr = %s" % stderr.strip()
#print "Stdout = %s" % stdout.strip()
(stderr, stdout) = RunExternal("condor_release %s" % factoryID)
print "Stderr = %s" % stderr.strip()
def GetNewStartdAds(self, site):
"""
Get the startd's that reported since last checked
@return list: List of ClassAd objects
"""
#def NewAds(data):
# if data.has_key("Offline") == False:
#if data["Offline"] == True and \
# int(data["DaemonStartTime"]) > (int(time.time()) - (int(time.time()) - self.lastupdatetime)) and \
# data[self.siteunique] == site:
# return True
# return False
#fetched = self.condor_status.fetchStored(NewAds)
cmd = "condor_status -l -const '(IsUndefined(Offline) == TRUE) && (DaemonStartTime > %(lastupdate)i) && (%(uniquesite)s =?= %(sitename)s)'"
query_opts = {
"lastupdate":
int(time.time()) - (int(time.time()) - self.lastupdatetime),
"uniquesite":
self.siteunique,
"sitename":
site
}
new_cmd = cmd % query_opts
(stdout, stderr) = RunExternal(new_cmd)
ad_list = []
for str_classad in stdout.split('\n\n'):
if len(str_classad) > 0:
ad_list.append(ClassAd(str_classad))
return ad_list
def GetOfflineAds(self, site):
"""
Get the full classads of the offline startds
@param site: The site to restrict the offline ads
@return: list of ClassAd objects
"""
#def OfflineAds(data):
# if data.has_key("Offline"):
# if data["Offline"] == True and data[self.siteunique] == site:
# return True
# return False
#fetched = self.condor_status.fetchStored(OfflineAds)
cmd = "condor_status -l -const '(IsUndefined(Offline) == FALSE) && (Offline == true) && (%(uniquesite)s =?= %(sitename)s)'"
query_opts = {"uniquesite": self.siteunique, "sitename": site}
new_cmd = cmd % query_opts
(stdout, stderr) = RunExternal(new_cmd)
ad_list = []
for str_classad in stdout.split('\n\n'):
if len(str_classad) > 0:
ad_list.append(ClassAd(str_classad))
return ad_list
def GetIdle(self):
self.idle = 0
self.found = False
# Get the xml from the collector
to_parse, stderr = RunExternal(self.command)
formatted_to_parse = "<doc>%s</doc>" % to_parse
# Parse the data
try:
xml.sax.parseString(formatted_to_parse, self)
except xml.sax._exceptions.SAXParseException as inst:
logging.error("Error parsing:")
logging.error("command = %s" % self.command)
logging.error("stderr = %s" % stderr)
logging.error("stdout = %s" % to_parse)
logging.error("Error: %s - %s" % (str(inst), inst.args))
if not self.found and (len(stderr) != 0):
logging.error("No valid output received from command: %s" %
self.command)
logging.error("stderr = %s" % stderr)
logging.error("stdout = %s" % to_parse)
return None
return self.idle
def Intialize(self):
"""
Function to initialize the factory's variables such as configuration
and logging
"""
# Set the sighup signal handler
signal.signal(signal.SIGHUP, self.Intialize)
# Read in the configuration file
self.config_file = self.options.config
files_read = set_config_file(self.config_file)
# check if no files read in
if len(files_read) < 1:
sys.stderr.write("No configuration files found. Location = %s\n" %
self.config_file)
sys.exit(1)
self._SetLogging()
if os.getuid() == 0 or get_option("factory_user"):
logging.info("Detected that factory should change user")
self._DropPriv()
if get_option("useoffline", "false").lower() == "true":
self.UseOffline = True
else:
self.UseOffline = False
self.cluster_list = []
# Get the cluster lists
if get_option("clusterlist", "") is not "":
logging.debug(
"Using the cluster list in the campus factory configuration.")
for cluster_id in get_option("clusterlist").split(','):
self.cluster_list.append(
Cluster(cluster_id, useOffline=self.UseOffline))
else:
# Check for the bosco cluster command
(stdout, stderr) = RunExternal("bosco_cluster -l")
if len(stdout) != 0 and stdout is not "No clusters configured":
logging.debug("Using the cluster list installed with BOSCO")
for cluster_id in stdout.split("\n"):
if len(cluster_id) > 0 and cluster_id != "":
self.cluster_list.append(
Cluster(cluster_id, useOffline=self.UseOffline))
else:
# Initialize as empty, which infers to submit 'here'
self.cluster_list = [
Cluster(get_option("CONDOR_HOST"),
useOffline=self.UseOffline)
]
# Tar up the executables
wrangler = DaemonWrangler()
wrangler.Package()
def Stop(self):
status = ClusterStatus()
# Get the factory id
factoryID = status.GetFactoryID()
# Remove the factory job
(stderr, stdout) = RunExternal("condor_rm %s" % factoryID)
print "Stderr = %s" % stderr.strip()
def SingleSubmit(self, filename):
"""
Submit a single glidein job
@param filename: The file (string) to submit
"""
# Get the cluster specific information
# First, the cluster tmp directory
cluster_tmp = self.get_option("worker_tmp", "/tmp")
remote_factory_location = self.get_option("remote_factory",
"~/bosco/campus_factory")
# If we are submtiting to ourselves, then don't need remote cluster
if self.get_option("CONDOR_HOST") == self.cluster_unique:
remote_cluster = ""
else:
remote_cluster = self.cluster_entry
# Get any custom attributes that are defined in the configuration
custom_options = {}
custom_options_raw = self.get_option("custom_condor_submit")
if (custom_options_raw is not None):
split_options = custom_options_raw.split(";")
for option in split_options:
(lside, rside) = option.split("=")
custom_options[lside.strip()] = rside.strip()
# TODO: These options should be moved to a better location
predetermined_options = {"WN_TMP": cluster_tmp, \
"GLIDEIN_HOST": self.get_option("COLLECTOR_HOST"), \
"GLIDEIN_Site": self.cluster_unique, \
"BOSCOCluster": self.cluster_unique, \
"REMOTE_FACTORY": remote_factory_location, \
"REMOTE_CLUSTER": remote_cluster, \
"REMOTE_SCHEDULER": self.cluster_type, \
"GLIDEIN_DIR": self.get_option("GLIDEIN_DIRECTORY"), \
"PASSWDFILE_LOCATION": self.get_option("SEC_PASSWORD_FILE")}
# Combine the custom options with the pre-determined options. Prefer
# custom options over pre-determined
options = dict(predetermined_options.items() + custom_options.items())
options_str = ""
for key in options.keys():
options_str += " -a %s=\"%s\"" % (key, options[key])
(stdout,
stderr) = RunExternal("condor_submit %s %s" % (filename, options_str))
logging.debug("stdout: %s" % stdout)
logging.debug("stderr: %s" % stderr)
def DeAdvertiseAds(self, ads):
"""
DeAdvertise ads to the collector
@param ads: List of ClassAd objects to deadvertise
"""
cmd = "condor_advertise INVALIDATE_STARTD_ADS"
for ad in ads:
str_query = "MyType = \"Query\"\n"
str_query += "TargetType = \"Machine\"\n"
str_query += "Requirements = Name == %s\n\n" % ad["Name"]
RunExternal(cmd, str_query)
def AdvertiseAds(self, ads):
"""
Advertise ads to the collector
@param ads: List of ClassAd objects to advertise
"""
cmd = "condor_advertise UPDATE_STARTD_AD"
for ad in ads:
(stdout, stderr) = RunExternal(cmd, str(ad))
logging.debug("stdin: %s", str(ad))
logging.debug("stdout: %s", stdout)
logging.debug("stderr: %s", stderr)
def GetIdle(self):
self.idle = 0
self.found = False
# Get the xml from the collector
to_parse, stderr = RunExternal(self.command)
formatted_to_parse = "<doc>%s</doc>" % to_parse
# Parse the data
try:
xml.sax.parseString(formatted_to_parse, self)
except xml.sax._exceptions.SAXParseException, inst:
logging.error("Error parsing:")
logging.error("command = %s" % self.command)
logging.error("stderr = %s" % stderr)
logging.error("stdout = %s" % to_parse)
logging.error("Error: %s - %s" % ( str(inst), inst.args ))
def get(self, key):
"""
@param key: string key
@return: string - value corresponding to key or "" if key is non-valid
"""
if self.config_dict.has_key(key):
return self.config_dict[key]
else:
(stdout, stderr) = RunExternal("condor_config_val %s" % key)
if len(stdout) == 0:
logging.error(
"Unable to get any output from condor_config_val. Is key = %s correct?"
% key)
logging.error("Stderr: %s" % stderr)
self.config_dict[key] = stdout.strip()
return stdout.strip()
def __init__(self):
self.config_dict = {}
(stdout, stderr) = RunExternal("condor_config_val -dump")
if len(stdout) == 0:
error_msg = "Unable to get any output from condor_config_val. Is condor binaries in the path?"
raise EnvironmentError(error_msg)
# Parse the stdout
line_re = re.compile("([\w|\d]+)\s+=\s+(.*)\Z")
config_dict = {}
for line in stdout.split('\n'):
match = line_re.search(line)
if match == None:
continue
(key, value) = match.groups()
config_dict[key] = value
logging.info("Got %s values from condor_config_val" %
len(config_dict.keys()))
def _ldd(self, file):
"""
Given a file return all the libraries referenced by the file
@type file: string
@param file: Full path to the file
@return: List containing linked libraries required by the file
@rtype: list
"""
rlist = []
if os.path.exists(file):
(stdout, stderr) = RunExternal("ldd %s" % file)
for line in stdout.split('\n'):
tokens = line.split('=>')
if len(tokens) == 2:
lib_loc = ((tokens[1].strip()).split(' '))[0].strip()
if os.path.exists(lib_loc):
rlist.append(os.path.abspath(lib_loc))
return rlist
def SingleSubmit(self, filename):
"""
Submit a single glidein job
@param filename: The file (string) to submit
"""
# Get the cluster specific information
# First, the cluster tmp directory
cluster_tmp = self._GetClusterSpecificConfig("worker_tmp", "/tmp")
remote_factory_location = self._GetClusterSpecificConfig("remote_factory", "~/bosco/campus_factory")
# If we are submtiting to ourselves, then don't need remote cluster
if get_option("CONDOR_HOST") == self.cluster_unique:
remote_cluster = ""
else:
remote_cluster = self.cluster_entry
# TODO: These options should be moved to a better location
options = {"WN_TMP": cluster_tmp, \
"GLIDEIN_HOST": get_option("COLLECTOR_HOST"), \
"GLIDEIN_Site": self.cluster_unique, \
"BOSCOCluster": self.cluster_unique, \
"REMOTE_FACTORY": remote_factory_location, \
"REMOTE_CLUSTER": remote_cluster, \
"REMOTE_SCHEDULER": self.cluster_type, \
"GLIDEIN_DIR": get_option("GLIDEIN_DIRECTORY"), \
"PASSWDFILE_LOCATION": get_option("SEC_PASSWORD_FILE")}
options_str = ""
for key in options.keys():
options_str += " -a %s=\"%s\"" % (key, options[key])
(stdout, stderr) = RunExternal("condor_submit %s %s" % (filename, options_str))
logging.debug("stdout: %s" % stdout)
logging.debug("stderr: %s" % stderr)