class Launcher(object):
"""
Connects to Redis instance, listens for jobs, and spawns new threads using defined
launcher_adapter
"""
adapter = None
ip_address = None
launcher = None
tag = None
pool = None
def __init__(self, site, tag="", logger=None, overwatch_id=False):
"""
Initialize the Launcher instance
Keyword arguments:
site -- site object with relevant information to run
tag -- optional string describing launcher. Defined in site.LAUNCHER_REGISTER
logger -- logger instance (default = None)
overwatch_id -- id for optional overwatcher instance
"""
# Get the logger Instance
self.logger = logger
# Save passed-in variables
self.site = site
self.tag = tag
self.overwatch_id = overwatch_id
# Retrieve settings for this Launcher
self.get_settings()
# Check if additional params in self.launcher
self.check_settings()
# Load the adapter
self.load_adapter()
# Connect to Redis for communications
self.connect_to_redis()
# For loop in self.run
self.running = True
self.run()
def run(self):
"""The core process of the Launcher instance"""
# Set up overwatcher
if self.overwatch_id:
self.ow_registrar = Registrar(site=self.site,
ow_type="launcher",
ow_id=self.overwatch_id)
self.ow_registrar.register({
"site_id":
json.dumps(self.launcher.get('site_tag')),
"job_list":
self.job_list
})
try:
timer = 0
# This is the server portion of the code
while self.running:
# Have Registrar update status every second
if round(timer % 1, 1) in (0.0, 1.0):
if self.overwatch_id:
#self.ow_registrar.update({"site_id":self.site.ID,
self.ow_registrar.update({
"site_id":
json.dumps(self.launcher.get('site_tag')),
"job_list":
self.job_list
})
#self.ow_registrar.update({"job_list":self.job_list})
# Look for a new command
# This will throw a redis.exceptions.ConnectionError if redis is unreachable
#command = self.redis.brpop(["RAPD_JOBS",], 5)
try:
while self.redis.llen(self.job_list) != 0:
command = self.redis.rpop(self.job_list)
# Handle the message
if command:
self.handle_command(json.loads(command))
# Only run 1 command
# self.running = False
# break
# sleep a little when jobs aren't coming in.
time.sleep(0.2)
timer += 0.2
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception(
"Remote Redis is not up. Waiting for Sentinal to switch to new host"
)
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
"""Stop everything smoothly."""
self.running = False
# Close the file lock handle
close_lock_file()
# Try to close the pool. Python bugs gives errors!
if self.pool:
self.pool.close()
self.pool.join()
# Tell overwatch it is closing
if self.overwatch_id:
self.ow_registrar.stop()
def connect_to_redis(self):
"""Connect to the redis instance"""
redis_database = importlib.import_module('database.redis_adapter')
self.redis = redis_database.Database(
settings=self.site.CONTROL_DATABASE_SETTINGS, logger=self.logger)
def handle_command(self, command):
"""
Handle an incoming command
Keyword arguments:
command -- command from redis
"""
print "handle_command"
pprint(command)
# Split up the command
message = command
if self.logger:
self.logger.debug("Command received channel:%s message: %s",
self.job_list, message)
# Use the adapter to launch
#self.adapter(self.site, message, self.launcher)
# If running thru a shell limit the number of running processes
if self.pool:
self.pool.apply_async(
self.adapter(self.site, message, self.launcher))
else:
self.adapter(self.site, message, self.launcher)
def get_settings(self):
"""
Get the settings for this Launcher based on ip address and tag
"""
# Get IP Address
self.ip_address = utils.site.get_ip_address()
#print self.ip_address
if self.logger:
self.logger.debug("Found ip address to be %s", self.ip_address)
# Save typing
launchers = self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]
# Look for the launcher matching this ip_address and the input tag
possible_tags = []
for launcher in launchers:
#print launcher
if launcher.get('ip_address') == self.ip_address and launcher.get(
'tag') == self.tag:
self.launcher = launcher
break
elif launcher.get('ip_address') == self.ip_address:
possible_tags.append(launcher.get('tag'))
# No launcher adapter
if self.launcher is None:
# No launchers for this IP address
if len(possible_tags) == 0:
print " There are no launcher adapters registered for this ip address"
# IP Address in launchers, but not the input tag
else:
print text.error + "There is a launcher adapter registered for thi\
s IP address (%s), but not for the input tag (%s)" % (self.ip_address,
self.tag)
print " Available tags for this IP address:"
for tag in possible_tags:
print " %s" % tag
print text.stop
# Exit in error state
sys.exit(9)
else:
# Get the job_list to watch for this launcher
self.job_list = self.launcher.get('job_list')
def check_settings(self):
"""Check if additional params in self.launcher need setup."""
# Check if a multiprocessing.Pool needs to be setup for launcher adapter.
if self.tag == 'shell':
if self.launcher.get('pool_size', False):
size = self.launcher.get('pool_size')
else:
size = total_nproc() - 1
# Make sure its an integer
self.pool = mp_pool(int(size))
def load_adapter(self):
"""Find and load the adapter"""
# Import the database adapter as database module
self.adapter = load_module(
seek_module=self.launcher["adapter"],
directories=self.site.LAUNCHER_SETTINGS[
"RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter
if self.logger:
self.logger.debug(self.adapter)
class Launcher_Manager(Thread):
"""
Listens to the 'RAPD_JOBS'list and sends jobs to proper
launcher.
"""
def __init__(self, site, logger=False, overwatch_id=False):
"""
Initialize the Launcher instance
Keyword arguments:
site -- site object with relevant information to run
redis -- Redis instance for communication
logger -- logger instance (default = None)
overwatch_id -- id for optional overwatcher instance
"""
# If logger is passed in from main use that...
if logger:
self.logger = logger
else:
# Otherwise, get the rapd logger
self.logger = logging.getLogger("RAPDLogger")
# Initialize the thread
Thread.__init__(self)
# Save passed-in variables
self.site = site
self.overwatch_id = overwatch_id
self.running = True
self.timer = 0
self.job_list = []
self.connect_to_redis()
self.start()
def run(self):
"""The core process of the Launcher instance"""
# Set up overwatcher
if self.overwatch_id:
self.ow_registrar = Registrar(site=self.site,
ow_type="launch_manager",
ow_id=self.overwatch_id)
self.ow_registrar.register()
# Get the initial possible jobs lists
full_job_list = [
x.get('job_list')
for x in self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]
]
try:
# This is the server portion of the code
while self.running:
# Get updated job list by checking which launchers are running
# Reassign jobs if launcher(s) status changes
if round(self.timer % TIMER, 1) == 1.0:
try:
# Have Registrar update status
if self.overwatch_id:
self.ow_registrar.update()
# Check which launchers are running
temp = [
l for l in full_job_list
if self.redis.get("OW:" + l)
]
# Determine which launcher(s) went offline
offline = [
line for line in self.job_list
if temp.count(line) == False
]
if len(offline) > 0:
# Pop waiting jobs off their job_lists and push back in RAPD_JOBS for reassignment.
for _l in offline:
while self.redis.llen(_l) != 0:
self.redis.rpoplpush(_l, 'RAPD_JOBS')
# Determine which launcher(s) came online (Also runs at startup!)
online = [
line for line in temp
if self.job_list.count(line) == False
]
if len(online) > 0:
# Pop jobs off RAPD_JOBS_WAITING and push back onto RAPD_JOBS for reassignment.
while self.redis.llen('RAPD_JOBS_WAITING') != 0:
self.redis.rpoplpush('RAPD_JOBS_WAITING',
'RAPD_JOBS')
# Update the self.job_list
self.job_list = temp
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception(
"Remote Redis is not up. Waiting for Sentinal to switch to new host"
)
time.sleep(1)
# Look for a new command
# This will throw a redis.exceptions.ConnectionError if redis is unreachable
#command = self.redis.brpop(["RAPD_JOBS",], 5)
try:
while self.redis.llen("RAPD_JOBS") != 0:
command = self.redis.rpop("RAPD_JOBS")
# Handle the message
if command:
self.push_command(json.loads(command))
# Only run 1 command
# self.running = False
# break
# sleep a little when jobs aren't coming in.
time.sleep(0.2)
self.timer += 0.2
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception(
"Remote Redis is not up. Waiting for Sentinal to switch to new host"
)
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
if self.logger:
self.logger.debug('shutting down launcher manager')
self.running = False
if self.overwatch_id:
self.ow_registrar.stop()
def set_launcher(self, command=False, site_tag=False):
"""Find the correct running launcher to launch a specific job COMMAND"""
# list of commands to look for in the 'job_types'
# If the highest prioity launcher is 'ALL', it is chosen.
search = ['ALL']
if command:
search.append(command)
# Search through launchers
for x in self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]:
# Is launcher running?
if x.get('job_list') in self.job_list:
# check if its job type matches the command
for j in search:
if j in x.get('job_types'):
# Check if launcher is accepting jobs for this beamline
if site_tag:
if site_tag in x.get('site_tag'):
return (x.get('job_list'), x.get('launch_dir'))
else:
return (x.get('job_list'), x.get('launch_dir'))
# Return False if no running launchers are appropriate
return (False, False)
def push_command(self, command):
"""
Handle an incoming command
Keyword arguments:
command -- command from redis
"""
print "push_command"
#pprint(command)
# Split up the command
message = command
if self.logger:
self.logger.debug(
"Command received channel:RAPD_JOBS message: %s", message)
# get the site_tag from the image header to determine beamline where is was collected.
site_tag = launch_tools.get_site_tag(message)
# get the correct running launcher and launch_dir
launcher, launch_dir = self.set_launcher(message['command'], site_tag)
if message['command'].startswith('INTEGRATE'):
print 'type: %s...%s' % (message['preferences']['xdsinp'][:100],
message['preferences']['xdsinp'][-100:])
if launcher:
# Update preferences to be in server run mode
if not message.get("preferences"):
message["preferences"] = {}
message["preferences"]["run_mode"] = "server"
# Pass along the Launch directory
if not message.get("directories"):
message["directories"] = {}
message["directories"]["launch_dir"] = launch_dir
# Push the job on the correct launcher job list
self.redis.lpush(launcher, json.dumps(message))
if self.logger:
self.logger.debug("Command sent channel:%s message: %s",
launcher, message)
else:
self.redis.lpush('RAPD_JOBS_WAITING', json.dumps(message))
if self.logger:
self.logger.debug(
"Could not find a running launcher for this job. Putting job on RAPD_JOBS_WAITING list"
)
def connect_to_redis(self):
"""Connect to the redis instance"""
redis_database = importlib.import_module('database.redis_adapter')
#self.redis_db = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
#self.redis = self.redis_db.connect_to_redis()
#self.redis = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
self.redis = redis_database.Database(
settings=self.site.CONTROL_DATABASE_SETTINGS, logger=self.logger)
class Launcher(object):
"""
Connects to Redis instance, listens for jobs, and spawns new threads using defined
launcher_adapter
"""
adapter = None
#adapter_file = None
#database = None
# address = None
ip_address = None
#job_types = None
launcher = None
#port = None
tag = None
def __init__(self, site, tag="", logger=None, overwatch_id=False):
"""
Initialize the Launcher instance
Keyword arguments:
site -- site object with relevant information to run
tag -- optional string describing launcher. Defined in site.LAUNCHER_REGISTER
logger -- logger instance (default = None)
overwatch_id -- id for optional overwatcher instance
"""
# Get the logger Instance
self.logger = logger
# Save passed-in variables
self.site = site
self.tag = tag
self.overwatch_id = overwatch_id
# Retrieve settings for this Launcher
self.get_settings()
# Load the adapter
self.load_adapter()
# Connect to Redis for communications
self.connect_to_redis()
# For loop in self.run
self.running = True
self.run()
def run(self):
"""The core process of the Launcher instance"""
# Set up overwatcher
if self.overwatch_id:
self.ow_registrar = Registrar(site=self.site,
ow_type="launcher",
ow_id=self.overwatch_id)
self.ow_registrar.register({"site_id":self.site.ID,
"job_list":self.job_list})
try:
# This is the server portion of the code
while self.running:
# Have Registrar update status
if self.overwatch_id:
self.ow_registrar.update({"site_id":self.site.ID,
"job_list":self.job_list})
# Look for a new command
# This will throw a redis.exceptions.ConnectionError if redis is unreachable
#command = self.redis.brpop(["RAPD_JOBS",], 5)
try:
while self.redis.llen(self.job_list) != 0:
command = self.redis.rpop(self.job_list)
# Handle the message
if command:
self.handle_command(json.loads(command))
# Only run 1 command
# self.running = False
# break
# sleep a little when jobs aren't coming in.
time.sleep(0.2)
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception("Remote Redis is not up. Waiting for Sentinal to switch to new host")
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
"""Stop everything smoothly."""
self.running = False
if self.overwatch_id:
self.ow_registrar.stop()
self.redis_database.stop()
def connect_to_redis(self):
"""Connect to the redis instance"""
redis_database = importlib.import_module('database.redis_adapter')
#self.redis_database = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
#self.redis = self.redis_database.connect_to_redis()
#self.redis = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
self.redis = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS,
logger=self.logger)
def handle_command(self, command):
"""
Handle an incoming command
Keyword arguments:
command -- command from redis
"""
print "handle_command"
pprint(command)
# Split up the command
message = command
if self.logger:
self.logger.debug("Command received channel:%s message: %s", self.job_list, message)
# Use the adapter to launch
self.adapter(self.site, message, self.launcher)
#Thread(target=self.adapter, args=(self.site, message, self.launcher)).start()
def get_settings(self):
"""
Get the settings for this Launcher based on ip address and tag
"""
# Save typing
#launchers = self.site.LAUNCHER_SETTINGS["LAUNCHER_REGISTER"]
# Get IP Address
self.ip_address = utils.site.get_ip_address()
#print self.ip_address
if self.logger:
self.logger.debug("Found ip address to be %s", self.ip_address)
"""
# Look for the launcher matching this ip_address and the input tag
possible_tags = []
for launcher in launchers:
if launcher[0] == self.ip_address and launcher[1] == self.tag:
self.launcher = launcher
break
elif launcher[0] == self.ip_address:
possible_tags.append(launcher[1])
"""
# Save typing
launchers = self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]
# Look for the launcher matching this ip_address and the input tag
possible_tags = []
for launcher in launchers:
#print launcher
if launcher.get('ip_address') == self.ip_address and launcher.get('tag') == self.tag:
self.launcher = launcher
break
elif launcher.get('ip_address') == self.ip_address:
possible_tags.append(launcher.get('tag'))
# No launcher adapter
if self.launcher is None:
# No launchers for this IP address
if len(possible_tags) == 0:
print " There are no launcher adapters registered for this ip address"
# IP Address in launchers, but not the input tag
else:
print text.error + "There is a launcher adapter registered for thi\
s IP address (%s), but not for the input tag (%s)" % (self.ip_address, self.tag)
print " Available tags for this IP address:"
for tag in possible_tags:
print " %s" % tag
print text.stop
# Exit in error state
sys.exit(9)
else:
# Get the job_list to watch for this launcher
self.job_list = self.launcher.get('job_list')
def load_adapter(self):
"""Find and load the adapter"""
# Import the database adapter as database module
self.adapter = load_module(
seek_module=self.launcher["adapter"],
directories=self.site.LAUNCHER_SETTINGS["RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter
if self.logger:
self.logger.debug(self.adapter)
class Launcher_Manager(threading.Thread):
"""
Listens to the 'RAPD_JOBS'list and sends jobs to proper
launcher.
"""
def __init__(self, site, logger=False, overwatch_id=False):
"""
Initialize the Launcher instance
Keyword arguments:
site -- site object with relevant information to run
redis -- Redis instance for communication
logger -- logger instance (default = None)
overwatch_id -- id for optional overwatcher instance
"""
# If logger is passed in from main use that...
if logger:
self.logger = logger
else:
# Otherwise, get the rapd logger
self.logger = logging.getLogger("RAPDLogger")
# Initialize the thread
threading.Thread.__init__(self)
# Save passed-in variables
self.site = site
self.overwatch_id = overwatch_id
self.running = True
self.timer = 0
self.job_list = []
self.connect_to_redis()
self.start()
def run(self):
"""The core process of the Launcher instance"""
# Set up overwatcher
if self.overwatch_id:
self.ow_registrar = Registrar(site=self.site,
ow_type="launch_manager",
ow_id=self.overwatch_id)
self.ow_registrar.register()
# Get the initial possible jobs lists
full_job_list = [x.get('job_list') for x in self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]]
try:
# This is the server portion of the code
while self.running:
# Have Registrar update status
#if self.overwatch_id:
# self.ow_registrar.update()
# Get updated job list by checking which launchers are running
# Reassign jobs if launcher(s) status changes
if round(self.timer%TIMER,1) == 1.0:
try:
# Have Registrar update status
if self.overwatch_id:
self.ow_registrar.update()
# Check which launchers are running
temp = [l for l in full_job_list if self.redis.get("OW:"+l)]
# Determine which launcher(s) went offline
offline = [line for line in self.job_list if temp.count(line) == False]
if len(offline) > 0:
# Pop waiting jobs off their job_lists and push back in RAPD_JOBS for reassignment.
for _l in offline:
while self.redis.llen(_l) != 0:
self.redis.rpoplpush(_l, 'RAPD_JOBS')
# Determine which launcher(s) came online (Also runs at startup!)
online = [line for line in temp if self.job_list.count(line) == False]
if len(online) > 0:
# Pop jobs off RAPD_JOBS_WAITING and push back onto RAPD_JOBS for reassignment.
while self.redis.llen('RAPD_JOBS_WAITING') != 0:
self.redis.rpoplpush('RAPD_JOBS_WAITING', 'RAPD_JOBS')
# Update the self.job_list
self.job_list = temp
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception("Remote Redis is not up. Waiting for Sentinal to switch to new host")
time.sleep(1)
# Look for a new command
# This will throw a redis.exceptions.ConnectionError if redis is unreachable
#command = self.redis.brpop(["RAPD_JOBS",], 5)
try:
while self.redis.llen("RAPD_JOBS") != 0:
command = self.redis.rpop("RAPD_JOBS")
# Handle the message
if command:
#self.push_command(json.loads(command))
self.push_command(json.loads(command))
# Only run 1 command
# self.running = False
# break
# sleep a little when jobs aren't coming in.
time.sleep(0.2)
self.timer += 0.2
except redis.exceptions.ConnectionError:
if self.logger:
self.logger.exception("Remote Redis is not up. Waiting for Sentinal to switch to new host")
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
if self.logger:
self.logger.debug('shutting down launcher manager')
self.running = False
if self.overwatch_id:
self.ow_registrar.stop()
self.redis_db.stop()
def set_launcher(self, command=False, site_tag=False):
"""Find the correct running launcher to launch a specific job COMMAND"""
# list of commands to look for in the 'job_types'
# If the highest prioity launcher is 'ALL', it is chosen.
search = ['ALL']
if command:
search.append(command)
# Search through launchers
for x in self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"]:
# Is launcher running?
if x.get('job_list') in self.job_list:
# check if its job type matches the command
for j in search:
if j in x.get('job_types'):
# Check if launcher is accepting jobs for this beamline
if site_tag:
if site_tag in x.get('site_tag'):
return (x.get('job_list'), x.get('launch_dir'))
else:
return (x.get('job_list'), x.get('launch_dir'))
# Return False if no running launchers are appropriate
return (False, False)
def push_command(self, command):
"""
Handle an incoming command
Keyword arguments:
command -- command from redis
"""
print "push_command"
#pprint(command)
# Split up the command
message = command
if self.logger:
self.logger.debug("Command received channel:RAPD_JOBS message: %s", message)
# get the site_tag from the image header to determine beamline where is was collected.
site_tag = launch_tools.get_site_tag(message)
# get the correct running launcher and launch_dir
launcher, launch_dir = self.set_launcher(message['command'], site_tag)
if message['command'].startswith('INTEGRATE'):
print 'type: %s'%message['preferences']['xdsinp']
if launcher:
# Update preferences to be in server run mode
if not message.get("preferences"):
message["preferences"] = {}
message["preferences"]["run_mode"] = "server"
# Pass along the Launch directory
if not message.get("directories"):
message["directories"] = {}
message["directories"]["launch_dir"] = launch_dir
# Push the job on the correct launcher job list
self.redis.lpush(launcher, json.dumps(message))
if self.logger:
self.logger.debug("Command sent channel:%s message: %s", launcher, message)
else:
self.redis.lpush('RAPD_JOBS_WAITING', json.dumps(message))
if self.logger:
self.logger.debug("Could not find a running launcher for this job. Putting job on RAPD_JOBS_WAITING list")
def connect_to_redis(self):
"""Connect to the redis instance"""
redis_database = importlib.import_module('database.redis_adapter')
#self.redis_db = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
#self.redis = self.redis_db.connect_to_redis()
#self.redis = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)
self.redis = redis_database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS,
logger=self.logger)