class JobHandlerStopQueue(object):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__(self, app, dispatcher):
self.app = app
self.dispatcher = dispatcher
self.sa_session = app.model.context
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread(
name="JobHandlerStopQueue.monitor_thread", target=self.monitor)
self.monitor_thread.setDaemon(True)
self.monitor_thread.start()
log.info("job handler stop queue started")
def monitor(self):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep(10)
while self.running:
try:
self.monitor_step()
except:
log.exception("Exception in monitor_step")
# Sleep
self.sleeper.sleep(1)
def monitor_step(self):
"""
Called repeatedly by `monitor` to stop jobs.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.DELETED_NEW ) \
& ( model.Job.handler == self.app.config.server_name ) ).all()
for job in newly_deleted_jobs:
jobs_to_check.append((job, job.stderr))
# Also pull from the queue (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append(
(self.sa_session.query(model.Job).get(job_id), error_msg))
except Empty:
pass
for job, error_msg in jobs_to_check:
if error_msg is not None:
job.state = job.states.ERROR
job.info = error_msg
else:
job.state = job.states.DELETED
self.sa_session.add(job)
self.sa_session.flush()
if job.job_runner_name is not None:
# tell the dispatcher to stop the job
self.dispatcher.stop(job)
def put(self, job_id, error_msg=None):
self.queue.put((job_id, error_msg))
def shutdown(self):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info("sending stop signal to worker thread")
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put(self.STOP_SIGNAL)
self.sleeper.wake()
log.info("job handler stop queue stopped")
class JobHandlerStopQueue( object ):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__( self, app, dispatcher ):
self.app = app
self.dispatcher = dispatcher
self.sa_session = app.model.context
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.monitor )
self.monitor_thread.start()
log.info( "job handler stop queue started" )
def monitor( self ):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def monitor_step( self ):
"""
Called repeatedly by `monitor` to stop jobs.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.DELETED_NEW ) \
& ( model.Job.handler == self.app.config.server_name ) ).all()
for job in newly_deleted_jobs:
jobs_to_check.append( ( job, None ) )
# Also pull from the queue (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) )
except Empty:
pass
for job, error_msg in jobs_to_check:
if error_msg is not None:
job.state = job.states.ERROR
job.info = error_msg
else:
job.state = job.states.DELETED
self.sa_session.add( job )
self.sa_session.flush()
if job.job_runner_name is not None:
# tell the dispatcher to stop the job
self.dispatcher.stop( job )
def put( self, job_id, error_msg=None ):
self.queue.put( ( job_id, error_msg ) )
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job handler stop queue stopped" )
class JobManagerStopQueue( object ):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__( self, app, job_handler ):
self.app = app
self.job_handler = job_handler
self.sa_session = app.model.context
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( name="JobManagerStopQueue.monitor_thread", target=self.monitor )
self.monitor_thread.setDaemon( True )
self.monitor_thread.start()
log.info( "job manager stop queue started" )
def monitor( self ):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def monitor_step( self ):
"""
Called repeatedly by `monitor` to stop jobs.
"""
jobs_to_check = []
# Pull from the queue even if tracking in the database (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) )
except Empty:
pass
# If tracking in the database, the handler will pick up the stop itself. Otherwise, notify the handler.
for job, error_msg in jobs_to_check:
self.job_handler.job_stop_queue.put( job.id, error_msg )
def put( self, job_id, error_msg=None ):
self.queue.put( ( job_id, error_msg ) )
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job manager stop queue stopped" )
class DistributedObjectStore(ObjectStore):
"""
ObjectStore that defers to a list of backends, for getting objects the
first store where the object exists is used, objects are created in a
store selected randomly, but with weighting.
"""
def __init__(self, config):
super(DistributedObjectStore, self).__init__()
self.distributed_config = config.distributed_object_store_config_file
assert self.distributed_config is not None, "distributed object store ('object_store = distributed') " \
"requires a config file, please set one in " \
"'distributed_object_store_config_file')"
self.backends = {}
self.weighted_backend_ids = []
self.original_weighted_backend_ids = []
self.max_percent_full = {}
self.global_max_percent_full = 0.0
random.seed()
self.__parse_distributed_config(config)
if self.global_max_percent_full or filter(lambda x: x is not None, self.max_percent_full.values()):
self.sleeper = Sleeper()
self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor)
self.filesystem_monitor_thread.start()
log.info("Filesystem space monitor started")
def __parse_distributed_config(self, config):
tree = util.parse_xml(self.distributed_config)
root = tree.getroot()
log.debug('Loading backends for distributed object store from %s' % self.distributed_config)
self.global_max_percent_full = float(root.get('maxpctfull', 0))
for elem in [ e for e in root if e.tag == 'backend' ]:
id = elem.get('id')
weight = int(elem.get('weight', 1))
maxpctfull = float(elem.get('maxpctfull', 0))
if elem.get('type', 'disk'):
path = None
extra_dirs = {}
for sub in elem:
if sub.tag == 'files_dir':
path = sub.get('path')
elif sub.tag == 'extra_dir':
type = sub.get('type')
extra_dirs[type] = sub.get('path')
self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs)
self.max_percent_full[id] = maxpctfull
log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path))
if extra_dirs:
log.debug(" Extra directories:")
for type, dir in extra_dirs.items():
log.debug(" %s: %s" % (type, dir))
for i in range(0, weight):
# The simplest way to do weighting: add backend ids to a
# sequence the number of times equalling weight, then randomly
# choose a backend from that sequence at creation
self.weighted_backend_ids.append(id)
self.original_weighted_backend_ids = self.weighted_backend_ids
def __filesystem_monitor(self):
while self.running:
new_weighted_backend_ids = self.original_weighted_backend_ids
for id, backend in self.backends.items():
maxpct = self.max_percent_full[id] or self.global_max_percent_full
pct = backend.get_store_usage_percent()
if pct > maxpct:
new_weighted_backend_ids = filter(lambda x: x != id, new_weighted_backend_ids)
self.weighted_backend_ids = new_weighted_backend_ids
self.sleeper.sleep(120) # Test free space every 2 minutes
def shutdown(self):
super(DistributedObjectStore, self).shutdown()
self.sleeper.wake()
def exists(self, obj, **kwargs):
return self.__call_method('exists', obj, False, False, **kwargs)
def file_ready(self, obj, **kwargs):
return self.__call_method('file_ready', obj, False, False, **kwargs)
def create(self, obj, **kwargs):
"""
create() is the only method in which obj.object_store_id may be None
"""
if obj.object_store_id is None or not self.exists(obj, **kwargs):
if obj.object_store_id is None or obj.object_store_id not in self.weighted_backend_ids:
try:
obj.object_store_id = random.choice(self.weighted_backend_ids)
except IndexError:
raise ObjectInvalid()
object_session( obj ).add( obj )
object_session( obj ).flush()
log.debug("Selected backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id))
else:
log.debug("Using preferred backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id))
self.backends[obj.object_store_id].create(obj, **kwargs)
def empty(self, obj, **kwargs):
return self.__call_method('empty', obj, True, False, **kwargs)
def size(self, obj, **kwargs):
return self.__call_method('size', obj, 0, False, **kwargs)
def delete(self, obj, **kwargs):
return self.__call_method('delete', obj, False, False, **kwargs)
def get_data(self, obj, **kwargs):
return self.__call_method('get_data', obj, ObjectNotFound, True, **kwargs)
def get_filename(self, obj, **kwargs):
return self.__call_method('get_filename', obj, ObjectNotFound, True, **kwargs)
def update_from_file(self, obj, **kwargs):
if kwargs.get('create', False):
self.create(obj, **kwargs)
kwargs['create'] = False
return self.__call_method('update_from_file', obj, ObjectNotFound, True, **kwargs)
def get_object_url(self, obj, **kwargs):
return self.__call_method('get_object_url', obj, None, False, **kwargs)
def __call_method(self, method, obj, default, default_is_exception, **kwargs):
object_store_id = self.__get_store_id_for(obj, **kwargs)
if object_store_id is not None:
return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs)
if default_is_exception:
raise default()
else:
return default
def __get_store_id_for(self, obj, **kwargs):
if obj.object_store_id is not None and obj.object_store_id in self.backends:
return obj.object_store_id
else:
# if this instance has been switched from a non-distributed to a
# distributed object store, or if the object's store id is invalid,
# try to locate the object
log.warning('The backend object store ID (%s) for %s object with ID %s is invalid' % (obj.object_store_id, obj.__class__.__name__, obj.id))
for id, store in self.backends.items():
if store.exists(obj, **kwargs):
log.warning('%s object with ID %s found in backend object store with ID %s' % (obj.__class__.__name__, obj.id, id))
obj.object_store_id = id
object_session( obj ).add( obj )
object_session( obj ).flush()
return id
return None
class JobManagerQueue( object ):
"""
Job manager, waits for jobs to be runnable and then dispatches to a
JobHandler.
"""
STOP_SIGNAL = object()
def __init__( self, app, job_handler ):
self.app = app
self.job_handler = job_handler # the (singular) handler if we are passing jobs in memory
self.sa_session = app.model.context
self.job_lock = False
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.__monitor )
# Recover jobs at startup
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
log.info( "job manager queue started" )
def __check_jobs_at_startup( self ):
"""
Checks all jobs that are in the 'new', 'queued' or 'running' state in
the database and requeues or cleans up as necessary. Only run as the
job manager starts.
"""
for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( ( model.Job.state == model.Job.states.NEW ) \
| ( model.Job.state == model.Job.states.RUNNING ) \
| ( model.Job.state == model.Job.states.QUEUED ) ) \
& ( model.Job.handler == None ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) )
JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' )
else:
job.handler = self.__get_handler( job ) # handler's recovery method will take it from here
log.info( "(%d) Job in '%s' state had no handler at job manager startup, assigned '%s' handler" % ( job.id, job.state, job.handler ) )
if self.sa_session.dirty:
self.sa_session.flush()
def __monitor( self ):
"""
Continually iterate the waiting jobs and dispatch to a handler
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.__monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def __monitor_step( self ):
"""
Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then assigns a
handler.
"""
# Do nothing if the queue is locked
if self.job_lock:
log.info( 'Job queue is administratively locked, sleeping...' )
time.sleep( 10 )
return
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.NEW ) \
& ( model.Job.handler == None ) ).all()
else:
# Get job objects and append to watch queue for any which were
# previously waiting
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, tool_id = message
# Get the job object and append to watch queue
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
except Empty:
pass
for job in jobs_to_check:
job.handler = self.__get_handler( job )
log.debug( "(%s) Job assigned to handler '%s'" % ( job.id, job.handler ) )
self.sa_session.add( job )
# If tracking in the database, handlers will pick up the job now
self.sa_session.flush()
time.sleep( 5 )
# This only does something in the case that there is only one handler and it is this Galaxy process
for job in jobs_to_check:
self.job_handler.job_queue.put( job.id, job.tool_id )
def __get_handler( self, job ):
try:
params = None
if job.params:
params = from_json_string( job.params )
return self.app.toolbox.tools_by_id.get( job.tool_id, None ).get_job_handler( params )
except:
log.exception( "(%s) Caught exception attempting to get tool-specific job handler for tool '%s', selecting at random from available handlers instead:" % ( job.id, job.tool_id ) )
return random.choice( self.app.config.job_handlers )
def put( self, job_id, tool ):
"""Add a job to the queue (by job identifier)"""
if not self.app.config.track_jobs_in_database:
self.queue.put( ( job_id, tool.id ) )
self.sleeper.wake()
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job manager queue stopped" )
class JobManagerQueue(object):
"""
Job manager, waits for jobs to be runnable and then dispatches to a
JobHandler.
"""
STOP_SIGNAL = object()
def __init__(self, app, job_handler):
self.app = app
self.job_handler = job_handler # the (singular) handler if we are passing jobs in memory
self.sa_session = app.model.context
self.job_lock = False
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread(target=self.__monitor)
# Recover jobs at startup
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
log.info("job manager queue started")
def __check_jobs_at_startup(self):
"""
Checks all jobs that are in the 'new', 'queued' or 'running' state in
the database and requeues or cleans up as necessary. Only run as the
job manager starts.
"""
for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( ( model.Job.state == model.Job.states.NEW ) \
| ( model.Job.state == model.Job.states.RUNNING ) \
| ( model.Job.state == model.Job.states.QUEUED ) ) \
& ( model.Job.handler == None ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning(
"(%s) Tool '%s' removed from tool config, unable to recover job"
% (job.id, job.tool_id))
JobWrapper(job, self).fail(
'This tool was disabled before the job completed. Please contact your Galaxy administrator.'
)
else:
job.handler = self.__get_handler(
job) # handler's recovery method will take it from here
log.info(
"(%d) Job in '%s' state had no handler at job manager startup, assigned '%s' handler"
% (job.id, job.state, job.handler))
if self.sa_session.dirty:
self.sa_session.flush()
def __monitor(self):
"""
Continually iterate the waiting jobs and dispatch to a handler
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep(10)
while self.running:
try:
self.__monitor_step()
except:
log.exception("Exception in monitor_step")
# Sleep
self.sleeper.sleep(1)
def __monitor_step(self):
"""
Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then assigns a
handler.
"""
# Do nothing if the queue is locked
if self.job_lock:
log.info('Job queue is administratively locked, sleeping...')
time.sleep(10)
return
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.NEW ) \
& ( model.Job.handler == None ) ).all()
else:
# Get job objects and append to watch queue for any which were
# previously waiting
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, tool_id = message
# Get the job object and append to watch queue
jobs_to_check.append(
self.sa_session.query(model.Job).get(job_id))
except Empty:
pass
for job in jobs_to_check:
job.handler = self.__get_handler(job)
log.debug("(%s) Job assigned to handler '%s'" %
(job.id, job.handler))
self.sa_session.add(job)
# If tracking in the database, handlers will pick up the job now
self.sa_session.flush()
time.sleep(5)
# This only does something in the case that there is only one handler and it is this Galaxy process
for job in jobs_to_check:
self.job_handler.job_queue.put(job.id, job.tool_id)
def __get_handler(self, job):
try:
params = None
if job.params:
params = from_json_string(job.params)
return self.app.toolbox.tools_by_id.get(
job.tool_id, None).get_job_handler(params)
except:
log.exception(
"(%s) Caught exception attempting to get tool-specific job handler for tool '%s', selecting at random from available handlers instead:"
% (job.id, job.tool_id))
return random.choice(self.app.config.job_handlers)
def put(self, job_id, tool):
"""Add a job to the queue (by job identifier)"""
if not self.app.config.track_jobs_in_database:
self.queue.put((job_id, tool.id))
self.sleeper.wake()
def shutdown(self):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info("sending stop signal to worker thread")
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put(self.STOP_SIGNAL)
self.sleeper.wake()
log.info("job manager queue stopped")
class DistributedObjectStore(ObjectStore):
"""
ObjectStore that defers to a list of backends, for getting objects the
first store where the object exists is used, objects are created in a
store selected randomly, but with weighting.
"""
def __init__(self, config, fsmon=False):
super(DistributedObjectStore, self).__init__()
self.distributed_config = config.distributed_object_store_config_file
assert self.distributed_config is not None, "distributed object store ('object_store = distributed') " \
"requires a config file, please set one in " \
"'distributed_object_store_config_file')"
self.backends = {}
self.weighted_backend_ids = []
self.original_weighted_backend_ids = []
self.max_percent_full = {}
self.global_max_percent_full = 0.0
random.seed()
self.__parse_distributed_config(config)
self.sleeper = None
if fsmon and ( self.global_max_percent_full or filter( lambda x: x != 0.0, self.max_percent_full.values() ) ):
self.sleeper = Sleeper()
self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor)
self.filesystem_monitor_thread.setDaemon( True )
self.filesystem_monitor_thread.start()
log.info("Filesystem space monitor started")
def __parse_distributed_config(self, config):
tree = util.parse_xml(self.distributed_config)
root = tree.getroot()
log.debug('Loading backends for distributed object store from %s' % self.distributed_config)
self.global_max_percent_full = float(root.get('maxpctfull', 0))
for elem in [ e for e in root if e.tag == 'backend' ]:
id = elem.get('id')
weight = int(elem.get('weight', 1))
maxpctfull = float(elem.get('maxpctfull', 0))
if elem.get('type', 'disk'):
path = None
extra_dirs = {}
for sub in elem:
if sub.tag == 'files_dir':
path = sub.get('path')
elif sub.tag == 'extra_dir':
type = sub.get('type')
extra_dirs[type] = sub.get('path')
self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs)
self.max_percent_full[id] = maxpctfull
log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path))
if extra_dirs:
log.debug(" Extra directories:")
for type, dir in extra_dirs.items():
log.debug(" %s: %s" % (type, dir))
for i in range(0, weight):
# The simplest way to do weighting: add backend ids to a
# sequence the number of times equalling weight, then randomly
# choose a backend from that sequence at creation
self.weighted_backend_ids.append(id)
self.original_weighted_backend_ids = self.weighted_backend_ids
def __filesystem_monitor(self):
while self.running:
new_weighted_backend_ids = self.original_weighted_backend_ids
for id, backend in self.backends.items():
maxpct = self.max_percent_full[id] or self.global_max_percent_full
pct = backend.get_store_usage_percent()
if pct > maxpct:
new_weighted_backend_ids = filter(lambda x: x != id, new_weighted_backend_ids)
self.weighted_backend_ids = new_weighted_backend_ids
self.sleeper.sleep(120) # Test free space every 2 minutes
def shutdown(self):
super(DistributedObjectStore, self).shutdown()
if self.sleeper is not None:
self.sleeper.wake()
def exists(self, obj, **kwargs):
return self.__call_method('exists', obj, False, False, **kwargs)
def file_ready(self, obj, **kwargs):
return self.__call_method('file_ready', obj, False, False, **kwargs)
def create(self, obj, **kwargs):
"""
create() is the only method in which obj.object_store_id may be None
"""
if obj.object_store_id is None or not self.exists(obj, **kwargs):
if obj.object_store_id is None or obj.object_store_id not in self.weighted_backend_ids:
try:
obj.object_store_id = random.choice(self.weighted_backend_ids)
except IndexError:
raise ObjectInvalid( 'objectstore.create, could not generate obj.object_store_id: %s, kwargs: %s'
%( str( obj ), str( kwargs ) ) )
object_session( obj ).add( obj )
object_session( obj ).flush()
log.debug("Selected backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id))
else:
log.debug("Using preferred backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id))
self.backends[obj.object_store_id].create(obj, **kwargs)
def empty(self, obj, **kwargs):
return self.__call_method('empty', obj, True, False, **kwargs)
def size(self, obj, **kwargs):
return self.__call_method('size', obj, 0, False, **kwargs)
def delete(self, obj, **kwargs):
return self.__call_method('delete', obj, False, False, **kwargs)
def get_data(self, obj, **kwargs):
return self.__call_method('get_data', obj, ObjectNotFound, True, **kwargs)
def get_filename(self, obj, **kwargs):
return self.__call_method('get_filename', obj, ObjectNotFound, True, **kwargs)
def update_from_file(self, obj, **kwargs):
if kwargs.get('create', False):
self.create(obj, **kwargs)
kwargs['create'] = False
return self.__call_method('update_from_file', obj, ObjectNotFound, True, **kwargs)
def get_object_url(self, obj, **kwargs):
return self.__call_method('get_object_url', obj, None, False, **kwargs)
def __call_method(self, method, obj, default, default_is_exception, **kwargs):
object_store_id = self.__get_store_id_for(obj, **kwargs)
if object_store_id is not None:
return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs)
if default_is_exception:
raise default( 'objectstore, __call_method failed: %s on %s, kwargs: %s'
%( method, str( obj ), str( kwargs ) ) )
else:
return default
def __get_store_id_for(self, obj, **kwargs):
if obj.object_store_id is not None and obj.object_store_id in self.backends:
return obj.object_store_id
else:
# if this instance has been switched from a non-distributed to a
# distributed object store, or if the object's store id is invalid,
# try to locate the object
log.warning('The backend object store ID (%s) for %s object with ID %s is invalid' % (obj.object_store_id, obj.__class__.__name__, obj.id))
for id, store in self.backends.items():
if store.exists(obj, **kwargs):
log.warning('%s object with ID %s found in backend object store with ID %s' % (obj.__class__.__name__, obj.id, id))
obj.object_store_id = id
object_session( obj ).add( obj )
object_session( obj ).flush()
return id
return None
