def __init__(self, config, config_xml): if boto is None: raise Exception(NO_BOTO_ERROR_MESSAGE) super(S3ObjectStore, self).__init__(config) self.staging_path = self.config.file_path self.transfer_progress = 0 self._parse_config_xml(config_xml) self._configure_connection() self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread( target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache try: subprocess.call('axel') self.use_axel = True except OSError: self.use_axel = False
def __init__(self, config, config_xml=None, fsmon=False): """ Create a new DistributedObjectStore. config -- Python object with any number of attributes. Most likely populated from `galaxy/config.ini`. The DistributedObjectStore specific attributes are: distributed_object_store_config_file config_xml -- An ElementTree object fsmon -- boolean. If true, monitor the file system for free space, removing backends when they get too full. """ super(DistributedObjectStore, self).__init__(config, config_xml=config_xml) if config_xml is None: self.distributed_config = config.distributed_object_store_config_file assert self.distributed_config is not None, \ "distributed object store ('object_store = distributed') " \ "requires a config file, please set one in " \ "'distributed_object_store_config_file')" self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} self.global_max_percent_full = 0.0 random.seed() self.__parse_distributed_config(config, config_xml) self.sleeper = None if fsmon and ( self.global_max_percent_full or filter( lambda x: x != 0.0, self.max_percent_full.values() ) ): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor) self.filesystem_monitor_thread.setDaemon( True ) self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started")
def __init__(self, app, dispatcher): """Initializes the Job Handler Queue, creates (unstarted) monitoring thread""" self.app = app self.dispatcher = dispatcher self.sa_session = app.model.context self.track_jobs_in_database = self.app.config.track_jobs_in_database # Initialize structures for handling job limits self.__clear_job_count() # Keep track of the pid that started the job manager, only it # has valid threads self.parent_pid = os.getpid() # Contains new jobs. Note this is not used if track_jobs_in_database is True self.queue = Queue() # Contains jobs that are waiting (only use from monitor thread) self.waiting_jobs = [] # Contains wrappers of jobs that are limited or ready (so they aren't created unnecessarily/multiple times) self.job_wrappers = {} # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True self.monitor_thread = threading.Thread( name="JobHandlerQueue.monitor_thread", target=self.__monitor) self.monitor_thread.setDaemon(True)
def _initialize(self): if boto is None: raise Exception(NO_BOTO_ERROR_MESSAGE) # for multipart upload self.s3server = { 'access_key': self.access_key, 'secret_key': self.secret_key, 'is_secure': self.is_secure, 'max_chunk_size': self.max_chunk_size, 'host': self.host, 'port': self.port, 'use_rr': self.use_rr, 'conn_path': self.conn_path } self._configure_connection() self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread( target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache if which('axel'): self.use_axel = True else: self.use_axel = False
def __init__(self, config, config_dict, fsmon=False): """ :type config: object :param config: An object, most likely populated from `galaxy/config.ini`, having the same attributes needed by :class:`NestedObjectStore` plus: * distributed_object_store_config_file :type config_xml: ElementTree :type fsmon: bool :param fsmon: If True, monitor the file system for free space, removing backends when they get too full. """ super(DistributedObjectStore, self).__init__(config, config_dict) self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} self.global_max_percent_full = config_dict.get( "global_max_percent_full", 0) random.seed() backends_def = config_dict["backends"] for backend_def in backends_def: backened_id = backend_def["id"] file_path = backend_def["files_dir"] extra_dirs = backend_def.get("extra_dirs", []) maxpctfull = backend_def.get("max_percent_full", 0) weight = backend_def["weight"] disk_config_dict = dict(files_dir=file_path, extra_dirs=extra_dirs) self.backends[backened_id] = DiskObjectStore( config, disk_config_dict) self.max_percent_full[backened_id] = maxpctfull log.debug( "Loaded disk backend '%s' with weight %s and file_path: %s" % (backened_id, weight, file_path)) for i in range(0, weight): # The simplest way to do weighting: add backend ids to a # sequence the number of times equalling weight, then randomly # choose a backend from that sequence at creation self.weighted_backend_ids.append(backened_id) self.original_weighted_backend_ids = self.weighted_backend_ids self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread( target=self.__filesystem_monitor) self.filesystem_monitor_thread.setDaemon(True) self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started")
def start_cache_monitor(self): # Clean cache only if value is set in galaxy.ini if self.cache_size != -1 and self.enable_cache_monitor: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread(target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started")
def __init__(self, config, config_dict, fsmon=False): """ :type config: object :param config: An object, most likely populated from `galaxy/config.ini`, having the same attributes needed by :class:`NestedObjectStore` plus: * distributed_object_store_config_file :type config_xml: ElementTree :type fsmon: bool :param fsmon: If True, monitor the file system for free space, removing backends when they get too full. """ super().__init__(config, config_dict) self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} self.global_max_percent_full = config_dict.get( "global_max_percent_full", 0) self.search_for_missing = config_dict.get("search_for_missing", True) random.seed() for backend_def in config_dict["backends"]: backened_id = backend_def["id"] maxpctfull = backend_def.get("max_percent_full", 0) weight = backend_def["weight"] backend = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon) self.backends[backened_id] = backend self.max_percent_full[backened_id] = maxpctfull for _ in range(0, weight): # The simplest way to do weighting: add backend ids to a # sequence the number of times equalling weight, then randomly # choose a backend from that sequence at creation self.weighted_backend_ids.append(backened_id) self.original_weighted_backend_ids = self.weighted_backend_ids self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread( target=self.__filesystem_monitor, args=[self.sleeper]) self.filesystem_monitor_thread.daemon = True self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started")
def __init__(self, app): self.app = app self.sa_session = app.model.context.current self.queue = Queue() self.plugins = {} self._load_plugins() self.sleeper = Sleeper() self.running = True self.waiting_jobs = [] self.__check_jobs_at_startup() self.monitor_thread = threading.Thread(target=self.__monitor) self.monitor_thread.start() log.info('Deferred job queue started')
def _initialize(self): if BlockBlobService is None: raise Exception(NO_BLOBSERVICE_ERROR_MESSAGE) self._configure_connection() # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread( target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started")
def __init__(self, config, config_xml): if BlockBlobService is None: raise Exception(NO_BLOBSERVICE_ERROR_MESSAGE) super(AzureBlobObjectStore, self).__init__(config) self.staging_path = self.config.file_path self.transfer_progress = 0 self._parse_config_xml(config_xml) self._configure_connection() # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread(target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started")
def _initialize(self): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) self.conn = self._get_connection(self.provider, self.credentials) self.bucket = self._get_bucket(self.bucket_name) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison self.cache_size = self.cache_size * 1073741824 # Helper for interruptable sleep self.sleeper = Sleeper() self.cache_monitor_thread = threading.Thread(target=self.__cache_monitor) self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache try: subprocess.call('axel') self.use_axel = True except OSError: self.use_axel = False
def __init__(self, config, config_xml=None, fsmon=False): super(DistributedObjectStore, self).__init__(config, config_xml=config_xml) if config_xml is None: self.distributed_config = config.distributed_object_store_config_file assert self.distributed_config is not None, "distributed object store ('object_store = distributed') " \ "requires a config file, please set one in " \ "'distributed_object_store_config_file')" self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} self.global_max_percent_full = 0.0 random.seed() self.__parse_distributed_config(config, config_xml) self.sleeper = None if fsmon and ( self.global_max_percent_full or filter( lambda x: x != 0.0, self.max_percent_full.values() ) ): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor) self.filesystem_monitor_thread.setDaemon( True ) self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started")
def __init__( self, app, dispatcher ): self.app = app self.dispatcher = dispatcher self.sa_session = app.model.context # Keep track of the pid that started the job manager, only it # has valid threads self.parent_pid = os.getpid() # Contains new jobs. Note this is not used if track_jobs_in_database is True self.queue = Queue() # Contains jobs that are waiting (only use from monitor thread) self.waiting = [] # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True self.monitor_thread = threading.Thread( name="JobHandlerStopQueue.monitor_thread", target=self.monitor ) self.monitor_thread.setDaemon( True ) self.monitor_thread.start() log.info( "job handler stop queue started" )
def __init__(self, config, config_xml=None, fsmon=False): """ :type config: object :param config: An object, most likely populated from `galaxy/config.ini`, having the same attributes needed by :class:`NestedObjectStore` plus: * distributed_object_store_config_file :type config_xml: ElementTree :type fsmon: bool :param fsmon: If True, monitor the file system for free space, removing backends when they get too full. """ super(DistributedObjectStore, self).__init__(config, config_xml=config_xml) if config_xml is None: self.distributed_config = config.distributed_object_store_config_file assert self.distributed_config is not None, \ "distributed object store ('object_store = distributed') " \ "requires a config file, please set one in " \ "'distributed_object_store_config_file')" self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} self.global_max_percent_full = 0.0 random.seed() self.__parse_distributed_config(config, config_xml) self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread( target=self.__filesystem_monitor) self.filesystem_monitor_thread.setDaemon(True) self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started")