def _more_init(self): """ Executed after the process has started. Set here non-picklable attributes """ self.name += "_%s" % self.ident self.logger = self.logger = logging.getLogger("FR.%s" % self.getName()) self.connector = StorageConnector(self.warebox, self.cfg)
def _more_init(self): """ Executed after the process has started. Set here non-picklable attributes """ self.warebox = warebox.Warebox(self.cfg) self.connector = StorageConnector(self.warebox, self.cfg) self.logger.debug(u"Hello, my PID is %s" % self.pid)
def _get_ready_for_service(self): """Perform further initialization. Here are initialized those components that need any information available at runtime and thus couldn't be initialized by the constructor. """ # TODO: do we need this at all? self.logger.debug(u"Initializing Warebox...") self._warebox = Warebox(self.cfg) self.logger.debug(u"Initializing Warebox Cache...") session_queue = MultiQueue([ 'servermessage', # Messages sent by the server 'operation', # PathnameOperation objects to handle 'usercommand', # Commands sent by the user 'sessioncommand', # ServerSession internal use commands 'systemcommand' # Commands sent by other client components ]) self.logger.debug(u"Initializing Event Queue...") self.queue = EventsQueue(self._internal_facade, session_queue) self.logger.debug(u"Initializing Storage Connector...") self.connector = StorageConnector(self._warebox, self.cfg) self.logger.debug(u"Initializing FileSystem Watcher...") self.FSWatcher = filesystemwatcher.watcher_class(self._warebox, self.queue, start_suspended=True) self.logger.debug(u"Initializing Startup Synchronization...") self.startup_synchronization = StartupSynchronization( self._warebox, self.storage_cache, self.queue) self.logger.debug(u"Initializing Server Session...") self._server_session = ServerSession(self.cfg, self._warebox, self.storage_cache, self.startup_synchronization, self.FSWatcher, self.linker, self._metadata_db, self.hashesDB, self._internal_facade, self._ui_controller, self.lockfile_fd, auto_start=self.auto_start, input_queue=session_queue, scheduler=self._scheduler) self.logger.debug(u"Initialization completed successfully")
class WorkerChild(Thread): """ Handle the upload and download of files Communicates with the core within multiprocess queues """ def __init__(self, warebox, inputQueue, communicationQueue, terminationEvent, percentage_callback, cfg, pool): """ @param inputQueue: multiprocess queue used to send pathname_operation to the child @param communcationQueue: multiprocess queue used to send back results @param terminationEvent: multiprocess event used to stop the upload/download @param logs_queue: multiprocess queue used to send back log messages @param cfg: instance of filerockclient.config.ConfigManager """ super(WorkerChild, self).__init__(name=self.__class__.__name__) self.inputQueue = inputQueue self.communicationQueue = communicationQueue self.terminationEvent = terminationEvent self.cfg = cfg self.up_bandwidth = pool.up_bandwidth self.down_bandwidth = pool.down_bandwidth self.percentage_callback = percentage_callback self.warebox = warebox def _check_download_dir(self, download_dir): if os.path.exists(download_dir) and os.path.isdir(download_dir): return True elif os.path.exists(download_dir) and not os.path.isdir(download_dir): os.unlink(download_dir) elif not os.path.exists(download_dir): os.makedirs(download_dir) def _get_download_dir(self): temp_dir = self.cfg.get('Application Paths', 'temp_dir') temp_dir = os.path.join(temp_dir, DOWNLOAD_DIR) self._check_download_dir(temp_dir) return temp_dir def _get_temp_file(self, file_operation): if file_operation.verb == 'DOWNLOAD': temp_dir = self._get_download_dir() temp_fd, temp_pathname = mkstemp(dir=temp_dir) os.close(temp_fd) file_operation.temp_pathname = temp_pathname file_operation.temp_fd = temp_fd def _more_init(self): """ Executed after the process has started. Set here non-picklable attributes """ self.name += "_%s" % self.ident self.logger = self.logger = logging.getLogger("FR.%s" % self.getName()) self.connector = StorageConnector(self.warebox, self.cfg) def run(self): """ Handles file operation received through the inputQueue until a poison pill is received, sends back logs and results through communicationQueue """ try: self._more_init() termination = False while not termination: operation, file_operation = self.inputQueue.get() self.logger.debug('==> Operation type %s, content %s' % (operation, file_operation)) if operation == 'FileOperation': self.terminationEvent.clear() self.logger.debug(u'Started to handle %s' % file_operation) self._get_temp_file(file_operation) result = self._handle_operation(file_operation) if result['status'] == SUCCESS: self.logger.debug( u'Operation completed: %s. Returning.' % file_operation) self.communicationQueue.put(('completed', result)) elif result['status'] == INTERRUPTED: self.logger.debug(u'Failed performing operation %s. ' 'INTERRUPTED.' % file_operation) self.communicationQueue.put(('interrupted', None)) elif result['status'] == FAILED: self.logger.debug(u'Failed performing operation %s. ' 'Returning.' % file_operation) self.communicationQueue.put(('failed', None)) elif operation == 'PoisonPill': termination = True self.logger.debug(u"I'm going to die") self.communicationQueue.put(('ShuttingDown', None)) except Exception: self.communicationQueue.put(('DIED', None)) raise def _handle_operation(self, file_operation): """ Handles a file_operation, uploading o downloading the associated file @param file_operation: instance of filerockclient.pathname_operation """ try: if file_operation.verb == 'UPLOAD': result = self._handle_upload_operation(file_operation) return result elif file_operation.verb == 'DOWNLOAD': result = self._handle_download_operation(file_operation) return result else: self.logger.debug(u'Unsupported verb for operation,' ' giving up: %s' % file_operation) result = {'status': FAILED} return result except Exception as e: self.logger.debug(u'Exception caught: %s\n%s' % (e, traceback.format_exc())) result = {'status': FAILED} return FAILED def _handle_upload_operation(self, file_operation): """ Handles upload operation Prepares useful data and delegates to the connector the upload @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_encrypt: pathname = file_operation.encrypted_pathname open_function = open etag = file_operation.storage_etag size = file_operation.storage_size iv = file_operation.iv else: pathname = file_operation.pathname open_function = self.warebox.open etag = file_operation.warebox_etag size = file_operation.warebox_size iv = None args = [ pathname, file_operation.upload_info['remote_pathname'], file_operation.upload_info['remote_ip_address'], file_operation.upload_info['bucket'], file_operation.upload_info['auth_token'], file_operation.upload_info['auth_date'], open_function, etag, size, iv ] percentage_callback = \ lambda percentage: \ self.percentage_callback(file_operation, PStatuses.UPLOADING, percentage) do_upload = \ lambda event: \ self.connector.upload_file(*args, terminationEvent=event, percentageQueue=percentage_callback, logger=self.logger, bandwidth=self.up_bandwidth) return self._perform_network_transfer(do_upload, file_operation) def _handle_download_operation(self, file_operation): """ Handles download operation Prepares useful data and delegates to the connector the download @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_decrypt: pathname = file_operation.encrypted_pathname open_function = open else: pathname = file_operation.temp_pathname open_function = open args = [ pathname, file_operation.pathname, file_operation.download_info['remote_ip_address'], file_operation.download_info['bucket'], file_operation.download_info['auth_token'], file_operation.download_info['auth_date'], open_function ] percentage_callback = \ lambda percentage: \ self.percentage_callback(file_operation, PStatuses.DOWNLOADING, percentage) def do_download(event): result = self.connector.download_file( *args, terminationEvent=event, percentageQueue=percentage_callback, logger=self.logger, bandwidth=self.down_bandwidth) return result return self._perform_network_transfer(do_download, file_operation) def _perform_network_transfer(self, transfer_strategy, file_operation): """Does a limited number of attempts to perform the given transfer. In case of failure a certain time interval is awaited and another attempt is performed. The waiting time is doubled each time until the maximum amount of attempts is reached. The transfer could be interrupted in any time by setting terminationEvent. @param transfer_strategy: lambda function wrapping the transfer method. @param file_operation: instance of filerockclient.pathname_operation. """ max_attempts = 10 waiting_time = 1 attempts = 0 while not self.terminationEvent.is_set() and attempts <= max_attempts: self.logger.debug(u'Started network transfer for: %s "%s":' % (file_operation.verb, file_operation.pathname)) response = transfer_strategy(self.terminationEvent) if response['success']: self.logger.debug( u'Successfully ended network transfer' ' for: %s "%s":' % (file_operation.verb, file_operation.pathname)) result = {'status': SUCCESS} if file_operation.verb == 'DOWNLOAD': result['actual_etag'] = response['etag'] return result elif 'termination' in response['details']: result = {'status': INTERRUPTED} return result self.logger.warning( u'HTTP %s failed for operation: %s. ' 'Retrying in %s seconds...' % (file_operation.verb, file_operation, waiting_time)) self.logger.debug(u'Response details: %s' % (response['details'])) waiting_time = stoppable_exponential_backoff_waiting( waiting_time, self.terminationEvent) attempts += 1 if self.terminationEvent.is_set(): # termination requested from outside result = {'status': INTERRUPTED} return result self.logger.error(u'Ok, I have tried performing %s for %d times.' ' I am done now. Put that stuff in a FedEx box and' ' send it via mail.' % (file_operation, max_attempts)) result = {'status': FAILED} return result
class WorkerChild(multiprocessing.Process): """ Handle the upload and download of files Communicates with the core within multiprocess queues """ def __init__(self, inputQueue, communicationQueue, terminationEvent, logs_queue, cfg): """ @param inputQueue: multiprocess queue used to send pathname_operation to the child @param communcationQueue: multiprocess queue used to send back results @param terminationEvent: multiprocess event used to stop the upload/download @param logs_queue: multiprocess queue used to send back log messages @param cfg: instance of filerockclient.config.ConfigManager """ # Note: executed before the process has started. Every attribute set here must be picklable. multiprocessing.Process.__init__(self) self.logger = SubprocessLogger(logs_queue) self.inputQueue = inputQueue self.communicationQueue = communicationQueue self.terminationEvent = terminationEvent self.cfg = cfg def __close_temp_file_fd(self, file_operation): if file_operation.verb == 'DOWNLOAD': if os.path.exists(file_operation.temp_pathname): os.close(file_operation.temp_fd) self.logger.debug(u'Temp fd %s closed' % file_operation.temp_fd) def __check_download_dir(self, download_dir): if os.path.exists(download_dir) and os.path.isdir(download_dir): return True elif os.path.exists(download_dir) and not os.path.isdir(download_dir): os.unlink(download_dir) elif not os.path.exists(download_dir): os.makedirs(download_dir) def __get_download_dir(self): temp_dir = os.path.join(self.cfg.get('User','temp_dir'), DOWNLOAD_DIR) self.__check_download_dir(temp_dir) return temp_dir def __get_temp_file(self, file_operation): if file_operation.verb == 'DOWNLOAD': temp_dir = self.__get_download_dir() temp_fd, temp_pathname = mkstemp(dir=temp_dir) file_operation.temp_pathname = temp_pathname file_operation.temp_fd = temp_fd def __rm_temp_file(self, file_operation): if file_operation.verb == 'DOWNLOAD': if os.path.exists(file_operation.temp_pathname): try: os.remove(file_operation.temp_pathname) self.logger.debug(u'Temp file %s deleted' % file_operation.temp_pathname) except: pass def _more_init(self): """ Executed after the process has started. Set here non-picklable attributes """ self.warebox = warebox.Warebox(self.cfg) self.connector = StorageConnector(self.warebox, self.cfg) self.logger.debug(u"Hello, my PID is %s" % self.pid) def run(self): """ Handles file operation received through the inputQueue until a poison pill is received, sends back logs and results through communicationQueue """ try: self._more_init() termination = False while not termination: try: operation, file_operation = self.inputQueue.get() self.logger.debug('==> Operation type %s, content %s' % (operation, file_operation)) if operation == 'FileOperation': self.terminationEvent.clear() self.logger.debug(u'Started to handle %s' % (file_operation)) self.__get_temp_file(file_operation) result, interrupted = self._handle_operation(file_operation) self.__rm_temp_file(file_operation) if result: self.logger.debug(u'Operation completed: %s. Returning.' % (file_operation)) self.communicationQueue.put(('result','completed')) elif not result and interrupted: self.logger.debug(u'Failed performing operation %s. INTERRUPTED.' % (file_operation)) self.communicationQueue.put(('result', 'interrupted')) elif not result and not interrupted: self.logger.debug(u'Failed performing operation %s. Returning.' % (file_operation)) self.communicationQueue.put(('result', 'failed')) elif operation == 'PoisonPill': termination = True self.logger.debug(u"I'm going to die") self.communicationQueue.put(('ShuttingDown', None)) except KeyboardInterrupt: pass except Exception: self.communicationQueue.put(('DIED',None)) # self.percentuageQueue.put(None) def _handle_operation(self, file_operation): """ Handles a file_operation, uploading o downloading the associated file @param file_operation: instance of filerockclient.pathname_operation """ try: if file_operation.verb == 'UPLOAD': result = self._handle_upload_operation(file_operation) return result elif file_operation.verb == 'DOWNLOAD': result, interrupted = self._handle_download_operation(file_operation) if not interrupted: if result and file_operation.to_decrypt: CryptoHelpers.decrypt(file_operation, self.warebox, self.cfg, self.logger) self.__close_temp_file_fd(file_operation) self.warebox.move(file_operation.temp_pathname, file_operation.pathname, file_operation.conflicted) return (result, interrupted) else: self.logger.debug(u'Unsupported verb for operation, giving up: %s' % (file_operation)) return False, False except Exception as e: self.logger.debug(u'Exception caught: %s\n%s' % (e, traceback.format_exc())) return False, False except KeyboardInterrupt: self.logger.debug(u'Caught a KeyboardInterrupt, this means that the application is closing. I give up.') return False, True def _handle_upload_operation(self, file_operation): """ Handles upload operation Prepares useful data and delegates to the connector the upload @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_encrypt: pathname = file_operation.encrypted_pathname open_function = open mode = 'rb' etag = file_operation.storage_etag size = file_operation.storage_size iv = file_operation.iv else: pathname = file_operation.pathname open_function = self.warebox.open mode = 'r' etag = file_operation.warebox_etag size = file_operation.warebox_size iv = None args = [ pathname, file_operation.upload_info['remote_pathname'], file_operation.upload_info['remote_ip_address'], file_operation.upload_info['bucket'], file_operation.upload_info['auth_token'], file_operation.upload_info['auth_date'], open_function, etag, size, iv ] do_upload = lambda event: self.connector.upload_file(*args, terminationEvent=event, percentageQueue=self.communicationQueue, logger=self.logger) return self._perform_network_transfer(do_upload, file_operation) def _handle_download_operation(self, file_operation): """ Handles download operation Prepares useful data and delegates to the connector the download @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_decrypt: pathname = file_operation.encrypted_pathname open_function = open mode = 'wb' else: pathname = file_operation.temp_pathname open_function= open mode = 'wb' args = [ pathname, file_operation.pathname, file_operation.download_info['remote_ip_address'], file_operation.download_info['bucket'], file_operation.download_info['auth_token'], file_operation.download_info['auth_date'], open_function ] do_download = lambda event: self.connector.download_file(*args, terminationEvent=event, percentageQueue=self.communicationQueue, logger=self.logger) return self._perform_network_transfer(do_download, file_operation) def _perform_network_transfer(self, transfer_strategy, file_operation): """ Does a limited number of attempts to perform the given transfer. In case of failure a certain time interval is awaited and another attempt is performed. The waiting time is doubled each time until the maximum amount of attempts is reached. The transfer could be interrupted in any time by setting terminationEvent @param transfer_strategy: lambda function wrapping the transfer method @param file_operation: instance of filerockclient.pathname_operation """ # # if self.terminationQueue is not None: # event = threading.Event() # termination = TerminationThread(self.terminationQueue, event) # termination.start() max_attempts = 10 waiting_time = 1 attempts = 0 while not self.terminationEvent.is_set() and attempts <= max_attempts: self.logger.debug(u'Started network transfer for: %s "%s":' % (file_operation.verb, file_operation.pathname)) response = transfer_strategy(self.terminationEvent) if response['success']: self.logger.debug(u'Successfully ended network transfer for: %s "%s":' % (file_operation.verb, file_operation.pathname)) return (True, False) else: if 'termination' in response['details']: return (False, True) self.logger.warning(u'HTTP %s failed for operation: %s. Retrying in %s seconds...' % (file_operation.verb, file_operation, waiting_time)) self.logger.debug(u'Response details: %s' % (response['details'])) waiting_time = stoppable_exponential_backoff_waiting(waiting_time, self.terminationEvent) attempts += 1 if self.terminationEvent.is_set(): # termination required from outside return (False, True) self.logger.error(u'Ok, I have tried performing %s for %d times. I am done now. Put that stuff in a FedEx box and send it via mail.' % (file_operation, max_attempts)) return (False, False)
class WorkerChild(Thread): """ Handle the upload and download of files Communicates with the core within multiprocess queues """ def __init__(self, warebox, inputQueue, communicationQueue, terminationEvent, percentage_callback, cfg, pool): """ @param inputQueue: multiprocess queue used to send pathname_operation to the child @param communcationQueue: multiprocess queue used to send back results @param terminationEvent: multiprocess event used to stop the upload/download @param logs_queue: multiprocess queue used to send back log messages @param cfg: instance of filerockclient.config.ConfigManager """ super(WorkerChild, self).__init__(name=self.__class__.__name__) self.inputQueue = inputQueue self.communicationQueue = communicationQueue self.terminationEvent = terminationEvent self.cfg = cfg self.up_bandwidth = pool.up_bandwidth self.down_bandwidth = pool.down_bandwidth self.percentage_callback = percentage_callback self.warebox = warebox def _check_download_dir(self, download_dir): if os.path.exists(download_dir) and os.path.isdir(download_dir): return True elif os.path.exists(download_dir) and not os.path.isdir(download_dir): os.unlink(download_dir) elif not os.path.exists(download_dir): os.makedirs(download_dir) def _get_download_dir(self): temp_dir = self.cfg.get('Application Paths', 'temp_dir') temp_dir = os.path.join(temp_dir, DOWNLOAD_DIR) self._check_download_dir(temp_dir) return temp_dir def _get_temp_file(self, file_operation): if file_operation.verb == 'DOWNLOAD': temp_dir = self._get_download_dir() temp_fd, temp_pathname = mkstemp(dir=temp_dir) os.close(temp_fd) file_operation.temp_pathname = temp_pathname file_operation.temp_fd = temp_fd def _more_init(self): """ Executed after the process has started. Set here non-picklable attributes """ self.name += "_%s" % self.ident self.logger = self.logger = logging.getLogger("FR.%s" % self.getName()) self.connector = StorageConnector(self.warebox, self.cfg) def run(self): """ Handles file operation received through the inputQueue until a poison pill is received, sends back logs and results through communicationQueue """ try: self._more_init() termination = False while not termination: operation, file_operation = self.inputQueue.get() self.logger.debug('==> Operation type %s, content %s' % (operation, file_operation)) if operation == 'FileOperation': self.terminationEvent.clear() self.logger.debug(u'Started to handle %s' % file_operation) self._get_temp_file(file_operation) result = self._handle_operation(file_operation) if result['status'] == SUCCESS: self.logger.debug( u'Operation completed: %s. Returning.' % file_operation) self.communicationQueue.put(('completed', result)) elif result['status'] == INTERRUPTED: self.logger.debug( u'Failed performing operation %s. ' 'INTERRUPTED.' % file_operation) self.communicationQueue.put(('interrupted', None)) elif result['status'] == FAILED: self.logger.debug( u'Failed performing operation %s. ' 'Returning.' % file_operation) self.communicationQueue.put(('failed', None)) elif operation == 'PoisonPill': termination = True self.logger.debug(u"I'm going to die") self.communicationQueue.put(('ShuttingDown', None)) except Exception: self.communicationQueue.put(('DIED', None)) raise def _handle_operation(self, file_operation): """ Handles a file_operation, uploading o downloading the associated file @param file_operation: instance of filerockclient.pathname_operation """ try: if file_operation.verb == 'UPLOAD': result = self._handle_upload_operation(file_operation) return result elif file_operation.verb == 'DOWNLOAD': result = self._handle_download_operation(file_operation) return result else: self.logger.debug(u'Unsupported verb for operation,' ' giving up: %s' % file_operation) result = {'status': FAILED} return result except Exception as e: self.logger.debug(u'Exception caught: %s\n%s' % (e, traceback.format_exc())) result = {'status': FAILED} return FAILED def _handle_upload_operation(self, file_operation): """ Handles upload operation Prepares useful data and delegates to the connector the upload @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_encrypt: pathname = file_operation.encrypted_pathname open_function = open etag = file_operation.storage_etag size = file_operation.storage_size iv = file_operation.iv else: pathname = file_operation.pathname open_function = self.warebox.open etag = file_operation.warebox_etag size = file_operation.warebox_size iv = None args = [ pathname, file_operation.upload_info['remote_pathname'], file_operation.upload_info['remote_ip_address'], file_operation.upload_info['bucket'], file_operation.upload_info['auth_token'], file_operation.upload_info['auth_date'], open_function, etag, size, iv ] percentage_callback = \ lambda percentage: \ self.percentage_callback(file_operation, PStatuses.UPLOADING, percentage) do_upload = \ lambda event: \ self.connector.upload_file(*args, terminationEvent=event, percentageQueue=percentage_callback, logger=self.logger, bandwidth=self.up_bandwidth) return self._perform_network_transfer(do_upload, file_operation) def _handle_download_operation(self, file_operation): """ Handles download operation Prepares useful data and delegates to the connector the download @param file_operation: instance of filerockclient.pathname_operation """ if file_operation.to_decrypt: pathname = file_operation.encrypted_pathname open_function = open else: pathname = file_operation.temp_pathname open_function = open args = [ pathname, file_operation.pathname, file_operation.download_info['remote_ip_address'], file_operation.download_info['bucket'], file_operation.download_info['auth_token'], file_operation.download_info['auth_date'], open_function ] percentage_callback = \ lambda percentage: \ self.percentage_callback(file_operation, PStatuses.DOWNLOADING, percentage) def do_download(event): result = self.connector.download_file( *args, terminationEvent=event, percentageQueue=percentage_callback, logger=self.logger, bandwidth=self.down_bandwidth) return result return self._perform_network_transfer(do_download, file_operation) def _perform_network_transfer(self, transfer_strategy, file_operation): """Does a limited number of attempts to perform the given transfer. In case of failure a certain time interval is awaited and another attempt is performed. The waiting time is doubled each time until the maximum amount of attempts is reached. The transfer could be interrupted in any time by setting terminationEvent. @param transfer_strategy: lambda function wrapping the transfer method. @param file_operation: instance of filerockclient.pathname_operation. """ max_attempts = 10 waiting_time = 1 attempts = 0 while not self.terminationEvent.is_set() and attempts <= max_attempts: self.logger.debug(u'Started network transfer for: %s "%s":' % (file_operation.verb, file_operation.pathname)) response = transfer_strategy(self.terminationEvent) if response['success']: self.logger.debug(u'Successfully ended network transfer' ' for: %s "%s":' % (file_operation.verb, file_operation.pathname)) result = {'status': SUCCESS} if file_operation.verb == 'DOWNLOAD': result['actual_etag'] = response['etag'] return result elif 'termination' in response['details']: result = {'status': INTERRUPTED} return result self.logger.warning(u'HTTP %s failed for operation: %s. ' 'Retrying in %s seconds...' % (file_operation.verb, file_operation, waiting_time)) self.logger.debug(u'Response details: %s' % (response['details'])) waiting_time = stoppable_exponential_backoff_waiting( waiting_time, self.terminationEvent) attempts += 1 if self.terminationEvent.is_set(): # termination requested from outside result = {'status': INTERRUPTED} return result self.logger.error(u'Ok, I have tried performing %s for %d times.' ' I am done now. Put that stuff in a FedEx box and' ' send it via mail.' % (file_operation, max_attempts)) result = {'status': FAILED} return result