def connect(self, reflect=True, existing_metadata=None): """ Connect to the database. When a connection has been established, the `on_connection_established` is invoked with the metadata. If connecting fails, the `on_connection_failed` is invoked with the failure. :param reflect: If true, reflects the tables and their relations. :param existing_metadata: If provided, use this existing metadata instead of creating a metadata instance. :param reconnect_on_failure: If true, keeps reconnecting if the initial attempt fails. """ if self.is_connected: return self._should_reflect = reflect self._configure_driver() self._make_metadata(existing_metadata) try: self._try_connecting() except sa.exc.SQLAlchemyError, e: log.error('Could not connect to database "%s": %s' % (self.database_name, e)) reactor.callFromThread(self.on_connection_failed, failure.Failure()) raise
def produce_ticks(self): while self.running: try: self._waiting_for_pipeline = self.dependencies.wait_for_resource('pipeline') pipeline = yield self._waiting_for_pipeline baton = self._create_baton() self._waiting_for_pipeline = pipeline.process(baton) yield self._waiting_for_pipeline except Exception as e: log.error() finally: self._waiting_for_pipeline = None # we might have stopped running while waiting for the pipeline to finish processing if not self.running: return # the processing might have taken some time, so subtract the time taken from the interval before waiting # we set the minimum sleep time to 0 in order to at least wait 1 reactor iteration between every # produced baton. processing_duration = time.time()-self._previous_tick sleep_time = max(self.interval-processing_duration, 0) try: self._sleeping = util.wait(sleep_time) yield self._sleeping except defer.CancelledError: return finally: self._sleeping = None
def startService(self): """ Start the reqest in the reactors thread pool. """ self.running = True self.running_service = True while self.running_service: try: yield threads.deferToThread(self._run) except Exception as e: log.error()
def __del__(self): # It's really important to close sockets before the context is # attempted terminated (which it is when it's garbage # collected). Any open sockets will block the termination, # effectively causing the entire Python-process to hang. for socket_name, socket in self._socket_by_name.items(): try: socket.close() except: # Nothing we can do, but we'll want to try to close the other sockets. log.error('Could not close ZeroMQ-socket named "%s"' % socket_name) util.reraise_if_should_exit()
def lineLengthExceeded(self, line): e_msg = 'Line length exceeded. Buffer has been dropped.' hint = 'While reading from stdin, too much data were read without encountering a delimiter.' detail = ('Make sure the expected stdin.delimiter are being used (currently %s), or adjust ' 'the stdin.max_length value (currently %s)') detail = detail%(repr(self.delimiter), self.MAX_LENGTH) try: raise StdinException(e_msg, hint, detail) except: log.error()
def lineLengthExceeded(self, line): e_msg = 'Line length exceeded by %s. Buffer has been dropped.'%self.process_protocol.process_name detail = 'While reading from %s, too much data were read without encountering a delimiter.'%self.type hint = ('Make sure the expected delimiter are being used (currently %(delimiter)s), or adjust ' 'the max_length value (currently %(max_length)s)') hint = hint%dict(delimiter=repr(self.delimiter), max_length=self.MAX_LENGTH) try: raise exceptions.PipedError(e_msg, detail, hint) except: log.error()
def _persist_context(self, context_name, context): kind = self.persisted_contexts[context_name].get('kind', 'json') file_path = self._get_filepath_for_persisted_context(context_name) assert kind in ('json', 'pickle') try: if kind == 'json': yield threads.deferToThread(json.dump, context, file_path.open('w')) else: yield threads.deferToThread(pickle.dump, context, file_path.open('w')) except Exception, e: log.error('Could not persist context "%s": %s' % (context_name, e.args[0])) raise
def _test_connection_until_working(self): """ Keep trying to connect. Calls itself every `wait_between_reconnect_tries` seconds. """ while self.running: try: log.debug('Trying to connect to database "%s"' % self.database_name) yield threads.deferToThread(self.test_connectivity) yield threads.deferToThread(self.reconnected) break except sa.exc.SQLAlchemyError, e: reactor.callFromThread(self.on_connection_failed, failure.Failure()) log.error('Could not connect to database "%s": %s' % (self.database_name, e)) yield util.wait(self.reconnect_wait)
def handle_messages(self, message_batch): try: for message in message_batch: try: pipeline = yield self.pipeline_dependency.wait_for_resource() # TODO: We ought to have a cooperator somewhere # else, for all the pipelines regardless of where # the batons come from. cooperative = task.cooperate(iter([pipeline.process(message).addErrback(self._log_process_error)])) yield cooperative.whenDone() except: util.reraise_if_should_exit() log.error() finally: self._register_in_poller()
def __init__(self, database_configuration): self.database_configuration = database_configuration for key in ('database_name', 'user', 'password'): setattr(self, key, database_configuration.get(key)) # Options for which we'll provide a default: for key, default in (('protocol', 'postgresql'), ('pool_size', 10), ('timeout', 1), ('reconnect_wait', 10), ('host', '')): # The empty string as a host means use local socket setattr(self, key, database_configuration.get(key, default)) if not self.protocol in ('postgresql', 'mysql'): raise ValueError('The provider currently only works with PostgreSQL and MySQL, not ' + self.protocol) # Options to provide to the underlying DBAPI-driver. self.dbapi_options = database_configuration.get('dbapi_options', {}) self.is_connected = False self.engine = None self.port = None # The default port is set by _configure_(postgres|mysql) self.on_connection_established = event.Event() #: Event called when a connection has been established. Provides a metadata instance as argument. self.on_connection_lost = event.Event() self.on_connection_failed = event.Event() log_failure = lambda failure: log.error('A connection to "%s" failed: %s' % (self.database_name, failure.getErrorMessage())) self.on_connection_failed += log_failure self.on_connection_failed += lambda failure: self.keep_reconnecting_and_report() self._reconnect_deferred = None self._should_reflect = False self._has_reflected = False self.metadata = None self.metadata_factory = sa.MetaData
def process(self, baton): status_text = util.dict_get_path(baton, self.status_path, Ellipsis) if status_text is not Ellipsis: # get the api by waiting for the dependency to become ready api = yield self.api_dependency.wait_for_resource() try: status = yield api.update_status(status_text) except error.TweepError as e: log.error('Error updating the status: %s' % e) else: log.warn('Updated the status of %r to %r' % (status.user.screen_name, status.text)) else: log.warn('No value at %r in the baton. Not updating the status.' % self.status_path) # return the baton unchanged defer.returnValue(baton)
def process_input(self, input, baton): try: return self.lambda_(input) except: log.error('Failed "%s"' % self.lambda_definition) raise
def _log_process_error(self, reason): log.error(reason)
def _log_error_getting_root_object(self, reason): self.on_failed_getting_root_object(reason) log.error(reason)