def connect(self, reflect=True, existing_metadata=None):
""" Connect to the database.
When a connection has been established, the
`on_connection_established` is invoked with the metadata.
If connecting fails, the `on_connection_failed` is invoked
with the failure.
:param reflect: If true, reflects the tables and their relations.
:param existing_metadata: If provided, use this existing
metadata instead of creating a metadata instance.
:param reconnect_on_failure: If true, keeps reconnecting if the initial
attempt fails.
"""
if self.is_connected:
return
self._should_reflect = reflect
self._configure_driver()
self._make_metadata(existing_metadata)
try:
self._try_connecting()
except sa.exc.SQLAlchemyError, e:
log.error('Could not connect to database "%s": %s' % (self.database_name, e))
reactor.callFromThread(self.on_connection_failed, failure.Failure())
raise
def produce_ticks(self):
while self.running:
try:
self._waiting_for_pipeline = self.dependencies.wait_for_resource('pipeline')
pipeline = yield self._waiting_for_pipeline
baton = self._create_baton()
self._waiting_for_pipeline = pipeline.process(baton)
yield self._waiting_for_pipeline
except Exception as e:
log.error()
finally:
self._waiting_for_pipeline = None
# we might have stopped running while waiting for the pipeline to finish processing
if not self.running:
return
# the processing might have taken some time, so subtract the time taken from the interval before waiting
# we set the minimum sleep time to 0 in order to at least wait 1 reactor iteration between every
# produced baton.
processing_duration = time.time()-self._previous_tick
sleep_time = max(self.interval-processing_duration, 0)
try:
self._sleeping = util.wait(sleep_time)
yield self._sleeping
except defer.CancelledError:
return
finally:
self._sleeping = None
def startService(self):
""" Start the reqest in the reactors thread pool. """
self.running = True
self.running_service = True
while self.running_service:
try:
yield threads.deferToThread(self._run)
except Exception as e:
log.error()
def __del__(self):
# It's really important to close sockets before the context is
# attempted terminated (which it is when it's garbage
# collected). Any open sockets will block the termination,
# effectively causing the entire Python-process to hang.
for socket_name, socket in self._socket_by_name.items():
try:
socket.close()
except:
# Nothing we can do, but we'll want to try to close the other sockets.
log.error('Could not close ZeroMQ-socket named "%s"' % socket_name)
util.reraise_if_should_exit()
def lineLengthExceeded(self, line):
e_msg = 'Line length exceeded. Buffer has been dropped.'
hint = 'While reading from stdin, too much data were read without encountering a delimiter.'
detail = ('Make sure the expected stdin.delimiter are being used (currently %s), or adjust '
'the stdin.max_length value (currently %s)')
detail = detail%(repr(self.delimiter), self.MAX_LENGTH)
try:
raise StdinException(e_msg, hint, detail)
except:
log.error()
def lineLengthExceeded(self, line):
e_msg = 'Line length exceeded by %s. Buffer has been dropped.'%self.process_protocol.process_name
detail = 'While reading from %s, too much data were read without encountering a delimiter.'%self.type
hint = ('Make sure the expected delimiter are being used (currently %(delimiter)s), or adjust '
'the max_length value (currently %(max_length)s)')
hint = hint%dict(delimiter=repr(self.delimiter), max_length=self.MAX_LENGTH)
try:
raise exceptions.PipedError(e_msg, detail, hint)
except:
log.error()
def _persist_context(self, context_name, context):
kind = self.persisted_contexts[context_name].get('kind', 'json')
file_path = self._get_filepath_for_persisted_context(context_name)
assert kind in ('json', 'pickle')
try:
if kind == 'json':
yield threads.deferToThread(json.dump, context, file_path.open('w'))
else:
yield threads.deferToThread(pickle.dump, context, file_path.open('w'))
except Exception, e:
log.error('Could not persist context "%s": %s' % (context_name, e.args[0]))
raise
def _test_connection_until_working(self):
""" Keep trying to connect. Calls itself every
`wait_between_reconnect_tries` seconds. """
while self.running:
try:
log.debug('Trying to connect to database "%s"' % self.database_name)
yield threads.deferToThread(self.test_connectivity)
yield threads.deferToThread(self.reconnected)
break
except sa.exc.SQLAlchemyError, e:
reactor.callFromThread(self.on_connection_failed, failure.Failure())
log.error('Could not connect to database "%s": %s' % (self.database_name, e))
yield util.wait(self.reconnect_wait)
def handle_messages(self, message_batch):
try:
for message in message_batch:
try:
pipeline = yield self.pipeline_dependency.wait_for_resource()
# TODO: We ought to have a cooperator somewhere
# else, for all the pipelines regardless of where
# the batons come from.
cooperative = task.cooperate(iter([pipeline.process(message).addErrback(self._log_process_error)]))
yield cooperative.whenDone()
except:
util.reraise_if_should_exit()
log.error()
finally:
self._register_in_poller()
def __init__(self, database_configuration):
self.database_configuration = database_configuration
for key in ('database_name', 'user', 'password'):
setattr(self, key, database_configuration.get(key))
# Options for which we'll provide a default:
for key, default in (('protocol', 'postgresql'), ('pool_size', 10), ('timeout', 1), ('reconnect_wait', 10),
('host', '')): # The empty string as a host means use local socket
setattr(self, key, database_configuration.get(key, default))
if not self.protocol in ('postgresql', 'mysql'):
raise ValueError('The provider currently only works with PostgreSQL and MySQL, not ' + self.protocol)
# Options to provide to the underlying DBAPI-driver.
self.dbapi_options = database_configuration.get('dbapi_options', {})
self.is_connected = False
self.engine = None
self.port = None # The default port is set by _configure_(postgres|mysql)
self.on_connection_established = event.Event() #: Event called when a connection has been established. Provides a metadata instance as argument.
self.on_connection_lost = event.Event()
self.on_connection_failed = event.Event()
log_failure = lambda failure: log.error('A connection to "%s" failed: %s' % (self.database_name, failure.getErrorMessage()))
self.on_connection_failed += log_failure
self.on_connection_failed += lambda failure: self.keep_reconnecting_and_report()
self._reconnect_deferred = None
self._should_reflect = False
self._has_reflected = False
self.metadata = None
self.metadata_factory = sa.MetaData
def process(self, baton):
status_text = util.dict_get_path(baton, self.status_path, Ellipsis)
if status_text is not Ellipsis:
# get the api by waiting for the dependency to become ready
api = yield self.api_dependency.wait_for_resource()
try:
status = yield api.update_status(status_text)
except error.TweepError as e:
log.error('Error updating the status: %s' % e)
else:
log.warn('Updated the status of %r to %r' % (status.user.screen_name, status.text))
else:
log.warn('No value at %r in the baton. Not updating the status.' % self.status_path)
# return the baton unchanged
defer.returnValue(baton)
def process_input(self, input, baton):
try:
return self.lambda_(input)
except:
log.error('Failed "%s"' % self.lambda_definition)
raise
def _log_process_error(self, reason):
log.error(reason)
def _log_error_getting_root_object(self, reason):
self.on_failed_getting_root_object(reason)
log.error(reason)
