def import_email(self, email_id):
mailbox = self.mailbox
# print "running fetch for message: "+email_id
try:
status, message_data = mailbox.uid('fetch', email_id, "(RFC822)")
if not is_ok((status,)):
raise ClientError(message_data)
# print repr(message_data)
for response_part in message_data:
if isinstance(response_part, tuple):
message_string = response_part[1]
if not message_string:
raise ClientError()
try:
if self.source.message_ok_to_import(message_string):
(email_object, dummy, error) = self.source.parse_email(message_string)
if error:
raise ReaderError(error)
self.source.db.add(email_object)
else:
print "Skipped message with imap id %s (bounce or vacation message)" % (email_id)
# print "Setting self.source.last_imported_email_uid to "+email_id
self.source.last_imported_email_uid = email_id
self.source.db.commit()
finally:
self.source = ContentSource.get(self.source.id)
except IMAP4.abort as e:
raise IrrecoverableError(e)
except IMAP4.error as e:
raise ClientError(e)
def read(self, source_id, reimport=False, force_restart=False, **kwargs):
from assembl.models import ContentSource
reader = self.readers.get(source_id, None)
if force_restart and reader is not None:
reader.shutdown()
self.readers.pop(source_id)
reader = None
if not (reader and reader.is_connected()):
source = ContentSource.get(source_id)
if not source:
return False
if source.connection_error == ReaderStatus.IRRECOVERABLE_ERROR and not force_restart:
return False
if force_restart:
source.reset_errors()
reader = source.make_reader()
self.readers[source_id] = reader
if reader is None:
return False
reader.setup_read(reimport, **kwargs)
reader.start()
return True
if reader is None:
return False
# We know it is connected by now.
reader.setup_read(reimport, **kwargs)
reader.wake()
return True
def setup(self):
from assembl.models import ContentSource
self.source = ContentSource.get(self.source_id)
connection_error = self.source.connection_error
self.error_backoff_until = self.source.error_backoff_until
if connection_error:
self.status = connection_error
def import_email(self, email_id):
mailbox = self.mailbox
# log.debug( "running fetch for message: "+email_id)
try:
messages = self.mailbox.fetch([email_id], [b"RFC822"])
# log.debug( repr(messages))
message_string = messages[email_id][b"RFC822"]
assert message_string
message_string = message_string.decode('ascii')
try:
if self.source.message_ok_to_import(message_string):
(email_object, dummy, error) = self.source.parse_email(message_string)
if error:
raise ReaderError(error)
self.source.db.add(email_object)
else:
log.info("Skipped message with imap id %s (bounce or vacation message)" % (email_id))
# log.debug( "Setting self.source.last_imported_email_uid to "+email_id)
self.source.last_imported_email_uid = email_id
self.source.db.commit()
finally:
self.source = ContentSource.get(self.source.id)
except IMAP4.abort as e:
raise IrrecoverableError(e)
except IMAP4.error as e:
raise ClientError(e)
def read(self, source_id, reimport=False, force_restart=False, **kwargs):
from assembl.models import ContentSource
reader = self.readers.get(source_id, None)
if force_restart and reader is not None:
reader.shutdown()
reader = None
if not (reader and reader.is_connected()):
source = ContentSource.get(source_id)
if not source:
return False
if source.connection_error == ReaderStatus.IRRECOVERABLE_ERROR and not force_restart:
return False
if force_restart:
source.reset_errors()
reader = source.make_reader()
self.readers[source_id] = reader
if reader is None:
return False
reader.setup_read(reimport, **kwargs)
reader.start()
return True
if reader is None:
return False
# We know it is connected by now.
reader.setup_read(reimport, **kwargs)
reader.wake()
return True
def setup(self):
from assembl.models import ContentSource
backoff = 0.5 + uniform(0, 0.5)
while self.source is None:
try:
self.source = ContentSource.get(self.source_id)
except TimeoutError:
# Ran out of connection pool
log.error("TimeoutError for " + self.source_id)
sleep(backoff)
backoff *= 2
connection_error = (ReaderStatus(self.source.connection_error)
if self.source.connection_error else None)
self.error_backoff_until = self.source.error_backoff_until
if connection_error:
self.status = connection_error
def setup(self):
from assembl.models import ContentSource
backoff = 0.5 + uniform(0, 0.5)
while self.source is None:
try:
self.source = ContentSource.get(self.source_id)
except TimeoutError:
# Ran out of connection pool
log.error("TimeoutError for " + self.source_id)
sleep(backoff)
backoff *= 2
connection_error = (
ReaderStatus(self.source.connection_error)
if self.source.connection_error else None)
self.error_backoff_until = self.source.error_backoff_until
if connection_error:
self.status = connection_error
def read(self, source_id):
from assembl.models import ContentSource
source = ContentSource.get(source_id)
if source_id not in self.readers:
self.readers[source_id] = self.make_reader(source)
reader = self.readers[source_id]
if reader.status == IRRECOVERABLE_ERROR:
return False
if not reader.is_alive():
reader.status = CLIENT_ERROR
if reader.status in (CLIENT_ERROR, CLOSED):
if reader.status == CLIENT_ERROR:
reader.do_close() # Just in case.
reader = self.make_reader(source)
self.readers[source_id] = reader
reader.prod()
return True
def refresh_source(self):
from assembl.models import ContentSource
# after a commit, refresh the source so it's still usable
self.source = ContentSource.get(self.source_id)