def backend_process(self, function, args, kwargs):
backend = self.tasks.get()
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s', backend,
function)
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception as error:
self.logger.debug(
'%s: Called function %s raised an error: %r', backend,
function, error)
self.errors.append((backend, error, get_backtrace(error)))
else:
self.logger.debug('%s: Called function %s returned: %r',
backend, function, result)
if hasattr(result, '__iter__') and not isinstance(
result, basestring):
# Loop on iterator
try:
for subresult in result:
self.store_result(backend, subresult)
except Exception as error:
self.errors.append(
(backend, error, get_backtrace(error)))
else:
self.store_result(backend, result)
finally:
self.tasks.task_done()
def backend_process(self, function, args, kwargs):
"""
Internal method to run a method of a backend.
As this method may be blocking, it should be run on its own thread.
"""
backend = self.tasks.get()
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s', backend, function)
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception as error:
self.logger.debug('%s: Called function %s raised an error: %r', backend, function, error)
self.errors.append((backend, error, get_backtrace(error)))
else:
self.logger.debug('%s: Called function %s returned: %r', backend, function, result)
if hasattr(result, '__iter__') and not isinstance(result, (bytes, basestring)):
# Loop on iterator
try:
for subresult in result:
self.store_result(backend, subresult)
if self.stop_event.is_set():
break
except Exception as error:
self.errors.append((backend, error, get_backtrace(error)))
else:
self.store_result(backend, result)
finally:
self.tasks.task_done()
def backend_process(self, function, args, kwargs):
backend = self.tasks.get()
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s' % (backend, function))
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception as error:
self.logger.debug('%s: Called function %s raised an error: %r' % (backend, function, error))
self.errors.append((backend, error, get_backtrace(error)))
else:
self.logger.debug('%s: Called function %s returned: %r' % (backend, function, result))
if hasattr(result, '__iter__') and not isinstance(result, basestring):
# Loop on iterator
try:
for subresult in result:
self.store_result(backend, subresult)
except Exception as error:
self.errors.append((backend, error, get_backtrace(error)))
else:
self.store_result(backend, result)
finally:
self.tasks.task_done()
def run(self):
while not self.finished.isSet():
try:
self.function(*self.args, **self.kwargs)
except Exception:
# do not stop timer because of an exception
print get_backtrace()
self.finished.wait(self.interval)
self.finished.set()
def run(self):
while not self.finished.isSet():
try:
self.function(*self.args, **self.kwargs)
except Exception:
# do not stop timer because of an exception
print get_backtrace()
self.finished.wait(self.interval)
self.finished.set()
def on_boobid(self, boobid):
_id, backend_name = boobid.split('@', 1)
if backend_name in self.weboob.backend_instances:
backend = self.weboob.backend_instances[backend_name]
for cap in backend.iter_caps():
func = 'obj_info_%s' % cap.__name__[4:].lower()
if hasattr(self, func):
try:
for msg in getattr(self, func)(backend, _id):
yield msg
except Exception as e:
print get_backtrace()
yield 'Oops: [%s] %s' % (type(e).__name__, e)
break
def on_boobid(self, boobid):
_id, backend_name = boobid.split('@', 1)
if backend_name in self.weboob.backend_instances:
backend = self.weboob.backend_instances[backend_name]
for cap in backend.iter_caps():
func = 'obj_info_%s' % cap.__name__[4:].lower()
if hasattr(self, func):
try:
for msg in getattr(self, func)(backend, _id):
yield msg
except Exception as e:
print get_backtrace()
yield u'Oops: [%s] %s' % (type(e).__name__, e)
break
def on_url(self, url):
url = fixurl(url)
try:
content_type, hsize, title = BoobotBrowser().urlinfo(url)
if title:
yield u'URL: %s' % title
elif hsize:
yield u'URL (file): %s, %s' % (content_type, hsize)
else:
yield u'URL (file): %s' % content_type
except BrowserUnavailable as e:
yield u'URL (error): %s' % e
except Exception as e:
print get_backtrace()
yield 'Oops: [%s] %s' % (type(e).__name__, e)
def on_url(self, url):
url = fixurl(url)
try:
content_type, hsize, title = BoobotBrowser().urlinfo(url)
if title:
yield u'URL: %s' % title
elif hsize:
yield u'URL (file): %s, %s' % (content_type, hsize)
else:
yield u'URL (file): %s' % content_type
except BrowserUnavailable as e:
yield u'URL (error): %s' % e
except Exception as e:
print get_backtrace()
yield u'Oops: [%s] %s' % (type(e).__name__, e)
def build_index(self, path, filename):
"""
Rebuild index of modules of repository.
:param path: path of the repository
:type path: str
:param filename: file to save index
:type filename: str
"""
self.logger.debug('Rebuild index')
self.modules.clear()
self.errors.clear()
if os.path.isdir(os.path.join(path, self.KEYDIR)):
self.signed = True
self.key_update = self.get_tree_mtime(
os.path.join(path, self.KEYDIR), True)
else:
self.signed = False
self.key_update = 0
for name in sorted(os.listdir(path)):
module_path = os.path.join(path, name)
if not os.path.isdir(
module_path
) or '.' in name or name == self.KEYDIR or not os.path.exists(
os.path.join(module_path, '__init__.py')):
continue
try:
fp, pathname, description = imp.find_module(name, [path])
try:
module = LoadedModule(
imp.load_module(name, fp, pathname, description))
finally:
if fp:
fp.close()
except Exception as e:
self.logger.warning('Unable to build module %s: [%s] %s' %
(name, type(e).__name__, e))
bt = get_backtrace(e)
self.logger.debug(bt)
self.errors[name] = bt
else:
m = ModuleInfo(module.name)
m.version = self.get_tree_mtime(module_path)
m.capabilities = list(
set([c.__name__ for c in module.iter_caps()]))
m.description = module.description
m.maintainer = module.maintainer
m.license = module.license
m.icon = module.icon or ''
self.modules[module.name] = m
self.update = int(datetime.now().strftime('%Y%m%d%H%M'))
self.save(filename)
def backend_process(self, function, args, kwargs):
"""
Internal method to run a method of a backend.
As this method may be blocking, it should be run on its own thread.
"""
backend = self.tasks.get()
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s', backend,
function)
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception as error:
self.logger.debug(
'%s: Called function %s raised an error: %r', backend,
function, error)
self.errors.append((backend, error, get_backtrace(error)))
else:
self.logger.debug('%s: Called function %s returned: %r',
backend, function, result)
if hasattr(result, '__iter__') and not isinstance(
result, (bytes, basestring)):
# Loop on iterator
try:
for subresult in result:
self.store_result(backend, subresult)
if self.stop_event.is_set():
break
except Exception as error:
self.errors.append(
(backend, error, get_backtrace(error)))
else:
self.store_result(backend, result)
finally:
self.tasks.task_done()
def build_index(self, path, filename):
"""
Rebuild index of modules of repository.
:param path: path of the repository
:type path: str
:param filename: file to save index
:type filename: str
"""
self.logger.debug('Rebuild index')
self.modules.clear()
self.errors.clear()
if os.path.isdir(os.path.join(path, self.KEYDIR)):
self.signed = True
self.key_update = self.get_tree_mtime(os.path.join(path, self.KEYDIR), True)
else:
self.signed = False
self.key_update = 0
for name in sorted(os.listdir(path)):
module_path = os.path.join(path, name)
if not os.path.isdir(module_path) or '.' in name or name == self.KEYDIR or not os.path.exists(os.path.join(module_path, '__init__.py')):
continue
try:
fp, pathname, description = imp.find_module(name, [path])
try:
module = LoadedModule(imp.load_module(name, fp, pathname, description))
finally:
if fp:
fp.close()
except Exception as e:
self.logger.warning('Unable to build module %s: [%s] %s' % (name, type(e).__name__, e))
bt = get_backtrace(e)
self.logger.debug(bt)
self.errors[name] = bt
else:
m = ModuleInfo(module.name)
m.version = self.get_tree_mtime(module_path)
m.capabilities = list(set([c.__name__ for c in module.iter_caps()]))
m.description = module.description
m.maintainer = module.maintainer
m.license = module.license
m.icon = module.icon or ''
self.modules[module.name] = m
self.update = int(datetime.now().strftime('%Y%m%d%H%M'))
self.save(filename)
def process_incoming_mail(self, msg):
to = self.get_email_address_ident(msg, 'To')
sender = msg.get('From')
reply_to = self.get_email_address_ident(msg, 'In-Reply-To')
title = msg.get('Subject')
if title:
new_title = u''
for part in decode_header(title):
if part[1]:
new_title += unicode(part[0], part[1])
else:
new_title += unicode(part[0])
title = new_title
content = u''
for part in msg.walk():
if part.get_content_type() == 'text/plain':
s = part.get_payload(decode=True)
charsets = part.get_charsets() + msg.get_charsets()
for charset in charsets:
try:
if charset is not None:
content += unicode(s, charset)
else:
content += unicode(s)
except UnicodeError as e:
self.logger.warning('Unicode error: %s' % e)
continue
except Exception as e:
self.logger.exception(e)
continue
else:
break
if len(content) == 0:
print('Unable to send an empty message', file=self.stderr)
return 1
# remove signature
content = content.split(u'\n-- \n')[0]
parent_id = None
if reply_to is None:
# This is a new message
if '.' in to:
backend_name, thread_id = to.split('.', 1)
else:
backend_name = to
thread_id = None
else:
# This is a reply
try:
backend_name, id = reply_to.split('.', 1)
thread_id, parent_id = id.rsplit('.', 1)
except ValueError:
print('In-Reply-To header might be in form <backend.thread_id.message_id>', file=self.stderr)
return 1
# Default use the To header field to know the backend to use.
if to and backend_name != to:
backend_name = to
try:
backend = self.weboob.backend_instances[backend_name]
except KeyError:
print('Backend %s not found' % backend_name, file=self.stderr)
return 1
if not backend.has_caps(CapMessagesPost):
print('The backend %s does not implement CapMessagesPost' % backend_name, file=self.stderr)
return 1
thread = Thread(thread_id)
message = Message(thread,
0,
title=title,
sender=sender,
receivers=[to],
parent=Message(thread, parent_id) if parent_id else None,
content=content)
try:
backend.post_message(message)
except Exception as e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % to_unicode(e)
if logging.root.level <= logging.DEBUG:
content += u'\n%s\n' % to_unicode(get_backtrace(e))
self.send_email(backend.name, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, parent_id) if parent_id else None,
content=content))
thread_id, msg_id = id.rsplit('.', 1)
thread = Thread(thread_id)
message = Message(thread,
0,
title=title,
sender=None,
receivers=None,
parent=Message(thread, msg_id),
content=content)
try:
backend.post_message(message)
except Exception, e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % e
if logging.root.level == logging.DEBUG:
content += u'\n%s\n' % get_backtrace(e)
self.send_email(backend, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, msg_id),
content=content))
def do_run(self, line):
"""
run
Run the fetching daemon.
"""
self.weboob.repeat(int(self.config.get('interval')), self.process)
self.weboob.loop()
def process_incoming_mail(self, msg):
to = self.get_email_address_ident(msg, 'To')
sender = msg.get('From')
reply_to = self.get_email_address_ident(msg, 'In-Reply-To')
title = msg.get('Subject')
if title:
new_title = u''
for part in decode_header(title):
if part[1]:
new_title += unicode(part[0], part[1])
else:
new_title += unicode(part[0])
title = new_title
content = u''
for part in msg.walk():
if part.get_content_type() == 'text/plain':
s = part.get_payload(decode=True)
charsets = part.get_charsets() + msg.get_charsets()
for charset in charsets:
try:
if charset is not None:
content += unicode(s, charset)
else:
content += unicode(s)
except UnicodeError as e:
self.logger.warning('Unicode error: %s' % e)
continue
except Exception as e:
self.logger.exception(e)
continue
else:
break
if len(content) == 0:
print('Unable to send an empty message', file=self.stderr)
return 1
# remove signature
content = content.split(u'\n-- \n')[0]
parent_id = None
if reply_to is None:
# This is a new message
if '.' in to:
backend_name, thread_id = to.split('.', 1)
else:
backend_name = to
thread_id = None
else:
# This is a reply
try:
backend_name, id = reply_to.split('.', 1)
thread_id, parent_id = id.rsplit('.', 1)
except ValueError:
print(
'In-Reply-To header might be in form <backend.thread_id.message_id>',
file=self.stderr)
return 1
# Default use the To header field to know the backend to use.
if to and backend_name != to:
backend_name = to
try:
backend = self.weboob.backend_instances[backend_name]
except KeyError:
print('Backend %s not found' % backend_name, file=self.stderr)
return 1
if not backend.has_caps(CapMessagesPost):
print('The backend %s does not implement CapMessagesPost' %
backend_name,
file=self.stderr)
return 1
thread = Thread(thread_id)
message = Message(
thread,
0,
title=title,
sender=sender,
receivers=[to],
parent=Message(thread, parent_id) if parent_id else None,
content=content)
try:
backend.post_message(message)
except Exception as e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % to_unicode(e)
if logging.root.level <= logging.DEBUG:
content += u'\n%s\n' % to_unicode(get_backtrace(e))
self.send_email(
backend.name,
Message(
thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, parent_id) if parent_id else None,
content=content))
def _store_error(self, backend, error):
with self.mutex:
backtrace = get_backtrace(error)
self.errors.append((backend, error, backtrace))
thread = Thread(thread_id)
message = Message(thread,
0,
title=title,
sender=sender,
receivers=[to],
parent=Message(thread, parent_id) if parent_id else None,
content=content)
try:
backend.post_message(message)
except Exception, e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % to_unicode(e)
if logging.root.level == logging.DEBUG:
content += u'\n%s\n' % to_unicode(get_backtrace(e))
self.send_email(backend, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, parent_id) if parent_id else None,
content=content))
def do_run(self, line):
"""
run
Run the fetching daemon.
"""
self.weboob.repeat(self.config.get('interval'), self.process)
self.weboob.loop()
