def __init__(self, conf=None, initconf=None, update=None):
'''Initialize this object, loading all configuration.
The last item in the list(def_conf+conf) is used for saving this obejct.
/conf/ An optional list of config files, the last of which is used
by save() to write to.
/initconf/ Initial configuration, either a dict or an object with dict().
/update/ Optional update to initconf and conf file(s).
'''
conf = [] if conf is None else list(conf if isinstance(conf, list) else [conf])
self.CONF = conf_update(jso.Storage(self.defaults), initconf)
xf = None
to_include = list()
self.loaded = list()
for xf in self.def_conf+conf:
try:
if isinstance(xf, str):
self.load(xf)
else:
self.update(xf)
more_include = self.config('include')
if more_include:
to_include.append(more_include)
except IOError as ex:
logger.debug('Exception {0} loading config {1}'.format(ex, xf))
for xf in to_include:
self.include(xf)
self.save_to = xf if isinstance(xf, str) else None
self.update(update)
def detect_prompt(self, prompt0=[]):
'''This sets the remote prompt to something more unique than # or $.
This makes it easier for the prompt() method to match the shell prompt
unambiguously. This method is called automatically by the login()
method, but you may want to call it manually if you somehow reset the
shell prompt. For example, if you 'su' to a different user then you
will need to manually reset the prompt. This sends shell commands to
the remote host to set the prompt, so this assumes the remote host is
ready to receive commands.
Alternatively, you may use your own prompt pattern. Just set the PROMPT
attribute to a regular expression that matches it. In this case you
should call login() with auto_prompt_reset=False; then set the PROMPT
attribute. After that the prompt() method will try to match your prompt
pattern.
The idea is copied from 'pxssh' in pexpect.'''
self.flush2(prompt0 if prompt0 else [EOF])
self.sendline()
prompt_wait = self.prompt_wait
if prompt_wait:
time.sleep(prompt_wait)
self.flush2([EOF])
self.sendline()
if prompt_wait:
time.sleep(prompt_wait)
ps1 = self.flush2([EOF])
logger.debug(
'Expecting prompt, flush2(newline) received: [{0}]'.format(ps1))
self.prompt = Prompt(self.newline.split(ps1)[-1])
self.flush()
def __init__(self, prompt):
'''Initialize this pattern, escape any character that regex uses.
'''
for chr in '\\$[]{}().:':
prompt = ('\\' + chr).join(prompt.split(chr))
logger.debug("Prompt set to [%s]" % (prompt))
super(Prompt, self).__init__(prompt)
def sendline(self, cmd=''):
'''Send a line to the server.
'''
if len(cmd) == 0 or cmd[-1] != '\n':
cmd += '\n'
self.channel.send(cmd)
logger.debug("Sent line: [%s]" % (cmd[:-1]))
def sendline(self, cmd=''):
'''Send a line to the server.
'''
if len(cmd) == 0 or cmd[-1] != '\n':
cmd += '\n'
self.channel.send(cmd)
logger.debug("Sent line: [%s]" % (cmd[:-1]))
def __init__(self, prompt):
'''Initialize this pattern, escape any character that regex uses.
'''
for chr in '\\$[]{}().:':
prompt = ('\\'+chr).join(prompt.split(chr))
logger.debug("Prompt set to [%s]" % (prompt))
super(Prompt, self).__init__(prompt)
def __init__(self, conf=None, initconf=None, update=None):
'''Initialize this object, loading all configuration.
The last item in the list(def_conf+conf) is used for saving this obejct.
/conf/ An optional list of config files, the last of which is used
by save() to write to.
/initconf/ Initial configuration, either a dict or an object with dict().
/update/ Optional update to initconf and conf file(s).
'''
conf = [] if conf is None else list(
conf if isinstance(conf, list) else [conf])
self.CONF = conf_update(jso.Storage(self.defaults), initconf)
xf = None
to_include = list()
self.loaded = set()
for xf in self.def_conf + conf:
try:
if isinstance(xf, str):
logger.debug('...loading config {0}'.format(xf))
self.load(xf)
else:
self.update(xf)
more_include = self.config('include')
if more_include:
to_include.append(more_include)
except IOError as ex:
logger.debug('Exception {0} loading config {1}'.format(ex, xf))
for xf in to_include:
self.include(xf)
self.save_to = xf if isinstance(xf, str) else None
self.update(update)
def __init__(self, xdev=None, **kwargs):
'''
Initialize the SNMP API, with session (self.S).
@param xdev May be either another SNMPAPI object, or a host (name or IP).
@param kwargs Arguments for netsnmp.Session.
In case 'xdev' is another SNMP object, its session information is copied.
Otherwise, i.e., 'xdev' is a host,
'''
logger.debug("SNMPAPI.__init__: xdev = %s" % (xdev))
if isinstance(xdev, SNMPAPI):
self.engine = xdev.engine
self.host = xdev.host
else:
SNMPAPI.__add_MIB_path(kwargs.get('mibdirs'))
SNMPAPI.__add_MIB(kwargs.get('mibs'))
if not SNMPAPI.config:
self.__init(**kwargs)
# HACK to accomondate SNMPAPI.config being c9r.jsonpy.Thingy:
config = SNMPAPI.config
config = dict(
config if isinstance(config, dict) else config.dict())
self.host = xdev
logger.debug('New session: SNMPAPI.config = {0}'.format(config))
self.engine = SnmpEngine()
self.mib = {}
def text(self):
'''Return text parsed from HTML document.'''
lineno = 0
for txt in self.segments:
lineno += 1
logger.debug("%d: %s"%(lineno, txt))
return ''.join([str(sx.encode('utf-8', 'xmlcharrefreplace'), 'utf-8') for sx in self.segments])
def run(self):
for arg in self.args or ['-']:
logger.debug('-- Processing {0}:'.format(arg))
jso.save_storage(
jso.load_storage(sys.stdin if arg == '-' else arg), sys.stdout,
self.indent)
pass
def get_credentials(self, host, **kwargs):
'''Get login credentials for the given host.
General practice in networking is that authentication is ceneralized, allowing
role (group) and subnet based access control. This function is intended to be
used as an interface to that mechanism.
Login credentials may be configured in the app's config file.
TO-DO: Implement subnet-based credential sets.
'''
conf = self.config()
cat = conf.subnets.get(host, self.default).split('.')
logger.debug('{0} is device of the "{1}" category'.format(host, cat))
if len(cat) > 1:
profile = conf.get('profiles', {}).get(cat[0])
cat = cat[1]
params = conf.logins.get(cat) ## Get host subnet specific credentials
params.update(kwargs)
try:
pw = params['password']
params['password'] = base64.b64decode(pw)
except:
# If decoding fails, assume it is already decoded.
pass
return profile, params
def get_credentials(self, host, **kwargs):
'''Get login credentials for the given host.
General practice in networking is that authentication is ceneralized, allowing
role (group) and subnet based access control. This function is intended to be
used as an interface to that mechanism.
Login credentials may be configured in the app's config file.
TO-DO: Implement subnet-based credential sets.
'''
conf = self.config()
cat = conf.subnets.get(host, self.default).split('.')
logger.debug('{0} is device of the "{1}" category'.format(host, cat))
if len(cat) > 1:
profile = conf.get('profiles', {}).get(cat[0])
cat = cat[1]
params = conf.logins.get(cat) ## Get host subnet specific credentials
params.update(kwargs)
try:
pw = params['password']
params['password'] = base64.b64decode(pw)
except:
# If decoding fails, assume it is already decoded.
pass
return profile, params
def __init__(self, host=None, **kwargs):
'''Initialization.
/host/ Optional hostname. Automatically connects if given.
/kwargs/ Optional, parameters for connect().
'''
super(SecureShell, self).__init__()
if not hasattr(SecureShell, 'CFG'):
SecureShell.CFG = Config(conf='~/.ssh/cli-conf.json',
initconf={
'known_hosts': '~/.ssh/known_hosts',
'missing_key': 'add',
'subnets': {}
})
self.cfg = Config(initconf=SecureShell.CFG, update=kwargs)
self.timeout = self.config('timeout', 10)
self.prompt_wait = self.config('prompt_wait') or 0.01
self.default = self.config('default', 'default.default')
logger.debug('Default profile.category = {0}.'.format(self.default))
self.newline = re.compile(self.config('newline', "[\r\n]+"))
sshc = pyssh.SSHClient()
known_hosts = self.config('known_hosts')
if known_hosts != None:
sshc.load_system_host_keys()
sshc.load_host_keys(os.path.expanduser(known_hosts))
sshc.set_missing_host_key_policy({
'accept': AcceptKeyPolicy,
'add': pyssh.client.AutoAddPolicy,
'warn': pyssh.client.WarningPolicy,
}.get(self.config('missing_key'), pyssh.client.RejectPolicy)())
self.sshc = sshc
self.channel = None
self.flush()
if host is not None:
self.connect(host, **kwargs)
def detect_prompt(self, prompt0=[]):
'''This sets the remote prompt to something more unique than # or $.
This makes it easier for the prompt() method to match the shell prompt
unambiguously. This method is called automatically by the login()
method, but you may want to call it manually if you somehow reset the
shell prompt. For example, if you 'su' to a different user then you
will need to manually reset the prompt. This sends shell commands to
the remote host to set the prompt, so this assumes the remote host is
ready to receive commands.
Alternatively, you may use your own prompt pattern. Just set the PROMPT
attribute to a regular expression that matches it. In this case you
should call login() with auto_prompt_reset=False; then set the PROMPT
attribute. After that the prompt() method will try to match your prompt
pattern.
The idea is copied from 'pxssh' in pexpect.'''
self.flush2(prompt0 if prompt0 else [EOF])
self.sendline()
prompt_wait = self.prompt_wait
if prompt_wait:
time.sleep(prompt_wait)
self.flush2([EOF])
self.sendline()
if prompt_wait:
time.sleep(prompt_wait)
ps1 = self.flush2([EOF])
logger.debug('Expecting prompt, flush2(newline) received: [{0}]'.format(ps1))
self.prompt = Prompt(self.newline.split(ps1)[-1])
self.flush()
def send(self, msg, msguid):
"""
Resend a given message that has been edited as configured.
"""
# Build TO:, CC: and BCC: lists into one list:
addr = {'to': set(), 'cc': set(), 'bcc': set()}
header = []
for xk, xv in msg.items():
xklo = xk.lower()
if xklo in addr:
logger.debug("mailmon.send: key = %s, value = %s" % (xk, xv))
addr[xklo] |= set(re.split('[\s,:|]+', xv))
elif xk != 'body': # -- excluding anything not belong in the header
header.append(xk + ': ' + xv)
# Get rid of any duplicates in the "to", "cc" and "bcc" list:
if addr['to']:
header.append('to: ' + ','.join(addr['to']))
addr['cc'] -= addr['to']
addr['bcc'] -= addr['to']
if addr['cc']:
header.append('cc: ' + ','.join(addr['cc']))
addr['bcc'] -= addr['cc']
addr['to'] |= addr['cc']
if addr['bcc']:
header.append('bcc: ' + ','.join(addr['bcc']))
addr['to'] |= addr['bcc']
nl = self.newline
self.smtp.sendmail(msg['from'], addr['to'],
nl.join(header) + nl + nl + msg['body'])
def detect_prompt(self, prompt0=[]):
'''This sets the remote prompt to something more unique than # or $.
This makes it easier for the prompt() method to match the shell prompt
unambiguously. This method is called automatically by the login()
method, but you may want to call it manually if you somehow reset the
shell prompt. For example, if you 'su' to a different user then you
will need to manually reset the prompt. This sends shell commands to
the remote host to set the prompt, so this assumes the remote host is
ready to receive commands.
Alternatively, you may use your own prompt pattern. Just set the PROMPT
attribute to a regular expression that matches it. In this case you
should call login() with auto_prompt_reset=False; then set the PROMPT
attribute. After that the prompt() method will try to match your prompt
pattern.
The idea is copied from 'pxssh' in pexpect.'''
if prompt0:
self.flush2(prompt0)
self.sendline()
self.flush2([self.newline])
self.sendline()
self.flush2()
prompt = self.newline.split(self.before)
self.prompt = prompt[-1].strip()
logger.debug("Prompt set to [%s]" % (self.prompt))
def detect_prompt(self, prompt0=[]):
'''This sets the remote prompt to something more unique than # or $.
This makes it easier for the prompt() method to match the shell prompt
unambiguously. This method is called automatically by the login()
method, but you may want to call it manually if you somehow reset the
shell prompt. For example, if you 'su' to a different user then you
will need to manually reset the prompt. This sends shell commands to
the remote host to set the prompt, so this assumes the remote host is
ready to receive commands.
Alternatively, you may use your own prompt pattern. Just set the PROMPT
attribute to a regular expression that matches it. In this case you
should call login() with auto_prompt_reset=False; then set the PROMPT
attribute. After that the prompt() method will try to match your prompt
pattern.
The idea is copied from 'pxssh' in pexpect.'''
if prompt0:
self.flush2(prompt0)
self.sendline()
self.flush2([self.newline])
self.sendline()
self.flush2()
prompt = self.newline.split(self.before)
self.prompt = prompt[-1].strip()
logger.debug("Prompt set to [%s]" % (self.prompt))
def __call__(self):
logger.debug('Default gateway = {0}'.format(def_gw))
cmd = cdp.details()
ssh = issh.SecureShell(def_gw)
_,result = ssh.run(cmd.command)[0]
# Parse the result next
result = cmd.parse(result)
print(result)
def __call__(self, content):
'''Parse an email message in "content", which is a string or a text input object.
/content/ Standard encoded email message content.
Returns parsed message in a dict of (subject, date, body, html, from, to, attachments).
'''
msgobj = self.parse(StringIO(content) if isinstance(content, str) else content)
subject = parse_header('Subject', msgobj)
date = parse_header('Date', msgobj)
received = []
for part in msgobj.get_all('Received'):
lx = self.re_received.split(part)
tmp = dict(zip(lx[1::2], [ x.strip() for x in lx[2::2] ]))
tx = tmp.get(';')
if tx: tmp['time'] = parse_time(tx)
received.append(tmp)
fromaddr = parse_addr(msgobj, 'From')
if date:
date = date.replace(',', '')
logger.debug('Parsing message: Date={0}, Subject={1}'.format(date, subject))
#-------- Parsing attachments:
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
else: # parse text content
content_type = part.get_content_type()
if content_type[0:5] == 'text/':
payload = unicode(part.get_payload(decode=True),
part.get_content_charset() or 'ascii',
'replace').encode('utf8','replace')
if content_type == "text/plain":
if body is None:
body = ''
body += payload
elif content_type == "text/html":
if html is None:
html = ''
html += payload
else:
logger.debug('Ignored: Content_type "{0}" in message "{1}" from {2}, Date={3}'.format(content_type, subject, fromaddr, date))
return {
'subject' : subject,
'date' : date,
'received': received,
# 'received': sorted(received, key=lambda k: k['time']),
'body' : body,
'html' : html,
'from' : fromaddr,
'to' : parse_addr(msgobj, 'To'),
'cc' : parse_addr(msgobj, 'CC'),
'bcc' : parse_addr(msgobj, 'BCC'),
'attachments': attachments
}
def open(self):
'''To start a filter thread/greenlet.
'''
if not self.is_open:
self.is_open = True
if self.que is None:
self.que = Queue()
logger.debug('{0}: open - Queue created'.format(type(self).__name__))
return self
def open(self):
'''To start a filter thread/greenlet.
'''
if not self.is_open:
self.is_open = True
if self.que is None:
self.que = Queue()
logger.debug('{0}: open - Queue created'.format(
type(self).__name__))
return self
def load(self, filename):
'''Load a given file, if not loaded yet.
/filename/ Path to file to be loaded.
'''
fn = os.path.expanduser(filename)
if fn in self.loaded:
logger.debug('Config file "{0}" already loaded.'.format(xf))
else:
self.update(jso.load_storage(fn))
def text(self):
'''Return text parsed from HTML document.'''
lineno = 0
for txt in self.segments:
lineno += 1
logger.debug("%d: %s" % (lineno, txt))
return ''.join([
str(sx.encode('utf-8', 'xmlcharrefreplace'), 'utf-8')
for sx in self.segments
])
def load(self, filename):
'''Load a given file, if not loaded yet.
/filename/ Path to file to be loaded.
'''
fn = os.path.expanduser(filename)
if fn in self.loaded:
logger.debug('Config file "{0}" already loaded.'.format(xf))
else:
self.update(jso.load_storage(fn))
def register_dialects(dialects):
'''Register a list of CSV dialects (dict of dicts).
'''
if dialects is None:
return
for name, params in dialects.items():
quoting = params.get('quoting')
if quoting and not isinstance(quoting, int):
params['quoting'] = getattr(csv, quoting)
csv.register_dialect(name, **params)
logger.debug('CSV dialect "{0}" registered'.format(name))
def register_dialects(dialects):
'''Register a list of CSV dialects (dict of dicts).
'''
if dialects is None:
return
for name, params in dialects.iteritems():
quoting = params.get('quoting')
if quoting and not isinstance(quoting, int):
params['quoting'] = getattr(csv, quoting)
csv.register_dialect(name, **params)
logger.debug('CSV dialect "{0}" registered'.format(name))
def main():
'''
'''
app = Main()
if app.dryrun and os.access('test/logfiler.txt', os.R_OK):
logger.debug('Configuration tested OK. Running more module tests.')
import doctest
doctest.testfile('test/logfiler.txt')
else:
app()
exit(0)
def search(self, xcriteria):
'''
Search for messages with given criteria.
@param xsearch Search criteria: either a string, or a list of search elements.
@return xres Result of the search: OK, NO, or BAD.
xuid A space separated list of UIDs if the search is OK.
'''
xsearch = xcriteria if isinstance(xcriteria, str) else '('+' '.join(xcriteria)+')'
logger.debug("imap4.search: %s" % (xsearch))
xres, xuid = self.uid('search', None, xsearch)
return xres, xuid
def close(self):
'''To exit this filter thread.
Try to close the next filter in chain after the thread is done.
'''
klass = type(self).__name__
qu = self.que
logger.debug('{0}: closing, que size = {1}, output count = {2}'.format(klass, None if qu is None else qu.qsize(), self.count))
self.__call__()
self.try_next('close')
logger.debug('{0}: closed, que size = {1}, output count = {2}'.format(klass, None if qu is None else qu.qsize(), self.count))
def path(self, name):
'''Create a full path to file with given name.
'''
cname = os.path.normpath(name)
cname = name if name[0] == os.sep else os.path.join(self.top, name)
head, tail = os.path.split(cname)
logger.debug('Path "{0}" split to {1} : {2}'.format(cname, head, tail))
if tail != name and not os.path.isdir(head):
logger.debug('Creating folder: {0}'.format(head))
fu.forge_path(head)
return cname
def __init__(self, dcon):
'''Initialize an IMAP4 client with given configuration "dcon".
Also, if an password is not provided in the given config, attempt to
load /app/etc/logins.json as a
'''
imaplib.IMAP4_SSL.__init__(self, dcon.imap.server, dcon.imap.port)
uid = dcon.user
logger.debug('IMAP = ({0}), user = {1}'.format(dcon.imap, uid))
self.login(uid, PIM().get(uid, dcon.password))
self.select()
def archive(self, auid, abox=''):
"""
Archive a given message to the configured archive mail box. The mail box is created if it does not exist.
"""
if not abox:
abox = self.CONF.get('archive', '')
if not abox:
return None
xcopy = self.mbox.archive(auid, abox)
logger.debug("Archive: copy('%s') == %s" % (auid, xcopy))
return xcopy
def connect(self, host, **kwargs):
profile, params = self.get_credentials(host, **kwargs)
self.sshc.connect(host, **params)
logger.debug('Connected to {0} as user "{1}".'.format(host, params.get('username', '<N/A>')))
self.channel = self.sshc.invoke_shell()
# self.stderr = self.channel.makefile_stderr('rb')
self.detect_prompt()
if profile:
init = profile.get('init')
if init:
self.run(init)
self.flush()
def __init(self, **kwargs):
'''
Run-once initialization to configure this object.
'''
xc = kwargs.get('session')
if not xc:
raise Exception(t('SNMPAPI.config not found!'))
domains = xc.get('domain')
if isinstance(domains, str):
xc.domain = domains.split("\r\n")
logger.debug("SNMPAPI.__init: config({0}) = {1}".format(type(xc), xc))
SNMPAPI.config = xc
def get_object(self, xoid, prefix=''):
"""
Get a single value of a given object on this device.
@param xoid ID of the object to get value for.
@param prefix Optional prefix to remove from the value.
@return Value of retrieved MIB object.
"""
xo = self.get(xoid)
logger.debug("SNMPAPI.get_object: oid = {0}, val = {1}".format(
xoid, xo))
return Prefix(prefix)(xo[0]) if xo and xo[0] else None
def assign(self, value):
'''Assigning a given value to self.
'''
try:
clear = b64decode(value)
except TypeError:
clear = value
value = b64encode(clear)
if not value is self.value:
logger.debug('Assigning value: {0}'.format(value))
self.value = value
return clear.decode("utf-8")
def __init__(self, next_filter, header, write_header=True, dialect=None):
'''The CSV writer that writes data to CSV format with given
header.
A csv.DictWriter is used to write out each row received in this writer.
Extra fields in each row is ignored.
'''
Filter.__init__(self, next_filter)
self.csvo = csv.DictWriter(self.Shim(self), header, extrasaction='ignore',
dialect=(dialect or 'excel'))
self.header = header if write_header else None # Header to write
logger.debug('write_header={0}, header={1}'.format(write_header, header))
def close(self):
'''To exit this filter thread.
Try to close the next filter in chain after the thread is done.
'''
klass = type(self).__name__
qu = self.que
logger.debug('{0}: closing, que size = {1}, output count = {2}'.format(
klass, None if qu is None else qu.qsize(), self.count))
self.__call__()
self.try_next('close')
logger.debug('{0}: closed, que size = {1}, output count = {2}'.format(
klass, None if qu is None else qu.qsize(), self.count))
def next(self):
'''Return the last line if the input is backed up.
Otherwise, read the next line from file.
'''
if self.read:
self.line = self.input.next()
else:
self.read = True
line = self.line
if self.ends and self.ends.match(line):
logger.debug('Ends proessing at: {0})'.format(line))
raise StopIteration
return line
def get(self, id, cpw=None):
'''Get the credential for "id".
'''
if cpw:
pw = cpw
else:
ux = self[id]
pw = ux if isinstance(ux,
str) else ux.get('password') if ux else None
if pw is None:
raise Exception("Password for id '{0}' not found.".format(id))
logger.debug('Credential resolved for user ({0})'.format(id))
return TextPassword(pw).cleartext()
def connect(self, host, **kwargs):
profile, params = self.get_credentials(host, **kwargs)
self.sshc.connect(host, **params)
logger.debug('Connected to {0} as user "{1}".'.format(
host, params.get('username', '<N/A>')))
self.channel = self.sshc.invoke_shell()
# self.stderr = self.channel.makefile_stderr('rb')
self.detect_prompt()
if profile:
init = profile.get('init')
if init:
self.run(init)
self.flush()
def __init__(self, host='', conf=Null()):
options = conf.get('options', {})
if isinstance(options, Null):
options = options.__dict__
## {timeout, maxread, searchwindowsize, logfile, cwd, env}
logger.debug("CSSH: spawning with option = %s" % (format(options)))
spawn.__init__(self, None, **options)
self.C = conf
prompt0 = conf.get('prompt', '[#$>]')
self.prompt0 = re.compile(prompt0)
self.newline = re.compile("[\r\n]+")
if host:
self.login(host)
logger.debug("CSSH: prompt0 = '%s'" % (format(prompt0)))
def __init__(self, host='', conf=Null()):
options = conf.get('options', {})
if isinstance(options, Null):
options = options.__dict__
## {timeout, maxread, searchwindowsize, logfile, cwd, env}
logger.debug("CSSH: spawning with option = %s" % (format(options)))
spawn.__init__(self, None, **options)
self.C = conf
prompt0 = conf.get('prompt', '[#$>]')
self.prompt0 = re.compile(prompt0)
self.newline = re.compile("[\r\n]+")
if host:
self.login(host)
logger.debug("CSSH: prompt0 = '%s'" % (format(prompt0)))
def __init__(self, conf, default_days=60):
'''Configure this object.
.days Only delete mails older than given number of days. Defaults to 60.
.limit Only delete up to given number of messages. Defaults to 1000.
.stop True if not continuing to process the messages found by this task.
'''
MailMonitor.__init__(self, conf)
self.limit = conf.get('limit', 1000)
self.stop = conf.get('stop', True)
self.fetching = conf.get('fetch') or False
if not self.archive(): self.CONF['archive'] = False
logger.debug("MailMonitor.{0}: days = {1}, limit = {2}".format(
type(self).__name__, self.days, self.limit))
def walk_object(self, xoid, context=''):
'''
Walk through SNMP objects and fetch their values
@param xoid Starting point of the walk -- an object ID.
@param context Optional context.
@return An array of values.
'''
xvar = self.walk(xoid, context)
xobj = {}
for v in xvar:
xobj[v.iid] = v.val
logger.debug("SNMPAPI.walk_object: oid = %s, val = %s" % (xoid, xobj))
return xobj
def archive(self, auid, abox):
"""
Archive a given message to the configured archive mail box.
The mail box is created if it does not exist.
"""
xcopy = self.uid('COPY', auid, abox)
if xcopy[0] == 'OK':
mov, data = self.delete(auid)
elif xcopy[0] == 'NO' and xcopy[1][0][:11] == '[TRYCREATE]':
xdir = self.create(abox)
logger.debug("Created mailbox: {0} = {1}".format(abox, xdir))
if xdir[0] == 'OK':
xcopy = self.archive(auid, abox)
return xcopy
def __next__(self):
'''Return the last line if the input is backed up.
Otherwise, read the next line from file.
'''
if self.read:
self.line = next(self.input)
else:
self.read = True
line = self.line
line = line.decode('utf-8') if isinstance(line, bytes) else line
if self.ends and self.ends.match(line):
logger.debug('Ends proessing at: {0})'.format(line))
raise StopIteration
return line
def __init__(self):
'''
A command line application object to monitor email.
The class must either have no doc, or have something in the format of c9r.app.Command.
'''
Command.__init__(self)
xc = self.CONF
smtp = self.CONF.smtp
try:
self.smtp = smtplib.SMTP(smtp.server, smtp.port)
except Exception as xerr:
raise Exception(
'Failed to connect SMTP server for sending email: %s' % (xerr))
self.task = []
plugin = xc.get('plugin', 'MailMonitor')
mmmod = __import__(plugin + '.' + plugin)
base_klass = getattr(getattr(mmmod, plugin, None), plugin, None)
logger.debug("mailmon: Plugin base '%s' loaded as %s from %s." %
(plugin, format(base_klass), format(mmmod)))
for xtask in xc.get('tasklist', []):
conf = xc.get(xtask)
action = conf.action
if not action:
logger.error("mailmon: Task '%s' with no action is ignored." %
(xtask))
continue
klass = getattr(globals(), action,
None) # "action" class may be in globally defined
if not klass and mmmod:
mod = __import__(
plugin + '.' +
action) # "action" class may also be in a plugin module
if isinstance(mod, type(mmmod)):
klass = getattr(getattr(mod, action, None), action, None)
logger.debug("mailmon: Action '%s' imported as %s." %
(action, str(klass)))
if callable(klass) and issubclass(klass, base_klass):
conf.util = self
self.task.append(klass(conf))
else:
logger.error("mailmon: Action module '%s' is not callable." %
(format(klass)))
logger.error(
"mailmon: Action module '%s' not found, task '%s' is ignored."
% (action, xtask))
self.parse = parser.Parser()
self.cleanup_set = set()
atexit.register(self.cleanup)
def include(self, path, ext='.json'):
'''To include a file or files in a folder that ends with given extension.
/path/ Path to file or directory to include.
/ext/ Extension to limit files to include.
'''
logger.debug("To include: {0}".format(path))
lx = len(ext)
for fn in (os.listdir(path) if os.path.isdir(path) else [path]):
if not fn is path:
fn = os.path.join(path, fn)
fn = os.path.expanduser(fn)
if fn[-lx:] == ext and os.path.isfile(fn):
logger.debug("Including file: {0}".format(fn))
self.load(fn)
def __init__(self, next_filter, header, write_header=True, dialect=None):
'''The CSV writer that writes data to CSV format with given
header.
A csv.DictWriter is used to write out each row received in this writer.
Extra fields in each row is ignored.
'''
Filter.__init__(self, next_filter)
self.csvo = csv.DictWriter(self.Shim(self),
header,
extrasaction='ignore',
dialect=(dialect or 'excel'))
self.header = header if write_header else None # Header to write
logger.debug('write_header={0}, header={1}'.format(
write_header, header))
def include(self, path, ext='.json'):
'''To include a file or files in a folder that ends with given extension.
/path/ Path to file or directory to include.
/ext/ Extension to limit files to include.
'''
logger.debug("To include: {0}".format(path))
lx = len(ext)
for fn in (os.listdir(path) if os.path.isdir(path) else [path]):
if not fn is path:
fn = os.path.join(path, fn)
fn = os.path.expanduser(fn)
if fn[-lx:] == ext and os.path.isfile(fn):
logger.debug("Including file: {0}".format(fn))
self.load(fn)
def __call__(self):
'''Go through list of files to monitor and fix them.
Each configured task is started as "concurrently" in a greenlet.
'''
os.chdir(self.config('path', '.'))
tasks = self.config('tasks', {})
for pat,cfg in tasks.iteritems():
jobqu.put((cfg, pat))
tasks = min(self.config('threads', 10), jobqu.qsize())
cwd = os.getcwd()
logger.debug('Spawning {0} task threads, CWD = {1}.'.format(tasks, cwd))
tasks = [ gevent.spawn(task, cwd) for x in range(0, tasks) ]
logger.debug('Waiting for {0} task threads to complete.'.format(len(tasks)))
#jobqu.join()
gevent.joinall(tasks)
def __call__(self):
'''Go through list of files to monitor and fix them.
Each configured task is started as "concurrently" in a greenlet.
'''
os.chdir(self.config('path', '.'))
tasks = self.config('tasks', {})
for pat,cfg in tasks.items():
jobqu.put((cfg, pat))
tasks = min(self.config('threads', 10), jobqu.qsize())
cwd = os.getcwd()
logger.debug('Spawning {0} task threads, CWD = {1}.'.format(tasks, cwd))
tasks = [ gevent.spawn(task, cwd) for x in range(0, tasks) ]
logger.debug('Waiting for {0} task threads to complete.'.format(len(tasks)))
#jobqu.join()
gevent.joinall(tasks)
def write(self, data):
'''Run given /data/ through the csv.DictWriter to convert from
a dict to a CSV row with its writerow() function.
'''
try:
if self.header != None:
# TBD: To use csv.DictWriter.writeheader()
# SuSE has Python 2.6.x, where the function does not exist.
# self.csvo.writeheader()
self.next_filter.write(','.join(self.header) + '\r\n')
self.header = None
logger.debug('Wrote header to {0}'.format(self.next_filter))
self.csvo.writerow(data)
self.flush()
except Exception as ex:
logger.debug('Got Exception {1}, data={0}'.format(data, ex))
raise
def write(self, data):
'''Run given /data/ through the csv.DictWriter to convert from
a dict to a CSV row with its writerow() function.
'''
try:
if self.header != None:
# TBD: To use csv.DictWriter.writeheader()
# SuSE has Python 2.6.x, where the function does not exist.
# self.csvo.writeheader()
self.next_filter.write(','.join(self.header)+'\r\n')
self.header = None
logger.debug('Wrote header to {0}'.format(self.next_filter))
self.csvo.writerow(data)
self.flush()
except Exception as ex:
logger.debug('Got Exception {1}, data={0}'.format(data, ex))
raise
def atexit_process(filename, act):
'''Post process a file with given /act/.
'''
if act == 'delete':
return atexit_delete(filename)
if act in set([
'bzip2', # Compress the file using bzip2;
'gzip', # Compress using gzip;
'xz', # Compress using xz;
'zip' # Compress using zip.
]):
cmd = (act if isinstance(act, list) else [act])+[filename]
try:
subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
logger.error('Error from command {0}\n{1}'.format(cmd, err.output))
return
logger.debug('Unknown postprocess action: "{0}" "{1}"'.format(act, filename))
def __call__(self):
'''Run the job as configured and use Popen.communicate() to get outputs,
assuming that the data is not super large.
If the return code from the rsync command is not zero (0), then any error
Returns results for the jobs, each in a dict of dict[status, out, err]
variable, keyed by job names.
'''
rsync = Rsync(self.config())
results = {}
for job in self.config('jobs', []):
run = rsync(self.config(job))
fout,ferr = run.communicate()
rc = run.wait()
logger.debug('Job "{0}":\nReturn code = {1}\n\nOutput: {2}\n\nError: {3}'.format(job, rc, fout, ferr))
results[job] = dict(status=rc, out=fout.split('\n'), err=ferr)
return results
def main():
'''Use SecureShell as command runner.
'''
cmd = ISSH()
if cmd.dryrun: # Run unit test
import doctest
doctest.testfile('test/issh.txt')
return
if not cmd.args:
return
host = cmd.args[0]
logger.debug('Connecting to {0}'.format(host))
ssh = SecureShell()
results = ssh.run(cmd.args[1:], host=host)
logger.debug('Got {0} result(s)'.format(len(results)))
for cmd,output in results.items():
print(cmd)
for line in StringIO(output):
print('\t'+line.rstrip())
def __call__(self, command, echo=None, prompt='', timeout=10):
'''
Send the specified (command) to the remote host and wait for (prompt).
Return what comes before (prompt) as command output.
'''
logger.debug("CSSH(): sending command [%s]" % (command))
self.sendline(command)
if echo is None:
echo = command.strip()
if echo:
logger.debug("CSSH(): waiting for command echo [%s]" % (echo))
xi = self.expect([echo, TIMEOUT, EOF], timeout=timeout)
if prompt is not None:
if not prompt:
prompt = self.prompt
logger.debug("CSSH(): waiting for prompt [%s]" % (prompt))
xi = self.expect([prompt, TIMEOUT, EOF], timeout=timeout)
logger.debug("CSSH.expect returned %d" % (xi))
output = self.before[:-2]
logger.debug("Remote output = [%s]" % (output))
return output
