def __init__(self, conf=None, initconf=None, update=None): '''Initialize this object, loading all configuration. The last item in the list(def_conf+conf) is used for saving this obejct. /conf/ An optional list of config files, the last of which is used by save() to write to. /initconf/ Initial configuration, either a dict or an object with dict(). /update/ Optional update to initconf and conf file(s). ''' conf = [] if conf is None else list(conf if isinstance(conf, list) else [conf]) self.CONF = conf_update(jso.Storage(self.defaults), initconf) xf = None to_include = list() self.loaded = list() for xf in self.def_conf+conf: try: if isinstance(xf, str): self.load(xf) else: self.update(xf) more_include = self.config('include') if more_include: to_include.append(more_include) except IOError as ex: logger.debug('Exception {0} loading config {1}'.format(ex, xf)) for xf in to_include: self.include(xf) self.save_to = xf if isinstance(xf, str) else None self.update(update)
def detect_prompt(self, prompt0=[]): '''This sets the remote prompt to something more unique than # or $. This makes it easier for the prompt() method to match the shell prompt unambiguously. This method is called automatically by the login() method, but you may want to call it manually if you somehow reset the shell prompt. For example, if you 'su' to a different user then you will need to manually reset the prompt. This sends shell commands to the remote host to set the prompt, so this assumes the remote host is ready to receive commands. Alternatively, you may use your own prompt pattern. Just set the PROMPT attribute to a regular expression that matches it. In this case you should call login() with auto_prompt_reset=False; then set the PROMPT attribute. After that the prompt() method will try to match your prompt pattern. The idea is copied from 'pxssh' in pexpect.''' self.flush2(prompt0 if prompt0 else [EOF]) self.sendline() prompt_wait = self.prompt_wait if prompt_wait: time.sleep(prompt_wait) self.flush2([EOF]) self.sendline() if prompt_wait: time.sleep(prompt_wait) ps1 = self.flush2([EOF]) logger.debug( 'Expecting prompt, flush2(newline) received: [{0}]'.format(ps1)) self.prompt = Prompt(self.newline.split(ps1)[-1]) self.flush()
def __init__(self, prompt): '''Initialize this pattern, escape any character that regex uses. ''' for chr in '\\$[]{}().:': prompt = ('\\' + chr).join(prompt.split(chr)) logger.debug("Prompt set to [%s]" % (prompt)) super(Prompt, self).__init__(prompt)
def sendline(self, cmd=''): '''Send a line to the server. ''' if len(cmd) == 0 or cmd[-1] != '\n': cmd += '\n' self.channel.send(cmd) logger.debug("Sent line: [%s]" % (cmd[:-1]))
def __init__(self, prompt): '''Initialize this pattern, escape any character that regex uses. ''' for chr in '\\$[]{}().:': prompt = ('\\'+chr).join(prompt.split(chr)) logger.debug("Prompt set to [%s]" % (prompt)) super(Prompt, self).__init__(prompt)
def __init__(self, conf=None, initconf=None, update=None): '''Initialize this object, loading all configuration. The last item in the list(def_conf+conf) is used for saving this obejct. /conf/ An optional list of config files, the last of which is used by save() to write to. /initconf/ Initial configuration, either a dict or an object with dict(). /update/ Optional update to initconf and conf file(s). ''' conf = [] if conf is None else list( conf if isinstance(conf, list) else [conf]) self.CONF = conf_update(jso.Storage(self.defaults), initconf) xf = None to_include = list() self.loaded = set() for xf in self.def_conf + conf: try: if isinstance(xf, str): logger.debug('...loading config {0}'.format(xf)) self.load(xf) else: self.update(xf) more_include = self.config('include') if more_include: to_include.append(more_include) except IOError as ex: logger.debug('Exception {0} loading config {1}'.format(ex, xf)) for xf in to_include: self.include(xf) self.save_to = xf if isinstance(xf, str) else None self.update(update)
def __init__(self, xdev=None, **kwargs): ''' Initialize the SNMP API, with session (self.S). @param xdev May be either another SNMPAPI object, or a host (name or IP). @param kwargs Arguments for netsnmp.Session. In case 'xdev' is another SNMP object, its session information is copied. Otherwise, i.e., 'xdev' is a host, ''' logger.debug("SNMPAPI.__init__: xdev = %s" % (xdev)) if isinstance(xdev, SNMPAPI): self.engine = xdev.engine self.host = xdev.host else: SNMPAPI.__add_MIB_path(kwargs.get('mibdirs')) SNMPAPI.__add_MIB(kwargs.get('mibs')) if not SNMPAPI.config: self.__init(**kwargs) # HACK to accomondate SNMPAPI.config being c9r.jsonpy.Thingy: config = SNMPAPI.config config = dict( config if isinstance(config, dict) else config.dict()) self.host = xdev logger.debug('New session: SNMPAPI.config = {0}'.format(config)) self.engine = SnmpEngine() self.mib = {}
def text(self): '''Return text parsed from HTML document.''' lineno = 0 for txt in self.segments: lineno += 1 logger.debug("%d: %s"%(lineno, txt)) return ''.join([str(sx.encode('utf-8', 'xmlcharrefreplace'), 'utf-8') for sx in self.segments])
def run(self): for arg in self.args or ['-']: logger.debug('-- Processing {0}:'.format(arg)) jso.save_storage( jso.load_storage(sys.stdin if arg == '-' else arg), sys.stdout, self.indent) pass
def get_credentials(self, host, **kwargs): '''Get login credentials for the given host. General practice in networking is that authentication is ceneralized, allowing role (group) and subnet based access control. This function is intended to be used as an interface to that mechanism. Login credentials may be configured in the app's config file. TO-DO: Implement subnet-based credential sets. ''' conf = self.config() cat = conf.subnets.get(host, self.default).split('.') logger.debug('{0} is device of the "{1}" category'.format(host, cat)) if len(cat) > 1: profile = conf.get('profiles', {}).get(cat[0]) cat = cat[1] params = conf.logins.get(cat) ## Get host subnet specific credentials params.update(kwargs) try: pw = params['password'] params['password'] = base64.b64decode(pw) except: # If decoding fails, assume it is already decoded. pass return profile, params
def __init__(self, host=None, **kwargs): '''Initialization. /host/ Optional hostname. Automatically connects if given. /kwargs/ Optional, parameters for connect(). ''' super(SecureShell, self).__init__() if not hasattr(SecureShell, 'CFG'): SecureShell.CFG = Config(conf='~/.ssh/cli-conf.json', initconf={ 'known_hosts': '~/.ssh/known_hosts', 'missing_key': 'add', 'subnets': {} }) self.cfg = Config(initconf=SecureShell.CFG, update=kwargs) self.timeout = self.config('timeout', 10) self.prompt_wait = self.config('prompt_wait') or 0.01 self.default = self.config('default', 'default.default') logger.debug('Default profile.category = {0}.'.format(self.default)) self.newline = re.compile(self.config('newline', "[\r\n]+")) sshc = pyssh.SSHClient() known_hosts = self.config('known_hosts') if known_hosts != None: sshc.load_system_host_keys() sshc.load_host_keys(os.path.expanduser(known_hosts)) sshc.set_missing_host_key_policy({ 'accept': AcceptKeyPolicy, 'add': pyssh.client.AutoAddPolicy, 'warn': pyssh.client.WarningPolicy, }.get(self.config('missing_key'), pyssh.client.RejectPolicy)()) self.sshc = sshc self.channel = None self.flush() if host is not None: self.connect(host, **kwargs)
def detect_prompt(self, prompt0=[]): '''This sets the remote prompt to something more unique than # or $. This makes it easier for the prompt() method to match the shell prompt unambiguously. This method is called automatically by the login() method, but you may want to call it manually if you somehow reset the shell prompt. For example, if you 'su' to a different user then you will need to manually reset the prompt. This sends shell commands to the remote host to set the prompt, so this assumes the remote host is ready to receive commands. Alternatively, you may use your own prompt pattern. Just set the PROMPT attribute to a regular expression that matches it. In this case you should call login() with auto_prompt_reset=False; then set the PROMPT attribute. After that the prompt() method will try to match your prompt pattern. The idea is copied from 'pxssh' in pexpect.''' self.flush2(prompt0 if prompt0 else [EOF]) self.sendline() prompt_wait = self.prompt_wait if prompt_wait: time.sleep(prompt_wait) self.flush2([EOF]) self.sendline() if prompt_wait: time.sleep(prompt_wait) ps1 = self.flush2([EOF]) logger.debug('Expecting prompt, flush2(newline) received: [{0}]'.format(ps1)) self.prompt = Prompt(self.newline.split(ps1)[-1]) self.flush()
def send(self, msg, msguid): """ Resend a given message that has been edited as configured. """ # Build TO:, CC: and BCC: lists into one list: addr = {'to': set(), 'cc': set(), 'bcc': set()} header = [] for xk, xv in msg.items(): xklo = xk.lower() if xklo in addr: logger.debug("mailmon.send: key = %s, value = %s" % (xk, xv)) addr[xklo] |= set(re.split('[\s,:|]+', xv)) elif xk != 'body': # -- excluding anything not belong in the header header.append(xk + ': ' + xv) # Get rid of any duplicates in the "to", "cc" and "bcc" list: if addr['to']: header.append('to: ' + ','.join(addr['to'])) addr['cc'] -= addr['to'] addr['bcc'] -= addr['to'] if addr['cc']: header.append('cc: ' + ','.join(addr['cc'])) addr['bcc'] -= addr['cc'] addr['to'] |= addr['cc'] if addr['bcc']: header.append('bcc: ' + ','.join(addr['bcc'])) addr['to'] |= addr['bcc'] nl = self.newline self.smtp.sendmail(msg['from'], addr['to'], nl.join(header) + nl + nl + msg['body'])
def detect_prompt(self, prompt0=[]): '''This sets the remote prompt to something more unique than # or $. This makes it easier for the prompt() method to match the shell prompt unambiguously. This method is called automatically by the login() method, but you may want to call it manually if you somehow reset the shell prompt. For example, if you 'su' to a different user then you will need to manually reset the prompt. This sends shell commands to the remote host to set the prompt, so this assumes the remote host is ready to receive commands. Alternatively, you may use your own prompt pattern. Just set the PROMPT attribute to a regular expression that matches it. In this case you should call login() with auto_prompt_reset=False; then set the PROMPT attribute. After that the prompt() method will try to match your prompt pattern. The idea is copied from 'pxssh' in pexpect.''' if prompt0: self.flush2(prompt0) self.sendline() self.flush2([self.newline]) self.sendline() self.flush2() prompt = self.newline.split(self.before) self.prompt = prompt[-1].strip() logger.debug("Prompt set to [%s]" % (self.prompt))
def __call__(self): logger.debug('Default gateway = {0}'.format(def_gw)) cmd = cdp.details() ssh = issh.SecureShell(def_gw) _,result = ssh.run(cmd.command)[0] # Parse the result next result = cmd.parse(result) print(result)
def __call__(self, content): '''Parse an email message in "content", which is a string or a text input object. /content/ Standard encoded email message content. Returns parsed message in a dict of (subject, date, body, html, from, to, attachments). ''' msgobj = self.parse(StringIO(content) if isinstance(content, str) else content) subject = parse_header('Subject', msgobj) date = parse_header('Date', msgobj) received = [] for part in msgobj.get_all('Received'): lx = self.re_received.split(part) tmp = dict(zip(lx[1::2], [ x.strip() for x in lx[2::2] ])) tx = tmp.get(';') if tx: tmp['time'] = parse_time(tx) received.append(tmp) fromaddr = parse_addr(msgobj, 'From') if date: date = date.replace(',', '') logger.debug('Parsing message: Date={0}, Subject={1}'.format(date, subject)) #-------- Parsing attachments: attachments = [] body = None html = None for part in msgobj.walk(): attachment = parse_attachment(part) if attachment: attachments.append(attachment) else: # parse text content content_type = part.get_content_type() if content_type[0:5] == 'text/': payload = unicode(part.get_payload(decode=True), part.get_content_charset() or 'ascii', 'replace').encode('utf8','replace') if content_type == "text/plain": if body is None: body = '' body += payload elif content_type == "text/html": if html is None: html = '' html += payload else: logger.debug('Ignored: Content_type "{0}" in message "{1}" from {2}, Date={3}'.format(content_type, subject, fromaddr, date)) return { 'subject' : subject, 'date' : date, 'received': received, # 'received': sorted(received, key=lambda k: k['time']), 'body' : body, 'html' : html, 'from' : fromaddr, 'to' : parse_addr(msgobj, 'To'), 'cc' : parse_addr(msgobj, 'CC'), 'bcc' : parse_addr(msgobj, 'BCC'), 'attachments': attachments }
def open(self): '''To start a filter thread/greenlet. ''' if not self.is_open: self.is_open = True if self.que is None: self.que = Queue() logger.debug('{0}: open - Queue created'.format(type(self).__name__)) return self
def open(self): '''To start a filter thread/greenlet. ''' if not self.is_open: self.is_open = True if self.que is None: self.que = Queue() logger.debug('{0}: open - Queue created'.format( type(self).__name__)) return self
def load(self, filename): '''Load a given file, if not loaded yet. /filename/ Path to file to be loaded. ''' fn = os.path.expanduser(filename) if fn in self.loaded: logger.debug('Config file "{0}" already loaded.'.format(xf)) else: self.update(jso.load_storage(fn))
def text(self): '''Return text parsed from HTML document.''' lineno = 0 for txt in self.segments: lineno += 1 logger.debug("%d: %s" % (lineno, txt)) return ''.join([ str(sx.encode('utf-8', 'xmlcharrefreplace'), 'utf-8') for sx in self.segments ])
def register_dialects(dialects): '''Register a list of CSV dialects (dict of dicts). ''' if dialects is None: return for name, params in dialects.items(): quoting = params.get('quoting') if quoting and not isinstance(quoting, int): params['quoting'] = getattr(csv, quoting) csv.register_dialect(name, **params) logger.debug('CSV dialect "{0}" registered'.format(name))
def register_dialects(dialects): '''Register a list of CSV dialects (dict of dicts). ''' if dialects is None: return for name, params in dialects.iteritems(): quoting = params.get('quoting') if quoting and not isinstance(quoting, int): params['quoting'] = getattr(csv, quoting) csv.register_dialect(name, **params) logger.debug('CSV dialect "{0}" registered'.format(name))
def main(): ''' ''' app = Main() if app.dryrun and os.access('test/logfiler.txt', os.R_OK): logger.debug('Configuration tested OK. Running more module tests.') import doctest doctest.testfile('test/logfiler.txt') else: app() exit(0)
def search(self, xcriteria): ''' Search for messages with given criteria. @param xsearch Search criteria: either a string, or a list of search elements. @return xres Result of the search: OK, NO, or BAD. xuid A space separated list of UIDs if the search is OK. ''' xsearch = xcriteria if isinstance(xcriteria, str) else '('+' '.join(xcriteria)+')' logger.debug("imap4.search: %s" % (xsearch)) xres, xuid = self.uid('search', None, xsearch) return xres, xuid
def close(self): '''To exit this filter thread. Try to close the next filter in chain after the thread is done. ''' klass = type(self).__name__ qu = self.que logger.debug('{0}: closing, que size = {1}, output count = {2}'.format(klass, None if qu is None else qu.qsize(), self.count)) self.__call__() self.try_next('close') logger.debug('{0}: closed, que size = {1}, output count = {2}'.format(klass, None if qu is None else qu.qsize(), self.count))
def path(self, name): '''Create a full path to file with given name. ''' cname = os.path.normpath(name) cname = name if name[0] == os.sep else os.path.join(self.top, name) head, tail = os.path.split(cname) logger.debug('Path "{0}" split to {1} : {2}'.format(cname, head, tail)) if tail != name and not os.path.isdir(head): logger.debug('Creating folder: {0}'.format(head)) fu.forge_path(head) return cname
def __init__(self, dcon): '''Initialize an IMAP4 client with given configuration "dcon". Also, if an password is not provided in the given config, attempt to load /app/etc/logins.json as a ''' imaplib.IMAP4_SSL.__init__(self, dcon.imap.server, dcon.imap.port) uid = dcon.user logger.debug('IMAP = ({0}), user = {1}'.format(dcon.imap, uid)) self.login(uid, PIM().get(uid, dcon.password)) self.select()
def archive(self, auid, abox=''): """ Archive a given message to the configured archive mail box. The mail box is created if it does not exist. """ if not abox: abox = self.CONF.get('archive', '') if not abox: return None xcopy = self.mbox.archive(auid, abox) logger.debug("Archive: copy('%s') == %s" % (auid, xcopy)) return xcopy
def connect(self, host, **kwargs): profile, params = self.get_credentials(host, **kwargs) self.sshc.connect(host, **params) logger.debug('Connected to {0} as user "{1}".'.format(host, params.get('username', '<N/A>'))) self.channel = self.sshc.invoke_shell() # self.stderr = self.channel.makefile_stderr('rb') self.detect_prompt() if profile: init = profile.get('init') if init: self.run(init) self.flush()
def __init(self, **kwargs): ''' Run-once initialization to configure this object. ''' xc = kwargs.get('session') if not xc: raise Exception(t('SNMPAPI.config not found!')) domains = xc.get('domain') if isinstance(domains, str): xc.domain = domains.split("\r\n") logger.debug("SNMPAPI.__init: config({0}) = {1}".format(type(xc), xc)) SNMPAPI.config = xc
def get_object(self, xoid, prefix=''): """ Get a single value of a given object on this device. @param xoid ID of the object to get value for. @param prefix Optional prefix to remove from the value. @return Value of retrieved MIB object. """ xo = self.get(xoid) logger.debug("SNMPAPI.get_object: oid = {0}, val = {1}".format( xoid, xo)) return Prefix(prefix)(xo[0]) if xo and xo[0] else None
def assign(self, value): '''Assigning a given value to self. ''' try: clear = b64decode(value) except TypeError: clear = value value = b64encode(clear) if not value is self.value: logger.debug('Assigning value: {0}'.format(value)) self.value = value return clear.decode("utf-8")
def __init__(self, next_filter, header, write_header=True, dialect=None): '''The CSV writer that writes data to CSV format with given header. A csv.DictWriter is used to write out each row received in this writer. Extra fields in each row is ignored. ''' Filter.__init__(self, next_filter) self.csvo = csv.DictWriter(self.Shim(self), header, extrasaction='ignore', dialect=(dialect or 'excel')) self.header = header if write_header else None # Header to write logger.debug('write_header={0}, header={1}'.format(write_header, header))
def close(self): '''To exit this filter thread. Try to close the next filter in chain after the thread is done. ''' klass = type(self).__name__ qu = self.que logger.debug('{0}: closing, que size = {1}, output count = {2}'.format( klass, None if qu is None else qu.qsize(), self.count)) self.__call__() self.try_next('close') logger.debug('{0}: closed, que size = {1}, output count = {2}'.format( klass, None if qu is None else qu.qsize(), self.count))
def next(self): '''Return the last line if the input is backed up. Otherwise, read the next line from file. ''' if self.read: self.line = self.input.next() else: self.read = True line = self.line if self.ends and self.ends.match(line): logger.debug('Ends proessing at: {0})'.format(line)) raise StopIteration return line
def get(self, id, cpw=None): '''Get the credential for "id". ''' if cpw: pw = cpw else: ux = self[id] pw = ux if isinstance(ux, str) else ux.get('password') if ux else None if pw is None: raise Exception("Password for id '{0}' not found.".format(id)) logger.debug('Credential resolved for user ({0})'.format(id)) return TextPassword(pw).cleartext()
def connect(self, host, **kwargs): profile, params = self.get_credentials(host, **kwargs) self.sshc.connect(host, **params) logger.debug('Connected to {0} as user "{1}".'.format( host, params.get('username', '<N/A>'))) self.channel = self.sshc.invoke_shell() # self.stderr = self.channel.makefile_stderr('rb') self.detect_prompt() if profile: init = profile.get('init') if init: self.run(init) self.flush()
def __init__(self, host='', conf=Null()): options = conf.get('options', {}) if isinstance(options, Null): options = options.__dict__ ## {timeout, maxread, searchwindowsize, logfile, cwd, env} logger.debug("CSSH: spawning with option = %s" % (format(options))) spawn.__init__(self, None, **options) self.C = conf prompt0 = conf.get('prompt', '[#$>]') self.prompt0 = re.compile(prompt0) self.newline = re.compile("[\r\n]+") if host: self.login(host) logger.debug("CSSH: prompt0 = '%s'" % (format(prompt0)))
def __init__(self, conf, default_days=60): '''Configure this object. .days Only delete mails older than given number of days. Defaults to 60. .limit Only delete up to given number of messages. Defaults to 1000. .stop True if not continuing to process the messages found by this task. ''' MailMonitor.__init__(self, conf) self.limit = conf.get('limit', 1000) self.stop = conf.get('stop', True) self.fetching = conf.get('fetch') or False if not self.archive(): self.CONF['archive'] = False logger.debug("MailMonitor.{0}: days = {1}, limit = {2}".format( type(self).__name__, self.days, self.limit))
def walk_object(self, xoid, context=''): ''' Walk through SNMP objects and fetch their values @param xoid Starting point of the walk -- an object ID. @param context Optional context. @return An array of values. ''' xvar = self.walk(xoid, context) xobj = {} for v in xvar: xobj[v.iid] = v.val logger.debug("SNMPAPI.walk_object: oid = %s, val = %s" % (xoid, xobj)) return xobj
def archive(self, auid, abox): """ Archive a given message to the configured archive mail box. The mail box is created if it does not exist. """ xcopy = self.uid('COPY', auid, abox) if xcopy[0] == 'OK': mov, data = self.delete(auid) elif xcopy[0] == 'NO' and xcopy[1][0][:11] == '[TRYCREATE]': xdir = self.create(abox) logger.debug("Created mailbox: {0} = {1}".format(abox, xdir)) if xdir[0] == 'OK': xcopy = self.archive(auid, abox) return xcopy
def __next__(self): '''Return the last line if the input is backed up. Otherwise, read the next line from file. ''' if self.read: self.line = next(self.input) else: self.read = True line = self.line line = line.decode('utf-8') if isinstance(line, bytes) else line if self.ends and self.ends.match(line): logger.debug('Ends proessing at: {0})'.format(line)) raise StopIteration return line
def __init__(self): ''' A command line application object to monitor email. The class must either have no doc, or have something in the format of c9r.app.Command. ''' Command.__init__(self) xc = self.CONF smtp = self.CONF.smtp try: self.smtp = smtplib.SMTP(smtp.server, smtp.port) except Exception as xerr: raise Exception( 'Failed to connect SMTP server for sending email: %s' % (xerr)) self.task = [] plugin = xc.get('plugin', 'MailMonitor') mmmod = __import__(plugin + '.' + plugin) base_klass = getattr(getattr(mmmod, plugin, None), plugin, None) logger.debug("mailmon: Plugin base '%s' loaded as %s from %s." % (plugin, format(base_klass), format(mmmod))) for xtask in xc.get('tasklist', []): conf = xc.get(xtask) action = conf.action if not action: logger.error("mailmon: Task '%s' with no action is ignored." % (xtask)) continue klass = getattr(globals(), action, None) # "action" class may be in globally defined if not klass and mmmod: mod = __import__( plugin + '.' + action) # "action" class may also be in a plugin module if isinstance(mod, type(mmmod)): klass = getattr(getattr(mod, action, None), action, None) logger.debug("mailmon: Action '%s' imported as %s." % (action, str(klass))) if callable(klass) and issubclass(klass, base_klass): conf.util = self self.task.append(klass(conf)) else: logger.error("mailmon: Action module '%s' is not callable." % (format(klass))) logger.error( "mailmon: Action module '%s' not found, task '%s' is ignored." % (action, xtask)) self.parse = parser.Parser() self.cleanup_set = set() atexit.register(self.cleanup)
def include(self, path, ext='.json'): '''To include a file or files in a folder that ends with given extension. /path/ Path to file or directory to include. /ext/ Extension to limit files to include. ''' logger.debug("To include: {0}".format(path)) lx = len(ext) for fn in (os.listdir(path) if os.path.isdir(path) else [path]): if not fn is path: fn = os.path.join(path, fn) fn = os.path.expanduser(fn) if fn[-lx:] == ext and os.path.isfile(fn): logger.debug("Including file: {0}".format(fn)) self.load(fn)
def __init__(self, next_filter, header, write_header=True, dialect=None): '''The CSV writer that writes data to CSV format with given header. A csv.DictWriter is used to write out each row received in this writer. Extra fields in each row is ignored. ''' Filter.__init__(self, next_filter) self.csvo = csv.DictWriter(self.Shim(self), header, extrasaction='ignore', dialect=(dialect or 'excel')) self.header = header if write_header else None # Header to write logger.debug('write_header={0}, header={1}'.format( write_header, header))
def __call__(self): '''Go through list of files to monitor and fix them. Each configured task is started as "concurrently" in a greenlet. ''' os.chdir(self.config('path', '.')) tasks = self.config('tasks', {}) for pat,cfg in tasks.iteritems(): jobqu.put((cfg, pat)) tasks = min(self.config('threads', 10), jobqu.qsize()) cwd = os.getcwd() logger.debug('Spawning {0} task threads, CWD = {1}.'.format(tasks, cwd)) tasks = [ gevent.spawn(task, cwd) for x in range(0, tasks) ] logger.debug('Waiting for {0} task threads to complete.'.format(len(tasks))) #jobqu.join() gevent.joinall(tasks)
def __call__(self): '''Go through list of files to monitor and fix them. Each configured task is started as "concurrently" in a greenlet. ''' os.chdir(self.config('path', '.')) tasks = self.config('tasks', {}) for pat,cfg in tasks.items(): jobqu.put((cfg, pat)) tasks = min(self.config('threads', 10), jobqu.qsize()) cwd = os.getcwd() logger.debug('Spawning {0} task threads, CWD = {1}.'.format(tasks, cwd)) tasks = [ gevent.spawn(task, cwd) for x in range(0, tasks) ] logger.debug('Waiting for {0} task threads to complete.'.format(len(tasks))) #jobqu.join() gevent.joinall(tasks)
def write(self, data): '''Run given /data/ through the csv.DictWriter to convert from a dict to a CSV row with its writerow() function. ''' try: if self.header != None: # TBD: To use csv.DictWriter.writeheader() # SuSE has Python 2.6.x, where the function does not exist. # self.csvo.writeheader() self.next_filter.write(','.join(self.header) + '\r\n') self.header = None logger.debug('Wrote header to {0}'.format(self.next_filter)) self.csvo.writerow(data) self.flush() except Exception as ex: logger.debug('Got Exception {1}, data={0}'.format(data, ex)) raise
def write(self, data): '''Run given /data/ through the csv.DictWriter to convert from a dict to a CSV row with its writerow() function. ''' try: if self.header != None: # TBD: To use csv.DictWriter.writeheader() # SuSE has Python 2.6.x, where the function does not exist. # self.csvo.writeheader() self.next_filter.write(','.join(self.header)+'\r\n') self.header = None logger.debug('Wrote header to {0}'.format(self.next_filter)) self.csvo.writerow(data) self.flush() except Exception as ex: logger.debug('Got Exception {1}, data={0}'.format(data, ex)) raise
def atexit_process(filename, act): '''Post process a file with given /act/. ''' if act == 'delete': return atexit_delete(filename) if act in set([ 'bzip2', # Compress the file using bzip2; 'gzip', # Compress using gzip; 'xz', # Compress using xz; 'zip' # Compress using zip. ]): cmd = (act if isinstance(act, list) else [act])+[filename] try: subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: logger.error('Error from command {0}\n{1}'.format(cmd, err.output)) return logger.debug('Unknown postprocess action: "{0}" "{1}"'.format(act, filename))
def __call__(self): '''Run the job as configured and use Popen.communicate() to get outputs, assuming that the data is not super large. If the return code from the rsync command is not zero (0), then any error Returns results for the jobs, each in a dict of dict[status, out, err] variable, keyed by job names. ''' rsync = Rsync(self.config()) results = {} for job in self.config('jobs', []): run = rsync(self.config(job)) fout,ferr = run.communicate() rc = run.wait() logger.debug('Job "{0}":\nReturn code = {1}\n\nOutput: {2}\n\nError: {3}'.format(job, rc, fout, ferr)) results[job] = dict(status=rc, out=fout.split('\n'), err=ferr) return results
def main(): '''Use SecureShell as command runner. ''' cmd = ISSH() if cmd.dryrun: # Run unit test import doctest doctest.testfile('test/issh.txt') return if not cmd.args: return host = cmd.args[0] logger.debug('Connecting to {0}'.format(host)) ssh = SecureShell() results = ssh.run(cmd.args[1:], host=host) logger.debug('Got {0} result(s)'.format(len(results))) for cmd,output in results.items(): print(cmd) for line in StringIO(output): print('\t'+line.rstrip())
def __call__(self, command, echo=None, prompt='', timeout=10): ''' Send the specified (command) to the remote host and wait for (prompt). Return what comes before (prompt) as command output. ''' logger.debug("CSSH(): sending command [%s]" % (command)) self.sendline(command) if echo is None: echo = command.strip() if echo: logger.debug("CSSH(): waiting for command echo [%s]" % (echo)) xi = self.expect([echo, TIMEOUT, EOF], timeout=timeout) if prompt is not None: if not prompt: prompt = self.prompt logger.debug("CSSH(): waiting for prompt [%s]" % (prompt)) xi = self.expect([prompt, TIMEOUT, EOF], timeout=timeout) logger.debug("CSSH.expect returned %d" % (xi)) output = self.before[:-2] logger.debug("Remote output = [%s]" % (output)) return output