def __call__(self):
'''
'''
product = self.config('product_url')
if product is None:
logger.error('No product information page configured. Exiting.')
return
info = requests.get(self.config('product_url'))
if info.status_code != 200:
logger.error('Error getting product information: status = {0}'.format(info.status_code))
return
mp = None
for line in StringIO(info.content):
mx = self.regex.search(line)
if mx is None: continue
link = mx.group(1)
logger.debug(link)
mp = self.repath.search(link)
if mp: break
if mp is None:
print('Did not find anything to download on page:\n\t{0}\n'.format(product))
return
pkginfo = mp.groupdict()
path = self.config('download_base')+self.config('path')
logger.debug('Product download folder: '+path)
logger.debug('Retrieving configured packages: ----------')
for prod, val in self.config('products').iteritems():
pkginfo['product'] = prod
for os, package in val.iteritems():
for pkg in package if isinstance(package, list) else [package]:
pkginfo['os'] = os
link = (path+'/'+pkg).format(**pkginfo)
print('wget {0}'.format(link))
def write(self, data):
'''Normalize given data before writing to the pipe.
/data/ is expected to be a dictionary-type object, with.
'''
val = dict(self.assigned_values)
emac = apname = None
try:
emac = data.get("EthernetMAC")
if not emac:
return 0
val['macAddress'] = MACFormat.none(emac)
rmac = data.get('BaseRadioMAC')
if rmac:
data['BaseRadioMAC'] = MACFormat.none(rmac)
apname = data.get("APName")
val.update(parse_apname(apname))
for kd, ks in self.map.items():
val[kd] = data[ks]
except Exception as err:
logger.error('Error {0} - MAC:{1}, AP:{2}'.format(
err, emac, apname))
if not emac:
return 0
data.update(val)
return Filter.write(self, data)
def __call__(self, msg):
xc = self.CONF.smtp
try:
self.smtp = smtplib.SMTP(xc.server, xc.port)
raise Exception('MailMonitor.Forward is not completed yet.')
except Exception as xerr:
logger.error(
'Failed to connect SMTP server for sending email: %s' % (xerr))
return -2
def write(self, data):
'''Normalize given data before writing to the pipe.
/data/ is expected to be a dictionary-type object, with.
'''
val = dict(self.assigned_values)
try:
val['macAddress'] = MACFormat.none(data["EthernetMAC"])
val.update(parse_apname(data["APName"]))
for kd, ks in self.map.iteritems():
val[kd] = data[ks]
except InvalidMACAddress:
logger.error('Invalid MAC address for data: {0}'.format(data))
data.update(val)
return Filter.write(self, data)
def atexit_process(filename, act):
'''Post process a file with given /act/.
'''
if act == 'delete':
return atexit_delete(filename)
if act in set([
'bzip2', # Compress the file using bzip2;
'gzip', # Compress using gzip;
'xz', # Compress using xz;
'zip' # Compress using zip.
]):
cmd = (act if isinstance(act, list) else [act])+[filename]
try:
subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
logger.error('Error from command {0}\n{1}'.format(cmd, err.output))
return
logger.debug('Unknown postprocess action: "{0}" "{1}"'.format(act, filename))
def atexit_process(filename, act):
'''Post process a file with given /act/.
'''
if act == 'delete':
return atexit_delete(filename)
if act in set([
'bzip2', # Compress the file using bzip2;
'gzip', # Compress using gzip;
'xz', # Compress using xz;
'zip' # Compress using zip.
]):
cmd = (act if isinstance(act, list) else [act])+[filename]
try:
subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
logger.error('Error from command {0}\n{1}'.format(cmd, err.output))
return
logger.debug('Unknown postprocess action: "{0}" "{1}"'.format(act, filename))
def test_app():
dir = os.path.dirname(__file__) if __file__ else 'tests'
sys.argv += ['-c', dir + '/app.json', '-c', 'no-such.json', '-c', '--1']
app = Command()
logger.debug('app.config() = {0}'.format(repr(app.config())))
logger.debug(format(sys.argv))
logger.debug('Testing DEBUG logging')
logger.debug('Testing DEBUG logging with stdout')
logger.info('INFO message')
logger.warn('WARNING message')
logger.error('ERROR message')
logger.fatal('FATAL message')
assert len(logger.handlers) == 1
assert app.config().__module__ == 'c9r.jsonpy'
#print(app.config().__module__, file=sys.stderr)
assert app.config().__class__.__name__ == 'Thingy'
assert app.config('true') == True
assert app.config('key') == 'value'
assert app.config('none') == None
app.clear_config()
assert app.config() == None
def test_app():
dir = os.path.dirname(__file__) if __file__ else 'tests'
sys.argv += [ '-c', dir+'/app.json', '-c', 'no-such.json', '-c', '--1' ]
app = Command()
logger.debug('app.config() = {0}'.format(repr(app.config())))
logger.debug(format(sys.argv))
logger.debug('Testing DEBUG logging')
logger.debug('Testing DEBUG logging with stdout')
logger.info('INFO message')
logger.warn('WARNING message')
logger.error('ERROR message')
logger.fatal('FATAL message')
assert len(logger.handlers) == 1
assert app.config().__module__ == 'c9r.jsonpy'
#print(app.config().__module__, file=sys.stderr)
assert app.config().__class__.__name__ == 'Thingy'
assert app.config('true') == True
assert app.config('key') == 'value'
assert app.config('none') == None
app.clear_config()
assert app.config() == None
def __init__(self):
'''
A command line application object to monitor email.
The class must either have no doc, or have something in the format of c9r.app.Command.
'''
Command.__init__(self)
xc = self.CONF
smtp = self.CONF.smtp
try:
self.smtp = smtplib.SMTP(smtp.server, smtp.port)
except Exception as xerr:
raise Exception(
'Failed to connect SMTP server for sending email: %s' % (xerr))
self.task = []
plugin = xc.get('plugin', 'MailMonitor')
mmmod = __import__(plugin + '.' + plugin)
base_klass = getattr(getattr(mmmod, plugin, None), plugin, None)
logger.debug("mailmon: Plugin base '%s' loaded as %s from %s." %
(plugin, format(base_klass), format(mmmod)))
for xtask in xc.get('tasklist', []):
conf = xc.get(xtask)
action = conf.action
if not action:
logger.error("mailmon: Task '%s' with no action is ignored." %
(xtask))
continue
klass = getattr(globals(), action,
None) # "action" class may be in globally defined
if not klass and mmmod:
mod = __import__(
plugin + '.' +
action) # "action" class may also be in a plugin module
if isinstance(mod, type(mmmod)):
klass = getattr(getattr(mod, action, None), action, None)
logger.debug("mailmon: Action '%s' imported as %s." %
(action, str(klass)))
if callable(klass) and issubclass(klass, base_klass):
conf.util = self
self.task.append(klass(conf))
else:
logger.error("mailmon: Action module '%s' is not callable." %
(format(klass)))
logger.error(
"mailmon: Action module '%s' not found, task '%s' is ignored."
% (action, xtask))
self.parse = parser.Parser()
self.cleanup_set = set()
atexit.register(self.cleanup)
def normalize(self, data):
'''Here the data fields will be normalized before returned:
o MAC addresses are rid of delimiters;
o Values such as "Unknown" and "Not Supported" are removed;
o /Last Session Length/ values are converted to number of seconds;
o /User/ IDs are normalized to "domain/userid" format, also handing
(wrongly-)escaped special characters embedded.
'''
for xk in [ 'APMACAddress', 'MACAddress' ]:
xv = data.get(xk)
if xv:
data[xk] = MACFormat.none(xv)
for xk in [ 'EndpointType' ]:
xv = data.get(xk, '')
if xv == 'Unknown':
data[xk] = ''
elif len(xv) > 0:
data[xk] = xv.strip()
#
# Wi-Fi tracking data:
#
for xk in [ 'CCX', 'E2E' ]:
if data.get(xk, '') == 'Not Supported':
data[xk] = ''
try:
it = self.retime.findall(data['LastSessionLength'])
except TypeError:
it = []
sec = 0
for x in it:
xsec = 0
try:
ui, ut = x
xsec = int(ui)*{ 'days': 86400, 'hrs': 3600, 'min': 60, 'sec': 1 }[ut]
except KeyError:
logger.error('Invalid time unit: {0}'.format(ui))
except ValueError:
logger.error('Invalid time value: {0}'.format(ut))
except StopIteration:
pass
sec += xsec
if sec > 0:
data['LastSessionLength'] = sec
#
# Normalize vendor names
#
vendor_name = data.get('Vendor', '')
vendor = self.vendor_map.get(vendor_name.lower())
if vendor is None:
vendor = self.reclean.sub(self.cleansub, vendor_name)
if vendor != vendor_name:
data['Vendor'] = vendor
# Process 'User' ID:
#-- Hacky but no better way: Must deal with '\n', '\t' individually so not to
# convert '\' to '\\'.
#-- Normalize to "domain/userid" format
uid = data.get('User', '')
if uid:
for xt in [ [ '\n', '\\n' ], [ '\r', '\\r' ], ['\t', '\\t' ] ]:
uid = uid.replace(xt[0], xt[1])
uid = uid.replace('\\', '/', 1)
data['User'] = uid
port = data.get('InterfaceName')
if port:
data['InterfaceName'] = port = nickname(port)
if not 'Port' in data:
data['Port'] = port
return data
def normalize(self, data):
'''Here the data fields will be normalized before returned:
o MAC addresses are rid of delimiters;
o Values such as "Unknown" and "Not Supported" are removed;
o /Last Session Length/ values are converted to number of seconds;
o /User/ IDs are normalized to "domain/userid" format, also handing
(wrongly-)escaped special characters embedded.
'''
for xk in ['APMACAddress', 'MACAddress']:
xv = data.get(xk)
if xv:
data[xk] = MACFormat.none(xv)
for xk in ['EndpointType']:
xv = data.get(xk, '')
if xv == 'Unknown':
data[xk] = ''
elif len(xv) > 0:
data[xk] = xv.strip()
#
# Wi-Fi tracking data:
#
for xk in ['CCX', 'E2E']:
if data.get(xk, '') == 'Not Supported':
data[xk] = ''
try:
it = self.retime.findall(data['LastSessionLength'])
except Exception:
it = []
sec = 0
for x in it:
xsec = 0
try:
ui, ut = x
xsec = int(ui) * {
'days': 86400,
'hrs': 3600,
'min': 60,
'sec': 1
}[ut]
except KeyError:
logger.error('Invalid time unit: {0}'.format(ui))
except ValueError:
logger.error('Invalid time value: {0}'.format(ut))
except StopIteration:
pass
sec += xsec
if sec > 0:
data['LastSessionLength'] = sec
#
# Normalize vendor names
#
vendor_name = data.get('Vendor', '')
vendor = self.vendor_map.get(vendor_name.lower())
if vendor is None:
vendor = self.reclean.sub(self.cleansub, vendor_name)
if vendor != vendor_name:
data['Vendor'] = vendor
# Process 'User' ID:
#-- Hacky but no better way: Must deal with '\n', '\t' individually so not to
# convert '\' to '\\'.
#-- Normalize to "domain/userid" format
uid = data.get('User', '')
if uid:
for xt in [['\n', '\\n'], ['\r', '\\r'], ['\t', '\\t']]:
uid = uid.replace(xt[0], xt[1])
uid = uid.replace('\\', '/', 1)
data['User'] = uid
port = data.get('InterfaceName')
if port:
data['InterfaceName'] = port = nickname(port)
if not 'Port' in data:
data['Port'] = port
return data
def log_error(self, msg=None):
if msg is None:
msg = format(sys.exc_info()[0])
logger.error(msg)
def task(cwd):
'''Task as a gevent Greenlet that processes one file name pattern.
/cwd/ Current working directory.
'''
while True:
# config = A dict containing configuration for the task;
# pattern = Pattern to match for input file names.
try:
config, pattern = jobqu.get(timeout=10)
except Empty:
break
if pattern == '' or config.get('disabled', False):
logger.debug('CSVFixer: Ignore empty pattern or disabled task')
continue
pattern = config.get('pattern', pattern) # 'pattern' may be configured inside task
dest = config.get('destination', cwd)
linkfolder = config.get('link-folder')
forge_path(dest)
process = Pipeline(config)
keep_times = config.get('times', False)
rename = [ (re.compile(xk),xv) for xk,xv in config.get('rename', {}).items() ]
logger.debug('CSVFixer: task = %s, destination = "%s"' % (pattern, dest))
for zipfn in glob.glob(pattern):
stinfo = os.stat(zipfn)
logger.debug('CSVFixer: Fixing file "{0}", mtime = {1}'.format(
zipfn, time.strftime('%c', time.localtime(stinfo.st_mtime))))
if zipfn[-4:] != '.zip':
## Assume that it is a text CSV file if file name does not end with .zip:
zipf = None
ziplist = [zipfn]
else:
try:
zipf = ZipFile(zipfn)
ziplist = zipf.namelist()
logger.debug('CSVFixer: Found list in zip file = %s' % (format(ziplist)))
except BadZipfile:
logger.warning('CSVFixer: zip file "%s" is bad.' % (zipfn))
continue
fbasename = fwpath = ''
for fn in ziplist:
if fwpath == '' or config.get('file-mode') != 'a':
fwname = fn
for rex, fmt in rename:
mx = rex.search(fwname)
if mx:
try:
fwname = fmt.format(*mx.groups())
except Exception as ex:
logger.warning('Exception fixing "{0}" with "{1}" and groups = {2}'.format(fn, fmt, mx.groups()))
break
fbasename = os.path.basename(fwname)
fwpath = os.path.join(dest, fbasename)
logger.debug('Processing file "{0}" to "{1}"'.format(fn, fwname))
lines = process(open(fn, 'r') if zipf is None else zipf.open(fn, 'r'), fwpath)
logger.debug('{0} lines processed in file "{1}"'.format(lines, fn))
# Set fixed file's timestamps if so configured:
if keep_times:
os.utime(fwpath, (stinfo.st_mtime, stinfo.st_mtime))
logger.debug('Set file "{0}" atime and mtime to {1}'.format(
fwpath, time.strftime('%c', time.localtime(stinfo.st_mtime))))
# Archive the .zip file if configured so
if config.get('delete', False):
logger.debug('File "%s" registered to be deleted' % (zipfn))
atexit.register(atexit_delete, zipfn)
else:
act = config.get('postprocess')
if act != None:
logger.debug('File "%s" registered to be postprocessed with "%s"' % (zipfn, act))
atexit.register(atexit_process, zipfn, act)
# Delete empty file if so configured:
if fwpath != '' and config.get('delete-empty', True) and os.stat(fwpath).st_size < 1:
os.unlink(fwpath)
logger.debug('Deleted empty output file "{0}"'.format(fwpath))
elif linkfolder:
try:
os.link(fwpath, os.path.join(linkfolder, fbasename))
except Exception as err:
logger.error('Error link file "{0}" to folder {1}: {2}'.format(fwpath, linkfolder, err))
jobqu.task_done()
logger.debug('Task "{0}" completed'.format(pattern))
def log_error(self, msg=None):
if msg is None:
msg = format(sys.exc_info()[0])
logger.error(msg)