def parse(self, stdout):
if isPythonMinVersion(3):
output = [_ for _ in str(stdout).split(r'\n') if _]
else:
output = [_ for _ in str(stdout).split('\n') if _]
log.debug('output = %s', output)
if len(output) < 2:
raise CriticalError("docker image '{repo}' not found! Does not exist or has not been pulled yet?"\
.format(repo=self.docker_image))
tags = set()
for line in output[1:]:
log.debug('line: %s', line)
line_parts = line.split()
if len(line_parts) > 1:
tags.add(line_parts[1])
tags = [tag for tag in tags if tag and tag != '<none>']
tags = sorted(list(tags))
if log.isEnabledFor(logging.DEBUG):
for tag in tags:
log.debug('found tag: %s', tag)
if len(tags) > 1:
raise UnknownError('too many results returned - did you forget to suffix a specific :tag to ' + \
'--docker-image? (eg. :latest, :1.1). The following tags were found: ' + \
', '.join(tags)
)
header_line = output[0]
docker_image_line = output[1]
image_header = ' '.join(header_line.split()[2:4])
log.debug('image header column: %s', image_header)
if image_header != 'IMAGE ID':
raise UnknownError("3rd column in header '{0}' is not 'IMAGE ID' as expected, parsing failed!"\
.format(image_header))
self.msg = "docker image '{repo}'".format(repo=self.docker_image)
self.check_id(docker_image_line)
self.check_size(docker_image_line)
def anonymize(self, line):
#log.debug('anonymize: line: %s', line)
match = self.re_line_ending.search(line)
line_ending = ''
if match:
line_ending = match.group(1)
if self.strip_cr:
line_ending = '\n'
if not isPythonMinVersion(3):
line = line.decode('utf-8').encode('ascii', errors='replace')
line = strip_ansi_escape_codes(line)
line = self.re_line_ending.sub('', line)
for _ in self.anonymizations:
if not self.anonymizations[_]:
continue
method = None
try:
#log.debug('checking for anonymize_%s', _)
method = getattr(self, 'anonymize_' + _)
except AttributeError:
#log.debug('anonymize_%s not found', _)
pass
if method:
#log.debug('found anonymize_%s: %s', _, method)
line = method(line)
else:
line = self.anonymize_dynamic(_, line)
if line is None:
if method:
raise AssertionError('anonymize_{} returned None'.format(_))
else:
raise AssertionError('anonymize_dynamic({}, line)'.format(_))
line += line_ending
return line
def run(self):
quoting = csv.QUOTE_ALL
if self.quotechar == '':
quoting = csv.QUOTE_NONE
#fieldnames = ['database', 'table', 'user']
fieldnames = ['database', 'table']
self.csv_writer = csv.DictWriter(sys.stdout,
delimiter=self.delimiter,
quotechar=self.quotechar,
escapechar=self.escapechar,
quoting=quoting,
fieldnames=fieldnames)
mode = 'rtU'
if isPythonMinVersion(3):
mode = 'rt'
# open(..., encoding="utf-8") is Python 3 only - uses system default otherwise
for filename in self.args:
if filename.endswith('.gz'):
log.debug("processing gzip'd file: %s", filename)
with gzip.open(filename, mode) as filehandle:
self.process_file(filehandle)
else:
log.debug("processing file: %s", filename)
with open(filename, mode) as filehandle:
self.process_file(filehandle)
# pylint: disable=wrong-import-position
from harisekhon.utils import isLinux, isMac, isIP, isPythonMinVersion, ERRORS, printerr, warn, pyspark_path
except ImportError as _:
print('module import failed: %s' % _, file=sys.stderr)
print("Did you remember to build the project by running 'make'?",
file=sys.stderr)
print(
"Alternatively perhaps you tried to copy this program out without it's adjacent libraries?",
file=sys.stderr)
sys.exit(4)
pyspark_path()
__author__ = 'Hari Sekhon'
__version__ = '0.3.2'
if not isPythonMinVersion(2.7):
warn(
'Python < 2.7 - IPython may not be available on this version of Python '
+ '(supplied auto-build will likely have failed for this module)\n')
try:
# pylint: disable=wrong-import-position
from IPython.lib import passwd
except ImportError as _:
printerr("""failed to import from IPython.lib
Perhaps you need to 'pip install \"ipython[notebook]\"'
Exception message: %s""" % _)
if not isPythonMinVersion(2.7):
printerr(
'Python < 2.7 - the supplied make auto build with this tool probably failed '
import os
import sys
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
sys.path.append(libdir)
try:
# pylint: disable=wrong-import-position
from harisekhon import CLI
from harisekhon.utils import isPythonMinVersion
except ImportError as _:
print('module import failed: %s' % _, file=sys.stderr)
print("Did you remember to build the project by running 'make'?", file=sys.stderr)
print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
sys.exit(4)
# pylint: disable=no-name-in-module,import-error
if isPythonMinVersion(3):
from urllib.parse import quote_plus as quote
else:
from urllib import quote_plus as quote
__author__ = 'Hari Sekhon'
__version__ = '0.1.1'
class URLEncode(CLI):
def __init__(self):
# Python 2.x
super(URLEncode, self).__init__()
# Python 3.x
# super().__init__()
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
sys.path.append(libdir)
try:
# pylint: disable=wrong-import-position
from harisekhon.utils import isLinux, isMac, isIP, isPythonMinVersion, ERRORS, printerr, warn, pyspark_path
except ImportError as _:
print('module import failed: %s' % _, file=sys.stderr)
print("Did you remember to build the project by running 'make'?", file=sys.stderr)
print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
sys.exit(4)
pyspark_path()
__author__ = 'Hari Sekhon'
__version__ = '0.3.2'
if not isPythonMinVersion(2.7):
warn('Python < 2.7 - IPython may not be available on this version of Python ' +
'(supplied auto-build will likely have failed for this module)\n')
try:
# pylint: disable=wrong-import-position
from IPython.lib import passwd
except ImportError as _:
printerr("""failed to import from IPython.lib
Perhaps you need to 'pip install \"ipython[notebook]\"'
Exception message: %s""" % _)
if not isPythonMinVersion(2.7):
printerr('Python < 2.7 - the supplied make auto build with this tool probably failed ' +
'to install IPython because IPython requires Python >= 2.7')
sys.exit(ERRORS['UNKNOWN'])
