def run(self):
self.logger = log.get_logger('Load')
self.process_options()
self.validate_conditions()
self.logger.debug("Produced spec:\n%s", json.pformat(self.job.spec))
self.queue_job()
def run(self):
self.logger = log.get_logger('Load')
self.process_options()
self.validate_conditions()
self.logger.debug("Produced spec:\n%s", json.pformat(self.job.spec))
self.queue_job()
def process_options(self):
""" This function validates the command line options and converts the options
into a spec file. The spec file is then passed through the job schema validator
for further validation."""
if self.options.help:
self.options.subparser.print_help()
sys.exit(0)
log.update_verbosity(debug=self.options.debug)
if self.options.spec:
try:
with open(self.options.spec, 'r') as f:
base_spec = json.loads(f.read())
except JSONDecodeError:
print >>sys.stderr, "Failed to load spec file '%s': invalid JSON" % self.options.spec
sys.exit(1)
except IOError as e:
print >>sys.stderr, "Unable to open spec file '%s': %s" % (self.options.spec, str(e))
sys.exit(1)
else:
base_spec = {}
self.pre_process_options(self.options, self.logger)
if self.options.password == _PasswordNotSpecified:
password = getpass.getpass('Enter password: '******'t break anything.
try:
paths = merged_spec['source']['paths']
except KeyError:
paths = []
for path in paths:
if path.startswith('s3://'):
schema.DEFAULT_AWS_ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID')
schema.DEFAULT_AWS_SECRET_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
break
try:
self.job = Job(merged_spec)
self.logger.debug("Produced spec:\n%s", json.pformat(self.job.spec))
except V.Invalid as err:
if isinstance(err, V.MultipleInvalid):
errors = err.errors
else:
errors = [err]
error_msgs = []
seen_paths = []
if self.options.spec:
er_config_validation = """\
Invalid specification:
%(formatted_spec)s
Error(s):
%(error_msg)s"""
er_msg_fmt = " - Error [%(error_path)s] (or %(error_cmd_line)s on the command line): %(message)s."
else:
er_config_validation = """\
Invalid command line options for load:
%(error_msg)s"""
er_msg_fmt = " - Invalid value for %(error_cmd_line)s: %(message)s."
for e in errors:
extra_key = (e.message == 'extra keys not allowed')
error_path = '.'.join([str(leg) for leg in e.path])
cmd_line_opt = schema.get_command_line_mapping([x for x in e.path])
if cmd_line_opt == 'paths':
cmd_line_opt = "the path argument (positional)"
else:
cmd_line_opt = '--' + cmd_line_opt
if any((error_path in seen_path) for seen_path in seen_paths):
# we do this because voluptuous triggers missing
# key errors for any required key that has a sub-error
continue
seen_paths.append(error_path)
error_msgs.append(
er_msg_fmt % {
'error_path': error_path, 'error_cmd_line': cmd_line_opt,
'message': 'key %s is not allowed' % error_path if extra_key else e.error_message})
self.logger.error(er_config_validation % {
'formatted_spec': json.pformat(merged_spec),
'error_msg': "\n".join(error_msgs)
})
sys.exit(1)
if self.options.print_spec:
print json.pformat(self.job.spec)
sys.exit(0)
def process_options(self):
""" This function validates the command line options and converts the options
into a spec file. The spec file is then passed through the job schema validator
for further validation."""
if self.options.help:
self.options.subparser.print_help()
sys.exit(0)
log.update_verbosity(debug=self.options.debug)
if self.options.spec:
try:
with open(self.options.spec, 'r') as f:
base_spec = json.loads(f.read())
except JSONDecodeError:
print >> sys.stderr, "Failed to load spec file '%s': invalid JSON" % self.options.spec
sys.exit(1)
except IOError as e:
print >> sys.stderr, "Unable to open spec file '%s': %s" % (
self.options.spec, str(e))
sys.exit(1)
else:
base_spec = {}
self.pre_process_options(self.options, self.logger)
if self.options.password == _PasswordNotSpecified:
password = getpass.getpass('Enter password: '******'t break anything.
try:
paths = merged_spec['source']['paths']
except KeyError:
paths = []
for path in paths:
if path.startswith('s3://'):
schema.DEFAULT_AWS_ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID')
schema.DEFAULT_AWS_SECRET_KEY = os.getenv(
'AWS_SECRET_ACCESS_KEY')
break
try:
self.job = Job(merged_spec)
self.logger.debug("Produced spec:\n%s",
json.pformat(self.job.spec))
except V.Invalid as err:
if isinstance(err, V.MultipleInvalid):
errors = err.errors
else:
errors = [err]
error_msgs = []
seen_paths = []
if self.options.spec:
er_config_validation = """\
Invalid specification:
%(formatted_spec)s
Error(s):
%(error_msg)s"""
er_msg_fmt = " - Error [%(error_path)s] (or %(error_cmd_line)s on the command line): %(message)s."
else:
er_config_validation = """\
Invalid command line options for load:
%(error_msg)s"""
er_msg_fmt = " - Invalid value for %(error_cmd_line)s: %(message)s."
for e in errors:
extra_key = (e.message == 'extra keys not allowed')
error_path = '.'.join([str(leg) for leg in e.path])
cmd_line_opt = schema.get_command_line_mapping(
[x for x in e.path])
if cmd_line_opt == 'paths':
cmd_line_opt = "the path argument (positional)"
else:
cmd_line_opt = '--' + cmd_line_opt
if any((error_path in seen_path) for seen_path in seen_paths):
# we do this because voluptuous triggers missing
# key errors for any required key that has a sub-error
continue
seen_paths.append(error_path)
error_msgs.append(
er_msg_fmt % {
'error_path':
error_path,
'error_cmd_line':
cmd_line_opt,
'message':
'key %s is not allowed' %
error_path if extra_key else e.error_message
})
self.logger.error(
er_config_validation % {
'formatted_spec': json.pformat(merged_spec),
'error_msg': "\n".join(error_msgs)
})
sys.exit(1)
if self.options.print_spec:
print json.pformat(self.job.spec)
sys.exit(0)
