def _refresh(self): with self.storage.cursor() as cursor: row = apsw_helpers.get(cursor, ''' SELECT * FROM %s WHERE id = :task_id AND execution_id = :execution_id AND last_contact > datetime(:now, 'unixepoch', '-%s second') ''' % (self._queue.table_name, self._queue.execution_ttl), now=unix_timestamp(datetime.utcnow()), task_id=self.task_id, execution_id=self.execution_id) if not row: raise TaskDoesNotExist() self.task_id = row.id self.data = json.loads(row.data) self.result = row.result self.job_id = row.job_id self.file_id = row.file_id self.md5 = row.md5 self.bytes_total = row.bytes_total self.bytes_downloaded = row.bytes_downloaded self.download_rate = row.download_rate self.steps = self._load_steps(json.loads(row.steps)) self.started = row.started self.finished = row.finished
def get(self, job_id): with self.storage.transaction() as cursor: job = apsw_helpers.get( cursor, 'SELECT id, spec FROM jobs WHERE id = ?', job_id) if job is not None: job['spec'] = json.loads(job.spec) return Job(job.spec, job.id)
def query_target(self, host, port, database, table): with self.storage.cursor() as cursor: result = apsw_helpers.query(cursor, 'SELECT id, spec FROM jobs') ret = [] for job in result: spec = json.loads(job.spec) if spec['connection']['host'] != host: continue if spec['connection']['port'] != port: continue if spec['target']['database'] != database: continue if spec['target']['table'] != table: continue ret.append(Job(spec, job.id)) return ret
def all(self): with self.storage.cursor() as cursor: result = apsw_helpers.query( cursor, 'SELECT id, spec FROM jobs ORDER BY created ASC') return [Job(json.loads(job.spec), job.id) for job in result]
def process_options(self): """ This function validates the command line options and converts the options into a spec file. The spec file is then passed through the job schema validator for further validation.""" if self.options.help: self.options.subparser.print_help() sys.exit(0) log.update_verbosity(debug=self.options.debug) if self.options.spec: try: with open(self.options.spec, 'r') as f: base_spec = json.loads(f.read()) except JSONDecodeError: print >>sys.stderr, "Failed to load spec file '%s': invalid JSON" % self.options.spec sys.exit(1) except IOError as e: print >>sys.stderr, "Unable to open spec file '%s': %s" % (self.options.spec, str(e)) sys.exit(1) else: base_spec = {} self.pre_process_options(self.options, self.logger) if self.options.password == _PasswordNotSpecified: password = getpass.getpass('Enter password: '******'t break anything. try: paths = merged_spec['source']['paths'] except KeyError: paths = [] for path in paths: if path.startswith('s3://'): schema.DEFAULT_AWS_ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID') schema.DEFAULT_AWS_SECRET_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') break try: self.job = Job(merged_spec) self.logger.debug("Produced spec:\n%s", json.pformat(self.job.spec)) except V.Invalid as err: if isinstance(err, V.MultipleInvalid): errors = err.errors else: errors = [err] error_msgs = [] seen_paths = [] if self.options.spec: er_config_validation = """\ Invalid specification: %(formatted_spec)s Error(s): %(error_msg)s""" er_msg_fmt = " - Error [%(error_path)s] (or %(error_cmd_line)s on the command line): %(message)s." else: er_config_validation = """\ Invalid command line options for load: %(error_msg)s""" er_msg_fmt = " - Invalid value for %(error_cmd_line)s: %(message)s." for e in errors: extra_key = (e.message == 'extra keys not allowed') error_path = '.'.join([str(leg) for leg in e.path]) cmd_line_opt = schema.get_command_line_mapping([x for x in e.path]) if cmd_line_opt == 'paths': cmd_line_opt = "the path argument (positional)" else: cmd_line_opt = '--' + cmd_line_opt if any((error_path in seen_path) for seen_path in seen_paths): # we do this because voluptuous triggers missing # key errors for any required key that has a sub-error continue seen_paths.append(error_path) error_msgs.append( er_msg_fmt % { 'error_path': error_path, 'error_cmd_line': cmd_line_opt, 'message': 'key %s is not allowed' % error_path if extra_key else e.error_message}) self.logger.error(er_config_validation % { 'formatted_spec': json.pformat(merged_spec), 'error_msg': "\n".join(error_msgs) }) sys.exit(1) if self.options.print_spec: print json.pformat(self.job.spec) sys.exit(0)
def process_options(self): """ This function validates the command line options and converts the options into a spec file. The spec file is then passed through the job schema validator for further validation.""" if self.options.help: self.options.subparser.print_help() sys.exit(0) log.update_verbosity(debug=self.options.debug) if self.options.spec: try: with open(self.options.spec, 'r') as f: base_spec = json.loads(f.read()) except JSONDecodeError: print >> sys.stderr, "Failed to load spec file '%s': invalid JSON" % self.options.spec sys.exit(1) except IOError as e: print >> sys.stderr, "Unable to open spec file '%s': %s" % ( self.options.spec, str(e)) sys.exit(1) else: base_spec = {} self.pre_process_options(self.options, self.logger) if self.options.password == _PasswordNotSpecified: password = getpass.getpass('Enter password: '******'t break anything. try: paths = merged_spec['source']['paths'] except KeyError: paths = [] for path in paths: if path.startswith('s3://'): schema.DEFAULT_AWS_ACCESS_KEY = os.getenv('AWS_ACCESS_KEY_ID') schema.DEFAULT_AWS_SECRET_KEY = os.getenv( 'AWS_SECRET_ACCESS_KEY') break try: self.job = Job(merged_spec) self.logger.debug("Produced spec:\n%s", json.pformat(self.job.spec)) except V.Invalid as err: if isinstance(err, V.MultipleInvalid): errors = err.errors else: errors = [err] error_msgs = [] seen_paths = [] if self.options.spec: er_config_validation = """\ Invalid specification: %(formatted_spec)s Error(s): %(error_msg)s""" er_msg_fmt = " - Error [%(error_path)s] (or %(error_cmd_line)s on the command line): %(message)s." else: er_config_validation = """\ Invalid command line options for load: %(error_msg)s""" er_msg_fmt = " - Invalid value for %(error_cmd_line)s: %(message)s." for e in errors: extra_key = (e.message == 'extra keys not allowed') error_path = '.'.join([str(leg) for leg in e.path]) cmd_line_opt = schema.get_command_line_mapping( [x for x in e.path]) if cmd_line_opt == 'paths': cmd_line_opt = "the path argument (positional)" else: cmd_line_opt = '--' + cmd_line_opt if any((error_path in seen_path) for seen_path in seen_paths): # we do this because voluptuous triggers missing # key errors for any required key that has a sub-error continue seen_paths.append(error_path) error_msgs.append( er_msg_fmt % { 'error_path': error_path, 'error_cmd_line': cmd_line_opt, 'message': 'key %s is not allowed' % error_path if extra_key else e.error_message }) self.logger.error( er_config_validation % { 'formatted_spec': json.pformat(merged_spec), 'error_msg': "\n".join(error_msgs) }) sys.exit(1) if self.options.print_spec: print json.pformat(self.job.spec) sys.exit(0)