Example #1
0
    def run(self):
        self.logger = log.get_logger('Tasks')
        self.tasks_api = TasksApi()

        if not self.options.job_id and not self.options.last_job:
            print 'You must specify a job ID or use the --last-job option.'
            sys.exit(1)

        if self.options.last_job:
            jobs = Jobs()
            job_list = jobs.all()
            if not job_list:
                print 'No jobs found.'
                sys.exit(1)
            self.options.job_id = job_list[-1].id

        try:
            result = self.tasks_api.query({ k: v for k, v in {
                'job_id': self.options.job_id,
                'state': self.options.state,
                'order': self.options.order,
                'order_by': self.options.order_by,
                'page_size': self.options.page_size,
                'page': self.options.page,
            }.iteritems() if v })
        except exceptions.ApiException as e:
            print e.message
            sys.exit(1)

        if result:
            tablefmt = TableFormat.JSON if self.options.json else TableFormat.TABLE
            print PrettyPrinter(result, columns=TasksApi.SORTABLE_COLUMNS, format=tablefmt).format()
        else:
            print 'No tasks found that match this query'
            sys.exit(1)
Example #2
0
    def run(self):
        self.logger = log.get_logger('Job')
        self.job_api = JobApi()
        self.tasks_api = TasksApi()

        try:
            result = self.job_api.query({ 'job_id': self.options.job_id })
        except exceptions.ApiException as e:
            print e.message
            sys.exit(1)

        if self.options.spec:
            print json.dumps(result.spec, sort_keys=True, indent=4 * ' ')
        else:
            try:
                finished_tasks = self.tasks_api.query({
                    'job_id': self.options.job_id,
                    'state': 'SUCCESS'
                })
            except exceptions.ApiException as e:
                print e.message
                sys.exit(1)

            files_loaded = len(finished_tasks)
            rows_loaded = reduce(lambda x, y: x + y.get('data', {}).get('row_count', 0), finished_tasks, 0)
            avg_rows_per_file = None
            avg_rows_per_second = None

            if files_loaded > 0:
                avg_rows_per_file = rows_loaded / files_loaded

                min_start_time = datetime.datetime.max
                max_stop_time = datetime.datetime.min
                for row in finished_tasks:
                    for step in row.steps:
                        if step['name'] == 'download':
                            min_start_time = min(min_start_time, step['start'])
                            max_stop_time = max(max_stop_time, step['stop'])
                            break
                    else:
                        continue
                avg_rows_per_second = rows_loaded / (max_stop_time - min_start_time).total_seconds()

            result['stats'] = { k: v for k, v in {
                'files_loaded': files_loaded,
                'rows_loaded': rows_loaded,
                'avg_rows_per_file': avg_rows_per_file,
                'avg_rows_per_second': avg_rows_per_second
            }.iteritems() if v is not None }

            if result.tasks_total > 0:
                result['stats'].update({
                    'success_rate': result.tasks_succeeded * 1.0 / result.tasks_total,
                    'error_rate': result.tasks_errored * 1.0 / result.tasks_total
                })

            result["database"] = result.spec["target"]["database"]
            result["table"] = result.spec["target"]["table"]

            result = dict(result)
            del result['spec']

            result = { k: str(v) if isinstance(v, SuperEnum.Element) else v for k, v in result.iteritems() }
            print json.dumps(result, sort_keys=True, indent=4 * ' ')
Example #3
0
class Job(Command):
    @staticmethod
    def configure(parser, subparsers):
        subparser = subparsers.add_parser('job', help='Show information about a single job')
        subparser.set_defaults(command=Job)

        subparser.add_argument('job_id',
            help='The ID of the job to lookup')

        subparser.add_argument('--spec', action='store_true', default=False,
            help='Output only the spec of this job')

    def run(self):
        self.logger = log.get_logger('Job')
        self.job_api = JobApi()
        self.tasks_api = TasksApi()

        try:
            result = self.job_api.query({ 'job_id': self.options.job_id })
        except exceptions.ApiException as e:
            print e.message
            sys.exit(1)

        if self.options.spec:
            print json.dumps(result.spec, sort_keys=True, indent=4 * ' ')
        else:
            try:
                finished_tasks = self.tasks_api.query({
                    'job_id': self.options.job_id,
                    'state': 'SUCCESS'
                })
            except exceptions.ApiException as e:
                print e.message
                sys.exit(1)

            files_loaded = len(finished_tasks)
            rows_loaded = reduce(lambda x, y: x + y.get('data', {}).get('row_count', 0), finished_tasks, 0)
            avg_rows_per_file = None
            avg_rows_per_second = None

            if files_loaded > 0:
                avg_rows_per_file = rows_loaded / files_loaded

                min_start_time = datetime.datetime.max
                max_stop_time = datetime.datetime.min
                for row in finished_tasks:
                    for step in row.steps:
                        if step['name'] == 'download':
                            min_start_time = min(min_start_time, step['start'])
                            max_stop_time = max(max_stop_time, step['stop'])
                            break
                    else:
                        continue
                avg_rows_per_second = rows_loaded / (max_stop_time - min_start_time).total_seconds()

            result['stats'] = { k: v for k, v in {
                'files_loaded': files_loaded,
                'rows_loaded': rows_loaded,
                'avg_rows_per_file': avg_rows_per_file,
                'avg_rows_per_second': avg_rows_per_second
            }.iteritems() if v is not None }

            if result.tasks_total > 0:
                result['stats'].update({
                    'success_rate': result.tasks_succeeded * 1.0 / result.tasks_total,
                    'error_rate': result.tasks_errored * 1.0 / result.tasks_total
                })

            result["database"] = result.spec["target"]["database"]
            result["table"] = result.spec["target"]["table"]

            result = dict(result)
            del result['spec']

            result = { k: str(v) if isinstance(v, SuperEnum.Element) else v for k, v in result.iteritems() }
            print json.dumps(result, sort_keys=True, indent=4 * ' ')
Example #4
0
class Tasks(Command):
    @staticmethod
    def configure(parser, subparsers):
        subparser = subparsers.add_parser('tasks', help='Show information about all child tasks of a job')
        subparser.set_defaults(command=Tasks)

        subparser.add_argument('job_id', type=str, nargs='?', default=None,
                               help='List tasks corresponding to this particular job.')

        subparser.add_argument('-s', '--state',
            choices=shared.TaskState.elements.keys(),
            type=lambda s: s.upper(),
            action='append',
            help='Only show tasks matching a particular set of states')

        subparser.add_argument('-o', '--order',
            choices=shared.SortDirection.elements.keys(),
            type=lambda s: s.upper(),
            default=shared.SortDirection.ASC,
            help='The order to display the results in')

        subparser.add_argument('-b', '--order-by', dest='order_by',
            choices=TasksApi.SORTABLE_COLUMNS,
            help='The column to sort the results by')

        subparser.add_argument('-l', '--page-size', type=int,
            help='The number of results to return')

        subparser.add_argument('-p', '--page', type=int,
            help='The page of results to return')

        subparser.add_argument('--json', action='store_true', default=False,
            help='Display results in JSON format')

        subparser.add_argument('--last-job', action='store_true', default=False,
            help='Show the tasks for the most recent job.')

    def run(self):
        self.logger = log.get_logger('Tasks')
        self.tasks_api = TasksApi()

        if not self.options.job_id and not self.options.last_job:
            print 'You must specify a job ID or use the --last-job option.'
            sys.exit(1)

        if self.options.last_job:
            jobs = Jobs()
            job_list = jobs.all()
            if not job_list:
                print 'No jobs found.'
                sys.exit(1)
            self.options.job_id = job_list[-1].id

        try:
            result = self.tasks_api.query({ k: v for k, v in {
                'job_id': self.options.job_id,
                'state': self.options.state,
                'order': self.options.order,
                'order_by': self.options.order_by,
                'page_size': self.options.page_size,
                'page': self.options.page,
            }.iteritems() if v })
        except exceptions.ApiException as e:
            print e.message
            sys.exit(1)

        if result:
            tablefmt = TableFormat.JSON if self.options.json else TableFormat.TABLE
            print PrettyPrinter(result, columns=TasksApi.SORTABLE_COLUMNS, format=tablefmt).format()
        else:
            print 'No tasks found that match this query'
            sys.exit(1)