コード例 #1
0
def cleanup_old_jobs_pipelines():
    """Clean the pipelines of old jobs."""
    num_cleaned = 0
    max_age_msec = (
        MAX_MAPREDUCE_METADATA_RETENTION_MSECS + 7 * 24 * 60 * 60 * 1000)
    # Only consider jobs that started at most 1 week before recency_msec.
    # The latest start time that a job scheduled for cleanup may have.
    max_start_time_msec = (
        utils.get_current_time_in_millisecs() -
        MAX_MAPREDUCE_METADATA_RETENTION_MSECS
    )
    # Get all pipeline ids from jobs that started between max_age_msecs
    # and max_age_msecs + 1 week, before now.
    pipeline_id_to_job_instance = {}

    job_instances = job_models.JobModel.get_recent_jobs(1000, max_age_msec)
    for job_instance in job_instances:
        if (
                job_instance.time_started_msec < max_start_time_msec and
                not job_instance.has_been_cleaned_up
        ):
            if 'root_pipeline_id' in job_instance.metadata:
                pipeline_id = job_instance.metadata['root_pipeline_id']
                pipeline_id_to_job_instance[pipeline_id] = job_instance

    # Clean up pipelines.
    for pline in pipeline.get_root_list()['pipelines']:
        pipeline_id = pline['pipelineId']
        job_definitely_terminated = (
            pline['status'] == 'done' or
            pline['status'] == 'aborted' or
            pline['currentAttempt'] > pline['maxAttempts']
        )
        have_start_time = 'startTimeMs' in pline
        job_started_too_long_ago = (
            have_start_time and
            pline['startTimeMs'] < max_start_time_msec
        )

        if (job_started_too_long_ago or
                (not have_start_time and job_definitely_terminated)):
            # At this point, the map/reduce pipeline is either in a
            # terminal state, or has taken so long that there's no
            # realistic possibility that there might be a race condition
            # between this and the job actually completing.
            if pipeline_id in pipeline_id_to_job_instance:
                job_instance = pipeline_id_to_job_instance[pipeline_id]
                job_instance.has_been_cleaned_up = True
                job_instance.update_timestamps()
                job_instance.put()

            # This enqueues a deferred cleanup item.
            p = pipeline.Pipeline.from_id(pipeline_id)
            if p:
                p.cleanup()
                num_cleaned += 1

    logging.warning('%s MR jobs cleaned up.' % num_cleaned)
コード例 #2
0
ファイル: model_analytics.py プロジェクト: thejeshgn/seek
 def _get_cloudstore_paths(self, course_name):
     ret = set()
     with common_utils.Namespace('ns_' + course_name):
         for state in pipeline.get_root_list()['pipelines']:
             root_key = db.Key.from_path(
                 pipeline_models._PipelineRecord.kind(), state['pipelineId'])
             paths = (mapreduce_module.CronMapreduceCleanupHandler
                      ._collect_cloudstore_paths(root_key))
             ret = ret.union(paths)
     return ret
コード例 #3
0
 def _get_cloudstore_paths(self, course_name):
     ret = set()
     with common_utils.Namespace('ns_' + course_name):
         for state in pipeline.get_root_list()['pipelines']:
             root_key = db.Key.from_path(
                 pipeline_models._PipelineRecord.kind(), state['pipelineId'])
             paths = (mapreduce_module.CronMapreduceCleanupHandler
                      ._collect_cloudstore_paths(root_key))
             ret = ret.union(paths)
     return ret
コード例 #4
0
    def get(self):
        """Clean up intermediate data items for completed M/R jobs that
        started more than MAX_MAPREDUCE_METADATA_RETENTION_MSECS milliseconds
        ago.

        Map/reduce runs leave around a large number of rows in several
        tables.  This data is useful to have around for a while:
        - it helps diagnose any problems with jobs that may be occurring
        - it shows where resource usage is occurring
        However, after a few days, this information is less relevant, and
        should be cleaned up.
        """
        recency_msec = MAX_MAPREDUCE_METADATA_RETENTION_MSECS

        num_cleaned = 0

        min_age_msec = recency_msec
        # Only consider jobs that started at most 1 week before recency_msec.
        max_age_msec = recency_msec + 7 * 24 * 60 * 60 * 1000
        # The latest start time that a job scheduled for cleanup may have.
        max_start_time_msec = (utils.get_current_time_in_millisecs() -
                               min_age_msec)

        # Get all pipeline ids from jobs that started between max_age_msecs
        # and max_age_msecs + 1 week, before now.
        pipeline_id_to_job_instance = {}

        job_instances = job_models.JobModel.get_recent_jobs(1000, max_age_msec)
        for job_instance in job_instances:
            if (job_instance.time_started_msec < max_start_time_msec
                    and not job_instance.has_been_cleaned_up):
                if 'root_pipeline_id' in job_instance.metadata:
                    pipeline_id = job_instance.metadata['root_pipeline_id']
                    pipeline_id_to_job_instance[pipeline_id] = job_instance

        # Clean up pipelines.
        for pline in pipeline.get_root_list()['pipelines']:
            pipeline_id = pline['pipelineId']
            job_definitely_terminated = (
                pline['status'] == 'done' or pline['status'] == 'aborted'
                or pline['currentAttempt'] > pline['maxAttempts'])
            have_start_time = 'startTimeMs' in pline
            job_started_too_long_ago = (
                have_start_time and pline['startTimeMs'] < max_start_time_msec)

            if (job_started_too_long_ago
                    or (not have_start_time and job_definitely_terminated)):
                # At this point, the map/reduce pipeline is either in a
                # terminal state, or has taken so long that there's no
                # realistic possibility that there might be a race condition
                # between this and the job actually completing.
                if pipeline_id in pipeline_id_to_job_instance:
                    job_instance = pipeline_id_to_job_instance[pipeline_id]
                    job_instance.has_been_cleaned_up = True
                    job_instance.put()

                # This enqueues a deferred cleanup item.
                p = pipeline.Pipeline.from_id(pipeline_id)
                if p:
                    p.cleanup()
                    num_cleaned += 1

        logging.warning('%s MR jobs cleaned up.' % num_cleaned)

        if job_models.JobModel.do_unfinished_jobs_exist(
                jobs.JobCleanupManager.__name__):
            logging.warning('A previous cleanup job is still running.')
        else:
            jobs.JobCleanupManager.enqueue(
                jobs.JobCleanupManager.create_new(),
                additional_job_params={
                    jobs.MAPPER_PARAM_MAX_START_TIME_MSEC: max_start_time_msec
                })
            logging.warning('Deletion jobs for auxiliary entities kicked off.')
コード例 #5
0
    def _clean_mapreduce(cls, max_age):
        """Separated as internal function to permit tests to pass max_age."""
        num_cleaned = 0

        # If job has a start time before this, it has been running too long.
        min_start_time_datetime = datetime.datetime.utcnow() - max_age
        min_start_time_millis = int(
            (min_start_time_datetime -
             datetime.datetime(1970, 1, 1)).total_seconds() * 1000)

        # Iterate over all namespaces in the installation
        for course_context in sites.get_all_courses():
            with Namespace(course_context.get_namespace_name()):

                # Index map/reduce jobs in this namespace by pipeline ID.
                jobs_by_pipeline_id = {}
                for job_class in data_sources.Registry.get_generator_classes():
                    if issubclass(job_class, jobs.MapReduceJob):
                        job = job_class(course_context)
                        pipe_id = jobs.MapReduceJob.get_root_pipeline_id(
                            job.load())
                        jobs_by_pipeline_id[pipe_id] = job

                # Clean up pipelines
                for state in pipeline.get_root_list()['pipelines']:
                    pipeline_id = state['pipelineId']
                    job_definitely_terminated = (
                        state['status'] == 'done'
                        or state['status'] == 'aborted'
                        or state['currentAttempt'] > state['maxAttempts'])
                    have_start_time = 'startTimeMs' in state
                    job_started_too_long_ago = (
                        have_start_time
                        and state['startTimeMs'] < min_start_time_millis)

                    if (job_started_too_long_ago or
                        (not have_start_time and job_definitely_terminated)):
                        # At this point, the map/reduce pipeline is
                        # either in a terminal state, or has taken so long
                        # that there's no realistic possibility that there
                        # might be a race condition between this and the
                        # job actually completing.
                        if pipeline_id in jobs_by_pipeline_id:
                            jobs_by_pipeline_id[pipeline_id].mark_cleaned_up()

                        p = pipeline.Pipeline.from_id(pipeline_id)
                        if p:
                            # Pipeline cleanup, oddly, does not go clean up
                            # relevant blobstore items.  They have a TODO,
                            # but it has not been addressed as of Sep 2014.
                            # pylint: disable=protected-access
                            root_key = db.Key.from_path(
                                pipeline_models._PipelineRecord.kind(),
                                pipeline_id)
                            cls._clean_cloudstore_paths_for_key(
                                root_key, min_start_time_millis)

                            # This only enqueues a deferred cleanup item, so
                            # transactionality with marking the job cleaned is
                            # not terribly important.
                            p.cleanup()
                        num_cleaned += 1
        return num_cleaned
コード例 #6
0
 def _get_num_root_jobs(self, course_name):
     with common_utils.Namespace('ns_' + course_name):
         return len(pipeline.get_root_list()['pipelines'])
コード例 #7
0
 def _get_num_root_jobs(self, course_name):
     with common_utils.Namespace('ns_' + course_name):
         return len(pipeline.get_root_list()['pipelines'])
コード例 #8
0
    def _clean_mapreduce(cls, max_age):
        """Separated as internal function to permit tests to pass max_age."""
        num_cleaned = 0

        # If job has a start time before this, it has been running too long.
        min_start_time_datetime = datetime.datetime.utcnow() - max_age
        min_start_time_millis = int(
            (min_start_time_datetime - datetime.datetime(1970, 1, 1))
            .total_seconds() * 1000)

        # Iterate over all namespaces in the installation
        for course_context in sites.get_all_courses():
            with Namespace(course_context.get_namespace_name()):

                # Index map/reduce jobs in this namespace by pipeline ID.
                jobs_by_pipeline_id = {}
                for job_class in data_sources.Registry.get_generator_classes():
                    if issubclass(job_class, jobs.MapReduceJob):
                        job = job_class(course_context)
                        pipe_id = jobs.MapReduceJob.get_root_pipeline_id(
                            job.load())
                        jobs_by_pipeline_id[pipe_id] = job

                # Clean up pipelines
                for state in pipeline.get_root_list()['pipelines']:
                    pipeline_id = state['pipelineId']
                    job_definitely_terminated = (
                        state['status'] == 'done' or
                        state['status'] == 'aborted' or
                        state['currentAttempt'] > state['maxAttempts'])
                    have_start_time = 'startTimeMs' in state
                    job_started_too_long_ago = (
                        have_start_time and
                        state['startTimeMs'] < min_start_time_millis)

                    if (job_started_too_long_ago or
                        (not have_start_time and job_definitely_terminated)):
                        # At this point, the map/reduce pipeline is
                        # either in a terminal state, or has taken so long
                        # that there's no realistic possibility that there
                        # might be a race condition between this and the
                        # job actually completing.
                        if pipeline_id in jobs_by_pipeline_id:
                            jobs_by_pipeline_id[pipeline_id].mark_cleaned_up()

                        p = pipeline.Pipeline.from_id(pipeline_id)
                        if p:
                            # Pipeline cleanup, oddly, does not go clean up
                            # relevant blobstore items.  They have a TODO,
                            # but it has not been addressed as of Sep 2014.
                            # pylint: disable=protected-access
                            root_key = db.Key.from_path(
                                pipeline_models._PipelineRecord.kind(),
                                pipeline_id)
                            cls._clean_cloudstore_paths_for_key(
                                root_key, min_start_time_millis)

                            # This only enqueues a deferred cleanup item, so
                            # transactionality with marking the job cleaned is
                            # not terribly important.
                            p.cleanup()
                        num_cleaned += 1
        return num_cleaned
コード例 #9
0
ファイル: cron.py プロジェクト: DSeanLaw/oppia
    def get(self):
        """Clean up intermediate data items for completed M/R jobs that
        started more than MAX_MAPREDUCE_METADATA_RETENTION_MSECS milliseconds
        ago.

        Map/reduce runs leave around a large number of rows in several
        tables.  This data is useful to have around for a while:
        - it helps diagnose any problems with jobs that may be occurring
        - it shows where resource usage is occurring
        However, after a few days, this information is less relevant, and
        should be cleaned up.
        """
        recency_msec = MAX_MAPREDUCE_METADATA_RETENTION_MSECS

        num_cleaned = 0

        min_age_msec = recency_msec
        # Only consider jobs that started at most 1 week before recency_msec.
        max_age_msec = recency_msec + 7 * 24 * 60 * 60 * 1000
        # The latest start time that a job scheduled for cleanup may have.
        max_start_time_msec = (
            utils.get_current_time_in_millisecs() - min_age_msec)

        # Get all pipeline ids from jobs that started between max_age_msecs
        # and max_age_msecs + 1 week, before now.
        pipeline_id_to_job_instance = {}

        job_instances = job_models.JobModel.get_recent_jobs(1000, max_age_msec)
        for job_instance in job_instances:
            if (job_instance.time_started_msec < max_start_time_msec and not
                    job_instance.has_been_cleaned_up):
                if 'root_pipeline_id' in job_instance.metadata:
                    pipeline_id = job_instance.metadata['root_pipeline_id']
                    pipeline_id_to_job_instance[pipeline_id] = job_instance

        # Clean up pipelines.
        for pline in pipeline.get_root_list()['pipelines']:
            pipeline_id = pline['pipelineId']
            job_definitely_terminated = (
                pline['status'] == 'done' or
                pline['status'] == 'aborted' or
                pline['currentAttempt'] > pline['maxAttempts'])
            have_start_time = 'startTimeMs' in pline
            job_started_too_long_ago = (
                have_start_time and
                pline['startTimeMs'] < max_start_time_msec)

            if (job_started_too_long_ago or
                    (not have_start_time and job_definitely_terminated)):
                # At this point, the map/reduce pipeline is either in a
                # terminal state, or has taken so long that there's no
                # realistic possibility that there might be a race condition
                # between this and the job actually completing.
                if pipeline_id in pipeline_id_to_job_instance:
                    job_instance = pipeline_id_to_job_instance[pipeline_id]
                    job_instance.has_been_cleaned_up = True
                    job_instance.put()

                # This enqueues a deferred cleanup item.
                p = pipeline.Pipeline.from_id(pipeline_id)
                if p:
                    p.cleanup()
                    num_cleaned += 1

        logging.warning('%s MR jobs cleaned up.' % num_cleaned)

        if job_models.JobModel.do_unfinished_jobs_exist(
                jobs.JobCleanupManager.__name__):
            logging.warning('A previous cleanup job is still running.')
        else:
            jobs.JobCleanupManager.enqueue(
                jobs.JobCleanupManager.create_new(), additional_job_params={
                    jobs.MAPPER_PARAM_MAX_START_TIME_MSEC: max_start_time_msec
                })
            logging.warning('Deletion jobs for auxiliary entities kicked off.')