コード例 #1
0
ファイル: cache_utils.py プロジェクト: lyttam/boac
def refresh_request_handler(term_id, load_only=False):
    """Handle a start refresh admin request by returning an error status or starting the job on a background thread."""
    job_state = JobProgress().get()
    if job_state is None or (not job_state['start']) or job_state['end']:
        job_type = 'Load' if load_only else 'Refresh'
        JobProgress().start({
            'job_type': job_type,
            'term_id': term_id,
        })
        app.logger.warn('About to start background thread')
        thread = Thread(
            target=background_thread_refresh,
            daemon=True,
            kwargs={
                'app_arg': app._get_current_object(),
                'term_id': term_id,
                'job_type': job_type,
            },
        )
        thread.start()
        return {
            'progress': JobProgress().get(),
        }
    else:
        return {
            'error': 'Cannot start a new refresh job',
            'progress': job_state,
        }
コード例 #2
0
 def test_start_and_end(self):
     assert JobProgress().get() is None
     progress = JobProgress().start()
     assert progress['start'].startswith(str(date.today()))
     assert progress['end'] is None
     progress = JobProgress().end()
     assert progress['end'].startswith(str(date.today()))
コード例 #3
0
def continue_request_handler():
    """Continue an interrupted cache refresh or load job (skipping the optional ASC import)."""
    # WARNING: There is currently no protection against duplicate continuation requests. Admins need to ensure
    # that the refresh job is currently inactive.

    job_state = JobProgress().get()
    if job_state and job_state['start'] and not job_state['end']:
        term_id = job_state['term_id']
        thread = Thread(
            target=background_thread_refresh,
            daemon=True,
            kwargs={
                'app_arg': app._get_current_object(),
                'term_id': term_id,
                'continuation': True,
            },
        )
        thread.start()
        return {
            'progress': JobProgress().get(),
        }
    else:
        return {
            'error': 'Cannot continue this refresh job',
            'progress': job_state,
        }
コード例 #4
0
ファイル: test_job_progress.py プロジェクト: ssilverm/boac
 def test_start_with_stored_properties(self):
     assert JobProgress().get() is None
     JobProgress().start({'job_type': 'refresh', 'term_id': '2178'})
     progress = JobProgress().get()
     assert progress['start'].startswith(str(date.today()))
     assert progress['job_type'] == 'refresh'
     assert progress['term_id'] == '2178'
コード例 #5
0
def refresh_request_handler(term_id, load_only=False):
    """Handle a start refresh admin request by returning an error status or starting the job on a background thread."""
    job_state = JobProgress().get()

    if job_state and job_state['start'] and not job_state['end']:
        app.logger.error(
            f'Previous refresh job did not finish normally: {job_state}')
        JobProgress().delete()

    job_type = 'Load' if load_only else 'Refresh'
    JobProgress().start({
        'job_type': job_type,
        'term_id': term_id,
    })
    app.logger.warn('About to start background thread')
    thread = Thread(
        target=background_thread_refresh,
        daemon=True,
        kwargs={
            'app_arg': app._get_current_object(),
            'term_id': term_id,
            'job_type': job_type,
        },
    )
    thread.start()
    return {
        'progress': JobProgress().get(),
    }
コード例 #6
0
ファイル: cache_utils.py プロジェクト: lyttam/boac
def cancel_refresh_in_progress(term_id):
    progress = JobProgress().get()
    if progress and progress['job_type'] == 'Refresh':
        # Drop the staging table.
        json_cache.drop_staging_table()
    progress = JobProgress().delete()
    return {
        'progressDeleted': progress,
    }
コード例 #7
0
 def test_updates_a_started_job(self):
     assert JobProgress().start()
     progress = JobProgress().update('First step')
     assert len(progress['steps']) == 1
     assert progress['steps'][0].startswith(str(date.today()))
     assert progress['steps'][0].endswith('First step')
     progress = JobProgress().update('Next step')
     assert len(progress['steps']) == 2
     assert progress['steps'][1].endswith('Next step')
コード例 #8
0
def background_thread_refresh(app_arg, term_id):
    with app_arg.app_context():
        try:
            load_term(term_id)
            JobProgress().end()
        except Exception as e:
            app.logger.exception(e)
            app.logger.error('Background thread is stopping')
            JobProgress().update(f'An unexpected error occured: {e}')
            raise e
コード例 #9
0
def load_all_terms():
    job_progress = JobProgress().get()
    terms_done = job_progress.get('terms_done', [])
    all_terms = berkeley.all_term_ids()
    while terms_done != all_terms:
        if len(terms_done) == len(all_terms):
            app.logger.error(f'Unexpected terms_done value; stopping load: {terms_done}')
            return
        term_id = next(t for t in all_terms if t not in terms_done)
        load_term(term_id)
        terms_done.append(term_id)
        JobProgress().update(f'Term {term_id} loaded', properties={'terms_done': terms_done})
コード例 #10
0
ファイル: cache_utils.py プロジェクト: lyttam/boac
def background_thread_refresh(app_arg, term_id, job_type, continuation=False):
    with app_arg.app_context():
        try:
            if job_type == 'Refresh':
                refresh_term(term_id, continuation)
            else:
                load_term(term_id)
            JobProgress().end()
        except Exception as e:
            app.logger.exception(e)
            app.logger.error('Background thread is stopping')
            JobProgress().update(f'An unexpected error occured: {e}')
            raise e
コード例 #11
0
ファイル: cache_utils.py プロジェクト: lyttam/boac
def load_term(term_id=berkeley.current_term_id()):
    if term_id == 'all':
        load_all_terms()
        return

    JobProgress().update(f'About to refresh alerts for term {term_id}')
    refresh_alerts(term_id)

    if term_id == berkeley.current_term_id():
        JobProgress().update(f'About to load filtered cohort counts')
        load_filtered_cohort_counts()
        JobProgress().update(f'About to update curated cohort memberships')
        update_curated_cohort_lists()
コード例 #12
0
ファイル: cache_utils.py プロジェクト: lyttam/boac
def refresh_term(term_id=berkeley.current_term_id(), continuation=False):
    if not continuation or not json_cache.staging_table_exists():
        JobProgress().update(f'About to drop/create staging table')
        json_cache.drop_staging_table()
        json_cache.create_staging_table(exclusions_for_term(term_id))
    json_cache.set_staging(True)
    load_term(term_id)
    JobProgress().update(f'About to refresh from staging table')
    refresh_count = json_cache.refresh_from_staging(inclusions_for_term(term_id))
    # TODO Currently we're not looping anything into the staging table, so we expect refresh count to be zero.
    # If a more considered set of cache entries comes back into the loop, this error message should come back
    # too.
    # if refresh_count == 0:
    #     JobProgress().update('ERROR: No cache entries copied from staging')
    JobProgress().update(f'{refresh_count} cache entries copied from staging')
コード例 #13
0
def load_term(term_id=berkeley.current_term_id()):
    if term_id == 'all':
        load_all_terms()
        return

    JobProgress().update(f'About to refresh alerts for term {term_id}')
    refresh_alerts(term_id)

    if term_id == berkeley.current_term_id():
        JobProgress().update(f'About to refresh department memberships')
        refresh_department_memberships()
        JobProgress().update(f'About to refresh CalNet attributes for active users')
        refresh_calnet_attributes()
        JobProgress().update(f'About to load filtered cohort counts')
        load_filtered_cohort_counts()
        JobProgress().update(f'About to update curated group memberships')
        update_curated_group_lists()
コード例 #14
0
def clear_cachejob():
    progress = JobProgress().delete()
    return tolerant_jsonify({
        'progressDeleted': progress,
    })
コード例 #15
0
def get_cachejob_status():
    progress = JobProgress().get()
    return tolerant_jsonify({
        'progress': progress,
    })
コード例 #16
0
 def test_multiple_job_names(self):
     assert JobProgress('alphonse').start()
     assert JobProgress('alphonse').start() is False
     assert JobProgress('gaston').start()
     assert JobProgress('alphonse').delete()
     assert JobProgress('gaston').start() is False
コード例 #17
0
 def test_delete_job(self):
     assert JobProgress().start()
     assert JobProgress().start() is False
     assert JobProgress().delete()
     assert JobProgress().start()
コード例 #18
0
 def test_does_not_update_an_ended_job(self):
     assert JobProgress().start()
     assert JobProgress().end()
     progress = JobProgress().update('Into the wall')
     assert progress is False
コード例 #19
0
 def test_does_not_update_an_unstarted_job(self):
     assert JobProgress().get() is None
     progress = JobProgress().update('False step')
     assert progress is False