Python harvest_source_list Examples

Example #1

File: create.py Project: metaodi/ckanext-harvest

def harvest_job_create_all(context, data_dict):
    '''
    Creates a Harvest Job for all Harvest Sources and runs them (by
    putting them on the gather queue)

    :param source_id:
    :type param: string
    :param run: whether to also run the jobs or not (default: True)
    :type run: bool
    '''

    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all',context,data_dict)

    run = data_dict.get('run', True)

    data_dict.update({'only_active':True})

    # Get all active sources
    sources = harvest_source_list(context,data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        exists = _check_for_existing_jobs(context, source['id'])
        if exists:
            log.info('Skipping source %s as it already has a pending job', source['id'])
            continue

        job = harvest_job_create(
            context, {'source_id': source['id'], 'run': run})
        jobs.append(job)

    log.info('Created jobs for %s%i harvest sources',
             'and run ' if run else '', len(jobs))
    return jobs

Example #2

File: create.py Project: tbalaz/test

def harvest_job_create_all(context,data_dict):
    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all',context,data_dict)

    data_dict.update({'only_active':True})

    # Get all active sources
    sources = harvest_source_list(context,data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        data_dict ={
            'source_id':source['id'],
            'status':u'New'
        }

        exists = harvest_job_list(context,data_dict)
        if len(exists):
            continue

        job = harvest_job_create(context,{'source_id':source['id']})
        jobs.append(job)

    log.info('Created jobs for %i harvest sources', len(jobs))
    return jobs

Example #3

File: create.py Project: uk-gov-mirror/datagovuk.ckanext-harvest

def harvest_job_create_all(context,data_dict):
    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all',context,data_dict)

    data_dict.update({'only_active':True})

    # Get all active sources
    sources = harvest_source_list(context,data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        data_dict ={
            'source_id':source['id'],
            'status':u'New'
        }

        exists = harvest_job_list(context,data_dict)
        if len(exists):
            continue

        job = harvest_job_create(context,{'source_id':source['id']})
        jobs.append(job)

    log.info('Created jobs for %i harvest sources', len(jobs))
    return jobs

Example #4

File: create.py Project: CodeForAfricaLabs/ckanext-harvest

def harvest_job_create_all(context,data_dict):
    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all',context,data_dict)

    data_dict.update({'only_active':True})

    # Get all active sources
    sources = harvest_source_list(context,data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        exists = _check_for_existing_jobs(context, source['id'])
        if exists:
            log.info('Skipping source %s as it already has a pending job', source['id'])
            continue

        job = harvest_job_create(context,{'source_id':source['id']})
        jobs.append(job)

    log.info('Created jobs for %i harvest sources', len(jobs))
    return jobs

Example #5

File: create.py Project: ngds/ckanext-harvest

def harvest_job_create_all(context, data_dict):
    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all', context, data_dict)

    data_dict.update({'only_active': True})

    # Get all active sources
    sources = harvest_source_list(context, data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        exists = _check_for_existing_jobs(context, source['id'])
        if exists:
            log.info('Skipping source %s as it already has a pending job',
                     source['id'])
            continue

        job = harvest_job_create(context, {'source_id': source['id']})
        jobs.append(job)

    log.info('Created jobs for %i harvest sources', len(jobs))
    return jobs

Example #6

File: create.py Project: heungshik/ckanext-harvest

def harvest_job_create_all(context, data_dict):
    '''
    Creates a Harvest Job for all Harvest Sources and runs them (by
    putting them on the gather queue)

    :param source_id:
    :type param: string
    :param run: whether to also run the jobs or not (default: True)
    :type run: bool
    '''

    log.info('Harvest job create all: %r', data_dict)
    check_access('harvest_job_create_all', context, data_dict)

    run = data_dict.get('run', True)

    data_dict.update({'only_active': True})

    # Get all active sources
    sources = harvest_source_list(context, data_dict)
    jobs = []
    # Create a new job for each, if there isn't already one
    for source in sources:
        exists = _check_for_existing_jobs(context, source['id'])
        if exists:
            log.info('Skipping source %s as it already has a pending job',
                     source['id'])
            continue

        job = harvest_job_create(context, {
            'source_id': source['id'],
            'run': run
        })
        jobs.append(job)

    log.info('Created jobs for %s%i harvest sources',
             'and run ' if run else '', len(jobs))
    return jobs