def harvest_object_create(context, data_dict):
""" Create a new harvest object
:type guid: string (optional)
:type content: string (optional)
:type job_id: string
:type source_id: string (optional)
:type package_id: string (optional)
:type extras: dict (optional)
"""
check_access('harvest_object_create', context, data_dict)
data, errors = _validate(data_dict, harvest_object_create_schema(), context)
if errors:
raise logic.ValidationError(errors)
obj = HarvestObject(
guid=data.get('guid'),
content=data.get('content'),
job=data['job_id'],
harvest_source_id=data.get('source_id'),
package_id=data.get('package_id'),
extras=[ HarvestObjectExtra(key=k, value=v)
for k, v in data.get('extras', {}).items() ]
)
obj.save()
return harvest_object_dictize(obj, context)
def harvest_object_create(context, data_dict):
""" Create a new harvest object
:type guid: string (optional)
:type content: string (optional)
:type job_id: string
:type source_id: string (optional)
:type package_id: string (optional)
:type extras: dict (optional)
"""
check_access('harvest_object_create', context, data_dict)
data, errors = _validate(data_dict, harvest_object_create_schema(),
context)
if errors:
raise logic.ValidationError(errors)
obj = HarvestObject(guid=data.get('guid'),
content=data.get('content'),
job=data['job_id'],
harvest_source_id=data.get('source_id'),
package_id=data.get('package_id'),
extras=[
HarvestObjectExtra(key=k, value=v)
for k, v in data.get('extras', {}).items()
])
obj.save()
return harvest_object_dictize(obj, context)
def harvest_object_show(context, data_dict):
p.toolkit.check_access('harvest_object_show', context, data_dict)
id = data_dict.get('id')
dataset_id = data_dict.get('dataset_id')
if id:
attr = data_dict.get('attr', None)
obj = HarvestObject.get(id, attr=attr)
elif dataset_id:
model = context['model']
pkg = model.Package.get(dataset_id)
if not pkg:
raise p.toolkit.ObjectNotFound('Dataset not found')
obj = model.Session.query(HarvestObject) \
.filter(HarvestObject.package_id == pkg.id) \
.filter(
HarvestObject.current == True # noqa: E711
).first()
else:
raise p.toolkit.ValidationError(
'Please provide either an "id" or a "dataset_id" parameter')
if not obj:
raise p.toolkit.ObjectNotFound('Harvest object not found')
return harvest_object_dictize(obj, context)
def harvest_object_show(context,data_dict):
p.toolkit.check_access('harvest_object_show', context, data_dict)
id = data_dict.get('id')
dataset_id = data_dict.get('dataset_id')
if id:
attr = data_dict.get('attr',None)
obj = HarvestObject.get(id,attr=attr)
elif dataset_id:
model = context['model']
pkg = model.Package.get(dataset_id)
if not pkg:
raise p.toolkit.ObjectNotFound('Dataset not found')
obj = model.Session.query(HarvestObject) \
.filter(HarvestObject.package_id == pkg.id) \
.filter(HarvestObject.current == True) \
.first()
else:
raise p.toolkit.ValidationError(
'Please provide either an "id" or a "dataset_id" parameter')
if not obj:
raise p.toolkit.ObjectNotFound('Harvest object not found')
return harvest_object_dictize(obj, context)
def harvest_object_show(context, data_dict):
p.toolkit.check_access("harvest_object_show", context, data_dict)
id = data_dict.get("id")
dataset_id = data_dict.get("dataset_id")
if id:
attr = data_dict.get("attr", None)
obj = HarvestObject.get(id, attr=attr)
elif dataset_id:
model = context["model"]
pkg = model.Package.get(dataset_id)
if not pkg:
raise p.toolkit.ObjectNotFound("Dataset not found")
obj = (
model.Session.query(HarvestObject)
.filter(HarvestObject.package_id == pkg.id)
.filter(HarvestObject.current == True)
.first()
)
else:
raise p.toolkit.ValidationError('Please provide either an "id" or a "dataset_id" parameter')
if not obj:
raise p.toolkit.ObjectNotFound("Harvest object not found")
return harvest_object_dictize(obj, context)
def harvest_object_show(context,data_dict):
check_access('harvest_object_show',context,data_dict)
id = data_dict.get('id')
attr = data_dict.get('attr',None)
obj = HarvestObject.get(id,attr=attr)
if not obj:
raise NotFound
return harvest_object_dictize(obj,context)
def harvest_object_show(context, data_dict):
check_access('harvest_object_show', context, data_dict)
id = data_dict.get('id')
attr = data_dict.get('attr', None)
obj = HarvestObject.get(id, attr=attr)
if not obj:
raise NotFound
return harvest_object_dictize(obj, context)
def harvest_objects_import(context,data_dict):
'''
Reimports the current harvest objects
It performs the import stage with the last fetched objects, optionally
belonging to a certain source.
Please note that no objects will be fetched from the remote server.
It will only affect the last fetched objects already present in the
database.
'''
log.info('Harvest objects import: %r', data_dict)
check_access('harvest_objects_import',context,data_dict)
model = context['model']
session = context['session']
source_id = data_dict.get('source_id',None)
if source_id:
source = HarvestSource.get(source_id)
if not source:
log.error('Harvest source %s does not exist', source_id)
raise NotFound('Harvest source %s does not exist' % source_id)
if not source.active:
log.warn('Harvest source %s is not active.', source_id)
raise Exception('This harvest source is not active')
last_objects_ids = session.query(HarvestObject.id) \
.join(HarvestSource).join(Package) \
.filter(HarvestObject.source==source) \
.filter(HarvestObject.current==True) \
.filter(Package.state==u'active') \
.all()
else:
last_objects_ids = session.query(HarvestObject.id) \
.join(Package) \
.filter(HarvestObject.current==True) \
.filter(Package.state==u'active') \
.all()
last_objects = []
for obj_id in last_objects_ids:
obj = session.query(HarvestObject).get(obj_id)
for harvester in PluginImplementations(IHarvester):
if harvester.info()['name'] == obj.source.type:
if hasattr(harvester,'force_import'):
harvester.force_import = True
harvester.import_stage(obj)
break
last_objects.append(harvest_object_dictize(obj,context))
log.info('Harvest objects imported: %r', last_objects)
return last_objects
