def add_timestamps(self, item):
"""Adds firstcreated and versioncreated timestamps
:param item:
"""
item['firstcreated'] = utc.localize(item['firstcreated']) if item.get('firstcreated') else utcnow()
item['versioncreated'] = utc.localize(item['versioncreated']) if item.get('versioncreated') else utcnow()
def add_timestamps(self, item):
"""
Adds _created, firstcreated, versioncreated and _updated timestamps
:param item:
:return:
"""
item['firstcreated'] = utc.localize(item['firstcreated'])
item['versioncreated'] = utc.localize(item['versioncreated'])
def add_timestamps(self, item):
"""
Adds _created, firstcreated, versioncreated and _updated timestamps
:param item:
:return:
"""
item['firstcreated'] = utc.localize(item['firstcreated'])
item['versioncreated'] = utc.localize(item['versioncreated'])
def fetch_ingest(self, guid):
items = self.get_items(guid)
result_items = []
while items:
item = items.pop()
result_items.append(item)
item['_created'] = item['firstcreated'] = utc.localize(item['firstcreated'])
item['_updated'] = item['versioncreated'] = utc.localize(item['versioncreated'])
items.extend(self.fetch_assets(item))
return result_items
def add_timestamps(self, item):
"""Adds firstcreated and versioncreated timestamps
:param item:
"""
item['firstcreated'] = utc.localize(
item['firstcreated']) if item.get('firstcreated') else utcnow()
item['versioncreated'] = utc.localize(
item['versioncreated']) if item.get(
'versioncreated') else utcnow()
def fetch_ingest(self, guid):
items = self.get_items(guid)
result_items = []
while items:
item = items.pop()
result_items.append(item)
item['_created'] = item['firstcreated'] = utc.localize(
item['firstcreated'])
item['_updated'] = item['versioncreated'] = utc.localize(
item['versioncreated'])
items.extend(self.fetch_assets(item))
return result_items
def archive_item(self, guid, provider_id, user, trigger_events, task_id=None):
try:
# For CELERY_ALWAYS_EAGER=True the current request context is
# empty but already initialized one is on request_stack
if app.config["CELERY_ALWAYS_EAGER"]:
self.request_stack.pop()
crt_task_id = self.request.id
if not task_id:
task_id = crt_task_id
if not self.request.retries:
update_status(*add_subtask_to_progress(task_id))
provider = superdesk.apps["ingest_providers"].find_one(req=None, _id=provider_id)
if provider is None:
message = "For ingest with guid= %s, failed to retrieve provider with _id=%s" % (guid, provider_id)
raise_fail(task_id, message)
service_provider = superdesk.io.providers[provider.get("type")]
service_provider.provider = provider
item = None
old_item = False
try:
items = service_provider.get_items(guid)
except LookupError:
ingest_doc = superdesk.apps["ingest"].find_one(req=None, _id=guid)
if not ingest_doc:
message = "Not found the ingest with guid: %s for provider %s" % (guid, provider.get("type"))
raise_fail(task_id, message)
else:
old_item = True
ingest_doc.pop("_id")
items = [ingest_doc]
except Exception:
raise self.retry(countdown=2)
for item_it in items:
if "guid" in item_it and item_it["guid"] == guid:
item = item_it
break
if item is None:
message = "Returned ingest but not found the ingest with guid: %s for provider %s" % (
guid,
provider.get("type"),
)
raise_fail(task_id, message)
if not old_item:
item["created"] = item["firstcreated"] = utc.localize(item["firstcreated"])
item["updated"] = item["versioncreated"] = utc.localize(item["versioncreated"])
superdesk.apps["archive"].update(guid, item, trigger_events=trigger_events)
tasks = []
for group in item.get("groups", []):
for ref in group.get("refs", []):
if "residRef" in ref:
doc = {
"guid": ref.get("residRef"),
"ingest_provider": provider_id,
"user": user,
"task_id": crt_task_id,
}
archived_doc = superdesk.apps["archive"].find_one(req=None, guid=doc.get("guid"))
# check if task already started
if not archived_doc:
doc.setdefault("_id", doc.get("guid"))
superdesk.apps["archive"].create([doc], trigger_events=trigger_events)
elif archived_doc.get("task_id") == crt_task_id:
# it is a retry so continue
archived_doc.update(doc)
superdesk.apps["archive"].update(
archived_doc.get("_id"), archived_doc, trigger_events=trigger_events
)
else:
# there is a cyclic dependency, skip it
continue
ingest_set_archived(doc.get("guid"))
tasks.append(archive_item.s(ref["residRef"], provider.get("_id"), user, trigger_events, task_id))
for rendition in item.get("renditions", {}).values():
href = service_provider.prepare_href(rendition["href"])
if rendition["rendition"] == "baseImage":
tasks.append(archive_media.s(task_id, guid, href, trigger_events))
else:
tasks.append(archive_rendition.s(task_id, guid, rendition["rendition"], href, trigger_events))
update_status(*finish_subtask_from_progress(task_id))
if tasks:
chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
elif task_id == crt_task_id:
update_status(*finish_task_for_progress(task_id))
except Exception:
logger.error(traceback.format_exc())
def archive_item(
self,
guid,
provider_id,
user,
trigger_events,
task_id=None,
):
try:
# For CELERY_ALWAYS_EAGER=True the current request context is
# empty but already initialized one is on request_stack
if app.config['CELERY_ALWAYS_EAGER']:
self.request_stack.pop()
crt_task_id = self.request.id
if not task_id:
task_id = crt_task_id
if not self.request.retries:
update_status(*add_subtask_to_progress(task_id))
provider = superdesk.apps['ingest_providers'].find_one(req=None,
_id=provider_id)
if provider is None:
message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (
guid, provider_id)
raise_fail(task_id, message)
service_provider = superdesk.io.providers[provider.get('type')]
service_provider.provider = provider
item = None
old_item = False
try:
items = service_provider.get_items(guid)
except LookupError:
ingest_doc = superdesk.apps['ingest'].find_one(req=None, _id=guid)
if not ingest_doc:
message = 'Not found the ingest with guid: %s for provider %s' % (
guid, provider.get('type'))
raise_fail(task_id, message)
else:
old_item = True
ingest_doc.pop('_id')
items = [ingest_doc]
except Exception:
raise self.retry(countdown=2)
for item_it in items:
if 'guid' in item_it and item_it['guid'] == guid:
item = item_it
break
if item is None:
message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
% (guid, provider.get('type'))
raise_fail(task_id, message)
if not old_item:
item['created'] = item['firstcreated'] = utc.localize(
item['firstcreated'])
item['updated'] = item['versioncreated'] = utc.localize(
item['versioncreated'])
superdesk.apps['archive'].update(guid,
item,
trigger_events=trigger_events)
tasks = []
for group in item.get('groups', []):
for ref in group.get('refs', []):
if 'residRef' in ref:
doc = {
'guid': ref.get('residRef'),
'ingest_provider': provider_id,
'user': user,
'task_id': crt_task_id
}
archived_doc = superdesk.apps['archive'].find_one(
req=None, guid=doc.get('guid'))
# check if task already started
if not archived_doc:
doc.setdefault('_id', doc.get('guid'))
superdesk.apps['archive'].create(
[doc], trigger_events=trigger_events)
elif archived_doc.get('task_id') == crt_task_id:
# it is a retry so continue
archived_doc.update(doc)
superdesk.apps['archive'].update(
archived_doc.get('_id'),
archived_doc,
trigger_events=trigger_events)
else:
# there is a cyclic dependency, skip it
continue
ingest_set_archived(doc.get('guid'))
tasks.append(
archive_item.s(ref['residRef'], provider.get('_id'),
user, trigger_events, task_id))
for rendition in item.get('renditions', {}).values():
href = service_provider.prepare_href(rendition['href'])
if rendition['rendition'] == 'baseImage':
tasks.append(
archive_media.s(task_id, guid, href, trigger_events))
else:
tasks.append(
archive_rendition.s(task_id, guid, rendition['rendition'],
href, trigger_events))
update_status(*finish_subtask_from_progress(task_id))
if tasks:
chord((task for task in tasks),
update_item.s(crt_task_id == task_id, task_id,
guid)).delay()
elif task_id == crt_task_id:
update_status(*finish_task_for_progress(task_id))
except Exception:
logger.error(traceback.format_exc())
def archive_item(self, guid, provider_id, user, task_id=None):
try:
# For CELERY_ALWAYS_EAGER=True the current request context is
# empty but already initialized one is on request_stack
if app.config['CELERY_ALWAYS_EAGER']:
self.request_stack.pop()
crt_task_id = self.request.id
if not task_id:
task_id = crt_task_id
if not self.request.retries:
update_status(*add_subtask_to_progress(task_id))
provider = superdesk.get_resource_service('ingest_providers').find_one(req=None, _id=provider_id)
if provider is None:
message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
raise_fail(task_id, message)
service_provider = superdesk.io.providers[provider.get('type')]
service_provider.provider = provider
item = None
old_item = False
try:
items = service_provider.get_items(guid)
except LookupError:
ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=guid)
if not ingest_doc:
message = 'Not found the ingest with guid: %s for provider %s' % (guid, provider.get('type'))
raise_fail(task_id, message)
else:
old_item = True
ingest_doc.pop('_id')
items = [ingest_doc]
except Exception:
raise self.retry(countdown=2)
for item_it in items:
if 'guid' in item_it and item_it['guid'] == guid:
item = item_it
break
if item is None:
message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
% (guid, provider.get('type'))
raise_fail(task_id, message)
if not old_item:
item['created'] = item['firstcreated'] = utc.localize(item['firstcreated'])
item['updated'] = item['versioncreated'] = utc.localize(item['versioncreated'])
'''
Necessary because flask.g.user is None while fetching packages the for grouped items or
while patching in archive collection. Without this version_creator is set None which doesn't make sense.
'''
flask.g.user = user
remove_unwanted(item)
superdesk.get_resource_service(ARCHIVE).patch(guid, item)
tasks = []
for group in item.get('groups', []):
for ref in group.get('refs', []):
if 'residRef' in ref:
resid_ref = ref.get('residRef')
doc = {'guid': resid_ref, 'ingest_provider': provider_id, 'task_id': crt_task_id}
archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, guid=doc.get('guid'))
# check if task already started
if not archived_doc:
doc.setdefault('_id', doc.get('guid'))
superdesk.get_resource_service(ARCHIVE).post([doc])
elif archived_doc.get('task_id') == crt_task_id:
# it is a retry so continue
archived_doc.update(doc)
remove_unwanted(archived_doc)
superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get('_id'), archived_doc)
else:
# there is a cyclic dependency, skip it
continue
mark_ingest_as_archived(doc.get('guid'))
tasks.append(archive_item.s(resid_ref, provider.get('_id'), user, task_id))
for rendition in item.get('renditions', {}).values():
href = service_provider.prepare_href(rendition['href'])
if rendition['rendition'] == 'baseImage':
tasks.append(archive_media.s(task_id, guid, href))
else:
tasks.append(archive_rendition.s(task_id, guid, rendition['rendition'], href))
update_status(*finish_subtask_from_progress(task_id))
if tasks:
chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
else:
insert_into_versions(guid, task_id)
if task_id == crt_task_id:
update_status(*finish_task_for_progress(task_id))
except Exception:
logger.error(traceback.format_exc())
def archive_item(self, guid, provider_id, user, trigger_events, task_id=None, ):
try:
# For CELERY_ALWAYS_EAGER=True the current request context is
# empty but already initialized one is on request_stack
if app.config['CELERY_ALWAYS_EAGER']:
self.request_stack.pop()
crt_task_id = self.request.id
if not task_id:
task_id = crt_task_id
if not self.request.retries:
update_status(*add_subtask_to_progress(task_id))
provider = superdesk.apps['ingest_providers'].find_one(req=None, _id=provider_id)
if provider is None:
message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
raise_fail(task_id, message)
service_provider = providers[provider.get('type')]
service_provider.provider = provider
item = None
old_item = False
try:
items = service_provider.get_items(guid)
except LookupError:
ingest_doc = superdesk.apps['ingest'].find_one(req=None, _id=guid)
if not ingest_doc:
message = 'Not found the ingest with guid: %s for provider %s' % (guid, provider.get('type'))
raise_fail(task_id, message)
else:
old_item = True
ingest_doc.pop('_id')
items = [ingest_doc]
except Exception:
raise self.retry(countdown=2)
for item_it in items:
if 'guid' in item_it and item_it['guid'] == guid:
item = item_it
break
if item is None:
message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
% (guid, provider.get('type'))
raise_fail(task_id, message)
if not old_item:
item['created'] = item['firstcreated'] = utc.localize(item['firstcreated'])
item['updated'] = item['versioncreated'] = utc.localize(item['versioncreated'])
superdesk.apps['archive'].update(guid, item, trigger_events=trigger_events)
tasks = []
for group in item.get('groups', []):
for ref in group.get('refs', []):
if 'residRef' in ref:
doc = {'guid': ref.get('residRef'), 'ingest_provider': provider_id,
'user': user, 'task_id': crt_task_id}
archived_doc = superdesk.apps['archive'].find_one(req=None, guid=doc.get('guid'))
# check if task already started
if not archived_doc:
doc.setdefault('_id', doc.get('guid'))
superdesk.apps['archive'].create([doc], trigger_events=trigger_events)
elif archived_doc.get('task_id') == crt_task_id:
# it is a retry so continue
archived_doc.update(doc)
superdesk.apps['archive'].update(archived_doc.get('_id'), archived_doc,
trigger_events=trigger_events)
else:
# there is a cyclic dependency, skip it
continue
ingest_set_archived(doc.get('guid'))
tasks.append(archive_item.s(ref['residRef'], provider.get('_id'), user, trigger_events, task_id))
for rendition in item.get('renditions', {}).values():
href = service_provider.prepare_href(rendition['href'])
if rendition['rendition'] == 'baseImage':
tasks.append(archive_media.s(task_id, guid, href, trigger_events))
else:
tasks.append(archive_rendition.s(task_id, guid, rendition['rendition'], href, trigger_events))
update_status(*finish_subtask_from_progress(task_id))
if tasks:
chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
elif task_id == crt_task_id:
update_status(*finish_task_for_progress(task_id))
except Exception:
logger.error(traceback.format_exc())
def archive_item(self, guid, provider_id, user, task_id=None):
try:
# For CELERY_ALWAYS_EAGER=True the current request context is
# empty but already initialized one is on request_stack
if app.config['CELERY_ALWAYS_EAGER']:
self.request_stack.pop()
crt_task_id = self.request.id
if not task_id:
task_id = crt_task_id
if not self.request.retries:
update_status(*add_subtask_to_progress(task_id))
provider = superdesk.get_resource_service('ingest_providers').find_one(
req=None, _id=provider_id)
if provider is None:
message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (
guid, provider_id)
raise_fail(task_id, message)
service_provider = superdesk.io.providers[provider.get('type')]
service_provider.provider = provider
item = None
old_item = False
try:
items = service_provider.get_items(guid)
except LookupError:
ingest_doc = superdesk.get_resource_service('ingest').find_one(
req=None, _id=guid)
if not ingest_doc:
message = 'Not found the ingest with guid: %s for provider %s' % (
guid, provider.get('type'))
raise_fail(task_id, message)
else:
old_item = True
ingest_doc.pop('_id')
items = [ingest_doc]
except Exception:
raise self.retry(countdown=2)
for item_it in items:
if 'guid' in item_it and item_it['guid'] == guid:
item = item_it
break
if item is None:
message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
% (guid, provider.get('type'))
raise_fail(task_id, message)
if not old_item:
item['created'] = item['firstcreated'] = utc.localize(
item['firstcreated'])
item['updated'] = item['versioncreated'] = utc.localize(
item['versioncreated'])
'''
Necessary because flask.g.user is None while fetching packages the for grouped items or
while patching in archive collection. Without this version_creator is set None which doesn't make sense.
'''
flask.g.user = user
remove_unwanted(item)
superdesk.get_resource_service(ARCHIVE).patch(guid, item)
tasks = []
for group in item.get('groups', []):
for ref in group.get('refs', []):
if 'residRef' in ref:
resid_ref = ref.get('residRef')
doc = {
'guid': resid_ref,
'ingest_provider': provider_id,
'task_id': crt_task_id
}
archived_doc = superdesk.get_resource_service(
ARCHIVE).find_one(req=None, guid=doc.get('guid'))
# check if task already started
if not archived_doc:
doc.setdefault('_id', doc.get('guid'))
superdesk.get_resource_service(ARCHIVE).post([doc])
elif archived_doc.get('task_id') == crt_task_id:
# it is a retry so continue
archived_doc.update(doc)
remove_unwanted(archived_doc)
superdesk.get_resource_service(ARCHIVE).patch(
archived_doc.get('_id'), archived_doc)
else:
# there is a cyclic dependency, skip it
continue
mark_ingest_as_archived(doc.get('guid'))
tasks.append(
archive_item.s(resid_ref, provider.get('_id'), user,
task_id))
for rendition in item.get('renditions', {}).values():
href = service_provider.prepare_href(rendition['href'])
if rendition['rendition'] == 'baseImage':
tasks.append(archive_media.s(task_id, guid, href))
else:
tasks.append(
archive_rendition.s(task_id, guid, rendition['rendition'],
href))
update_status(*finish_subtask_from_progress(task_id))
if tasks:
chord((task for task in tasks),
update_item.s(crt_task_id == task_id, task_id,
guid)).delay()
else:
insert_into_versions(guid, task_id)
if task_id == crt_task_id:
update_status(*finish_task_for_progress(task_id))
except Exception:
logger.error(traceback.format_exc())
