def publish_resource(project_id, entity_uuid=None):
"""Publish a resource.
Retrieves a project and/or an entity and set any saved DOIs
as published. If no DOIs are saved in the specified project or entity
it will fail silently. We need to specify the project id because
this function also changes the status of the locally saved publication
to `"published"` that way it shows up in the published listing.
:param str project_id: Project Id to publish.
:param str entity_uuid: Entity uuid to publish.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
entity = None
if entity_uuid:
entity = mgr.get_entity_by_uuid(entity_uuid)
responses = []
for doi in prj.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
if entity:
for doi in entity.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
pub = BaseESPublication(project_id=project_id)
pub.update(status='published')
for res in responses:
LOG.info("DOI published: %(doi)s", {"doi": res['data']['id']})
return responses
def publish_resource(project_id, entity_uuids=None, publish_dois=False, revision=None):
"""Publish a resource.
Retrieves a project and/or an entity and set any saved DOIs
as published. If no DOIs are saved in the specified project or entity
it will fail silently. We need to specify the project id because
this function also changes the status of the locally saved publication
to `"published"` that way it shows up in the published listing.
If publish_dois is False Datacite will keep the newly created DOIs in
"DRAFT" status, and not "PUBLISHED". A DOI on DataCite can only be
deleted if it is in "DRAFT" status. Once a DOI is set to "PUBLISHED"
or "RESERVED" it can't be deleted.
:param str project_id: Project Id to publish.
:param list entity_uuids: list of str Entity uuids to publish.
:param int revision: Revision number to publish.
"""
es_client = new_es_client()
# If revision number passed, set status to "published" for specified revision and
# set status to "revised" for old versions
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
responses = []
if publish_dois:
if entity_uuids:
for ent_uuid in entity_uuids:
entity = None
if ent_uuid:
entity = mgr.get_entity_by_uuid(ent_uuid)
if entity:
for doi in entity.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
for doi in prj.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
pub = BaseESPublication(project_id=project_id, revision=revision, using=es_client)
pub.update(status='published', using=es_client)
IndexedPublication._index.refresh(using=es_client)
if revision:
# Revising a publication sets the status of the previous document to 'archived'
last_revision = revision - 1 if revision > 2 else 0
archived_pub = BaseESPublication(project_id=project_id, revision=last_revision)
archived_pub.update(status='archived')
for res in responses:
logger.info(
"DOI published: %(doi)s",
{"doi": res['data']['id']}
)
return responses
def freeze_project_and_entity_metadata(project_id, entity_uuids=None):
"""Freeze project and entity metadata.
Given a project id and an entity uuid (should be a main entity) this function
retrieves all metadata related to these entities and stores it into Elasticsearch
as :class:`~designafe.libs.elasticsearch.docs.publications.BaseESPublication`
:param str project_id: Project id.
:param list of entity_uuid strings: Entity uuids.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
pub_doc = BaseESPublication(project_id=project_id)
publication = pub_doc.to_dict()
if entity_uuids:
# clear any existing entities in publication
entity = mgr.get_entity_by_uuid(entity_uuids[0])
pub_entities_field_name = FIELD_MAP[entity.name]
publication[pub_entities_field_name] = []
for ent_uuid in entity_uuids:
entity = None
entity = mgr.get_entity_by_uuid(ent_uuid)
entity_json = entity.to_body_dict()
if entity:
pub_entities_field_name = FIELD_MAP[entity.name]
publication['authors'] = entity_json['value']['authors'][:]
entity_json['authors'] = []
_populate_entities_in_publication(entity, publication)
_transform_authors(entity_json, publication)
if entity_json['value']['dois']:
entity_json['doi'] = entity_json['value']['dois'][-1]
_delete_unused_fields(entity_json)
publication[pub_entities_field_name].append(entity_json)
prj_json = prj.to_body_dict()
_delete_unused_fields(prj_json)
award_number = publication.get('project', {}).get('value', {}).pop(
'awardNumber', []) or []
if not isinstance(award_number, list):
award_number = []
prj_json['value']['awardNumbers'] = award_number
prj_json['value'].pop('awardNumber', None)
if publication.get('project'):
publication['project'].update(prj_json)
else:
publication['project'] = prj_json
pub_doc.update(**publication)
return pub_doc
def amend_publication(project_id, amendments=None, authors=None, revision=None):
"""Amend a Publication
Update Amendable fields on a publication and the corrosponding DataCite
records. These changes do not produce a new version of a publication, but
they do allow for limited changes to a published project. This is currently
configured to support "Other" publications only.
:param str project_id: Project uuid to amend
:param int revision: Revision number to amend
"""
es_client = new_es_client()
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
pub = BaseESPublication(project_id=project_id, revision=revision, using=es_client)
prj_dict = prj.to_body_dict()
pub_dict = pub.to_dict()
_delete_unused_fields(prj_dict)
if pub.project.value.projectType != 'other':
pub_entity_uuids = pub.entities()
for uuid in pub_entity_uuids:
if uuid in amendments:
entity = amendments[uuid]
else:
entity = mgr.get_entity_by_uuid(uuid)
entity = entity.to_body_dict()
_delete_unused_fields(entity)
for pub_ent in pub_dict[FIELD_MAP[entity['name']]]:
if pub_ent['uuid'] == entity['uuid']:
for key in entity['value']:
ent_type = 'entity' if 'dois' in entity['value'] else 'subentity'
if key not in UNAMENDABLE_FIELDS[ent_type]:
pub_ent['value'][key] = entity['value'][key]
if 'authors' in entity['value']:
pub_ent['value']['authors'] = authors[entity['uuid']]
_set_authors(pub_ent, pub_dict)
# weird key swap for old issues with awardnumber(s)
award_number = prj.award_number or []
if not isinstance(award_number, list):
award_number = []
prj_dict['value']['awardNumbers'] = award_number
prj_dict['value'].pop('awardNumber', None)
for key in prj_dict['value']:
if key not in UNAMENDABLE_FIELDS['project']:
pub_dict['project']['value'][key] = prj_dict['value'][key]
if authors and prj_dict['value']['projectType'] == 'other':
pub_dict['project']['value']['teamOrder'] = authors
pub.update(**pub_dict)
IndexedPublication._index.refresh(using=es_client)
return pub
def publish_resource(project_id, entity_uuids=None, publish_dois=False):
"""Publish a resource.
Retrieves a project and/or an entity and set any saved DOIs
as published. If no DOIs are saved in the specified project or entity
it will fail silently. We need to specify the project id because
this function also changes the status of the locally saved publication
to `"published"` that way it shows up in the published listing.
If publish_dois is False Datacite will keep the newly created DOIs in
"DRAFT" status, but they will not be set to "PUBLISHED". A DOI on
DataCite can only be deleted if it is in "DRAFT" status. Once a DOI
is set to "PUBLISHED" or "RESERVED" it can't be deleted.
:param str project_id: Project Id to publish.
:param list entity_uuids: list of str Entity uuids to publish.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
responses = []
if publish_dois:
for ent_uuid in entity_uuids:
entity = None
if ent_uuid:
entity = mgr.get_entity_by_uuid(ent_uuid)
if entity:
for doi in entity.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
for doi in prj.dois:
res = DataciteManager.publish_doi(doi)
responses.append(res)
pub = BaseESPublication(project_id=project_id)
pub.update(status='published')
for res in responses:
LOG.info(
"DOI published: %(doi)s",
{"doi": res['data']['id']}
)
return responses
def draft_publication(
project_id,
main_entity_uuid=None,
project_doi=None,
main_entity_doi=None,
upsert_project_doi=False,
upsert_main_entity_doi=True,
):
"""Reserve a publication.
A publication is reserved by creating a DOI through Datacite.
For some of the projects a DOI is only created for the main entity
e.g. Mission or Simulation. For some other projects we also (or only)
get a DOI for the project.
- If :param:`project_doi` and/or :param:`main_entity_doi` values are given
then those dois will be updated (or created if they don't exist in datacite).
- If :param:`upsert_project_doi` and/or :param:`upsert_main_entity_doi` are
set to `True` then any saved DOIs will be updated (even if there's multiple
unless a specific DOI is given). If there are no saved DOIs then a new DOI
will be created. Meaning, it will act as update or insert.
- If :param:`project_id` is given **but** :param:`main_entity_uuid` is ``None``
then a project DOI will be created or updated.
.. warning:: This funciton only creates a *Draft* DOI and not a public one.
.. warning:: An entity *might* have multiple DOIs, if this is the case and
:param:`upsert_project_doi` or :param:`upsert_main_entity_doi` are set to True
then *all* saved dois will be updated.
.. note:: In theory a single resource *should not* have multiple DOIs
but we don't know how this will change in the future, hence, we are
supporting multiple DOIs.
.. note:: If no :param:`main_entity_uuid` is given then a project DOI will be
created.
:param str project_id: Project Id
:param str main_entity_uuid: Uuid of main entity.
:param str project_doi: Custom doi for project.
:param str main_entity_doi: Custom doi for main entity.
:param bool upsert_project_doi: Update or insert project doi.
:param bool upsert_main_entity_doi: Update or insert main entity doi.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
entity = None
if main_entity_uuid:
entity = mgr.get_entity_by_uuid(main_entity_uuid)
else:
upsert_project_doi = True
responses = []
prj_url = TARGET_BASE.format(project_id=project_id)
if entity:
entity_url = ENTITY_TARGET_BASE.format(project_id=project_id,
entity_uuid=main_entity_uuid)
prj_datacite_json = prj.to_datacite_json()
prj_datacite_json['url'] = prj_url
if entity:
ent_datacite_json = entity.to_datacite_json()
ent_datacite_json['url'] = entity_url
if upsert_project_doi and project_doi:
prj_res = DataciteManager.create_or_update_doi(prj_datacite_json,
project_doi)
prj.dois += [project_doi]
prj.dois = list(set(prj.dois))
prj.save(service_account())
responses.append(prj_res)
elif upsert_project_doi and prj.dois:
for doi in prj.dois:
prj_res = DataciteManager.create_or_update_doi(
prj_datacite_json, doi)
responses.append(prj_res)
elif upsert_project_doi and not prj.dois:
prj_res = DataciteManager.create_or_update_doi(prj_datacite_json)
prj.dois += [prj_res['data']['id']]
prj.save(service_account())
responses.append(prj_res)
if entity and upsert_main_entity_doi and main_entity_doi:
me_res = DataciteManager.create_or_update_doi(ent_datacite_json,
main_entity_doi)
entity.dois += [main_entity_doi]
entity.dois = list(set(entity.dois))
entity.save(service_account())
responses.append(me_res)
elif entity and upsert_main_entity_doi and entity.dois:
for doi in entity.dois:
me_res = DataciteManager.create_or_update_doi(
ent_datacite_json, doi)
responses.append(me_res)
elif entity and upsert_main_entity_doi and not entity.dois:
me_res = DataciteManager.create_or_update_doi(ent_datacite_json)
entity.dois += [me_res['data']['id']]
entity.save(service_account())
responses.append(me_res)
for res in responses:
LOG.info("DOI created or updated: %(doi)s", {"doi": res['data']['id']})
return responses
def freeze_project_and_entity_metadata(project_id, entity_uuids=None):
"""Freeze project and entity metadata.
Given a project id and an entity uuid (should be a main entity) this function
retrieves all metadata related to these entities and stores it into Elasticsearch
as :class:`~designafe.libs.elasticsearch.docs.publications.BaseESPublication`
When publishing for the first time or publishing over an existing publication. We
will clear any existing entities (if any) from the published metadata. We'll use entity_uuids
(the entities getting DOIs) to rebuild the rest of the publication. These entities
usually do not have files associated to them (except published reports/documents).
:param str project_id: Project id.
:param list of entity_uuid strings: Entity uuids.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
pub_doc = BaseESPublication(project_id=project_id)
publication = pub_doc.to_dict()
if entity_uuids:
# clear any existing sub entities in publication and keep updated fileObjs
fields_to_clear = []
entities_with_files = []
for key in list(FIELD_MAP.keys()):
if FIELD_MAP[key] in list(publication.keys()):
fields_to_clear.append(FIELD_MAP[key])
fields_to_clear = set(fields_to_clear)
for field in fields_to_clear:
for ent in publication[field]:
if 'fileObjs' in ent:
entities_with_files.append(ent)
if ent['uuid'] in entity_uuids:
publication[field] = []
for ent_uuid in entity_uuids:
entity = None
entity = mgr.get_entity_by_uuid(ent_uuid)
if entity:
entity_json = entity.to_body_dict()
pub_entities_field_name = FIELD_MAP[entity.name]
for e in entities_with_files:
if e['uuid'] == entity_json['uuid']:
entity_json['fileObjs'] = e['fileObjs']
publication['authors'] = list(entity_json['value']['authors'])
entity_json['authors'] = []
_populate_entities_in_publication(entity, publication)
_transform_authors(entity_json, publication)
if entity_json['value']['dois']:
entity_json['doi'] = entity_json['value']['dois'][-1]
_delete_unused_fields(entity_json)
publication[pub_entities_field_name].append(entity_json)
prj_json = prj.to_body_dict()
_delete_unused_fields(prj_json)
award_number = publication.get('project', {}).get('value', {}).pop(
'awardNumber', []
) or []
if not isinstance(award_number, list):
award_number = []
prj_json['value']['awardNumbers'] = award_number
prj_json['value'].pop('awardNumber', None)
if publication.get('project'):
publication['project'].update(prj_json)
else:
publication['project'] = prj_json
pub_doc.update(**publication)
return pub_doc
class NcoProjectsManager(object):
"""Nco Projects Manager."""
def __init__(self, user):
"""Initialize.
:param user: Django user instance.
"""
self.user = user
if user.is_authenticated:
self._ac = user.agave_oauth.client
else:
self._ac = service_account()
self._mp = MongoProjectsHelper(self._ac)
self._pm = ProjectsManager(self._ac)
def _process_time_filter(self, value):
"""Convert from time filter value to date range."""
date = datetime.utcnow()
def this_week():
"""Get this week range."""
start = date - timedelta(days=date.weekday())
end = start + timedelta(days=6)
return {"dateStart": {"$gt": start, "$lt": end}}
def this_month():
"""Get this month range."""
month_range = calendar.monthrange(date.year, date.month)
start = date - timedelta(days=date.day - 1)
end = start + timedelta(days=month_range[1] - 1)
return {"dateStart": {"$gt": start, "$lt": end}}
def last_n_days(days):
"""Last N days range."""
def last_range():
start = date - timedelta(days=days)
end = date
return {"dateStart": {"$gt": start, "$lt": end}}
return last_range
def in_n_days(days):
"""In N days range."""
def in_range():
start = date + timedelta(days=days)
end = date
return {"dateStart": {"$gt": start, "$lt": end}}
return in_range
range_switch = {
"Happening This Week": this_week,
"Happening This Month": this_month,
"Event Started Last 7 Days": last_n_days(7),
"Event Started Last 14 Days": last_n_days(14),
"Event Started Last 30 Days": last_n_days(30),
"Event Will Start in 7 Days": in_n_days(7),
"Event Will Start in 14 Days": in_n_days(14),
"Event Will Start in 30 Days": in_n_days(30),
}
return range_switch[value]()
def _process_filters(self, filters):
"""Create mongo query from filters dict.
:param lsit query: Query as key/value,
e.g. [{"name": "facility", "value": "facility name"}, ...]
"""
if not filters:
return {}
query = {}
for fil in filters:
if fil["name"] == "facilities":
query["facility"] = fil["value"]
elif fil["name"] == "locations":
query["location"] = fil["value"]
elif fil["name"] == "time":
query.update(self._process_time_filter(fil["value"]))
return query
def _process_sort(self, sorts):
"""Create mongo sort from sorting value.
:param list sorts: Sort as key/value.
e.g. [{"name": "sort key", "value": 1}, ...].
"""
sort_switch = {
"Project Id A-Z": ("projectId", 1),
"Event Title A-Z": ("eventTitle", 1),
"Event Start Newer First": ("dateStart", -1),
"Event Start Older First": ("dateStart", 1),
}
return [sort_switch[val] for val in sorts]
def projects(self, filters=None, page_number=0, sorts=None, page_size=10):
"""Return projects list."""
filters = filters or []
sorts = sorts or []
query = self._process_filters(filters)
sort = self._process_sort(sorts)
prjs = [
prj for prj in self._mp.list_events(query=query,
sort=sort,
page_number=page_number,
page_size=page_size)
]
total = self._mp.count_events(query)
return total, prjs
def update_saved_projects(self):
"""Update saved projects in MongoDB.
First, it loops through all the documents saved in MongoDB
and keeps a list of all the project ids. Then, it get
the project data from Agave.
"""
ids = set()
for doc in self.projects():
ids.add(doc["projectId"])
self._mp._mc.scheduler.projects.delete_many(
{"projectId": doc["projectId"]})
for prj_id in ids:
self.save_projct_to_mongo(prj_id)
def save_projct_to_mongo(self, prj_id):
"""Save project from Agave to mongo.
:param str prj_id: Agave's prj id.
"""
prj = self._project_id_to_dict(prj_id)
return self._mp.import_project_to_mongo(prj)
def _project_id_to_dict(self, prj_id):
"""Given a project id return a JSON obj.
The JSON objec returned will be the nested object
stored in the Agave's metadata `value` field.
:param str project_id: Project id
"""
prj = self._pm.get_project_by_id(prj_id)
entities = []
related = None
if prj.project_type == 'experimental':
related = prj.experiment_set
if prj.project_type == 'simulation':
related = prj.simulation_set
if prj.project_type == 'hybrid_simulation':
related = prj.hybridsimulation_set
if prj.project_type == 'field_recon':
related = prj.mission_set
if related:
entities = [self._process_entity(ent) for ent in related(self._ac)]
prj_dict = prj.to_body_dict()['value']
prj_dict['entities'] = entities
return prj_dict
def _process_entity(self, entity):
"""Process entity."""
doc = entity.to_body_dict()["value"]
doc = self._parse_datetime(doc)
doc["uuid"] = entity.uuid
return doc
def _parse_datetime(self, doc):
"""Parse any string into a datetime obj."""
for key in doc:
if not isinstance(doc[key], six.string_types):
continue
try:
dt = datetime_parse(doc[key])
doc[key] = dt
except ValueError:
pass
return doc
def filters(self):
"""Return a list of possible filters."""
return self._mp.filters()
def draft_publication(
project_id,
main_entity_uuids=None,
project_doi=None,
main_entity_doi=None,
upsert_project_doi=False,
upsert_main_entity_doi=True,
revision=None,
revised_authors=None
):
"""Reserve a publication.
A publication is reserved by creating a DOI through Datacite.
For some of the projects a DOI is only created for the main entity
e.g. Mission or Simulation. For some other projects we also (or only)
get a DOI for the project.
- If :param:`project_doi` and/or :param:`main_entity_doi` values are given
then those dois will be updated (or created if they don't exist in datacite).
- If :param:`upsert_project_doi` and/or :param:`upsert_main_entity_doi` are
set to `True` then any saved DOIs will be updated (even if there's multiple
unless a specific DOI is given). If there are no saved DOIs then a new DOI
will be created. Meaning, it will act as update or insert.
- If :param:`project_id` is given **but** :param:`main_entity_uuids` is ``None``
then a project DOI will be created or updated.
.. warning:: This funciton only creates a *Draft* DOI and not a public one.
.. warning:: An entity *might* have multiple DOIs, if this is the case and
:param:`upsert_project_doi` or :param:`upsert_main_entity_doi` are set to True
then *all* saved dois will be updated.
.. note:: In theory a single resource *should not* have multiple DOIs
but we don't know how this will change in the future, hence, we are
supporting multiple DOIs.
.. note:: If no :param:`main_entity_uuids` is given then a project DOI will be
created.
:param str project_id: Project Id
:param list main_entity_uuids: uuid strings of main entities.
:param str project_doi: Custom doi for project.
:param str main_entity_doi: Custom doi for main entity.
:param bool upsert_project_doi: Update or insert project doi.
:param bool upsert_main_entity_doi: Update or insert main entity doi.
"""
mgr = ProjectsManager(service_account())
prj = mgr.get_project_by_id(project_id)
responses = []
if main_entity_uuids:
### Draft Entity DOI(s) ###
pub = BaseESPublication(project_id=project_id, revision=revision)
for ent_uuid in main_entity_uuids:
entity = mgr.get_entity_by_uuid(ent_uuid)
if entity:
if revision:
entity_url = ENTITY_TARGET_BASE.format(
project_id='{}v{}'.format(project_id, revision),
entity_uuid=ent_uuid
)
original_entities = getattr(pub, FIELD_MAP[entity.name])
pub_ent = next(ent for ent in original_entities if ent.uuid == ent_uuid)
entity.title = pub_ent.value.title
entity.authors = revised_authors[ent_uuid]
else:
entity_url = ENTITY_TARGET_BASE.format(
project_id=project_id,
entity_uuid=ent_uuid
)
ent_datacite_json = entity.to_datacite_json()
ent_datacite_json['url'] = entity_url
# ent_datacite_json['version'] = str(revision) # omitting version number per Maria
if upsert_main_entity_doi and main_entity_doi:
me_res = DataciteManager.create_or_update_doi(
ent_datacite_json,
main_entity_doi
)
entity.dois += [main_entity_doi]
entity.dois = list(set(entity.dois))
entity.save(service_account())
responses.append(me_res)
elif upsert_main_entity_doi and entity.dois:
for doi in entity.dois:
me_res = DataciteManager.create_or_update_doi(
ent_datacite_json,
doi
)
responses.append(me_res)
elif upsert_main_entity_doi and not entity.dois:
me_res = DataciteManager.create_or_update_doi(
ent_datacite_json
)
entity.dois += [me_res['data']['id']]
entity.save(service_account())
responses.append(me_res)
else:
### Draft Project DOI ###
upsert_project_doi = True
if revision:
# Versions should not update certain fields
# Add version number to DataCite info
prj_url = TARGET_BASE.format(project_id='{}v{}'.format(project_id, revision))
pub = BaseESPublication(project_id=project_id, revision=revision)
prj.title = pub.project.value.title
prj.team_order = pub.project.value.teamOrder
if revised_authors:
prj.team_order = revised_authors
prj_datacite_json = prj.to_datacite_json()
prj_datacite_json['url'] = prj_url
prj_datacite_json['version'] = str(revision)
# append links to previous versions in DOI...
relatedIdentifiers = []
for ver in range(1, revision):
id = '{}v{}'.format(project_id, ver) if ver!=1 else project_id
relatedIdentifiers.append(
{
'relatedIdentifierType': 'URL',
'relationType': 'IsNewVersionOf',
'relatedIdentifier': TARGET_BASE.format(project_id=id),
}
)
prj_datacite_json['relatedIdentifiers'] = relatedIdentifiers
else:
# format project for publication
prj_url = TARGET_BASE.format(project_id=project_id)
prj_datacite_json = prj.to_datacite_json()
prj_datacite_json['url'] = prj_url
if upsert_project_doi and project_doi:
prj_res = DataciteManager.create_or_update_doi(
prj_datacite_json,
project_doi
)
prj.dois += [project_doi]
prj.dois = list(set(prj.dois))
prj.save(service_account())
responses.append(prj_res)
elif upsert_project_doi and prj.dois:
for doi in prj.dois:
prj_res = DataciteManager.create_or_update_doi(
prj_datacite_json,
doi
)
responses.append(prj_res)
elif upsert_project_doi and not prj.dois:
prj_res = DataciteManager.create_or_update_doi(prj_datacite_json)
prj.dois += [prj_res['data']['id']]
prj.save(service_account())
responses.append(prj_res)
for res in responses:
logger.info(
"DOI created or updated: %(doi)s",
{"doi": res['data']['id']}
)
return responses