def _generate_system_metadata(scimeta_bytes, native_identifier_sid, record_date, symeta_settings_dict):
"""
:param scimeta_bytes: Bytes of the node's original metadata document.
:param native_identifier_sid: Node's system identifier for this object, which becomes the series ID.
:param record_date: Date metadata document was created/modified in the source system. Becomes dateUploaded.
:param sysmeta_settings_dict: A dict containing node-specific system metadata properties that will apply to all
science metadata documents loaded into GMN.
This function generates a system metadata document for describing the science metadata record being loaded. Some
of the fields, such as checksum and size, are based off the bytes of the science metadata object itself. Other
system metadata fields are passed to D1ClientManager in a dict which is configured in the main adapter program.
Note that the checksum is assigned as an arbitrary version identifier to accommodate the source system's mutable
content represented in the target system's immutable content standard. """
sys_meta = v2.systemMetadata()
sys_meta.seriesId = native_identifier_sid
sys_meta.formatId = symeta_settings_dict['formatId_custom']
sys_meta.size = len(scimeta_bytes)
sys_meta.checksum = dataoneTypes.checksum(hashlib.md5(scimeta_bytes).hexdigest())
sys_meta.checksum.algorithm = 'MD5'
sys_meta.identifier = sys_meta.checksum.value()
# sys_meta.dateUploaded = datetime.datetime.strptime(record_date, "%Y-%m-%dT%H:%M:%SZ")
sys_meta.dateUploaded = record_date
sys_meta.dateSysMetadataModified = datetime.datetime.now()
sys_meta.rightsHolder = symeta_settings_dict['rightsholder']
sys_meta.submitter = symeta_settings_dict['submitter']
sys_meta.authoritativeMemberNode = symeta_settings_dict['authoritativeMN']
sys_meta.originMemberNode = symeta_settings_dict['originMN']
sys_meta.accessPolicy = _generate_public_access_policy()
return sys_meta
def _generate_sysmeta(self, scimeta_str, pid, sid, record_datetime):
sysmeta_pyxb = v2.systemMetadata()
sysmeta_pyxb.seriesId = sid
sysmeta_pyxb.formatId = self.node_dict['sci_md_formatId']
sysmeta_pyxb.size = len(scimeta_str)
sysmeta_pyxb.checksum = d1_common.checksum.create_checksum_object_from_string(
scimeta_str
)
sysmeta_pyxb.identifier = pid
# sysmeta_pyxb.dateUploaded = datetime.datetime.strptime(record_datetime, "%Y-%m-%dT%H:%M:%SZ")
sysmeta_pyxb.dateUploaded = record_datetime
sysmeta_pyxb.dateSysMetadataModified = datetime.datetime.now()
sysmeta_pyxb.rightsHolder = self.node_dict['rightsholder']
sysmeta_pyxb.submitter = self.node_dict['submitter']
sysmeta_pyxb.authoritativeMemberNode = self.node_dict['authoritativeMN']
sysmeta_pyxb.originMemberNode = self.node_dict['originMN']
# Access policies for SlenderNode design pattern default to public read only
accessPolicy = v2.AccessPolicy()
accessRule = v2.AccessRule()
accessRule.subject.append(d1_common.const.SUBJECT_PUBLIC)
permission = v2.Permission('read')
accessRule.permission.append(permission)
accessPolicy.append(accessRule)
sysmeta_pyxb.accessPolicy = accessPolicy
return sysmeta_pyxb
def _generate_system_metadata(self, scimeta_bytes, sid, pid, record_date):
sys_meta = v2.systemMetadata()
sys_meta.seriesId = sid
sys_meta.formatId = 'http://www.isotc211.org/2005/gmd-ieda'
sys_meta.size = len(scimeta_bytes)
digest = hashlib.md5(scimeta_bytes).hexdigest()
sys_meta.checksum = v2.checksum(digest)
sys_meta.checksum.algorithm = 'MD5'
sys_meta.identifier = sys_meta.checksum.value()
sys_meta.dateUploaded = record_date
sys_meta.dateSysMetadataModified = dt.datetime.now()
sys_meta.identifier = pid
sys_meta.dateUploaded = d1_common.date_time.utc_now()
sys_meta.dateSysMetadataModified = dt.datetime.now()
sys_meta.rightsHolder = 'urn:node:IEDA'
sys_meta.submitter = 'urn:node:IEDA'
sys_meta.authoritativeMemberNode = 'urn:node:mnTestIEDA'
sys_meta.originMemberNode = 'urn:node:IEDA'
sys_meta.accessPolicy = v2.AccessPolicy()
return sys_meta
def generate_sys_meta(pid, size, md5, now):
sys_meta = dataoneTypes.systemMetadata()
sys_meta.identifier = pid
sys_meta.formatId = SYSMETA_FORMATID
sys_meta.size = size
sys_meta.rightsHolder = SYSMETA_RIGHTSHOLDER
sys_meta.checksum = dataoneTypes.checksum(md5)
sys_meta.checksum.algorithm = 'MD5'
sys_meta.dateUploaded = now
sys_meta.dateSysMetadataModified = now
sys_meta.accessPolicy = generate_public_access_policy()
return sys_meta
def _generate_sys_meta(self,
pid,
format_id,
size,
checksum,
rights_holder,
modified=None):
if modified is None:
modified = datetime.datetime.now()
sys_meta = dataoneTypes.systemMetadata()
sys_meta.identifier = pid
sys_meta.formatId = format_id
sys_meta.size = size
sys_meta.rightsHolder = rights_holder
sys_meta.checksum = checksum
sys_meta.dateUploaded = modified
sys_meta.dateSysMetadataModified = modified
sys_meta.accessPolicy = self.generate_public_access_policy()
return sys_meta
def generate_system_metadata(scimeta_bytes, pid, sid):
"""
:param scimeta_bytes: Bytes of the node's original metadata document.
:param native_identifier_sid: Node's system identifier for this object, which
becomes the series ID.
:param record_date: Date metadata document was created/modified in the source
system. Becomes dateUploaded.
:param sysmeta_settings_dict: A dict containing node-specific system metadata
properties that will apply to all science metadata documents loaded into GMN.
This function generates a system metadata document for describing the science
metadata record being loaded. Some of the fields, such as checksum and size,
are based off the bytes of the science metadata object itself. Other system
metadata fields are passed to D1ClientManager in a dict which is configured in
the main adapter program. Note that the checksum is assigned as an arbitrary
version identifier to accommodate the source system's mutable content
represented in the target system's immutable content standard.
"""
sysmeta_pyxb = v2.systemMetadata()
sysmeta_pyxb.seriesId = sid
sysmeta_pyxb.formatId = SCIMETA_FORMAT_ID
sysmeta_pyxb.size = len(scimeta_bytes)
sysmeta_pyxb.checksum = d1_common.checksum.create_checksum_object_from_string(
scimeta_bytes
)
sysmeta_pyxb.identifier = pid
sysmeta_pyxb.dateUploaded = d1_common.date_time.utc_now()
sysmeta_pyxb.dateSysMetadataModified = datetime.datetime.now()
sysmeta_pyxb.rightsHolder = SCIMETA_RIGHTS_HOLDER
sysmeta_pyxb.submitter = SCIMETA_SUBMITTER
sysmeta_pyxb.authoritativeMemberNode = SCIMETA_AUTHORITATIVE_MEMBER_NODE
sysmeta_pyxb.originMemberNode = SCIMETA_AUTHORITATIVE_MEMBER_NODE
sysmeta_pyxb.accessPolicy = v2.AccessPolicy()
with d1_common.wrap.access_policy.wrap_sysmeta_pyxb(sysmeta_pyxb) as ap:
ap.clear()
ap.add_public_read()
return sysmeta_pyxb
def _generate_sysmeta_pyxb(self, metadata_et, xml_str, checksum_pyxb, obsoletes_pid):
now = datetime.datetime.now()
sysmeta_pyxb = v2.systemMetadata()
sysmeta_pyxb.serialVersion = 1
sysmeta_pyxb.identifier = checksum_pyxb.value()
sysmeta_pyxb.seriesId = metadata_et.find(
'./gmd:fileIdentifier/gco:CharacterString', NS_DICT
).text
sysmeta_pyxb.formatId = SCIOBJ_FORMAT_STR
sysmeta_pyxb.size = len(xml_str)
sysmeta_pyxb.submitter = SUBMITTER_SUBJECT_STR
sysmeta_pyxb.rightsHolder = OWNER_SUBJECT_STR
sysmeta_pyxb.checksum = checksum_pyxb
sysmeta_pyxb.dateUploaded = d1_common.date_time.from_iso8601(
metadata_et.find('./gmd:dateStamp/gco:DateTime', NS_DICT).text
)
sysmeta_pyxb.dateSysMetadataModified = now
sysmeta_pyxb.originMemberNode = ORIGIN_MEMBER_NODE_URN
sysmeta_pyxb.authoritativeMemberNode = AUTHORITATIVE_MEMBER_NODE_URN
sysmeta_pyxb.obsoletes = obsoletes_pid
sysmeta_pyxb.accessPolicy = self._generate_public_access_policy_pyxb()
return sysmeta_pyxb
def _generate_system_metadata(scimeta_bytes, native_identifier_sid, version_pid, record_date, symeta_settings_dict):
"""This function generates a system metadata document for describing the science metadata record being loaded. Some
of the fields, such as checksum and size, are based off the bytes of the science metadata object itself. Other
system metadata fields are passed to D1ClientManager in a dict which is configured in the main adapter program."""
sys_meta = v2.systemMetadata()
sys_meta.seriesId = native_identifier_sid
sys_meta.identifier = version_pid
sys_meta.formatId = symeta_settings_dict['formatId']
sys_meta.size = len(scimeta_bytes)
# sys_meta.checksum = \
# d1_common.checksum.create_checksum_object_from_stream(StringIO.StringIO(scimeta_bytes), algorithm='MD5')
sys_meta.checksum = dataoneTypes.checksum(hashlib.md5(scimeta_bytes).hexdigest())
sys_meta.checksum.algorithm = 'MD5'
#sys_meta.dateUploaded = datetime.datetime.strptime(record_date, "%Y-%m-%d %H:%M:%SZ")
sys_meta.dateUploaded = record_date
sys_meta.dateSysMetadataModified = datetime.datetime.now()
sys_meta.rightsHolder = symeta_settings_dict['rightsholder']
sys_meta.submitter = symeta_settings_dict['submitter']
sys_meta.authoritativeMemberNode = symeta_settings_dict['authoritativeMN']
sys_meta.originMemberNode = symeta_settings_dict['originMN']
sys_meta.accessPolicy = _generate_public_access_policy()
return sys_meta
def generate_system_metadata(scimeta_bytes, pid, sid):
"""
:param scimeta_bytes: Bytes of the node's original metadata document.
:param native_identifier_sid: Node's system identifier for this object, which
becomes the series ID.
:param record_date: Date metadata document was created/modified in the source
system. Becomes dateUploaded.
:param sysmeta_settings_dict: A dict containing node-specific system metadata
properties that will apply to all science metadata documents loaded into GMN.
This function generates a system metadata document for describing the science
metadata record being loaded. Some of the fields, such as checksum and size,
are based off the bytes of the science metadata object itself. Other system
metadata fields are passed to D1ClientManager in a dict which is configured in
the main adapter program. Note that the checksum is assigned as an arbitrary
version identifier to accommodate the source system's mutable content
represented in the target system's immutable content standard.
"""
sysmeta_pyxb = v2.systemMetadata()
sysmeta_pyxb.seriesId = sid
sysmeta_pyxb.formatId = SCIMETA_FORMAT_ID
sysmeta_pyxb.size = len(scimeta_bytes)
sysmeta_pyxb.checksum = d1_common.checksum.create_checksum_object_from_string(
scimeta_bytes)
sysmeta_pyxb.identifier = pid
sysmeta_pyxb.dateUploaded = d1_common.date_time.utc_now()
sysmeta_pyxb.dateSysMetadataModified = datetime.datetime.now()
sysmeta_pyxb.rightsHolder = SCIMETA_RIGHTS_HOLDER
sysmeta_pyxb.submitter = SCIMETA_SUBMITTER
sysmeta_pyxb.authoritativeMemberNode = SCIMETA_AUTHORITATIVE_MEMBER_NODE
sysmeta_pyxb.originMemberNode = SCIMETA_AUTHORITATIVE_MEMBER_NODE
sysmeta_pyxb.accessPolicy = v2.AccessPolicy()
with d1_common.wrap.access_policy.wrap_sysmeta_pyxb(sysmeta_pyxb) as ap:
ap.clear()
ap.add_public_read()
return sysmeta_pyxb
def build_sysmeta(pid=None, obj=None, fmt=None, sid=None):
if pid is None:
return 1
if obj is None:
return 1
if fmt is None:
fmt = 'application/octet-stream'
obj_size = os.path.getsize(obj)
obj_md5 = md5(obj)
accessPolicy = dataoneTypes_v2_0.accessPolicy()
accessRule = dataoneTypes_v2_0.AccessRule()
accessRule.subject.append('gmn-local')
accessRule.permission.append('changePermission')
accessPolicy.append(accessRule)
accessRule = dataoneTypes_v2_0.AccessRule()
accessRule.subject.append('public')
accessRule.permission.append('read')
accessPolicy.append(accessRule)
d1_sys_meta = dataoneTypes_v2_0.systemMetadata()
d1_sys_meta.serialVersion = 1
d1_sys_meta.identifier = pid
d1_sys_meta.size = obj_size
d1_sys_meta.formatId = fmt
d1_sys_meta.rightsHolder = 'gmn-local'
d1_sys_meta.checksum = obj_md5
d1_sys_meta.checksum.algorithm = 'MD5'
d1_sys_meta.accessPolicy = accessPolicy
if sid is not None:
d1_sys_meta.seriesId = sid
return d1_sys_meta
def generate_system_metadata(self,
*,
scimeta_bytes=None,
native_identifier_sid=None,
record_date=None,
record_version=None):
"""
This function generates a system metadata document for describing
the science metadata record being loaded. Some of the fields,
such as checksum and size, are based off the bytes of the science
metadata object itself. Other system metadata fields are passed
to D1ClientManager in a dict which is configured in the main
adapter program. Note that the checksum is assigned as an
arbitrary version identifier to accommodate the source system's
mutable content represented in the target system's immutable
content standard.
This is the default case. It should be specialized by each slender
nodes object.
Parameters
----------
scimeta_bytes :
Bytes of the node's original metadata document.
native_identifier_sid :
Node's system identifier for this object, which becomes the series
ID, or sid.
record_date :
Date metadata document was created/modified in the source
system. Becomes dateUploaded.
record_version : str
Will be the pid.
Returns
-------
A dict containing node-specific system metadata properties that
will apply to all science metadata documents loaded into GMN.
"""
sys_meta = v2.systemMetadata()
sys_meta.seriesId = native_identifier_sid
sys_meta.formatId = self.sys_meta_dict['formatId_custom']
sys_meta.size = len(scimeta_bytes)
digest = hashlib.md5(scimeta_bytes).hexdigest()
sys_meta.checksum = dataoneTypes.checksum(digest)
sys_meta.checksum.algorithm = 'MD5'
if record_version is None:
# only if we have nothing else.
sys_meta.identifier = sys_meta.checksum.value()
else:
sys_meta.identifier = record_version
sys_meta.dateUploaded = record_date
sys_meta.dateSysMetadataModified = dt.datetime.now(dt.timezone.utc)
sys_meta.rightsHolder = self.sys_meta_dict['rightsholder']
sys_meta.submitter = self.sys_meta_dict['submitter']
sys_meta.authoritativeMemberNode = self.sys_meta_dict[
'authoritativeMN'] # noqa: E501
sys_meta.originMemberNode = self.sys_meta_dict['originMN']
sys_meta.accessPolicy = self.generate_public_access_policy()
return sys_meta
