def _gen_sys_meta(sys_meta, obj):
'''Return system metadata for GMN
:param sys_meta:
Source system metadata from either CN or MN
:param obj:
Science object
:return:
GMN system metadata
'''
# Compute new values for size and hash
size = len(obj)
md5 = hashlib.md5(obj).hexdigest()
# Merge desired system metadata from MN or CN into GMN
_sys_meta = dataoneTypes.systemMetadata()
_sys_meta.identifier = sys_meta.identifier
_sys_meta.formatId = sys_meta.formatId
_sys_meta.size = size
_sys_meta.rightsHolder = sys_meta.rightsHolder
_sys_meta.checksum = dataoneTypes.checksum(md5)
_sys_meta.checksum.algorithm = "MD5"
_sys_meta.dateUploaded = sys_meta.dateUploaded
_sys_meta.dateSysMetadataModified = sys_meta.dateSysMetadataModified
_sys_meta.accessPolicy = sys_meta.accessPolicy
# Only for test environments
_sys_meta.authoritativeMemberNode = "Nemo GMN localClient"
return _sys_meta
def generate_from_flo(flo, algorithm=None):
"""Generate a Checksum object for a file-like-object, using random
algorithm.
"""
if algorithm is None:
algorithm = random_checksum_algorithm()
hexdigest = calculate_checksum_of_flo(flo, algorithm)
checksum = dataoneTypes.checksum(hexdigest)
checksum.algorithm = algorithm
return checksum
def generate_sys_meta(pid, format_id, size, md5, now): sys_meta = dataoneTypes.systemMetadata() sys_meta.identifier = pid sys_meta.formatId = format_id sys_meta.size = size sys_meta.rightsHolder = SYSMETA_RIGHTSHOLDER sys_meta.checksum = dataoneTypes.checksum(md5) sys_meta.checksum.algorithm = 'MD5' sys_meta.dateUploaded = now sys_meta.dateSysMetadataModified = now sys_meta.accessPolicy = generate_public_access_policy() return sys_meta
def generate_sys_meta(pid, format_id, size, md5, now):
sys_meta = dataoneTypes.systemMetadata()
sys_meta.identifier = pid
sys_meta.formatId = format_id
sys_meta.size = size
sys_meta.rightsHolder = SYSMETA_RIGHTSHOLDER
sys_meta.checksum = dataoneTypes.checksum(md5)
sys_meta.checksum.algorithm = 'MD5'
sys_meta.dateUploaded = now
sys_meta.dateSysMetadataModified = now
sys_meta.accessPolicy = generate_public_access_policy()
return sys_meta
def create_system_metadata(pid, size, checksum, algorithm=None, format_id=None,
access_policy=None, replication_policy=None,
submitter=None, owner=None, orig_mn=None,
auth_mn=None):
sysmeta = dataoneTypes.systemMetadata()
sysmeta.serialVersion = 1
sysmeta.identifier = pid
sysmeta.size = size
sysmeta.checksum = dataoneTypes.checksum(checksum)
sysmeta.dateUploaded = datetime.datetime.utcnow()
sysmeta.dateSysMetadataModified = datetime.datetime.utcnow()
print "utils CONFIGURATION:", id(configuration), configuration
if algorithm is not None:
sysmeta.checksum.algorithm = algorithm
else:
sysmeta.checksum.algorithm = configuration.checksum_alg
if format_id is not None:
sysmeta.formatId = format_id
else:
sysmeta.formatId = configuration.format
if submitter is not None:
sysmeta.submitter = submitter
else:
sysmeta.submitter = configuration.submitter
if owner is not None:
sysmeta.rightsHolder = owner
else:
sysmeta.rightsHolder = configuration.owner
if orig_mn is not None:
sysmeta.originmn = orig_mn
else:
sysmeta.originmn = configuration.orig_mn
if auth_mn is not None:
sysmeta.authoritativemn = auth_mn
else:
sysmeta.authoritativemn = configuration.auth_mn
if access_policy is not None:
sysmeta.accessPolicy = access_policy.to_pyxb()
else:
sysmeta.accessPolicy = access_control_module.access_control().to_pyxb()
if replication_policy is not None:
sysmeta.replicationPolicy = replication_policy.to_pyxb()
else:
sysmeta.replicationPolicy = \
replication_policy_module.replication_policy().to_pyxb()
return sysmeta
def updatePackage(dcxString, isoXML, fileID, idx, client):
now = datetime.now()
# update metadata object
pids = ["dcx_" + fileID]
print "updating: " + pids[0] + "_" + str(idx)
sysMeta = create_sys_meta(
pids[0],
META_FORMAT_ID,
idx+1,
len(dcxString),
dataoneTypes.checksum(hashlib.sha1(dcxString).hexdigest()),
now)
oldpid = pids[0] + "_" + str(idx)
pids[0] = pids[0] + "_" + str(idx+1)
try:
sleep(0.1)
client.update(oldpid, StringIO.StringIO(dcxString), pids[0], sysMeta)
except d1_common.types.exceptions.DataONEException as e:
print "update of " + oldpid + " failed with exception:"
raise
else:
print "update of " + oldpid + " succeeded"
# update data object, the ISO 19139 metadata xml
pids = pids + ["iso19139_" + fileID]
print "updating: " + pids[-1] + "_" + str(idx)
sysMeta = create_sys_meta(
pids[-1],
DATA_FORMAT_ID,
idx+1,
len(isoXML),
dataoneTypes.checksum(hashlib.sha1(isoXML).hexdigest()),
now)
oldpid = pids[-1] + "_" + str(idx)
pids[-1] = pids[-1] + "_" + str(idx+1)
try:
sleep(0.1)
client.update(oldpid, StringIO.StringIO(isoXML), pids[-1], sysMeta)
except d1_common.types.exceptions.DataONEException as e:
print "manual intervention required due to inconsistent package state:"
print pids[0] + "has obsoleted dcx_" + fileID + str(idx) + ", but"
print "update of " + oldpid + " failed with exception:"
raise
else:
print "update of " + oldpid + " succeeded"
# update resource map
oldpid = fileID + "_" + str(idx)
newpid = fileID + "_" + str(idx+1)
print "updating: " + oldpid
rmapGenerator = d1_client.data_package.ResourceMapGenerator()
rmap = rmapGenerator.simple_generate_resource_map(newpid, pids[0], pids[1:])
sysMeta = create_sys_meta(
fileID,
RMAP_FORMAT_ID,
idx+1,
len(rmap),
dataoneTypes.checksum(hashlib.sha1(rmap).hexdigest()),
now)
try:
sleep(0.1)
client.update(oldpid, StringIO.StringIO(rmap), newpid, sysMeta)
except d1_common.types.exceptions.DataONEException as e:
print "manual intervention required due to inconsistent package state:"
print pids[0] + "has obsoleted dcx_" + fileID + str(idx) + ", and"
print pids[-1] + "has obsoleted iso19139_" + fileID + str(idx) + ", but"
print "update of " + oldpid + " failed with exception:"
raise
else:
print "update of " + oldpid + " succeeded"
print "package update for " + oldpid + " successful."
return True
def createInitialPackage(dcxString, isoXML, fileID, client):
now = datetime.now()
# create metadata object
pids = ["dcx_" + fileID]
print "creating metadata object " + pids[0] + "_0"
sysMeta = create_sys_meta(
pids[0],
META_FORMAT_ID,
0,
len(dcxString),
dataoneTypes.checksum(hashlib.sha1(dcxString).hexdigest()),
now)
pids[0] = pids[0] + "_0"
try:
sleep(0.1)
client.create(pids[0], StringIO.StringIO(dcxString), sysMeta)
except:
print "creation of metadata object " + pids[0] + " failed"
return False
# create data object, the ISO 19139 metadata xml
pids = pids + ["iso19139_" + fileID]
print "creating data object " + pids[-1] + "_0"
sysMeta = create_sys_meta(
pids[-1],
DATA_FORMAT_ID,
0,
len(isoXML),
dataoneTypes.checksum(hashlib.sha1(isoXML).hexdigest()),
now)
pids[-1] = pids[-1] + "_0"
try:
sleep(0.1)
client.create(pids[-1], StringIO.StringIO(isoXML), sysMeta)
except:
print "creation of data object " + pids[-1] + " failed"
print "rolling back..."
try:
sleep(0.1)
client.delete(pids[0])
print "rollback deletion of metadata object " + pids[0] + " succeeded"
except:
try: # again
sleep(0.1)
client.delete(pids[0])
print "rollback deletion of metadata object " + pids[0] + " succeeded"
except:
print "rollback deletion of metadata object " + pids[0] + " failed"
print "manual intervention required to delete object."
return False
# create resource map
pid = fileID + "_0"
print "creating resource map " + pid
rmapGenerator = d1_client.data_package.ResourceMapGenerator()
rmap = rmapGenerator.simple_generate_resource_map(pid, pids[0], pids[1:])
sysMeta = create_sys_meta(
fileID,
RMAP_FORMAT_ID,
0,
len(rmap),
dataoneTypes.checksum(hashlib.sha1(rmap).hexdigest()),
now)
try:
sleep(0.1)
client.create(pid, StringIO.StringIO(rmap), sysMeta)
except:
print "creation of resource map " + pid + " failed"
print "rolling back..."
try:
sleep(0.1)
client.delete(pids[-1])
print "rollback deletion of data object " + pids[-1] + " succeeded"
except:
try: # again
sleep(0.1)
client.delete(pids[-1])
print "rollback deletion of data object " + pids[-1] + " succeeded"
except:
print "rollback deletion of data object " + pids[-1] + " failed"
print "manual intervention required to delete object."
try:
sleep(0.1)
client.delete(pids[0])
print "rollback deletion of metadata object " + pids[0] + " succeeded"
except:
try: # again
sleep(0.1)
client.delete(pids[0])
print "rollback deletion of metadata object " + pids[0] + " succeeded"
except:
print "rollback deletion of metadata object " + pids[0] + " failed"
print "manual intervention required to delete object."
return False
# creation of resource map succeeded
else:
print "package creation for " + pid + " successful."
return True
