def _gen_sys_meta(sys_meta, obj): '''Return system metadata for GMN :param sys_meta: Source system metadata from either CN or MN :param obj: Science object :return: GMN system metadata ''' # Compute new values for size and hash size = len(obj) md5 = hashlib.md5(obj).hexdigest() # Merge desired system metadata from MN or CN into GMN _sys_meta = dataoneTypes.systemMetadata() _sys_meta.identifier = sys_meta.identifier _sys_meta.formatId = sys_meta.formatId _sys_meta.size = size _sys_meta.rightsHolder = sys_meta.rightsHolder _sys_meta.checksum = dataoneTypes.checksum(md5) _sys_meta.checksum.algorithm = "MD5" _sys_meta.dateUploaded = sys_meta.dateUploaded _sys_meta.dateSysMetadataModified = sys_meta.dateSysMetadataModified _sys_meta.accessPolicy = sys_meta.accessPolicy # Only for test environments _sys_meta.authoritativeMemberNode = "Nemo GMN localClient" return _sys_meta
def generate_from_flo(flo, algorithm=None): """Generate a Checksum object for a file-like-object, using random algorithm. """ if algorithm is None: algorithm = random_checksum_algorithm() hexdigest = calculate_checksum_of_flo(flo, algorithm) checksum = dataoneTypes.checksum(hexdigest) checksum.algorithm = algorithm return checksum
def generate_sys_meta(pid, format_id, size, md5, now): sys_meta = dataoneTypes.systemMetadata() sys_meta.identifier = pid sys_meta.formatId = format_id sys_meta.size = size sys_meta.rightsHolder = SYSMETA_RIGHTSHOLDER sys_meta.checksum = dataoneTypes.checksum(md5) sys_meta.checksum.algorithm = 'MD5' sys_meta.dateUploaded = now sys_meta.dateSysMetadataModified = now sys_meta.accessPolicy = generate_public_access_policy() return sys_meta
def create_system_metadata(pid, size, checksum, algorithm=None, format_id=None, access_policy=None, replication_policy=None, submitter=None, owner=None, orig_mn=None, auth_mn=None): sysmeta = dataoneTypes.systemMetadata() sysmeta.serialVersion = 1 sysmeta.identifier = pid sysmeta.size = size sysmeta.checksum = dataoneTypes.checksum(checksum) sysmeta.dateUploaded = datetime.datetime.utcnow() sysmeta.dateSysMetadataModified = datetime.datetime.utcnow() print "utils CONFIGURATION:", id(configuration), configuration if algorithm is not None: sysmeta.checksum.algorithm = algorithm else: sysmeta.checksum.algorithm = configuration.checksum_alg if format_id is not None: sysmeta.formatId = format_id else: sysmeta.formatId = configuration.format if submitter is not None: sysmeta.submitter = submitter else: sysmeta.submitter = configuration.submitter if owner is not None: sysmeta.rightsHolder = owner else: sysmeta.rightsHolder = configuration.owner if orig_mn is not None: sysmeta.originmn = orig_mn else: sysmeta.originmn = configuration.orig_mn if auth_mn is not None: sysmeta.authoritativemn = auth_mn else: sysmeta.authoritativemn = configuration.auth_mn if access_policy is not None: sysmeta.accessPolicy = access_policy.to_pyxb() else: sysmeta.accessPolicy = access_control_module.access_control().to_pyxb() if replication_policy is not None: sysmeta.replicationPolicy = replication_policy.to_pyxb() else: sysmeta.replicationPolicy = \ replication_policy_module.replication_policy().to_pyxb() return sysmeta
def updatePackage(dcxString, isoXML, fileID, idx, client): now = datetime.now() # update metadata object pids = ["dcx_" + fileID] print "updating: " + pids[0] + "_" + str(idx) sysMeta = create_sys_meta( pids[0], META_FORMAT_ID, idx+1, len(dcxString), dataoneTypes.checksum(hashlib.sha1(dcxString).hexdigest()), now) oldpid = pids[0] + "_" + str(idx) pids[0] = pids[0] + "_" + str(idx+1) try: sleep(0.1) client.update(oldpid, StringIO.StringIO(dcxString), pids[0], sysMeta) except d1_common.types.exceptions.DataONEException as e: print "update of " + oldpid + " failed with exception:" raise else: print "update of " + oldpid + " succeeded" # update data object, the ISO 19139 metadata xml pids = pids + ["iso19139_" + fileID] print "updating: " + pids[-1] + "_" + str(idx) sysMeta = create_sys_meta( pids[-1], DATA_FORMAT_ID, idx+1, len(isoXML), dataoneTypes.checksum(hashlib.sha1(isoXML).hexdigest()), now) oldpid = pids[-1] + "_" + str(idx) pids[-1] = pids[-1] + "_" + str(idx+1) try: sleep(0.1) client.update(oldpid, StringIO.StringIO(isoXML), pids[-1], sysMeta) except d1_common.types.exceptions.DataONEException as e: print "manual intervention required due to inconsistent package state:" print pids[0] + "has obsoleted dcx_" + fileID + str(idx) + ", but" print "update of " + oldpid + " failed with exception:" raise else: print "update of " + oldpid + " succeeded" # update resource map oldpid = fileID + "_" + str(idx) newpid = fileID + "_" + str(idx+1) print "updating: " + oldpid rmapGenerator = d1_client.data_package.ResourceMapGenerator() rmap = rmapGenerator.simple_generate_resource_map(newpid, pids[0], pids[1:]) sysMeta = create_sys_meta( fileID, RMAP_FORMAT_ID, idx+1, len(rmap), dataoneTypes.checksum(hashlib.sha1(rmap).hexdigest()), now) try: sleep(0.1) client.update(oldpid, StringIO.StringIO(rmap), newpid, sysMeta) except d1_common.types.exceptions.DataONEException as e: print "manual intervention required due to inconsistent package state:" print pids[0] + "has obsoleted dcx_" + fileID + str(idx) + ", and" print pids[-1] + "has obsoleted iso19139_" + fileID + str(idx) + ", but" print "update of " + oldpid + " failed with exception:" raise else: print "update of " + oldpid + " succeeded" print "package update for " + oldpid + " successful." return True
def createInitialPackage(dcxString, isoXML, fileID, client): now = datetime.now() # create metadata object pids = ["dcx_" + fileID] print "creating metadata object " + pids[0] + "_0" sysMeta = create_sys_meta( pids[0], META_FORMAT_ID, 0, len(dcxString), dataoneTypes.checksum(hashlib.sha1(dcxString).hexdigest()), now) pids[0] = pids[0] + "_0" try: sleep(0.1) client.create(pids[0], StringIO.StringIO(dcxString), sysMeta) except: print "creation of metadata object " + pids[0] + " failed" return False # create data object, the ISO 19139 metadata xml pids = pids + ["iso19139_" + fileID] print "creating data object " + pids[-1] + "_0" sysMeta = create_sys_meta( pids[-1], DATA_FORMAT_ID, 0, len(isoXML), dataoneTypes.checksum(hashlib.sha1(isoXML).hexdigest()), now) pids[-1] = pids[-1] + "_0" try: sleep(0.1) client.create(pids[-1], StringIO.StringIO(isoXML), sysMeta) except: print "creation of data object " + pids[-1] + " failed" print "rolling back..." try: sleep(0.1) client.delete(pids[0]) print "rollback deletion of metadata object " + pids[0] + " succeeded" except: try: # again sleep(0.1) client.delete(pids[0]) print "rollback deletion of metadata object " + pids[0] + " succeeded" except: print "rollback deletion of metadata object " + pids[0] + " failed" print "manual intervention required to delete object." return False # create resource map pid = fileID + "_0" print "creating resource map " + pid rmapGenerator = d1_client.data_package.ResourceMapGenerator() rmap = rmapGenerator.simple_generate_resource_map(pid, pids[0], pids[1:]) sysMeta = create_sys_meta( fileID, RMAP_FORMAT_ID, 0, len(rmap), dataoneTypes.checksum(hashlib.sha1(rmap).hexdigest()), now) try: sleep(0.1) client.create(pid, StringIO.StringIO(rmap), sysMeta) except: print "creation of resource map " + pid + " failed" print "rolling back..." try: sleep(0.1) client.delete(pids[-1]) print "rollback deletion of data object " + pids[-1] + " succeeded" except: try: # again sleep(0.1) client.delete(pids[-1]) print "rollback deletion of data object " + pids[-1] + " succeeded" except: print "rollback deletion of data object " + pids[-1] + " failed" print "manual intervention required to delete object." try: sleep(0.1) client.delete(pids[0]) print "rollback deletion of metadata object " + pids[0] + " succeeded" except: try: # again sleep(0.1) client.delete(pids[0]) print "rollback deletion of metadata object " + pids[0] + " succeeded" except: print "rollback deletion of metadata object " + pids[0] + " failed" print "manual intervention required to delete object." return False # creation of resource map succeeded else: print "package creation for " + pid + " successful." return True