def popolae(ctx, size): logger.info('COMMAND: Filling irods') com = basher() # system commands, only needed for this command remove_irods_existing = appconfig.mocking() from libs.service_supply import fill_irods_random fill_irods_random(com, ctx.obj['icom'], size, remove_irods_existing)
def pid_metadata(self, pid): """ Metadata derived only inside an Eudat enviroment """ # Binary included inside the neoicommands docker image com = 'epicc' credentials = './conf/credentials.json' args = ['os', credentials, 'read', pid] json_data = "" select = { 'location':'URL', 'checksum': 'CHECKSUM', 'parent_pid':'EUDAT/PPID', } metas = {} if appconfig.mocking(): # // TO FIX: empty = "" # Generate random # e.g. irods://130.186.13.14:1247/cinecaDMPZone/home/pdonorio/replica/test2 # e.g. sha2:dCdRWFfS2TGm/4BfKQPu1WdQSdBwxRoxCRMX3zan3SM= # e.g. 842/52ae4c2c-4feb-11e5-afd1-fa163e62896a pid_metas = { 'URL': empty, 'CHECKSUM': empty, 'EUDAT/PPID': empty, } # // TO REMOVE: # Fake, always the same metas = self.parse_rest_json(None, './tests/epic.pid.out') else: logger.debug("Epic client for %s " % args) json_data = self.execute_command(com, args).strip() if json_data.strip() == 'None': return {} # Get all epic metas metas = self.parse_rest_json(json_data) ## Meaningfull data pid_metas = {} for name, selection in select.items(): value = None if selection in metas: value = metas[selection] pid_metas[name] = value return pid_metas
def findconnect_frompid(graph, pid, ppid): # Replica pid_replica = graph.PID.nodes.get(code=pid) dobj_replica = pid_replica.identify.get() # Original copy (parent) if appconfig.mocking(): fake = {'name':'well', 'location':'fixed'} dobj_parent = graph.DataObject.get_or_create(fake).pop() else: pid_parent = graph.PID.nodes.get(code=ppid) dobj_parent = pid_parent.identify.get() # Relationship as neomodel relation = dobj_replica.replica.connect(dobj_parent) relation.ppid = ppid # // TO FIX: how to find ROR? relation.ror = relation.ppid logger.info("Saved replica relation for %s %s" % (pid, relation.ppid))
def register_pid(self, dataobj): """ Eudat rule for irods to register a PID to a Handle """ # Path fix dataobj = os.path.join(self._base_dir, dataobj) if appconfig.mocking(): #pid = "842/a72976e0-5177-11e5-b479-fa163e62896a" # 8 - 4 - 4 - 4 - 12 base = "842" code = string_generator(8) code += "-" + str(random.randint(1000,9999)) code += "-" + string_generator(4) + "-" + string_generator(4) code += "-" + string_generator(12) pid = base + "/" + code else: context = { 'irods_file': dataobj.center(len(dataobj)+2, '"') } pid = self.execute_rule_from_template('getpid', context) return pid
def fill_irods_random(com, icom, elements=10, clean_irods=True, \ prefix=DEFAULT_PREFIX, tmp_dir='itmp'): # Remove host data temporary dir if existing com.remove_directory(tmp_dir, ignore=True) # Create host data dir com.create_directory(tmp_dir) # Clean all fake dirs if requested if clean_irods: for mydir in fake_directories: # Clean if existing on iRODS if icom.check(mydir): logger.warning("Cleaning on server") icom.remove_directory(mydir) logger.info("Creating %s elements" % elements) # Create and save for i in range(1, elements+1): # Random directory, not so random pos = random.randint(0,len(fake_directories)-1) irods_dir = fake_directories[pos] icom.create_directory(irods_dir) logger.debug("Element n. %s " % i) # Create two strings r1 = string_generator() r2 = string_generator() # Write a random file filename = prefix + r1 + ".txt" hostfile = tmp_dir + "/" + filename with open(hostfile,'w') as f: f.write(r2) # Put into irods irods_file = irods_dir + "/" + filename icom.save(hostfile, irods_file) # Add random meta via imeta metas = {} metas_elements = random.randint(1,5) for j in range(0, metas_elements): # More randoms r3 = string_generator(j) metatag = random.randint(1,elements) # Meta key name = "meta-" + str(metatag) #str(r3) #to make unique? # Content value = r3 + r2 metas[name] = value ####################### ## PID if random.randint(0,1): # PID may not exists pid = icom.register_pid(irods_file) logger.info("Obtained PID %s" % pid) if appconfig.mocking(): # Save pid inside metadata # Automatic if using real Eudat service metas["PID"] = pid # Write metadata for key, value in metas.items(): # Could use batch insert instead icom.meta_write(irods_file, [key], [value]) logger.debug("Wrote key %s inside file %s " % (key, filename)) ####################### ## REPLICA # Random choise if replica or not if random.randint(0,1): logger.debug("Generating a REPLICA!") # Random number of replicas n = random.randint(1,3) if appconfig.mocking(): # irods simple replica icom.replica(irods_file, n) #icom.replica_list(irods_file) #exit() else: #// TO FIX: more copies icom.eudat_replica(irods_file) # # Eudat # while n > 1: # n -= 1 # icom.eudat_replica(irods_file) #exit() # Debug # print("Created ", metas_elements, "elements") # Clean host data com.remove_directory(tmp_dir, ignore=True) logger.info("COMPLETED\nGenerated n.%s elements" % elements)