def getDumpAsObjects(cls, jobId, agg = None): ret = Console.call("unicorn -dump %d" % int(jobId)) result = cls.dumpRegexp.search(ret[1]) if result is None: return None else: result = (result.groups(1))[0] path = os.path.join("/tmp", "tests", "dump", result) result = ow.toObjectsFromJSON(path, ignoreIds = False) result = sorted(result, key = lambda x: x.getObjectId()) if agg is None: os.remove(path) else: agg.aggregateFile(path) return result
def taskProcess(self): filepath = None jobId = self.currentTask.job taskId = self.currentTask.task_id for param in self.currentTask.parameters: if param.name == "uri": filepath = param.value continue if filepath is None: raise ParamException("No file path passed.") try: objects = ow.toObjectsFromJSON(filepath, True) objects2 = list() skipped = 0 for obj in objects: passed = True for attr in obj.getTypeStore().iteritems(): logging.info(attr) if attr[1] == "BYTES": logging.info("got %s" % attr[0]) key = getattr(obj, attr[0]).getKey() if isinstance(key, str) or isinstance(key, unicode): if os.path.exists(key): obj.addBytes( attr[0], long( self.dsAdapter.putFile( key, self.currentTask.job))) else: logging.warn("File not found.") passed = False if passed: objects2.append(obj) else: skipped += 1 newObjIds = self.osAdapter.objectsPut(jobId, taskId, objects2) self.newObjects.extend(newObjIds) except IOError as e: raise ParamException("IOError - %s." % e) #raise ParamException("File '%s' not found." % filepath) except ValueError as e: raise ParamException("Trouble processing file - '%s'" % e.message) return ["Skipped %d objects" % skipped]
def taskProcess(self): filepath = None jobId = self.currentTask.job taskId = self.currentTask.task_id for param in self.currentTask.parameters: if param.name == "uri": filepath = param.value continue if filepath is None: raise ParamException("No file path passed.") try: objects = ow.toObjectsFromJSON(filepath, True) objects2 = list() skipped = 0 for obj in objects: passed = True for attr in obj.getTypeStore().iteritems(): logging.info(attr) if attr[1] == "BYTES": logging.info("got %s" % attr[0]) key = getattr(obj,attr[0]).getKey() if isinstance(key, str) or isinstance(key, unicode): if os.path.exists(key): obj.addBytes(attr[0],long(self.dsAdapter.putFile(key, self.currentTask.job))) else: logging.warn("File not found.") passed = False if passed: objects2.append(obj) else: skipped += 1 newObjIds = self.osAdapter.objectsPut(jobId,taskId,objects2) self.newObjects.extend(newObjIds) except IOError as e: raise ParamException("IOError - %s." % e) #raise ParamException("File '%s' not found." % filepath) except ValueError as e: raise ParamException("Trouble processing file - '%s'" % e.message) return ["Skipped %d objects" % skipped]