def getDumpAsObjects(cls, jobId, agg = None):
ret = Console.call("unicorn -dump %d" % int(jobId))
result = cls.dumpRegexp.search(ret[1])
if result is None:
return None
else:
result = (result.groups(1))[0]
path = os.path.join("/tmp", "tests", "dump", result)
result = ow.toObjectsFromJSON(path, ignoreIds = False)
result = sorted(result, key = lambda x: x.getObjectId())
if agg is None:
os.remove(path)
else:
agg.aggregateFile(path)
return result
def taskProcess(self):
filepath = None
jobId = self.currentTask.job
taskId = self.currentTask.task_id
for param in self.currentTask.parameters:
if param.name == "uri":
filepath = param.value
continue
if filepath is None:
raise ParamException("No file path passed.")
try:
objects = ow.toObjectsFromJSON(filepath, True)
objects2 = list()
skipped = 0
for obj in objects:
passed = True
for attr in obj.getTypeStore().iteritems():
logging.info(attr)
if attr[1] == "BYTES":
logging.info("got %s" % attr[0])
key = getattr(obj, attr[0]).getKey()
if isinstance(key, str) or isinstance(key, unicode):
if os.path.exists(key):
obj.addBytes(
attr[0],
long(
self.dsAdapter.putFile(
key, self.currentTask.job)))
else:
logging.warn("File not found.")
passed = False
if passed:
objects2.append(obj)
else:
skipped += 1
newObjIds = self.osAdapter.objectsPut(jobId, taskId, objects2)
self.newObjects.extend(newObjIds)
except IOError as e:
raise ParamException("IOError - %s." % e)
#raise ParamException("File '%s' not found." % filepath)
except ValueError as e:
raise ParamException("Trouble processing file - '%s'" % e.message)
return ["Skipped %d objects" % skipped]
def taskProcess(self):
filepath = None
jobId = self.currentTask.job
taskId = self.currentTask.task_id
for param in self.currentTask.parameters:
if param.name == "uri":
filepath = param.value
continue
if filepath is None:
raise ParamException("No file path passed.")
try:
objects = ow.toObjectsFromJSON(filepath, True)
objects2 = list()
skipped = 0
for obj in objects:
passed = True
for attr in obj.getTypeStore().iteritems():
logging.info(attr)
if attr[1] == "BYTES":
logging.info("got %s" % attr[0])
key = getattr(obj,attr[0]).getKey()
if isinstance(key, str) or isinstance(key, unicode):
if os.path.exists(key):
obj.addBytes(attr[0],long(self.dsAdapter.putFile(key, self.currentTask.job)))
else:
logging.warn("File not found.")
passed = False
if passed:
objects2.append(obj)
else:
skipped += 1
newObjIds = self.osAdapter.objectsPut(jobId,taskId,objects2)
self.newObjects.extend(newObjIds)
except IOError as e:
raise ParamException("IOError - %s." % e)
#raise ParamException("File '%s' not found." % filepath)
except ValueError as e:
raise ParamException("Trouble processing file - '%s'" % e.message)
return ["Skipped %d objects" % skipped]
