def upload_workflow_collection(arvrunner, name, packed):
collection = arvados.collection.Collection(
api_client=arvrunner.api,
keep_client=arvrunner.keep_client,
num_retries=arvrunner.num_retries)
with collection.open("workflow.cwl", "w") as f:
f.write(
json.dumps(packed,
indent=2,
sort_keys=True,
separators=(',', ': ')))
filters = [["portable_data_hash", "=",
collection.portable_data_hash()], ["name", "like", name + "%"]]
if arvrunner.project_uuid:
filters.append(["owner_uuid", "=", arvrunner.project_uuid])
exists = arvrunner.api.collections().list(filters=filters).execute(
num_retries=arvrunner.num_retries)
if exists["items"]:
logger.info("Using collection %s", exists["items"][0]["uuid"])
else:
collection.save_new(name=name,
owner_uuid=arvrunner.project_uuid,
ensure_unique_name=True,
num_retries=arvrunner.num_retries)
logger.info("Uploaded to %s", collection.manifest_locator())
return collection.portable_data_hash()
def write_file(collection, pathprefix, fn, flush=False):
with open(os.path.join(pathprefix, fn), "rb") as src:
dst = collection.open(fn, "wb")
r = src.read(1024*128)
while r:
dst.write(r)
r = src.read(1024*128)
dst.close(flush=flush)
def write_file(collection, pathprefix, fn):
with open(os.path.join(pathprefix, fn)) as src:
dst = collection.open(fn, "w")
r = src.read(1024 * 128)
while r:
dst.write(r)
r = src.read(1024 * 128)
dst.close(flush=False)
def upload_workflow_collection(arvrunner, name, packed):
collection = arvados.collection.Collection(api_client=arvrunner.api,
keep_client=arvrunner.keep_client,
num_retries=arvrunner.num_retries)
with collection.open("workflow.cwl", "w") as f:
f.write(json.dumps(packed, indent=2, sort_keys=True, separators=(',',': ')))
filters = [["portable_data_hash", "=", collection.portable_data_hash()],
["name", "like", name+"%"]]
if arvrunner.project_uuid:
filters.append(["owner_uuid", "=", arvrunner.project_uuid])
exists = arvrunner.api.collections().list(filters=filters).execute(num_retries=arvrunner.num_retries)
if exists["items"]:
logger.info("Using collection %s", exists["items"][0]["uuid"])
else:
collection.save_new(name=name,
owner_uuid=arvrunner.project_uuid,
ensure_unique_name=True,
num_retries=arvrunner.num_retries)
logger.info("Uploaded to %s", collection.manifest_locator())
return collection.portable_data_hash()
def open(self, fn, mode, encoding=None):
collection, rest = self.get_collection(fn)
if collection is not None:
return collection.open(rest, mode, encoding=encoding)
else:
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
def open(self, fn, mode):
collection, rest = self.get_collection(fn)
if collection:
return collection.open(rest, mode)
else:
return open(self._abs(fn), mode)
def open(self, fn, mode):
collection, rest = self.get_collection(fn)
if collection:
return collection.open(rest, mode)
else:
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
def open(self, fn, mode):
collection, rest = self.get_collection(fn)
if collection:
return collection.open(rest, mode)
else:
return open(self._abs(fn), mode)
def open(self, fn, mode):
collection, rest = self.get_collection(fn)
if collection:
return collection.open(rest, mode)
else:
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
