def upload_workflow_collection(arvrunner, name, packed): collection = arvados.collection.Collection( api_client=arvrunner.api, keep_client=arvrunner.keep_client, num_retries=arvrunner.num_retries) with collection.open("workflow.cwl", "w") as f: f.write( json.dumps(packed, indent=2, sort_keys=True, separators=(',', ': '))) filters = [["portable_data_hash", "=", collection.portable_data_hash()], ["name", "like", name + "%"]] if arvrunner.project_uuid: filters.append(["owner_uuid", "=", arvrunner.project_uuid]) exists = arvrunner.api.collections().list(filters=filters).execute( num_retries=arvrunner.num_retries) if exists["items"]: logger.info("Using collection %s", exists["items"][0]["uuid"]) else: collection.save_new(name=name, owner_uuid=arvrunner.project_uuid, ensure_unique_name=True, num_retries=arvrunner.num_retries) logger.info("Uploaded to %s", collection.manifest_locator()) return collection.portable_data_hash()
def write_file(collection, pathprefix, fn, flush=False): with open(os.path.join(pathprefix, fn), "rb") as src: dst = collection.open(fn, "wb") r = src.read(1024*128) while r: dst.write(r) r = src.read(1024*128) dst.close(flush=flush)
def write_file(collection, pathprefix, fn): with open(os.path.join(pathprefix, fn)) as src: dst = collection.open(fn, "w") r = src.read(1024 * 128) while r: dst.write(r) r = src.read(1024 * 128) dst.close(flush=False)
def upload_workflow_collection(arvrunner, name, packed): collection = arvados.collection.Collection(api_client=arvrunner.api, keep_client=arvrunner.keep_client, num_retries=arvrunner.num_retries) with collection.open("workflow.cwl", "w") as f: f.write(json.dumps(packed, indent=2, sort_keys=True, separators=(',',': '))) filters = [["portable_data_hash", "=", collection.portable_data_hash()], ["name", "like", name+"%"]] if arvrunner.project_uuid: filters.append(["owner_uuid", "=", arvrunner.project_uuid]) exists = arvrunner.api.collections().list(filters=filters).execute(num_retries=arvrunner.num_retries) if exists["items"]: logger.info("Using collection %s", exists["items"][0]["uuid"]) else: collection.save_new(name=name, owner_uuid=arvrunner.project_uuid, ensure_unique_name=True, num_retries=arvrunner.num_retries) logger.info("Uploaded to %s", collection.manifest_locator()) return collection.portable_data_hash()
def open(self, fn, mode, encoding=None): collection, rest = self.get_collection(fn) if collection is not None: return collection.open(rest, mode, encoding=encoding) else: return super(CollectionFsAccess, self).open(self._abs(fn), mode)
def open(self, fn, mode): collection, rest = self.get_collection(fn) if collection: return collection.open(rest, mode) else: return open(self._abs(fn), mode)
def open(self, fn, mode): collection, rest = self.get_collection(fn) if collection: return collection.open(rest, mode) else: return super(CollectionFsAccess, self).open(self._abs(fn), mode)