def main(): logging.basicConfig(level=logging.INFO) hca_config = HCAConfig() hca_config[ "DSSClient"].swagger_url = f"https://dss.dev.data.humancellatlas.org/v1/swagger.json" dss = DSSClient(config=hca_config) projects = get_target_project_dirs(follow_links=True) for project in projects: log.info('Uploading %s', project) bundle_uuid = project.name assert str(UUID(bundle_uuid)) == bundle_uuid bundle = project / 'bundle' def file_uuid_callback(file_path: str): file_path = Path(file_path) file_name = file_path.name file_uuid = generate_file_uuid(bundle_uuid, file_name) log.info('Allocated UUID %s for file %s', file_uuid, file_path) if file_name.endswith('.json'): with file_path.open('rt') as f: document = json.load(f) if file_name == 'links.json': pass elif file_name == 'project_0.json': assert document['provenance'][ 'document_id'] == bundle_uuid else: assert document['provenance'][ 'document_id'] == file_uuid return file_uuid if bundle.is_dir(): response = dss.upload(src_dir=str(bundle), replica='aws', staging_bucket='lon-test-data', bundle_uuid=bundle_uuid, file_uuid_callback=file_uuid_callback) print( f'Successful upload. Bundle information is:\n{json.dumps(response, indent=4)}' ) else: log.warning('Skipping %s because metadata is missing', project)
from hca import HCAConfig from hca.dss import DSSClient import boto3 s3 = boto3.resource('s3') bucket = s3.Bucket('upload-test-unittest') hca_config = HCAConfig() hca_config[ "DSSClient"].swagger_url = f"https://dss.dev.data.humancellatlas.org/v1/swagger.json" dss = DSSClient(config=hca_config) print( dss.upload(src_dir="data/", replica="aws", staging_bucket="upload-test-unittest")) bucket.objects.all().delete() print("Upload successful")