def publish_snapshot(store, dataset, snapshot, cookies=None, realm=None): """Publish a snapshot tag to S3, GitHub or both.""" ds = store.get_dataset(dataset) siblings = ds.siblings() # if realm parameter is not included, find the best target if realm is None: # if the dataset has a public sibling, use this as the export target # otherwise, use the private as the export target public_bucket_name = DatasetRealm(DatasetRealm.PUBLIC).s3_remote has_public_bucket = get_sibling_by_name(public_bucket_name, siblings) if has_public_bucket: realm = DatasetRealm(DatasetRealm.PUBLIC) else: realm = DatasetRealm(DatasetRealm.PRIVATE) else: realm = get_s3_realm(realm=realm) # Create the sibling if it does not exist s3_sibling(ds, siblings) # Export to S3 and GitHub in another worker publish_s3_async \ .s(store.annex_path, dataset, snapshot, realm.s3_remote, realm.s3_bucket, cookies) \ .apply_async(queue=publish_queue()) # Public publishes to GitHub if realm == DatasetRealm.PUBLIC and DATALAD_GITHUB_EXPORTS_ENABLED: # Create Github sibling only if GitHub is enabled github_sibling(ds, dataset, siblings) publish_github_async \ .s(store.annex_path, dataset, snapshot, realm.github_remote) \ .apply_async(queue=publish_queue())
def publish_snapshot(store, dataset, snapshot, cookies=None, realm=None): """Publish a snapshot tag to S3, GitHub or both.""" dataset_id = dataset ds = store.get_dataset(dataset) siblings = ds.siblings() # if realm parameter is not included, find the best target if realm is None: # if the dataset has a public sibling, use this as the export target # otherwise, use the private as the export target public_bucket_name = DatasetRealm(DatasetRealm.PUBLIC).s3_remote has_public_bucket = get_sibling_by_name(public_bucket_name, siblings) if has_public_bucket: realm = DatasetRealm(DatasetRealm.PUBLIC) else: realm = DatasetRealm(DatasetRealm.PRIVATE) else: realm = get_s3_realm(realm=realm) s3_remote = s3_sibling(ds, siblings) publish_target(ds, realm.s3_remote, snapshot) versions = s3_versions(ds, realm, snapshot) if (len(versions)): r = requests.post(url=GRAPHQL_ENDPOINT, json=file_urls_mutation(dataset_id, snapshot, versions), cookies=cookies) if r.status_code != 200: raise Exception(r.text) # Public publishes to GitHub if realm == DatasetRealm.PUBLIC and DATALAD_GITHUB_EXPORTS_ENABLED: github_remote = github_sibling(ds, dataset_id, siblings) publish_target(ds, realm.github_remote, snapshot)
def migrate_to_bucket(store, dataset, cookies=None, realm='PUBLIC'): """Migrate a dataset and all snapshots to an S3 bucket""" realm = get_s3_realm(realm=realm) dataset_id = dataset ds = store.get_dataset(dataset) tags = [tag['name'] for tag in ds.repo.get_tags()] siblings = ds.siblings() s3_remote = s3_sibling(ds, siblings, realm=realm) for tag in tags: publish_target(ds, realm.s3_remote, tag) # Public publishes to GitHub if realm == DatasetRealm.PUBLIC and DATALAD_GITHUB_EXPORTS_ENABLED: github_remote = github_sibling(ds, dataset_id, siblings) publish_target(ds, realm.github_remote, tag)
def get_dataset_realm(ds, siblings, realm=None): # if realm parameter is not included, find the best target if realm is None: # if the dataset has a public sibling, use this as the export target # otherwise, use the private as the export target public_bucket_name = DatasetRealm(DatasetRealm.PUBLIC).s3_remote has_public_bucket = get_sibling_by_name(public_bucket_name, siblings) if has_public_bucket: realm = DatasetRealm(DatasetRealm.PUBLIC) else: realm = DatasetRealm(DatasetRealm.PRIVATE) else: realm = get_s3_realm(realm=realm) return realm
def migrate_to_bucket(store, dataset, cookies=None, realm='PUBLIC'): """Migrate a dataset and all snapshots to an S3 bucket""" realm = get_s3_realm(realm=realm) dataset_id = dataset ds = store.get_dataset(dataset) tags = [tag['name'] for tag in ds.repo.get_tags()] siblings = ds.siblings() s3_remote = s3_sibling(ds, siblings, realm=realm) for tag in tags: publish_target(ds, realm.s3_remote, tag) versions = s3_versions(ds, realm, tag) if (len(versions)): r = requests.post( url=GRAPHQL_ENDPOINT, json=file_urls_mutation(dataset_id, tag, versions), cookies=cookies) if r.status_code != 200: raise Exception(r.text) # Public publishes to GitHub if realm == DatasetRealm.PUBLIC and DATALAD_GITHUB_EXPORTS_ENABLED: github_remote = github_sibling(ds, dataset_id, siblings) publish_target(ds, realm.github_remote, tag)
def get_realm(ds, siblings): return get_s3_realm(realm=realm)