Example #1
0
def gcs_maker(populate=False, **kwargs):
    gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN, **kwargs)
    gcs.invalidate_cache()
    try:
        # ensure we're empty.
        try:
            gcs.rm(TEST_BUCKET, recursive=True)
        except FileNotFoundError:
            pass
        try:
            gcs.mkdir(
                TEST_BUCKET, default_acl="authenticatedread", acl="publicReadWrite"
            )
        except Exception:
            pass

        if populate:
            gcs.pipe({TEST_BUCKET + "/" + k: v for k, v in allfiles.items()})
        gcs.invalidate_cache()
        yield gcs
    finally:
        try:
            gcs.rm(gcs.find(TEST_BUCKET))
        except:  # noqa: E722
            pass
Example #2
0
def gcs_maker(populate=False):
    gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN)
    gcs.invalidate_cache()
    try:
        try:
            gcs.mkdir(TEST_BUCKET, default_acl="authenticatedread",
                      acl="publicReadWrite")
        except:
            pass
        for k in [a, b, c, d]:
            try:
                gcs.rm(k)
            except:
                pass
        if populate:
            for flist in [files, csv_files, text_files]:
                for fname, data in flist.items():
                    with gcs.open(TEST_BUCKET+'/'+fname, 'wb') as f:
                        f.write(data)
        yield gcs
    finally:
        for f in gcs.find(TEST_BUCKET):
            try:
                gcs.rm(f)
            except:
                pass
Example #3
0
def gcs_maker(populate=False):
    gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN)
    try:
        if not gcs.exists(TEST_BUCKET):
            gcs.mkdir(TEST_BUCKET)
        for k in [a, b, c, d]:
            try:
                gcs.rm(k)
            except:
                pass
        if populate:
            for flist in [files, csv_files, text_files]:
                for fname, data in flist.items():
                    with gcs.open(TEST_BUCKET + '/' + fname, 'wb') as f:
                        f.write(data)
        yield gcs
    finally:
        [gcs.rm(f) for f in gcs.walk(TEST_BUCKET)]
Example #4
0
def gcs_maker(populate=False):
    gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN)
    gcs.invalidate_cache()
    try:
        try:
            gcs.mkdir(TEST_BUCKET,
                      default_acl="authenticatedread",
                      acl="publicReadWrite")
        except gcsfs.utils.HttpError:
            pass

        # ensure we're empty.
        gcs.rm(TEST_BUCKET, recursive=True)

        for k in [a, b, c, d]:
            try:
                gcs.rm(k)
            except FileNotFoundError:
                pass
        if populate:
            for flist in [files, csv_files, text_files]:
                for fname, data in flist.items():
                    with gcs.open(TEST_BUCKET + "/" + fname, "wb") as f:
                        f.write(data)
        gcs.invalidate_cache()
        yield gcs
    finally:
        for f in gcs.find(TEST_BUCKET):
            try:
                gcs.rm(f)
            except:  # noqa: E722
                pass