def download_files_from_gcs(client, gspath, localpath=None, log=False):
"""Use boto client to download gcs path to a local file."""
components = gspath.split('/')
if localpath is None:
localpath = components[-1]
obj = client.storage_uri(gspath, "gs")
if log:
print "Downloading from GCS: %s" % gspath
with open(localpath, "w") as f:
obj.get_contents_to_file(f)
def download_files_from_gcs(client, gspath, localpath=None, log=False):
"""Use boto client to download gcs path to a local file."""
components = gspath.split('/')
if localpath is None:
localpath = components[-1]
obj = client.storage_uri(gspath, "gs")
if log:
print "Downloading from GCS: %s" % gspath
with open(localpath, "w") as f:
obj.get_contents_to_file(f)
def list_files_in_gcs_bucket(client, gspath):
"""List all GCS files (csv, txt and json) matching the provided path."""
components = gspath.split('/')
bucket = client.storage_uri(components[0], "gs")
if len(components) == 1:
filekeys = bucket.get_bucket().get_all_keys()
else:
filekeys = bucket.get_bucket().get_all_keys(
prefix='/'.join(components[1:]))
files = [f.name for f in filekeys if f.name.endswith('.csv') or
f.name.endswith('.json') or f.name.endswith('.txt')]
return files
def upload_files_to_gcs(client, filenames, localdir, gspath, log=False):
"""Use boto client to upload local files to GCS bucket."""
components = gspath.split('/')
for filename in filenames:
if log:
print "Uploading to GCS: %s" % filename
localname = os.path.join(localdir, filename)
if len(components) > 1:
destname = os.path.join('/'.join(components[1:]), filename)
else:
destname = filename
bucket = client.storage_uri(components[0], "gs").get_bucket()
new_file = bucket.new_key(destname)
new_file.set_contents_from_filename(localname)
def upload_files_to_gcs(client, filenames, localdir, gspath, log=False):
"""Use boto client to upload local files to GCS bucket."""
components = gspath.split('/')
for filename in filenames:
if log:
print "Uploading to GCS: %s" % filename
localname = os.path.join(localdir, filename)
if len(components) > 1:
destname = os.path.join('/'.join(components[1:]), filename)
else:
destname = filename
bucket = client.storage_uri(components[0], "gs").get_bucket()
new_file = bucket.new_key(destname)
new_file.set_contents_from_filename(localname)
def list_files_in_gcs_bucket(client, gspath):
"""List all GCS files (csv, txt and json) matching the provided path."""
components = gspath.split('/')
bucket = client.storage_uri(components[0], "gs")
if len(components) == 1:
filekeys = bucket.get_bucket().get_all_keys()
else:
filekeys = bucket.get_bucket().get_all_keys(
prefix='/'.join(components[1:]))
files = [
f.name for f in filekeys if f.name.endswith('.csv')
or f.name.endswith('.json') or f.name.endswith('.txt')
]
return files
