def download_files_from_gcs(client, gspath, localpath=None, log=False): """Use boto client to download gcs path to a local file.""" components = gspath.split('/') if localpath is None: localpath = components[-1] obj = client.storage_uri(gspath, "gs") if log: print "Downloading from GCS: %s" % gspath with open(localpath, "w") as f: obj.get_contents_to_file(f)
def list_files_in_gcs_bucket(client, gspath): """List all GCS files (csv, txt and json) matching the provided path.""" components = gspath.split('/') bucket = client.storage_uri(components[0], "gs") if len(components) == 1: filekeys = bucket.get_bucket().get_all_keys() else: filekeys = bucket.get_bucket().get_all_keys( prefix='/'.join(components[1:])) files = [f.name for f in filekeys if f.name.endswith('.csv') or f.name.endswith('.json') or f.name.endswith('.txt')] return files
def upload_files_to_gcs(client, filenames, localdir, gspath, log=False): """Use boto client to upload local files to GCS bucket.""" components = gspath.split('/') for filename in filenames: if log: print "Uploading to GCS: %s" % filename localname = os.path.join(localdir, filename) if len(components) > 1: destname = os.path.join('/'.join(components[1:]), filename) else: destname = filename bucket = client.storage_uri(components[0], "gs").get_bucket() new_file = bucket.new_key(destname) new_file.set_contents_from_filename(localname)
def list_files_in_gcs_bucket(client, gspath): """List all GCS files (csv, txt and json) matching the provided path.""" components = gspath.split('/') bucket = client.storage_uri(components[0], "gs") if len(components) == 1: filekeys = bucket.get_bucket().get_all_keys() else: filekeys = bucket.get_bucket().get_all_keys( prefix='/'.join(components[1:])) files = [ f.name for f in filekeys if f.name.endswith('.csv') or f.name.endswith('.json') or f.name.endswith('.txt') ] return files