def download_dir_contents(self, source_bucket, source_dir, dest_dir): """Recursively download contents of a Google Storage directory to local disk params: source_bucket: GS bucket to copy the files from source_dir: full path (Posix-style) within that bucket; read the files from this directory dest_dir: full path (local-OS-style) on local disk of directory to copy the files into The copy operates as a "merge with overwrite": any files in source_dir will be "overlaid" on top of the existing content in dest_dir. Existing files with the same names will be overwritten. TODO(epoger): Download multiple files simultaneously to reduce latency. """ _makedirs_if_needed(dest_dir) b = self._connect_to_bucket(bucket=source_bucket) (dirs, files) = self.list_bucket_contents( bucket=source_bucket, subdir=source_dir) for filename in files: key = Key(b) key.name = posixpath.join(source_dir, filename) dest_path = os.path.join(dest_dir, filename) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % ( b.name, key.name, dest_path)) raise
def download_dir_contents(self, source_bucket, source_dir, dest_dir): """Recursively download contents of a Google Storage directory to local disk params: source_bucket: GS bucket to copy the files from source_dir: full path (Posix-style) within that bucket; read the files from this directory dest_dir: full path (local-OS-style) on local disk of directory to copy the files into The copy operates as a "merge with overwrite": any files in source_dir will be "overlaid" on top of the existing content in dest_dir. Existing files with the same names will be overwritten. TODO(epoger): Download multiple files simultaneously to reduce latency. """ _makedirs_if_needed(dest_dir) b = self._connect_to_bucket(bucket=source_bucket) (dirs, files) = self.list_bucket_contents(bucket=source_bucket, subdir=source_dir) for filename in files: key = Key(b) key.name = posixpath.join(source_dir, filename) dest_path = os.path.join(dest_dir, filename) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = ( repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % (b.name, key.name, dest_path)) raise
def download_file(self, source_bucket, source_path, dest_path, create_subdirs_if_needed=False, source_generation=None): """Downloads a single file from Google Cloud Storage to local disk. Args: source_bucket: GS bucket to download the file from source_path: full path (Posix-style) within that bucket dest_path: full path (local-OS-style) on local disk to copy the file to create_subdirs_if_needed: boolean; whether to create subdirectories as needed to create dest_path source_generation: the generation version of the source """ b = self._connect_to_bucket(bucket=source_bucket) key = Key(b) key.name = source_path if source_generation: key.generation = source_generation if create_subdirs_if_needed: _makedirs_if_needed(os.path.dirname(dest_path)) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % ( b.name, source_path, dest_path)) raise
def download_file(self, source_bucket, source_path, dest_path, create_subdirs_if_needed=False, source_generation=None): """Downloads a single file from Google Cloud Storage to local disk. Args: source_bucket: GS bucket to download the file from source_path: full path (Posix-style) within that bucket dest_path: full path (local-OS-style) on local disk to copy the file to create_subdirs_if_needed: boolean; whether to create subdirectories as needed to create dest_path source_generation: the generation version of the source """ b = self._connect_to_bucket(bucket=source_bucket) key = Key(b) key.name = source_path if source_generation: key.generation = source_generation if create_subdirs_if_needed: _makedirs_if_needed(os.path.dirname(dest_path)) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % (b.name, source_path, dest_path)) raise