def store_fileobject(self, fileobject, path, interrupt_event=None): size = self.__get_size(fileobject) self.logger.info("Storing file object of size %s to %s", size, path) k = Key(self.bucket) k.key = path[1:] k.set_contents_from_file(fileobject) # does not return bytes written return int(time.time())
def download_file(self, source_bucket, source_path, dest_path, create_subdirs_if_needed=False, source_generation=None): """Downloads a single file from Google Cloud Storage to local disk. Args: source_bucket: GS bucket to download the file from source_path: full path (Posix-style) within that bucket dest_path: full path (local-OS-style) on local disk to copy the file to create_subdirs_if_needed: boolean; whether to create subdirectories as needed to create dest_path source_generation: the generation version of the source """ b = self._connect_to_bucket(bucket=source_bucket) key = Key(b) key.name = source_path if source_generation: key.generation = source_generation if create_subdirs_if_needed: _makedirs_if_needed(os.path.dirname(dest_path)) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % ( b.name, source_path, dest_path)) raise
def download_dir_contents(self, source_bucket, source_dir, dest_dir): """Recursively download contents of a Google Storage directory to local disk params: source_bucket: GS bucket to copy the files from source_dir: full path (Posix-style) within that bucket; read the files from this directory dest_dir: full path (local-OS-style) on local disk of directory to copy the files into The copy operates as a "merge with overwrite": any files in source_dir will be "overlaid" on top of the existing content in dest_dir. Existing files with the same names will be overwritten. TODO(epoger): Download multiple files simultaneously to reduce latency. """ _makedirs_if_needed(dest_dir) b = self._connect_to_bucket(bucket=source_bucket) (dirs, files) = self.list_bucket_contents( bucket=source_bucket, subdir=source_dir) for filename in files: key = Key(b) key.name = posixpath.join(source_dir, filename) dest_path = os.path.join(dest_dir, filename) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % ( b.name, key.name, dest_path)) raise
def download_dir_contents(self, source_bucket, source_dir, dest_dir): """Recursively download contents of a Google Storage directory to local disk params: source_bucket: GS bucket to copy the files from source_dir: full path (Posix-style) within that bucket; read the files from this directory dest_dir: full path (local-OS-style) on local disk of directory to copy the files into The copy operates as a "merge with overwrite": any files in source_dir will be "overlaid" on top of the existing content in dest_dir. Existing files with the same names will be overwritten. TODO(epoger): Download multiple files simultaneously to reduce latency. """ _makedirs_if_needed(dest_dir) b = self._connect_to_bucket(bucket=source_bucket) (dirs, files) = self.list_bucket_contents(bucket=source_bucket, subdir=source_dir) for filename in files: key = Key(b) key.name = posixpath.join(source_dir, filename) dest_path = os.path.join(dest_dir, filename) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = ( repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % (b.name, key.name, dest_path)) raise
def get_file(self, path_to_file): self.logger.info("getting file: %s", path_to_file) #self._raise_error_if_invalid_path(path_to_file) k = Key(self.bucket) k.key = path_to_file[1:] object_contents = k.get_contents_as_string() return object_contents
def WriteState(self): state = {'status': 'WAITING', 'update_time': time.time()} conn = boto.connect_gs(config.gs_access_key, config.gs_secret_key) bucket = conn.get_bucket(self._bucket) k = Key(bucket, '%s/%s/output/stitch.state' % (self._user, self._batch)) k.set_contents_from_string(simplejson.dumps(state, indent=2), policy='public-read', headers={'Content-Type': 'text/plain'})
def WriteState(self): state = { 'status': 'WAITING', 'update_time': time.time() } conn = boto.connect_gs(config.gs_access_key,config.gs_secret_key) bucket = conn.get_bucket(self._bucket) k = Key(bucket, '%s/%s/output/stitch.state' % (self._user, self._batch)) k.set_contents_from_string( simplejson.dumps(state, indent=2), policy='public-read',headers={'Content-Type': 'text/plain'})
def upload_to_GS(self, local_file_path, gs_file_path, replace=True): """ Args: local_file_path: local file path which user want to upload to google storage gs_file_path: file path that stored on google storage Return: upload result """ gs_file_path = self._normpath(gs_file_path) obj = Key(self.bucket) obj.key = gs_file_path obj.set_contents_from_filename(local_file_path, replace=replace) return True
def AddFile(self, filename, contents): part = 0 bytes_left = len(contents) parts = [] conn = boto.connect_gs(config.gs_access_key, config.gs_secret_key) bucket = conn.get_bucket(self._bucket) while bytes_left > 0: fname = '%s.%d' % (filename, part) parts.append(fname) offset = part * _CHUNK_SIZE k = Key(bucket, '%s/%s/input/%s' % (self._user, self._batch, fname)) k.set_contents_from_string(contents[offset:offset + _CHUNK_SIZE]) part += 1 bytes_left -= _CHUNK_SIZE self._files.append({'name': filename, 'chunks': parts})
def AddFile(self, filename, contents): part = 0 bytes_left = len(contents) parts = [] conn = boto.connect_gs(config.gs_access_key,config.gs_secret_key) bucket = conn.get_bucket(self._bucket) while bytes_left > 0: fname = '%s.%d' % (filename, part) parts.append(fname) offset = part * _CHUNK_SIZE k = Key(bucket, '%s/%s/input/%s' % (self._user, self._batch, fname)) k.set_contents_from_string( contents[offset:offset+_CHUNK_SIZE]) part += 1 bytes_left -= _CHUNK_SIZE self._files.append({'name': filename, 'chunks': parts})
def upload_photo(file_path): """Uploads single photo f""" file_name = os.path.basename(file_path) global i i += 1 print "Uploading {0} to Google Cloud Storage".format(file_name) k = Key(bucket) k.key = '{0}/{1}'.format(gs_bucket_destination_prefix, file_name) k.set_contents_from_filename(file_path) metadata = {'fileName': file_name} app.put('/{0}'.format(firebase_destination_prefix), '%.5i' % i, metadata) print "Moving {0}".format(file_name) os.rename(os.path.join(captured_path, file_name), os.path.join(uploaded_path, file_name))
def delete_file(self, bucket, path): """Delete a single file within a GS bucket. TODO(epoger): what if bucket or path does not exist? Should probably raise an exception. Implement, and add a test to exercise this. Params: bucket: GS bucket to delete a file from path: full path (Posix-style) of the file within the bucket to delete """ b = self._connect_to_bucket(bucket=bucket) key = Key(b) key.name = path try: key.delete() except BotoServerError, e: e.body = (repr(e.body) + ' while deleting gs://%s/%s' % (b.name, path)) raise
def download_file(self, source_bucket, source_path, dest_path, create_subdirs_if_needed=False, source_generation=None): """Downloads a single file from Google Cloud Storage to local disk. Args: source_bucket: GS bucket to download the file from source_path: full path (Posix-style) within that bucket dest_path: full path (local-OS-style) on local disk to copy the file to create_subdirs_if_needed: boolean; whether to create subdirectories as needed to create dest_path source_generation: the generation version of the source """ b = self._connect_to_bucket(bucket=source_bucket) key = Key(b) key.name = source_path if source_generation: key.generation = source_generation if create_subdirs_if_needed: _makedirs_if_needed(os.path.dirname(dest_path)) with open(dest_path, 'w') as f: try: key.get_contents_to_file(fp=f) except BotoServerError, e: e.body = (repr(e.body) + ' while downloading gs://%s/%s to local_path=%s' % (b.name, source_path, dest_path)) raise
def upload(self, package, data): key = Key(self.bucket) key.key = self.get_path(package) key.set_metadata('name', package.name) key.set_metadata('version', package.version) key.set_contents_from_string(data.read())
def _generate_url(self, package): """ Generate a signed url to the GCS file """ key = Key(self.bucket, self.get_path(package)) return key.generate_url(self.expire_after)
def delete(self, package): path = self.get_path(package) key = Key(self.bucket) key.key = path key.delete()
def upload_file(self, source_path, dest_bucket, dest_path, upload_if=UploadIf.ALWAYS, predefined_acl=None, fine_grained_acl_list=None): """Upload contents of a local file to Google Storage. params: source_path: full path (local-OS-style) on local disk to read from dest_bucket: GS bucket to copy the file to dest_path: full path (Posix-style) within that bucket upload_if: one of the UploadIf values, describing in which cases we should upload the file predefined_acl: which predefined ACL to apply to the file on Google Storage; must be one of the PredefinedACL values defined above. If None, inherits dest_bucket's default object ACL. fine_grained_acl_list: list of (id_type, id_value, permission) tuples to apply to the uploaded file (on top of the predefined_acl), or None if predefined_acl is sufficient TODO(epoger): Consider adding a do_compress parameter that would compress the file using gzip before upload, and add a "Content-Encoding:gzip" header so that HTTP downloads of the file would be unzipped automatically. See https://developers.google.com/storage/docs/gsutil/addlhelp/ WorkingWithObjectMetadata#content-encoding """ b = self._connect_to_bucket(bucket=dest_bucket) local_md5 = None # filled in lazily if upload_if == self.UploadIf.IF_NEW: old_key = b.get_key(key_name=dest_path) if old_key: print ('Skipping upload of existing file gs://%s/%s' % ( b.name, dest_path)) return elif upload_if == self.UploadIf.IF_MODIFIED: old_key = b.get_key(key_name=dest_path) if old_key: if not local_md5: local_md5 = _get_local_md5(path=source_path) if ('"%s"' % local_md5) == old_key.etag: print ( 'Skipping upload of unmodified file gs://%s/%s : %s' % ( b.name, dest_path, local_md5)) return elif upload_if != self.UploadIf.ALWAYS: raise Exception('unknown value of upload_if: %s' % upload_if) # Upload the file using a temporary name at first, in case the transfer # is interrupted partway through. if not local_md5: local_md5 = _get_local_md5(path=source_path) initial_key = Key(b) initial_key.name = dest_path + '-uploading-' + local_md5 try: initial_key.set_contents_from_filename(filename=source_path, policy=predefined_acl) except BotoServerError, e: e.body = (repr(e.body) + ' while uploading source_path=%s to gs://%s/%s' % ( source_path, b.name, initial_key.name)) raise
def upload_file(self, source_path, dest_bucket, dest_path, upload_if=UploadIf.ALWAYS, predefined_acl=None, fine_grained_acl_list=None): """Upload contents of a local file to Google Storage. params: source_path: full path (local-OS-style) on local disk to read from dest_bucket: GS bucket to copy the file to dest_path: full path (Posix-style) within that bucket upload_if: one of the UploadIf values, describing in which cases we should upload the file predefined_acl: which predefined ACL to apply to the file on Google Storage; must be one of the PredefinedACL values defined above. If None, inherits dest_bucket's default object ACL. fine_grained_acl_list: list of (id_type, id_value, permission) tuples to apply to the uploaded file (on top of the predefined_acl), or None if predefined_acl is sufficient TODO(epoger): Consider adding a do_compress parameter that would compress the file using gzip before upload, and add a "Content-Encoding:gzip" header so that HTTP downloads of the file would be unzipped automatically. See https://developers.google.com/storage/docs/gsutil/addlhelp/ WorkingWithObjectMetadata#content-encoding """ b = self._connect_to_bucket(bucket=dest_bucket) local_md5 = None # filled in lazily if upload_if == self.UploadIf.IF_NEW: old_key = b.get_key(key_name=dest_path) if old_key: print('Skipping upload of existing file gs://%s/%s' % (b.name, dest_path)) return elif upload_if == self.UploadIf.IF_MODIFIED: old_key = b.get_key(key_name=dest_path) if old_key: if not local_md5: local_md5 = _get_local_md5(path=source_path) if ('"%s"' % local_md5) == old_key.etag: print( 'Skipping upload of unmodified file gs://%s/%s : %s' % (b.name, dest_path, local_md5)) return elif upload_if != self.UploadIf.ALWAYS: raise Exception('unknown value of upload_if: %s' % upload_if) # Upload the file using a temporary name at first, in case the transfer # is interrupted partway through. if not local_md5: local_md5 = _get_local_md5(path=source_path) initial_key = Key(b) initial_key.name = dest_path + '-uploading-' + local_md5 try: initial_key.set_contents_from_filename(filename=source_path, policy=predefined_acl) except BotoServerError, e: e.body = (repr(e.body) + ' while uploading source_path=%s to gs://%s/%s' % (source_path, b.name, initial_key.name)) raise
def create_directory(self, directory): self.logger.info("creating directory %s", directory) k = Key(self.bucket) k.key = directory[1:]+'/' k.set_metadata('Content-Type', 'application/x-directory') k.set_contents_from_string('')