def tearDown(self): logger.info('Tearing down...') try: b = self.connection.conn.get_bucket(self.bucket) k = b.get_key(key) k.delete() except: pass
def setUp(self): logger.info('Setting up') self.payload = '0' * 1024 self.nkeys = 4 self.bucket = 'freshscape' self.prefix = 'test/key' self.keys = [self.prefix + repr(i) for i in range(self.nkeys)] self.connection = s3po.Connection( 'access_id', 'secret_key', async=False)
def setUp(self): logger.info('Setting up') # This is 1MB, and should not be uploaded with multipart uploads # automatically self.small = '0' * 1024 * 1024 # This is 11MB, and should be uploaded with multipart uploads # automatically self.large = '01234567890' * 1024 * 1024 self.bucket = 'blogscape' self.key = '000000000-testing-%i' % int(time.time()) self.connection = s3po.Connection('access_id', 'secret_key', async=False)
def setUp(self): logger.info('Setting up') # This is 1MB, and should not be uploaded with multipart uploads # automatically self.small = '0' * 1024 * 1024 # This is 11MB, and should be uploaded with multipart uploads # automatically self.large = '01234567890' * 1024 * 1024 self.bucket = 'blogscape' self.key = '000000000-testing-%i' % int(time.time()) self.connection = s3po.Connection( 'access_id', 'secret_key', async=False)
def compressToFile(inf, outf, compression): # Given an input stream that is uncompressed, write its compressed # contenst out to the provided file inf.seek(0) if compression == 'gzip': logger.info('Compressing as gzip...') gzip.GzipFile(fileobj=outf, mode='wb').writelines(inf) elif compression == 'zlib' or compression == 'deflate': logger.info('Compressing as zlib/deflate...') # Prepare to read uncompressed content tmp = zlib.compressobj() # Read in 1MB chunks data = inf.read(1024 * 1024) while data: outf.write(tmp.compress(data)) data = inf.read(1024 * 1024) outf.write(tmp.flush()) else: outf.writelines(inf) return True
def decompressToFile(inf, outf, compression): # Given an input stream that's compressed, and the compression type, # read in the compressed format, and write out the decompressed content inf.seek(0) if compression == 'gzip': logger.info('Decompressing gzip content...') # Make a gzip file reader, and then write its decompressed # contents out to the file tmp = gzip.GzipFile(fileobj=inf, mode='r') outf.writelines(tmp) elif compression == 'deflate' or compression == 'zlib': logger.info('Decompressing zlib/deflate content...') # Prepare to read compressed content tmp = zlib.decompressobj() # Read the result in 1MB chunks, decompress, and write to the output data = inf.read(1024 * 1024) while data: outf.write(tmp.decompress(data)) data = inf.read(1024 * 1024) outf.write(tmp.flush()) else: outf.writelines(inf) return True
def backoff(attempt): # How much should we backoff? Exponential with a # base of 30. Return the number of seconds to wait sleep = 30 * (2**attempt) logger.info('Sleeping %is after attempt %i' % (sleep, attempt)) time.sleep(sleep)
def backoff(attempt): # How much should we backoff? Exponential with a # base of 30. Return the number of seconds to wait sleep = 30 * (2 ** attempt) logger.info('Sleeping %is after attempt %i' % (sleep, attempt)) time.sleep(sleep)