def validate(self):
deferreds = []
for bucket, key in self.objects:
producer = AwsProducer(bucket, key)
self.producers.append(producer)
deferreds.append(producer.verify())
deferredlist = DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
return deferredlist
def test_fields(self):
filename = make_random_temp_file()
key = os.path.basename(filename)
self.tempfiles.append(filename)
self.s3keys.append(key)
with open(filename) as f:
data = f.read()
crc32 = binascii.crc32(data) & 0xffffffff
s3client = S3Client()
yield s3client.put_object('zippy2-dev', key, data, metadata={'crc32': '%x' % crc32})
consumer = TestConsumer()
producer = AwsProducer('zippy2-dev', key)
yield producer.beginProducing(consumer)
consumer.seek(0)
self.assertEquals(data, consumer.read())
producer_crc32 = yield producer.crc32()
producer_size = yield producer.size()
self.assertEquals(crc32, producer_crc32)
self.assertEquals(len(data), producer_size)
@inlineCallbacks
def main():
options = Options()
try:
options.parseOptions()
except usage.UsageError, error:
print '%s' % error
sys.exit(1)
client = S3Client()
bucket = yield client.get_bucket(options['bucket'])
for item in bucket.contents:
print 'Getting CRC for %s' % item.key
producer = AwsProducer(options['bucket'], item.key)
consumer = Crc32Consumer()
yield producer.beginProducing(consumer)
crc32 = '%x' % consumer.crc32()
print '%s: %s' % (item.key, crc32)
print 'Copying metadata to %s' % item.key
yield client.copy_object(options['bucket'], item.key, metadata={ 'crc32': crc32 }, amz_headers={ 'storage-class': 'REDUCED_REDUNDANCY', 'metadata-directive': 'REPLACE' })
reactor.stop()
if __name__ == '__main__':