def assert_updated_metadata(self, bucket, key_name):
key = Key(bucket, key_name)
key.get_contents_as_string()
# unchanged
self.assertEqual(key.get_metadata("uid"), "0")
# updated
self.assertEqual(key.content_disposition,
'attachment; filename="newname.txt"')
self.assertEqual(key.get_metadata("mtime"), "2222222222")
# removed
self.assertEqual(key.content_encoding, None)
self.assertEqual(key.get_metadata("with-hypen"), None)
# inserted
self.assertEqual(key.get_metadata("new-entry"), "NEW")
def push_file(filepath, relpath):
'''
Checks the mtime of the file and if it's been modified it pushes the entire file contents to the s3 bucket
:param str filepath: The filesystem path to the file pending for upload
:param str relpath: The root of the directory structure being backed up
'''
keyname = os.path.relpath(filepath, relpath)
logger.info("Accessing S3 key %s", keyname)
existing_key = key = bucket.get_key(keyname)
local_mtime = datetime.utcfromtimestamp(os.path.getmtime(filepath))
filestats = os.stat(filepath)
if key is None:
logger.info("Key doesn't exist, creating new key")
key = Key(bucket)
key.key = keyname
key.set_metadata('username', pwd.getpwuid(filestats.st_uid).pw_name)
key.set_metadata('unix-perms', oct(filestats.st_mode))
if key.get_metadata(
'mtime') != local_mtime.isoformat() or existing_key is None:
try:
logger.info("Pushing %s (%.2f MB) to S3", keyname,
os.path.getsize(filepath) * 1.0 / (1024 * 1024))
md5sum = hashfile(filepath, hashlib.md5())
logger.info("MD5 %s", md5sum.hexdigest())
key.set_metadata('mtime', local_mtime.isoformat())
key.set_metadata('md5', md5sum.hexdigest())
key.set_contents_from_filename(filepath)
except Exception as e:
logger.exception("Failed to upload %s", filepath)
else:
logger.info("No changes necessary")
def _upload_file(self,connection,bucket,localfile,bucketfile,ignoredates):
s3key = bucket.get_key(bucketfile)
if not s3key:
s3key = Key(bucket)
s3key.key = bucketfile
s3date = s3key.get_metadata("date")
if s3date: s3date = int(s3date)
lcdate = int(os.path.getmtime(localfile))
upload = False
if not s3date: upload = True
if s3date and lcdate > s3date: upload = True
if ignoredates: upload = True
if not upload: return ## don't upload, return
if self.dryrun: print "dry-run. %s : %s => %s" % (bucket.name,localfile,bucketfile)
else: print "%s : %s => %s" % (bucket.name,localfile,bucketfile)
filetype = localfile.split(".")[-1]
meta = self.get_metadata_for_filtetype(filetype)
if meta:
for metadata in meta:
for key in metadata:
print " => metdata: %s:%s" % (key,metadata[key])
if not self.dryrun:
s3key.set_metadata(key,metadata[key])
if not self.dryrun:
s3key.set_metadata("date",str(int(time.time())))
s3key.set_contents_from_filename(localfile)
def push_file(filepath, relpath):
'''
Checks the mtime of the file and if it's been modified it pushes the entire file contents to the s3 bucket
:param str filepath: The filesystem path to the file pending for upload
:param str relpath: The root of the directory structure being backed up
'''
keyname = os.path.relpath(filepath, relpath)
logger.info("Accessing S3 key %s", keyname)
existing_key = key = bucket.get_key(keyname)
local_mtime = datetime.utcfromtimestamp(os.path.getmtime(filepath))
filestats = os.stat(filepath)
if key is None:
logger.info("Key doesn't exist, creating new key")
key = Key(bucket)
key.key = keyname
key.set_metadata('username', pwd.getpwuid(filestats.st_uid).pw_name)
key.set_metadata('unix-perms', oct(filestats.st_mode))
if key.get_metadata('mtime') != local_mtime.isoformat() or existing_key is None:
try:
logger.info("Pushing %s (%.2f MB) to S3", keyname, os.path.getsize(filepath) * 1.0 / (1024 * 1024))
md5sum = hashfile(filepath, hashlib.md5())
logger.info("MD5 %s", md5sum.hexdigest())
key.set_metadata('mtime', local_mtime.isoformat())
key.set_metadata('md5', md5sum.hexdigest())
key.set_contents_from_filename(filepath)
except Exception as e:
logger.exception("Failed to upload %s", filepath)
else:
logger.info("No changes necessary")
def getKeyContent(self,name,key,string=True,path=None):
bucket=self.conn.get_bucket(name,validate=False)
k=Key(bucket)
k.key=key
if string:
return k.get_contents_as_string()
else:
if not path:
try:
if k.get_metadata('file'):
path=k.get_metadata('file')
except Exception:
print("Forgot to give the filename")
sys.exit(0)
return k.get_contents_to_filename(path)
else:
return k.get_contents_to_filename(path)
def getKeyContent(self,name,key,string=True,path=None):
bucket=self.conn.get_bucket(name,validate=False)
k=Key(bucket)
k.key=key
if string:
return k.get_contents_as_string()
else:
if not path:
try:
if k.get_metadata('file'):
path=k.get_metadata('file')
except Exception:
print "Forgot to give the filename"
sys.exit(0)
return k.get_contents_to_filename(path)
else:
return k.get_contents_to_filename(path)
def put(self, filedata, content_type, remote_path, force=False):
now = datetime.datetime.utcnow()
then = now + datetime.timedelta(self.expiration_days)
expires = then.strftime("%a, %d %b %Y %H:%M:%S GMT")
if self.aws_prefix:
remote_path = "%s/%s" % (self.aws_prefix, remote_path)
(hexdigest, b64digest) = mediasync.checksum(filedata)
raw_b64digest = b64digest # store raw b64digest to add as file metadata
# create initial set of headers
headers = {
"x-amz-acl":
"public-read",
"Content-Type":
content_type,
"Expires":
expires,
"Cache-Control":
'max-age=%d, public' % (self.expiration_days * 24 * 3600),
}
key = self._bucket.get_key(remote_path)
if key is None:
key = Key(self._bucket, remote_path)
key_meta = key.get_metadata('mediasync-checksum') or ''
s3_checksum = key_meta.replace(' ', '+')
if force or s3_checksum != raw_b64digest:
key.set_metadata('mediasync-checksum', raw_b64digest)
key.set_contents_from_string(filedata,
headers=headers,
md5=(hexdigest, b64digest))
# check to see if file should be gzipped based on content_type
# also check to see if filesize is greater than 1kb
if content_type in TYPES_TO_COMPRESS:
key = Key(self._bucket, "%s.gz" % remote_path)
filedata = mediasync.compress(filedata)
(hexdigest, b64digest) = mediasync.checksum(
filedata) # update checksum with compressed data
headers[
"Content-Disposition"] = 'inline; filename="%sgz"' % remote_path.split(
'/')[-1]
headers["Content-Encoding"] = 'gzip'
key.set_metadata('mediasync-checksum', raw_b64digest)
key.set_contents_from_string(filedata,
headers=headers,
md5=(hexdigest, b64digest))
return True
def photo_redirect(request, photo_id):
conn = boto.connect_s3()
bucket = conn.get_bucket('imagrphotostorage')
k = Key(bucket)
k.key = photo_id
picture = k.get_contents_as_string()
file_type = k.get_metadata('Content-Type')
response = HttpResponse(picture, content_type=file_type)
return response
def update_depts_if_necessary():
k = Key(bucket)
k.key = 'api-json'
old_time = k.get_metadata('timestamp')
if (int(old_time) + 300 < int(time.time())):
DEPARTMENTS = update_departments(dept_ids)
k.set_metadata('timestamp', str(int(time.time())))
k.set_contents_from_string(json.dumps(DEPARTMENTS))
else :
DEPARTMENTS = json.load(k.get_contents_as_string())
def test_write_over_key_with_meta(self):
"""
test that metadata does not persist when a key is written over
"""
key_name = "test-key"
test_string = os.urandom(1024)
test_string_1 = os.urandom(1024)
meta_key = "meta_key"
meta_value = "pork"
# create the bucket
bucket = self._s3_connection.create_unique_bucket()
self.assertTrue(bucket is not None)
# create an empty key
write_key = Key(bucket)
# set the name
write_key.name = key_name
# self.assertFalse(write_key.exists())
# set some metadata
write_key.set_metadata(meta_key, meta_value)
# upload some data
write_key.set_contents_from_string(test_string)
self.assertTrue(write_key.exists())
# create another key to write over the first key
write_key1 = Key(bucket, key_name)
# upload some data
write_key1.set_contents_from_string(test_string_1)
self.assertTrue(write_key.exists())
# create another key with the same name
read_key = Key(bucket, key_name)
# read back the data
returned_string = read_key.get_contents_as_string()
self.assertEqual(returned_string, test_string_1)
# get the metadata
returned_meta_value = read_key.get_metadata(meta_key)
self.assertEqual(returned_meta_value, None)
# delete the key
read_key.delete()
self.assertFalse(write_key.exists())
# delete the bucket
self._s3_connection.delete_bucket(bucket.name)
def test_write_over_key_with_meta(self):
"""
test that metadata does not persist when a key is written over
"""
key_name = "test-key"
test_string = os.urandom(1024)
test_string_1 = os.urandom(1024)
meta_key = "meta_key"
meta_value = "pork"
# create the bucket
bucket = self._s3_connection.create_unique_bucket()
self.assertTrue(bucket is not None)
# create an empty key
write_key = Key(bucket)
# set the name
write_key.name = key_name
# self.assertFalse(write_key.exists())
# set some metadata
write_key.set_metadata(meta_key, meta_value)
# upload some data
write_key.set_contents_from_string(test_string)
self.assertTrue(write_key.exists())
# create another key to write over the first key
write_key1 = Key(bucket, key_name)
# upload some data
write_key1.set_contents_from_string(test_string_1)
self.assertTrue(write_key.exists())
# create another key with the same name
read_key = Key(bucket, key_name)
# read back the data
returned_string = read_key.get_contents_as_string()
self.assertEqual(returned_string, test_string_1)
# get the metadata
returned_meta_value = read_key.get_metadata(meta_key)
self.assertEqual(returned_meta_value, None)
# delete the key
read_key.delete()
self.assertFalse(write_key.exists())
# delete the bucket
self._s3_connection.delete_bucket(bucket.name)
def restore_data_file(self, file_path, remote_key_name, remote_bucket=None):
fp = file(file_path, 'w')
key = Key(remote_bucket, remote_key_name)
self.get_contents_to_file(key, fp)
try:
uid = int(key.get_metadata('uid'))
gid = int(key.get_metadata('gid'))
os.chown(file_path, uid, gid)
except:
pass
key.close()
fp.close()
def put(self, filedata, content_type, remote_path, force=False):
now = datetime.datetime.utcnow()
then = now + datetime.timedelta(self.expiration_days)
expires = then.strftime("%a, %d %b %Y %H:%M:%S GMT")
if self.aws_prefix:
remote_path = "%s/%s" % (self.aws_prefix, remote_path)
(hexdigest, b64digest) = mediasync.checksum(filedata)
raw_b64digest = b64digest # store raw b64digest to add as file metadata
# create initial set of headers
headers = {
"x-amz-acl": "public-read",
"Content-Type": content_type,
"Expires": expires,
"Cache-Control": 'max-age=%d, public' % (self.expiration_days * 24 * 3600),
}
key = self._bucket.get_key(remote_path)
if key is None:
key = Key(self._bucket, remote_path)
key_meta = key.get_metadata('mediasync-checksum') or ''
s3_checksum = key_meta.replace(' ', '+')
if force or s3_checksum != raw_b64digest:
key.set_metadata('mediasync-checksum', raw_b64digest)
key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest))
# check to see if file should be gzipped based on content_type
# also check to see if filesize is greater than 1kb
if content_type in TYPES_TO_COMPRESS:
# Use a .gzt extension to avoid issues with Safari on OSX
key = Key(self._bucket, "%s.gzt" % remote_path)
filedata = mediasync.compress(filedata)
(hexdigest, b64digest) = mediasync.checksum(filedata) # update checksum with compressed data
headers["Content-Disposition"] = 'inline; filename="%sgzt"' % remote_path.split('/')[-1]
headers["Content-Encoding"] = 'gzip'
key.set_metadata('mediasync-checksum', raw_b64digest)
key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest))
return True
def put(self, filedata, content_type, remote_path, force=False):
now = datetime.datetime.utcnow()
then = now + datetime.timedelta(self.expiration_days)
expires = then.strftime("%a, %d %b %Y %H:%M:%S GMT")
if AWS_PREFIX:
remote_path = "%s/%s" % (AWS_PREFIX, remote_path)
(hexdigest, b64digest) = _checksum(filedata)
raw_b64digest = b64digest # store raw b64digest to add as file metadata
# create initial set of headers
headers = {
"x-amz-acl": "public-read",
"Content-Type": content_type,
"Expires": expires,
"Cache-Control": 'max-age=%d' % (self.expiration_days * 24 * 3600),
}
# check to see if file should be gzipped based on content_type
# also check to see if filesize is greater than 1kb
if content_type in TYPES_TO_COMPRESS and len(filedata) > 1024:
filedata = _compress(filedata)
headers["Content-Encoding"] = "gzip"
(hexdigest, b64digest) = _checksum(filedata) # update checksum with compressed data
key = self._bucket.get_key(remote_path)
if key is None:
key = Key(self._bucket)
key.key = remote_path
key_meta = key.get_metadata('mediasync-checksum') or ''
s3_checksum = key_meta.replace(' ', '+')
if force or s3_checksum != raw_b64digest:
key.set_metadata('mediasync-checksum', raw_b64digest)
key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest))
return True
def assert_metadata(self, bucket, key_name):
key = Key(bucket, key_name)
key.get_contents_as_string()
self.assertEqual(key.content_disposition,
'attachment; filename="metaname.txt"')
self.assertEqual(key.content_encoding, "identity")
# TODO: Expires header can be accessed by boto?
# self.assertEqual(key.expires, "Tue, 19 Jan 2038 03:14:07 GMT")
self.assertEqual(key.get_metadata("mtime"), "1364742057")
self.assertEqual(key.get_metadata("uid"), "0")
self.assertEqual(key.get_metadata("with-hypen"), "1")
# x-amz-meta-* headers should be normalized to lowercase
self.assertEqual(key.get_metadata("Mtime"), None)
self.assertEqual(key.get_metadata("MTIME"), None)
self.assertEqual(key.get_metadata("Uid"), None)
self.assertEqual(key.get_metadata("UID"), None)
self.assertEqual(key.get_metadata("With-Hypen"), None)
def main():
raw_input(
"I am about to create a bucket called 'test_bucket1' and a\n text file called 'HelloWorld.txt'. Press enter to continue."
)
print
with open("HelloWorld.txt", "w") as f:
f.writelines("I hope you can read this file!")
s3 = boto.connect_s3()
bucket1 = s3.create_bucket('test_bucket1')
#creates an s3 bucket.
print "'test_bucket1' should be created. GO CHECK! Press enter to continue."
raw_input()
#I am going to create two new keys
raw_input(
"I am going to add a textfile and picture to S3. Press enter to continue."
)
k = Key(bucket1)
picture = Key(bucket1)
picture.key = "picture"
picture.set_contents_from_filename("bearandi.jpg")
k.key = "helloWorld"
k.set_contents_from_filename("helloWorld.txt")
print
raw_input(
"Look at the files on S3. The Files will now be downloaded. Enter to continue."
)
print
#This line and the next download the files from S3
picture.get_contents_to_filename("newBear.jpg")
k.get_contents_to_filename("newHelloWorld.txt")
#delete a key
raw_input(
"File downloads 100% I am now going to delete the text file. Enter to continue."
)
print
#delete the text file.
bucket1.delete_key("helloWorld")
raw_input(
"The text file should now be deleted. I am now going to create 3 more buckets \nand delete one. Press enter to continue."
)
print
#create more buckets
bucket2 = s3.create_bucket("lab1_bucket2")
bucket3 = s3.create_bucket("lab1_bucket3")
bucket4 = s3.create_bucket("lab1_bucket4")
raw_input("The buckets were created. I will now delete lab1_bucket4.")
print
bucket4.delete()
raw_input(
"lab1_bucket4 deleted. I will now querry to see if buckets exist and if I have permision."
)
print
#find buckets
print "I am going to try the bucket names 'test_bucket1', which exists, and 'lab1_bucket4', which does not."
print
print "Here is a list of all buckets:"
print s3.get_all_buckets()
print
try:
print "test_bucket1:",
print bucket1.get_acl()
except NameError:
print "The bucket 'bucket1' name does not exist."
try:
print "lab1_bucket4:",
print bucket4.get_acl()
except:
print "That bucket 'lab1_bucket4' does not exist. Invalid name."
print
raw_input(
"I am now going to copy the picture from test_bucket1 to lab1_bucket2."
)
#move object
print
#kill object in 5 days
picture.copy("lab1_bucket2", "Bucket2Bear.jpg")
raw_input(
"There should now be a copied picture in lab1_bucket2.\nI will now add a new photo with a 5 day expiration and with reduced redundancy in bucket 3."
)
print
cycle = Lifecycle()
k3 = Key(bucket3)
cycle.add_rule("Five Days", "My Second picture", "Enabled", 5)
bucket3.configure_lifecycle(cycle)
k3.key = "My Second picture"
k3.set_contents_from_filename("GW2.jpg", reduced_redundancy=True)
raw_input(
"Check bucket3 for the new object with redundancy and an expiration.\nThe last bucket with versioning is going to be made."
)
print
#create last bucket
lastBucket = s3.create_bucket("last_bucket")
lastBucket.configure_versioning(True, False, None)
print "Version Status: ", #print versioning status
print lastBucket.get_versioning_status()
print
lastK = Key(lastBucket)
lastK.name = "MyFile"
lastK.set_contents_from_filename("helloWorld.txt")
#add original hello world
print "Added a hello world containing the string: '",
print lastK.get_contents_as_string()
print
#editted the same hello world
with open("helloWorld.txt", "a") as f:
f.writelines("\nI added some lines.\nLast Line.")
lastK.name = "MyFile"
lastK.set_contents_from_filename("helloWorld.txt")
print "Added a hello world containing the string: '",
print lastK.get_contents_as_string()
print
print "'.\nObject details: "
for version in lastBucket.list_versions():
print version.name
print version.version_id
print
print
toDelete = raw_input(
"There should now be two different versions. Type the version of the file you would like to delete: "
)
try:
print lastBucket.delete_key("MyFile", version_id=toDelete)
except:
print
raw_input("Version of the file you entered should be deleted.")
lastK.set_metadata("My meta data", "This is the meta data")
print
lastK.get_metadata("My meta data")
def main():
raw_input("I am about to create a bucket called 'test_bucket1' and a\n text file called 'HelloWorld.txt'. Press enter to continue.");
print;
with open("HelloWorld.txt", "w") as f:
f.writelines("I hope you can read this file!");
s3=boto.connect_s3();
bucket1=s3.create_bucket('test_bucket1'); #creates an s3 bucket.
print "'test_bucket1' should be created. GO CHECK! Press enter to continue.";
raw_input();
#I am going to create two new keys
raw_input("I am going to add a textfile and picture to S3. Press enter to continue.");
k=Key(bucket1);
picture=Key(bucket1);
picture.key="picture";
picture.set_contents_from_filename("bearandi.jpg");
k.key="helloWorld";
k.set_contents_from_filename("helloWorld.txt");
print;
raw_input("Look at the files on S3. The Files will now be downloaded. Enter to continue.");
print;
#This line and the next download the files from S3
picture.get_contents_to_filename("newBear.jpg");
k.get_contents_to_filename("newHelloWorld.txt");
#delete a key
raw_input("File downloads 100% I am now going to delete the text file. Enter to continue.");
print;
#delete the text file.
bucket1.delete_key("helloWorld");
raw_input("The text file should now be deleted. I am now going to create 3 more buckets \nand delete one. Press enter to continue.");
print;
#create more buckets
bucket2=s3.create_bucket("lab1_bucket2");
bucket3=s3.create_bucket("lab1_bucket3");
bucket4=s3.create_bucket("lab1_bucket4");
raw_input("The buckets were created. I will now delete lab1_bucket4.");
print;
bucket4.delete();
raw_input("lab1_bucket4 deleted. I will now querry to see if buckets exist and if I have permision.");
print;
#find buckets
print "I am going to try the bucket names 'test_bucket1', which exists, and 'lab1_bucket4', which does not."
print;
print "Here is a list of all buckets:";
print s3.get_all_buckets();
print;
try:
print "test_bucket1:",
print bucket1.get_acl();
except NameError:
print "The bucket 'bucket1' name does not exist.";
try:
print "lab1_bucket4:",
print bucket4.get_acl();
except :
print "That bucket 'lab1_bucket4' does not exist. Invalid name.";
print;
raw_input("I am now going to copy the picture from test_bucket1 to lab1_bucket2.");
#move object
print;
#kill object in 5 days
picture.copy("lab1_bucket2","Bucket2Bear.jpg");
raw_input("There should now be a copied picture in lab1_bucket2.\nI will now add a new photo with a 5 day expiration and with reduced redundancy in bucket 3.");
print;
cycle=Lifecycle();
k3=Key(bucket3);
cycle.add_rule("Five Days", "My Second picture", "Enabled", 5);
bucket3.configure_lifecycle(cycle);
k3.key="My Second picture";
k3.set_contents_from_filename("GW2.jpg", reduced_redundancy=True);
raw_input("Check bucket3 for the new object with redundancy and an expiration.\nThe last bucket with versioning is going to be made.");
print;
#create last bucket
lastBucket=s3.create_bucket("last_bucket");
lastBucket.configure_versioning(True, False, None);
print "Version Status: ", #print versioning status
print lastBucket.get_versioning_status();
print;
lastK=Key(lastBucket);
lastK.name="MyFile";
lastK.set_contents_from_filename("helloWorld.txt"); #add original hello world
print "Added a hello world containing the string: '",
print lastK.get_contents_as_string();
print;
#editted the same hello world
with open("helloWorld.txt", "a") as f:
f.writelines("\nI added some lines.\nLast Line.");
lastK.name="MyFile";
lastK.set_contents_from_filename("helloWorld.txt");
print "Added a hello world containing the string: '",
print lastK.get_contents_as_string();
print;
print "'.\nObject details: ";
for version in lastBucket.list_versions():
print version.name;
print version.version_id;
print;
print;
toDelete=raw_input("There should now be two different versions. Type the version of the file you would like to delete: ");
try:
print lastBucket.delete_key("MyFile", version_id=toDelete);
except:
print;
raw_input("Version of the file you entered should be deleted.");
lastK.set_metadata("My meta data", "This is the meta data");
print; lastK.get_metadata("My meta data");
