def test_tell(self):
url = self.get_url("test_tell.txt")
f = s3open(url)
f.write(self.lorem)
f.close()
f = s3open(url)
self.assertEqual(str(f.read(8), 'utf-8'), self.lorem[:8])
self.assertEqual(f.tell(), 8)
def test_tell(self):
url = self.get_url('test_tell.txt')
f = s3open(url, key=self.key, secret=self.secret)
f.write(LOREM)
f.close()
f = s3open(url, key=self.key, secret=self.secret)
self.assertEqual(f.read(8), LOREM[:8])
self.assertEqual(f.tell(), 8)
def test_closed(self): path = "test_closed.txt" url = self.get_url(path) f = s3open(url) self.assertEqual(False, f.closed) f.close() self.assertEqual(True, f.closed)
def test_closed(self):
path = 'test_closed.txt'
url = self.get_url(path)
f = s3open(url, key=self.key, secret=self.secret)
self.assertEqual(False, f.closed)
f.close()
self.assertEqual(True, f.closed)
def test_binary_write(self):
path = 'test_binary_write.txt'
bs = self._bin_str()
f = s3open(self.get_url(path), key=self.key, secret=self.secret)
f.write(bs)
f.close()
k = Key(self.bucket, path)
self.assertEqual(k.get_contents_as_string(), bs)
def test_context_manager(self):
path = 'test_write_cm.txt'
with s3open(self.get_url(path), key=self.key, secret=self.secret) as f:
f.write(LOREM)
k = Key(self.bucket, path)
self.assertEqual(k.get_contents_as_string(), LOREM)
def test_binary_read(self):
path = 'test_binary_read.txt'
bs = self._bin_str()
k = Key(self.bucket, path)
k.set_contents_from_string(bs)
f = s3open(self.get_url(path), key=self.key, secret=self.secret)
self.assertEqual(f.read(), bs)
f.close()
def test_binary_write(self): path = "test_binary_write.txt" bs = self._bin_str() f = s3open(self.get_url(path)) f.write(bs) f.close() k = self.resource.Object(self.bucket.name, path) read = str(k.get()["Body"].read(), 'utf-8') self.assertEqual(read, bs)
def test_context_manager(self): path = "test_write_cm.txt" with s3open(self.get_url(path)) as f: f.write(self.lorem) k = self.resource.Object(self.bucket.name, path) read = str(k.get()["Body"].read(), 'utf-8') self.assertEqual(read, self.lorem)
def test_large_binary_write(self): path = "test_large_binary_write.txt" bs = self._bin_str() for i in range(0, 10): bs += bs f = s3open(self.get_url(path)) f.write(bs) f.close() k = self.resource.Object(self.bucket.name, path) self.assertEqual(str(k.get()["Body"].read(), 'utf-8'), bs)
def test_writelines(self): path = "test_writelines.txt" url = self.get_url(path) f = s3open(url) lines = self.lorem_est() res = "".join(lines) f.writelines(lines) f.close() k = self.resource.Object(self.bucket.name, path) self.assertEqual(str(k.get()["Body"].read(), 'utf-8'), res)
def test_binary_read(self): path = "test_binary_read.txt" bs = self._bin_str() k = self.resource.Object(self.bucket.name, path) k.put(Body=bs) url = self.get_url(path) f = s3open(url) read = str(f.read(), 'utf-8') self.assertEqual(read, bs) f.close()
def test_writelines(self):
path = "test_writelines.txt"
url = self.get_url(path)
f = s3open(url, key=self.key, secret=self.secret)
lines = self.lorem_est()
res = "".join(lines)
f.writelines(lines)
f.close()
k = Key(self.bucket, path)
self.assertEqual(k.get_contents_as_string(), res)
def test_readline(self): path = "test_readline.txt" url = self.get_url(path) lines = self.lorem_est() res = "".join(lines) k = self.resource.Object(self.bucket.name, path) k.put(Body=res) f = s3open(url) rline = str(f.readline(), 'utf-8') f.close() self.assertEqual(rline, self.lorem + "\n")
def test_read(self):
path = 'test_read.txt'
# write using boto
k = Key(self.bucket, path)
k.set_contents_from_string(LOREM)
# check contents using s3file
f = s3open(self.get_url(path), key=self.key, secret=self.secret)
self.assertEqual(f.read(), LOREM)
f.close()
def test_readline(self):
path = 'test_readline.txt'
url = self.get_url(path)
lines = self.lorem_est()
res = "".join(lines)
k = Key(self.bucket, path)
k.set_contents_from_string(res)
f = s3open(url, key=self.key, secret=self.secret)
rline = f.readline()
f.close()
self.assertEqual(rline, LOREM + '\n')
def test_write(self):
path = 'test_write.txt'
# write using s3file
f = s3open(self.get_url(path), key=self.key, secret=self.secret)
f.write(LOREM)
f.close()
# check contents using boto
k = Key(self.bucket, path)
self.assertEqual(k.get_contents_as_string(), LOREM)
def test_read(self): path = "test_read.txt" # write using boto k = self.resource.Object(self.bucket.name, path) k.put(Body=self.lorem) # check contents using s3file f = s3open(self.get_url(path)) self.assertEqual(str(f.read(), 'utf-8'), self.lorem) f.close()
def test_write(self): path = "test_write.txt" # write using s3file f = s3open(self.get_url(path)) f.write(self.lorem) f.close() # check contents using boto k = self.resource.Object(self.bucket.name, path) self.assertEqual(str(k.get()["Body"].read(), 'utf-8'), self.lorem)
def test_large_binary_read(self): path = "test_large_binary_read.txt" bs = self._bin_str() for i in range(0, 10): bs += bs k = self.resource.Object(self.bucket.name, path) k.put(Body=bs) f = s3open(self.get_url(path)) read = str(f.read(), 'utf-8') self.assertEqual(read, bs) f.close()
def test_truncate(self): path = "test_truncate.txt" url = self.get_url(path) lines = self.lorem_est() res = "".join(lines) k = self.resource.Object(self.bucket.name, path) k.put(Body=res) f = s3open(url) dummy = f.read() # not convinced we should do this but down to the trucate in the write f.truncate(3) f.close() t = s3open(url) self.assertEqual(str(t.read(), 'utf-8'), res[:3]) t.seek(1, 0) t.truncate() t.close() f = s3open(url) self.assertEqual(str(f.read(), 'utf-8'), res[:1]) f.close()
def test_readlines(self): path = "test_readlines.txt" url = self.get_url(path) lines = self.lorem_est() res = "".join(lines) k = self.resource.Object(self.bucket.name, path) k.put(Body=res) f = s3open(url) rlines = f.readlines() rres = "".join([str(s, 'utf-8') for s in rlines]) f.close() self.assertEqual(res, rres)
def test_truncate(self):
path = 'test_truncate.txt'
url = self.get_url(path)
lines = self.lorem_est()
res = "".join(lines)
k = Key(self.bucket, path)
k.set_contents_from_string(res)
f = s3open(url, key=self.key, secret=self.secret)
dummy = f.read() # not convinced we should do this but down to the trucate in the write
f.truncate(3)
f.close()
t = s3open(url, key=self.key, secret=self.secret)
self.assertEqual(t.read(), res[:3])
t.seek(1,0)
t.truncate()
t.close()
f = s3open(url, key=self.key, secret=self.secret)
self.assertEqual(f.read(), res[:1])
f.close()
def test_flush(self): path = "test_flush.txt" url = self.get_url(path) fl = self.lorem + "\n" + self.lorem + "\n" fl2 = fl + fl f = s3open(url) f.write(fl) f.flush() k = self.resource.Object(self.bucket.name, path) self.assertEqual(str(k.get()["Body"].read(), 'utf-8'), fl) f.write(fl) f.close() self.assertEqual(str(k.get()["Body"].read(), 'utf-8'), fl2)
def test_flush(self):
path = 'test_flush.txt'
url = self.get_url(path)
fl = LOREM + "\n" + LOREM + "\n"
fl2 = fl + fl
f = s3open(url, key=self.key, secret=self.secret)
f.write(fl)
f.flush()
k = Key(self.bucket, path)
self.assertEqual(k.get_contents_as_string(), fl)
f.write(fl)
f.close()
self.assertEqual(k.get_contents_as_string(), fl2)
def test_xreadlines(self):
path = 'test_xreadlines.txt'
url = self.get_url(path)
lines = self.lorem_est()
res = "".join(lines)
k = Key(self.bucket, path)
k.set_contents_from_string(res)
f = s3open(url, key=self.key, secret=self.secret)
rres = ""
for lin in f.xreadlines():
rres += lin
f.close()
self.assertEqual(res, rres)
def save(self, thistag):
db = pymongo.Connection()[settings['database']]
table = db[settings['table']]
# if there's a file, store it in s3 and replace the file body
# with the s3 url
if thistag.get('file', None):
unique_filename = str(uuid.uuid4())
file_url = "http://assets.2d.sunlightlabs.com/%s" % unique_filename
f = s3open(file_url, settings['S3_KEY'], settings['S3_SECRET'])
f.write(thistag['file'])
f.close()
thistag['file'] = file_url
thistag['created'] = datetime.datetime.now()
if not thistag.get('id', None):
# create 'contents' as a list of items; people may append
# content to existing tags
tag = {'contents': [thistag,],
# last_updated will be updated each time a new item is
# added to this story
'last_updated' : thistag['created'],
'created' : thistag['created'],
}
_id = table.insert(tag, safe=True)
if self.get_secure_cookie("created"):
newvalue = self.get_secure_cookie("created") + "["+str(_id)+"]"
self.set_secure_cookie("created", newvalue)
else:
self.set_secure_cookie("created", "[%s]" % str(_id))
else:
_id = pymongo.objectid.ObjectId(thistag.get('id'))
record = table.find_one({'_id': _id})
# this is totally not the best way to do this; should
# design thistag data structure better...
del thistag['id']
record['contents'].append(thistag)
record['last_updated'] = thistag['created']
table.save(record)
if self.get_secure_cookie("updated"):
newvalue = self.get_secure_cookie("updated") + "["+str(_id)+"]"
self.set_secure_cookie("updated", newvalue)
else:
self.set_secure_cookie("updated", "[%s]" % str(_id))
db.connection.disconnect()
return _id
def pull_from_s3( s3_url, s3_index_file, output_dir ):
"""
Pull From s3: copy data from an S3 bucket given an index file.
s3_url: URL of the S3 bucket
s3_index_file: A link to an index file in an s3 bucket.
output_dor: Location where we're saving stuff locally
"""
# open index file
remote_index_handle = s3open( s3_url + "/" + s3_index_file )
for site_name in remote_index_handle.readlines():
site_name = site_name[:-1] # strip out newlines
site_posts = site_name + "/Posts.xml"
# make the output directory; if it exists simply continue
try:
os.makedirs( output_dir + "/" + site_name )
except OSError:
print( "Site directory '%s' exists locally, skipping." % site_name )
continue
# create pointers to the remote and local files
remote_posts_handle = s3open( s3_url + "/" + site_posts )
local_posts_handle = open( output_dir + "/" + site_posts, "w+" )
for line in remote_posts_handle.readlines():
local_posts_handle.write( line )
local_posts_handle.close()
remote_posts_handle.close()
remote_index_handle.close()
def download_s3(self, s3_bucket_name, video_dir='', chunk_size=8 * 1024, on_progress=None,
on_finish=None):
"""
Downloads video to S3
:param str s3_bucket_name:
S3 bucket name with output directory
"""
# Setup AWS S3 connections
s3_connection = boto.connect_s3()
video_key_name = "{0}.{1}".format(self.filename, self.extension)
if video_dir:
video_key_name = os.path.join(video_dir, video_key_name)
video_key = s3_connection.get_bucket(s3_bucket_name).new_key(video_key_name)
# Download the video
response = urlopen(self.url)
meta_data = dict(response.info().items())
file_size = int(meta_data.get("Content-Length") or
meta_data.get("content-length"))
self._bytes_received = 0
start = clock()
try:
with s3open(video_key.generate_url(60), create=False) as dst_file:
while True:
self._buffer = response.read(chunk_size)
# Check if the buffer is empty (aka no bytes remaining).
if not self._buffer:
if on_finish:
# TODO: We possibly want to flush the
# `_bytes_recieved`` buffer before we call
# ``on_finish()``.
on_finish(path)
break
self._bytes_received += len(self._buffer)
dst_file.write(self._buffer)
if on_progress:
on_progress(self._bytes_received, file_size, start)
except KeyboardInterrupt:
# TODO: Move this into the cli, ``KeyboardInterrupt`` handling
# should be taken care of by the client. Also you should be allowed
# to disable this.
video_key.delete()
raise KeyboardInterrupt(
"Interrupt signal given. Deleting incomplete video.")
def test_seek(self):
# needs start, relative, end
path = 'test_seek.txt'
url = self.get_url(path)
lines = self.lorem_est()
res = "".join(lines)
k = Key(self.bucket, path)
k.set_contents_from_string(res)
f = s3open(url, key=self.key, secret=self.secret)
f.seek(2,0)
self.assertEqual(f.read(8), res[2:10])
f.seek(1)
self.assertEqual(f.read(8), res[1:9])
f.seek(-1,1)
self.assertEqual(f.read(9), res[8:17])
f.seek(-10,2)
self.assertEqual(f.read(10), res[-10:])
f.close()
def download_s3(self,
s3_bucket_name,
aws_key,
aws_secret,
video_dir='',
chunk_size=8 * 1024,
on_progress=None,
on_finish=None,
acl_permission='public-read'):
"""
Downloads video to S3
:param str s3_bucket_name:
S3 bucket name with output directory
:param str aws_key:
AWS Key id for the account
:param str aws_secret:
AWS Secret Key for the account
:param str video_dir:
video directory to store the file within the bucket
:param int chunk_size:
File size (in bytes) to write to buffer at a time. By default,
this is set to 8 bytes.
:param func on_progress:
*Optional* function to be called every time the buffer is written
to. Arguments passed are the bytes recieved, file size, and start
datetime.
:param func on_finish:
*Optional* callback function when download is complete. Arguments
passed are the full path to downloaded the file.
:param str acl_permission:
*Optional* set the acl permissions for the new key saved to s3.
:return: tuple of s3_bucket_name, video_key_path
"""
# Setup AWS S3 connections
s3_connection = boto.connect_s3(aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret)
# TODO: Create bucket if it doesn't exist
video_key_path = "{0}.{1}".format(self.filename, self.extension)
if video_dir:
video_key_path = os.path.join(video_dir, video_key_path)
try:
video_key = s3_connection.get_bucket(s3_bucket_name).new_key(
video_key_path)
except:
print "S3 bucket does not exist. Cannot save file to S3"
return None, None
# Download the video
response = urlopen(self.url)
meta_data = dict(response.info().items())
file_size = int(
meta_data.get("Content-Length") or meta_data.get("content-length"))
self._bytes_received = 0
start = clock()
try:
with s3open(video_key.generate_url(60),
key=aws_key,
secret=aws_secret,
create=False) as dst_file:
while True:
self._buffer = response.read(chunk_size)
# Check if the buffer is empty (aka no bytes remaining).
if not self._buffer:
if on_finish:
# TODO: We possibly want to flush the
# `_bytes_recieved`` buffer before we call
# ``on_finish()``.
# on_finish(path)
pass
break
self._bytes_received += len(self._buffer)
dst_file.write(self._buffer)
if on_progress:
on_progress(self._bytes_received, file_size, start)
# Set key permissions to public read after download
video_key.set_acl(acl_permission)
return s3_bucket_name, video_key_path
except KeyboardInterrupt:
# TODO: Move this into the cli, ``KeyboardInterrupt`` handling
# should be taken care of by the client. Also you should be allowed
# to disable this.
video_key.delete()
raise KeyboardInterrupt(
"Interrupt signal given. Deleting incomplete video.")
def test_name(self):
path = 'test_name.txt'
url = self.get_url(path)
f = s3open(url, key=self.key, secret=self.secret)
self.assertEqual("s3://" + self.bucket.name + "/" + path, f.name)
f.close()
