def test__upload_uses_prefix(self, mock_write_files):
s3_mock = Mock()
local_path = '/local_path/static'
file_paths = ['/local_path/static/file1', '/local_path/static/file2']
files = {(local_path, '/static'): file_paths}
flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket')
expected_call = call(s3_mock,
self.app,
'/static',
local_path,
file_paths,
's3_bucket',
hashes=None)
self.assertEquals(mock_write_files.call_args_list, [expected_call])
for supported_prefix in ['foo', '/foo', 'foo/', '/foo/']:
mock_write_files.reset_mock()
self.app.config['FLASKS3_PREFIX'] = supported_prefix
flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket')
expected_call = call(s3_mock,
self.app,
'/foo/static',
local_path,
file_paths,
's3_bucket',
hashes=None)
self.assertEquals(mock_write_files.call_args_list, [expected_call])
def test__write_only_modified(self, key_mock):
""" Test that we only upload files that have changed """
self.app.config['FLASKS3_ONLY_MODIFIED'] = True
static_folder = tempfile.mkdtemp()
static_url_loc = static_folder
filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']]
expected = []
data_iter = count()
for filename in filenames:
# Write random data into files
with open(filename, 'wb') as f:
if six.PY3:
data = str(data_iter)
f.write(data.encode())
else:
data = str(data_iter.next())
f.write(data)
# We expect each file to be uploaded
expected.append(call.put_object(ACL='public-read',
Bucket=None,
Key=filename.lstrip("/"),
Body=data,
Metadata={},
Expires='Thu, 31 Dec 2037 23:59:59 GMT',
ContentEncoding='gzip'))
files = {(static_url_loc, static_folder): filenames}
hashes = flask_s3._upload_files(key_mock, self.app, files, None)
# All files are uploaded and hashes are returned
self.assertLessEqual(len(expected), len(key_mock.mock_calls))
self.assertEquals(len(hashes), len(filenames))
# We now modify the second file
with open(filenames[1], 'wb') as f:
data = str(next(data_iter))
if six.PY2:
f.write(data)
else:
f.write(data.encode())
# We expect only this file to be uploaded
expected.append(call.put_object(ACL='public-read',
Bucket=None,
Key=filenames[1].lstrip("/"),
Body=data,
Metadata={},
Expires='Thu, 31 Dec 2037 23:59:59 GMT',
ContentEncoding='gzip'))
new_hashes = flask_s3._upload_files(key_mock, self.app, files, None,
hashes=dict(hashes))
#import pprint
#pprint.pprint(zip(expected, key_mock.mock_calls))
self.assertEquals(len(expected), len(key_mock.mock_calls))
def test__write_only_modified(self, key_mock):
""" Test that we only upload files that have changed """
self.app.config['S3_ONLY_MODIFIED'] = True
static_folder = tempfile.mkdtemp()
static_url_loc = static_folder
filenames = [
os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']
]
expected = []
for filename in filenames:
# Write random data into files
with open(filename, 'wb') as f:
f.write(os.urandom(1024))
# We expect each file to be uploaded
expected.extend([
call(bucket=None, name=filename),
call().set_metadata('Expires',
'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filename),
call().make_public()
])
files = {(static_url_loc, static_folder): filenames}
hashes = flask_s3._upload_files(self.app, files, None)
# All files are uploaded and hashes are returned
self.assertLessEqual(expected, key_mock.mock_calls)
self.assertEquals(len(hashes), len(filenames))
# We now modify the second file
with open(filenames[1], 'wb') as f:
f.write(os.urandom(1024))
# We expect only this file to be uploaded
expected.extend([
call(bucket=None, name=filenames[1]),
call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filenames[1]),
call().make_public()
])
new_hashes = flask_s3._upload_files(self.app,
files,
None,
hashes=dict(hashes))
self.assertEqual(expected, key_mock.mock_calls)
def test__write_only_modified(self, key_mock):
""" Test that we only upload files that have changed """
self.app.config['S3_ONLY_MODIFIED'] = True
static_folder = tempfile.mkdtemp()
static_url_loc = static_folder
filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']]
expected = []
for filename in filenames:
# Write random data into files
with open(filename, 'wb') as f:
f.write(os.urandom(1024))
# We expect each file to be uploaded
expected.extend([call(bucket=None, name=filename),
call().set_metadata('Expires',
'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filename),
call().make_public()])
files = {(static_url_loc, static_folder): filenames}
hashes = flask_s3._upload_files(self.app, files, None)
# All files are uploaded and hashes are returned
self.assertLessEqual(expected, key_mock.mock_calls)
self.assertEquals(len(hashes), len(filenames))
# We now modify the second file
with open(filenames[1], 'wb') as f:
f.write(os.urandom(1024))
# We expect only this file to be uploaded
expected.extend([call(bucket=None, name=filenames[1]),
call().set_metadata('Expires',
'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filenames[1]),
call().make_public()])
new_hashes = flask_s3._upload_files(self.app, files, None,
hashes=dict(hashes))
self.assertEqual(expected, key_mock.mock_calls)
def test__upload_uses_prefix(self, mock_write_files):
s3_mock = Mock()
local_path = '/local_path/static'
file_paths = ['/local_path/static/file1', '/local_path/static/file2']
files = {(local_path, '/static'): file_paths}
flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket')
expected_call = call(
s3_mock, self.app, '/static', local_path, file_paths, 's3_bucket', hashes=None)
self.assertEquals(mock_write_files.call_args_list, [expected_call])
for supported_prefix in ['foo', '/foo', 'foo/', '/foo/']:
mock_write_files.reset_mock()
self.app.config['FLASKS3_PREFIX'] = supported_prefix
flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket')
expected_call = call(s3_mock, self.app, '/foo/static',
local_path, file_paths, 's3_bucket', hashes=None)
self.assertEquals(mock_write_files.call_args_list, [expected_call])
#!/usr/bin/env python
# Copyright 2013 Globo.com. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
import os
import re
import flask_s3
from boto.s3.connection import S3Connection
import app
location = None
endpoint = os.environ.get("TSURU_S3_ENDPOINT")
if endpoint:
location_regexp = re.compile("^https://([a-z0-9-]+)\.amazonaws\.com$")
m = location_regexp.match(endpoint)
if m:
location = m.groups()[0]
all_files = flask_s3._gather_files(app.app, False)
conn = S3Connection(os.environ.get("TSURU_S3_ACCESS_KEY_ID"),
os.environ.get("TSURU_S3_SECRET_KEY"))
bucket = conn.get_bucket(os.environ.get("TSURU_S3_BUCKET"))
print "Uploading static files to S3... ",
flask_s3._upload_files(app.app, all_files, bucket)
print "ok"