def test__upload_uses_prefix(self, mock_write_files): s3_mock = Mock() local_path = '/local_path/static' file_paths = ['/local_path/static/file1', '/local_path/static/file2'] files = {(local_path, '/static'): file_paths} flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket') expected_call = call(s3_mock, self.app, '/static', local_path, file_paths, 's3_bucket', hashes=None) self.assertEquals(mock_write_files.call_args_list, [expected_call]) for supported_prefix in ['foo', '/foo', 'foo/', '/foo/']: mock_write_files.reset_mock() self.app.config['FLASKS3_PREFIX'] = supported_prefix flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket') expected_call = call(s3_mock, self.app, '/foo/static', local_path, file_paths, 's3_bucket', hashes=None) self.assertEquals(mock_write_files.call_args_list, [expected_call])
def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['FLASKS3_ONLY_MODIFIED'] = True static_folder = tempfile.mkdtemp() static_url_loc = static_folder filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']] expected = [] data_iter = count() for filename in filenames: # Write random data into files with open(filename, 'wb') as f: if six.PY3: data = str(data_iter) f.write(data.encode()) else: data = str(data_iter.next()) f.write(data) # We expect each file to be uploaded expected.append(call.put_object(ACL='public-read', Bucket=None, Key=filename.lstrip("/"), Body=data, Metadata={}, Expires='Thu, 31 Dec 2037 23:59:59 GMT', ContentEncoding='gzip')) files = {(static_url_loc, static_folder): filenames} hashes = flask_s3._upload_files(key_mock, self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(len(expected), len(key_mock.mock_calls)) self.assertEquals(len(hashes), len(filenames)) # We now modify the second file with open(filenames[1], 'wb') as f: data = str(next(data_iter)) if six.PY2: f.write(data) else: f.write(data.encode()) # We expect only this file to be uploaded expected.append(call.put_object(ACL='public-read', Bucket=None, Key=filenames[1].lstrip("/"), Body=data, Metadata={}, Expires='Thu, 31 Dec 2037 23:59:59 GMT', ContentEncoding='gzip')) new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, hashes=dict(hashes)) #import pprint #pprint.pprint(zip(expected, key_mock.mock_calls)) self.assertEquals(len(expected), len(key_mock.mock_calls))
def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['S3_ONLY_MODIFIED'] = True static_folder = tempfile.mkdtemp() static_url_loc = static_folder filenames = [ os.path.join(static_folder, f) for f in ['foo.css', 'bar.css'] ] expected = [] for filename in filenames: # Write random data into files with open(filename, 'wb') as f: f.write(os.urandom(1024)) # We expect each file to be uploaded expected.extend([ call(bucket=None, name=filename), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename(filename), call().make_public() ]) files = {(static_url_loc, static_folder): filenames} hashes = flask_s3._upload_files(self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(expected, key_mock.mock_calls) self.assertEquals(len(hashes), len(filenames)) # We now modify the second file with open(filenames[1], 'wb') as f: f.write(os.urandom(1024)) # We expect only this file to be uploaded expected.extend([ call(bucket=None, name=filenames[1]), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename(filenames[1]), call().make_public() ]) new_hashes = flask_s3._upload_files(self.app, files, None, hashes=dict(hashes)) self.assertEqual(expected, key_mock.mock_calls)
def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['S3_ONLY_MODIFIED'] = True static_folder = tempfile.mkdtemp() static_url_loc = static_folder filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']] expected = [] for filename in filenames: # Write random data into files with open(filename, 'wb') as f: f.write(os.urandom(1024)) # We expect each file to be uploaded expected.extend([call(bucket=None, name=filename), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename(filename), call().make_public()]) files = {(static_url_loc, static_folder): filenames} hashes = flask_s3._upload_files(self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(expected, key_mock.mock_calls) self.assertEquals(len(hashes), len(filenames)) # We now modify the second file with open(filenames[1], 'wb') as f: f.write(os.urandom(1024)) # We expect only this file to be uploaded expected.extend([call(bucket=None, name=filenames[1]), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename(filenames[1]), call().make_public()]) new_hashes = flask_s3._upload_files(self.app, files, None, hashes=dict(hashes)) self.assertEqual(expected, key_mock.mock_calls)
def test__upload_uses_prefix(self, mock_write_files): s3_mock = Mock() local_path = '/local_path/static' file_paths = ['/local_path/static/file1', '/local_path/static/file2'] files = {(local_path, '/static'): file_paths} flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket') expected_call = call( s3_mock, self.app, '/static', local_path, file_paths, 's3_bucket', hashes=None) self.assertEquals(mock_write_files.call_args_list, [expected_call]) for supported_prefix in ['foo', '/foo', 'foo/', '/foo/']: mock_write_files.reset_mock() self.app.config['FLASKS3_PREFIX'] = supported_prefix flask_s3._upload_files(s3_mock, self.app, files, 's3_bucket') expected_call = call(s3_mock, self.app, '/foo/static', local_path, file_paths, 's3_bucket', hashes=None) self.assertEquals(mock_write_files.call_args_list, [expected_call])
#!/usr/bin/env python # Copyright 2013 Globo.com. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. import os import re import flask_s3 from boto.s3.connection import S3Connection import app location = None endpoint = os.environ.get("TSURU_S3_ENDPOINT") if endpoint: location_regexp = re.compile("^https://([a-z0-9-]+)\.amazonaws\.com$") m = location_regexp.match(endpoint) if m: location = m.groups()[0] all_files = flask_s3._gather_files(app.app, False) conn = S3Connection(os.environ.get("TSURU_S3_ACCESS_KEY_ID"), os.environ.get("TSURU_S3_SECRET_KEY")) bucket = conn.get_bucket(os.environ.get("TSURU_S3_BUCKET")) print "Uploading static files to S3... ", flask_s3._upload_files(app.app, all_files, bucket) print "ok"