def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
"""Uploads any assets that are in the given manifest and in our compiled
output dir but missing from our static assets bucket to that bucket on S3.
"""
# We will gather a set of (file_name, file_path) tuples to be uploaded
to_upload = set()
# We know we want to upload each asset block (these correspond to the
# assetman.include_* blocks in each template)
for depspec in manifest.blocks.itervalues():
file_name = depspec['versioned_path']
file_path = make_output_path(settings['compiled_asset_root'],
file_name)
assert os.path.isfile(
file_path), 'Missing compiled asset %s' % file_path
to_upload.add((file_name, file_path))
# And we know that we'll want to upload any statically-referenced assets
# (from assetman.static_url calls or referenced in any compiled assets),
# but we'll need to filter out other entries in the complete 'assets'
# block of the manifest.
should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search
for rel_path, depspec in manifest.assets.iteritems():
if should_skip(rel_path):
continue
file_path = make_absolute_static_path(settings['static_dir'], rel_path)
assert os.path.isfile(file_path), 'Missing static asset %s' % file_path
file_name = depspec['versioned_path']
to_upload.add((file_name, file_path))
logging.info('Found %d assets to upload to S3', len(to_upload))
if skip_s3_upload:
logging.info('Skipping asset upload to S3 %s', to_upload)
return to_upload
# Upload assets to S3 using 5 threads
queue = Queue.Queue()
errors = []
for i in xrange(5):
uploader = S3UploadThread(queue, errors, manifest, settings)
uploader.setDaemon(True)
uploader.start()
map(queue.put, to_upload)
queue.join()
if errors:
raise Exception(errors)
return to_upload
def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
"""Uploads any assets that are in the given manifest and in our compiled
output dir but missing from our static assets bucket to that bucket on S3.
"""
# We will gather a set of (file_name, file_path) tuples to be uploaded
to_upload = set()
# We know we want to upload each asset block (these correspond to the
# assetman.include_* blocks in each template)
for depspec in manifest.blocks.itervalues():
file_name = depspec['versioned_path']
file_path = make_output_path(settings['compiled_asset_root'], file_name)
assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path
to_upload.add((file_name, file_path))
# And we know that we'll want to upload any statically-referenced assets
# (from assetman.static_url calls or referenced in any compiled assets),
# but we'll need to filter out other entries in the complete 'assets'
# block of the manifest.
should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search
for rel_path, depspec in manifest.assets.iteritems():
if should_skip(rel_path):
continue
file_path = make_absolute_static_path(settings['static_dir'], rel_path)
assert os.path.isfile(file_path), 'Missing static asset %s' % file_path
file_name = depspec['versioned_path']
to_upload.add((file_name, file_path))
logging.info('Found %d assets to upload to S3', len(to_upload))
if skip_s3_upload:
logging.info('Skipping asset upload to S3 %s', to_upload)
return to_upload
# Upload assets to S3 using 5 threads
queue = Queue.Queue()
errors = []
for i in xrange(5):
uploader = S3UploadThread(queue, errors, manifest, settings)
uploader.setDaemon(True)
uploader.start()
map(queue.put, to_upload)
queue.join()
if errors:
raise Exception(errors)
return to_upload
def get_compiled_path(self):
"""Creates the output filename for the compiled assets of the given manager."""
return make_output_path(self.settings['compiled_asset_root'],
self.get_compiled_name())
def get_compiled_path(self):
"""Creates the output filename for the compiled assets of the given manager."""
return make_output_path(self.settings['compiled_asset_root'], self.get_compiled_name())