def upload_assets_to_s3(manifest, settings, skip_s3_upload=False): """Uploads any assets that are in the given manifest and in our compiled output dir but missing from our static assets bucket to that bucket on S3. """ # We will gather a set of (file_name, file_path) tuples to be uploaded to_upload = set() # We know we want to upload each asset block (these correspond to the # assetman.include_* blocks in each template) for depspec in manifest.blocks.itervalues(): file_name = depspec['versioned_path'] file_path = make_output_path(settings['compiled_asset_root'], file_name) assert os.path.isfile( file_path), 'Missing compiled asset %s' % file_path to_upload.add((file_name, file_path)) # And we know that we'll want to upload any statically-referenced assets # (from assetman.static_url calls or referenced in any compiled assets), # but we'll need to filter out other entries in the complete 'assets' # block of the manifest. should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search for rel_path, depspec in manifest.assets.iteritems(): if should_skip(rel_path): continue file_path = make_absolute_static_path(settings['static_dir'], rel_path) assert os.path.isfile(file_path), 'Missing static asset %s' % file_path file_name = depspec['versioned_path'] to_upload.add((file_name, file_path)) logging.info('Found %d assets to upload to S3', len(to_upload)) if skip_s3_upload: logging.info('Skipping asset upload to S3 %s', to_upload) return to_upload # Upload assets to S3 using 5 threads queue = Queue.Queue() errors = [] for i in xrange(5): uploader = S3UploadThread(queue, errors, manifest, settings) uploader.setDaemon(True) uploader.start() map(queue.put, to_upload) queue.join() if errors: raise Exception(errors) return to_upload
def upload_assets_to_s3(manifest, settings, skip_s3_upload=False): """Uploads any assets that are in the given manifest and in our compiled output dir but missing from our static assets bucket to that bucket on S3. """ # We will gather a set of (file_name, file_path) tuples to be uploaded to_upload = set() # We know we want to upload each asset block (these correspond to the # assetman.include_* blocks in each template) for depspec in manifest.blocks.itervalues(): file_name = depspec['versioned_path'] file_path = make_output_path(settings['compiled_asset_root'], file_name) assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path to_upload.add((file_name, file_path)) # And we know that we'll want to upload any statically-referenced assets # (from assetman.static_url calls or referenced in any compiled assets), # but we'll need to filter out other entries in the complete 'assets' # block of the manifest. should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search for rel_path, depspec in manifest.assets.iteritems(): if should_skip(rel_path): continue file_path = make_absolute_static_path(settings['static_dir'], rel_path) assert os.path.isfile(file_path), 'Missing static asset %s' % file_path file_name = depspec['versioned_path'] to_upload.add((file_name, file_path)) logging.info('Found %d assets to upload to S3', len(to_upload)) if skip_s3_upload: logging.info('Skipping asset upload to S3 %s', to_upload) return to_upload # Upload assets to S3 using 5 threads queue = Queue.Queue() errors = [] for i in xrange(5): uploader = S3UploadThread(queue, errors, manifest, settings) uploader.setDaemon(True) uploader.start() map(queue.put, to_upload) queue.join() if errors: raise Exception(errors) return to_upload
def get_compiled_path(self): """Creates the output filename for the compiled assets of the given manager.""" return make_output_path(self.settings['compiled_asset_root'], self.get_compiled_name())