def apply(cls, message, paths_to_content, write_func, batch_write_func, delete_func, threaded=True, batch_writes=False): if pool is None: text = 'Deployment is unavailable in this environment.' raise common_utils.UnavailableError(text) thread_pool = pool.ThreadPool(cls.POOL_SIZE) diff = message num_files = len(diff.adds) + len(diff.edits) + len(diff.deletes) text = 'Deploying: %(value)d/{} (in %(time_elapsed).9s)' widgets = [progressbar.FormatLabel(text.format(num_files))] progress = progressbar_non.create_progressbar( "Deploying...", widgets=widgets, max_value=num_files) def run_with_progress(func, *args): func(*args) progress.update(progress.value + 1) if batch_writes: writes_paths_to_contents = {} for file_message in diff.adds: writes_paths_to_contents[file_message.path] = \ paths_to_content[file_message.path] for file_message in diff.edits: writes_paths_to_contents[file_message.path] = \ paths_to_content[file_message.path] deletes_paths = [ file_message.path for file_message in diff.deletes] if writes_paths_to_contents: batch_write_func(writes_paths_to_contents) if deletes_paths: delete_func(deletes_paths) else: progress.start() for file_message in diff.adds: content = paths_to_content[file_message.path] if threaded: args = (write_func, file_message.path, content) thread_pool.apply_async(run_with_progress, args=args) else: run_with_progress(write_func, file_message.path, content) for file_message in diff.edits: content = paths_to_content[file_message.path] if threaded: args = (write_func, file_message.path, content) thread_pool.apply_async(run_with_progress, args=args) else: run_with_progress(write_func, file_message.path, content) for file_message in diff.deletes: if threaded: args = (delete_func, file_message.path) thread_pool.apply_async(run_with_progress, args=args) else: run_with_progress(delete_func, file_message.path) if threaded: thread_pool.close() thread_pool.join() if not batch_writes: progress.finish()
def __init__(self, *args, **kwargs): super(ScpDestination, self).__init__(*args, **kwargs) if paramiko is None: raise utils.UnavailableError( 'SCP deployments are not available in this environment.') self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.host = self.config.host self.port = self.config.port self.root_dir = self.config.root_dir self.username = self.config.username
def build_directory(self, sass_path, css_path, _root_sass=None, _root_css=None): if sass is None: raise utils.UnavailableError('The Sass compiler is not available in this environment.') if self.config.image_path: image_path = os.path.abspath(os.path.join(self.root, self.config.image_path.lstrip('/'))) else: image_path = None _root_sass = sass_path if _root_sass is None else _root_sass _root_css = css_path if _root_css is None else _root_css result = {} if not os.path.isdir(css_path): os.makedirs(css_path) for name in os.listdir(sass_path): if not SUFFIX_PATTERN.search(name) or name.startswith('_'): continue sass_fullname = os.path.join(sass_path, name) if os.path.isfile(sass_fullname): basename = os.path.splitext(name)[0] css_fullname = os.path.join(css_path, basename) + self.config.suffix try: kwargs = { 'filename': sass_fullname, 'include_paths': [_root_sass], 'output_style': self.config.output_style, } if self.config.output_style is not None: kwargs['output_style'] = self.config.output_style if image_path is not None: kwargs['image_path'] = image_path if self.config.image_path is not None: kwargs['image_path'] = image_path css = sass.compile(**kwargs) except sass.CompileError as e: logging.error(str(e)) return result with open(css_fullname, 'w') as css_file: if isinstance(css, unicode): css = css.encode('utf-8') css_file.write(css) result[sass_fullname] = css_fullname elif os.path.isdir(sass_fullname): css_fullname = os.path.join(css_path, name) subresult = self.build_directory(sass_fullname, css_fullname, _root_sass, _root_css) result.update(subresult) for sass_path, out_path in result.iteritems(): self.logger.info( 'Compiled: {} -> {}'.format(sass_path.replace(self.root, ''), out_path.replace(self.root, ''))) return result
def __init__(self, *args, **kwargs): super(WebReviewDestination, self).__init__(*args, **kwargs) if webreview is None: raise common_utils.UnavailableError('WebReview deployments are not available in this environment.') if self.config.name: print ('WARNING: The "name" parameter for webreview deployments is ' 'deprecated. Use "subdomain" instead, or use the "grow stage ' '--subdomain=<subdomain>" command.') self.config.subdomain = self.config.name if self.config.remote: self.config.server, self.config.project = self.config.remote.split('/', 1) if self.config.server.startswith('localhost:'): self.config.secure = False self._webreview = None
def apply(cls, message, paths_to_rendered_doc, write_func, batch_write_func, delete_func, threaded=True, batch_writes=False): if pool is None: text = 'Deployment is unavailable in this environment.' raise common_utils.UnavailableError(text) apply_errors = [] thread_pool = None if threaded: thread_pool = pool.ThreadPool(cls.POOL_SIZE) diff = message num_files = len(diff.adds) + len(diff.edits) + len(diff.deletes) text = 'Deploying: %(value)d/{} (in %(time_elapsed).9s)' widgets = [progressbar.FormatLabel(text.format(num_files))] progress = progressbar_non.create_progressbar("Deploying...", widgets=widgets, max_value=num_files) def run_func(kwargs): """Run an arbitrary function.""" try: kwargs['func'](*kwargs['args']) return True # pylint: disable=broad-except except Exception as err: _, _, err_tb = sys.exc_info() return DeploymentError( "Error deploying {}".format(kwargs['args']), err, err_tb) if batch_writes: writes_paths_to_rendered_doc = {} for file_message in diff.adds: writes_paths_to_rendered_doc[file_message.path] = \ paths_to_rendered_doc[file_message.path] for file_message in diff.edits: writes_paths_to_rendered_doc[file_message.path] = \ paths_to_rendered_doc[file_message.path] deletes_paths = [ file_message.path for file_message in diff.deletes ] if writes_paths_to_rendered_doc: batch_write_func(writes_paths_to_rendered_doc) if deletes_paths: delete_func(deletes_paths) else: progress.start() threaded_args = [] for file_message in diff.adds: rendered_doc = paths_to_rendered_doc[file_message.path] threaded_args.append({ 'func': write_func, 'args': (rendered_doc, ), }) for file_message in diff.edits: rendered_doc = paths_to_rendered_doc[file_message.path] threaded_args.append({ 'func': write_func, 'args': (rendered_doc, ), }) for file_message in diff.deletes: threaded_args.append({ 'func': delete_func, 'args': (file_message.path, ), }) if threaded: results = thread_pool.imap_unordered(run_func, threaded_args) for result in results: if isinstance(result, Exception): apply_errors.append(result) progress.update(progress.value + 1) else: for kwargs in threaded_args: try: kwargs['func'](*kwargs['args']) progress.update(progress.value + 1) except DeploymentError as err: apply_errors.append(err) if threaded: thread_pool.close() thread_pool.join() if not batch_writes: progress.finish() if apply_errors: for error in apply_errors: print(error.message) print(error.err) traceback.print_tb(error.err_tb) print('') text = 'There were {} errors during deployment.' raise DeploymentErrors(text.format(len(apply_errors)), apply_errors)