def _write_to_file(config, path): """Writes the configuration to the config file. Args: config: The configuration dictionary. path: Path of the file to write to. """ if os.path.exists(path): backup = '%s.backup.%d' % (path, base.now_ms()) os.rename(path, backup) os.chmod(path=backup, mode=0o600) logging.debug('Writing configuration to file: %r', path) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'wt') as f: json = base.json_encode(config) f.write(json) os.chmod(path=path, mode=0o600)
def run(self, args): # Generate a unique ID for this build. build_id = "build-%d" % base.now_ms() # Determine which workspace to build in: workspace = self.workspace if self.flags.clone_workspace: base.make_dir(FLAGS.build_workspace_dir) temp_workspace = os.path.join(tempfile.mkdtemp(dir=FLAGS.build_workspace_dir), build_id) workspace = self.workspace.Clone(temp_workspace) targets = [] if (self.flags.targets is not None) and (len(self.flags.targets) > 0): targets.extend(self.flags.targets.split(",")) targets.extend(args) targets = frozenset(targets) self._targets = targets http_monitor = None if (self.flags.http_monitor is not None) and (len(self.flags.http_monitor) > 0): (interface, port) = base.parse_host_port(self.flags.http_monitor) try: http_monitor = workflow.WorkflowHTTPMonitor(interface=interface, port=port) http_monitor.start() except OSError as os_error: if (os_error.errno == 48) or (os_error.errno == 98): raise Error( "Address {!s}:{!s} already in use. Specify a different address " "for the workflow http monitor with the --http-monitor flag." .format(interface, port) ) else: raise # Pass flags through the workspace config object. workspace.config.enable_incremental_testing = self.flags.enable_incremental_testing workspace.config.separate_test_logs = self.flags.separate_test_logs workspace.config.test_log_prefix = self.flags.test_log_prefix # Run the build: try: flow = workspace.make_workflow(force_build=self.flags.force_build) self._flow = flow self.adjust_workflow() flow = workspace.process( nworkers=self.flags.nworkers, http_monitor=http_monitor, ) # TODO(DEV-429): refactor these flags, they are unwieldy if self.flags.output_dot is not None: with open(self.flags.output_dot, "wt", encoding="UTF-8") as dot_file: dot_file.write(flow.dump_as_dot()) if self.flags.output_svg is not None: with open(self.flags.output_svg, "wt", encoding="UTF-8") as svg_file: svg_file.write(flow.dump_as_svg()) if self.flags.output_table is not None: with open(self.flags.output_table, "wt", encoding="UTF-8") as table_file: table_file.write(flow.dump_state_as_table()) return len(flow.failed_tasks) # 0 means success, non 0 means failure finally: if self.flags.clone_workspace and self.flags.delete_cloned_workspace: logging.info("Deleting cloned workspace %s", temp_workspace) shutil.rmtree(path=temp_workspace)
class Error(Exception): """Errors used in this module.""" pass class StaleDependencyError(Error): """Raised when a project has a stale dependency on a snapshot.""" pass # -------------------------------------------------------------------------------------------------- # Build ID (ms since the Epoch): BUILD_ID = base.now_ms() # Path to the local Maven repository: MAVEN_LOCAL_REPO = os.path.join(os.getenv('HOME'), '.m2', 'repository') OUTPUT_PACKAGES = frozenset(['com/moz/fiji', 'com/wibidata']) # -------------------------------------------------------------------------------------------------- FLAGS.add_string( name='build_workspace_dir', default=('/tmp/%s/fiji_build_workspaces/%d.%d' % (getpass.getuser(), os.getpid(), BUILD_ID)),
def run(self, args): # Generate a unique ID for this build. build_id = "build-%d" % base.now_ms() # Determine which workspace to build in: workspace = self.workspace if self.flags.clone_workspace: base.make_dir(FLAGS.build_workspace_dir) temp_workspace = os.path.join( tempfile.mkdtemp(dir=FLAGS.build_workspace_dir), build_id) workspace = self.workspace.Clone(temp_workspace) targets = [] if (self.flags.targets is not None) and (len(self.flags.targets) > 0): targets.extend(self.flags.targets.split(",")) targets.extend(args) targets = frozenset(targets) self._targets = targets http_monitor = None if (self.flags.http_monitor is not None) and (len(self.flags.http_monitor) > 0): (interface, port) = base.parse_host_port(self.flags.http_monitor) try: http_monitor = workflow.WorkflowHTTPMonitor( interface=interface, port=port) http_monitor.start() except OSError as os_error: if (os_error.errno == 48) or (os_error.errno == 98): raise Error( "Address {!s}:{!s} already in use. Specify a different address " "for the workflow http monitor with the --http-monitor flag." .format(interface, port)) else: raise # Pass flags through the workspace config object. workspace.config.enable_incremental_testing = self.flags.enable_incremental_testing workspace.config.separate_test_logs = self.flags.separate_test_logs workspace.config.test_log_prefix = self.flags.test_log_prefix # Run the build: try: flow = workspace.make_workflow(force_build=self.flags.force_build) self._flow = flow self.adjust_workflow() flow = workspace.process( nworkers=self.flags.nworkers, http_monitor=http_monitor, ) # TODO(DEV-429): refactor these flags, they are unwieldy if self.flags.output_dot is not None: with open(self.flags.output_dot, "wt", encoding="UTF-8") as dot_file: dot_file.write(flow.dump_as_dot()) if self.flags.output_svg is not None: with open(self.flags.output_svg, "wt", encoding="UTF-8") as svg_file: svg_file.write(flow.dump_as_svg()) if self.flags.output_table is not None: with open(self.flags.output_table, "wt", encoding="UTF-8") as table_file: table_file.write(flow.dump_state_as_table()) return len( flow.failed_tasks) # 0 means success, non 0 means failure finally: if self.flags.clone_workspace and self.flags.delete_cloned_workspace: logging.info("Deleting cloned workspace %s", temp_workspace) shutil.rmtree(path=temp_workspace)
class Error(Exception): """Errors used in this module.""" pass class StaleDependencyError(Error): """Raised when a project has a stale dependency on a snapshot.""" pass # -------------------------------------------------------------------------------------------------- # Build ID (ms since the Epoch): BUILD_ID = base.now_ms() # Path to the local Maven repository: MAVEN_LOCAL_REPO = os.path.join(os.getenv('HOME'), '.m2', 'repository') OUTPUT_PACKAGES = frozenset(['org/kiji', 'com/wibidata']) # -------------------------------------------------------------------------------------------------- FLAGS.add_string( name='build_workspace_dir', default=('/tmp/%s/kiji_build_workspaces/%d.%d' % (getpass.getuser(), os.getpid(), BUILD_ID)), help='Local temporary directory to use for isolated build workspaces.', )