def _write_to_file(config, path):
"""Writes the configuration to the config file.
Args:
config: The configuration dictionary.
path: Path of the file to write to.
"""
if os.path.exists(path):
backup = '%s.backup.%d' % (path, base.now_ms())
os.rename(path, backup)
os.chmod(path=backup, mode=0o600)
logging.debug('Writing configuration to file: %r', path)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wt') as f:
json = base.json_encode(config)
f.write(json)
os.chmod(path=path, mode=0o600)
def _write_to_file(config, path):
"""Writes the configuration to the config file.
Args:
config: The configuration dictionary.
path: Path of the file to write to.
"""
if os.path.exists(path):
backup = '%s.backup.%d' % (path, base.now_ms())
os.rename(path, backup)
os.chmod(path=backup, mode=0o600)
logging.debug('Writing configuration to file: %r', path)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wt') as f:
json = base.json_encode(config)
f.write(json)
os.chmod(path=path, mode=0o600)
def run(self, args):
# Generate a unique ID for this build.
build_id = "build-%d" % base.now_ms()
# Determine which workspace to build in:
workspace = self.workspace
if self.flags.clone_workspace:
base.make_dir(FLAGS.build_workspace_dir)
temp_workspace = os.path.join(tempfile.mkdtemp(dir=FLAGS.build_workspace_dir), build_id)
workspace = self.workspace.Clone(temp_workspace)
targets = []
if (self.flags.targets is not None) and (len(self.flags.targets) > 0):
targets.extend(self.flags.targets.split(","))
targets.extend(args)
targets = frozenset(targets)
self._targets = targets
http_monitor = None
if (self.flags.http_monitor is not None) and (len(self.flags.http_monitor) > 0):
(interface, port) = base.parse_host_port(self.flags.http_monitor)
try:
http_monitor = workflow.WorkflowHTTPMonitor(interface=interface, port=port)
http_monitor.start()
except OSError as os_error:
if (os_error.errno == 48) or (os_error.errno == 98):
raise Error(
"Address {!s}:{!s} already in use. Specify a different address "
"for the workflow http monitor with the --http-monitor flag."
.format(interface, port)
)
else:
raise
# Pass flags through the workspace config object.
workspace.config.enable_incremental_testing = self.flags.enable_incremental_testing
workspace.config.separate_test_logs = self.flags.separate_test_logs
workspace.config.test_log_prefix = self.flags.test_log_prefix
# Run the build:
try:
flow = workspace.make_workflow(force_build=self.flags.force_build)
self._flow = flow
self.adjust_workflow()
flow = workspace.process(
nworkers=self.flags.nworkers,
http_monitor=http_monitor,
)
# TODO(DEV-429): refactor these flags, they are unwieldy
if self.flags.output_dot is not None:
with open(self.flags.output_dot, "wt", encoding="UTF-8") as dot_file:
dot_file.write(flow.dump_as_dot())
if self.flags.output_svg is not None:
with open(self.flags.output_svg, "wt", encoding="UTF-8") as svg_file:
svg_file.write(flow.dump_as_svg())
if self.flags.output_table is not None:
with open(self.flags.output_table, "wt", encoding="UTF-8") as table_file:
table_file.write(flow.dump_state_as_table())
return len(flow.failed_tasks) # 0 means success, non 0 means failure
finally:
if self.flags.clone_workspace and self.flags.delete_cloned_workspace:
logging.info("Deleting cloned workspace %s", temp_workspace)
shutil.rmtree(path=temp_workspace)
class Error(Exception):
"""Errors used in this module."""
pass
class StaleDependencyError(Error):
"""Raised when a project has a stale dependency on a snapshot."""
pass
# --------------------------------------------------------------------------------------------------
# Build ID (ms since the Epoch):
BUILD_ID = base.now_ms()
# Path to the local Maven repository:
MAVEN_LOCAL_REPO = os.path.join(os.getenv('HOME'), '.m2', 'repository')
OUTPUT_PACKAGES = frozenset(['com/moz/fiji', 'com/wibidata'])
# --------------------------------------------------------------------------------------------------
FLAGS.add_string(
name='build_workspace_dir',
default=('/tmp/%s/fiji_build_workspaces/%d.%d'
% (getpass.getuser(), os.getpid(), BUILD_ID)),
def run(self, args):
# Generate a unique ID for this build.
build_id = "build-%d" % base.now_ms()
# Determine which workspace to build in:
workspace = self.workspace
if self.flags.clone_workspace:
base.make_dir(FLAGS.build_workspace_dir)
temp_workspace = os.path.join(
tempfile.mkdtemp(dir=FLAGS.build_workspace_dir), build_id)
workspace = self.workspace.Clone(temp_workspace)
targets = []
if (self.flags.targets is not None) and (len(self.flags.targets) > 0):
targets.extend(self.flags.targets.split(","))
targets.extend(args)
targets = frozenset(targets)
self._targets = targets
http_monitor = None
if (self.flags.http_monitor
is not None) and (len(self.flags.http_monitor) > 0):
(interface, port) = base.parse_host_port(self.flags.http_monitor)
try:
http_monitor = workflow.WorkflowHTTPMonitor(
interface=interface, port=port)
http_monitor.start()
except OSError as os_error:
if (os_error.errno == 48) or (os_error.errno == 98):
raise Error(
"Address {!s}:{!s} already in use. Specify a different address "
"for the workflow http monitor with the --http-monitor flag."
.format(interface, port))
else:
raise
# Pass flags through the workspace config object.
workspace.config.enable_incremental_testing = self.flags.enable_incremental_testing
workspace.config.separate_test_logs = self.flags.separate_test_logs
workspace.config.test_log_prefix = self.flags.test_log_prefix
# Run the build:
try:
flow = workspace.make_workflow(force_build=self.flags.force_build)
self._flow = flow
self.adjust_workflow()
flow = workspace.process(
nworkers=self.flags.nworkers,
http_monitor=http_monitor,
)
# TODO(DEV-429): refactor these flags, they are unwieldy
if self.flags.output_dot is not None:
with open(self.flags.output_dot, "wt",
encoding="UTF-8") as dot_file:
dot_file.write(flow.dump_as_dot())
if self.flags.output_svg is not None:
with open(self.flags.output_svg, "wt",
encoding="UTF-8") as svg_file:
svg_file.write(flow.dump_as_svg())
if self.flags.output_table is not None:
with open(self.flags.output_table, "wt",
encoding="UTF-8") as table_file:
table_file.write(flow.dump_state_as_table())
return len(
flow.failed_tasks) # 0 means success, non 0 means failure
finally:
if self.flags.clone_workspace and self.flags.delete_cloned_workspace:
logging.info("Deleting cloned workspace %s", temp_workspace)
shutil.rmtree(path=temp_workspace)
class Error(Exception):
"""Errors used in this module."""
pass
class StaleDependencyError(Error):
"""Raised when a project has a stale dependency on a snapshot."""
pass
# --------------------------------------------------------------------------------------------------
# Build ID (ms since the Epoch):
BUILD_ID = base.now_ms()
# Path to the local Maven repository:
MAVEN_LOCAL_REPO = os.path.join(os.getenv('HOME'), '.m2', 'repository')
OUTPUT_PACKAGES = frozenset(['org/kiji', 'com/wibidata'])
# --------------------------------------------------------------------------------------------------
FLAGS.add_string(
name='build_workspace_dir',
default=('/tmp/%s/kiji_build_workspaces/%d.%d' %
(getpass.getuser(), os.getpid(), BUILD_ID)),
help='Local temporary directory to use for isolated build workspaces.',
)
