def __init__(self, root_dir):
typecheck(root_dir, Directory)
self.__loaded_files = {}
self.__source_artifacts = {}
self.__derived_artifacts = {}
self.__root_dir = root_dir
def __init__(self, root_dir):
typecheck(root_dir, Directory)
self.__loaded_files = {}
self.__source_artifacts = {}
self.__derived_artifacts = {}
self.__root_dir = root_dir
def read(self, filename): typecheck(filename, basestring) path = os.path.join(self.__path, filename) f = open(path, "rU") result = f.read() f.close() return result
def add_rule(self, config, rule):
typecheck(rule, Rule)
rule.expand_once()
for artifact in rule.outputs:
self.add_artifact(config, artifact)
def build(self, action_runner):
self.__lock.acquire()
try:
typecheck(action_runner, ActionRunner)
while self.__num_pending > 0 and not self.failed:
if len(self.__action_queue) == 0:
# wait for actions
# TODO(kenton): Use a semaphore or something?
self.__lock.release()
try:
time.sleep(1)
finally:
self.__lock.acquire()
continue
action_state = self.__action_queue.popleft()
self.do_one_action(action_state.config, action_state.action,
action_runner)
except KeyboardInterrupt:
if not self.failed:
self.__console.write(
ColoredText(ColoredText.RED, "INTERRUPTED"))
self.failed = True
except:
self.failed = True
raise
finally:
self.__lock.release()
def _args_to_rules(loader, args):
"""Given a list of command-line arguments like 'foo/bar.sebs:baz', return an
iterator of rules which should be built."""
typecheck(args, list, basestring)
for arg in args:
if arg.startswith("src/") or arg.startswith("src\\"):
# For ease of use, we allow files to start with "src/", so tab completion
# can be used.
arg = arg[4:]
elif arg.startswith("//"):
# We also allow files to start with "//" which mimics to the syntax given
# to sebs.import_.
arg = arg[2:]
print arg
target = loader.load(arg)
if isinstance(target, BuildFile):
for name, value in target.__dict__.items():
if isinstance(value, Rule):
yield value
elif not isinstance(target, Rule):
raise UsageError("%s: Does not name a rule." % arg)
else:
yield target
def build(self, action_runner):
self.__lock.acquire()
try:
typecheck(action_runner, ActionRunner)
while self.__num_pending > 0 and not self.failed:
if len(self.__action_queue) == 0:
# wait for actions
# TODO(kenton): Use a semaphore or something?
self.__lock.release()
try:
time.sleep(1)
finally:
self.__lock.acquire()
continue
action_state = self.__action_queue.popleft()
self.do_one_action(
action_state.config, action_state.action, action_runner)
except KeyboardInterrupt:
if not self.failed:
self.__console.write(ColoredText(ColoredText.RED, "INTERRUPTED"))
self.failed = True
except:
self.failed = True
raise
finally:
self.__lock.release()
def enumerate_artifacts(self, artifact_enumerator):
typecheck(artifact_enumerator, ArtifactEnumerator)
value = artifact_enumerator.read(self.__condition_artifact)
if value == "true":
self.__true_command.enumerate_artifacts(artifact_enumerator)
elif value == "false" and self.__false_command is not None:
self.__false_command.enumerate_artifacts(artifact_enumerator)
def __init__(self, loader, context):
typecheck(loader, Loader)
typecheck(context, _ContextImpl)
self.Rule = Rule
self.Test = Test
self.ArgumentSpec = ArgumentSpec
self.Artifact = Artifact
self.Action = Action
self.DefinitionError = DefinitionError
self.typecheck = typecheck
self.Command = command.Command
self.EchoCommand = command.EchoCommand
self.EnvironmentCommand = command.EnvironmentCommand
self.DoAllCommand = command.DoAllCommand
self.ConditionalCommand = command.ConditionalCommand
self.SubprocessCommand = command.SubprocessCommand
self.DepFileCommand = command.DepFileCommand
self.MirrorCommand = command.MirrorCommand
self.__loader = loader
self.__context = context
parts = context.filename.rsplit("/", 1)
if len(parts) == 1:
self.__prefix = ""
else:
self.__prefix = parts[0] + "/"
def __init__(self, loader, context):
typecheck(loader, Loader)
typecheck(context, _ContextImpl)
self.Rule = Rule
self.Test = Test
self.ArgumentSpec = ArgumentSpec
self.Artifact = Artifact
self.Action = Action
self.DefinitionError = DefinitionError
self.typecheck = typecheck
self.Command = command.Command
self.EchoCommand = command.EchoCommand
self.EnvironmentCommand = command.EnvironmentCommand
self.DoAllCommand = command.DoAllCommand
self.ConditionalCommand = command.ConditionalCommand
self.SubprocessCommand = command.SubprocessCommand
self.DepFileCommand = command.DepFileCommand
self.MirrorCommand = command.MirrorCommand
self.__loader = loader
self.__context = context
parts = context.filename.rsplit("/", 1)
if len(parts) == 1:
self.__prefix = ""
else:
self.__prefix = parts[0] + "/"
def read(self, filename):
typecheck(filename, basestring)
path = os.path.join(self.__path, filename)
f = open(path, "rU")
result = f.read()
f.close()
return result
def add_rule(self, config, rule):
typecheck(rule, Rule)
rule.expand_once()
for artifact in rule.outputs:
self.add_artifact(config, artifact)
def enumerate_artifacts(self, artifact_enumerator):
typecheck(artifact_enumerator, ArtifactEnumerator)
value = artifact_enumerator.read(self.__condition_artifact)
if value == "true":
self.__true_command.enumerate_artifacts(artifact_enumerator)
elif value == "false" and self.__false_command is not None:
self.__false_command.enumerate_artifacts(artifact_enumerator)
def _args_to_rules(loader, args):
"""Given a list of command-line arguments like 'foo/bar.sebs:baz', return an
iterator of rules which should be built."""
typecheck(args, list, basestring)
for arg in args:
if arg.startswith("src/") or arg.startswith("src\\"):
# For ease of use, we allow files to start with "src/", so tab completion
# can be used.
arg = arg[4:]
elif arg.startswith("//"):
# We also allow files to start with "//" which mimics to the syntax given
# to sebs.import_.
arg = arg[2:]
print arg
target = loader.load(arg)
if isinstance(target, BuildFile):
for name, value in target.__dict__.items():
if isinstance(value, Rule):
yield value
elif not isinstance(target, Rule):
raise UsageError("%s: Does not name a rule." % arg)
else:
yield target
def mkdir(self, filename):
typecheck(filename, basestring)
if filename in self.__files:
raise os.error("Can't make directory because file exists: %s" % filename)
if filename != "":
self.mkdir(os.path.dirname(filename))
self.__dirs.add(filename)
def touch(self, filename, mtime=None):
typecheck(filename, basestring)
if filename not in self.__files:
raise os.error("File not found: " + filename)
if mtime is None:
mtime = time.time()
oldtime, content = self.__files[filename]
self.__files[filename] = (mtime, content)
def touch(self, filename, mtime=None):
typecheck(filename, basestring)
if filename not in self.__files:
raise os.error("File not found: " + filename)
if mtime is None:
mtime = time.time()
oldtime, content = self.__files[filename]
self.__files[filename] = (mtime, content)
def action_state(self, config, action):
typecheck(action, Action)
result = self.__actions.get((config, action))
if result is None:
result = _ActionState(action, config.root_dir, self, config)
self.__actions[(config, action)] = result
return result
def action_state(self, config, action):
typecheck(action, Action)
result = self.__actions.get((config, action))
if result is None:
result = _ActionState(action, config.root_dir, self, config)
self.__actions[(config, action)] = result
return result
def mkdir(self, filename):
typecheck(filename, basestring)
if filename in self.__files:
raise os.error("Can't make directory because file exists: %s" %
filename)
if filename != "":
self.mkdir(os.path.dirname(filename))
self.__dirs.add(filename)
def get_disk_path(self, filename):
"""If the file is a real, on-disk file, return its path, suitable to be
passed to open() or other file I/O routines. If the file is not on disk,
returns None. The file does not necessarily have to actually exist; if it
doesn't, this method will still return the path that the file would have
if it did exist."""
typecheck(filename, basestring)
return None
def source_artifact(self, filename):
typecheck(filename, basestring)
if filename in self.__source_artifacts:
return self.__source_artifacts[filename]
result = Artifact(filename, None)
self.__source_artifacts[filename] = result
return result
def execfile(self, filename, globals):
typecheck(filename, basestring)
if filename not in self.__files:
raise os.error("File not found: " + filename)
(mtime, content) = self.__files[filename]
# Can't just exec because we want the filename in tracebacks
# to exactly match the filename parameter to this method.
ast = compile(content, filename, "exec")
exec ast in globals
def action_state(self, action):
typecheck(action, Action)
result = self.__actions.get((action))
if result is None:
result = _ScriptActionState(action)
action.command.write_script(_ScriptWriterImpl(result, self))
self.__actions[(action)] = result
return result
def action_state(self, action):
typecheck(action, Action)
result = self.__actions.get((action))
if result is None:
result = _ScriptActionState(action)
action.command.write_script(_ScriptWriterImpl(result, self))
self.__actions[(action)] = result
return result
def enumerate_artifacts(self, artifact_enumerator):
typecheck(artifact_enumerator, ArtifactEnumerator)
if artifact_enumerator.read(self.__env_set_artifact) == "true":
artifact_enumerator.add_input(self.__env_artifact)
elif self.__default is not None and isinstance(self.__default, Artifact):
artifact_enumerator.add_input(self.__default)
artifact_enumerator.add_output(self.__output_artifact)
def get_disk_path(self, filename):
"""If the file is a real, on-disk file, return its path, suitable to be
passed to open() or other file I/O routines. If the file is not on disk,
returns None. The file does not necessarily have to actually exist; if it
doesn't, this method will still return the path that the file would have
if it did exist."""
typecheck(filename, basestring)
return None
def execfile(self, filename, globals):
typecheck(filename, basestring)
if filename not in self.__files:
raise os.error("File not found: " + filename)
(mtime, content) = self.__files[filename]
# Can't just exec because we want the filename in tracebacks
# to exactly match the filename parameter to this method.
ast = compile(content, filename, "exec")
exec ast in globals
def source_artifact(self, filename):
typecheck(filename, basestring)
if filename in self.__source_artifacts:
return self.__source_artifacts[filename]
result = Artifact(filename, None)
self.__source_artifacts[filename] = result
return result
def __init__(self, state_map, config, action):
typecheck(state_map, _StateMap)
typecheck(action, Action)
self.__state_map = state_map
self.__config = config
self.__action = action
self.inputs = []
self.outputs = []
self.disk_inputs = []
def enumerate_artifacts(self, artifact_enumerator):
typecheck(artifact_enumerator, ArtifactEnumerator)
if artifact_enumerator.read(self.__env_set_artifact) == "true":
artifact_enumerator.add_input(self.__env_artifact)
elif self.__default is not None and isinstance(self.__default,
Artifact):
artifact_enumerator.add_input(self.__default)
artifact_enumerator.add_output(self.__output_artifact)
def mkdir(self, filename):
typecheck(filename, basestring)
path = os.path.join(self.__path, filename)
# If the path exists and is a directory, we don't have to create anything,
# but makedirs() will raise an error if we call it. If the path exists
# but is *not* a directory, we still call makedirs() so that it raises an
# appropriate error.
if not os.path.exists(path) or not os.path.isdir(path):
os.makedirs(path)
def __init__(self, loader, filename, root_dir):
typecheck(loader, Loader)
typecheck(filename, basestring)
self.__loader = loader
self.filename = filename
self.full_filename = os.path.join("src", filename)
self.directory = os.path.dirname(filename)
self.timestamp = root_dir.getmtime(self.full_filename)
self.__root_dir = root_dir
def mkdir(self, filename):
typecheck(filename, basestring)
path = os.path.join(self.__path, filename)
# If the path exists and is a directory, we don't have to create anything,
# but makedirs() will raise an error if we call it. If the path exists
# but is *not* a directory, we still call makedirs() so that it raises an
# appropriate error.
if not os.path.exists(path) or not os.path.isdir(path):
os.makedirs(path)
def __init__(self, loader, filename, root_dir):
typecheck(loader, Loader)
typecheck(filename, basestring)
self.__loader = loader
self.filename = filename
self.full_filename = os.path.join("src", filename)
self.directory = os.path.dirname(filename)
self.timestamp = root_dir.getmtime(self.full_filename)
self.__root_dir = root_dir
def intermediate_artifact(self, filename, action, configured_name=None):
self.__validate_artifact_name(filename, configured_name)
typecheck(action, Action)
if configured_name is not None:
configured_name = ["tmp/%s/" % self.directory] + configured_name
return self.__loader.derived_artifact(
os.path.join("tmp", self.directory, filename), action,
configured_name = configured_name)
def __init__(self, state_map, config, action):
typecheck(state_map, _StateMap)
typecheck(action, Action)
self.__state_map = state_map
self.__config = config
self.__action = action
self.inputs = []
self.outputs = []
self.disk_inputs = []
def intermediate_artifact(self, filename, action, configured_name=None):
self.__validate_artifact_name(filename, configured_name)
typecheck(action, Action)
if configured_name is not None:
configured_name = ["tmp/%s/" % self.directory] + configured_name
return self.__loader.derived_artifact(os.path.join(
"tmp", self.directory, filename),
action,
configured_name=configured_name)
def add_artifact(self, config, artifact):
typecheck(artifact, Artifact)
artifact_state = self.__state_map.artifact_state(config, artifact)
if not artifact_state.is_dirty:
return # Source file; nothing to do.
# The artifact is dirty, therefore it must have an action. Note that
# artifact_state.artifact and artifact_state.config may differ from
# the local artifact and config if the artifact is a reference to one
# in another config.
self.add_action(artifact_state.config, artifact_state.artifact.action)
def add_artifact(self, config, artifact):
typecheck(artifact, Artifact)
artifact_state = self.__state_map.artifact_state(config, artifact)
if not artifact_state.is_dirty:
return # Source file; nothing to do.
# The artifact is dirty, therefore it must have an action. Note that
# artifact_state.artifact and artifact_state.config may differ from
# the local artifact and config if the artifact is a reference to one
# in another config.
self.add_action(artifact_state.config, artifact_state.artifact.action)
def __validate_artifact_name(self, filename, configured_name=None):
typecheck(filename, basestring)
normalized = os.path.normpath(filename).replace("\\", "/")
if filename != normalized:
raise DefinitionError(
"File '%s' is not a normalized path name. Please use '%s' "
"instead." % (filename, normalized))
if filename.startswith("../") or filename.startswith("/"):
raise DefinitionError(
"File '%s' points outside the surrounding directory. To "
"include a file from another directory, that directory must explicitly "
"export it." % filename)
def __validate_artifact_name(self, filename, configured_name=None):
typecheck(filename, basestring)
normalized = os.path.normpath(filename).replace("\\", "/")
if filename != normalized:
raise DefinitionError(
"File '%s' is not a normalized path name. Please use '%s' "
"instead." % (filename, normalized))
if filename.startswith("../") or filename.startswith("/"):
raise DefinitionError(
"File '%s' points outside the surrounding directory. To "
"include a file from another directory, that directory must explicitly "
"export it." % filename)
def run(self, context, log):
typecheck(context, CommandContext)
value = context.read(self.__condition_artifact)
if value == "true":
return self.__true_command.run(context, log)
elif value == "false":
if self.__false_command is not None:
return self.__false_command.run(context, log)
else:
return True
else:
log.write("Condition artifact was not true or false: %s\n" %
self.__condition_artifact)
return False
def __init__(self, console):
typecheck(console, Console)
self.__state_map = _StateMap()
self.__console = console
self.__lock = threading.Lock()
self.__num_pending = 0
# ActionStates which are ready but haven't been started.
self.__action_queue = collections.deque()
self.__tests = []
self.failed = False
def __init__(self, console):
typecheck(console, Console)
self.__state_map = _StateMap()
self.__console = console
self.__lock = threading.Lock()
self.__num_pending = 0
# ActionStates which are ready but haven't been started.
self.__action_queue = collections.deque()
self.__tests = []
self.failed = False
def local_filename(self, artifact):
if isinstance(artifact, str):
return artifact
typecheck(artifact, Artifact)
parts = artifact.filename.split("/")
if parts[0] not in ["src", "tmp", "mem"]:
return None
parts = parts[1:]
dir_parts = self.directory.split("/")
if len(parts) < len(dir_parts) or parts[:len(dir_parts)] != dir_parts:
return None
return "/".join(parts[len(dir_parts):])
def local_filename(self, artifact):
if isinstance(artifact, str):
return artifact
typecheck(artifact, Artifact)
parts = artifact.filename.split("/")
if parts[0] not in ["src", "tmp", "mem"]:
return None
parts = parts[1:]
dir_parts = self.directory.split("/")
if len(parts) < len(dir_parts) or parts[:len(dir_parts)] != dir_parts:
return None
return "/".join(parts[len(dir_parts):])
def run(self, context, log):
typecheck(context, CommandContext)
value = context.read(self.__condition_artifact)
if value == "true":
return self.__true_command.run(context, log)
elif value == "false":
if self.__false_command is not None:
return self.__false_command.run(context, log)
else:
return True
else:
log.write("Condition artifact was not true or false: %s\n" %
self.__condition_artifact)
return False
def add_test(self, config, test):
typecheck(test, Test)
test.expand_once()
self.add_artifact(config, test.test_result_artifact)
self.add_artifact(config, test.test_output_artifact)
action_state = self.__state_map.action_state(
config, test.test_result_artifact.action)
action_state.test = test
cached = not self.__state_map.artifact_state(
config, test.test_result_artifact).is_dirty
self.__tests.append((test.name, config, test, cached))
def derived_artifact(self, filename, action, configured_name=None):
typecheck(filename, basestring)
typecheck(action, Action)
if filename in self.__derived_artifacts:
raise DefinitionError(
"Two different rules claim to build file '%s'. Conflicting rules are "
"'%s' and '%s'." %
(filename, action.rule.name,
self.__derived_artifacts[filename].action.rule.name))
filename = os.path.normpath(filename).replace("\\", "/")
result = Artifact(filename, action, configured_name = configured_name)
self.__derived_artifacts[filename] = result
return result
def add_action(self, config, action):
typecheck(action, Action)
action_state = self.__state_map.action_state(config, action)
if action_state.is_pending:
# Already pending.
return
action_state.is_pending = True
self.__num_pending = self.__num_pending + 1
if action_state.is_ready:
self.__action_queue.append(action_state)
else:
for blocker in action_state.blocking:
self.add_action(blocker.config, blocker.action)
def import_(self, name):
typecheck(name, str)
if (self.__loader is None):
raise DefinitionError("Imports must occur at file load time.")
# Absolute imports start with "//".
if name.startswith("//"):
name = name[2:]
else:
name = self.__prefix + name
(result, timestamp) = self.__loader.load_with_timestamp(name)
if timestamp > self.__context.timestamp:
self.__context.timestamp = timestamp
return result
def add_action(self, config, action):
typecheck(action, Action)
action_state = self.__state_map.action_state(config, action)
if action_state.is_pending:
# Already pending.
return
action_state.is_pending = True
self.__num_pending = self.__num_pending + 1
if action_state.is_ready:
self.__action_queue.append(action_state)
else:
for blocker in action_state.blocking:
self.add_action(blocker.config, blocker.action)
def add_test(self, config, test):
typecheck(test, Test)
test.expand_once()
self.add_artifact(config, test.test_result_artifact)
self.add_artifact(config, test.test_output_artifact)
action_state = self.__state_map.action_state(
config, test.test_result_artifact.action)
action_state.test = test
cached = not self.__state_map.artifact_state(
config, test.test_result_artifact).is_dirty
self.__tests.append((test.name, config, test, cached))
def derived_artifact(self, filename, action, configured_name=None):
typecheck(filename, basestring)
typecheck(action, Action)
if filename in self.__derived_artifacts:
raise DefinitionError(
"Two different rules claim to build file '%s'. Conflicting rules are "
"'%s' and '%s'." %
(filename, action.rule.name,
self.__derived_artifacts[filename].action.rule.name))
filename = os.path.normpath(filename).replace("\\", "/")
result = Artifact(filename, action, configured_name=configured_name)
self.__derived_artifacts[filename] = result
return result
def import_(self, name):
typecheck(name, str)
if (self.__loader is None):
raise DefinitionError("Imports must occur at file load time.")
# Absolute imports start with "//".
if name.startswith("//"):
name = name[2:]
else:
name = self.__prefix + name
(result, timestamp) = self.__loader.load_with_timestamp(name)
if timestamp > self.__context.timestamp:
self.__context.timestamp = timestamp
return result
def artifact_state(self, config, artifact):
typecheck(artifact, Artifact)
while artifact.alt_artifact is not None:
config = config.alt_configs.get(artifact.alt_config)
if config is None:
raise DefinitionError(
"Artifact '%s' refers to unknown configuration '%s'." %
artifact, artifact.alt_config)
artifact = artifact.alt_artifact
result = self.__artifacts.get((config, artifact))
if result is None:
result = _ArtifactState(artifact, config.root_dir, self, config)
self.__artifacts[(config, artifact)] = result
return result
def artifact_state(self, config, artifact):
typecheck(artifact, Artifact)
while artifact.alt_artifact is not None:
config = config.alt_configs.get(artifact.alt_config)
if config is None:
raise DefinitionError(
"Artifact '%s' refers to unknown configuration '%s'." %
artifact, artifact.alt_config)
artifact = artifact.alt_artifact
result = self.__artifacts.get((config, artifact))
if result is None:
result = _ArtifactState(artifact, config.root_dir, self, config)
self.__artifacts[(config, artifact)] = result
return result
def run(self, context, log):
typecheck(context, CommandContext)
if context.read(self.__env_set_artifact) == "true":
value = context.read(self.__env_artifact)
elif self.__default is None:
log.write(self.__error_message_if_unset + "\n")
return False
elif isinstance(self.__default, Artifact):
value = context.read(self.__default)
else:
value = self.__default
context.write(self.__output_artifact, value)
if self.__set_status:
context.status(value)
return True
def run(self, context, log):
typecheck(context, CommandContext)
if context.read(self.__env_set_artifact) == "true":
value = context.read(self.__env_artifact)
elif self.__default is None:
log.write(self.__error_message_if_unset + "\n")
return False
elif isinstance(self.__default, Artifact):
value = context.read(self.__default)
else:
value = self.__default
context.write(self.__output_artifact, value)
if self.__set_status:
context.status(value)
return True