def test_timestamp_from_utc(self):
self.assertEqual(
"20141211-041314-654320-PST",
base.timestamp(
base.parse_timestamp_str("20141211-121314-654320-UTC")))
self.assertEqual(
"20140607-010910-123456-PDT",
base.timestamp(
base.parse_timestamp_str("20140607-080910-123456-UTC")))
def test_timestamp_from_utc(self):
self.assertEqual(
"20141211-041314-654320-PST",
base.timestamp(base.parse_timestamp_str("20141211-121314-654320-UTC"))
)
self.assertEqual(
"20140607-010910-123456-PDT",
base.timestamp(base.parse_timestamp_str("20140607-080910-123456-UTC"))
)
def test_est_timestamp_to_utc(self):
os.environ["TZ"] = "UTC+00"
time.tzset()
self.assertEqual(
"20141211-201314-654320-UTC",
base.timestamp(base.parse_timestamp_str("20141211-151314-654320-EST"))
)
self.assertEqual(
"20140607-150910-123456-UTC",
base.timestamp(base.parse_timestamp_str("20140607-110910-123456-EDT"))
)
def test_est_timestamp_to_utc(self):
os.environ["TZ"] = "UTC+00"
time.tzset()
self.assertEqual(
"20141211-201314-654320-UTC",
base.timestamp(
base.parse_timestamp_str("20141211-151314-654320-EST")))
self.assertEqual(
"20140607-150910-123456-UTC",
base.timestamp(
base.parse_timestamp_str("20140607-110910-123456-EDT")))
def format_task(task):
if task.start_time is None:
return task.task_id
elif task.end_time is None:
return '%s (start time: %s - elapsed: %s)' % (
task.task_id,
base.timestamp(task.start_time.timestamp()),
datetime.datetime.now() - task.start_time)
else:
return '%s (start time: %s - end time: %s - duration: %s)' % (
task.task_id,
base.timestamp(task.start_time.timestamp()),
base.timestamp(task.end_time.timestamp()),
task.end_time - task.start_time)
def log(self, level, fmt, *args, **kwargs):
"""Appends the specified log entry.
Args:
level: Level of the log entry to append (see the Level enum).
fmt: Format of the log entry message.
*args, **kwargs: Positional and named arguments of the log entry message format.
"""
(module, filepath, line, function) = get_caller_location()
if level < self._module_map.get(module, self._level):
return
# Normalize well-known log levels:
try:
level_str = Level(level).name
except ValueError:
level_str = "L{}".format(level)
message = fmt.format(*args, **kwargs)
entry = self._entry_format.format(
time=base.timestamp(),
level=level_str,
filepath=os.path.abspath(filepath),
line=line,
module=module,
function=function,
message=message,
)
for handler in self._handlers:
if level >= handler.level:
handler.stream.write(entry)
handler.stream.flush()
def log(self, level, fmt, *args, **kwargs):
"""Appends the specified log entry.
Args:
level: Level of the log entry to append (see the Level enum).
fmt: Format of the log entry message.
*args, **kwargs: Positional and named arguments of the log entry message format.
"""
(module, filepath, line, function) = get_caller_location()
if level < self._module_map.get(module, self._level):
return
# Normalize well-known log levels:
try:
level_str = Level(level).name
except ValueError:
level_str = "L{}".format(level)
message = fmt.format(*args, **kwargs)
entry = self._entry_format.format(
time=base.timestamp(),
level=level_str,
filepath=os.path.abspath(filepath),
line=line,
module=module,
function=function,
message=message,
)
for handler in self._handlers:
if level >= handler.level:
handler.stream.write(entry)
handler.stream.flush()
def test_timestamp(self):
dt = datetime.datetime(2014, 12, 11, 12, 13, 14, 654320)
tstamp_str = base.timestamp(dt.timestamp())
self.assertEqual("20141211-121314-654320-PST", tstamp_str)
self.assertEqual(
dt,
datetime.datetime.fromtimestamp(
base.parse_timestamp_str(tstamp_str)))
def test_timestamp(self):
dt = datetime.datetime(2014, 12, 11, 12, 13, 14, 654320)
tstamp_str = base.timestamp(dt.timestamp())
self.assertEqual("20141211-121314-654320-PST", tstamp_str)
self.assertEqual(
dt,
datetime.datetime.fromtimestamp(base.parse_timestamp_str(tstamp_str))
)
def test_timestamp_dst(self):
dt = datetime.datetime(2014, 6, 7, 8, 9, 10, 123456)
tstamp = dt.timestamp()
tstamp_str = base.timestamp(tstamp)
self.assertEqual("20140607-080910-123456-PDT", tstamp_str)
self.assertEqual(
dt,
datetime.datetime.fromtimestamp(base.parse_timestamp_str(tstamp_str))
)
def test_timestamp_dst(self):
dt = datetime.datetime(2014, 6, 7, 8, 9, 10, 123456)
tstamp = dt.timestamp()
tstamp_str = base.timestamp(tstamp)
self.assertEqual("20140607-080910-123456-PDT", tstamp_str)
self.assertEqual(
dt,
datetime.datetime.fromtimestamp(
base.parse_timestamp_str(tstamp_str)))
def run_with_io(self, output):
logging.info('Task1 is running NOW')
output.timestamp = base.timestamp()
return self.SUCCESS
def _make_workflow(self, config, force_build=False):
"""Builds the workflow tasks after the build graph defined by the workspace.
Args:
config: Workspace configuration record.
force_build: When true, force a rebuild irrespective of incremental changes.
Returns:
The workflow builder.
"""
workflow_name = "build.commit={commit}.timestamp={timestamp}".format(
commit=self.git.get_commit_hash(),
timestamp=base.timestamp(),
)
flow = workflow.Workflow(name=workflow_name)
TASK_CLASS_MAP = dict(
avro_java_library=workflow_task.AvroJavaLibraryTask,
generated_pom=workflow_task.GeneratePomTask,
java_binary=workflow_task.JavaBinaryTask,
java_library=workflow_task.JavaLibraryTask,
java_test=workflow_task.JavaTestTask,
java_super_binary=workflow_task.JavaSuperBinaryTask,
js_app=workflow_task.JSAppTask,
npm_install=workflow_task.NpmInstallTask,
bower_install=workflow_task.BowerInstallTask,
python_binary=workflow_task.PythonBinaryTask,
python_library=workflow_task.PythonLibraryTask,
python_test=workflow_task.PythonTestTask,
run_checkstyle=workflow_task.RunCheckstyleTask,
run_java_test=workflow_task.RunJavaTestTask,
run_python_test=workflow_task.RunPythonTestTask,
run_scala_test=workflow_task.RunJavaTestTask,
run_scalastyle=workflow_task.RunScalastyleTask,
scala_library=workflow_task.ScalaLibraryTask,
scala_test=workflow_task.ScalaTestTask,
)
for name, definition in self._build_defs.definitions.items():
task_class = TASK_CLASS_MAP[definition.kind]
task = task_class(
workflow=flow,
workspace=self,
name=name,
spec=definition,
force=force_build,
)
for dep in definition.get("deps", tuple()):
if isinstance(dep, artifact.Artifact):
# Direct Maven artifact dependencies are not reified as workflow tasks:
continue
if isinstance(dep, build_defs.PythonPyPIDep):
# Python PyPI dependencies are external dependencies:
continue
if isinstance(dep, build_defs.DynamicDep):
dep = dep.provider
if dep is None:
continue
task.bind_input_to_task_output(input_name=dep, task=dep)
return flow
def _make_workflow(self, config, force_build=False):
"""Builds the workflow tasks after the build graph defined by the workspace.
Args:
config: Workspace configuration record.
force_build: When true, force a rebuild irrespective of incremental changes.
Returns:
The workflow builder.
"""
workflow_name = "build.commit={commit}.timestamp={timestamp}".format(
commit=self.git.get_commit_hash(),
timestamp=base.timestamp(),
)
flow = workflow.Workflow(name=workflow_name)
TASK_CLASS_MAP = dict(
avro_java_library=workflow_task.AvroJavaLibraryTask,
generated_pom=workflow_task.GeneratePomTask,
java_binary=workflow_task.JavaBinaryTask,
java_library=workflow_task.JavaLibraryTask,
java_test=workflow_task.JavaTestTask,
java_super_binary=workflow_task.JavaSuperBinaryTask,
js_app=workflow_task.JSAppTask,
npm_install=workflow_task.NpmInstallTask,
bower_install=workflow_task.BowerInstallTask,
python_binary=workflow_task.PythonBinaryTask,
python_library=workflow_task.PythonLibraryTask,
python_test=workflow_task.PythonTestTask,
run_checkstyle=workflow_task.RunCheckstyleTask,
run_java_test=workflow_task.RunJavaTestTask,
run_python_test=workflow_task.RunPythonTestTask,
run_scala_test=workflow_task.RunJavaTestTask,
run_scalastyle=workflow_task.RunScalastyleTask,
scala_library=workflow_task.ScalaLibraryTask,
scala_test=workflow_task.ScalaTestTask,
)
for name, definition in self._build_defs.definitions.items():
task_class = TASK_CLASS_MAP[definition.kind]
task = task_class(
workflow=flow,
workspace=self,
name=name,
spec=definition,
force=force_build,
)
for dep in definition.get("deps", tuple()):
if isinstance(dep, artifact.Artifact):
# Direct Maven artifact dependencies are not reified as workflow tasks:
continue
if isinstance(dep, build_defs.PythonPyPIDep):
# Python PyPI dependencies are external dependencies:
continue
if isinstance(dep, build_defs.DynamicDep):
dep = dep.provider
if dep is None:
continue
task.bind_input_to_task_output(input_name=dep, task=dep)
return flow
def __init__(self,
*arglist,
args=None,
exit_code=None,
work_dir=None,
env=None,
log_dir=None,
direct_log=False,
collect_log=True,
start=True,
wait_for=True):
"""Runs a shell command.
The command runs in the specified working directory and with the given
environment.
The command takes no standard input.
Its output and error streams are captured in files, and are exposed as
properties once the process has completed.
Args:
*arglist: Command-line, as an array of command-line arguments.
First argument is the path to the executable.
args: Keyword argument, alternative to *arglist.
exit_code: Optional command exit code to require, or None.
work_dir: Working directory. None means current workding directory.
env: Optional environment variables for the subprocess, or None.
log_dir: Optional directory where to write files capturing the command output streams.
Defaults to the log directory (FLAGS.log_dir).
direct_log: When set, the command's output and error streams are directly written
to the log files, instead of being piped through this process.
collect_log: When set, log files are collected in memory and removed.
start: Whether to start running the command right away.
wait_for: Whether to wait for the command to complete.
Raises:
CommandError: if the sub-process exit code does not match exit_code.
"""
self._command_id = CommandID.get_new_id()
assert (args is None) ^ (len(arglist) == 0)
if args is None:
self._args = tuple(arglist)
else:
self._args = tuple(args)
self._required_exit_code = exit_code
self._work_dir = work_dir or os.getcwd()
self._env = env or os.environ
log_dir = log_dir or FLAGS.log_dir
name = os.path.basename(self._args[0])
timestamp = base.timestamp()
self._direct_log = direct_log
self._collect_log = collect_log
self._output_path = os.path.join(
log_dir, "%s.%s.%d.out" % (name, timestamp, os.getpid()))
self._error_path = os.path.join(
log_dir, "%s.%s.%d.err" % (name, timestamp, os.getpid()))
self._process = None
self._output_bytes = None
self._error_bytes = None
if start:
self.start(wait_for=wait_for)
def __init__(
self,
workspace,
config=None,
avro_version=None,
scala_version=None,
checkstyle_version=None,
scalastyle_version=None,
):
self._workspace = workspace
if config is None:
config = self.workspace.config
self._config = config
if avro_version is None:
avro_version = self.config.avro_version
if scala_version is None:
scala_version = self.config.scala_version
if checkstyle_version is None:
checkstyle_version = self.config.checkstyle_version
if scalastyle_version is None:
scalastyle_version = self.config.scalastyle_version
self._jar_path = get_executable_path("jar")
self._java_path = get_executable_path("java")
self._javac_path = get_executable_path("javac")
self._mvn_path = get_executable_path("mvn")
self._python_path = get_executable_path("python")
self._npm_path = get_executable_path("npm")
self._bower_path = get_executable_path("bower")
self._grunt_path = get_executable_path("grunt")
assert (self.java_path is not None), "Required 'java' executable not found."
assert (self.javac_path is not None), "Required 'javac' executable not found."
assert (self.python_path is not None), "Required 'python' executable not found."
logging.debug("Path for 'java' executable: %r", self.java_path)
logging.debug("Path for 'javac' executable: %r", self.javac_path)
logging.debug("Path for 'python' executable: %r", self.python_path)
logging.debug("Path for 'npm' executable: %r", self.npm_path)
logging.debug("Path for 'bower' executable: %r", self.bower_path)
logging.debug("Path for 'grunt' executable: %r", self.grunt_path)
self._avro_version = avro_version
self._scala_version = scala_version
self._checkstyle_version = checkstyle_version
self._scalastyle_version = scalastyle_version
self._avro_lib_classpath = None
self._avro_ipc_classpath = None
self._avro_tools_classpath = None
self._scalac_classpath = None
self._checkstyle_classpath = None
self._scalastyle_classpath = None
self._timestamp = base.timestamp()
self._maven_cloned_repo_dir = os.path.join(
self.workspace.temp_dir,
"maven_repository",
"%s.%s" % (self._timestamp, os.getpid()),
)
logging.debug("KijiBuild builtin fingerprint salt: %r", FINGERPRINT_SALT)
logging.debug("Java version: %r", self.java_version_text)
logging.debug("Javac version: %r", self.javac_version_text)
logging.debug("Python version: %r", self.python_version_text)
# Compute the fingerprint salt:
self._fingerprint_salt = "\n".join([
FINGERPRINT_SALT,
self.java_version_text,
self.javac_version_text,
self.python_version_text,
])
logging.debug("KijiBuild composite fingerprint: %r", self._fingerprint_salt)
def __init__(
self,
*arglist,
args=None,
exit_code=None,
work_dir=None,
env=None,
log_dir=None,
direct_log=False,
collect_log=True,
start=True,
wait_for=True
):
"""Runs a shell command.
The command runs in the specified working directory and with the given
environment.
The command takes no standard input.
Its output and error streams are captured in files, and are exposed as
properties once the process has completed.
Args:
*arglist: Command-line, as an array of command-line arguments.
First argument is the path to the executable.
args: Keyword argument, alternative to *arglist.
exit_code: Optional command exit code to require, or None.
work_dir: Working directory. None means current workding directory.
env: Optional environment variables for the subprocess, or None.
log_dir: Optional directory where to write files capturing the command output streams.
Defaults to the log directory (FLAGS.log_dir).
direct_log: When set, the command's output and error streams are directly written
to the log files, instead of being piped through this process.
collect_log: When set, log files are collected in memory and removed.
start: Whether to start running the command right away.
wait_for: Whether to wait for the command to complete.
Raises:
CommandError: if the sub-process exit code does not match exit_code.
"""
self._command_id = CommandID.get_new_id()
assert (args is None) ^ (len(arglist) == 0)
if args is None:
self._args = tuple(arglist)
else:
self._args = tuple(args)
self._required_exit_code = exit_code
self._work_dir = work_dir or os.getcwd()
self._env = env or os.environ
log_dir = log_dir or FLAGS.log_dir
name = os.path.basename(self._args[0])
timestamp = base.timestamp()
self._direct_log = direct_log
self._collect_log = collect_log
self._output_path = os.path.join(log_dir, "%s.%s.%d.out" % (name, timestamp, os.getpid()))
self._error_path = os.path.join(log_dir, "%s.%s.%d.err" % (name, timestamp, os.getpid()))
self._process = None
self._output_bytes = None
self._error_bytes = None
if start:
self.start(wait_for=wait_for)
