Ejemplo n.º 1
0
 def test_timestamp_from_utc(self):
     self.assertEqual(
         "20141211-041314-654320-PST",
         base.timestamp(
             base.parse_timestamp_str("20141211-121314-654320-UTC")))
     self.assertEqual(
         "20140607-010910-123456-PDT",
         base.timestamp(
             base.parse_timestamp_str("20140607-080910-123456-UTC")))
Ejemplo n.º 2
0
 def test_timestamp_from_utc(self):
     self.assertEqual(
         "20141211-041314-654320-PST",
         base.timestamp(base.parse_timestamp_str("20141211-121314-654320-UTC"))
     )
     self.assertEqual(
         "20140607-010910-123456-PDT",
         base.timestamp(base.parse_timestamp_str("20140607-080910-123456-UTC"))
     )
Ejemplo n.º 3
0
 def test_est_timestamp_to_utc(self):
     os.environ["TZ"] = "UTC+00"
     time.tzset()
     self.assertEqual(
         "20141211-201314-654320-UTC",
         base.timestamp(base.parse_timestamp_str("20141211-151314-654320-EST"))
     )
     self.assertEqual(
         "20140607-150910-123456-UTC",
         base.timestamp(base.parse_timestamp_str("20140607-110910-123456-EDT"))
     )
Ejemplo n.º 4
0
 def test_est_timestamp_to_utc(self):
     os.environ["TZ"] = "UTC+00"
     time.tzset()
     self.assertEqual(
         "20141211-201314-654320-UTC",
         base.timestamp(
             base.parse_timestamp_str("20141211-151314-654320-EST")))
     self.assertEqual(
         "20140607-150910-123456-UTC",
         base.timestamp(
             base.parse_timestamp_str("20140607-110910-123456-EDT")))
Ejemplo n.º 5
0
 def format_task(task):
     if task.start_time is None:
         return task.task_id
     elif task.end_time is None:
         return '%s (start time: %s - elapsed: %s)' % (
             task.task_id,
             base.timestamp(task.start_time.timestamp()),
             datetime.datetime.now() - task.start_time)
     else:
         return '%s (start time: %s - end time: %s - duration: %s)' % (
             task.task_id,
             base.timestamp(task.start_time.timestamp()),
             base.timestamp(task.end_time.timestamp()),
             task.end_time - task.start_time)
Ejemplo n.º 6
0
    def log(self, level, fmt, *args, **kwargs):
        """Appends the specified log entry.

        Args:
            level: Level of the log entry to append (see the Level enum).
            fmt: Format of the log entry message.
            *args, **kwargs: Positional and named arguments of the log entry message format.
        """
        (module, filepath, line, function) = get_caller_location()
        if level < self._module_map.get(module, self._level):
            return

        # Normalize well-known log levels:
        try:
            level_str = Level(level).name
        except ValueError:
            level_str = "L{}".format(level)

        message = fmt.format(*args, **kwargs)
        entry = self._entry_format.format(
            time=base.timestamp(),
            level=level_str,
            filepath=os.path.abspath(filepath),
            line=line,
            module=module,
            function=function,
            message=message,
        )

        for handler in self._handlers:
            if level >= handler.level:
                handler.stream.write(entry)
                handler.stream.flush()
Ejemplo n.º 7
0
Archivo: log.py Proyecto: yubobo/kiji
    def log(self, level, fmt, *args, **kwargs):
        """Appends the specified log entry.

        Args:
            level: Level of the log entry to append (see the Level enum).
            fmt: Format of the log entry message.
            *args, **kwargs: Positional and named arguments of the log entry message format.
        """
        (module, filepath, line, function) = get_caller_location()
        if level < self._module_map.get(module, self._level):
            return

        # Normalize well-known log levels:
        try:
            level_str = Level(level).name
        except ValueError:
            level_str = "L{}".format(level)

        message = fmt.format(*args, **kwargs)
        entry = self._entry_format.format(
            time=base.timestamp(),
            level=level_str,
            filepath=os.path.abspath(filepath),
            line=line,
            module=module,
            function=function,
            message=message,
        )

        for handler in self._handlers:
            if level >= handler.level:
                handler.stream.write(entry)
                handler.stream.flush()
Ejemplo n.º 8
0
 def test_timestamp(self):
     dt = datetime.datetime(2014, 12, 11, 12, 13, 14, 654320)
     tstamp_str = base.timestamp(dt.timestamp())
     self.assertEqual("20141211-121314-654320-PST", tstamp_str)
     self.assertEqual(
         dt,
         datetime.datetime.fromtimestamp(
             base.parse_timestamp_str(tstamp_str)))
Ejemplo n.º 9
0
 def test_timestamp(self):
     dt = datetime.datetime(2014, 12, 11, 12, 13, 14, 654320)
     tstamp_str = base.timestamp(dt.timestamp())
     self.assertEqual("20141211-121314-654320-PST", tstamp_str)
     self.assertEqual(
         dt,
         datetime.datetime.fromtimestamp(base.parse_timestamp_str(tstamp_str))
     )
Ejemplo n.º 10
0
 def test_timestamp_dst(self):
     dt = datetime.datetime(2014, 6, 7, 8, 9, 10, 123456)
     tstamp = dt.timestamp()
     tstamp_str = base.timestamp(tstamp)
     self.assertEqual("20140607-080910-123456-PDT", tstamp_str)
     self.assertEqual(
         dt,
         datetime.datetime.fromtimestamp(base.parse_timestamp_str(tstamp_str))
     )
Ejemplo n.º 11
0
 def test_timestamp_dst(self):
     dt = datetime.datetime(2014, 6, 7, 8, 9, 10, 123456)
     tstamp = dt.timestamp()
     tstamp_str = base.timestamp(tstamp)
     self.assertEqual("20140607-080910-123456-PDT", tstamp_str)
     self.assertEqual(
         dt,
         datetime.datetime.fromtimestamp(
             base.parse_timestamp_str(tstamp_str)))
Ejemplo n.º 12
0
 def run_with_io(self, output):
     logging.info('Task1 is running NOW')
     output.timestamp = base.timestamp()
     return self.SUCCESS
Ejemplo n.º 13
0
    def _make_workflow(self, config, force_build=False):
        """Builds the workflow tasks after the build graph defined by the workspace.

        Args:
            config: Workspace configuration record.
            force_build: When true, force a rebuild irrespective of incremental changes.
        Returns:
            The workflow builder.
        """
        workflow_name = "build.commit={commit}.timestamp={timestamp}".format(
            commit=self.git.get_commit_hash(),
            timestamp=base.timestamp(),
        )
        flow = workflow.Workflow(name=workflow_name)

        TASK_CLASS_MAP = dict(
            avro_java_library=workflow_task.AvroJavaLibraryTask,
            generated_pom=workflow_task.GeneratePomTask,
            java_binary=workflow_task.JavaBinaryTask,
            java_library=workflow_task.JavaLibraryTask,
            java_test=workflow_task.JavaTestTask,
            java_super_binary=workflow_task.JavaSuperBinaryTask,
            js_app=workflow_task.JSAppTask,
            npm_install=workflow_task.NpmInstallTask,
            bower_install=workflow_task.BowerInstallTask,
            python_binary=workflow_task.PythonBinaryTask,
            python_library=workflow_task.PythonLibraryTask,
            python_test=workflow_task.PythonTestTask,
            run_checkstyle=workflow_task.RunCheckstyleTask,
            run_java_test=workflow_task.RunJavaTestTask,
            run_python_test=workflow_task.RunPythonTestTask,
            run_scala_test=workflow_task.RunJavaTestTask,
            run_scalastyle=workflow_task.RunScalastyleTask,
            scala_library=workflow_task.ScalaLibraryTask,
            scala_test=workflow_task.ScalaTestTask,
        )

        for name, definition in self._build_defs.definitions.items():
            task_class = TASK_CLASS_MAP[definition.kind]
            task = task_class(
                workflow=flow,
                workspace=self,
                name=name,
                spec=definition,
                force=force_build,
            )
            for dep in definition.get("deps", tuple()):
                if isinstance(dep, artifact.Artifact):
                    # Direct Maven artifact dependencies are not reified as workflow tasks:
                    continue

                if isinstance(dep, build_defs.PythonPyPIDep):
                    # Python PyPI dependencies are external dependencies:
                    continue

                if isinstance(dep, build_defs.DynamicDep):
                    dep = dep.provider
                    if dep is None:
                        continue

                task.bind_input_to_task_output(input_name=dep, task=dep)

        return flow
Ejemplo n.º 14
0
    def _make_workflow(self, config, force_build=False):
        """Builds the workflow tasks after the build graph defined by the workspace.

        Args:
            config: Workspace configuration record.
            force_build: When true, force a rebuild irrespective of incremental changes.
        Returns:
            The workflow builder.
        """
        workflow_name = "build.commit={commit}.timestamp={timestamp}".format(
            commit=self.git.get_commit_hash(),
            timestamp=base.timestamp(),
        )
        flow = workflow.Workflow(name=workflow_name)

        TASK_CLASS_MAP = dict(
            avro_java_library=workflow_task.AvroJavaLibraryTask,
            generated_pom=workflow_task.GeneratePomTask,
            java_binary=workflow_task.JavaBinaryTask,
            java_library=workflow_task.JavaLibraryTask,
            java_test=workflow_task.JavaTestTask,
            java_super_binary=workflow_task.JavaSuperBinaryTask,

            js_app=workflow_task.JSAppTask,
            npm_install=workflow_task.NpmInstallTask,
            bower_install=workflow_task.BowerInstallTask,

            python_binary=workflow_task.PythonBinaryTask,
            python_library=workflow_task.PythonLibraryTask,
            python_test=workflow_task.PythonTestTask,
            run_checkstyle=workflow_task.RunCheckstyleTask,
            run_java_test=workflow_task.RunJavaTestTask,
            run_python_test=workflow_task.RunPythonTestTask,
            run_scala_test=workflow_task.RunJavaTestTask,
            run_scalastyle=workflow_task.RunScalastyleTask,
            scala_library=workflow_task.ScalaLibraryTask,
            scala_test=workflow_task.ScalaTestTask,
        )

        for name, definition in self._build_defs.definitions.items():
            task_class = TASK_CLASS_MAP[definition.kind]
            task = task_class(
                workflow=flow,
                workspace=self,
                name=name,
                spec=definition,
                force=force_build,
            )
            for dep in definition.get("deps", tuple()):
                if isinstance(dep, artifact.Artifact):
                    # Direct Maven artifact dependencies are not reified as workflow tasks:
                    continue

                if isinstance(dep, build_defs.PythonPyPIDep):
                    # Python PyPI dependencies are external dependencies:
                    continue

                if isinstance(dep, build_defs.DynamicDep):
                    dep = dep.provider
                    if dep is None:
                        continue

                task.bind_input_to_task_output(input_name=dep, task=dep)

        return flow
Ejemplo n.º 15
0
    def __init__(self,
                 *arglist,
                 args=None,
                 exit_code=None,
                 work_dir=None,
                 env=None,
                 log_dir=None,
                 direct_log=False,
                 collect_log=True,
                 start=True,
                 wait_for=True):
        """Runs a shell command.

        The command runs in the specified working directory and with the given
        environment.
        The command takes no standard input.
        Its output and error streams are captured in files, and are exposed as
        properties once the process has completed.

        Args:
            *arglist: Command-line, as an array of command-line arguments.
                First argument is the path to the executable.
            args: Keyword argument, alternative to *arglist.
            exit_code: Optional command exit code to require, or None.
            work_dir: Working directory. None means current workding directory.
            env: Optional environment variables for the subprocess, or None.
            log_dir: Optional directory where to write files capturing the command output streams.
                Defaults to the log directory (FLAGS.log_dir).
            direct_log: When set, the command's output and error streams are directly written
                to the log files, instead of being piped through this process.
            collect_log: When set, log files are collected in memory and removed.
            start: Whether to start running the command right away.
            wait_for: Whether to wait for the command to complete.
        Raises:
            CommandError: if the sub-process exit code does not match exit_code.
        """
        self._command_id = CommandID.get_new_id()
        assert (args is None) ^ (len(arglist) == 0)
        if args is None:
            self._args = tuple(arglist)
        else:
            self._args = tuple(args)
        self._required_exit_code = exit_code
        self._work_dir = work_dir or os.getcwd()
        self._env = env or os.environ
        log_dir = log_dir or FLAGS.log_dir

        name = os.path.basename(self._args[0])
        timestamp = base.timestamp()

        self._direct_log = direct_log
        self._collect_log = collect_log
        self._output_path = os.path.join(
            log_dir, "%s.%s.%d.out" % (name, timestamp, os.getpid()))
        self._error_path = os.path.join(
            log_dir, "%s.%s.%d.err" % (name, timestamp, os.getpid()))

        self._process = None
        self._output_bytes = None
        self._error_bytes = None

        if start:
            self.start(wait_for=wait_for)
Ejemplo n.º 16
0
    def __init__(
        self,
        workspace,
        config=None,
        avro_version=None,
        scala_version=None,
        checkstyle_version=None,
        scalastyle_version=None,
    ):
        self._workspace = workspace

        if config is None:
            config = self.workspace.config
        self._config = config

        if avro_version is None:
            avro_version = self.config.avro_version
        if scala_version is None:
            scala_version = self.config.scala_version
        if checkstyle_version is None:
            checkstyle_version = self.config.checkstyle_version
        if scalastyle_version is None:
            scalastyle_version = self.config.scalastyle_version

        self._jar_path = get_executable_path("jar")
        self._java_path = get_executable_path("java")
        self._javac_path = get_executable_path("javac")
        self._mvn_path = get_executable_path("mvn")
        self._python_path = get_executable_path("python")
        self._npm_path = get_executable_path("npm")
        self._bower_path = get_executable_path("bower")
        self._grunt_path = get_executable_path("grunt")

        assert (self.java_path is not None), "Required 'java' executable not found."
        assert (self.javac_path is not None), "Required 'javac' executable not found."
        assert (self.python_path is not None), "Required 'python' executable not found."

        logging.debug("Path for 'java' executable: %r", self.java_path)
        logging.debug("Path for 'javac' executable: %r", self.javac_path)
        logging.debug("Path for 'python' executable: %r", self.python_path)
        logging.debug("Path for 'npm' executable: %r", self.npm_path)
        logging.debug("Path for 'bower' executable: %r", self.bower_path)
        logging.debug("Path for 'grunt' executable: %r", self.grunt_path)

        self._avro_version = avro_version
        self._scala_version = scala_version
        self._checkstyle_version = checkstyle_version
        self._scalastyle_version = scalastyle_version

        self._avro_lib_classpath = None
        self._avro_ipc_classpath = None
        self._avro_tools_classpath = None
        self._scalac_classpath = None
        self._checkstyle_classpath = None
        self._scalastyle_classpath = None

        self._timestamp = base.timestamp()

        self._maven_cloned_repo_dir = os.path.join(
            self.workspace.temp_dir,
            "maven_repository",
            "%s.%s" % (self._timestamp, os.getpid()),
        )

        logging.debug("KijiBuild builtin fingerprint salt: %r", FINGERPRINT_SALT)
        logging.debug("Java version: %r", self.java_version_text)
        logging.debug("Javac version: %r", self.javac_version_text)
        logging.debug("Python version: %r", self.python_version_text)

        # Compute the fingerprint salt:
        self._fingerprint_salt = "\n".join([
            FINGERPRINT_SALT,
            self.java_version_text,
            self.javac_version_text,
            self.python_version_text,
        ])
        logging.debug("KijiBuild composite fingerprint: %r", self._fingerprint_salt)
Ejemplo n.º 17
0
    def __init__(
        self,
        *arglist,
        args=None,
        exit_code=None,
        work_dir=None,
        env=None,
        log_dir=None,
        direct_log=False,
        collect_log=True,
        start=True,
        wait_for=True
    ):
        """Runs a shell command.

        The command runs in the specified working directory and with the given
        environment.
        The command takes no standard input.
        Its output and error streams are captured in files, and are exposed as
        properties once the process has completed.

        Args:
            *arglist: Command-line, as an array of command-line arguments.
                First argument is the path to the executable.
            args: Keyword argument, alternative to *arglist.
            exit_code: Optional command exit code to require, or None.
            work_dir: Working directory. None means current workding directory.
            env: Optional environment variables for the subprocess, or None.
            log_dir: Optional directory where to write files capturing the command output streams.
                Defaults to the log directory (FLAGS.log_dir).
            direct_log: When set, the command's output and error streams are directly written
                to the log files, instead of being piped through this process.
            collect_log: When set, log files are collected in memory and removed.
            start: Whether to start running the command right away.
            wait_for: Whether to wait for the command to complete.
        Raises:
            CommandError: if the sub-process exit code does not match exit_code.
        """
        self._command_id = CommandID.get_new_id()
        assert (args is None) ^ (len(arglist) == 0)
        if args is None:
            self._args = tuple(arglist)
        else:
            self._args = tuple(args)
        self._required_exit_code = exit_code
        self._work_dir = work_dir or os.getcwd()
        self._env = env or os.environ
        log_dir = log_dir or FLAGS.log_dir

        name = os.path.basename(self._args[0])
        timestamp = base.timestamp()

        self._direct_log = direct_log
        self._collect_log = collect_log
        self._output_path = os.path.join(log_dir, "%s.%s.%d.out" % (name, timestamp, os.getpid()))
        self._error_path = os.path.join(log_dir, "%s.%s.%d.err" % (name, timestamp, os.getpid()))

        self._process = None
        self._output_bytes = None
        self._error_bytes = None

        if start:
            self.start(wait_for=wait_for)