Python RunJob示例

示例#1

文件： test_consumers.py 项目： celestialized/Square_Dessa_atlas

    def test_running_job_consumers(self):
        from foundations_events.producers.jobs import RunJob
        from time import time

        queued_job_key = "project:{}:jobs:queued".format(self._project_name)
        self._redis.sadd(queued_job_key, self._job_id)

        global_queued_job_key = "projects:global:jobs:queued"
        self._redis.sadd(global_queued_job_key, self._job_id)

        current_time = time()
        RunJob(self._message_router, self._context).push_message()

        queued_jobs = self._redis.smembers(queued_job_key)
        self.assertEqual(set(), queued_jobs)

        global_queued_jobs = self._redis.smembers(global_queued_job_key)
        self.assertEqual(set(), global_queued_jobs)

        job_state_key = "jobs:{}:state".format(self._job_id)
        state = self._redis.get(job_state_key)
        self.assertEqual(b"running", state)

        start_time_key = "jobs:{}:start_time".format(self._job_id)
        string_start_time = self._redis.get(start_time_key)
        start_time = float(string_start_time.decode())
        self.assertLess(current_time - start_time, 1)

示例#2

文件： test_job_listing.py 项目： celestialized/Square_Dessa_atlas

    def _make_job(self):
        from foundations_contrib.global_state import message_router
        from foundations_events.producers.jobs import QueueJob
        from foundations_events.producers.jobs import RunJob

        QueueJob(message_router, self._context).push_message()
        RunJob(message_router, self._context).push_message()

示例#3

文件： jobs_tests_helper_mixin_v2.py 项目： celestialized/Square_Dessa_atlas

 def _make_running_job(klass, job_name, user, tags=None, start_timestamp=None):
     klass._foundations_job.job_id = job_name
     klass._foundations_job.user_name = user
     QueueJob(klass._message_router, klass._foundations_job).push_message()
     klass._fake_start_time(start_timestamp)
     RunJob(klass._message_router, klass._foundations_job).push_message()
     klass._set_tags(job_name, tags)
     klass._restore_time(start_timestamp, None)

示例#4

文件： jobs_tests_helper_mixin_v2.py 项目： celestialized/Square_Dessa_atlas

 def _make_completed_job(klass, job_name, user, tags=None, start_timestamp=None, end_timestamp=None, **kwargs):
     klass._foundations_job.job_id = job_name
     klass._foundations_job.user_name = user
     klass._foundations_job.provenance.job_run_data = kwargs
     QueueJob(klass._message_router, klass._foundations_job).push_message()
     klass._fake_start_time(start_timestamp)
     RunJob(klass._message_router, klass._foundations_job).push_message()
     klass._set_tags(job_name, tags)
     klass._fake_end_time(end_timestamp)
     CompleteJob(klass._message_router, klass._foundations_job).push_message()
     klass._restore_time(start_timestamp, end_timestamp)

示例#5

文件： jobs_tests_helper_mixin_v2.py 项目： tomzhang/atlas-1

 def _make_running_job(klass,
                       job_name,
                       user,
                       tags=None,
                       start_timestamp=None):
     klass._pipeline_context.file_name = job_name
     klass._pipeline_context.provenance.user_name = user
     QueueJob(klass._message_router, klass._pipeline_context).push_message()
     klass._fake_start_time(start_timestamp)
     RunJob(klass._message_router, klass._pipeline_context).push_message()
     klass._set_tags(job_name, tags)
     klass._restore_time(start_timestamp, None)

示例#6

def set_up_job_environment():
    from foundations_events.producers.jobs import QueueJob
    from foundations_events.producers.jobs import RunJob
    from foundations_contrib.global_state import (
        current_foundations_context,
        message_router,
        config_manager,
    )
    import atexit

    config_manager["_is_deployment"] = True
    _get_logger().debug(
        f"Foundations has been run with the following configuration:\n"
        f"{yaml.dump(config_manager.config(), default_flow_style=False)}")
    pipeline_context = current_foundations_context().pipeline_context()
    _set_job_state(pipeline_context)

    QueueJob(message_router, pipeline_context).push_message()
    RunJob(message_router, pipeline_context).push_message()

    atexit.register(_at_exit_callback)
    _set_up_exception_handling()

示例#7

文件： in_job_log_metric_script.py 项目： celestialized/Square_Dessa_atlas

import os

import foundations
from foundations_contrib.global_state import current_foundations_job, message_router
from foundations_events.producers.jobs import RunJob

foundations.set_project_name('default')

job_id = os.environ['ACCEPTANCE_TEST_JOB_ID']
current_foundations_job().job_id = job_id

RunJob(message_router, current_foundations_job()).push_message()

foundations.log_metric('key', 'value')
print('Hello World!')

示例#8

import os

import foundations
from foundations_contrib.global_state import current_foundations_context, message_router
from foundations_events.producers.jobs import RunJob

foundations.set_project_name('default')

job_id = os.environ['ACCEPTANCE_TEST_JOB_ID']
pipeline_context = current_foundations_context().pipeline_context()
pipeline_context.file_name = job_id

RunJob(message_router, pipeline_context).push_message()

foundations.set_tag('model type', 'simple mlp')
foundations.set_tag('data set', 'out of time')
foundations.set_tag('what I was doing,', 'drinking tea')

print('Hello World!')