def test_running_job_consumers(self):
from foundations_events.producers.jobs import RunJob
from time import time
queued_job_key = "project:{}:jobs:queued".format(self._project_name)
self._redis.sadd(queued_job_key, self._job_id)
global_queued_job_key = "projects:global:jobs:queued"
self._redis.sadd(global_queued_job_key, self._job_id)
current_time = time()
RunJob(self._message_router, self._context).push_message()
queued_jobs = self._redis.smembers(queued_job_key)
self.assertEqual(set(), queued_jobs)
global_queued_jobs = self._redis.smembers(global_queued_job_key)
self.assertEqual(set(), global_queued_jobs)
job_state_key = "jobs:{}:state".format(self._job_id)
state = self._redis.get(job_state_key)
self.assertEqual(b"running", state)
start_time_key = "jobs:{}:start_time".format(self._job_id)
string_start_time = self._redis.get(start_time_key)
start_time = float(string_start_time.decode())
self.assertLess(current_time - start_time, 1)
def _make_job(self):
from foundations_contrib.global_state import message_router
from foundations_events.producers.jobs import QueueJob
from foundations_events.producers.jobs import RunJob
QueueJob(message_router, self._context).push_message()
RunJob(message_router, self._context).push_message()
def _make_running_job(klass, job_name, user, tags=None, start_timestamp=None):
klass._foundations_job.job_id = job_name
klass._foundations_job.user_name = user
QueueJob(klass._message_router, klass._foundations_job).push_message()
klass._fake_start_time(start_timestamp)
RunJob(klass._message_router, klass._foundations_job).push_message()
klass._set_tags(job_name, tags)
klass._restore_time(start_timestamp, None)
def _make_completed_job(klass, job_name, user, tags=None, start_timestamp=None, end_timestamp=None, **kwargs):
klass._foundations_job.job_id = job_name
klass._foundations_job.user_name = user
klass._foundations_job.provenance.job_run_data = kwargs
QueueJob(klass._message_router, klass._foundations_job).push_message()
klass._fake_start_time(start_timestamp)
RunJob(klass._message_router, klass._foundations_job).push_message()
klass._set_tags(job_name, tags)
klass._fake_end_time(end_timestamp)
CompleteJob(klass._message_router, klass._foundations_job).push_message()
klass._restore_time(start_timestamp, end_timestamp)
def _make_running_job(klass,
job_name,
user,
tags=None,
start_timestamp=None):
klass._pipeline_context.file_name = job_name
klass._pipeline_context.provenance.user_name = user
QueueJob(klass._message_router, klass._pipeline_context).push_message()
klass._fake_start_time(start_timestamp)
RunJob(klass._message_router, klass._pipeline_context).push_message()
klass._set_tags(job_name, tags)
klass._restore_time(start_timestamp, None)
def set_up_job_environment():
from foundations_events.producers.jobs import QueueJob
from foundations_events.producers.jobs import RunJob
from foundations_contrib.global_state import (
current_foundations_context,
message_router,
config_manager,
)
import atexit
config_manager["_is_deployment"] = True
_get_logger().debug(
f"Foundations has been run with the following configuration:\n"
f"{yaml.dump(config_manager.config(), default_flow_style=False)}")
pipeline_context = current_foundations_context().pipeline_context()
_set_job_state(pipeline_context)
QueueJob(message_router, pipeline_context).push_message()
RunJob(message_router, pipeline_context).push_message()
atexit.register(_at_exit_callback)
_set_up_exception_handling()
import os
import foundations
from foundations_contrib.global_state import current_foundations_job, message_router
from foundations_events.producers.jobs import RunJob
foundations.set_project_name('default')
job_id = os.environ['ACCEPTANCE_TEST_JOB_ID']
current_foundations_job().job_id = job_id
RunJob(message_router, current_foundations_job()).push_message()
foundations.log_metric('key', 'value')
print('Hello World!')
import os
import foundations
from foundations_contrib.global_state import current_foundations_context, message_router
from foundations_events.producers.jobs import RunJob
foundations.set_project_name('default')
job_id = os.environ['ACCEPTANCE_TEST_JOB_ID']
pipeline_context = current_foundations_context().pipeline_context()
pipeline_context.file_name = job_id
RunJob(message_router, pipeline_context).push_message()
foundations.set_tag('model type', 'simple mlp')
foundations.set_tag('data set', 'out of time')
foundations.set_tag('what I was doing,', 'drinking tea')
print('Hello World!')