def test_put_replace_config(self, httpsconnection_mock):
def _getresponse():
self.assertLessEqual(self._call_count, 1)
result = mock.Mock()
if self._call_count == 0:
result.status = httplib.OK
result.read.return_value = '{"type":"file", "sha":"abc"}'
else:
result.status = httplib.CREATED
self._call_count += 1
return result
conn = mock.Mock()
httpsconnection_mock.return_value = conn
conn.getresponse = _getresponse
repository = GithubRepository()
repository._put_config('/some_path', 'some_content')
self.assertEqual('GET', conn.request.call_args_list[0][0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args_list[0][0][1])
self.assertIsNone(conn.request.call_args_list[0][0][2])
self.assertEqual('PUT', conn.request.call_args_list[1][0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args_list[1][0][1])
encoded_content = base64.b64encode('some_content')
self.assertEqual('{"content": "%s", "committer": {"name": "Pinball", '
'"email": "*****@*****.**"}, "message": '
'"updating config", "sha": "abc"}' % encoded_content,
conn.request.call_args_list[1][0][2])
def test_put_replace_config(self, httpsconnection_mock):
def _getresponse():
self.assertLessEqual(self._call_count, 1)
result = mock.Mock()
if self._call_count == 0:
result.status = httplib.OK
result.read.return_value = '{"type":"file", "sha":"abc"}'
else:
result.status = httplib.CREATED
self._call_count += 1
return result
conn = mock.Mock()
httpsconnection_mock.return_value = conn
conn.getresponse = _getresponse
repository = GithubRepository()
repository._put_config('/some_path', 'some_content')
self.assertEqual('GET', conn.request.call_args_list[0][0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args_list[0][0][1])
self.assertIsNone(conn.request.call_args_list[0][0][2])
self.assertEqual('PUT', conn.request.call_args_list[1][0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args_list[1][0][1])
encoded_content = base64.b64encode('some_content')
self.assertEqual(
'{"content": "%s", "committer": {"name": "Pinball", '
'"email": "*****@*****.**"}, "message": '
'"updating config", "sha": "abc"}' % encoded_content,
conn.request.call_args_list[1][0][2])
def test_get_config(self, httpsconnection_mock):
conn = mock.Mock()
httpsconnection_mock.return_value = conn
response = mock.Mock()
conn.getresponse.return_value = response
response.status = httplib.OK
encoded_content = base64.b64encode('some_content')
response.read.return_value = '{{"content": "{0!s}"}}'.format(encoded_content)
repository = GithubRepository()
content = repository._get_config('/some_path')
self.assertEqual('some_content', content)
self.assertEqual('GET', conn.request.call_args[0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args[0][1])
self.assertIsNone(conn.request.call_args[0][2])
def test_list_directory(self, httpsconnection_mock):
conn = mock.Mock()
httpsconnection_mock.return_value = conn
response = mock.Mock()
conn.getresponse.return_value = response
response.status = httplib.OK
response.read.return_value = ('[{"type":"dir", "name":"some_dir"},'
' {"type":"file", "name":"some_file"}]')
repository = GithubRepository()
paths = repository._list_directory('/some_path/', False)
self.assertEqual(['some_dir/', 'some_file'], paths)
self.assertEqual('GET', conn.request.call_args[0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args[0][1])
self.assertIsNone(conn.request.call_args[0][2])
def test_get_config(self, httpsconnection_mock):
conn = mock.Mock()
httpsconnection_mock.return_value = conn
response = mock.Mock()
conn.getresponse.return_value = response
response.status = httplib.OK
encoded_content = base64.b64encode('some_content')
response.read.return_value = '{"content": "%s"}' % encoded_content
repository = GithubRepository()
content = repository._get_config('/some_path')
self.assertEqual('some_content', content)
self.assertEqual('GET', conn.request.call_args[0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args[0][1])
self.assertIsNone(conn.request.call_args[0][2])
def test_list_directory(self, httpsconnection_mock):
conn = mock.Mock()
httpsconnection_mock.return_value = conn
response = mock.Mock()
conn.getresponse.return_value = response
response.status = httplib.OK
response.read.return_value = ('[{"type":"dir", "name":"some_dir"},'
' {"type":"file", "name":"some_file"}]')
repository = GithubRepository()
paths = repository._list_directory('/some_path/', False)
self.assertEqual(['some_dir/', 'some_file'], paths)
self.assertEqual('GET', conn.request.call_args[0][0])
self.assertEqual('/api/v3/repos/data/configs/contents/some_path',
conn.request.call_args[0][1])
self.assertIsNone(conn.request.call_args[0][2])
def __init__(self, params=None):
self._repository = GithubRepository()
class RepositoryConfigParser(ConfigParser):
def __init__(self, params=None):
self._repository = GithubRepository()
@staticmethod
def _job_config_to_job(job_config, outputs):
"""Create a job from a job config.
Args:
job_config: The job config.
outputs: The list of job output names.
Returns:
Pinball job constructed from the config.
"""
# load template
job_template = load_path(job_config.template)(
job_config.job,
max_attempts=job_config.max_attempts,
emails=job_config.emails,
priority=job_config.priority)
return job_template.get_pinball_job(job_config.parents,
outputs,
job_config.template_params)
@staticmethod
def _condition_config_to_condition(condition_config, outputs):
"""Create a condition from a condition config.
Args:
condition_config: The condition config.
outputs: The list of condition output names.
Returns:
Pinball job constructed from the config.
"""
# load template
condition_template = load_path(condition_config.template)(
condition_config.job,
max_attempts=condition_config.max_attempts,
retry_delay_sec=condition_config.retry_delay_sec,
emails=condition_config.emails,
priority=condition_config.priority)
return condition_template.get_pinball_condition(
outputs, params=condition_config.template_params)
@staticmethod
def _job_config_to_token(workflow, instance, job_config, job_outputs):
"""Create a job token from a job config.
Args:
workflow: The workflow name.
instance: The workflow instance.
job_config: The job config to create token from.
job_outputs: The names of the job outputs.
Returns:
Job token constructed from the job config.
"""
if job_config.is_condition:
job = RepositoryConfigParser._condition_config_to_condition(
job_config, job_outputs)
else:
job = RepositoryConfigParser._job_config_to_job(job_config,
job_outputs)
name = Name(workflow=workflow, instance=instance,
job_state=Name.WAITING_STATE, job=job_config.job)
job_token = Token(name=name.get_job_token_name(),
data=pickle.dumps(job))
return job_token
def get_schedule_token(self, workflow):
schedule_config = self._repository.get_schedule(workflow)
timestamp = schedule_to_timestamp(schedule_config.time,
schedule_config.start_date)
recurrence = recurrence_str_to_sec(schedule_config.recurrence)
overrun_policy = OverrunPolicy.from_string(
schedule_config.overrun_policy)
schedule = WorkflowSchedule(
next_run_time=timestamp,
recurrence_seconds=recurrence,
overrun_policy=overrun_policy,
workflow=schedule_config.workflow,
emails=schedule_config.emails,
#TODO(mao): to make it flexible that allow users specify through UI
max_running_instances=PinballConfig.DEFAULT_MAX_WORKFLOW_RUNNING_INSTANCES)
schedule.advance_next_run_time()
timestamp = schedule.next_run_time
token_name = (
Name(workflow=schedule_config.workflow
).get_workflow_schedule_token_name())
return Token(name=token_name, owner='parser',
expirationTime=timestamp,
data=pickle.dumps(schedule))
def get_workflow_tokens(self, workflow):
# TODO(pawel): add workflow connectivity check.
job_configs = {}
top_level_job_names = []
job_names = self._repository.get_job_names(workflow)
for job_name in job_names:
job_config = self._repository.get_job(workflow, job_name)
job_configs[job_name] = job_config
if not job_config.parents:
top_level_job_names.append(job_name)
job_config.parents = [Name.WORKFLOW_START_INPUT]
job_outputs = collections.defaultdict(list)
for job_config in job_configs.values():
for parent_job_name in job_config.parents:
job_outputs[parent_job_name].append(job_config.job)
result = []
instance = get_unique_workflow_instance()
# Convert job configs to job tokens.
for job_config in job_configs.values():
token = RepositoryConfigParser._job_config_to_token(
workflow,
instance,
job_config,
job_outputs[job_config.job])
result.append(token)
# Create triggering events for top-level jobs.
for job_name in top_level_job_names:
event = Event(creator='repository_config_parser')
event_name = Name(workflow=workflow,
instance=instance,
job=job_name,
input_name=Name.WORKFLOW_START_INPUT,
event='workflow_start_event')
result.append(Token(name=event_name.get_event_token_name(),
data=pickle.dumps(event)))
return result
def get_workflow_names(self):
return self._repository.get_workflow_names()
def __init__(self, params=None):
self._repository = GithubRepository()
class RepositoryConfigParser(ConfigParser):
def __init__(self, params=None):
self._repository = GithubRepository()
@staticmethod
def _job_config_to_job(job_config, outputs):
"""Create a job from a job config.
Args:
job_config: The job config.
outputs: The list of job output names.
Returns:
Pinball job constructed from the config.
"""
# load template
job_template = load_path(job_config.template)(
job_config.job,
max_attempts=job_config.max_attempts,
emails=job_config.emails,
priority=job_config.priority)
return job_template.get_pinball_job(job_config.parents, outputs,
job_config.template_params)
@staticmethod
def _condition_config_to_condition(condition_config, outputs):
"""Create a condition from a condition config.
Args:
condition_config: The condition config.
outputs: The list of condition output names.
Returns:
Pinball job constructed from the config.
"""
# load template
condition_template = load_path(condition_config.template)(
condition_config.job,
max_attempts=condition_config.max_attempts,
retry_delay_sec=condition_config.retry_delay_sec,
emails=condition_config.emails,
priority=condition_config.priority)
return condition_template.get_pinball_condition(
outputs, params=condition_config.template_params)
@staticmethod
def _job_config_to_token(workflow, instance, job_config, job_outputs):
"""Create a job token from a job config.
Args:
workflow: The workflow name.
instance: The workflow instance.
job_config: The job config to create token from.
job_outputs: The names of the job outputs.
Returns:
Job token constructed from the job config.
"""
if job_config.is_condition:
job = RepositoryConfigParser._condition_config_to_condition(
job_config, job_outputs)
else:
job = RepositoryConfigParser._job_config_to_job(
job_config, job_outputs)
name = Name(workflow=workflow,
instance=instance,
job_state=Name.WAITING_STATE,
job=job_config.job)
job_token = Token(name=name.get_job_token_name(),
data=pickle.dumps(job))
return job_token
def get_schedule_token(self, workflow):
schedule_config = self._repository.get_schedule(workflow)
timestamp = schedule_to_timestamp(schedule_config.time,
schedule_config.start_date)
recurrence = recurrence_str_to_sec(schedule_config.recurrence)
overrun_policy = OverrunPolicy.from_string(
schedule_config.overrun_policy)
schedule = WorkflowSchedule(
next_run_time=timestamp,
recurrence_seconds=recurrence,
overrun_policy=overrun_policy,
workflow=schedule_config.workflow,
emails=schedule_config.emails,
#TODO(mao): to make it flexible that allow users specify through UI
max_running_instances=PinballConfig.
DEFAULT_MAX_WORKFLOW_RUNNING_INSTANCES)
schedule.advance_next_run_time()
timestamp = schedule.next_run_time
token_name = (Name(workflow=schedule_config.workflow).
get_workflow_schedule_token_name())
return Token(name=token_name,
owner='parser',
expirationTime=timestamp,
data=pickle.dumps(schedule))
def get_workflow_tokens(self, workflow):
# TODO(pawel): add workflow connectivity check.
job_configs = {}
top_level_job_names = []
job_names = self._repository.get_job_names(workflow)
for job_name in job_names:
job_config = self._repository.get_job(workflow, job_name)
job_configs[job_name] = job_config
if not job_config.parents:
top_level_job_names.append(job_name)
job_config.parents = [Name.WORKFLOW_START_INPUT]
job_outputs = collections.defaultdict(list)
for job_config in job_configs.values():
for parent_job_name in job_config.parents:
job_outputs[parent_job_name].append(job_config.job)
result = []
instance = get_unique_workflow_instance()
# Convert job configs to job tokens.
for job_config in job_configs.values():
token = RepositoryConfigParser._job_config_to_token(
workflow, instance, job_config, job_outputs[job_config.job])
result.append(token)
# Create triggering events for top-level jobs.
for job_name in top_level_job_names:
event = Event(creator='repository_config_parser')
event_name = Name(workflow=workflow,
instance=instance,
job=job_name,
input_name=Name.WORKFLOW_START_INPUT,
event='workflow_start_event')
result.append(
Token(name=event_name.get_event_token_name(),
data=pickle.dumps(event)))
return result
def get_workflow_names(self):
return self._repository.get_workflow_names()
