def check_dependencies_ready(dependencies, start_date, dependencies_to_ignore):
"""Checks if every dependent pipeline has completed
Args:
dependencies(dict): dict from id to name of pipelines it depends on
start_date(str): string representing the start date of the pipeline
dependencies_to_ignore(list of str): dependencies to ignore if failed
"""
print 'Checking dependency at ', str(datetime.now())
dependency_ready = True
# Convert date string to datetime object
start_date = datetime.strptime(start_date, '%Y-%m-%d')
for pipeline in dependencies.keys():
# Get instances of each pipeline
instances = list_pipeline_instances(pipeline)
failures = []
# Collect all pipeline instances that are scheduled for today
instances_today = []
for instance in instances:
date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
if date.date() == start_date.date():
instances_today.append(instance)
# Dependency pipeline has not started from today
if not instances_today:
dependency_ready = False
for instance in instances_today:
# One of the dependency failed/cancelled
if instance[STATUS] in FAILED_STATUSES:
if dependencies[pipeline] not in dependencies_to_ignore:
raise Exception(
'Pipeline %s (ID: %s) has bad status: %s'
% (dependencies[pipeline], pipeline, instance[STATUS])
)
else:
failures.append(dependencies[pipeline])
# Dependency is still running
elif instance[STATUS] != FINISHED:
dependency_ready = False
return dependency_ready, failures
def check_dependencies_ready(dependencies, start_date):
"""Checks if every dependent pipeline has completed
Args:
dependencies(list of str): list of pipeline name that it depends on
start_date(str): string representing the start date of the pipeline
"""
print 'Checking dependency at ', str(datetime.now())
dependency_ready = True
# Convert date string to datetime object
start_date = datetime.strptime(start_date, '%Y-%m-%d')
for pipeline in dependencies:
# Get instances of each pipeline
instances = list_pipeline_instances(pipeline)
# Collect all pipeline instances that are scheduled for today
instances_today = []
for instance in instances:
date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
if date.date() == start_date.date():
instances_today.append(instance)
# Dependency pipeline has not started from today
if not instances_today:
dependency_ready = False
for instance in instances_today:
# One of the dependency failed/cancelled
if instance[STATUS] in FAILED_STATUSES:
raise Exception(
'Pipeline %s has bad status: %s'
% (pipeline, instance[STATUS])
)
# Dependency is still running
elif instance[STATUS] != FINISHED:
dependency_ready = False
# All dependencies are done
return dependency_ready
def check_dependencies_ready(dependencies, start_date):
"""Checks if every dependent pipeline has completed
Args:
dependencies(list of str): list of pipeline name that it depends on
start_date(str): string representing the start date of the pipeline
"""
print 'Checking dependency at ', str(datetime.now())
dependency_ready = True
# Convert date string to datetime object
start_date = datetime.strptime(start_date, '%Y-%m-%d')
for pipeline in dependencies:
# Get instances of each pipeline
instances = list_pipeline_instances(pipeline)
# Collect all pipeline instances that are scheduled for today
instances_today = []
for instance in instances:
date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
if date.date() == start_date.date():
instances_today.append(instance)
# Dependency pipeline has not started from today
if not instances_today:
dependency_ready = False
for instance in instances_today:
# One of the dependency failed/cancelled
if instance[STATUS] in FAILED_STATUSES:
raise Exception('Pipeline %s has bad status: %s' %
(pipeline, instance[STATUS]))
# Dependency is still running
elif instance[STATUS] != FINISHED:
dependency_ready = False
# All dependencies are done
return dependency_ready