def check_dependencies_ready(dependencies, start_date, dependencies_to_ignore): """Checks if every dependent pipeline has completed Args: dependencies(dict): dict from id to name of pipelines it depends on start_date(str): string representing the start date of the pipeline dependencies_to_ignore(list of str): dependencies to ignore if failed """ print 'Checking dependency at ', str(datetime.now()) dependency_ready = True # Convert date string to datetime object start_date = datetime.strptime(start_date, '%Y-%m-%d') for pipeline in dependencies.keys(): # Get instances of each pipeline instances = list_pipeline_instances(pipeline) failures = [] # Collect all pipeline instances that are scheduled for today instances_today = [] for instance in instances: date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S') if date.date() == start_date.date(): instances_today.append(instance) # Dependency pipeline has not started from today if not instances_today: dependency_ready = False for instance in instances_today: # One of the dependency failed/cancelled if instance[STATUS] in FAILED_STATUSES: if dependencies[pipeline] not in dependencies_to_ignore: raise Exception( 'Pipeline %s (ID: %s) has bad status: %s' % (dependencies[pipeline], pipeline, instance[STATUS]) ) else: failures.append(dependencies[pipeline]) # Dependency is still running elif instance[STATUS] != FINISHED: dependency_ready = False return dependency_ready, failures
def check_dependencies_ready(dependencies, start_date): """Checks if every dependent pipeline has completed Args: dependencies(list of str): list of pipeline name that it depends on start_date(str): string representing the start date of the pipeline """ print 'Checking dependency at ', str(datetime.now()) dependency_ready = True # Convert date string to datetime object start_date = datetime.strptime(start_date, '%Y-%m-%d') for pipeline in dependencies: # Get instances of each pipeline instances = list_pipeline_instances(pipeline) # Collect all pipeline instances that are scheduled for today instances_today = [] for instance in instances: date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S') if date.date() == start_date.date(): instances_today.append(instance) # Dependency pipeline has not started from today if not instances_today: dependency_ready = False for instance in instances_today: # One of the dependency failed/cancelled if instance[STATUS] in FAILED_STATUSES: raise Exception( 'Pipeline %s has bad status: %s' % (pipeline, instance[STATUS]) ) # Dependency is still running elif instance[STATUS] != FINISHED: dependency_ready = False # All dependencies are done return dependency_ready
def check_dependencies_ready(dependencies, start_date): """Checks if every dependent pipeline has completed Args: dependencies(list of str): list of pipeline name that it depends on start_date(str): string representing the start date of the pipeline """ print 'Checking dependency at ', str(datetime.now()) dependency_ready = True # Convert date string to datetime object start_date = datetime.strptime(start_date, '%Y-%m-%d') for pipeline in dependencies: # Get instances of each pipeline instances = list_pipeline_instances(pipeline) # Collect all pipeline instances that are scheduled for today instances_today = [] for instance in instances: date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S') if date.date() == start_date.date(): instances_today.append(instance) # Dependency pipeline has not started from today if not instances_today: dependency_ready = False for instance in instances_today: # One of the dependency failed/cancelled if instance[STATUS] in FAILED_STATUSES: raise Exception('Pipeline %s has bad status: %s' % (pipeline, instance[STATUS])) # Dependency is still running elif instance[STATUS] != FINISHED: dependency_ready = False # All dependencies are done return dependency_ready