def check_dependencies_ready(dependencies, start_date, dependencies_to_ignore):
    """Checks if every dependent pipeline has completed

    Args:
        dependencies(dict): dict from id to name of pipelines it depends on
        start_date(str): string representing the start date of the pipeline
        dependencies_to_ignore(list of str): dependencies to ignore if failed
    """

    print 'Checking dependency at ', str(datetime.now())

    dependency_ready = True

    # Convert date string to datetime object
    start_date = datetime.strptime(start_date, '%Y-%m-%d')

    for pipeline in dependencies.keys():
        # Get instances of each pipeline
        instances = list_pipeline_instances(pipeline)
        failures = []

        # Collect all pipeline instances that are scheduled for today
        instances_today = []
        for instance in instances:
            date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
            if date.date() == start_date.date():
                instances_today.append(instance)

        # Dependency pipeline has not started from today
        if not instances_today:
            dependency_ready = False

        for instance in instances_today:
            # One of the dependency failed/cancelled
            if instance[STATUS] in FAILED_STATUSES:
                if dependencies[pipeline] not in dependencies_to_ignore:
                    raise Exception(
                        'Pipeline %s (ID: %s) has bad status: %s'
                        % (dependencies[pipeline], pipeline, instance[STATUS])
                    )
                else:
                    failures.append(dependencies[pipeline])
            # Dependency is still running
            elif instance[STATUS] != FINISHED:
                dependency_ready = False

    return dependency_ready, failures
def check_dependencies_ready(dependencies, start_date):
    """Checks if every dependent pipeline has completed

    Args:
        dependencies(list of str): list of pipeline name that it depends on
        start_date(str): string representing the start date of the pipeline
    """

    print 'Checking dependency at ', str(datetime.now())

    dependency_ready = True

    # Convert date string to datetime object
    start_date = datetime.strptime(start_date, '%Y-%m-%d')

    for pipeline in dependencies:
        # Get instances of each pipeline
        instances = list_pipeline_instances(pipeline)

        # Collect all pipeline instances that are scheduled for today
        instances_today = []
        for instance in instances:
            date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
            if date.date() == start_date.date():
                instances_today.append(instance)

        # Dependency pipeline has not started from today
        if not instances_today:
            dependency_ready = False

        for instance in instances_today:
            # One of the dependency failed/cancelled
            if instance[STATUS] in FAILED_STATUSES:
                raise Exception(
                    'Pipeline %s has bad status: %s'
                    % (pipeline, instance[STATUS])
                )
            # Dependency is still running
            elif instance[STATUS] != FINISHED:
                dependency_ready = False

    # All dependencies are done
    return dependency_ready
Exemple #3
0
def check_dependencies_ready(dependencies, start_date):
    """Checks if every dependent pipeline has completed

    Args:
        dependencies(list of str): list of pipeline name that it depends on
        start_date(str): string representing the start date of the pipeline
    """

    print 'Checking dependency at ', str(datetime.now())

    dependency_ready = True

    # Convert date string to datetime object
    start_date = datetime.strptime(start_date, '%Y-%m-%d')

    for pipeline in dependencies:
        # Get instances of each pipeline
        instances = list_pipeline_instances(pipeline)

        # Collect all pipeline instances that are scheduled for today
        instances_today = []
        for instance in instances:
            date = datetime.strptime(instance[START_TIME], '%Y-%m-%dT%H:%M:%S')
            if date.date() == start_date.date():
                instances_today.append(instance)

        # Dependency pipeline has not started from today
        if not instances_today:
            dependency_ready = False

        for instance in instances_today:
            # One of the dependency failed/cancelled
            if instance[STATUS] in FAILED_STATUSES:
                raise Exception('Pipeline %s has bad status: %s' %
                                (pipeline, instance[STATUS]))
            # Dependency is still running
            elif instance[STATUS] != FINISHED:
                dependency_ready = False

    # All dependencies are done
    return dependency_ready