def main(): """ Main Function """ parser = argparse.ArgumentParser() parser.add_argument( '--dependencies', type=str, nargs='+', default=None) parser.add_argument('--refresh_rate', dest='refresh_rate', default='900') parser.add_argument('--start_date', dest='start_date') args = parser.parse_args() # Exit if there are no dependencies if not args.dependencies: sys.exit() # Create mapping from pipeline name to id pipeline_name_to_id = dict( (pipeline['name'], pipeline['id']) for pipeline in list_pipelines() ) # Remove whitespace from dependency list dependencies = map(str.strip, args.dependencies) # Check if all dependencies are valid pipelines for dependency in dependencies: if dependency not in pipeline_name_to_id: raise Exception('Pipeline not found: %s.' % dependency) # Map from pipeline object to pipeline ID dependencies = [pipeline_name_to_id[dependency] for dependency in dependencies] print 'Start checking for dependencies' start_time = datetime.now() # Loop until all dependent pipelines have finished while not check_dependencies_ready(dependencies, args.start_date): print 'checking' time.sleep(float(args.refresh_rate)) print 'Finished checking for dependencies. Total time spent: ', print (datetime.now() - start_time).total_seconds(), ' seconds'
def main(): """ Main Function """ parser = argparse.ArgumentParser() parser.add_argument('--dependencies', type=str, nargs='+', default=None) parser.add_argument('--refresh_rate', dest='refresh_rate', default='900') parser.add_argument('--start_date', dest='start_date') args = parser.parse_args() # Exit if there are no dependencies if not args.dependencies: sys.exit() # Create mapping from pipeline name to id pipeline_name_to_id = dict( (pipeline['name'], pipeline['id']) for pipeline in list_pipelines()) # Remove whitespace from dependency list dependencies = map(str.strip, args.dependencies) # Check if all dependencies are valid pipelines for dependency in dependencies: if dependency not in pipeline_name_to_id: raise Exception('Pipeline not found: %s.' % dependency) # Map from pipeline object to pipeline ID dependencies = [ pipeline_name_to_id[dependency] for dependency in dependencies ] print 'Start checking for dependencies' start_time = datetime.now() # Loop until all dependent pipelines have finished while not check_dependencies_ready(dependencies, args.start_date): print 'checking' time.sleep(float(args.refresh_rate)) print 'Finished checking for dependencies. Total time spent: ', print(datetime.now() - start_time).total_seconds(), ' seconds'
def dependency_check(): parser = argparse.ArgumentParser() parser.add_argument('--dependencies', type=str, nargs='+', default=[]) parser.add_argument('--dependencies_ok_to_fail', type=str, nargs='+', default=[]) parser.add_argument('--pipeline_name', dest='pipeline_name') parser.add_argument('--refresh_rate', dest='refresh_rate', default='900') parser.add_argument('--start_date', dest='start_date') parser.add_argument('--sns_topic_arn', dest="sns_topic_arn") args = parser.parse_args() # Exit if there are no dependencies if not args.dependencies and not args.dependencies_ok_to_fail: sys.exit() # Create mapping from pipeline name to id pipeline_name_to_id = dict( (pipeline['name'], pipeline['id']) for pipeline in list_pipelines()) # Remove whitespace from dependency lists dependencies = map(str.strip, args.dependencies) dependencies_to_ignore = map(str.strip, args.dependencies_ok_to_fail) # Add the dependencies which can fail to the list of dependencies dependencies.extend(dependencies_to_ignore) # Check if all dependencies are valid pipelines for dependency in dependencies: if dependency not in pipeline_name_to_id: raise Exception('Pipeline not found: %s.' % dependency) # Map from dependency id to name dependencies = {pipeline_name_to_id[dep]: dep for dep in dependencies} print 'Start checking for dependencies' start_time = datetime.now() failures = [] dependencies_ready = False # Loop until all dependent pipelines have finished or failed while not dependencies_ready: print 'checking' dependencies_ready, new_failures = check_dependencies_ready( dependencies, args.start_date, dependencies_to_ignore) failures.extend(new_failures) if not dependencies_ready: time.sleep(float(args.refresh_rate)) # Send message through SNS if there are failures if failures: if args.sns_topic_arn: message = 'Failed dependencies: %s.' % ', '.join(set(failures)) subject = 'Dependency error for pipeline: %s.' % args.pipeline_name SNSConnection().publish(args.sns_topic_arn, message, subject) else: raise Exception('ARN for SNS topic not specified in ETL config') print 'Finished checking for dependencies. Total time spent: ', print(datetime.now() - start_time).total_seconds(), ' seconds'
def main(): """ Main Function """ parser = argparse.ArgumentParser() parser.add_argument( '--dependencies', type=str, nargs='+', default=[]) parser.add_argument( '--dependencies_ok_to_fail', type=str, nargs='+', default=[]) parser.add_argument('--pipeline_name', dest='pipeline_name') parser.add_argument('--refresh_rate', dest='refresh_rate', default='900') parser.add_argument('--start_date', dest='start_date') parser.add_argument('--sns_topic_arn', dest="sns_topic_arn") args = parser.parse_args() # Exit if there are no dependencies if not args.dependencies and not args.dependencies_ok_to_fail: sys.exit() # Create mapping from pipeline name to id pipeline_name_to_id = dict( (pipeline['name'], pipeline['id']) for pipeline in list_pipelines() ) # Remove whitespace from dependency lists dependencies = map(str.strip, args.dependencies) dependencies_to_ignore = map(str.strip, args.dependencies_ok_to_fail) # Add the dependencies which can fail to the list of dependencies dependencies.extend(dependencies_to_ignore) # Check if all dependencies are valid pipelines for dependency in dependencies: if dependency not in pipeline_name_to_id: raise Exception('Pipeline not found: %s.' % dependency) # Map from dependency id to name dependencies = {pipeline_name_to_id[dep]: dep for dep in dependencies} print 'Start checking for dependencies' start_time = datetime.now() failures = [] dependencies_ready = False # Loop until all dependent pipelines have finished or failed while not dependencies_ready: print 'checking' time.sleep(float(args.refresh_rate)) dependencies_ready, new_failures = check_dependencies_ready(dependencies, args.start_date, dependencies_to_ignore) failures.extend(new_failures) # Send message through SNS if there are failures if failures: if args.sns_topic_arn: message = 'Failed dependencies: %s.' % ', '.join(set(failures)) subject = 'Dependency error for pipeline: %s.' % args.pipeline_name SNSConnection().publish(args.sns_topic_arn, message, subject) else: raise Exception('ARN for SNS topic not specified in ETL config') print 'Finished checking for dependencies. Total time spent: ', print (datetime.now() - start_time).total_seconds(), ' seconds'