def run(args): check_project_root(args) # Init workflow cache cache = WorkflowCache(args) # validate infrabox.json data = load_infrabox_file(args.infrabox_file_path) if args.memory: logger.warn( 'WARNING: only int resource limits are supported right now. Using rounded int instead of provided value.' ) for job in data['jobs']: job['resources']['limits']['memory'] = int(args.memory) if args.cpu: logger.warn( 'WARNING: only int resource limits are supported right now. Using rounded int instead of provided value.' ) for job in data['jobs']: job['resources']['limits']['cpu'] = int(args.cpu) jobs = get_job_list(data, args, infrabox_context=args.project_root) if not args.job_name: # We run all jobs, so clear all cached jobs cache.clear() # Cache all jobs cache.add_jobs(jobs) for j in cache.get_jobs(job_name=args.job_name, children=args.children): build_and_run(args, j, cache)
def check_if_supported(job): if 'resources' not in job: return res_k8s = job['resources'].get('kubernetes', None) if res_k8s: logger.warn('Using kubernetes resources is not supported')
def build_and_run(args, job, cache): # check if dependency conditions are met for dep in job.get("depends_on", []): on = dep['on'] parent = get_parent_job(dep['job']) if not parent: continue if parent['state'] not in on: logger.info('Skipping job %s' % job['name']) track_as_parent(job, 'skipped') return job_type = job['type'] start_date = datetime.now() logger.info("Starting job %s" % job['name']) state = 'finished' try: if job_type == "docker-compose": build_and_run_docker_compose(args, job) elif job_type == "docker": build_and_run_docker(args, job) elif job_type == "docker-image": run_docker_image(args, job) elif job_type == "wait": # do nothing pass else: logger.error("Unknown job type") sys.exit(1) except Exception as e: state = 'failure' traceback.print_exc(file=sys.stdout) logger.warn("Job failed: %s" % e) sys.exit(1) if not job.get('directories', None): return # Dynamic child jobs infrabox_file = os.path.join(job['directories']['output'], 'infrabox.json') if not os.path.exists(infrabox_file): infrabox_file = os.path.join(job['directories']['output'], 'infrabox.yaml') jobs = [] if os.path.exists(infrabox_file): logger.info("Loading generated jobs") data = load_infrabox_file(infrabox_file) jobs = get_job_list(data, args, infrabox_context=os.path.join( args.project_root, '.infrabox', 'output')) end_date = datetime.now() track_as_parent(job, state, start_date, end_date) logger.info("Finished job %s" % job['name']) for j in jobs: # Prefix name with parent j['name'] = job['name'] + '/' + j['name'] # Add dependencies to all root jobs if not j.get('depends_on', None): j['depends_on'] = [{"on": ["finished"], "job": job['name']}] else: dependencies = copy.deepcopy(j['depends_on']) for d in dependencies: d['job'] = job['name'] + '/' + d['job'] j['depends_on'] = dependencies cache.add_job(j) if args.children: for j in jobs: build_and_run(args, j, cache)