def run(self, args): store = UnitStore(args.config) stats = dict((w.label, [0, 0, 0]) for w in args.config.workflows) missing = [] for wflow in args.config.workflows: logger.info('validating output files for {0}'.format(wflow.label)) delete, missed = self.process_workflow(store, stats, wflow) missing += missed if not args.dry_run and len(delete) > 0: fs.remove(*delete) logger.info('finished validating') if sum(sum(stats.values(), [])) == 0: logger.info('no files found to cleanup') else: self.print_stats(stats) if len(missing) > 0: if not args.dry_run: store.update_missing(missing) verb = 'would have' if args.dry_run else 'have' template = 'the following {0} been marked as failed because their output could not be found: {1}' logger.warning(template.format(verb, ', '.join(map(str, missing))))
def run(self, args): store = UnitStore(args.config) stats = dict((w.label, [0, 0, 0]) for w in args.config.workflows) missing = [] for wflow in args.config.workflows: logger.info('validating output files for {0}'.format(wflow.label)) delete, merged, missed = self.process_workflow(store, stats, wflow) missing += missed if not args.dry_run: fs.remove(*delete) if args.delete_merged: fs.remove(*merged) logger.info('finished validating') if sum(sum(stats.values(), [])) == 0: logger.info('no files found to cleanup') else: self.print_stats(stats) if len(missing) > 0: if not args.dry_run: store.update_missing(missing) verb = 'would have' if args.dry_run else 'have' template = 'the following {0} been marked as failed because their output could not be found: {1}' logger.warning(template.format(verb, ', '.join(map(str, missing))))
def release(self, tasks): fail_cleanup = [] merge_cleanup = [] input_cleanup = [] update = defaultdict(list) propagate = defaultdict(dict) input_files = defaultdict(set) summary = ReleaseSummary() transfers = defaultdict(lambda: defaultdict(Counter)) with self.measure('dash'): self.config.advanced.dashboard.update_task_status( (task.tag, dash.DONE) for task in tasks) for task in tasks: with self.measure('updates'): handler = self.__taskhandlers[task.tag] failed, task_update, file_update, unit_update = handler.process( task, summary, transfers) wflow = getattr(self.config.workflows, handler.dataset) with self.measure('elk'): if self.config.elk: self.config.elk.index_task(task) self.config.elk.index_task_update(task_update) with self.measure('handler'): if failed: faildir = util.move(wflow.workdir, handler.id, 'failed') summary.dir(str(handler.id), faildir) fail_cleanup.extend([lf for rf, lf in handler.outputs]) else: util.move(wflow.workdir, handler.id, 'successful') merge = isinstance(handler, MergeTaskHandler) if (wflow.merge_size <= 0 or merge) and len(handler.outputs) > 0: outfn = handler.outputs[0][1] outinfo = handler.output_info for dep in wflow.dependents: propagate[dep.label][outfn] = outinfo if merge: merge_cleanup.extend(handler.input_files) if wflow.cleanup_input: input_files[handler.dataset].update( set([f for (_, _, f) in file_update])) update[(handler.dataset, handler.unit_source)].append( (task_update, file_update, unit_update)) del self.__taskhandlers[task.tag] with self.measure('dash'): self.config.advanced.dashboard.update_task_status( (task.tag, dash.RETRIEVED) for task in tasks) if len(update) > 0: with self.measure('sqlite'): logger.info(summary) self.__store.update_units(update) with self.measure('cleanup'): if len(input_files) > 0: input_cleanup.extend(self.__store.finished_files(input_files)) for cleanup in [fail_cleanup, merge_cleanup + input_cleanup]: if len(cleanup) > 0: try: fs.remove(*cleanup) except (IOError, OSError): pass except ValueError as e: logger.error("error removing {0}:\n{1}".format( task.tag, e)) with self.measure('propagate'): for label, infos in propagate.items(): unique_args = getattr(self.config.workflows, label).unique_arguments self.__store.register_files(infos, label, unique_args) if len(transfers) > 0: with self.measure('transfers'): self.__store.update_transfers(transfers) if self.config.elk: with self.measure('elk'): try: self.config.elk.index_summary( self.__store.workflow_status()) except Exception as e: logger.error('ELK failed to index summary:\n{}'.format(e))
def release(self, tasks): fail_cleanup = [] merge_cleanup = [] input_cleanup = [] update = defaultdict(list) propagate = defaultdict(dict) input_files = defaultdict(set) summary = ReleaseSummary() transfers = defaultdict(lambda: defaultdict(Counter)) with self.measure('dash'): self.config.advanced.dashboard.update_task_status( (task.tag, dash.DONE) for task in tasks ) for task in tasks: with self.measure('updates'): handler = self.__taskhandlers[task.tag] failed, task_update, file_update, unit_update = handler.process(task, summary, transfers) wflow = getattr(self.config.workflows, handler.dataset) with self.measure('elk'): if self.config.elk: self.config.elk.index_task(task) self.config.elk.index_task_update(task_update) with self.measure('handler'): if failed: faildir = util.move(wflow.workdir, handler.id, 'failed') summary.dir(str(handler.id), faildir) fail_cleanup.extend([lf for rf, lf in handler.outputs]) else: util.move(wflow.workdir, handler.id, 'successful') merge = isinstance(handler, MergeTaskHandler) if (wflow.merge_size <= 0 or merge) and len(handler.outputs) > 0: outfn = handler.outputs[0][1] outinfo = handler.output_info for dep in wflow.dependents: propagate[dep.label][outfn] = outinfo if merge: merge_cleanup.extend(handler.input_files) if wflow.cleanup_input: input_files[handler.dataset].update(set([f for (_, _, f) in file_update])) update[(handler.dataset, handler.unit_source)].append((task_update, file_update, unit_update)) del self.__taskhandlers[task.tag] with self.measure('dash'): self.config.advanced.dashboard.update_task_status( (task.tag, dash.RETRIEVED) for task in tasks ) if len(update) > 0: with self.measure('sqlite'): logger.info(summary) self.__store.update_units(update) with self.measure('cleanup'): if len(input_files) > 0: input_cleanup.extend(self.__store.finished_files(input_files)) for cleanup in [fail_cleanup, merge_cleanup + input_cleanup]: if len(cleanup) > 0: try: fs.remove(*cleanup) except (IOError, OSError): pass except ValueError as e: logger.error("error removing {0}:\n{1}".format(task.tag, e)) with self.measure('propagate'): for label, infos in propagate.items(): unique_args = getattr(self.config.workflows, label).unique_arguments self.__store.register_files(infos, label, unique_args) if len(transfers) > 0: with self.measure('transfers'): self.__store.update_transfers(transfers) if self.config.elk: with self.measure('elk'): try: self.config.elk.index_summary(self.__store.workflow_status()) except Exception as e: logger.error('ELK failed to index summary:\n{}'.format(e))