class ExperimentInfo: def __init__(self, threads, block_size, path=None): self.threads = threads self.block_size = block_size self._path = path self._parser = Parser() def get_n_blocks(self, input_size): return np.ceil(input_size / self.block_size) def get_n_tasks(self, stage): task_amounts = self._get_ns_tasks(stage) assert len(task_amounts) == 1, 'More than 1 value: {}'.format( task_amounts) return task_amounts[0] def _get_ns_tasks(self, stage): amounts = set() for app in self._get_apps(): tasks = app.stages[stage].tasks n = sum(1 for t in tasks if not t.failed) amounts.add(len(app.stages[stage].tasks)) return list(amounts) def _get_apps(self): return self._parser.parse_folder(self._path)
def _build_all_tasks(self): parser = Parser() apps = parser.parse_folder(self._folder) first, nonfirst, columns = self._get_task_records(apps) return (pd.DataFrame.from_records(first, columns=columns), pd.DataFrame.from_records(nonfirst, columns=columns))
def _build(self): all_durs, all_sizes, all_workers = [], [], [] parser = Parser() apps = parser.parse_folder(self._folder) sizes = self._get_app_sizes() for app, size in zip(apps, sizes): duration = self._get_app_duration(app) all_durs.append(duration) all_sizes.append(size) all_workers.append(app.slaves) return _get_df(all_workers, all_sizes, all_durs)
def _build_first_nonfirst_tasks(self): durations = {'first': [], 'nonfirst': []} sizes = {'first': [], 'nonfirst': []} workers = {'first': [], 'nonfirst': []} parser = Parser() apps = parser.parse_folder(self._folder) apps_sizes = self._get_app_sizes() for app, size in zip(apps, apps_sizes): first, nonfirst = self._get_first_nonfirst_tasks_durations(app) durations['first'].extend(first) durations['nonfirst'].extend(nonfirst) sizes['first'].extend([size] * len(first)) sizes['nonfirst'].extend([size] * len(nonfirst)) workers['first'].extend([app.slaves] * len(first)) workers['nonfirst'].extend([app.slaves] * len(nonfirst)) first = _get_df(workers, sizes, durations, 'first') nonfirst = _get_df(workers, sizes, durations, 'nonfirst') return first, nonfirst