def from_dict(cls, dict_value: Dict) -> 'Job': """Create a `Job` from some dict read from JSON/YAML :param dict_value: a Dictionary :returns: A job instance based on data inside the dictionary. """ mounts = dict_value.get('mount', dict_value.get('mounts')) mounts = [ Volume(m['path'], m['name'], m['claim']) for m in listify(mounts) ] if mounts is not None else None secrets = dict_value.get('secret', dict_value.get('secrets')) secrets = [populate_secret(s) for s in listify(secrets)] if secrets is not None else None config_maps = dict_value.get('config_map', dict_value.get('config_maps')) config_maps = [populate_config_map(cm) for cm in listify(config_maps) ] if config_maps is not None else None return Task( dict_value['name'], dict_value['image'], dict_value['command'], dict_value.get('args', []), mounts, secrets, config_maps, dict_value.get('num_gpus', 0), dict_value.get('pull_policy', 'IfNotPresent'), dict_value.get('node_selector'), dict_value.get('resource_type', "Pod"), dict_value.get('num_workers', 1), dict_value.get('inputs'), dict_value.get('outputs'), )
def list_results(self, task, param_dict, user, metric, sort, event_type): if event_type is None or event_type == 'None': event_type = 'test_events' metrics = [x for x in listify(metric) if x.strip()] users = [x for x in listify(user) if x.strip()] if users: param_dict.update({'username': users}) coll = self.db[task] query = self._update_query({}, **param_dict) all_results = list(coll.find(query)) if not all_results: return BackendError(message='no information available for {} in task database [{}]' .format(param_dict, task)) experiments = mongo_to_experiment_set(task, all_results, event_type=event_type, metrics=metrics) if type(experiments) == BackendError: return experiments if sort is None or sort == 'None': return experiments else: if event_type == 'test_events': if sort in METRICS_SORT_ASCENDING: return experiments.sort(sort, reverse=False) else: return experiments.sort(sort) else: return BackendError(message='experiments can only be sorted when event_type=test_events')
def list_results(self, task, param_dict, user, metric, sort, event_type): session = self.Session() data_experiments = [] if event_type is None or event_type == 'None': event_type = 'test_events' metrics = [x for x in listify(metric) if x.strip()] users = [x for x in listify(user) if x.strip()] if not param_dict: hits = session.query(SqlExperiment).filter(SqlExperiment.task == task) else: hits = session.query(SqlExperiment).filter(SqlExperiment.task == task) for prop, value in param_dict.items(): hits = hits.filter(getattr(SqlExperiment, prop) == value) if users: hits = hits.filter(SqlExperiment.username.in_(users)) if hits.first() is None: return BackendError('No results in {} database for {}'.format(task, param_dict)) for exp in hits: data_experiment = self.sql_result_to_data_experiment(exp, event_type, metrics) if type(data_experiment) is BackendError: return data_experiment else: data_experiments.append(data_experiment) experiment_set = self.get_data_experiment_set(data_experiments) if sort is None or sort == 'None': return experiment_set else: if event_type == 'test_events': if sort in METRICS_SORT_ASCENDING: return experiment_set.sort(sort, reverse=False) else: return experiment_set.sort(sort) else: return BackendError(message='experiments can only be sorted when event_type=test_events')
def experiment_details(self, user, metric, sort, task, event_type, sha1, n): session = self.Session() results = [] metrics = listify(metric) users = listify(user) metrics_to_add = [metrics[0]] if len(metrics) == 1 else [] phase = self.event2phase(event_type) hits = session.query(Experiment).filter(Experiment.sha1 == sha1). \ filter(Experiment.task == task) for exp in hits: for event in exp.events: if event.phase == phase: result = [ exp.id, exp.username, exp.label, exp.dataset, exp.sha1, exp.date ] for m in self._get_filtered_metrics( event.metrics, metrics): result += [m.value] if m.label not in metrics_to_add: metrics_to_add += [m.label] results.append(result) cols = ['id', 'username', 'label', 'dataset', 'sha1', 'date' ] + metrics_to_add result_frame = pd.DataFrame(results, columns=cols) return df_experimental_details(result_frame, sha1, users, sort, metric, n)
def __init__(self, **kwargs): super().__init__(**kwargs) self.field = kwargs.get('fields', kwargs.get('field', 'text')) self.label = kwargs.get('label', 'label') self.emit_begin_toks = listify( kwargs.get('emit_begin_tok', [Offsets.VALUES[Offsets.PAD]])) self.emit_end_toks = listify( kwargs.get('emit_end_tok', [Offsets.VALUES[Offsets.PAD]]))
def experiment_details(self, user, metric, sort, task, event_type, sha1, n): metrics = listify(metric) coll = self.db[task] users = listify(user) query = self._update_query({}, username=users, sha1=sha1) projection = self._update_projection(event_type=event_type) result_frame = self._generate_data_frame(coll, metrics=metrics, query=query, projection=projection, event_type=event_type) return df_experimental_details(result_frame, sha1, users, sort, metric, n)
def __init__(self, **kwargs): super().__init__(**kwargs) self.field = kwargs.get('fields', kwargs.get('field')) self.primary_feature = kwargs.get('primary_feature', 'text') self.emit_begin_toks = listify( kwargs.get('emit_begin_tok', [Offsets.VALUES[Offsets.PAD]])) self.emit_end_toks = listify( kwargs.get('emit_end_tok', [Offsets.VALUES[Offsets.PAD]])) self.apply_all_subwords = kwargs.get('apply_all_subwords', True)
def __init__(self, **kwargs): super().__init__(kwargs.get('transform_fn')) self.max_seen = 128 self.tokenizer = WordpieceTokenizer( self.read_vocab(kwargs.get('vocab_file'))) self.mxlen = kwargs.get('mxlen', -1) self.dtype = kwargs.get('dtype', 'int') self._special_tokens = {"[CLS]", "<unk>", "<EOS>"} self.emit_begin_toks = listify(kwargs.get('emit_begin_tok', ['[CLS]'])) self.emit_end_toks = listify(kwargs.get('emit_end_tok', ['[SEP]']))
def create_graph( # pylint: disable=too-many-nested-blocks,too-many-branches task_list: List[Dict], external_inputs: Dict = {}) -> Graph: """Convert task list into a graph. There is a link between a chores if a chore references another in any of it's inputs or if the chore is listed in a key called `depends` (`depends` is used for forced control flow when not a explicit input dep) :param task_list: A list of dictionaries describing tasks :param external_inputs: A dictionary of external inputs :raises ValueError: If a task name contains a `.` or there is a phantom dependency :returns: A DAG """ graph: Graph = defaultdict(set) name2idx = { task.get('_name', task.get('name')): i for i, task in enumerate(task_list) } idx2name = {i: k for k, i in name2idx.items()} for name in name2idx: if '.' in name: raise ValueError(f"Names cannot contain `.` found {name} ") for dst, task in enumerate(task_list): if DEPENDENCY_KEY in task: for src in listify(task[DEPENDENCY_KEY]): src = src[1:] if src.startswith('^') else src if src not in name2idx: raise ValueError( f"Dependency `{src}` of node `{idx2name[dst]}` not found in graph." ) graph[name2idx[src]].add(dst) for values in task.values(): values = listify(values) for value in values: if is_reference(value): lookups = parse_reference(value) src = lookups[0] if src not in external_inputs: if src not in name2idx: raise ValueError( f"Dependency `{src}` of node `{idx2name[dst]}` not found in graph." ) graph[name2idx[src]].add(dst) else: LOGGER.info( "No dependency required in this graph from %s to %s", dst, src) for dst in range(len(task_list)): if dst not in graph: graph[dst] = set() return graph
def build_vocab(self, files, **kwargs): """Take a directory (as a string), or an array of files and build a vocabulary Take in a directory or an array of individual files (as a list). If the argument is a string, it may be a directory, in which case, all files in the directory will be loaded to form a vocabulary. :param files: Either a directory (str), or an array of individual files :return: """ vocab_file = kwargs.get('vocab_file') label_file = kwargs.get('label_file') if vocab_file is not None and label_file is not None: _vocab_allowed(self.vectorizers) vocab = _build_vocab_for_col(0, listify(vocab_file), self.vectorizers) labels = Counter(chain(*_read_from_col(0, listify(label_file)))) self.label2index = {l: i for i, l in enumerate(labels)} return vocab, self.get_labels() label_idx = len(self.label2index) if isinstance(files, six.string_types): if os.path.isdir(files): base = files files = filter( os.path.isfile, [os.path.join(base, x) for x in os.listdir(base)]) else: files = [files] vocab = {k: Counter() for k in self.vectorizers.keys()} for file_name in files: if file_name is None: continue with codecs.open(file_name, encoding='utf-8', mode='r') as f: for il, line in enumerate(f): label, text = TSVSeqLabelReader.label_and_sentence( line, self.clean_fn) if len(text) == 0: continue for k, vectorizer in self.vectorizers.items(): vocab_file = vectorizer.count(text) vocab[k].update(vocab_file) if label not in self.label2index: self.label2index[label] = label_idx label_idx += 1 vocab = _filter_vocab(vocab, kwargs.get('min_f', {})) return vocab, self.get_labels()
def __init__(self, **kwargs): """Loads a BPE tokenizer""" super().__init__(kwargs.get('transform_fn')) self.max_seen = 128 self.model_file = kwargs.get('model_file') self.vocab_file = kwargs.get('vocab_file') self.tokenizer = SavableFastBPE(self.model_file, self.vocab_file) self.mxlen = kwargs.get('mxlen', -1) self._vocab = { k: i for i, k in enumerate(self.read_vocab(self.vocab_file)) } self.emit_begin_toks = listify(kwargs.get('emit_begin_tok', [])) self.emit_end_toks = listify(kwargs.get('emit_end_tok', [])) self._special_tokens = {"[CLS]", "<unk>", "<EOS>"}
def build_vocab(self, files, **kwargs): """Take a directory (as a string), or an array of files and build a vocabulary Take in a directory or an array of individual files (as a list). If the argument is a string, it may be a directory, in which case, all files in the directory will be loaded to form a vocabulary. :param files: Either a directory (str), or an array of individual files :return: """ vocab_file = kwargs.get('vocab_file') label_file = kwargs.get('label_file') if vocab_file is not None and label_file is not None: _vocab_allowed(self.vectorizers) vocab = _build_vocab_for_col(0, listify(vocab_file), self.vectorizers) labels = Counter(chain(*_read_from_col(0, listify(label_file)))) self.label2index = {l: i for i, l in enumerate(labels)} return vocab, self.get_labels() label_idx = len(self.label2index) if isinstance(files, six.string_types): if os.path.isdir(files): base = files files = filter(os.path.isfile, [os.path.join(base, x) for x in os.listdir(base)]) else: files = [files] vocab = {k: Counter() for k in self.vectorizers.keys()} for file_name in files: if file_name is None: continue with codecs.open(file_name, encoding='utf-8', mode='r') as f: for il, line in enumerate(f): label, text = TSVSeqLabelReader.label_and_sentence(line, self.clean_fn) if len(text) == 0: continue for k, vectorizer in self.vectorizers.items(): vocab_file = vectorizer.count(text) vocab[k].update(vocab_file) if label not in self.label2index: self.label2index[label] = label_idx label_idx += 1 vocab = _filter_vocab(vocab, kwargs.get('min_f', {})) return vocab, self.get_labels()
def generate_export_task( # pylint: disable=too-many-locals template: Dict, odin_image: str, claim: str, models: List[str], task: str, dataset_name: str, depends: List[str], metric: str = 'acc', export_policy: Optional[str] = None, pull_policy: str = ALWAYS, ) -> Dict: """Generate a task (an element in the task list for the pipeline) that is a export job. :param template: The skeleton version of the export task. :param odin_image: The name of the odin docker image to use. :param claim: The name of the pvc to use. :param models: A list of models to evaluate. :param task: The name of the task that was used in training. :param dataset_name: The name of the dataset used to train on. :param depends: The tasks that need to be run before this one :param metric: The metric to use when comparing models. :param export_policy: How to make a decision if we should export things. :param pull_policy: Should k8s repull your containers. :returns: Dict, The task the represents a export job to run in the odin pipeline. """ if not export_policy: return {} depends = listify(depends) template = deepcopy(template) template['image'] = odin_image template['mounts'][0]['claim'] = claim template['pull_policy'] = pull_policy # For each model we trained add it to `args` just after `--models` models_idx = template['args'].index('--models') + 1 template['args'].pop(models_idx) for model in (f"${{PIPE_ID}}--{name}" for name in models): template['args'].insert(models_idx, model) template['args'][template['args'].index('--task') + 1] = task template['args'][template['args'].index('--type') + 1] = export_policy template['args'][template['args'].index('--metric') + 1] = metric template['depends'] = deepcopy(listify(depends)) return template
def stacked(self, pooled, init, **kwargs): """Stack 1 or more hidden layers, optionally (forming an MLP) :param pooled: The fixed representation of the model :param init: The tensorflow initializer :param kwargs: See below :Keyword Arguments: * *hsz* -- (``int``) The number of hidden units (defaults to `100`) :return: The final layer """ hszs = listify(kwargs.get('hsz', [])) if len(hszs) == 0: return pooled in_layer = pooled for i, hsz in enumerate(hszs): with tf.variable_scope('fc-{}'.format(i)): with tf.contrib.slim.arg_scope([fully_connected], weights_initializer=init): fc = fully_connected(in_layer, hsz, activation_fn=tf.nn.relu) in_layer = tf.nn.dropout(fc, self.pkeep) return in_layer
def build_vocab(self, files, **kwargs): label_idx = len(self.label2index) files = listify(files) vocab = {k: Counter() for k in self.vectorizers.keys()} for file_name in files: if file_name is None: continue with codecs.open(file_name + self.data, encoding='utf-8', mode='r') as data_file: with codecs.open(file_name + self.labels, encoding='utf-8', mode='r') as label_file: for d, l in zip(data_file, label_file): if d.strip() == "": continue label = l.rstrip() text = ParallelSeqLabelReader.get_sentence(d, self.clean_fn) if len(text) == 0: continue for k, vectorizer in self.vectorizers.items(): vocab_file = vectorizer.count(text) vocab[k].update(vocab_file) if label not in self.label2index: self.label2index[label] = label_idx label_idx += 1 vocab = _filter_vocab(vocab, kwargs.get('min_f', {})) return vocab, self.get_labels()
def register_model(cls, task, name=None): """Register a function as a plug-in""" if name is None: name = cls.__name__ names = listify(name) if task not in BASELINE_MODELS: BASELINE_MODELS[task] = {} if task not in BASELINE_LOADERS: BASELINE_LOADERS[task] = {} if hasattr(cls, 'create'): def create(*args, **kwargs): return cls.create(*args, **kwargs) else: def create(*args, **kwargs): return cls(*args, **kwargs) for alias in names: if alias in BASELINE_MODELS[task]: raise Exception('Error: attempt to re-define previously registered handler {} (old: {}, new: {}) for task {} in registry'.format(alias, BASELINE_MODELS[task], cls, task)) BASELINE_MODELS[task][alias] = create if hasattr(cls, 'load'): BASELINE_LOADERS[task][alias] = cls.load return cls
def get_results(self, task, dataset, event_type, num_exps=None, num_exps_per_config=None, metric=None, sort=None, id=None, label=None): session = self.Session() results = [] metrics = listify(metric) metrics_to_add = [metrics[0]] if len(metrics) == 1 else [] phase = self.event2phase(event_type) if id is not None: hit = session.query(Experiment).get(id) if hit is None: return None hits = [hit] elif label is not None: hits = session.query(Experiment).filter(Experiment.label == label). \ filter(Experiment.dataset == dataset). \ filter(Experiment.task == task) else: hits = session.query(Experiment).filter(Experiment.dataset == dataset). \ filter(Experiment.task == task) for exp in hits: for event in exp.events: if event.phase == phase: result = [exp.id, exp.username, exp.label, exp.dataset, exp.sha1, exp.date] for m in self._get_filtered_metrics(event.metrics, metrics): result += [m.value] if m.label not in metrics_to_add: metrics_to_add += [m.label] results.append(result) cols = ['id', 'username', 'label', 'dataset', 'sha1', 'date'] + metrics_to_add result_frame = pd.DataFrame(results, columns=cols) if not result_frame.empty: return df_get_results(result_frame, dataset, num_exps, num_exps_per_config, metric, sort) return None
def df_get_results(result_frame, dataset, num_exps, num_exps_per_config, metric, sort): datasets = result_frame.dataset.unique() if dataset not in datasets: return None dsr = result_frame[result_frame.dataset == dataset] if dsr.empty: return None df = pd.DataFrame() if num_exps_per_config is not None: for gname, rframe in result_frame.groupby("sha1"): rframe = rframe.copy() rframe['date'] =pd.to_datetime(rframe.date) rframe = rframe.sort_values(by='date', ascending=False).head(int(num_exps_per_config)) df = df.append(rframe) result_frame = df result_frame = result_frame.drop(["id"], axis=1) result_frame = result_frame.groupby("sha1").agg([len, np.mean, np.std, np.min, np.max])\ .rename(columns={'len': 'num_exps', 'amean': 'mean', 'amin': 'min', 'amax': 'max'}) metrics = listify(metric) if len(metrics) == 1: result_frame = result_frame.sort_values([(metrics[0], 'mean')], ascending=sort_ascending(metric)) if sort: result_frame = result_frame.sort_values([(sort, 'mean')], ascending=sort_ascending(metric)) if result_frame.empty: return None if num_exps is not None: result_frame = result_frame.head(num_exps) return result_frame
def task_summary(self, task, dataset, metric, event_type): metrics = listify(metric) coll = self.db[task] query = self._update_query({}, [], dataset) projection = self._update_projection(event_type=event_type) result_frame = self._generate_data_frame(coll, metrics, query, projection, event_type=event_type) if not result_frame.empty: datasets = result_frame.dataset.unique() if dataset not in datasets: return None dsr = result_frame[result_frame.dataset == dataset].sort_values( metric, ascending=False) result = dsr[metric].iloc[0] user = dsr.username.iloc[0] sha1 = dsr.sha1.iloc[0] date = dsr.date.iloc[0] summary = "For dataset {}, the best {} is {:0.3f} reported by {} on {}. " \ "The sha1 for the config file is {}.".format(dataset, metric, result, user, date, sha1) return summary return None
def get_results(self, task, param_dict, reduction_dim, metric, sort, numexp_reduction_dim, event_type): metrics = [x for x in listify(metric)] if event_type is None or event_type == 'None': event_type = 'test_events' reduction_dim = reduction_dim if reduction_dim is not None else 'sha1' coll = self.db[task] if 'dataset' in param_dict.keys(): value = self.get_related_datasets(task, param_dict['dataset']) param_dict['dataset'] = value query = self._update_query({}, **param_dict) all_results = list(coll.find(query)) if not all_results: return BackendError(message='no information available for {} in task database [{}]' .format(param_dict, task)) resultset = mongo_to_experiment_set(task, all_results, event_type=event_type, metrics=metrics) if type(resultset) is BackendError: return resultset experiment_aggregate_set = aggregate_results(resultset, reduction_dim, event_type, numexp_reduction_dim, param_dict) if sort is None or sort == 'None': return experiment_aggregate_set else: if event_type == 'test_events': if sort in METRICS_SORT_ASCENDING: return experiment_aggregate_set.sort(sort, reverse=False) else: return experiment_aggregate_set.sort(sort) else: return BackendError(message='experiments can only be sorted when event_type=test_events')
def build_vocab(self, files, **kwargs): label_idx = len(self.label2index) files = listify(files) vocab = {k: Counter() for k in self.vectorizers.keys()} for file_name in files: if file_name is None: continue with codecs.open(file_name + self.data, encoding='utf-8', mode='r') as data_file: with codecs.open(file_name + self.labels, encoding='utf-8', mode='r') as label_file: for d, l in zip(data_file, label_file): if d.strip() == "": continue label = l.rstrip() text = ParallelSeqLabelReader.get_sentence( d, self.clean_fn) if len(text) == 0: continue for k, vectorizer in self.vectorizers.items(): vocab_file = vectorizer.count(text) vocab[k].update(vocab_file) if label not in self.label2index: self.label2index[label] = label_idx label_idx += 1 vocab = _filter_vocab(vocab, kwargs.get('min_f', {})) return vocab, self.get_labels()
def generate_chore_task(template: Dict, odin_image: str, claim: str, depends: Optional[Union[str, List[str]]], pull_policy: str = ALWAYS) -> Dict: """Generate a task (an element in the task list in main.yaml) that is a chore job. :param template: The skeleton of a chore task to fill in. :param odin_image: The name of the odin docker image to use. :param claim: The name of the pvc to use. :param depends: Tasks that need to run before this one. :param pull_policy: Should k8s repull your containers. :returns: Dict, The task that will run a single chore file in an odin pipeline. """ template = deepcopy(template) template['image'] = odin_image template['mount']['claim'] = claim template['pull_policy'] = pull_policy if depends: template['depends'] = deepcopy(listify(depends)) return template
def build_vocab(self, files, **kwargs): if _all_predefined_vocabs(self.vectorizers): logger.info("Skipping building vocabulary. All vectorizers have predefined vocabs!") return {k: v.vocab for k, v in self.vectorizers.items()} vocab_file = kwargs.get('vocab_file') if vocab_file is not None: _vocab_allowed(self.vectorizers) return _build_vocab_for_col(0, listify(vocab_file), self.vectorizers) vocabs = {k: Counter() for k in self.vectorizers.keys()} for file in files: if file is None: continue with codecs.open(file, encoding='utf-8', mode='r') as f: sentences = [] for line in f: sentences += line.split() + ['<EOS>'] for k, vectorizer in self.vectorizers.items(): vocabs[k].update(vectorizer.count(sentences)) vocabs = _filter_vocab(vocabs, kwargs.get('min_f', {})) return vocabs
def get_results(self, task, dataset, event_type, num_exps=None, num_exps_per_config=None, metric=None, sort=None): session = self.Session() results = [] metrics = listify(metric) metrics_to_add = [metrics[0]] if len(metrics) == 1 else [] phase = self.event2phase(event_type) hits = session.query(Experiment).filter(Experiment.dataset == dataset). \ filter(Experiment.task == task) for exp in hits: for event in exp.events: if event.phase == phase: result = [ exp.id, exp.username, exp.label, exp.dataset, exp.sha1, exp.date ] for m in self._get_filtered_metrics( event.metrics, metrics): result += [m.value] if m.label not in metrics_to_add: metrics_to_add += [m.label] results.append(result) cols = ['id', 'username', 'label', 'dataset', 'sha1', 'date' ] + metrics_to_add result_frame = pd.DataFrame(results, columns=cols) if not result_frame.empty: return df_get_results(result_frame, dataset, num_exps, num_exps_per_config, metric, sort) return None
def _generate_configmaps(self, task: Task) -> Optional[List[ConfigMap]]: """Generate configmaps based on the requirements of the job. Eventually we can support custom configmaps by having the job create configmaps from the yaml config. Then this function will combine configmaps on the job with these injected configmaps to yield the final full list. :param task: The job we are running and want to add configmaps too. :type task: Task :returns: A list of configmaps or `None` :rtype: Optional[List[ConfigMap]] """ configmaps = task.config_maps if task.config_maps is not None else [] command = listify(task.command) if command[0].startswith('odin-chores'): try: # Check that the ssh-config configmap exists _ = self.core_api.read_namespaced_config_map( name='ssh-config', namespace=self.namespace) # Inject an ssh_config that will use the ssh key we inject with a secret ssh_config = ConfigMap('/etc/ssh/ssh_config', 'ssh-config', 'ssh_config') # Inject a known hosts file so it can find our gitlab server. known_hosts = ConfigMap('/etc/ssh/ssh_known_hosts', 'ssh-config', 'known_hosts') configmaps.extend((ssh_config, known_hosts)) except client.rest.ApiException: pass return configmaps if configmaps else None
def build_vocabs(self, files, **kwargs): vocab_file = kwargs.get('vocab_file') if vocab_file is not None: all_vects = self.src_vectorizers.copy() all_vects['tgt'] = self.tgt_vectorizer _vocab_allowed(all_vects) # Only read the file once. text = _read_from_col(0, listify(vocab_file)) src_vocab = _build_vocab_for_col(None, None, self.src_vectorizers, text=text) tgt_vocab = _build_vocab_for_col(None, None, {'tgt': self.tgt_vectorizer}, text=text) return src_vocab, tgt_vocab['tgt'] src_vocab = _build_vocab_for_col(0, [f + self.src_suffix for f in files], self.src_vectorizers) tgt_vocab = _build_vocab_for_col(0, [f + self.tgt_suffix for f in files], {'tgt': self.tgt_vectorizer}) min_f = kwargs.get('min_f', {}) tgt_min_f = {'tgt': min_f.pop('tgt', -1)} src_vocab = _filter_vocab(src_vocab, min_f) tgt_vocab = _filter_vocab(tgt_vocab, tgt_min_f) return src_vocab, tgt_vocab['tgt']
def register_model(cls, task, name=None): """Register a function as a plug-in""" if name is None: name = cls.__name__ names = listify(name) if task not in BASELINE_MODELS: BASELINE_MODELS[task] = {} if task not in BASELINE_LOADERS: BASELINE_LOADERS[task] = {} if hasattr(cls, 'create'): def create(*args, **kwargs): return cls.create(*args, **kwargs) else: def create(*args, **kwargs): return cls(*args, **kwargs) for alias in names: if alias in BASELINE_MODELS[task]: raise Exception( 'Error: attempt to re-define previously registered handler {} (old: {}, new: {}) for task {} in registry' .format(alias, BASELINE_MODELS[task], cls, task)) BASELINE_MODELS[task][alias] = create if hasattr(cls, 'load'): BASELINE_LOADERS[task][alias] = cls.load return cls
def fit(model, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('tagger', 'dynet', kwargs.get('basedir')) conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_metric')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) #validation_improvement_fn = kwargs.get('validation_improvement', None) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New max %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) trainer = create_trainer(model, **kwargs) trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test')
def stacked(self, pooled, init, **kwargs): """Stack 1 or more hidden layers, optionally (forming an MLP) :param pooled: The fixed representation of the model :param init: The tensorflow initializer :param kwargs: See below :Keyword Arguments: * *hsz* -- (``int``) The number of hidden units (defaults to `100`) :return: The final layer """ hszs = listify(kwargs.get('hsz', [])) if len(hszs) == 0: return pooled in_layer = pooled for i, hsz in enumerate(hszs): fc = tf.layers.dense(in_layer, hsz, activation=tf.nn.relu, kernel_initializer=init, name='fc-{}'.format(i)) in_layer = tf.layers.dropout(fc, self.pdrop_value, training=TRAIN_FLAG(), name='fc-dropout-{}'.format(i)) return in_layer
def fit(model, ts, vs, es, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs conll_file = kwargs.get('conll_file', None) txts = kwargs.get('txts', None) model_file = kwargs['outfile'] if 'outfile' in kwargs and kwargs[ 'outfile'] is not None else './tagger-model-tf' after_train_fn = kwargs[ 'after_train_fn'] if 'after_train_fn' in kwargs else None trainer = TaggerTrainerTf(model, **kwargs) init = tf.global_variables_initializer() model.sess.run(init) saver = tf.train.Saver() model.save_using(saver) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', basic_reporting)) print('reporting', reporting_fns) max_metric = 0 last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() model.save(model_file) elif test_metrics[early_stopping_metric] > max_metric: last_improved = epoch max_metric = test_metrics[early_stopping_metric] print('New max %.3f' % max_metric) trainer.checkpoint() model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on max_metric %.3f at epoch %d' % (max_metric, last_improved)) if es is not None: trainer.recover_last_checkpoint() evaluator = TaggerEvaluatorTf(model) test_metrics = evaluator.test(es, conll_file=conll_file, txts=txts) for reporting in reporting_fns: reporting(test_metrics, 0, 'Test')
def _configure_reporting(self, reporting, config_file, **kwargs): """Configure all `reporting_hooks` specified in the mead settings or overridden at the command line :param reporting: :param kwargs: :return: """ default_reporting = self.mead_settings_config.get( 'reporting_hooks', {}) # Add default reporting information to the reporting settings. for report_type in default_reporting: if report_type in reporting: for report_arg, report_val in default_reporting[ report_type].items(): if report_arg not in reporting[report_type]: reporting[report_type][report_arg] = report_val reporting_hooks = list(reporting.keys()) for settings in reporting.values(): for module in listify( settings.get('module', settings.get('modules', []))): import_user_module(module) self.reporting = baseline.create_reporting( reporting_hooks, reporting, { 'config_file': config_file, 'task': self.__class__.task_name(), 'base_dir': self.get_basedir() }) self.config_params['train']['reporting'] = [ x.step for x in self.reporting ] logging.basicConfig(level=logging.DEBUG)
def df_get_results(result_frame, dataset, num_exps, num_exps_per_config, metric, sort): datasets = result_frame.dataset.unique() if dataset not in datasets: return None dsr = result_frame[result_frame.dataset == dataset] if dsr.empty: return None df = pd.DataFrame() if num_exps_per_config is not None: for gname, rframe in result_frame.groupby("sha1"): rframe = rframe.copy() rframe['date'] = pd.to_datetime(rframe.date) rframe = rframe.sort_values(by='date', ascending=False).head( int(num_exps_per_config)) df = df.append(rframe) result_frame = df result_frame = result_frame.drop(["id"], axis=1) result_frame = result_frame.groupby("sha1").agg([len, np.mean, np.std, np.min, np.max])\ .rename(columns={'len': 'num_exps', 'amean': 'mean', 'amin': 'min', 'amax': 'max'}) metrics = listify(metric) if len(metrics) == 1: result_frame = result_frame.sort_values( [(metrics[0], 'mean')], ascending=sort_ascending(metric)) if sort: result_frame = result_frame.sort_values( [(sort, 'mean')], ascending=sort_ascending(metric)) if result_frame.empty: return None if num_exps is not None: result_frame = result_frame.head(num_exps) return result_frame
def build_vocabs(self, files, **kwargs): if _all_predefined_vocabs(self.src_vectorizers) and isinstance( self.tgt_vectorizer, HasPredefinedVocab): logger.info( "Skipping building vocabulary. All vectorizers have predefined vocabs!" ) return {k: v.vocab for k, v in self.src_vectorizers.items()}, { 'tgt': self.tgt_vectorizer.vocab } vocab_file = kwargs.get('vocab_file') if vocab_file is not None: all_vects = self.src_vectorizers.copy() all_vects['tgt'] = self.tgt_vectorizer _vocab_allowed(all_vects) # Only read the file once text = _read_from_col(0, listify(vocab_file)) src_vocab = _build_vocab_for_col(None, None, self.src_vectorizers, text=text) tgt_vocab = _build_vocab_for_col(None, None, {'tgt': self.tgt_vectorizer}, text=text) return src_vocab, tgt_vocab['tgt'] src_vocab = _build_vocab_for_col(self.src_col_num, files, self.src_vectorizers) tgt_vocab = _build_vocab_for_col(self.tgt_col_num, files, {'tgt': self.tgt_vectorizer}) min_f = kwargs.get('min_f', {}) tgt_min_f = {'tgt': min_f.pop('tgt', -1)} src_vocab = _filter_vocab(src_vocab, min_f) tgt_vocab = _filter_vocab(tgt_vocab, tgt_min_f) return src_vocab, tgt_vocab['tgt']
def fit(model, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file(kwargs, 'tagger', 'pytorch') conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', basic_reporting)) print('reporting', reporting_fns) #validation_improvement_fn = kwargs.get('validation_improvement', None) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(TaggerTrainerPyTorch, model, **kwargs) last_improved = 0 max_metric = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif test_metrics[early_stopping_metric] > max_metric: #if validation_improvement_fn is not None: # validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved) last_improved = epoch max_metric = test_metrics[early_stopping_metric] print('New max %.3f' % max_metric) model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on max_metric %.3f at epoch %d' % (max_metric, last_improved)) if es is not None: print('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(TaggerTrainerPyTorch, model, **kwargs) trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test')
def hash_args(command: Union[str, List[str]], args: List[str]) -> str: """Hash the command and arguments of a container. :param command: The command that k8s will give the container. :param args: The arguments the container receives. :returns: The hash of the container args. """ return sha1("".join(chain(listify(command), args)).encode('utf-8')).hexdigest()
def encode(self, embedseq, **kwargs): nlayers = kwargs.get('layers', 1) hsz = int(kwargs['hsz']) filts = kwargs.get('wfiltsz', None) if filts is None: filts = 5 cnnout = stacked_cnn(embedseq, hsz, self.pdrop_value, nlayers, filts=listify(filts), training=TRAIN_FLAG()) return cnnout
def fit(model, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir')) conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) #validation_improvement_fn = kwargs.get('validation_improvement', None) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): #if validation_improvement_fn is not None: # validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved) last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test') return test_metrics
def register_remote(cls, name=None): """Register a class as a plug-in""" if name is None: name = cls.__name__ names = listify(name) for alias in names: if alias in BASELINE_REMOTES: raise Exception('Error: attempt to re-define previously registered hander {} (old: {}, new: {}) in registry'.format(alias, BASELINE_REMOTES, cls)) BASELINE_REMOTES[alias] = cls return cls
def get_results(self, task, dataset, event_type, num_exps=None, num_exps_per_config=None, metric=None, sort=None, id=None, label=None): metrics = listify(metric) coll = self.db[task] query = self._update_query({}, dataset=dataset, id=id, label=label) projection = self._update_projection(event_type=event_type) result_frame = self._generate_data_frame(coll, metrics=metrics, query=query, projection=projection, event_type=event_type) if not result_frame.empty: return df_get_results(result_frame, dataset, num_exps, num_exps_per_config, metric, sort) return None
def experiment_details(self, user, metric, sort, task, event_type, sha1, n): session = self.Session() results = [] metrics = listify(metric) users = listify(user) metrics_to_add = [metrics[0]] if len(metrics) == 1 else [] phase = self.event2phase(event_type) hits = session.query(Experiment).filter(Experiment.sha1 == sha1). \ filter(Experiment.task == task) for exp in hits: for event in exp.events: if event.phase == phase: result = [exp.id, exp.username, exp.label, exp.dataset, exp.sha1, exp.date] for m in self._get_filtered_metrics(event.metrics, metrics): result += [m.value] if m.label not in metrics_to_add: metrics_to_add += [m.label] results.append(result) cols = ['id', 'username', 'label', 'dataset', 'sha1', 'date'] + metrics_to_add result_frame = pd.DataFrame(results, columns=cols) return df_experimental_details(result_frame, sha1, users, sort, metric, n)
def _stacked(self, pooled, **kwargs): pdrop = kwargs.get('dropout', 0.5) hszs = listify(kwargs.get('hsz', [])) activation_type = kwargs.get('activation', 'relu') if len(hszs) == 0: return pooled last_layer = pooled for i, hsz in enumerate(hszs): last_layer = Dense(units=hsz, activation=activation_type)(last_layer) last_layer = Dropout(rate=pdrop)(last_layer) return last_layer
def fit(model, ts, vs, es=None, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('seq2seq', 'pytorch', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'bleu') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: model.load(model_file) trainer = Seq2SeqTrainerPyTorch(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def init_stacked(self, input_dim, **kwargs): hszs = listify(kwargs.get('hsz', [])) if len(hszs) == 0: self.stacked_layers = None return input_dim self.stacked_layers = nn.Sequential() layers = [] in_layer_sz = input_dim for i, hsz in enumerate(hszs): layers.append(nn.Linear(in_layer_sz, hsz)) layers.append(nn.ReLU()) layers.append(nn.Dropout(self.pdrop)) in_layer_sz = hsz append2seq(self.stacked_layers, layers) return in_layer_sz
def fit(model, ts, vs, es=None, epochs=5, do_early_stopping=True, early_stopping_metric='bleu', **kwargs): patience = int(kwargs.get('patience', epochs)) after_train_fn = kwargs.get('after_train_fn', None) model_file = get_model_file('seq2seq', 'dy', kwargs.get('basedir')) trainer = create_trainer(model, **kwargs) best_metric = 0 if do_early_stopping: early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) logger.info("Doing early stopping on [%s] with patience [%d]", early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info("New best %.3f", best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info("Stopping due to persistent failures to improve") break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def fit(model, ts, vs, es, epochs=20, do_early_stopping=True, early_stopping_metric='acc', **kwargs): autobatchsz = kwargs.get('autobatchsz', 1) verbose = kwargs.get('verbose', {'print': kwargs.get('verbose_print', False), 'file': kwargs.get('verbose_file', None)}) model_file = get_model_file('classify', 'dynet', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: patience = kwargs.get('patience', epochs) early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics
def create_lr_scheduler(**kwargs): """Create a learning rate scheduler. :Keyword Arguments: * *lr_scheduler_type* `str` or `list` The name of the learning rate scheduler if list then the first scheduler should be a warmup scheduler. """ sched_type = kwargs.get('lr_scheduler_type') if sched_type is None: return None sched_type = listify(sched_type) if len(sched_type) == 2: warm = BASELINE_LR_SCHEDULERS.get(sched_type[0])(**kwargs) assert isinstance(warm, WarmupLearningRateScheduler), "First LR Scheduler must be a warmup scheduler." rest = BASELINE_LR_SCHEDULERS.get(sched_type[1])(**kwargs) return BASELINE_LR_SCHEDULERS.get('composite')(warm=warm, rest=rest, **kwargs) Constructor = BASELINE_LR_SCHEDULERS.get(sched_type[0]) lrs = Constructor(**kwargs) logger.info(lrs) return lrs
def init_stacked(self, input_dim, **kwargs): hszs = listify(kwargs.get('hsz', [])) if len(hszs) == 0: return input_dim, None stacked_layers = [] isz = input_dim for i, hsz in enumerate(hszs): stacked_layers.append(Linear(hsz, isz, self.pc)) stacked_layers.append(dy.rectify) stacked_layers.append(self.dropout) isz = hsz def call_stacked(input_): for layer in stacked_layers: input_ = layer(input_) return input_ return hsz, call_stacked
def build_vocab(self, files, **kwargs): vocab_file = kwargs.get('vocab_file') if vocab_file is not None: _vocab_allowed(self.vectorizers) return _build_vocab_for_col(0, listify(vocab_file), self.vectorizers) vocabs = {k: Counter() for k in self.vectorizers.keys()} for file in files: if file is None: continue with codecs.open(file, encoding='utf-8', mode='r') as f: sentences = [] for line in f: sentences += line.split() + ['<EOS>'] for k, vectorizer in self.vectorizers.items(): vocabs[k].update(vectorizer.count(sentences)) vocabs = _filter_vocab(vocabs, kwargs.get('min_f', {})) return vocabs
def fit(model, ts, vs, es=None, **kwargs): """ Train a classifier using Keras :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model-keras * *patience* (``int``) -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *optim* -- What optimizer to use. Defaults to `adam` :return: """ trainer = create_trainer(model, **kwargs) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'keras', kwargs.get('basedir')) if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) max_metric = 0 last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: model.save(model_file) elif test_metrics[early_stopping_metric] > max_metric: last_improved = epoch max_metric = test_metrics[early_stopping_metric] logger.info('New max %.3f', max_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on max_metric %.3f at epoch %d', max_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) trainer = ClassifyTrainerKeras(model, **kwargs) trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs model_file = get_model_file('lm', 'tf', kwargs.get('basedir')) after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) init = tf.global_variables_initializer() feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} model.sess.run(init, feed_dict) saver = tf.train.Saver() model.set_saver(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 1000 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def add_min_max(example, key, value): key = (key,) c = value.get('constraints', DEFAULT_CONSTRAINTS.get(key[-1], DEFAULT_CONSTRAINTS['default'])) example[key] = [listify(c), value['min'], value['max']]
def add_normal(example, key, value): key = (key,) c = value.get('constraints', DEFAULT_CONSTRAINTS.get(key[-1], DEFAULT_CONSTRAINTS['default'])) example[key] = [listify(c), value['mu'], value['sigma']]
def __init__(self, **kwargs): super(Dict2DVectorizer, self).__init__(**kwargs) self.fields = listify(kwargs.get('fields', 'text')) self.delim = kwargs.get('token_delim', '@@')
def fit(model, ts, vs, es, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} init = tf.global_variables_initializer() model.sess.run(init, feed_dict) saver = tf.train.Saver() model.save_using(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = bool(kwargs.get('verbose', False)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() # What to do about overloading this?? evaluator = TaggerEvaluatorTf(model, span_type, verbose) start = time.time() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = time.time() - start for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration}) return test_metrics
def fit(model, ts, vs, es=None, **kwargs): """ Train a classifier using TensorFlow :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * Additional arguments are supported, see :func:`baseline.tf.optimize` for full list :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) ema = True if kwargs.get('ema_decay') is not None else False best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} model.sess.run(tf.global_variables_initializer(), feed_dict) model.set_saver(tf.train.Saver()) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics
def train(model, ts, vs, es=None, **kwargs): """ Train a classifier using TensorFlow :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * Additional arguments are supported, see :func:`baseline.tf.optimize` for full list :return: """ n = int(kwargs.get('test_epochs', 5)) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) model.sess.run(tf.global_variables_initializer()) model.set_saver(tf.train.Saver()) max_metric = 0 last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if epoch > 0 and epoch % n == 0 and epoch < epochs - 1: print(color('Running test', Colors.GREEN)) trainer.test(es, reporting_fns, phase='Test') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif test_metrics[early_stopping_metric] > max_metric: last_improved = epoch max_metric = test_metrics[early_stopping_metric] print('New max %.3f' % max_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print(color('Stopping due to persistent failures to improve', Colors.RED)) break if do_early_stopping is True: print('Best performance on max_metric %.3f at epoch %d' % (max_metric, last_improved)) if es is not None: print(color('Reloading best checkpoint', Colors.GREEN)) trainer.recover_last_checkpoint() trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es, **kwargs): """ Train a classifier using PyTorch :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *optim* -- Optimizer to use, defaults to `sgd` * *eta, lr* (``float``) -- Learning rate, defaults to 0.01 * *mom* (``float``) -- Momentum (SGD only), defaults to 0.9 if optim is `sgd` :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'pytorch', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('eatly_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics