def run_classifier_labels(hdfs_input_pos, hdfs_input_neg, hdfs_output, classifier_name, classifier_extra, local_labels, classifier, **kw): labels = {} try: labels = file_parse.load(local_labels) except IOError: pass hdfs_output_pos = hdfs_output + '/pos' hdfs_output_neg = hdfs_output + '/neg' hadoopy.launch_frozen(hdfs_input_pos, hdfs_output_pos, 'collect_keys.py') hadoopy.launch_frozen(hdfs_input_neg, hdfs_output_neg, 'collect_keys.py') pos_keys = sum((x[1] for x in hadoopy.readtb(hdfs_output_pos)), []) neg_keys = sum((x[1] for x in hadoopy.readtb(hdfs_output_neg)), []) labels[classifier_name] = {'labels': {'1': pos_keys, '-1': neg_keys}, 'classifier': classifier, 'classifier_extra': classifier_extra} file_parse.dump(labels, local_labels)
def __init__(self): self._labels = file_parse.load(os.environ['LOCAL_LABELS_FN'])
def __init__(self): labels = file_parse.load(os.environ['LOCAL_LABELS_FN']) self._hash_class_labels = self._invert_labels(labels)
def __init__(self): self._classifiers = [(x, classifiers.loads(y)) for x, y in file_parse.load(os.environ['CLASSIFIERS_FN'])]