def run_classifier_labels(hdfs_input_pos, hdfs_input_neg, hdfs_output, classifier_name, classifier_extra, local_labels, classifier, **kw):
labels = {}
try:
labels = file_parse.load(local_labels)
except IOError:
pass
hdfs_output_pos = hdfs_output + '/pos'
hdfs_output_neg = hdfs_output + '/neg'
hadoopy.launch_frozen(hdfs_input_pos, hdfs_output_pos, 'collect_keys.py')
hadoopy.launch_frozen(hdfs_input_neg, hdfs_output_neg, 'collect_keys.py')
pos_keys = sum((x[1] for x in hadoopy.readtb(hdfs_output_pos)), [])
neg_keys = sum((x[1] for x in hadoopy.readtb(hdfs_output_neg)), [])
labels[classifier_name] = {'labels': {'1': pos_keys, '-1': neg_keys},
'classifier': classifier,
'classifier_extra': classifier_extra}
file_parse.dump(labels, local_labels)
def __init__(self):
self._labels = file_parse.load(os.environ['LOCAL_LABELS_FN'])
def __init__(self):
labels = file_parse.load(os.environ['LOCAL_LABELS_FN'])
self._hash_class_labels = self._invert_labels(labels)
def __init__(self):
self._classifiers = [(x, classifiers.loads(y))
for x, y in file_parse.load(os.environ['CLASSIFIERS_FN'])]
