def split_train_test_highest(fname, codeprefixlen, mincodeoccurences, filtered_by):
#prepare generators
rec_generator = lambda:gen_record(fname, filtered_by)
prefixed_rec_generator = lambda:gen_record_prefixed(rec_generator, codeprefixlen)
prefix_code_generator = lambda:gen_lmc(prefixed_rec_generator)
#generate labels
PRINTER('generating labels...')
labels = get_labels_min_occurence(prefix_code_generator, mincodeoccurences)
PRINTER('labels generated:')
PRINTER(str(labels))
#gen filtered records:
labelsset = set(labels)
prefix_code_generator = lambda:gen_record_filteredbylabels(prefixed_rec_generator, labelsset)
PRINTER('counting elements...')
elements_count = len(list(prefix_code_generator()))
PRINTER('number of elements' +str(elements_count))
#split into training and testing samples
PRINTER('splitting into training and testing...')
train_inds, test_inds = randomly_divide(elements_count, int(elements_count / 10))
train_generator = lambda:gen_record_fromshifts(prefix_code_generator, train_inds)
test_generator = lambda:gen_record_fromshifts(prefix_code_generator, test_inds)
PRINTER('splitted.')
elements_count = len(list(prefix_code_generator()))
return train_generator, test_generator, elements_count, labels, elements_count
def load_labels_codegen_elemcnt(fname, codeprefixlen, mincodeoccurences, filtered_by):
#prepare generators
rec_generator = lambda:gen_record(fname, filtered_by)
prefixed_rec_generator = lambda:gen_record_prefixed(rec_generator, codeprefixlen)
prefix_code_generator = lambda:gen_lmc(prefixed_rec_generator)
#generate labels
PRINTER('generating labels...')
labels = get_labels_min_occurence(prefix_code_generator, mincodeoccurences)
PRINTER('labels generated:')
PRINTER(str(labels))
#gen filtered records:
labelsset = set(labels)
prefix_code_generator = lambda:gen_record_filteredbylabels(prefixed_rec_generator, labelsset)
PRINTER('counting elements...')
elements_count = len(list(prefix_code_generator()))
PRINTER('number of elements' +str(elements_count))
return labels, labelsset, prefix_code_generator, elements_count
def load_labels_codegen_elemcnt(fname, codeprefixlen, mincodeoccurences,
filtered_by):
#prepare generators
rec_generator = lambda: gen_record(fname, filtered_by)
prefixed_rec_generator = lambda: gen_record_prefixed(
rec_generator, codeprefixlen)
prefix_code_generator = lambda: gen_lmc(prefixed_rec_generator)
#generate labels
PRINTER('generating labels...')
labels = get_labels_min_occurence(prefix_code_generator, mincodeoccurences)
PRINTER('labels generated:')
PRINTER(str(labels))
#gen filtered records:
labelsset = set(labels)
prefix_code_generator = lambda: gen_record_filteredbylabels(
prefixed_rec_generator, labelsset)
PRINTER('counting elements...')
elements_count = len(list(prefix_code_generator()))
PRINTER('number of elements' + str(elements_count))
return labels, labelsset, prefix_code_generator, elements_count
print "mincodeoccurences", mincodeoccurences
print "k:", k
print "smoothingparam:", smoothingparam
print "distancetrainingsteps:", distancetrainingsteps
print "filtered_by:", filtered_by
print "save_hierarchical_path:", save_hierarchical_path
print "save_train_generator_path:", save_train_generator_path
print "save_lenlabels_path:", save_lenlabels_path
log_level = logging.INFO
logging.basicConfig(level=log_level)
#prepare generators
rec_generator = lambda: gen_record(fname, filtered_by)
prefixed_rec_generator = lambda: gen_record_prefixed(rec_generator, codeprefixlen)
prefix_code_generator = lambda: gen_lmc(prefixed_rec_generator)
#generate labels
print "generating labels..."
labels = get_labels_min_occurence(prefix_code_generator, mincodeoccurences)
labelsset = set(labels)
print "labels generated."
print labels
#gen filtered records:
prefix_code_generator = lambda: gen_record_filteredbylabels(prefixed_rec_generator, labelsset)
print "counting elements..."
elements_count = len(list(prefix_code_generator()))
print "number of elements:", elements_count
sys.exit(1)
try:
filtered_by = sys.argv[7:]
except:
print '7th argument: list of the fields to exist in considered records.'
sys.exit(1)
#prepare generators
rec_generator_first = lambda: gen_record(fname, filtered_by)
#choosing shuffling_cnt elements in random:
PRINTER("shuffling in random")
import random
chosen_records = random.sample(list(rec_generator_first()), shuffling_cnt)
rec_generator = lambda: chosen_records
prefixed_rec_generator = lambda: gen_record_prefixed(rec_generator, codeprefixlen)
prefix_code_generator = lambda: gen_lmc(prefixed_rec_generator)
#generate labels
PRINTER("generating labels...")
labels_counts = get_labels_counts(prefix_code_generator, mincodeoccurences)
#PRINTER("labels generated."
#PRINTER(sorted(labels_counts, key = lambda x: x[1], reverse = True)
biggest_labels = map(lambda x: x[0], sorted(labels_counts, key = lambda x: x[1],
reverse = True))[:biggest_labels_cnt]
labelsset = set(biggest_labels)
PRINTER(biggest_labels)
#gen filtered records:
prefix_code_generator = lambda: gen_record_filteredbylabels(prefixed_rec_generator, labelsset)
PRINTER("counting elements...")