def tf(query, doc_id):
doc = serializer.load('../data/processed/' + str(doc_id) + '.dat')
tf = np.zeros(len(query))
for i, word in enumerate(query):
tf[i] = doc.count(word)
tf /= len(doc)
return tf
def loadGame(self, save_name):
self.unloadMap()
self.world = serializer.load("saves/" + save_name + ".sav")
self.world.application = self
self.battle_controller = BattleController(self, self.world)
self.ai = AI(self, self.world)
self.createVisual()
print "* Game loaded!"
def main():
opts = read_opts()
if not opts.c_backend:
print "WARNING: training in pure python. Run with -c option to enable the (much faster) C++ backend"
feature_descriptors = faces.list_feature_descriptors((16, 16))
data = []
print "loading faces..."
faces.load_data_dir("Face16", 1, feature_descriptors, data, opts.num_faces, opts.c_backend)
faces.load_data_dir("Nonface16", -1, feature_descriptors, data, opts.num_other, opts.c_backend)
print "suffling..."
random.shuffle(data)
if opts.sample_size:
train_data = data[: opts.sample_size]
validation_data = data[opts.sample_size : opts.sample_size + opts.validate_size]
elif opts.validate_size:
train_data = []
validation_data = data[: opts.validate_size]
else:
train_data = data
validation_data = []
if opts.load_classifier:
with open(opts.load_classifier) as in_file:
classifier = serializer.load(in_file)
else:
print "training boosted classifier..."
if not train_data:
print "specify some training data with the -s flag."
exit(1)
classifier = boost.train_classifier(train_data, opts.num_iterations)
print classifier
if train_data:
print "training error:"
classify(classifier, train_data)
if validation_data:
print "validation error:"
if opts.each_iter:
classify_with_all_iterations(classifier, validation_data, opts.plot_iters)
else:
classify(classifier, validation_data)
if opts.plot_features:
plot_features(classifier, feature_descriptors)
if opts.search_image:
search.search(classifier, opts.search_image, feature_descriptors, opts.c_backend)
if opts.save_classifier:
with open(opts.save_classifier, "w") as out_file:
serializer.dump(classifier, out_file)
def main():
opts = read_opts()
if not opts.c_backend:
print "WARNING: training in pure python. Run with -c option to enable the (much faster) C++ backend"
feature_descriptors = faces.list_feature_descriptors((16,16))
data = []
print "loading faces..."
faces.load_data_dir('Face16', 1, feature_descriptors, data, opts.num_faces, opts.c_backend)
faces.load_data_dir('Nonface16', -1, feature_descriptors, data, opts.num_other, opts.c_backend)
print "suffling..."
random.shuffle(data)
if opts.sample_size:
train_data = data[:opts.sample_size]
validation_data = data[opts.sample_size : opts.sample_size + opts.validate_size]
elif opts.validate_size:
train_data = []
validation_data = data[:opts.validate_size]
else:
train_data = data
validation_data = []
if opts.load_classifier:
with open(opts.load_classifier) as in_file:
classifier = serializer.load(in_file)
else:
print "training boosted classifier..."
if not train_data:
print "specify some training data with the -s flag."
exit(1)
classifier = boost.train_classifier(train_data, opts.num_iterations)
print classifier
if train_data:
print "training error:"
classify(classifier, train_data)
if validation_data:
print "validation error:"
if opts.each_iter:
classify_with_all_iterations(classifier, validation_data, opts.plot_iters)
else:
classify(classifier, validation_data)
if opts.plot_features:
plot_features(classifier, feature_descriptors)
if opts.search_image:
search.search(classifier, opts.search_image, feature_descriptors, opts.c_backend)
if opts.save_classifier:
with open(opts.save_classifier, "w") as out_file:
serializer.dump(classifier, out_file)
def idf(self):
if self._idf is None:
data_dir = os.path.join(os.path.dirname(__file__), '../data/')
idf_filepath = os.path.join(data_dir, 'idf.dat')
logger.info('Loading IDF from {}'.format(idf_filepath))
self._idf = serializer.load(idf_filepath)
if self._idf is not None:
return self._idf
else:
raise ValueError('IDF is not define')
def upload_data():
global popular_words, distinct_words
popular_words, distinct_words = load('data/key_words.pck', False)
langauge_model.load()
print '# Loading data'
print ' * Popular words loaded.\n *\ttotal count :', len(popular_words)
print ' * Distinct words loaded.'
print ' *\tmostly not in spam:', len(distinct_words[0])
print ' *\tmostly in spam: ', len(distinct_words[1])
print ' * Langauge model loaded.\n *\ttotal n_grams :', len(langauge_model.struct_3)
popular_words = decode(popular_words)
distinct_words = [decode(distinct_words[0]), decode(distinct_words[1])]
def load_preprocessed_data(self):
data = {}
for counter, filepath in enumerate(os.listdir('../data/tmp'), 1):
logger.info('Part {} loading'.format(counter))
update = serializer.load('../data/tmp/' + filepath)
if not update:
logger.error(
'Part {} loading error. Tereminate process'.format(
counter))
break
for key, value in update.items():
serializer.save(value,
'../data/processed/' + str(key) + '.dat')
logger.info('Part {} loaded'.format(counter))
return data
def load(cls, path):
log.debug("Config loading class", cls.__name__)
if not os.path.isfile(path):
raise IOError("File not found")
obj = serializer.load(path)
# Secure the path.
utils.secure_file(const.ConfigFile)
# If we get to here, we have loaded the config.
# Now check for version upgrade
import appconfig
appconfig.check_version(obj)
return obj
def assert_vars(obj):
for language in serializer.get_formats():
loader = serializer.get_serializer(language)
read_mode = loader.read_mod
write_mode = loader.write_mod
name = f"test.{language}"
with open(name, write_mode) as fp:
serializer.dump(obj, fp, language)
with open(name, read_mode) as fp:
try:
restored = serializer.load(fp)
finally:
os.remove(name)
for var in vars(obj):
if var not in ("__dict__", "__weakref__", "__module__"):
assert getattr(obj, var) == getattr(restored, var)
def CreateIDF(self):
logger.info('Creating IDF')
data_dir = os.path.join(os.path.dirname(__file__), '../data/')
data_dir = os.path.join(data_dir, 'tmp')
data_list = os.listdir(data_dir)
if not data_list:
raise Exception('Data not preprocessed')
self._idf = Counter()
for cnt, filepath in enumerate(data_list, 1):
logger.info('Part {} processing'.format(cnt))
for word_list in serializer.load(os.path.join(data_dir,
filepath)).values():
self._idf.update(word_list)
idf_filepath = os.path.join(data_dir, 'idf.dat')
if os.path.isfile(idf_filepath):
ans = input(
'file {} alredy exist, replace it? (Y/N)'.format(idf_filepth))
if ans.lower() != 'y':
return
serializer.save(self._idf, idf_filepath)
def prepareLoadGame(self, save_name):
self.unloadMap()
self.world = serializer.load("saves/" + save_name + ".sav")
#self.real_timeline.addTimer(Timer("load game", 50, 1, self.loadGame, args=save_name))
self.gui.loading.showFade(lambda: self.loadGame(save_name),
map=self.world.current_map_name)
def loadCommands(self):
self.commands = serializer.load("saves/replaycommands.replay")
self.current_command = 0
def loadWorld(self):
return serializer.load("saves/replaystart.sav")
