def tf(query, doc_id): doc = serializer.load('../data/processed/' + str(doc_id) + '.dat') tf = np.zeros(len(query)) for i, word in enumerate(query): tf[i] = doc.count(word) tf /= len(doc) return tf
def loadGame(self, save_name): self.unloadMap() self.world = serializer.load("saves/" + save_name + ".sav") self.world.application = self self.battle_controller = BattleController(self, self.world) self.ai = AI(self, self.world) self.createVisual() print "* Game loaded!"
def main(): opts = read_opts() if not opts.c_backend: print "WARNING: training in pure python. Run with -c option to enable the (much faster) C++ backend" feature_descriptors = faces.list_feature_descriptors((16, 16)) data = [] print "loading faces..." faces.load_data_dir("Face16", 1, feature_descriptors, data, opts.num_faces, opts.c_backend) faces.load_data_dir("Nonface16", -1, feature_descriptors, data, opts.num_other, opts.c_backend) print "suffling..." random.shuffle(data) if opts.sample_size: train_data = data[: opts.sample_size] validation_data = data[opts.sample_size : opts.sample_size + opts.validate_size] elif opts.validate_size: train_data = [] validation_data = data[: opts.validate_size] else: train_data = data validation_data = [] if opts.load_classifier: with open(opts.load_classifier) as in_file: classifier = serializer.load(in_file) else: print "training boosted classifier..." if not train_data: print "specify some training data with the -s flag." exit(1) classifier = boost.train_classifier(train_data, opts.num_iterations) print classifier if train_data: print "training error:" classify(classifier, train_data) if validation_data: print "validation error:" if opts.each_iter: classify_with_all_iterations(classifier, validation_data, opts.plot_iters) else: classify(classifier, validation_data) if opts.plot_features: plot_features(classifier, feature_descriptors) if opts.search_image: search.search(classifier, opts.search_image, feature_descriptors, opts.c_backend) if opts.save_classifier: with open(opts.save_classifier, "w") as out_file: serializer.dump(classifier, out_file)
def main(): opts = read_opts() if not opts.c_backend: print "WARNING: training in pure python. Run with -c option to enable the (much faster) C++ backend" feature_descriptors = faces.list_feature_descriptors((16,16)) data = [] print "loading faces..." faces.load_data_dir('Face16', 1, feature_descriptors, data, opts.num_faces, opts.c_backend) faces.load_data_dir('Nonface16', -1, feature_descriptors, data, opts.num_other, opts.c_backend) print "suffling..." random.shuffle(data) if opts.sample_size: train_data = data[:opts.sample_size] validation_data = data[opts.sample_size : opts.sample_size + opts.validate_size] elif opts.validate_size: train_data = [] validation_data = data[:opts.validate_size] else: train_data = data validation_data = [] if opts.load_classifier: with open(opts.load_classifier) as in_file: classifier = serializer.load(in_file) else: print "training boosted classifier..." if not train_data: print "specify some training data with the -s flag." exit(1) classifier = boost.train_classifier(train_data, opts.num_iterations) print classifier if train_data: print "training error:" classify(classifier, train_data) if validation_data: print "validation error:" if opts.each_iter: classify_with_all_iterations(classifier, validation_data, opts.plot_iters) else: classify(classifier, validation_data) if opts.plot_features: plot_features(classifier, feature_descriptors) if opts.search_image: search.search(classifier, opts.search_image, feature_descriptors, opts.c_backend) if opts.save_classifier: with open(opts.save_classifier, "w") as out_file: serializer.dump(classifier, out_file)
def idf(self): if self._idf is None: data_dir = os.path.join(os.path.dirname(__file__), '../data/') idf_filepath = os.path.join(data_dir, 'idf.dat') logger.info('Loading IDF from {}'.format(idf_filepath)) self._idf = serializer.load(idf_filepath) if self._idf is not None: return self._idf else: raise ValueError('IDF is not define')
def upload_data(): global popular_words, distinct_words popular_words, distinct_words = load('data/key_words.pck', False) langauge_model.load() print '# Loading data' print ' * Popular words loaded.\n *\ttotal count :', len(popular_words) print ' * Distinct words loaded.' print ' *\tmostly not in spam:', len(distinct_words[0]) print ' *\tmostly in spam: ', len(distinct_words[1]) print ' * Langauge model loaded.\n *\ttotal n_grams :', len(langauge_model.struct_3) popular_words = decode(popular_words) distinct_words = [decode(distinct_words[0]), decode(distinct_words[1])]
def load_preprocessed_data(self): data = {} for counter, filepath in enumerate(os.listdir('../data/tmp'), 1): logger.info('Part {} loading'.format(counter)) update = serializer.load('../data/tmp/' + filepath) if not update: logger.error( 'Part {} loading error. Tereminate process'.format( counter)) break for key, value in update.items(): serializer.save(value, '../data/processed/' + str(key) + '.dat') logger.info('Part {} loaded'.format(counter)) return data
def load(cls, path): log.debug("Config loading class", cls.__name__) if not os.path.isfile(path): raise IOError("File not found") obj = serializer.load(path) # Secure the path. utils.secure_file(const.ConfigFile) # If we get to here, we have loaded the config. # Now check for version upgrade import appconfig appconfig.check_version(obj) return obj
def assert_vars(obj): for language in serializer.get_formats(): loader = serializer.get_serializer(language) read_mode = loader.read_mod write_mode = loader.write_mod name = f"test.{language}" with open(name, write_mode) as fp: serializer.dump(obj, fp, language) with open(name, read_mode) as fp: try: restored = serializer.load(fp) finally: os.remove(name) for var in vars(obj): if var not in ("__dict__", "__weakref__", "__module__"): assert getattr(obj, var) == getattr(restored, var)
def CreateIDF(self): logger.info('Creating IDF') data_dir = os.path.join(os.path.dirname(__file__), '../data/') data_dir = os.path.join(data_dir, 'tmp') data_list = os.listdir(data_dir) if not data_list: raise Exception('Data not preprocessed') self._idf = Counter() for cnt, filepath in enumerate(data_list, 1): logger.info('Part {} processing'.format(cnt)) for word_list in serializer.load(os.path.join(data_dir, filepath)).values(): self._idf.update(word_list) idf_filepath = os.path.join(data_dir, 'idf.dat') if os.path.isfile(idf_filepath): ans = input( 'file {} alredy exist, replace it? (Y/N)'.format(idf_filepth)) if ans.lower() != 'y': return serializer.save(self._idf, idf_filepath)
def prepareLoadGame(self, save_name): self.unloadMap() self.world = serializer.load("saves/" + save_name + ".sav") #self.real_timeline.addTimer(Timer("load game", 50, 1, self.loadGame, args=save_name)) self.gui.loading.showFade(lambda: self.loadGame(save_name), map=self.world.current_map_name)
def loadCommands(self): self.commands = serializer.load("saves/replaycommands.replay") self.current_command = 0
def loadWorld(self): return serializer.load("saves/replaystart.sav")