def __init__(self): super(OptionsScreen, self).__init__() self.font = utils.getFont('VolterGoldfish', 44) self.infofont = utils.getFont('VolterGoldfish', 18) self.keyfont = utils.getFont('VolterGoldfish', 24) self.background = pygame.image.load("../img/backgrounds/options.png").convert() self.soundBar = pygame.image.load("../img/backgrounds/soundBar.png").convert_alpha() self.musicBar = pygame.image.load("../img/backgrounds/soundBar.png").convert_alpha() self.keyImg = pygame.image.load("../img/backgrounds/key.png").convert_alpha() self.volumeTest = pygame.mixer.Sound("../sounds/volumeTest.wav") self.menuEntries = ["Music","Sound","Controls","Back"] self.menuPositions = [(200,100),(200,200),(200,300),(200,500)] self.menuChoice = 0 self.activeColor = THECOLORS["black"] self.inactiveColor = THECOLORS["grey64"] self.soundLevel = 0 if exist('soundLevel'): self.soundLevel = load('soundLevel') self.musicLevel = 0 if exist('musicLevel'): self.musicLevel = load('musicLevel') self.actionKeys = ['slow time','shield','trap','left','jump','right'] self.keys = ['w','x','c','<','^','>'] self.actionKeysPos = [(200,450),(330,450),(430,450),(530,450),(630,450),(730,450)]
def test(): B0 = utils.load('blk/0.pkl') B1 = utils.load('blk/1.pkl') B2 = utils.load('blk/4.pkl') B3 = utils.load('blk/5.pkl') BM = [[B0,B1],[B2,B3]] P = Block() P.merge_from_matrix(BM) utils.save(P, 'mb.pkl') print P
def prepare(self, block_path, imgs_path, camera_list_path, plane_path): self.block = utils.load(block_path) for i in self.block.cameras: self.imgs[i] = np.array(Image.open(imgs_path + str(i)+'.jpg')) self.camera_list = utils.load(camera_list_path) self.plane = utils.load(plane_path) self.h, self.w = self.block.height * self.scale, self.block.width*self.scale self.h, self.w = int(self.h), int(self.w) self.labels = cp.deepcopy(self.block.labels) self.img = zeros((self.h,self.w,3),uint8)
def test(): #camera_list = CameraList(10, (4912,7360)) #camera_list.read_from_bundle('bundle.out') #print camera_list #saveCameraList(camera_list, 'camera_list.pkl') C = utils.load('camera_list.pkl') print C
def make_tests(test_descr=TRANSFORMS_TESTINFO): """generate tests classes from test info return the list of generated test classes """ tests = [] for _transform, tr_input, tr_output, _normalize, _subobjects in test_descr: # load transform if necessary if type(_transform) is type(''): try: _transform = load(_transform).register() except MissingBinary: # we are not interessted in tests with missing binaries continue except: import traceback traceback.print_exc() continue if TR_NAMES is not None and not _transform.name() in TR_NAMES: print 'skip test for', _transform.name() continue class TransformTestSubclass(TransformTest): input = input_file_path(tr_input) output = output_file_path(tr_output) transform = _transform normalize = lambda x, y: _normalize(y) subobjects = _subobjects tests.append(TransformTestSubclass) tests.append(PILTransformsTest) return tests
def f_load(): try: return load(sess, saver, checkpoint_dir, name) except: print("Loading weights via Keras") G.load_weights(checkpoint_dir+"/G_weights.keras") D.load_weights(checkpoint_dir+"/D_weights.keras") E.load_weights(checkpoint_dir+"/E_weights.keras")
def codebook_to_csv(k=128, des_name=constants.ORB_FEAT_NAME): if not os.path.exists(constants.FILES_DIR_NAME): os.makedirs(constants.FILES_DIR_NAME) codebook = utils.load(filenames.codebook(k, des_name)) filename = "{0}/codebook_{1}_{2}.csv".format(constants.FILES_DIR_NAME, k, des_name) utils.save_csv(filename, codebook) print("Copied codebook into the file with name {0}. Press any key to exit...".format(filename)) cv2.waitKey()
def test_imageMatch(self): img_prot = load('orig/kueche.jpg') rgb_prot = img_prot.getdata() hsv_prot = rgb2hsv(rgb_prot) nc = 16 filt = Filter(nc) hc = clust1d([x[0] for x in hsv_prot], nc, 1000) add_palettes(img_prot, hc) img_prot.show() filt.hues = hc filt.update()
def train(self, svm_kernel, codebook, des_option=constants.ORB_FEAT_OPTION, is_interactive=True): """ Gets the descriptors for the training set and then calculates the SVM for them. Args: svm_kernel (constant): The kernel of the SVM that will be created. codebook (NumPy float matrix): Each row is a center of a codebook of Bag of Words approach. des_option (integer): The option of the feature that is going to be used as local descriptor. is_interactive (boolean): If it is the user can choose to load files or generate. Returns: cv2.SVM: The Support Vector Machine obtained in the training phase. """ des_name = constants.ORB_FEAT_NAME if des_option == constants.ORB_FEAT_OPTION else constants.SIFT_FEAT_NAME k = len(codebook) x_filename = filenames.vlads_train(k, des_name) if is_interactive: data_option = input("Enter [1] to calculate VLAD vectors for the training set or [2] to load them.\n") else: data_option = constants.GENERATE_OPTION if data_option == constants.GENERATE_OPTION: # Getting the global vectors for all of the training set print("Getting global descriptors for the training set.") start = time.time() x, y = self.get_data_and_labels(self.dataset.get_train_set(), codebook, des_option) utils.save(x_filename, x) end = time.time() print("VLADs training vectors saved on file {0}".format(x_filename)) self.log.train_vlad_time(end - start) else: # Loading the global vectors for all of the training set print("Loading global descriptors for the training set.") x = utils.load(x_filename) y = self.dataset.get_train_y() x = np.matrix(x, dtype=np.float32) svm = cv2.SVM() svm_filename = filenames.svm(k, des_name, svm_kernel) if is_interactive: svm_option = input("Enter [1] for generating a SVM or [2] to load one\n") else: svm_option = constants.GENERATE_OPTION if svm_option == constants.GENERATE_OPTION: # Calculating the Support Vector Machine for the training set print("Calculating the Support Vector Machine for the training set...") svm_params = dict(kernel_type=svm_kernel, svm_type=cv2.SVM_C_SVC, C=1) start = time.time() svm.train_auto(x, y, None, None, svm_params) end = time.time() self.log.svm_time(end - start) # Storing the SVM in a file svm.save(svm_filename) else: svm.load(svm_filename) return svm
def _process_path(self, subfile, current_encoding=None) : try: text = load(subfile) except (URLError, HTTPError, IOError) as e: print '[SubsLoader] "{0}" ,cannot load: {1}'.format(subfile, e) raise LoadError(subfile) try: decoded_list, encoding = decode(text, self._encodings, current_encoding) except Exception as e: print '[SubsLoader] "{0}" cannot decode: {1}'.format(subfile, e) raise DecodeError(subfile) return decoded_list, encoding
def loadOutput(self, name): """ Load output from dataset. Usage: JS = loadOutput(img_name) Input: img - Full path of image to load. It's useful to use something like: img_name = getName (basename, 'img') Output: JS - JS structure for this image """ return utils.load(name, 'JS')
def loadSummary (self, name): """ Load scene summary from dataset. Usage: mask = loadSummary(name) Input: name - Full path of Summary to load. It's useful to use something like: mask_name = getName (basename, 'mask') Output: JS - JS structure containing only a summary of regions in JS.CompactReg. """ return utils.load(name, 'JS')
def loadSeq (self, name): """ Load data Sequence containing info about a subset of images. Usage: seq = loadSeq(name) Input: name - Full path of sequence to load. It's useful to use something like: name = getName (basename, 'seq') Output: seq - RGBDSequence() object containing a list of files and their poses. """ return utils.load(name, 'seq')
def load(self, name = None, path = None): """ load data from mask, and update name/path if necessary. """ if name is not None: self.name = name if path is not None: self.path = path # BaseMask assumes we're reading masks from an image type file = os.path.join(self.path, self.name) if os.path.exists(file): self.data = utils.load(file, 'mask', Matlab_compatible = True)
def load_output(self, count): """ Load particle data from dumped output file. Parameters ---------- count : string The iteration time from which to load the data. If time is '?' then list of available data files is returned else the latest available data file is used Notes ----- Data is loaded from the :py:attr:`output_directory` using the same format as stored by the :py:meth:`dump_output` method. Proper functioning required that all the relevant properties of arrays be dumped """ # get the list of available files available_files = [i.rsplit('_',1)[1][:-4] for i in os.listdir(self.output_directory) if i.startswith(self.fname) and i.endswith('.npz')] if count == '?': return sorted(set(available_files), key=int) else: if not count in available_files: msg = """File with iteration count `%s` does not exist"""%(count) msg += "\nValid iteration counts are %s"%(sorted(set(available_files), key=int)) #print msg raise IOError(msg) array_names = [pa.name for pa in self.particles.arrays] # load the output file data = load(os.path.join(self.output_directory, self.fname+'_'+str(count)+'.npz')) arrays = [ data["arrays"][i] for i in array_names ] # set the Particle's arrays self.particles.arrays = arrays # call the particle's initialize self.particles.initialize() self.t = float(data["solver_data"]['t']) self.count = int(data["solver_data"]['count'])
def __init__(self, levelInd): super(GameScreen, self).__init__() self.fadeScreen = pygame.image.load("../img/backgrounds/blackscreen.png").convert() self.font = getFont("VolterGoldfish", 20) self.nextColorIcon = pygame.image.load("../img/hud/nextColor23.png").convert_alpha() self.progressIcon = pygame.image.load("../img/hud/progress.png").convert_alpha() self.timebar = pygame.image.load("../img/hud/timebar.png").convert_alpha() self.shieldbar = pygame.image.load("../img/hud/shieldbar.png").convert_alpha() self.lifebar_ = pygame.image.load("../img/hud/lifebar_.png").convert() self.level = Level(levelInd) self.camera = Camera(640, 800, 3200) # running = True self.retry = False self.frame_number = 0 self.anims = [] self.stopDelay = 0 self.stopped = False self.killFragments = [] self.dust = [] self.starting = True self.ending = False self.alpha = 255 # Music load # pygame.mixer.music.load("../sounds/piano.wav") # pygame.mixer.music.play(-1) pygame.mixer.music.set_volume(0.5) # Player self.player = Player() self.fire = Rect((0,3200),(800,5)) self.playerDown = False self.answerPos = [(100,300),(100,400),(100,500)] self.answerChoice = 0 self.activecolor = THECOLORS['white'] self.inactivecolor = THECOLORS['grey21'] if exist('powers'): self.player.timePower, self.player.mines, self.player.shields = load('powers')
def extrapolation(what, **kwargs): what = 'extrap_'+what fname = utils.filename(what, **kwargs) if path.exists(fname): return utils.load(fname) else: import pairfield_correlations, density_correlations, density, energy if what == 'extrap_pairfield': return utils.load_or_evaluate(what, pairfield_correlations.evaluate_extrap_at, **kwargs) elif what == 'extrap_densdens': return utils.load_or_evaluate(what, density_correlations.evaluate_extrap_at, **kwargs) elif what == 'extrap_density': return utils.load_or_evaluate(what, density.evaluate_extrap_at, **kwargs) elif what == 'extrap_energy': return utils.load_or_evaluate(what, energy.evaluate_extrap, **kwargs) else: raise Exception('`%s` is not a valid measurement.' % what)
def _process_path(self, subfile, current_encoding=None) : filename = os.path.basename(subfile) size = getFileSize(subfile) if size and size > SUBTITLES_FILE_MAX_SIZE: self.log.error("<%s> not supported subtitles size ({%d}KB > {%d}KB)!", filename, size / 1024, SUBTITLES_FILE_MAX_SIZE / 1024) raise LoadError('"%s" - not supported subtitles size: "%dKB"' % (toString(os.path.basename(subfile)), size / 1024)) try: text = load(subfile) except (URLError, HTTPError, IOError) as e: self.log.error("<%s> %s", filename, str(e)) raise LoadError(subfile) try: decoded_text, encoding = decode(text, self._encodings, current_encoding) except Exception as e: self.log.error("<%s> %s", filename, "cannot decode") raise DecodeError(subfile) return decoded_text, encoding
def test(self, codebook, svm, des_option = constants.ORB_FEAT_OPTION, is_interactive=True): """ Gets the descriptors for the testing set and use the svm given as a parameter to predict all the elements Args: codebook (NumPy matrix): Each row is a center of a codebook of Bag of Words approach. svm (cv2.SVM): The Support Vector Machine obtained in the training phase. des_option (integer): The option of the feature that is going to be used as local descriptor. is_interactive (boolean): If it is the user can choose to load files or generate. Returns: NumPy float array: The result of the predictions made. NumPy float array: The real labels for the testing set. """ des_name = constants.ORB_FEAT_NAME if des_option == constants.ORB_FEAT_OPTION else constants.SIFT_FEAT_NAME k = len(codebook) x_filename = filenames.vlads_test(k, des_name) if is_interactive: data_option = input("Enter [1] to calculate VLAD vectors for the testing set or [2] to load them.\n") else: data_option = constants.GENERATE_OPTION if data_option == constants.GENERATE_OPTION: # Getting the global vectors for all of the testing set print("Getting global descriptors for the testing set...") start = time.time() x, y = self.get_data_and_labels(self.dataset.get_test_set(), codebook, des_option) utils.save(x_filename, x) end = time.time() print("VLADs testing vectors saved on file {0}".format(x_filename)) self.log.test_vlad_time(end - start) else: # Loading the global vectors for all of the testing set print("Loading global descriptors for the testing set.") x = utils.load(x_filename.format(des_name)) y = self.dataset.get_test_y() x = np.matrix(x, dtype=np.float32) # Predicting the testing set using the SVM start = time.time() result = svm.predict_all(x) end = time.time() self.log.predict_time(end - start) mask = result == y correct = np.count_nonzero(mask) accuracy = (correct * 100.0 / result.size) self.log.accuracy(accuracy) return result, y
def load(): import config try: data = utils.load(cfgfilename()) except: a = ConfigWindow() a.exec_loop() return lines = data.split('\n') for line in lines: pair = line.split('=') if pair: if pair[0] == "name": config.name = pair[1] elif pair[0] == "email": config.email = pair[1] elif pair[0] == "pspec_folder": config.pspec_folder = pair[1]
def result(what, **kwargs): fname = utils.filename(what, **kwargs) if path.exists(fname): return utils.load(fname) else: try: import pyalps_dset import pairfield_correlations, density_correlations, density, amplitudes if what == 'pairfield': return utils.load_or_evaluate(what, pairfield_correlations.evaluate, **kwargs) elif what == 'densdens': return utils.load_or_evaluate(what, density_correlations.evaluate, **kwargs) elif what == 'density': return utils.load_or_evaluate(what, density.evaluate, **kwargs) elif what == 'density_fit': return utils.load_or_evaluate(what, density.evaluate_fit, **kwargs) elif what == 'density_amplitudes': return utils.load_or_evaluate(what, amplitudes.evaluate, **kwargs) else: raise Exception('`%s` is not a valid measurement.' % what) except ImportError, e: print 'To execute new evaluations you need the ALPS.Python library.' print e
def run(self, restore=False): g.init() for event in pygame.event.get(): if event.type == pygame.QUIT: return elif event.type == pygame.VIDEORESIZE: pygame.display.set_mode(event.size, pygame.RESIZABLE) break if not self.journal: utils.load() self.trip = trip.Trip(self.sugar, self.label, self.colors[0]) self.trip.setup() load_save.retrieve() self.buttons_setup() if self.canvas is not None: self.canvas.grab_focus() ctrl = False pygame.key.set_repeat(600, 120) key_ms = pygame.time.get_ticks() going = True while going: if self.journal: # Pump GTK messages. while Gtk.events_pending(): Gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: # only in standalone version if not self.journal: utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.pos = event.pos self.trip.check_mouse() g.redraw = True if self.canvas is not None: self.canvas.grab_focus() elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True if event.button == 1: bu = buttons.check() if bu != '': self.do_button(bu) else: if self.trip.check_mouse(): self.do_click() self.flush_queue() elif event.type == pygame.KEYDOWN: # throttle keyboard repeat if pygame.time.get_ticks() - key_ms > 110: key_ms = pygame.time.get_ticks() if ctrl: if event.key == pygame.K_q: if not self.journal: utils.save() going = False break else: ctrl = False if event.key in (pygame.K_LCTRL, pygame.K_RCTRL): ctrl = True break self.do_key(event.key) g.redraw = True self.flush_queue() elif event.type == pygame.KEYUP: ctrl = False if not going: break if g.redraw: self.display() if g.version_display: utils.version_display() g.screen.blit(g.pointer, g.pos) pygame.display.flip() g.redraw = False if self.trip.delay: pygame.time.delay(1000) self.trip.delay = False self.trip.gone() g.redraw = True self.trip.complete() g.clock.tick(40)
# c_svm_param # knn # naive_bayes # ols # ridge_reg # lasso # adaboost # voting # random_forest # bow start = time.time() print time.ctime() X_tr, X_te, y_tr, y_te, sorted_files_tr, sorted_files_te = \ load(load_from_folder, display_collage, submission) N_tr = X_tr.shape[0] N_te = X_te.shape[0] D = X_tr.shape[1] print "Training : [Inputs x Features ] = [%d x %d]" % (N_tr,D) print "Test : [Inputs x Features ] = [%d x %d]" % (N_te,D) ####################### Feature Expansion ################################ if classifier!="nn" and classifier!="bow": X_tr = feature_exp(X_tr) X_te = feature_exp(X_te) D = X_tr.shape[1] print "After Feature Expansion: Training : [Inputs x Features ] = [%d x %d]" % (N_tr,D) print "After Feature Expansion: Test : [Inputs x Features ] = [%d x %d]" % (N_te,D)
""" script_dir = Path(os.path.abspath(__file__)).parent data_dir = script_dir.parent / "data" models_path = data_dir / "models" if not models_path.exists(): print("please download models to %s." % models_path) sys.exit(1) # Load all models into a list loaded = load([ (NetPredictor, "v3/sep/1"), (NetPredictor, "v3/sep/2"), (NetPredictor, "v3/sep/4"), (NetPredictor, "v3/blkx/2"), (NetPredictor, "v3/blkx/4"), (NetPredictor, "v3/blkx/5"), ], models_path=models_path) sep_predictor = VotingPredictor(loaded["v3/sep/1"], loaded["v3/sep/2"], loaded["v3/sep/4"]) blkx_predictor = VotingPredictor(loaded["v3/blkx/2"], loaded["v3/blkx/4"], loaded["v3/blkx/5"]) for page_path in (data_dir / "pages").iterdir(): if page_path.suffix == ".jpg": page = Page.Page(page_path) sep_res = sep_predictor(page)
def main(): def _str_to_bool(s): """Convert string to bool (in argparse context).""" if s.lower() not in ['true', 'false']: raise ValueError('Argument needs to be a boolean, got {}'.format(s)) return {'true': True, 'false': False}[s.lower()] parser = argparse.ArgumentParser(description='WaveNet example network') DATA_DIRECTORY = '.\\data\\moon,.\\data\\son' parser.add_argument('--data_dir', type=str, default=DATA_DIRECTORY, help='The directory containing the VCTK corpus.') LOGDIR = None #LOGDIR = './/logdir-wavenet//train//2018-12-21T22-58-10' parser.add_argument('--logdir', type=str, default=LOGDIR,help='Directory in which to store the logging information for TensorBoard. If the model already exists, it will restore the state and will continue training. Cannot use with --logdir_root and --restore_from.') parser.add_argument('--logdir_root', type=str, default=None,help='Root directory to place the logging output and generated model. These are stored under the dated subdirectory of --logdir_root. Cannot use with --logdir.') parser.add_argument('--restore_from', type=str, default=None,help='Directory in which to restore the model from. This creates the new model under the dated directory in --logdir_root. Cannot use with --logdir.') CHECKPOINT_EVERY = 1000 # checkpoint 저장 주기 parser.add_argument('--checkpoint_every', type=int, default=CHECKPOINT_EVERY,help='How many steps to save each checkpoint after. Default: ' + str(CHECKPOINT_EVERY) + '.') config = parser.parse_args() # command 창에서 입력받을 수 있는 조건 config.data_dir = config.data_dir.split(",") try: directories = validate_directories(config,hparams) except ValueError as e: print("Some arguments are wrong:") print(str(e)) return logdir = directories['logdir'] restore_from = directories['restore_from'] # Even if we restored the model, we will treat it as new training # if the trained model is written into an arbitrary location. is_overwritten_training = logdir != restore_from log_path = os.path.join(logdir, 'train.log') infolog.init(log_path, logdir) global_step = tf.Variable(0, name='global_step', trainable=False) # Create coordinator. coord = tf.train.Coordinator() num_speakers = len(config.data_dir) # Load raw waveform from VCTK corpus. with tf.name_scope('create_inputs'): # Allow silence trimming to be skipped by specifying a threshold near # zero. silence_threshold = hparams.silence_threshold if hparams.silence_threshold > EPSILON else None gc_enable = num_speakers > 1 # AudioReader에서 wav 파일을 잘라 input값을 만든다. receptive_field길이만큼을 앞부분에 pad하거나 앞조각에서 가져온다. (receptive_field+ sample_size)크기로 자른다. reader = DataFeederWavenet(coord,config.data_dir,batch_size=hparams.wavenet_batch_size,receptive_field=WaveNetModel.calculate_receptive_field(hparams.filter_width, hparams.dilations,hparams.scalar_input, hparams.initial_filter_width), gc_enable= gc_enable) if gc_enable: audio_batch, lc_batch, gc_id_batch = reader.inputs_wav, reader.local_condition, reader.speaker_id else: audio_batch, lc_batch = reader.inputs_wav, self.local_condition # Create network. net = WaveNetModel( batch_size=hparams.wavenet_batch_size, dilations=hparams.dilations, filter_width=hparams.filter_width, residual_channels=hparams.residual_channels, dilation_channels=hparams.dilation_channels, quantization_channels=hparams.quantization_channels, out_channels =hparams.out_channels, skip_channels=hparams.skip_channels, use_biases=hparams.use_biases, # True scalar_input=hparams.scalar_input, initial_filter_width=hparams.initial_filter_width, global_condition_channels=hparams.gc_channels, global_condition_cardinality=num_speakers, local_condition_channels=hparams.num_mels, upsample_factor=hparams.upsample_factor, train_mode=True) if hparams.l2_regularization_strength == 0: hparams.l2_regularization_strength = None net.add_loss(input_batch=audio_batch,local_condition=lc_batch, global_condition_batch=gc_id_batch, l2_regularization_strength=hparams.l2_regularization_strength) net.add_optimizer(hparams,global_step) run_metadata = tf.RunMetadata() # Set up session sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) # log_device_placement=False --> cpu/gpu 자동 배치. init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=hparams.max_checkpoints) # 최대 checkpoint 저장 갯수 지정 try: start_step = load(saver, sess, restore_from) # checkpoint load if is_overwritten_training or start_step is None: # The first training step will be saved_global_step + 1, # therefore we put -1 here for new or overwritten trainings. zero_step_assign = tf.assign(global_step, 0) sess.run(zero_step_assign) except: print("Something went wrong while restoring checkpoint. We will terminate training to avoid accidentally overwriting the previous model.") raise ########### start_step = sess.run(global_step) last_saved_step = start_step try: reader.start_in_session(sess,start_step) while not coord.should_stop(): start_time = time.time() if hparams.store_metadata and step % 50 == 0: # Slow run that stores extra information for debugging. log('Storing metadata') run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) step, loss_value, _ = sess.run([global_step, net.loss, net.optimize],options=run_options,run_metadata=run_metadata) tl = timeline.Timeline(run_metadata.step_stats) timeline_path = os.path.join(logdir, 'timeline.trace') with open(timeline_path, 'w') as f: f.write(tl.generate_chrome_trace_format(show_memory=True)) else: step, loss_value, _ = sess.run([global_step,net.loss, net.optimize]) duration = time.time() - start_time log('step {:d} - loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration)) if step % config.checkpoint_every == 0: save(saver, sess, logdir, step) last_saved_step = step if step >= hparams.num_steps: # error message가 나오지만, 여기서 멈춘 것은 맞다. raise Exception('End xxx~~~yyy') except Exception as e: print('finally') #if step > last_saved_step: # save(saver, sess, logdir, step) coord.request_stop(e)
# START !!! {} PID: {} #============================================================================== """.format(__file__, os.getpid())) import pandas as pd import utils from nltk import stem pt = stem.PorterStemmer().stem from gensim.models import Doc2Vec d2v = Doc2Vec.load('../nlp_source/d2v/enwiki_dbow/doc2vec.bin') from gensim.models import KeyedVectors w2v = KeyedVectors.load_word2vec_format('../nlp_source/w2v/GoogleNews-vectors-negative300.bin.gz', binary=True) train, test = utils.load(4,1) stop_verbs = {'do','does','did','done','is','am','are','be','been', 'being','was','were',"'s","'m"} """ nltk.pos_tag(nltk.word_tokenize('How can I be a good geologist ?')) s='How do I use Twitter as a business source ? ' s='What is the best way to gain confidence ? ' s='How can I find an IT job in Japan ? ' s='How I can speak English fluently ? ' s='Should I switch from Spotify to Apple Music ? ' s='What`s the best way to get rid of p**n addiction ? '
def main(): #if not torch.cuda.is_available(): # logging.info('no gpu device available') # sys.exit(1) np.random.seed(args.seed) ''' torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) ''' device = torch.device("cpu") torch.manual_seed(args.seed) logging.info('use cpu') logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) param_list = list(model.parameters()) # print(model) # model = model.cuda() model.to(device) utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() #criterion = criterion.cuda() criterion.to(device) _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob METRICS = Metrics(list(model.parameters()), p=1) PERFORMANCE_STATISTICS = {} ARCH_STATISTICS = {} metrics_path = './metrics_stat_test.xlsx' weights_path = './weights_stat_test.xlsx' epoch = 0 io_metrics = METRICS.evaluate(epoch) PERFORMANCE_STATISTICS[f'in_S_epoch_{epoch}'] = io_metrics.input_channel_S PERFORMANCE_STATISTICS[ f'out_S_epoch_{epoch}'] = io_metrics.output_channel_S PERFORMANCE_STATISTICS[ f'mode12_S_epoch_{epoch}'] = io_metrics.mode_12_channel_S PERFORMANCE_STATISTICS[f'fc_S_epoch_{epoch}'] = io_metrics.fc_S PERFORMANCE_STATISTICS[ f'in_rank_epoch_{epoch}'] = io_metrics.input_channel_rank PERFORMANCE_STATISTICS[ f'out_rank_epoch_{epoch}'] = io_metrics.output_channel_rank PERFORMANCE_STATISTICS[ f'mode12_rank_epoch_{epoch}'] = io_metrics.mode_12_channel_rank PERFORMANCE_STATISTICS[f'fc_rank_epoch_{epoch}'] = io_metrics.fc_rank PERFORMANCE_STATISTICS[ f'in_condition_epoch_{epoch}'] = io_metrics.input_channel_condition PERFORMANCE_STATISTICS[ f'out_condition_epoch_{epoch}'] = io_metrics.output_channel_condition PERFORMANCE_STATISTICS[ f'mode12_condition_epoch_{epoch}'] = io_metrics.mode_12_channel_condition # write metrics data to xls file metrics_df = pd.DataFrame(data=PERFORMANCE_STATISTICS) metrics_df.to_excel(metrics_path) test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
y.append(pred) prob.append(joint_prob[pred] / np.sum(joint_prob)) input_prob.append(np.sum(joint_prob)) return np.array(apply_index2raw(y, self.feat2index_dict_y)), prob, input_prob if __name__ == "__main__": p_x_train = os.path.join("data", "x_train.pkl") p_y_train = os.path.join("data", "y_train.pkl") p_x_test = os.path.join("data", "x_test.pkl") p_y_test = os.path.join("data", "y_test.pkl") X_train = load(p_x_train) y_train = load(p_y_train) X_test = load(p_x_test) y_test = load(p_y_test) model = Naive_bayes() # we are going to split dimension of age, BMI and charges model.fit(X_train, y_train, discretize_dim=(0, 2, 5), split_num=(5, 5, 5)) pred, _, _ = model.predict(X_train) train_acc = np.sum((pred == y_train)) / y_train.shape[0] pred, _, _ = model.predict(X_test) test_acc = np.sum((pred == y_test)) / y_test.shape[0] print("train accuracy: {}, test accuracy: {}".format(train_acc, test_acc))
def print_max_significance(discovery_sig_df): max_sig_idx = discovery_sig_df['ZA'].idxmax() max_sig_data = discovery_sig_df.iloc[max_sig_idx] max_sig = max_sig_data['ZA'] max_sig_cut = max_sig_data['t_cut'] max_sig_str = 'Max significance {:.2f} for cut value {}'.format(max_sig, max_sig_cut) print max_sig_str if __name__ == '__main__': DATA_F = 'atlas-higgs-challenge-2014-v2.csv' CLF_F = 'BDT.pkl' data_path = os.path.join('data', DATA_F) log.info('Reading data from {}'.format(data_path)) data, labels, weights = get_data_labels_weights(data_path) log.info('Splitting into signal and background samples') X_sig, X_bkg, w_sig, w_bkg = split_sig_bkg(labels, data, weights) clf = load(CLF_F) log.info('Calculating classifier predictions') p_sig = clf.decision_function(X_sig) p_bkg = clf.decision_function(X_bkg) log.info('Scanning discovery significance over cut values') ds = discovery_significance(p_sig, p_bkg, w_sig, w_bkg) print_max_significance(ds)
import task2 import utils from matplotlib import pyplot as plt import numpy as np tcases = utils.load('testcases.pkl') for tcase in tcases: img_patches, shape, reconstructed = tcases[tcase] cleaned_img = task2.reconstruct_from_noisy_patches(img_patches, shape) #if tcase == 3 : #print(img_patches) #print(shape) #print(reconstructed.shape) #key = [(197, 207, 214, 222), (167, 207, 197, 222), (153, 207, 167, 222), (137, 207, 153, 222)] #for i in key: # utils.make_fig(img_patches[i], cmap='gray', title='Yours') try: if np.allclose(cleaned_img, reconstructed): print('testcase# {} passed'.format(tcase)) utils.make_fig(reconstructed, cmap='gray', title='Correct') utils.make_fig(cleaned_img, cmap='gray', title='Yours') plt.show() else: print('testcase# {} failed'.format(tcase)) utils.make_fig(reconstructed, cmap='gray', title='Correct') utils.make_fig(cleaned_img, cmap='gray', title='Yours') plt.show() except Exception as e: print('testcase# {} failed'.format(tcase))
import linear_model as lm import utils ''' 聽說取二次也算 linear regression ''' train_path = os.path.join(os.path.dirname(__file__), "./data/train.csv") test_path = os.path.join(os.path.dirname(__file__), "./data/test.csv") output_path = os.path.join(os.path.dirname(__file__), "./ans.csv") model_path = os.path.join(os.path.dirname(__file__), "./model") scaler_path = os.path.join(os.path.dirname(__file__), "./scaler_sqr") fea_select, y_pos = (0, 4, 5, 6, 7, 8, 9, 16), 70 x, y = utils.load(train_path, mode='train', fea_select=fea_select, y_pos=y_pos) # 讀出所有 data 、擷取 feature 、劃分 9 天成一筆 x = np.concatenate((x, x**2), axis=1) # 加入平方當特徵 x, max, min = utils.rescaling(x) # 作 rescaling , 在 [0, 1] 間 x, y = utils.shuffle(x, y) x_train, y_train, x_val, y_val = utils.validation(x, y, ratio=0.1) b, w = lm.LinearRegression(x, y, lr=100000, epoch=1000000, lr_method='adagrad', x_val=x_val, y_val=y_val) x_test = utils.load(test_path, mode='test', fea_select=fea_select, y_pos=y_pos) x_test = np.concatenate((x_test, x_test**2), axis=1)
def knndis(t, X_M): Mg = X_M.shape[1] dist = np.sum((np.transpose(np.matlib.repmat(t, Mg, 1)) - X_M)**2, 0) dist = np.sort(dist) return sum(dist[0:10]) sess = tf.compat.v1.Session(config=config) # Initialize for Tensorflow restore_var = [v for v in tf.global_variables()] loader = tf.train.Saver(var_list=restore_var) ckpt = const.SNAPSHOT_DIR load(loader, sess, ckpt) # Yolo Initializer config_file = "/content/test/cfg/yolov4.cfg" weights = "/content/test/yolov4.weights" data_file = "/content/test/cfg/coco.data" batch_size = 1 thresh = 0.25 # remove object with confidence score < 0.25 save_labels = False # do not export output to .txt file random.seed(3) # deterministic bbox colors network, class_names, class_colors = darknet.load_network( config_file, data_file, weights, batch_size=batch_size) count = 4
def run(self): g.init() if not self.journal: utils.load() else: g.ok = self.ok g.rule = self.rule self.cards = cards.Cards() self.cards.ok = g.ok self.cards.rule = g.rule self.cards.redeal() bx = g.sx(17.4) by1 = g.sy(15.5) by2 = g.sy(18.5) self.yes_bu = [] self.no_bu = [] for i in range(3): bu = buttons.Button("yes", (bx, by1), True) bu.active = False self.yes_bu.append(bu) bu = buttons.Button("no", (bx, by2), True) bu.active = False self.no_bu.append(bu) bx += self.cards.width + g.sy(.5) bx = g.sx(29.9) by = g.sy(14.2) buttons.Button("redeal", (bx, by), True) if self.journal: # Sugar only a, b, c, d = pygame.cursors.load_xbm('my_cursor.xbm', 'my_cursor_mask.xbm') pygame.mouse.set_cursor(a, b, c, d) going = True while going: # Pump GTK messages. while gtk.events_pending(): gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: if not self.journal: g.ok = self.cards.ok g.rule = self.cards.rule utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.redraw = True elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True if event.button == 2: # centre button if not self.journal: g.version_display = not g.version_display break bu = buttons.check() if bu <> '': self.button(bu) break self.click() if not going: break self.update() if g.redraw: self.display() if g.version_display: utils.version_display() pygame.display.flip() g.redraw = False tf = False if pygame.mouse.get_focused(): tf = True pygame.mouse.set_visible(tf) g.clock.tick(40) # be ready for xo quit at any time self.ok = self.cards.ok self.rule = self.cards.rule
def main(): if not torch.cuda.is_available(): logging.info('No GPU device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('GPU device = %d' % args.gpu) logging.info("args = %s", args) gpu_logger = GpuLogThread([args.gpu], writer, seconds=15 if not args.test else 1) gpu_logger.start() logging.debug(locals()) model = None # prepare dataset train_transform, valid_transform = utils.data_transforms(args.dataset, args.cutout, args.cutout_length) if args.dataset == "CIFAR100": train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) elif args.dataset == "CIFAR10": train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) elif args.dataset == 'MIT67': dset_cls = dset.ImageFolder data_path = '%s/MIT67/train' % args.tmp_data_dir # 'data/MIT67/train' val_path = '%s/MIT67/test' % args.tmp_data_dir # 'data/MIT67/val' train_data = dset_cls(root=data_path, transform=train_transform) valid_data = dset_cls(root=val_path, transform=valid_transform) elif args.dataset == 'Sport8': dset_cls = dset.ImageFolder data_path = '%s/Sport8/train' % args.tmp_data_dir # 'data/Sport8/train' val_path = '%s/Sport8/test' % args.tmp_data_dir # 'data/Sport8/val' train_data = dset_cls(root=data_path, transform=train_transform) valid_data = dset_cls(root=val_path, transform=valid_transform) elif args.dataset == "flowers102": dset_cls = dset.ImageFolder data_path = '%s/flowers102/train' % args.tmp_data_dir val_path = '%s/flowers102/test' % args.tmp_data_dir train_data = dset_cls(root=data_path, transform=train_transform) valid_data = dset_cls(root=val_path, transform=valid_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) import random;random.shuffle(indices) train_iterator = utils.DynamicBatchSizeLoader(torch.utils.data.DataLoader( train_data, batch_size=args.batch_multiples, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=args.workers), args.batch_size_min) valid_iterator = utils.DynamicBatchSizeLoader(torch.utils.data.DataLoader( train_data, batch_size=args.batch_multiples, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=args.workers), args.batch_size_min) # build Network logging.debug('building network') criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() num_graph_edges = sum(list(range(2, 2 + args.blocks))) switches_normal = SwitchManager(num_graph_edges, copy.deepcopy(PRIMITIVES), 'normal') switches_reduce = SwitchManager(num_graph_edges, copy.deepcopy(PRIMITIVES), 'reduce') total_epochs = 0 for cycle in parse_cycles(): logging.debug('new cycle %s' % repr(cycle)) print('\n' * 3, '-' * 100) print(cycle) print('', '-' * 100, '\n') writer.add_scalar('cycle/net_layers', cycle.net_layers, cycle.num) writer.add_scalar('cycle/net_init_c', cycle.net_init_c, cycle.num) writer.add_scalar('cycle/net_dropout', cycle.net_dropout, cycle.num) writer.add_scalar('cycle/ops_keep', cycle.ops_keep, cycle.num) writer.add_scalar('cycle/epochs', cycle.epochs, cycle.num) writer.add_scalar('cycle/grace_epochs', cycle.grace_epochs, cycle.num) writer.add_scalar('cycle/morphs', cycle.morphs, cycle.num) switches_normal.plot_ops(logging.info, writer, cycle.num) switches_reduce.plot_ops(logging.info, writer, cycle.num) # rebuild the model in each cycle, clean up the cache... logging.debug('building model') del model torch.cuda.empty_cache() if args.dataset == "CIFAR100": CLASSES = 100 elif args.dataset == "CIFAR10": CLASSES = 10 elif args.dataset == 'MIT67': dset_cls = dset.ImageFolder CLASSES = 67 elif args.dataset == 'Sport8': dset_cls = dset.ImageFolder CLASSES = 8 elif args.dataset == "flowers102": dset_cls = dset.ImageFolder CLASSES = 102 model = Network(cycle.net_init_c, CLASSES, cycle.net_layers, criterion, switches_normal=switches_normal, switches_reduce=switches_reduce, steps=args.blocks, p=cycle.net_dropout, largemode=True if args.dataset in utils.LARGE_DATASETS else False) gpu_logger.reset_recent() if cycle.load: utils.load(model, model_path) if args.reset_alphas: model.reset_alphas() if args.test: model.randomize_alphas() if cycle.init_morphed: model.init_morphed(switches_normal, switches_reduce) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) logging.debug('building optimizers') optimizer = torch.optim.SGD(model.net_parameters, args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) optimizer_a = torch.optim.Adam(model.arch_parameters, lr=args.arch_learning_rate, betas=(0.5, 0.999), weight_decay=args.arch_weight_decay) logging.debug('building scheduler') scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, int(cycle.epochs), eta_min=args.learning_rate_min) if args.batch_size_max > args.batch_size_min: train_iterator.set_batch_size(args.batch_size_min) valid_iterator.set_batch_size(args.batch_size_min) sm_dim = -1 scale_factor = 0.2 for epoch in range(cycle.epochs): lr = scheduler.get_lr()[0] logging.info('Epoch: %d lr: %e', epoch, lr) epoch_start = time.time() # training if epoch < cycle.grace_epochs: model.update_p(cycle.net_dropout * (cycle.epochs - epoch - 1) / cycle.epochs) else: model.update_p(cycle.net_dropout * np.exp(-(epoch - cycle.grace_epochs) * scale_factor)) train_acc, train_obj = train(train_iterator, valid_iterator, model, criterion, optimizer, optimizer_a, gpu_logger, train_arch=epoch >= cycle.grace_epochs) epoch_duration = time.time() - epoch_start # log info logging.info('Train_acc %f', train_acc) logging.info('Epoch time: %ds', epoch_duration) writer.add_scalar('train/accuracy', train_acc, total_epochs) writer.add_scalar('train/loss', train_obj, total_epochs) writer.add_scalar('epoch/lr', lr, total_epochs) writer.add_scalar('epoch/seconds', epoch_duration, total_epochs) writer.add_scalar('epoch/model.p', model.p, total_epochs) writer.add_scalar('epoch/batch_size', train_iterator.batch_size, total_epochs) # validation, only for the last 5 epochs in a cycle if cycle.epochs - epoch < 5: valid_acc, valid_obj = infer(valid_iterator, model, criterion) logging.info('Valid_acc %f', valid_acc) writer.add_scalar('valid/accuracy', valid_acc, total_epochs) writer.add_scalar('valid/loss', valid_obj, total_epochs) total_epochs += 1 gpu_logger.reset_recent() scheduler.step() utils.save(model, model_path) print('\n' * 2, '------Dropping/morphing paths------') # Save switches info for s-c refinement. if cycle.is_last: switches_normal_copy = switches_normal.copy() switches_reduce_copy = switches_reduce.copy() # drop operations with low architecture weights, add morphed ones arch_param = model.arch_parameters normal_prob = F.softmax(arch_param[0], dim=sm_dim).data.cpu().numpy() switches_normal.drop_and_morph(normal_prob, cycle.ops_keep, writer, cycle.num, num_morphs=cycle.morphs, no_zero=cycle.is_last and args.restrict_zero, keep_morphable=not cycle.is_last) reduce_prob = F.softmax(arch_param[1], dim=sm_dim).data.cpu().numpy() switches_reduce.drop_and_morph(reduce_prob, cycle.ops_keep, writer, cycle.num, num_morphs=cycle.morphs, no_zero=cycle.is_last and args.restrict_zero, keep_morphable=not cycle.is_last) logging.info('switches_normal = \n%s', switches_normal) logging.info('switches_reduce = \n%s', switches_reduce) # end last cycle with shortcut/zero pruning and save the genotype if cycle.is_last: #import ipdb;ipdb.set_trace() arch_param = model.arch_parameters normal_prob = F.softmax(arch_param[0], dim=sm_dim).data.cpu().numpy() reduce_prob = F.softmax(arch_param[1], dim=sm_dim).data.cpu().numpy() normal_final = [0 for _ in range(num_graph_edges)] reduce_final = [0 for _ in range(num_graph_edges)] # Generate Architecture keep_normal = [0, 1] keep_reduce = [0, 1] n = 3 start = 2 for i in range(3): end = start + n tbsn = normal_final[start:end] tbsr = reduce_final[start:end] edge_n = sorted(range(n), key=lambda x: tbsn[x]) keep_normal.append(edge_n[-1] + start) keep_normal.append(edge_n[-2] + start) edge_r = sorted(range(n), key=lambda x: tbsr[x]) keep_reduce.append(edge_r[-1] + start) keep_reduce.append(edge_r[-2] + start) start = end n = n + 1 for i in range(num_graph_edges): if i not in keep_normal: for j in range(len(switches_normal.current_ops)): switches_normal[i][j] = False if i not in keep_reduce: for j in range(len(switches_reduce.current_ops)): switches_reduce[i][j] = False switches_normal.keep_2_branches(normal_prob) switches_reduce.keep_2_branches(reduce_prob) switches_normal.plot_ops(logging.info, writer, cycle.num + 1) switches_reduce.plot_ops(logging.info, writer, cycle.num + 1) genotype = parse_network(switches_normal, switches_reduce) logging.info(genotype) save_genotype(args.save + 'genotype.json', genotype) with open(args.save + "/best_genotype.txt", "w") as f: f.write(str(genotype)) gpu_logger.stop()
def create_zips(args): def get_filepaths(subset): filepaths = [] tids = tracks.index[tracks['set', 'subset'] <= subset] for tid in tids: filepaths.append(utils.get_audio_path('', tid)) return filepaths def get_checksums(base_dir, filepaths): """Checksums are assumed to be stored in order for efficiency.""" checksums = [] with open(os.path.join(args.path, base_dir, 'checksums')) as f: for filepath in filepaths: exist = False for line in f: if filepath == line[42:-1]: exist = True break if not exist: raise ValueError('checksum not found: {}'.format(filepath)) checksums.append(line) return checksums def create_zip(zip_filename, base_dir, filepaths): # Audio: all compressions are the same. # CSV: stored > deflated > BZIP2 > LZMA. # LZMA is close to BZIP2 and too recent to be widely available (unzip). compression = zipfile.ZIP_BZIP2 zip_filepath = os.path.join(args.path, zip_filename) with zipfile.ZipFile(zip_filepath, 'x', compression) as zf: def info(name): name = os.path.join(zip_filename[:-4], name) info = zipfile.ZipInfo(name, (2017, 4, 1, 0, 0, 0)) info.external_attr = 0o444 << 16 | 0o2 << 30 return info zf.writestr(info('README.txt'), README, compression) checksums = get_checksums(base_dir, filepaths) zf.writestr(info('checksums'), ''.join(checksums), compression) for filepath in tqdm(filepaths): src = os.path.join(args.path, base_dir, filepath) dst = os.path.join(zip_filename[:-4], filepath) zf.write(src, dst) os.chmod(zip_filepath, 0o444) os.utime(zip_filepath, (TIME, TIME)) METADATA = [ 'not_found.pickle', 'raw_genres.csv', 'raw_albums.csv', 'raw_artists.csv', 'raw_tracks.csv', 'tracks.csv', 'genres.csv', 'raw_echonest.csv', 'echonest.csv', 'features.csv', ] create_zip('fma_metadata.zip', 'fma_metadata', METADATA) tracks = utils.load('tracks.csv') create_zip('fma_small.zip', 'fma_large', get_filepaths('small')) create_zip('fma_medium.zip', 'fma_large', get_filepaths('medium')) create_zip('fma_large.zip', 'fma_large', get_filepaths('large')) create_zip('fma_full.zip', 'fma_full', get_filepaths('large'))
def run(self): g.init() if not self.journal: utils.load() self.didg = didg.Didg() g.carry_shape = didg.shapes[0] self.didg.make_carry() load_save.retrieve() self.buttons_setup() if self.canvas <> None: self.canvas.grab_focus() ctrl = False pygame.key.set_repeat(600, 120) key_ms = pygame.time.get_ticks() need_carry = False going = True while going: if self.journal: # Pump GTK messages. while Gtk.events_pending(): Gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: if not self.journal: utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.pos = event.pos need_carry = True if self.canvas <> None: self.canvas.grab_focus() elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True if event.button == 1: if self.do_click(1): pass else: bu = buttons.check() if bu != '': self.do_button(bu) self.flush_queue() if event.button == 3: self.do_click(3) elif event.type == pygame.KEYDOWN: # throttle keyboard repeat if pygame.time.get_ticks() - key_ms > 110: key_ms = pygame.time.get_ticks() if ctrl: if event.key == pygame.K_q: if not self.journal: utils.save() going = False break else: ctrl = False if event.key in (pygame.K_LCTRL, pygame.K_RCTRL): ctrl = True break self.do_key(event.key) g.redraw = True self.flush_queue() elif event.type == pygame.KEYUP: ctrl = False if not going: break if g.redraw: g.carry_save = False self.display() if g.version_display: utils.version_display() self.carry() need_carry = False pygame.display.flip() g.redraw = False elif need_carry: self.carry() pygame.display.flip() need_carry = False g.clock.tick(40)
import os print( """#============================================================================== # START !!! {} PID: {} #============================================================================== """.format(__file__, os.getpid())) #import pandas as pd #import numpy as np from itertools import product #from collections import Counter import multiprocessing as mp total_proc = 2 import utils train, test = utils.load(3, 1) #============================================================================== # def #============================================================================== def set_q(s): return set(s.lower().split()) - utils.kigou def _and_(s1, s2): return s1 & s2 def set_diff(s1, s2): return s1 - s2
def main(): ''' set default hyperparams in default_hyperparams.py ''' parser = argparse.ArgumentParser() # Required arguments parser.add_argument('-d', '--dataset', choices=supported.datasets, required=True) parser.add_argument('--algorithm', required=True, choices=supported.algorithms) parser.add_argument( '--root_dir', required=True, help= 'The directory where [dataset]/data can be found (or should be downloaded to, if it does not exist).' ) # Dataset parser.add_argument( '--split_scheme', help= 'Identifies how the train/val/test split is constructed. Choices are dataset-specific.' ) parser.add_argument('--dataset_kwargs', nargs='*', action=ParseKwargs, default={}) parser.add_argument( '--download', default=False, type=parse_bool, const=True, nargs='?', help= 'If true, tries to downloads the dataset if it does not exist in root_dir.' ) parser.add_argument( '--frac', type=float, default=1.0, help= 'Convenience parameter that scales all dataset splits down to the specified fraction, for development purposes.' ) # Loaders parser.add_argument('--loader_kwargs', nargs='*', action=ParseKwargs, default={}) parser.add_argument('--train_loader', choices=['standard', 'group']) parser.add_argument('--uniform_over_groups', type=parse_bool, const=True, nargs='?') parser.add_argument('--distinct_groups', type=parse_bool, const=True, nargs='?') parser.add_argument('--n_groups_per_batch', type=int) parser.add_argument('--batch_size', type=int) parser.add_argument('--eval_loader', choices=['standard'], default='standard') # Model parser.add_argument('--model', choices=supported.models) parser.add_argument( '--model_kwargs', nargs='*', action=ParseKwargs, default={}, help= 'keyword arguments for model initialization passed as key1=value1 key2=value2' ) # Transforms parser.add_argument('--train_transform', choices=supported.transforms) parser.add_argument('--eval_transform', choices=supported.transforms) parser.add_argument( '--target_resolution', nargs='+', type=int, help= 'target resolution. for example --target_resolution 224 224 for standard resnet.' ) parser.add_argument('--resize_scale', type=float) parser.add_argument('--max_token_length', type=int) # Objective parser.add_argument('--loss_function', choices=supported.losses) # Algorithm parser.add_argument('--groupby_fields', nargs='+') parser.add_argument('--group_dro_step_size', type=float) parser.add_argument('--coral_penalty_weight', type=float) parser.add_argument('--irm_lambda', type=float) parser.add_argument('--irm_penalty_anneal_iters', type=int) parser.add_argument('--algo_log_metric') parser.add_argument('--hsic_beta', type=float) parser.add_argument('--sd_penalty_lamb', type=float) parser.add_argument('--grad_penalty_lamb', type=float) parser.add_argument( '--params_regex', type=str, help='Regular expression specifying which gradients to penalize.') parser.add_argument('--label_cond', type=parse_bool, const=True, nargs='?', default=False) parser.add_argument('--dann_lamb', type=float) parser.add_argument('--dann_dc_name', type=str) # Model selection parser.add_argument('--val_metric') parser.add_argument('--val_metric_decreasing', type=parse_bool, const=True, nargs='?') # Optimization parser.add_argument('--n_epochs', type=int) parser.add_argument('--optimizer', choices=supported.optimizers) parser.add_argument('--lr', type=float) parser.add_argument('--weight_decay', type=float) parser.add_argument('--max_grad_norm', type=float) parser.add_argument('--optimizer_kwargs', nargs='*', action=ParseKwargs, default={}) # Scheduler parser.add_argument('--scheduler', choices=supported.schedulers) parser.add_argument('--scheduler_kwargs', nargs='*', action=ParseKwargs, default={}) parser.add_argument('--scheduler_metric_split', choices=['train', 'val'], default='val') parser.add_argument('--scheduler_metric_name') # Evaluation parser.add_argument('--evaluate_all_splits', type=parse_bool, const=True, nargs='?', default=True) parser.add_argument('--eval_splits', nargs='+', default=[]) parser.add_argument('--eval_only', type=parse_bool, const=True, nargs='?', default=False) parser.add_argument('--eval_epoch', default=None, type=int) parser.add_argument('--save_z', type=parse_bool, const=True, nargs='?', default=False) # Misc parser.add_argument('--device', type=int, default=0) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--log_dir', default='./logs') parser.add_argument('--log_every', default=50, type=int) parser.add_argument('--save_step', type=int) parser.add_argument('--save_best', type=parse_bool, const=True, nargs='?', default=True) parser.add_argument('--save_last', type=parse_bool, const=True, nargs='?', default=True) parser.add_argument('--no_group_logging', type=parse_bool, const=True, nargs='?') parser.add_argument('--use_wandb', type=parse_bool, const=True, nargs='?', default=False) parser.add_argument('--progress_bar', type=parse_bool, const=True, nargs='?', default=False) parser.add_argument('--resume', type=parse_bool, const=True, nargs='?', default=False) config = parser.parse_args() config = populate_defaults(config) # set device config.device = torch.device("cuda:" + str( config.device)) if torch.cuda.is_available() else torch.device("cpu") ## Initialize logs if os.path.exists(config.log_dir) and config.resume: resume = True mode = 'a' elif os.path.exists(config.log_dir) and config.eval_only: resume = False mode = 'a' else: resume = False mode = 'w' if not os.path.exists(config.log_dir): os.makedirs(config.log_dir) logger = Logger(os.path.join(config.log_dir, 'log.txt'), mode) # Record config log_config(config, logger) # Set random seed set_seed(config.seed) # Data full_dataset = supported.datasets[config.dataset]( root_dir=config.root_dir, download=config.download, split_scheme=config.split_scheme, **config.dataset_kwargs) # To implement data augmentation (i.e., have different transforms # at training time vs. test time), modify these two lines: train_transform = initialize_transform( transform_name=config.train_transform, config=config, dataset=full_dataset) eval_transform = initialize_transform(transform_name=config.eval_transform, config=config, dataset=full_dataset) train_grouper = CombinatorialGrouper(dataset=full_dataset, groupby_fields=config.groupby_fields) datasets = defaultdict(dict) for split in full_dataset.split_dict.keys(): if split == 'train': transform = train_transform verbose = True elif split == 'val': transform = eval_transform verbose = True else: transform = eval_transform verbose = False # Get subset datasets[split]['dataset'] = full_dataset.get_subset( split, frac=config.frac, transform=transform) if split == 'train': datasets[split]['loader'] = get_train_loader( loader=config.train_loader, dataset=datasets[split]['dataset'], batch_size=config.batch_size, uniform_over_groups=config.uniform_over_groups, grouper=train_grouper, distinct_groups=config.distinct_groups, n_groups_per_batch=config.n_groups_per_batch, **config.loader_kwargs) else: datasets[split]['loader'] = get_eval_loader( loader=config.eval_loader, dataset=datasets[split]['dataset'], grouper=train_grouper, batch_size=config.batch_size, **config.loader_kwargs) # Set fields datasets[split]['split'] = split datasets[split]['name'] = full_dataset.split_names[split] datasets[split]['verbose'] = verbose # Loggers # Loggers datasets[split]['eval_logger'] = BatchLogger( os.path.join(config.log_dir, f'{split}_eval.csv'), mode=mode, use_wandb=(config.use_wandb and verbose)) datasets[split]['algo_logger'] = BatchLogger( os.path.join(config.log_dir, f'{split}_algo.csv'), mode=mode, use_wandb=(config.use_wandb and verbose)) if config.use_wandb: initialize_wandb(config) # Logging dataset info if config.no_group_logging and full_dataset.is_classification and full_dataset.y_size == 1: log_grouper = CombinatorialGrouper(dataset=full_dataset, groupby_fields=['y']) elif config.no_group_logging: log_grouper = None else: log_grouper = train_grouper log_group_data(datasets, log_grouper, logger) ## Initialize algorithm algorithm = initialize_algorithm(config=config, datasets=datasets, train_grouper=train_grouper) if not config.eval_only: ## Load saved results if resuming resume_success = False if resume: save_path = os.path.join(config.log_dir, 'last_model.pth') if not os.path.exists(save_path): epochs = [ int(file.split('_')[0]) for file in os.listdir(config.log_dir) if file.endswith('.pth') ] if len(epochs) > 0: latest_epoch = max(epochs) save_path = os.path.join(config.log_dir, f'{latest_epoch}_model.pth') try: prev_epoch, best_val_metric = load(algorithm, save_path) epoch_offset = prev_epoch + 1 logger.write( f'Resuming from epoch {epoch_offset} with best val metric {best_val_metric}' ) resume_success = True except FileNotFoundError: pass if resume_success == False: epoch_offset = 0 best_val_metric = None train(algorithm=algorithm, datasets=datasets, general_logger=logger, config=config, epoch_offset=epoch_offset, best_val_metric=best_val_metric) else: if config.eval_epoch is None: eval_model_path = os.path.join(config.log_dir, 'best_model.pth') else: eval_model_path = os.path.join(config.log_dir, f'{config.eval_epoch}_model.pth') best_epoch, best_val_metric = load(algorithm, eval_model_path) if config.eval_epoch is None: epoch = best_epoch else: epoch = config.eval_epoch evaluate(algorithm=algorithm, datasets=datasets, epoch=epoch, general_logger=logger, config=config) logger.close() for split in datasets: datasets[split]['eval_logger'].close() datasets[split]['algo_logger'].close()
def test_SdA(finetune_lr=0.1, pretraining_epochs=15, pretrain_lr=0.001, training_epochs=1000, dataset='mnist.pkl.gz', batch_size=32, finetuning= False, selvar=False): """ :type learning_rate: float :param learning_rate: learning rate used in the finetune stage (factor for the stochastic gradient) :type pretraining_epochs: int :param pretraining_epochs: number of epoch to do pretraining :type pretrain_lr: float :param pretrain_lr: learning rate to be used during pre-training :type n_iter: int :param n_iter: maximal number of iterations ot run the optimizer :type dataset: string :param dataset: path the the pickled dataset """ #datasets = load_data(dataset) datasets = load_otto_data("data/train.csv", selected=selvar) data = load_otto_data('data/test.csv', train=False, selected=selvar) data_x, data_y = data[0] print data_x.get_value().shape train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_train_batches /= batch_size n_data_batches = data_x.get_value(borrow=True).shape[0] n_data_batches /= batch_size # numpy random generator # start-snippet-3 numpy_rng = numpy.random.RandomState(89677) newmodel = True ## check if model exists and resume otherwise rebuild if os.path.isfile("./tmp/sda"): print ("Loading existing model...") sda = ut.load("sda", "./tmp/") newmodel = False print ("done.") return sda else: # construct the stacked denoising autoencoder class print '... building the model' sda = SdA( numpy_rng=numpy_rng, n_ins=93, #784, # 28 * 28, hidden_layers_sizes=[60], # [1000, 1000, 1000], corruption_levels=[0.1, 0.2, 0.2, 0.2], n_outs=10 #9 ) # train only if we just created a new model if newmodel: ######################### # PRETRAINING THE MODEL # ######################### print '... getting the pretraining functions' if finetuning: pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size) else: pretraining_fns = sda.pretraining_functions(train_set_x=data_x, batch_size=batch_size) print '... pre-training the model' start_time = time.clock() ## Pre-train layer-wise corruption_levels = [0.1, 0.1, 0.2] #[.1, .2, .3, .2, .2, .2] for i in xrange(sda.n_layers): # go through pretraining epochs for epoch in xrange(pretraining_epochs): # go through the training set c = [] #for batch_index in xrange(n_train_batches): for batch_index in xrange(n_data_batches): c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) end_time = time.clock() print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)) ######################## # FINETUNING THE MODEL # ######################## if finetuning: # get the training, validation and testing function for the model print '... getting the finetuning functions' train_fn, validate_model, test_model = sda.build_finetune_functions( datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr ) print '... finetunning the model' # early-stopping parameters patience = 20 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = numpy.inf test_score = 0. start_time = time.clock() done_looping = False epoch = 0 while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in xrange(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() this_validation_loss = numpy.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough if ( this_validation_loss < best_validation_loss * improvement_threshold ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number best_validation_loss = this_validation_loss best_iter = iter # test it on the test set test_losses = test_model() test_score = numpy.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) if patience <= iter: done_looping = True break end_time = time.clock() print( ( 'Optimization complete with best validation score of %f %%, ' 'on iteration %i, ' 'with test performance %f %%' ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.) ) print >> sys.stderr, ('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)) print "Saving model to disk" ut.save(sda, "sda") return sda
#%% import re from utils import load raw = load("day24.txt") direction = { "e": 1.0, "se": 0.5 - 1j, "sw": -0.5 - 1j, "w": -1.0, "nw": -0.5 + 1j, "ne": 0.5 + 1j, } def run_init() -> set[complex]: # black: 0 white: 1 locs: set[complex] = set() for line in raw: locs ^= { sum(direction[step] for step in re.findall(r"([ns]?[ew])", line)) } return locs def test1() -> None: locs = run_init() assert len(locs) == 528
def main(argv): learnModel = True # by default we learn each model again try: opts, args = getopt.getopt(argv,"hl:",["load="]) except getopt.GetoptError: print 'analyticSuite.py -l <modelpath>' sys.exit(2) for opt, arg in opts: if opt == '-h': print 'analyticSuite.py -l <path>' sys.exit() elif opt in ("-l", "--load"): modelpath = arg learnModel = False # do not learn, load an existing model ################################################ # load_data returns data in tuple form (x,y) ################################################ if not learnModel: unseen_data = md.load_data('data/test.csv', datatype="test", verbose=True) ################################################ # GradientBoosting classifier ################################################ if learnModel: # load training dataset and split train,test,valid = md.load_data( 'data/train.csv', datatype="train", verbose=True) x_train,y_train = train x_test, y_test = test x_valid, y_valid = valid params = {'n_estimators': 800, 'max_depth': 3, 'subsample': 0.5, 'learning_rate': 0.1, 'min_samples_leaf': 1} # train Gradient Boosting Classifier gbc = ensemble.GradientBoostingClassifier(**params) gbc.fit(x_train, y_train) train_acc, test_acc = evalError(gbc, (x_train,y_train), (x_test,y_test) ) print "GradientBoostingClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc) ut.save(gbc, "GradientBoosting", verbose=True) else: print "Loading Gradient Boosting model" gbc = ut.load("GradientBoosting", modelpath) #ut.writePredictionFile(gbc.predict(unseen_data), modeltype="gbc") ut.writePredictionFile(gbc.predict_proba(unseen_data), modeltype="gbc") # free memory del gbc gc.collect() ############################################## ## Support Vector Machine ## ############################################## if learnModel: svmc = svm.SVC(probability=True) svmc.fit(x_train, y_train) ut.save(svmc, "SupportVectorMachine", verbose=True) train_acc, test_acc = evalError(svmc, (x_train,y_train), (x_test,y_test)) print "ExtraTreeClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc) #svmscores = cross_val_score(svmc, x_train, y_train) #print "SupportVectorMachine mean score %f" %svmscores.mean() else: print "Loading SVM model" svmc = ut.load("SupportVectorMachine", modelpath) ut.writePredictionFile(svmc.predict(unseen_data), modeltype="svm") # free memory del svmc gc.collect() ############################################ # ExtraTreesClassifier ############################################ if learnModel: etc = ensemble.ExtraTreesClassifier(n_estimators=600, max_depth=None, min_samples_split=1, random_state=0, n_jobs=-1) etc.fit(x_train, y_train) ut.save(etc, "ExtraTreesClassifier", verbose=True) train_acc, test_acc = evalError(etc, (x_train,y_train), (x_test,y_test)) print "ExtraTreeClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc) #scores = cross_val_score(etc, x_train, y_train) else: print "Loading ExtraTrees Classifier model" etc = ut.load("ExtraTreesClassifier", modelpath) #ut.writePredictionFile(etc.predict(unseen_data), modeltype="etc") ut.writePredictionFile(etc.predict_proba(unseen_data), modeltype="etc") # free memory del etc gc.collect()
def run(self, restore=False): self.g_init() if not self.journal: utils.load() load_save.retrieve() if restore: self.restore_pattern() else: g.delay = (3 - g.level) * 400 self.tu = my_turtle.TurtleClass() self.tu.current = [1, 1, 1, 3, 2] self.get_goal() if g.pattern == 1: self.tu.current = utils.copy_list(g.goal) self.tu.setup(self.colors[0]) g.numbers = utils.copy_list(self.tu.current) #buttons x = g.sx(7.3) y = g.sy(16.5) dx = g.sy(2.6) if not self.sugar: buttons.Button("cyan", (x, y), True) x += dx buttons.off('cyan') buttons.Button("green", (x, y), True) x += dx buttons.Button("red", (x, y), True) x += dx buttons.Button("black", (x, y), True) x += dx self.slider = slider.Slider(g.sx(23.5), g.sy(21), 3, utils.YELLOW) self.mouse_1st_no() # to 1st number if self.canvas is not None: self.canvas.grab_focus() ctrl = False pygame.key.set_repeat(600, 120) key_ms = pygame.time.get_ticks() going = True while going: if self.journal: # Pump GTK messages. while gtk.events_pending(): gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: if not self.journal: utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.pos = event.pos g.redraw = True if self.canvas is not None: self.canvas.grab_focus() elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True if g.big: g.big = False else: bu = buttons.check() if bu != '': self.do_button(bu) self.flush_queue() elif not self.sugar: if utils.mouse_on_img1(g.magician, g.magician_c): self.help2() elif utils.mouse_in(g.x1, g.y0, g.x1 + g.bw, g.y0 + g.bw): self.big_pic() elif self.slider.mouse(): g.delay = (3 - g.level) * 400 else: g.show_help = False self.check_nos(event.button) else: g.show_help = False self.check_nos(event.button) elif event.type == pygame.KEYDOWN: # throttle keyboard repeat if pygame.time.get_ticks() - key_ms > 110: key_ms = pygame.time.get_ticks() if ctrl: if event.key == pygame.K_q: if not self.journal: utils.save() going = False break else: ctrl = False if event.key in (pygame.K_LCTRL, pygame.K_RCTRL): ctrl = True break self.do_key(event.key) g.redraw = True self.flush_queue() elif event.type == pygame.KEYUP: ctrl = False if not going: break if self.tu.running: self.tu.move() if not g.crash_drawn: g.crash_drawn = True g.redraw = True if g.redraw: self.display() if g.version_display: utils.version_display() g.screen.blit(g.pointer, g.pos) pygame.display.flip() g.redraw = False g.clock.tick(40)
def _create_particles(self, particle_factory, *args, **kw): """ Create particles given a callable `particle_factory` and any arguments to it. This will also automatically distribute the particles among processors if this is a parallel run. Returns a list of particle arrays that are created. """ solver = self._solver num_procs = self.num_procs options = self.options rank = self.rank comm = self.comm # particle array info that is used to create dummy particles # on non-root processors particles_info = {} # Only master creates the particles. if rank == 0: if options.restart_file is not None: data = load(options.restart_file) arrays = data["arrays"] solver_data = data["solver_data"] # arrays and particles particles = [] for array_name in arrays: particles.append(arrays[array_name]) # save the particles list self.particles = particles # time, timestep and solver iteration count at restart t, dt, count = solver_data["t"], solver_data["dt"], solver_data["count"] # rescale dt at restart dt *= options.rescale_dt solver.t, solver.dt, solver.count = t, dt, count else: self.particles = particle_factory(*args, **kw) # get the array info which will be b'casted to other procs particles_info = utils.get_particles_info(self.particles) # Broadcast the particles_info to other processors for parallel runs if self.num_procs > 1: particles_info = self.comm.bcast(particles_info, root=0) # now all processors other than root create dummy particle arrays if rank != 0: self.particles = utils.create_dummy_particles(particles_info) # Instantiate the Parallel Manager here and do an initial LB self.pm = None if num_procs > 1: options = self.options if options.with_zoltan: if not (Has_Zoltan and Has_MPI): raise RuntimeError("Cannot run in parallel!") else: raise ValueError( """Sorry. You're stuck with Zoltan for now use the option '--with_zoltan' for parallel runs """ ) # create the parallel manager obj_weight_dim = "0" if options.zoltan_weights: obj_weight_dim = "1" zoltan_lb_method = options.zoltan_lb_method zoltan_debug_level = options.zoltan_debug_level zoltan_obj_wgt_dim = obj_weight_dim # ghost layers ghost_layers = options.ghost_layers # radius scale for the parallel update radius_scale = options.parallel_scale_factor * solver.kernel.radius_scale self.pm = pm = ZoltanParallelManagerGeometric( dim=solver.dim, particles=self.particles, comm=comm, lb_method=zoltan_lb_method, obj_weight_dim=obj_weight_dim, ghost_layers=ghost_layers, update_cell_sizes=options.update_cell_sizes, radius_scale=radius_scale, ) ### ADDITIONAL LOAD BALANCING FUNCTIONS FOR ZOLTAN ### # RCB lock directions if options.zoltan_rcb_lock_directions: pm.set_zoltan_rcb_lock_directions() if options.zoltan_rcb_reuse: pm.set_zoltan_rcb_reuse() if options.zoltan_rcb_rectilinear: pm.set_zoltan_rcb_rectilinear_blocks() if options.zoltan_rcb_set_direction > 0: pm.set_zoltan_rcb_directions(str(options.zoltan_rcb_set_direction)) # set zoltan options pm.pz.Zoltan_Set_Param("DEBUG_LEVEL", options.zoltan_debug_level) pm.pz.Zoltan_Set_Param("DEBUG_MEMORY", "0") # do an initial load balance pm.update() pm.initial_update = False # set subsequent load balancing frequency lb_freq = options.lb_freq if lb_freq < 1: raise ValueError("Invalid lb_freq %d" % lb_freq) pm.set_lb_freq(lb_freq) # wait till the initial partition is done comm.barrier() # set the solver's parallel manager solver.set_parallel_manager(self.pm) return self.particles
LR = 0.01 GPUS = 1 RAW_MODEL = None TASK = "CNN@%s_AR@%d_DISC@%d_CTC@%d_MTO@%s_DLOSS@%s_MARGIN@%.2f_INIT@%d_BN@%d"%(RES_TYPE,AR_EN,DISC_EN,CTC_EN,MANY_TO_ONE, METRIC_LOSS,MARGIN,1 if RAW_MODEL else 0,BN_DIM) MODEL_DIR = "exp/%s"%TASK if not os.path.isdir(MODEL_DIR):os.mkdir(MODEL_DIR) lr_reducer = ReduceLROnPlateau(factor=0.3, cooldown=0, patience=2, min_lr=1e-5, monitor='val_y_accent_acc', mode='max', min_delta=0.001, verbose=1) early_stopper = EarlyStopping(patience=3, monitor='val_y_accent_acc', mode='max', min_delta=0.001, verbose=1) csv_logger = CSVLogger('%s/train.csv' % MODEL_DIR) # generator and data DATA_DCT = us.load("array/data_scp.pkl") ACCENT_DCT = us.load("array/accent_scp.pkl") TRANS_DCT = us.load("array/trans_scp.pkl") train_lst = us.read_lines("train.lst") dev_lst = us.read_lines("dev.lst")[:100] N_BATCHS = len(train_lst)//BATCH_SIZE train_generator = us.data_generator(train_lst, batch_size=BATCH_SIZE, ctc_enable=CTC_EN, ar_enable=AR_EN, disc_enable=DISC_EN, data_dct=DATA_DCT, accent_dct=ACCENT_DCT, trans_dct=TRANS_DCT, max_input_len=MAX_INPUT_LEN,
def run_trials(args): """ Run args["conf_trials"] trials and survive args["max_attempts"] failed attempts. """ print(f"Arguments: {args}") if args["no_rand"]: utils.set_rand_seed() out_dir = args["out_dir"] if not path.isdir(out_dir): print(f"Output directory does not exist. Creating it: {out_dir}") os.makedirs(out_dir) net_tmp = models.MODELS[args["model"]]() # Verify that the necessary supplemental parameters are present. for param in net_tmp.params: assert param in args, f"\"{param}\" not in args: {args}" # Assemble the output filepath. out_flp = path.join( args["out_dir"], (utils.args_to_str(args, order=sorted(defaults.DEFAULTS.keys()))) + ( # Determine the proper extension based on the type of # model. ".pickle" if isinstance(net_tmp, models.SvmSklearnWrapper) else ".pth")) # If custom features are specified, then overwrite the model's # default features. fets = args["features"] if fets: net_tmp.in_spc = fets else: assert "arrival time us" not in args["features"] args["features"] = net_tmp.in_spc # If a trained model file already exists, then delete it. if path.exists(out_flp): os.remove(out_flp) # Load or geenrate training data. dat_flp = path.join(out_dir, "data.npz") scl_prms_flp = path.join(out_dir, "scale_params.json") # Check for the presence of both the data and the scaling # parameters because the resulting model is useless without the # proper scaling parameters. if (not args["regen_data"] and path.exists(dat_flp) and path.exists(scl_prms_flp)): print("Found existing data!") dat_in, dat_out, dat_out_raw, dat_out_oracle, num_flws = utils.load( dat_flp) dat_in_shape = dat_in.shape dat_out_shape = dat_out.shape assert dat_in_shape[0] == dat_out_shape[0], \ f"Data has invalid shapes! in: {dat_in_shape}, out: {dat_out_shape}" else: print("Regenerating data...") dat_in, dat_out, dat_out_raw, dat_out_oracle, num_flws = (gen_data( net_tmp, args, dat_flp, scl_prms_flp)) print(f"Number of input features: {len(dat_in.dtype.names)}") # Visualaize the ground truth data. utils.visualize_classes(net_tmp, dat_out) # TODO: Parallelize attempts. trls = args["conf_trials"] apts = 0 apts_max = args["max_attempts"] ress = [] while trls > 0 and apts < apts_max: apts += 1 res = (run_sklearn if isinstance(net_tmp, models.SvmSklearnWrapper) else run_torch)(args, dat_in, dat_out, dat_out_raw, dat_out_oracle, num_flws, out_dir, out_flp) if res[0] == 100: print(( f"Training failed (attempt {apts}/{apts_max}). Trying again!")) else: ress.append(res) trls -= 1 if ress: print(("Resulting accuracies: " f"{', '.join([f'{acc:.2f}' for acc, _ in ress])}")) max_acc, tim_s = max(ress, key=lambda p: p[0]) print(f"Maximum accuracy: {max_acc:.2f}") # Return the minimum error instead of the maximum accuracy. return 1 - max_acc, tim_s print(f"Model cannot be trained with args: {args}") return float("NaN"), float("NaN")
def FID(img): with torch.no_grad(): batch_size = 32 cfg['batch_size']['train'] = batch_size dataset = fetch_dataset(cfg['data_name'], cfg['subset'], verbose=False) real_data_loader = make_data_loader(dataset)['train'] generated_data_loader = DataLoader(img, batch_size=batch_size) if cfg['data_name'] in ['COIL100', 'Omniglot']: model = models.classifier().to(cfg['device']) model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier'] model_tag = '_'.join(filter(None, model_tag)) checkpoint = load( './metrics_tf/res/classifier/{}_best.pt'.format(model_tag)) model.load_state_dict(checkpoint['model_dict']) model.train(False) real_feature = [] for i, input in enumerate(real_data_loader): input = collate(input) input = to_device(input, cfg['device']) real_feature_i = model.feature(input) real_feature.append(real_feature_i.cpu().numpy()) real_feature = np.concatenate(real_feature, axis=0) generated_feature = [] for i, input in enumerate(generated_data_loader): input = { 'img': input, 'label': input.new_zeros(input.size(0)).long() } input = to_device(input, cfg['device']) generated_feature_i = model.feature(input) generated_feature.append(generated_feature_i.cpu().numpy()) generated_feature = np.concatenate(generated_feature, axis=0) else: model = inception_v3(pretrained=True, transform_input=False).to(cfg['device']) up = nn.Upsample(size=(299, 299), mode='bilinear', align_corners=False) model.feature = nn.Sequential(*[ up, model.Conv2d_1a_3x3, model.Conv2d_2a_3x3, model.Conv2d_2b_3x3, nn.MaxPool2d(kernel_size=3, stride=2), model.Conv2d_3b_1x1, model.Conv2d_4a_3x3, nn.MaxPool2d(kernel_size=3, stride=2), model.Mixed_5b, model.Mixed_5c, model.Mixed_5d, model.Mixed_6a, model.Mixed_6b, model.Mixed_6c, model.Mixed_6d, model.Mixed_6e, model.Mixed_7a, model.Mixed_7b, model.Mixed_7c, nn.AdaptiveAvgPool2d(1), nn.Flatten() ]) model.train(False) real_feature = [] for i, input in enumerate(real_data_loader): input = collate(input) input = to_device(input, cfg['device']) real_feature_i = model.feature(input['img']) real_feature.append(real_feature_i.cpu().numpy()) real_feature = np.concatenate(real_feature, axis=0) generated_feature = [] for i, input in enumerate(generated_data_loader): input = to_device(input, cfg['device']) generated_feature_i = model.feature(input) generated_feature.append(generated_feature_i.cpu().numpy()) generated_feature = np.concatenate(generated_feature, axis=0) mu1 = np.mean(real_feature, axis=0) sigma1 = np.cov(real_feature, rowvar=False) mu2 = np.mean(generated_feature, axis=0) sigma2 = np.cov(generated_feature, rowvar=False) mu1 = np.atleast_1d(mu1) mu2 = np.atleast_1d(mu2) sigma1 = np.atleast_2d(sigma1) sigma2 = np.atleast_2d(sigma2) assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths" assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions" diff = mu1 - mu2 # product might be almost singular covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False) if not np.isfinite(covmean).all(): offset = np.eye(sigma1.shape[0]) * 1e-6 covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset)) # numerical error might give slight imaginary component if np.iscomplexobj(covmean): if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3): m = np.max(np.abs(covmean.imag)) raise ValueError("Imaginary component {}".format(m)) covmean = covmean.real tr_covmean = np.trace(covmean) fid = diff.dot(diff) + np.trace(sigma1) + np.trace( sigma2) - 2 * tr_covmean fid = fid.item() return fid
def load(self): utils.load(self.model, os.path.join(self.args.save, 'weights.pt'))
#%% import re from typing import Callable from utils import load raw = load("day04.txt", split="\n\n") # %% def test1() -> list[str]: valid: list[str] = list() for line in raw: test = ( len(re.findall("[bie]yr", line)) == 3, re.findall("hgt", line), len(re.findall("[he]cl", line)) == 2, re.findall("pid", line), ) if all(test): valid.append(line) assert len(valid) == 190 return valid # %% def proc_hgt(x: str) -> bool: if x.endswith("cm"): return 150 <= int(x[:-2]) <= 193 elif x.endswith("in"): return 59 <= int(x[:-2]) <= 76
else: raise NotImplementedError return y, x if __name__ == '__main__': import os import anyconfig from utils import parse_config, load, get_parameter_number config = anyconfig.load( open("config/imagedataset_None_VGG_RNN_CTC.yaml", 'rb')) if 'base' in config: config = parse_config(config) if os.path.isfile(config['dataset']['alphabet']): config['dataset']['alphabet'] = load(config['dataset']['alphabet']) device = torch.device('cpu') config['arch']['backbone']['in_channels'] = 3 config['arch']['head']['n_class'] = 95 net = Model(config['arch']).to(device) print(net.name, len(config['dataset']['alphabet'])) a = torch.randn(2, 3, 32, 320).to(device) import time text_for_pred = torch.LongTensor(2, 25 + 1).fill_(0) tic = time.time() for i in range(1): b = net(a, text_for_pred)[0] print(b.shape)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Tue Nov 27 23:44:00 2018 @author: dhruvinpatel """ from sklearn.cluster import DBSCAN import numpy as np from utils import load, cal_sse import matplotlib.pyplot as plt text_vectors = load('text_vectors.p') clustering = DBSCAN(eps=.5, min_samples=10,metric='cosine').fit(text_vectors) labels = clustering.labels_ n_labels = set(labels) clusters = {} for n in n_labels: if n == -1: continue else: clusters[n] = [] for i in range(len(labels)): if labels[i] == -1: continue else: clist = clusters[labels[i]] clist.append(text_vectors[i])
import os from os import path import utils dumpdir = path.join('..', 'dumps') try: data = utils.load(path.join(dumpdir, 'datadict.pkl')) ingredient_dict = utils.load( path.join(dumpdir, 'ingredient_dictionary.pkl')) except Exception as e: print( "pkl files not found. Please run 'ingredient_dictionary.py' first. Exiting..." ) exit(1) def ranker(input_ingredients): def printer(*args): print("ranker: ", end="") print(*args) unigrams = set(input_ingredients) candidates = [(ingredient_dict[uni], len(ingredient_dict[uni])) for uni in unigrams if uni in ingredient_dict] candidate_scores = {} for candidate_list, score in candidates: for candidate in candidate_list: if candidate not in candidate_scores: candidate_scores[candidate] = 0.0 candidate_scores[candidate] += 1.0 / score
def load_model(self, model_path): self.Q = utils.load(model_path)
Command: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python kaggle_otto_nn.py On EC2 g2.2xlarge instance: 19s/epoch. 6-7 minutes total training time. Best validation score at epoch 21: 0.4881 Try it at home: - with/without BatchNormalization (BatchNormalization helps!) - with ReLU or with PReLU (PReLU helps!) - with smaller layers, largers layers - with more layers, less layers - with different optimizers (SGD+momentum+decay is probably better than Adam!) ''' np.random.seed(1337) # for reproducibility sort_idx = ut.load("sort_idx") sort_idx = sort_idx[-93:] print(type(sort_idx)) print(sort_idx.shape) ## check if raw data exist if os.path.isfile("./tmp/X-extra-features-0"): print ("Loading existing data (with extra features)...") #print(X.shape) for i in range(6): print("Loading subset %d"%i) objname = "X-extra-features-%d"%i labelsname = "labels-%d"%i if i==0:
raw_path = params['RAW_MODEL_PATH'] spectro_path = params['SPECTRUM_MODEL_PATH'] test_dataset_path = params['TEST_PICKLE'] batch_size = BATCH_SIZE try: with open(params['DICT_JSON'], mode='r', encoding='utf-8') as fin: class_dict = json.load(fin) except Exception as e: print(e) print("We have lost correpsondances, aborting...") sys.exit(0) test_set = load(test_dataset_path) if test_set is None: print("No Test data, aborting...") sys.exit(0) random.shuffle(test_set) test_lens = get_class_numbers(test_set, class_dict) test_data = get_reduced_set(test_set, test_lens, 'min') def _DScnn(x, rawnet, spectronet): c1 = rawnet(x[0]) c2 = spectronet(x[1]) c1 = tf.nn.softmax(c1) c2 = tf.nn.softmax(c2)
def run(self, restore=False): self.g_init() if not self.journal: utils.load() load_save.retrieve() self.aim = simon.Simon(1200) # arg is glow time if self.sugar: self.set_delay(800) else: self.set_delay() self.player = simon.Simon(200) if restore: self.restore_pattern() self.aim.started = True if self.sugar: self.green_button.set_sensitive(True) self.back_button.set_sensitive(False) else: bx = g.sx(22.42) by = g.sy(20.8) buttons.Button('green', (bx, by), True) buttons.Button('back', (bx, by), True) buttons.off('back') self.slider = slider.Slider(g.sx(9), g.sy(20.8), 3, utils.BLUE) self.rc = rc_skip_last.RC(3, 5) if self.canvas is not None: self.canvas.grab_focus() ctrl = False pygame.key.set_repeat(600, 120) key_ms = pygame.time.get_ticks() going = True while going: if self.journal: # Pump GTK messages. while gtk.events_pending(): gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: if not self.journal: utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.pos = event.pos g.redraw = True if self.canvas is not None: self.canvas.grab_focus() self.mouse_set() elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True if event.button == 1: self.do_click() self.flush_queue() elif event.type == pygame.KEYDOWN: # throttle keyboard repeat if pygame.time.get_ticks() - key_ms > 110: key_ms = pygame.time.get_ticks() if ctrl: if event.key == pygame.K_q: if not self.journal: utils.save() going = False break else: ctrl = False if event.key in (pygame.K_LCTRL, pygame.K_RCTRL): ctrl = True break self.do_key(event.key) g.redraw = True self.flush_queue() elif event.type == pygame.KEYUP: ctrl = False if not going: break if self.sugar: if g.player_n == 0 and not self.green_button.get_sensitive(): self.back_button.set_sensitive(True) else: if g.player_n == 0 and not buttons.active('green'): buttons.on('back') self.player.do() self.aim.do() if g.redraw: self.display() if g.version_display: utils.version_display() if self.aim.running or self.aim.glow_active: pass else: g.screen.blit(g.pointer, g.pos) pygame.display.flip() g.redraw = False g.clock.tick(40)
def run(self, restore=False): self.black = False g.init() if not self.journal: utils.load() load_save.retrieve() x = g.sx(26) y = g.sy(11.2) if not self.sugar: buttons.Button("new", (x, y)) x, y = g.cxy2 dx = g.sy(4) self.back_button = buttons.Button("back", (x, y)) x += dx buttons.Button("plus", (x, y)) x += dx buttons.Button("times", (x, y)) x += dx buttons.Button("equals", (x, y)) self.ops = ['back', 'plus', 'times', 'equals'] if not self.sugar: self.slider = slider.Slider(g.sx(22.4), g.sy(20.5), 10, utils.GREEN) self.mouse_auto = True self.anim_ms = None self.level1() # initial animation self.scored = False if restore: g.score = self.save_score ctrl = False pygame.key.set_repeat(600, 120) key_ms = pygame.time.get_ticks() going = True if self.canvas is not None: self.canvas.grab_focus() while going: if self.journal: # Pump GTK messages. while Gtk.events_pending(): Gtk.main_iteration() # Pump PyGame messages. for event in pygame.event.get(): if event.type == pygame.QUIT: if not self.journal: utils.save() going = False elif event.type == pygame.MOUSEMOTION: g.pos = event.pos g.redraw = True if self.canvas is not None: self.canvas.grab_focus() elif event.type == pygame.MOUSEBUTTONDOWN: g.redraw = True self.anim_end() if event.button == 1: bu = buttons.check() if bu == '': if not self.check_numbers(): if not self.sugar: if self.slider.mouse(): self.level1() else: self.do_button(bu) # eg do_button('plus') self.flush_queue() elif event.type == pygame.KEYDOWN: self.anim_end() # throttle keyboard repeat if pygame.time.get_ticks() - key_ms > 110: key_ms = pygame.time.get_ticks() if ctrl: if event.key == pygame.K_q: if not self.journal: utils.save() going = False break else: ctrl = False if event.key in (pygame.K_LCTRL, pygame.K_RCTRL): ctrl = True break self.do_key(event.key) g.redraw = True self.flush_queue() elif event.type == pygame.KEYUP: ctrl = False if not going: break self.animation() if g.redraw: self.display() if g.version_display: utils.version_display() if not self.black: g.screen.blit(g.pointer, g.pos) pygame.display.flip() g.redraw = False g.clock.tick(40)
#!/usr/bin/env python from utils import (load, take, show, bgr, image, like, bounds, channels, crop, scale, color, avail, colorPicker) from proto import alias, sharpen, group, find, edge, center, distance from PIL import Image print "# fast stuff" img = load('samples/abstract/colors.png') #b = take() show(img) b, g, r = bgr(img) img = image(b,b,b) test = like(img) bound = bounds(b) channel = channels(b) coord = (0,0,50,50) closer = crop(img, coord) bigger = scale(closer, 2.0) eyedrop = color(img, 0, 30) pallet = avail(img) colorPicker(img,0,30) print "# slow stuff" res1 = alias(img, .3) res2 = sharpen(img, .3) blob1 = group(img) mask = Image.new("RGB", (50, 10), "white") blob3 = find(img,mask,(3,3)) coords1 = edge(img) coords2 = center(blob1) dist = distance(0,3)
def _parse_embedding(self): return load(self.store_folder / self.pkl_file)
def main(args): # loading configurations with open(args.config) as f: config = yaml.safe_load(f)["configuration"] name = config["Name"] # Construct or load embeddings print("Initializing embeddings ...") vocab_size = config["embeddings"]["vocab_size"] embed_size = config["embeddings"]["embed_size"] embeddings = init_embeddings(vocab_size, embed_size, name=name) print("\tDone.") # Build the model and compute losses source_ids = tf.placeholder(tf.int32, [None, None], name="source") target_ids = tf.placeholder(tf.int32, [None, None], name="target") sequence_mask = tf.placeholder(tf.bool, [None, None], name="mask") attn_wrappers = { "None": None, "Attention": AttentionWrapper, } attn_wrapper = attn_wrappers.get(config["decoder"]["wrapper"]) (enc_num_layers, enc_num_units, enc_cell_type, enc_bidir, dec_num_layers, dec_num_units, dec_cell_type, state_pass, infer_batch_size, infer_type, beam_size, max_iter, attn_num_units, l2_regularize) = get_model_config(config) print("Building model architecture ...") CE, loss, logits, infer_outputs = compute_loss( source_ids, target_ids, sequence_mask, embeddings, enc_num_layers, enc_num_units, enc_cell_type, enc_bidir, dec_num_layers, dec_num_units, dec_cell_type, state_pass, infer_batch_size, infer_type, beam_size, max_iter, attn_wrapper, attn_num_units, l2_regularize, name) print("\tDone.") # Set up session restore_from = config["training"]["restore_from"] gpu_fraction = config["training"]["gpu_fraction"] gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction) sess = tf.Session(config=tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options)) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.trainable_variables()) try: saved_global_step = load(saver, sess, restore_from) if saved_global_step is None: raise ValueError("Cannot find the checkpoint to restore from.") except Exception: print("Something went wrong while restoring checkpoint. ") raise # ##### Inference ##### # Load data print("Loading inference data ...") # id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding embed_shift = 3 filename = config["inference"]["infer_source_file"] max_leng = config["inference"]["infer_source_max_length"] source_data = loadfile(filename, is_source=True, max_length=max_leng) + embed_shift print("\tDone.") # Inference print("Start inferring ...") final_result = [] n_data = source_data.shape[0] n_pad = n_data % infer_batch_size if n_pad > 0: n_pad = infer_batch_size - n_pad pad = np.zeros((n_pad, max_leng), dtype=np.int32) source_data = np.concatenate((source_data, pad)) for ith in range(int(len(source_data) / infer_batch_size)): start = ith * infer_batch_size end = (ith + 1) * infer_batch_size batch = source_data[start:end] result = sess.run(infer_outputs, feed_dict={source_ids: batch}) result = result.ids[:, :, 0] if result.shape[1] < max_iter: l_pad = max_iter - result.shape[1] result = np.concatenate( (result, np.ones((infer_batch_size, l_pad))), axis=1) final_result.append(result) final_result = np.concatenate(final_result)[:n_data] - embed_shift final_result[final_result < 0] = -1 final_result = final_result.astype(str).tolist() final_result = list(map(lambda t: " ".join(t), final_result)) df = pd.DataFrame(data={"0": final_result}) df.to_csv(config["inference"]["output_path"], header=None, index=None) print("\tDone.")
import os import utils from processing import calculate_relations from models import Theme, Teacher, Student if __name__ == '__main__': os.makedirs('output', exist_ok=True) # Загрузка тем themes_data = utils.load('../data/themes.csv') themes = utils.extract_models(Theme, themes_data) utils.print_multiline(themes) # Загрузка преподавателей teachers_data = utils.load('../data/teachers.csv') teachers = utils.extract_models(Teacher, teachers_data) utils.print_multiline(teachers[:10]) # Загрузка студентов students_data = utils.load('../data/students.csv') students = utils.extract_models(Student, students_data) utils.print_multiline(students[:10]) students, teachers, lines = calculate_relations(students, teachers) utils.dump(lines, 'output/relations.csv') print('- ' * 30) for idx, _teacher in enumerate(teachers):