def demux(self): ''' Demux video, audio, subs Return an object with the filenames ''' src_dir_series = os.path.join(self.src_dir_top, self.series) for r in self._regions(): if r not in self.demux_map: continue src_dir = os.path.join(src_dir_series, r) dest_dir = os.path.join(self.temp_dir, r) create_dir(dest_dir) logger.info('Demuxing %s %s %s...', self.series, self.number, r) self.files[r] = demux(self, src_dir, dest_dir, self.demux_map[r], novid=((r == 'R1' or r == 'PIONEER') and not self.demux_r1_vid), nosub=(r == 'R2'), sub_only=self.sub_only, orange_brick=( r == 'R1' and self.series == 'DBZ' and not self.is_special)) if not self.sub_only and 'R2' in self.files: self.r2_chapters = _load_r2_chapters( self.files['R2']['chapters'][0], self.series, self.is_special)
def extract(self): batch_time = utils.AverageMeter() self.model.eval() end = time.time() for batch_idx, (imgs, target, img_files, class_ids) in tqdm.tqdm( enumerate(self.val_loader), total=len(self.val_loader), desc='Extract', ncols=80, leave=False): gc.collect() if self.cuda: imgs = imgs.cuda() imgs = Variable(imgs, volatile=True) output = self.model(imgs) # N C H W torch.Size([1, 1, 401, 600]) if self.flatten_feature: output = output.view(output.size(0), -1) output = output.data.cpu().numpy() assert output.shape[0] == len(img_files) for i, img_file in enumerate(img_files): base_name = os.path.splitext(img_file)[0] feature_file = os.path.join(self.feature_dir, base_name + ".npy") utils.create_dir(os.path.dirname(feature_file)) np.save(feature_file, output[i]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % self.print_freq == 0: log_str = 'Extract: [{0}/{1}]\tTime: {batch_time.val:.3f} ({batch_time.avg:.3f})'.format( batch_idx, len(self.val_loader), batch_time=batch_time) print(log_str) self.print_log(log_str)
def generate_lgs(inkmls, path): create_dir(path) path = path + '/' if not path.endswith('/') else path for inkml in inkmls: fname = inkml.src.rstrip('inkml') + "lg" fname = path + os.path.basename(fname) open(fname, 'w+').write(inkml.get_lg())
def download_from_url(client_id, url, base_dir, override=False): """Download the given playlist""" downloaded = 0 skipped = 0 errors = 0 # Retrieve playlist data client = soundcloud.Client(client_id=client_id) playlist = client.get('/resolve', url=url) # Create dir playlist_title = playlist.title dir = os.path.join(base_dir, playlist_title) utils.create_dir(dir) # Download tracks for trak in playlist.tracks: try: #done = song.down(client, track, dir, override) done = track.download_from_id(client_id, trak['id'], dir, override) if done: downloaded = downloaded + 1 else: skipped = skipped + 1 except requests.exceptions.HTTPError, err: if err.response.status_code == 404: print 'Error: could not download' errors = errors + 1 else: raise
def test_one_dir(self): utils.create_dir("dir1") response = self.client.request("GET", "/") (status, reason, body, headers) = response self.assertEqual(200, status) body = eval(body) self.assertTrue(isinstance(body,dict)) body = body["dir"] self.assertEqual(1, len(body)) self.assertEqual("dir1", body[0]["name"]) self.assertEqual(True, body[0]["is_dir"]) response = self.client.request("GET", "/dir1/") (status, reason, body, headers) = response self.assertEqual(200, status) body = eval(body) body = body["dir"] self.assertTrue(isinstance(body,list)) self.assertEqual(0, len(body))
def compute_target(answers_dset, ans2label, name, cache_root='data/cache'): """Augment answers_dset with soft score as label ***answers_dset should be preprocessed*** Write result into a cache file """ target = [] for ans_entry in answers_dset: answers = ans_entry['answers'] answer_count = {} for answer in answers: answer_ = answer['answer'] answer_count[answer_] = answer_count.get(answer_, 0) + 1 labels = [] scores = [] for answer in answer_count: if answer not in ans2label: continue labels.append(ans2label[answer]) score = get_score(answer_count[answer]) scores.append(score) target.append({ 'question_id': ans_entry['question_id'], 'image_id': ans_entry['image_id'], 'labels': labels, 'scores': scores }) utils.create_dir(cache_root) cache_file = os.path.join(cache_root, name+'_target.pkl') cPickle.dump(target, open(cache_file, 'wb')) return target
def init_visit_dir(self): """Create results and logs directories for this visit.""" visit_name = str(self.instance_num) self.visit_dir = os.path.join(self.base_dir, visit_name) ut.create_dir(self.visit_dir) self.visit_log_dir = os.path.join(self.visit_dir, 'logs') ut.create_dir(self.visit_log_dir)
def save(config, data): """Save data and create backup, creating a new data file if necessary.""" create_dir(os.path.dirname(config['data_path'])) # atomically save by writing to temporary file and moving to destination try: # write to temp file with open( config['tmp_path'], 'w', encoding='utf-8', errors='replace') as f: for path, weight in data.items(): if is_python3(): f.write(("%s\t%s\n" % (weight, path))) else: f.write(unicode( "%s\t%s\n" % (weight, path)).encode('utf-8')) f.flush() os.fsync(f) except IOError as ex: print("Error saving autojump data (disk full?)" % ex, file=sys.stderr) sys.exit(1) # create backup file if it doesn't exist or is older than BACKUP_THRESHOLD if not os.path.exists(config['backup_path']) or \ (time() - os.path.getmtime(config['backup_path']) > BACKUP_THRESHOLD): move_file(config['data_path'], config['backup_path']) # move temp_file -> autojump.txt move_file(config['tmp_path'], config['data_path'])
def add_sim_modules_to_project(tags, sim_dict, user_paths): #utils.pretty_print_dict(tags) #utils.pretty_print_dict(sim_dict) #Get the directory of where to put the sim modules base_dir = utils.resolve_path(tags["BASE_DIR"]) project_dir = tags["PROJECT_NAME"] out_dir = os.path.join(base_dir, "sim", "sim_modules") if not os.path.exists(out_dir): utils.create_dir(out_dir) #Find all the file locations module_filename = utils.find_module_filename(sim_dict["name"], user_paths) module_filepath = utils.find_rtl_file_location(module_filename, user_paths) out_file_path = os.path.join(out_dir, module_filename) #print "copy %s > %s" % (module_filepath, out_file_path) shutil.copy2(module_filepath, os.path.join(out_dir, out_file_path)) #Get the locations for each of the auxilary files for f in sim_dict["aux_files"]: module_path = utils.find_rtl_file_location(f) out_file_path = os.path.join(out_dir, f) #print "copy %s > %s" % (module_path, out_file_path) shutil.copy2(module_path, os.path.join(out_dir, f))
def run_STAR(args): read1 = ','.join(args.read1) read2 = ','.join(args.read2) output_dir = args.outdir + '/' + args.prefix + '/' utils.create_dir(output_dir) STAR_args = ['STAR', '--runThreadN', args.cores, '--genomeDir', args.genome_index, '--outFileNamePrefix ',output_dir , '--outSAMunmapped', 'Within'] if args.outdir is None: args.outdir = 'STARmapped_%s_out' % time.strftime('%Y-%m-%d:%H-%M-%S') if args.optional_args is not None: STAR_args.extend([args.optional_args]) STAR_args.extend(['--readFilesIn',read1, read2 ]) #to make sure we convert all the args to string STAR_args = map(str,STAR_args) #run STAR utils.run_external_command(STAR_args, logger=args.logger) #expected output sam file expected_sam_file = output_dir + 'Aligned.out.sam' #convert sam to bam bamFile = bamUtils.sam_to_bam(expected_sam_file, delete_sam=True)
def setup_crawl_dirs(self, test_url=TEST_URL): crawl_name = ut.append_timestamp("crawl") self.crawl_dir = ut.create_dir(join(cm.TEST_FILES_DIR, crawl_name)) batch_dir = ut.create_dir(join(self.crawl_dir, str(self.batch_num))) self.site_dir = ut.create_dir(join(batch_dir, ut.get_filename_from_url(test_url, self.site_num)))
def write_file(self, location = "", filename=""): """write_file Search through the specified location, if the location doesn't exist then create the location. then write out the specified file Args: location: the location where the file is to be written filename: the name of the output file to write Returns: Nothing Raises: IOError """ home = False location = utils.resolve_path(location) if not os.path.exists(location): utils.create_dir(location) fname = os.path.join(location, filename) fileout = open(fname, "w") fileout.write(self.buf) fileout.close() return
def __init__(self, api_key = "", output_dir = "", text_list_file = "", *args, **kwargs): super(GetTextsSpider, self).__init__(*args, **kwargs) utils.create_dir(output_dir) if len(api_key) == 0: raise scrapy.exceptions.CloseSpider("expected 'api_key' to be a string") if len(output_dir) == 0: raise scrapy.exceptions.CloseSpider("expected 'output_dir' to be a string") if len(text_list_file) == 0: raise scrapy.exceptions.CloseSpider("expected 'text_list_file' to be a string") self.api_key = api_key self.output_dir = output_dir with open(text_list_file, "r") as f: self.all_texts = json.load(f)['texts'] if utils.is_file_exists(output_dir+"/master_record.json"): with open(output_dir+"/master_record.json", "r") as f: json_file = json.load(f) self.master_record = json_file["master_record"] self.last_unallocated_number = json_file["last_unallocated_number"] else: self.master_record = {text["gid"]: "" for text in self.all_texts} self.last_unallocated_number = 1 self.text_name_prefix = "Guardian" self.text_name_length = int(math.floor(math.log(len(self.all_texts))/math.log(10)) + 1)
def main(): colorama.init() config = load_config_file() args, wtf = create_args().parse_known_args() if (wtf): logger.error('Unknown argument %s', wtf[0]) sys.exit(1) init_logging(args.verbose) # don't proceed if paths aren't right/programs missing pre_check(args, config) try: working_dir = config.get(APP_NAME, 'working_dir') except configparser.Error: working_dir = None if working_dir: if not os.path.isdir(working_dir): create_dir(working_dir) tempfile.tempdir = working_dir tmp_dir = tempfile.mkdtemp() logger.debug('Episode temp folder: %s', tmp_dir) atexit.register(delete_temp, tmp_dir) start, end, special = validate_args(args) print(WELCOME_MSG) for ep in range(start, end + 1): start_time = time.clock() episode = Episode(ep, config, args, tmp_dir, special) if not args.no_demux: episode.demux() else: if args.sub_only: detect_streams(os.path.join(config.get(APP_NAME, 'output_dir'), args.series, str(ep if not special else special).zfill(3), 'R1', 'Subtitle.idx')) if not args.no_retime: episode.retime_subs() episode.retime_audio() if not args.no_demux and args.no_mux: # move files to destination folder episode.move_demuxed_files() if not args.no_mux: episode.mux() if args.make_avs: # only works on files generated with --no-mux episode.make_avs() delete_temp(episode.temp_dir) elapsed = time.clock() - start_time logger.debug('Elapsed time: %s seconds', elapsed) logger.info('Finished!')
def create_crawl_dir(self): """Create a timestamped crawl.""" ut.create_dir(self.output) # ensure that we've a results dir crawl_dir_wo_ts = os.path.join(self.output, 'crawl') crawl_dir = ut.create_dir(ut.append_timestamp(crawl_dir_wo_ts)) crawl_logs_dir = os.path.join(crawl_dir, 'logs') ut.create_dir(crawl_logs_dir) return crawl_dir, crawl_logs_dir
def get_profile_photos(browser, username): save_dir = 'profiles/' + username + '/' create_dir(save_dir) # need to fix these names i = 0 for image in browser.find_elements_by_xpath('//div[@id="profile_thumbs"]//img[@src]'): image_url = image.get_attribute('src') save_path = save_dir + str(i) + '.webp' # probably a bad idea to assume the img format download_image(image_url, save_path) i += 1 # using 'i' is probably not the clean way to name these images
def setup_env(tbb_rec_ver=None): """Initialize the tbb directory and import TBB signing keys. Download recommended TBB version and verify it. """ import_tbb_signing_keys() ut.create_dir(TBB_BASE_DIR) if not tbb_rec_ver: tbb_rec_ver = get_recommended_tbb_version() download_tbb_tarball(tbb_rec_ver)
def test_dir(self): utils.create_dir("dir1") response = self.client.request("GET", "/dir1") (status, reason, body, headers) = response location = utils.get_header("location", headers) self.assertEqual(307, status) self.assertEqual("http://localhost:%s/dir1/" % utils.get_port(), location)
def __init__(self, number, config, args, tmp_dir, special): series = args.series self.is_movie = False if args.movie: series = 'MOVIES' number = series_to_movie(args.series, number) self.is_movie = True if special: ep_str = special self.is_special = True else: ep_str = str(number).zfill(pad_zeroes(series)) self.is_special = False if args.r1_dbox: frame_data, op_offset = load_frame_data('DBoxZ', ep_str) else: frame_data, op_offset = load_frame_data(series, ep_str) # config stuff self.pgcdemux = config.get(APP_NAME, 'pgcdemux') self.vsrip = config.get(APP_NAME, 'vsrip') self.delaycut = config.get(APP_NAME, 'delaycut') self.dgindex = config.get(APP_NAME, 'dgindex') self.mkvmerge = config.get(APP_NAME, 'mkvmerge') self.restream = config.get(APP_NAME, 'restream') self.src_dir_top = config.get(APP_NAME, 'source_dir') self.output_dir = config.get(APP_NAME, 'output_dir') # special flags self.is_r1dbox = args.series == 'DBZ' and args.r1_dbox self.is_pioneer = args.pioneer self.no_funi = args.no_funi self.demux_r1_vid = args.r1_vid self.verbose = args.verbose # options self.no_mux = args.no_mux self.sub_only = args.sub_only self.temp_dir = os.path.join(tmp_dir, ep_str) create_dir(self.temp_dir) self.number = ep_str self.series = series self.offsets = _combine_framedata(frame_data, op_offset) self.pioneer_offsets = None if not self.is_pioneer else load_frame_data(series, str(number+50))[0] self.r2_chapters = {} if not args.no_demux: self.demux_map = load_demux_map(series, ep_str) if self.is_r1dbox: self.demux_map['R1_DBOX'] = auto_detect(self) else: self.demux_map = {'R1': {'audio': ['en', 'jp']}} self.files = self._init_files() if args.no_demux else {}
def test_html(self): file1contents = "file 1 contents" utils.create_dir("dir1") utils.write_file("dir1/file1.txt", file1contents) response = self.client.request("GET", "/dir1/", {"Accept": "text/html"}) (status, reason, body, headers) = response content_type = utils.get_header("content-type", headers) self.assertEqual(200, status) self.assertEqual("text/html", content_type)
def download(client, track, dir, override=False): """Download a track using the given SC client""" title = fix_title(track.title, track.user['username']) print '"%s"' % title if not dir: dir = 'mp3' utils.create_dir(dir) file_name = utils.build_file_name(dir, title) if not override and os.path.exists(file_name): print "File already exists, skipped" return False stream_url = client.get(track.stream_url, allow_redirects=False) urllib.urlretrieve(stream_url.location, file_name) return True
def edit_ratio_histogram(self): utils.log('creating edit histogram %s' % self.lang) f_out = utils.create_dir('results/ratio_histograms') df = pd.read_csv(self.db_path) df.page_id = df.page_id.astype(float) df = df.loc[df['linked_id'] != None] df.linked_id = df.linked_id.astype(float) df = self.drop_dups(df) utils.log('dropped %s duplicates' % len(df.set_index('page_id',drop=False).index.get_duplicates())) df = df.drop_duplicates(subset='page_id',keep=False) if self.drop1: df = df.loc[(df['len'] > 1)] for r in self.revert: utils.log('%s %s' % (self.lang,r)) utils.log('%s pages' % len(df)) n0 = df.loc[(df['namespace'] == 0)].set_index('page_id',drop=False) n1 = df.loc[(df['namespace'] == 1)].set_index('linked_id',drop=False) utils.log('%s articles' % len(n0)) utils.log('%s talk' % len(n1)) ratio = n0[r].divide(n1[r],axis='index',fill_value=-1).to_frame() ratio.columns = ['ratio'] ratio.ratio = ratio.ratio.astype(int) ratio = n0.join(ratio).set_index('page_id') ratio = ratio.loc[ratio['ratio'] >= 0] utils.log('%s ratios' % len(ratio)) result = ratio['ratio'].value_counts().to_frame() result = result.sort_index(ascending=True) result.columns = ['pages'] result.to_csv('%s/%s_%s.csv' % (f_out,self.lang,r),encoding='utf-8',index_label='edit_ratio')
def _create_dex_PRE(self): if self._check_classes() and \ (True if self.dx_bin else False) and \ (True if self.project_path else False): # Create bin folder if doesn't exist create_dir(os.path.join(self.project_path, 'bin')) log.info('Generating DEX executable') if self._need_to_re_compile_java_classes() or \ not self._check_dex(): self._create_dex_task() else: log.info('No new/modified classes to re-generate DEX executable') self._create_dex_POST() else: log.warn('Missing compiled classes and/or build tools!')
def __do_instance(self): for self.job.visit in xrange(self.job.visits): ut.create_dir(self.job.path) wl_log.info("*** Visit #%s to %s ***", self.job.visit, self.job.url) with self.driver.launch(): try: self.driver.set_page_load_timeout(cm.SOFT_VISIT_TIMEOUT) except WebDriverException as seto_exc: wl_log.error("Setting soft timeout %s", seto_exc) self.__do_visit() if self.screenshots: try: self.driver.get_screenshot_as_file(self.job.png_file) except WebDriverException: wl_log.error("Cannot get screenshot.") sleep(float(self.job.config['pause_between_visits'])) self.post_visit()
def _compile_java_code_PRE(self): if self._check_sources() and \ (True if self.javac_bin else False) and \ (True if self.project_path else False) and \ (True if self.android_jar else False): # Create obj folder if doesn't exist create_dir(os.path.join(self.project_path, 'obj')) log.info('Compiling java source') if self._need_to_re_compile_java_sources() or \ not self._check_classes(): self._compile_java_code_task() else: log.info('No new/modified sources to re-compile sources') self._compile_java_code_POST() else: log.warn('Missing sources and/or build tools!')
def test_one_file_in_dir(self): file1contents = "file 1 contents" utils.create_dir("dir1") utils.write_file("dir1/file1.txt", file1contents) response = self.client.request("GET", "/dir1/") (status, reason, body, headers) = response self.assertEqual(200, status) body = eval(body) body = body["dir"] self.assertTrue(isinstance(body,list)) self.assertEqual(1, len(body)) self.assertEqual("file1.txt", body[0]["name"]) self.assertEqual(False, body[0]["is_dir"])
def render_flame_file(batch_name, output_path='./output/'): u""" Dado un nombre de lote y una ruta de output opcional, renderiza el archivo flame con flam3 y guarda las imágenes resultates en la ruta definida por el output y el nombre del lote. :param batch_name: nombre del lote :param output_path: directorio de salida para las imágenes :return: boolean: Indica si la función ha tenido éxito """ # Crear la ruta de carpetas para el conjunto de flames que se van a renderizar current_batch_dir = output_path + batch_name + '/' create_dir(current_batch_dir) current_batch_flame_file = current_batch_dir + batch_name + '.flame' # Renderizar el archivo XML de flames con flam3 try: flame_file = open(current_batch_flame_file, 'r') except IOError: print 'Error: El archivo "' + current_batch_flame_file + '" no existe' return False else: # Establecer las variables de entorno para flam3 # Flam3 utiliza variables de entorno en lugar de parámetros envy = os.environ.copy() envy['prefix'] = current_batch_dir envy['name_enable'] = '0' # Debe ser cero para que la opción 'prefix' funcione envy['bits'] = '32' # Utilizar 32-bits integer # Por defecto 33 (32-bits flotante) y se generan # imágenes de muy baja calidad para 'quality' bajos # Llamar a flam3 y renderizar todos los flames process = subprocess.Popen('./flam3/flam3-render.exe', stdin=flame_file, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=envy) # Imprimir la salida de flam3 para ver que está pasando for e in iter(lambda: process.stderr.read(1), ''): sys.stdout.write(e) return True
def generate_session_pages(self): self.date_data = get_date_dataset() self.date_data.reverse() if self.fast_run: COUNTER = 0 for leg, sess, num, d, dpub, page_start, page_end in self.date_data: dateobj = parse_iso_date(d) session = get_session_from_legsessnum(leg, sess, num) if not session: log.warn("File for %s-%s-%s is missing from the transcripts dataset!" % (leg, sess, num)) continue target_dir = "%s%d/%02d/%02d" % (self.sessions_path, dateobj.year, dateobj.month, dateobj.day) filename = "%s/index.html" % target_dir info = get_session_info(leg, sess, num) create_dir(os.path.join(self.output_dir, target_dir)) if type(session) in (str, unicode): # sessão em texto simples context = {'date': dateobj, 'year_number': dateobj.year, 'leg': leg, 'sess': sess, 'num': num, 'text': session, 'monthnames': MESES, 'pdf_url': 'xpto', 'page_name': 'sessoes', } if info: context['session_info'] = info self.render_template_into_file('session_plaintext.html', filename, context) elif type(session) in (dict, OrderedDict): # usar entradas do .json como contexto session['date'] = parse_iso_date(session['session_date']) session['monthnames'] = MESES session['page_name'] = 'sessoes' self.render_template_into_file('session.html', filename, session) if self.fast_run: COUNTER += 1 if COUNTER > self.fast_run_count: break
def test_simple_in_dir(self): utils.create_dir("a-dir") file1contents = "file 1 contents" utils.write_file("a-dir/file.txt", file1contents) response = self.client.request("GET", "/a-dir/file.txt") (status, reason, body, headers) = response content_length = utils.get_header("content-length", headers) etag = utils.get_header("etag", headers) etag_pattern = re.compile(r'^"[^"]+"$') self.assertEqual(200, status) self.assertEqual(file1contents, body) self.assertEqual(len(file1contents), int(content_length)) self.assertTrue(etag != None) self.assertTrue(etag_pattern.match(etag) != None)
def __init__(self, molID=None, dset=None, run_name=None, tset_path=None): if molID and dset and not run_name and not tset_path: if not __class__._run_name or not __class__._tset_path: msg = 'run_name has to be specified before to instance a'\ ' useful class!' lg.critical(msg) raise RuntimeError(msg) self._inout_path = os.path.join(__class__._run_name, dset, 'inout', __class__._inout_id) self._inout_path = os.path.abspath(self._inout_path) create_dir(self._inout_path) self._xyzp = os.path.join(__class__._tset_path, dset, 'geometry', molID.split('.')[1] + '.xyz') self._inout_inp_path = os.path.join(self._inout_path, molID + '.inp') self._inout_out_path = os.path.join(self._inout_path, molID + '.log') self.molID = molID create_dir(config['densities_repo']) self._wb97x_saves = os.path.join(config['densities_repo'], self.molID + '.wb97x') self._ddsc_saves = os.path.join(config['densities_repo'], self.molID + '.ddsc') self._sbatch_file = \ os.path.join(config['sbatch_script_prefix'], self.molID) elif not molID and not dset and run_name and tset_path: __class__._run_name = run_name __class__._tset_path = tset_path else: msg = 'You cannot specify run_name, dset_path and molID all'\ ' together in Run' lg.critical(msg) raise AttributeError(msg)
def compute_coeff(airfoil, reynolds=500000, mach=0, alpha=3, n_iter=200, tmp_dir='tmp'): create_dir(tmp_dir) gc.collect() safe_remove('{}/airfoil.log'.format(tmp_dir)) fname = '{}/airfoil.dat'.format(tmp_dir) with open(fname, 'wb') as f: np.savetxt(f, airfoil) try: # Has error: Floating point exception (core dumped) # This is the "empty input file: 'tmp/airfoil.log'" warning in other approaches child = pexpect.spawn('xfoil') timeout = 10 child.expect('XFOIL c> ', timeout) child.sendline('load {}/airfoil.dat'.format(tmp_dir)) child.expect('Enter airfoil name s> ', timeout) child.sendline('af') child.expect('XFOIL c> ', timeout) child.sendline('OPER') child.expect('.OPERi c> ', timeout) child.sendline('VISC {}'.format(reynolds)) child.expect('.OPERv c> ', timeout) child.sendline('ITER {}'.format(n_iter)) child.expect('.OPERv c> ', timeout) child.sendline('MACH {}'.format(mach)) child.expect('.OPERv c> ', timeout) child.sendline('PACC') child.expect('Enter polar save filename OR <return> for no file s> ', timeout) child.sendline('{}/airfoil.log'.format(tmp_dir)) child.expect('Enter polar dump filename OR <return> for no file s> ', timeout) child.sendline() child.expect('.OPERva c> ', timeout) child.sendline('ALFA {}'.format(alpha)) child.expect('.OPERva c> ', timeout) child.sendline() child.expect('XFOIL c> ', timeout) child.sendline('quit') child.expect(pexpect.EOF) child.close() res = np.loadtxt('{}/airfoil.log'.format(tmp_dir), skiprows=12) if len(res) == 9: CL = res[1] CD = res[2] else: CL = np.nan CD = np.nan except Exception as ex: # print(ex) print('XFoil error!') CL = np.nan CD = np.nan safe_remove(':00.bl') return CL, CD
:param lines: list of ranges, [article1_start, article1_end, article2_start, article2_end, ...] :return: vector with binary values, where 1 is on position of the topic (article) change """ # dimension of y_ is the index of last article line -1 (-1 because there can be n-1 topic changes, # where n is number of rows in the prediction matrix) dim = lines[-1] - 1 y_ = np.zeros((dim, 1)) for i in range(1, len(lines) - 1, 2): y_[lines[i] - 1] = 1 return y_ def get_next_data(data_names): for name in data_names: yield config.get_seg_data(name) if __name__ == '__main__': print('Loading the instances') classifier = load_pickle(config.classifier) for data in get_next_data(config.data.keys()): print('Processing ' + data['name'] + ' data') create_dir(data['dir']) y_true = line_map_to_y(load_pickle(data['line_map'])) print('Saving the data') np.save(data['y_true_lm'], y_true)
TF_LOG_PATH = os.path.join(TF_LOG_DIR, 'alphanum', str(TEST_IDX)) KERACT_PATH = os.path.join(KERACT_DIR, 'alphanum', str(TEST_IDX)) MODEL_PATH = os.path.join(MODEL_DIR, 'alphanum') MODEL_IMG_PATH = os.path.join(MODEL_PATH, f'model_{TEST_IDX:3}.png') MODEL_STRUCT_PATH = os.path.join(MODEL_PATH, f'model_{TEST_IDX:3}.h5') DATA_PATH = os.path.join(DATA_DIR, 'alphanum') TEST_DATA_PATH = os.path.join(DATA_PATH, 'test') TRAIN_DATA_PATH = os.path.join(DATA_PATH, 'train') create_clean_dir(TF_LOG_PATH) create_clean_dir(KERACT_PATH) create_dir(MODEL_PATH) model = Sequential() model.add( Conv2D(filters=12, kernel_size=(3, 3), strides=(1, 1), padding='valid', input_shape=(CHAR_IMG_HEIGHT, CHAR_IMG_WIDTH, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2), padding='valid')) model.add(Flatten()) model.add(Dense(512, activation='sigmoid')) model.add(Dense(256, activation='sigmoid')) model.add(Dense(NUM_ALPHANUM_CHARS, activation='softmax')) model.compile(optimizer=SGD(lr=0.05),
from os import listdir, remove, path from shutil import rmtree from utils import load_json, call, check_skip, create_dir, extract, move import json import re config = load_json('cdn-config.json') print '[INFO] Rebuild CDN collection.' for target in config['targets']: print '[INFO] Collect %s libraries.' % target target_dir = target[target.find('/') + 1:] if target.find('/') > 0 else target create_dir("%s/%s" % (config['directory'], target_dir)) lib_info = call("bower info %s -j --allow-root" % target) if not lib_info: print '[ERROR] Cannot collect information about library' break versions = [ version for version in json.loads(lib_info)['versions'] if not check_skip(version, config['skipWords']) ] for version in versions: target_directory = "%s/%s/%s" % (config['directory'], target_dir, version) if not create_dir(target_directory) and listdir(target_directory):
from utils import first_option, create_dir, save_pickle if __name__ == '__main__': # Number of vectors in one sequence, input data structure [samples, time_steps, features] time_steps = 200 if Path(config.lstm_model_1).is_file() and first_option( 'Do you want to continue training the saved model?', 'y', 'n'): print("Loading new model") model = load_model(config.lstm_model_1) else: print("Building new model") model = build_model(time_steps, 1) # Create dir for histories create_dir(config.hist_dir) print("Loading the data") train = config.get_seg_data('train') held_out = config.get_seg_data('held_out') X_train_or = np.load(train['y']) y_train_or = np.load(train['y_true_lm']) X_held_out = np.load(held_out['y']) y_held_out = np.load(held_out['y_true_lm']) print("Computing the distances on test data") X_held_out = compute_distance(X_held_out, cosine_distances) # Split the 2D matrix to 3D matrix of dimensions [samples, time_steps, features]
parser = argparse.ArgumentParser(description='Transformer NER') # parser.add_argument('--corpus-data', type=str, default='../data/auto_only-nav-distance_BOI.txt', # help='path to corpus data') parser.add_argument('--save-dir', type=str, default='./data_char/', help='path to save processed data') parser.add_argument('--onnx-dir', type=str, default='./model_onnx/', help='path to save processed data') parser.add_argument('--pre-w2v', type=str, default='../data/w2v') args = parser.parse_args() create_dir(args.onnx_dir) pre_w2v = torch.load(args.save_dir + 'pre_w2v') pre_w2v = torch.Tensor(pre_w2v).to(device) model_ckpt = torch.load(os.path.join( args.save_dir, '{}.pyt'.format("Transformer_NER_best")), map_location=torch.device(device)) config = load_obj(args.save_dir + 'Config.json') model = Transformer_Mix(config, pre_w2v).to(device) model.load_state_dict(model_ckpt['model']) # Initialize the DataLoader data_loader = DataLoader_test(args.save_dir)
def train_coco(model, train_loader, eval_loader, num_epochs, output,s_epoch=0): lr_default=0.001 grad_clip = .25 utils.create_dir(output) lr_decay_step = 2 lr_decay_rate = .5 lr_decay_epochs = range(8, 12, lr_decay_step) gradual_warmup_steps = [0.5 * lr_default, 1.0 * lr_default,2.0*lr_default] optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) logger = utils.Logger(os.path.join(output, 'log.txt')) best_eval_score = 0 utils.print_model(model, logger) for epoch in range(s_epoch, num_epochs): total_loss = 0 train_score = 0 total_norm = 0 count_norm = 0 t = time.time() N = len(train_loader.dataset) if epoch < len(gradual_warmup_steps): optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch] logger.write('gradual warmup lr: %.4f' % optim.param_groups[0]['lr']) elif epoch in lr_decay_epochs: optim.param_groups[0]['lr'] *= lr_decay_rate logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr']) else: logger.write('lr: %.4f' % optim.param_groups[0]['lr']) for i, (v, b, q, a,question_id,image_id,types) in enumerate(train_loader): v = Variable(v).cuda() b = Variable(b).cuda() q = Variable(q).cuda() a = Variable(a).cuda() pred,att_1,att_2= model(v, b, q, a) loss = instance_bce_with_logits(pred, a) loss.backward() total_norm += nn.utils.clip_grad_norm(model.parameters(), grad_clip) count_norm += 1 optim.step() optim.zero_grad() batch_score = compute_score_with_logits(pred, a.data).sum() total_loss += loss.data[0] * v.size(0) train_score += batch_score total_loss /= N train_score = 100 * train_score / N if None != eval_loader: model.train(False) eval_score,eval_loss= evaluate(model, eval_loader) model.train(True) logger.write('epoch %d, time: %.2f' % (epoch, time.time()-t)) logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' % (total_loss, total_norm/count_norm, train_score)) if eval_loader is not None: logger.write('\teval score: %.2f,eval loss:%.2f' % (100 * eval_score,eval_loss)) if (eval_loader is not None and eval_score > best_eval_score) or (eval_loader is None and epoch>=0): model_path = os.path.join(output, 'model_epoch%d.pth' % epoch) utils.save_model(model_path, model, epoch, optim) if eval_loader is not None: best_eval_score = eval_score
def train(args, model, question_model, train_loader, eval_loader, s_opt=None, s_epoch=0): device = args.device model = model.to(device) question_model = question_model.to(device) # create packet for output utils.create_dir(args.output) # for every train, create a packet for saving .pth and .log run_timestamp = datetime.now().strftime("%Y%b%d-%H%M%S") ckpt_path = os.path.join(args.output, run_timestamp) utils.create_dir(ckpt_path) # create logger logger = utils.Logger(os.path.join(ckpt_path, 'medVQA.log')).get_logger() logger.info(">>>The net is:") logger.info(model) logger.info(">>>The args is:") logger.info(args.__repr__()) # Adamax optimizer optim = torch.optim.Adamax(params=model.parameters()) # Scheduler learning rate #lr_decay = lr_scheduler.CosineAnnealingLR(optim,T_max=len(train_loader)) # only fit for sgdr # Loss function criterion = torch.nn.BCEWithLogitsLoss() ae_criterion = torch.nn.MSELoss() best_eval_score = 0 best_epoch = 0 # Epoch passing in training phase for epoch in range(s_epoch, args.epochs): total_loss = 0 train_score = 0 number = 0 model.train() # Predicting and computing score for i, (v, q, a, answer_type, question_type, phrase_type, answer_target) in enumerate(train_loader): #lr_decay.step() optim.zero_grad() if args.maml: v[0] = v[0].reshape(v[0].shape[0], 84, 84).unsqueeze(1) v[0] = v[0].to(device) if args.autoencoder: v[1] = v[1].reshape(v[1].shape[0], 128, 128).unsqueeze(1) v[1] = v[1].to(device) if args.other_model: v = v.to(device) q = q.to(device) a = a.to(device) # MEVF loss computation if args.autoencoder: last_output_close, last_output_open, a_close, a_open, decoder = model( v, q, a, answer_target) else: last_output_close, last_output_open, a_close, a_open = model( v, q, a, answer_target) preds_close, preds_open = model.classify(last_output_close, last_output_open) #loss loss_close = criterion(preds_close.float(), a_close) loss_open = criterion(preds_open.float(), a_open) loss = loss_close + loss_open if args.autoencoder: loss_ae = ae_criterion(v[1], decoder) loss = loss + (loss_ae * args.ae_alpha) # loss /= answers.size()[0] loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 0.25) optim.step() #compute the acc for open and close batch_close_score = compute_score_with_logits( preds_close, a_close.data).sum() batch_open_score = compute_score_with_logits( preds_open, a_open.data).sum() total_loss += loss.item() train_score += batch_close_score + batch_open_score number += q.shape[0] total_loss /= len(train_loader) train_score = 100 * train_score / number logger.info('-------[Epoch]:{}-------'.format(epoch)) logger.info('[Train] Loss:{:.6f} , Train_Acc:{:.6f}%'.format( total_loss, train_score)) # Evaluation if eval_loader is not None: eval_score = evaluate_classifier(model, question_model, eval_loader, args, logger) if eval_score > best_eval_score: best_eval_score = eval_score best_epoch = epoch # Save the best acc epoch model_path = os.path.join(ckpt_path, '{}.pth'.format(best_epoch)) utils.save_model(model_path, model, best_epoch, optim) logger.info('[Result] The best acc is {:.6f}% at epoch {}'.format( best_eval_score, best_epoch))
def create_rate_subs(lang, mech_name, therm_name=None, last_spec=None): """Create rate subroutines from mechanism. Parameters ---------- lang : {'c', 'cuda', 'fortran', 'matlab'} Language type. mech_name : str Reaction mechanism filename (e.g. 'mech.dat'). therm_name : str, optional Thermodynamic database filename (e.g. 'therm.dat') or nothing if info in mechanism file. last_spec : str, optional If specified, the species to assign to the last index. Typically should be N2, Ar, He or another inert bath gas Returns ------- None """ lang = lang.lower() if lang not in utils.langs: print('Error: language needs to be one of: ') for l in utils.langs: print(l) sys.exit() if lang in ['fortran', 'matlab']: print('WARNING: Fortran and Matlab support incomplete.') # create output directory if none exists build_path = './out/' utils.create_dir(build_path) # Interpret reaction mechanism file, depending on Cantera or # Chemkin format. if mech_name.endswith(tuple(['.cti', '.xml'])): [elems, specs, reacs] = mech.read_mech_ct(mech_name) else: [elems, specs, reacs] = mech.read_mech(mech_name, therm_name) # Check to see if the last_spec is specified if last_spec is not None: # Find the index if possible isp = next((i for i, sp in enumerate(specs) if sp.name.lower() == last_spec.lower().strip()), None) if isp is None: print('Warning: User specified last species {} ' 'not found in mechanism.' ' Attempting to find a default species.'.format(last_spec)) last_spec = None else: last_spec = isp else: print('User specified last species not found or not specified. ' 'Attempting to find a default species') if last_spec is None: wt = chem.get_elem_wt() #check for N2, Ar, He, etc. candidates = [('N2', wt['n'] * 2.), ('Ar', wt['ar']), ('He', wt['he'])] for sp in candidates: match = next( (isp for isp, spec in enumerate(specs) if sp[0].lower() == spec.name.lower() and sp[1] == spec.mw), None) if match is not None: last_spec = match break if last_spec is not None: print('Default last species ' '{} found.'.format(specs[last_spec].name)) if last_spec is None: print('Warning: Neither a user specified or default last species ' 'could be found. Proceeding using the last species in the ' 'base mechanism: {}'.format(specs[-1].name)) last_spec = len(specs) - 1 # ordering of species and reactions not changed. fwd_rxn_mapping = range(len(reacs)) spec_maps = utils.get_species_mappings(len(specs), last_spec) fwd_spec_mapping, reverse_spec_mapping = spec_maps #pick up the last_spec and drop it at the end temp = specs[:] for i in range(len(specs)): specs[i] = temp[fwd_spec_mapping[i]] # Reassign reaction's product / reactant / third body list # to integer indexes for speed utils.reassign_species_lists(reacs, specs) ## Now begin writing subroutines # print reaction rate subroutine rate.write_rxn_rates(build_path, lang, specs, reacs, fwd_rxn_mapping) # if third-body/pressure-dependent reactions, # print modification subroutine if next((r for r in reacs if (r.thd_body or r.pdep)), None): rate.write_rxn_pressure_mod(build_path, lang, specs, reacs, fwd_rxn_mapping) # write species rates subroutine rate.write_spec_rates(build_path, lang, specs, reacs, fwd_spec_mapping, fwd_rxn_mapping) # write chem_utils subroutines rate.write_chem_utils(build_path, lang, specs) # write derivative subroutines rate.write_derivs(build_path, lang, specs, reacs) # write mass-mole fraction conversion subroutine rate.write_mass_mole(build_path, lang, specs) write_header(build_path, lang, specs) write_main(build_path, lang, specs) return 0
#!/usr/bin/env python import time import os import subprocess import itertools from utils import SLURM_TEMPLATE, create_dir SAVE_DIR = "/home/gridsan/apjacob/sweeps" JOB_SWEEP_NAME = "multitask_fixed_sweep_test" time = time.strftime("%Y%m%d-%H%M%S") job_output_dir = os.path.join(SAVE_DIR, f"{JOB_SWEEP_NAME}-{time}") print(f"Job output directory: {job_output_dir}") # Make top level directories create_dir(job_output_dir) # Sweep params rules = [3, 7, 12, 14, 21, 28, 80] rnns = ["zero", "rnn"] train_modes = ["coach", "executor", "both"] for rule, rnn, train_mode in itertools.product(rules, rnns, train_modes): job_name = f"rule-{rule}-rnns-{rnn}-train_mode-{train_mode}" job_file_name = os.path.join(job_output_dir, f"{job_name}.job") job_log_file = os.path.join(job_output_dir, f"{job_name}.log") if rnn == "zero" and train_mode != "executor": continue python_command = "python -u /home/gridsan/apjacob/minirts/scripts/self_play/multitask-fixed.py --coach1 rnn500 " \ "--coach2 rnn500 " \
def main(): opts = {} opts['random_seed'] = 66 opts['dataset'] = 'guitars' # gmm, circle_gmm, mnist, mnist3 ... opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images) opts['unrolling_steps'] = 5 # Used only if unrolled = True opts['data_dir'] = 'mnist' opts['trained_model_path'] = 'models' opts[ 'mnist_trained_model_file'] = 'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000' opts['gmm_max_val'] = 15. opts['toy_dataset_size'] = 10000 opts['toy_dataset_dim'] = 2 opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500 opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels opts['input_normalize_sym'] = True # Normalize data to [-1, 1] opts['adagan_steps_total'] = 3 opts['samples_per_component'] = 1000 # 50000 opts['work_dir'] = FLAGS.workdir opts['is_bagging'] = FLAGS.is_bagging opts['beta_heur'] = 'uniform' # uniform, constant opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk opts['beta_constant'] = 0.5 opts['topk_constant'] = 0.5 opts["init_std"] = FLAGS.init_std opts["init_bias"] = 0.0 opts['latent_space_distr'] = 'normal' # uniform, normal opts['optimizer'] = 'adam' # sgd, adam opts["batch_size"] = 64 opts["d_steps"] = 1 opts["g_steps"] = 1 opts["verbose"] = True opts['tf_run_batch_size'] = 100 opts['gmm_modes_num'] = 5 opts['latent_space_dim'] = FLAGS.zdim opts["gan_epoch_num"] = 50 opts["mixture_c_epoch_num"] = 5 opts['opt_learning_rate'] = FLAGS.learning_rate opts['opt_d_learning_rate'] = FLAGS.d_learning_rate opts['opt_g_learning_rate'] = FLAGS.g_learning_rate opts["opt_beta1"] = FLAGS.adam_beta1 opts['batch_norm_eps'] = 1e-05 opts['batch_norm_decay'] = 0.9 opts['d_num_filters'] = 64 opts['g_num_filters'] = 64 opts['conv_filters_dim'] = 4 opts["early_stop"] = -1 # set -1 to run normally opts["plot_every"] = 1 # set -1 to run normally opts["eval_points_num"] = 1000 # 25600 opts['digit_classification_threshold'] = 0.999 opts['inverse_metric'] = True # Use metric from the Unrolled GAN paper? opts['inverse_num'] = 64 # Number of real points to inverse. if opts['verbose']: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s') utils.create_dir(opts['work_dir']) with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text: text.write('Parameters:\n') for key in opts: text.write('%s : %s\n' % (key, opts[key])) data = DataHandler(opts) assert data.num_points >= opts['batch_size'], 'Training set too small' adagan = AdaGan(opts, data) metrics = Metrics() for step in range(opts["adagan_steps_total"]): logging.info('Running step {} of AdaGAN'.format(step + 1)) adagan.make_step(opts, data) num_fake = opts['eval_points_num'] logging.debug('Sampling fake points') fake_points = adagan.sample_mixture(num_fake) logging.debug('Sampling more fake points') more_fake_points = adagan.sample_mixture(500) logging.debug('Plotting results') if opts['dataset'] == 'gmm': metrics.make_plots(opts, step, data.data[:500], fake_points[0:100], adagan._data_weights[:500]) logging.debug('Evaluating results') (likelihood, C) = metrics.evaluate(opts, step, data.data[:500], fake_points, more_fake_points, prefix='') else: metrics.make_plots(opts, step, data.data, fake_points[:6 * 16], adagan._data_weights) logging.debug('Evaluating results') if opts['inverse_metric']: l2 = np.min(adagan._invert_losses[:step + 1], axis=0) logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2))) res = metrics.evaluate(opts, step, data.data[:500], fake_points, more_fake_points, prefix='') logging.debug("AdaGan finished working!")
print('# lambda0 = {}'.format(lambda0)) print('# lambda1 = {}'.format(lambda1)) print('# disc_lr = {}'.format(disc_lr)) print('# gen_lr = {}'.format(gen_lr)) print('#################################') # Read dataset data_fname = './data/xs_train.npy' perf_fname = './data/ys_train.npy' X = np.load(data_fname) Y = np.load(perf_fname) N = X.shape[0] assert N == Y.shape[0] # Prepare save directory create_dir('./trained_gan') save_dir = './trained_gan/{}_{}'.format(lambda0, lambda1) create_dir(save_dir) # print('Plotting training samples ...') # samples = X[np.random.choice(N, size=36, replace=False)] # plot_samples(None, samples, scale=1.0, scatter=False, lw=1.2, alpha=.7, c='k', fname='{}/samples'.format(save_dir)) # Train surrogate_dir = './surrogate/trained_surrogate' model = BezierGAN(latent_dim, noise_dim, X.shape[1], bezier_degree, bounds, lambda0, lambda1) if args.mode == 'train': safe_remove(save_dir) timer = ElapsedTimer() model.train(X, batch_size=batch_size, train_steps=train_steps, disc_lr=disc_lr, gen_lr=gen_lr, save_interval=save_interval, directory=save_dir, surrogate_dir=surrogate_dir)
img_degraded_wzwb = process(img_degraded, wb_chosen, ccm) img_label_wzwb = process(img_unprocessed[pad_length : H+pad_length, pad_length : W+pad_length, :], wb_chosen, ccm) """save the degradation results""" tifffile.imwrite(os.path.join(input_dir, (filename + '.tiff')), (img_degraded_wzwb * 65535).astype(np.uint16)) print('Image is saved in path: %s' % (os.path.join(input_dir, filename))) tifffile.imwrite(os.path.join(label_dir, (filename + '.tiff')), (img_label_wzwb * 65535).astype(np.uint16)) print('Image is saved in path: %s' % (os.path.join(label_dir, filename))) if __name__ == "__main__": # input image path # label8bit_dir = '~/train_datasets/label_8bit' label8bit_dir = '~/valid_datasets/label_8bit' # label raw path # labelraw_dir = '~/train_datasets/label_rgb' labelraw_dir = '~/valid_datasets/label_rgb' create_dir(labelraw_dir) # output image path # inputraw_dir = '~/train_datasets/input_rgb' inputraw_dir = '~/valid_datasets/input_rgb' create_dir(inputraw_dir) # kernel path kernel_path = '~/kernel/kernel.mat' # interval of the patch, 10pixels patch_itv = 50 # size of the patch, 100pixels patch_size = 150 # generate the image list img_list = sorted(glob.glob(label8bit_dir + '/*.png')) ccm = np.array([[ 1.93994141, -0.73925781, -0.20068359], [-0.28857422, 1.59741211, -0.30883789],
def main(): opts = {} # Utility opts['random_seed'] = 66 opts['dataset'] = 'cifar10' # gmm, circle_gmm, mnist, mnist3 ... opts['data_dir'] = 'cifar10' opts['trained_model_path'] = None #'models' opts[ 'mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000' opts['work_dir'] = FLAGS.workdir opts['ckpt_dir'] = 'checkpoints' opts["verbose"] = 1 opts['tf_run_batch_size'] = 128 opts["early_stop"] = -1 # set -1 to run normally opts["plot_every"] = 150 opts["save_every_epoch"] = 10 opts['gmm_max_val'] = 15. # Datasets opts['toy_dataset_size'] = 10000 opts['toy_dataset_dim'] = 2 opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500 opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels opts['input_normalize_sym'] = False # Normalize data to [-1, 1] opts['gmm_modes_num'] = 5 # AdaGAN parameters opts['adagan_steps_total'] = 1 opts['samples_per_component'] = 1000 opts['is_bagging'] = FLAGS.is_bagging opts['beta_heur'] = 'uniform' # uniform, constant opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk opts['beta_constant'] = 0.5 opts['topk_constant'] = 0.5 opts["mixture_c_epoch_num"] = 5 opts["eval_points_num"] = 25600 opts['digit_classification_threshold'] = 0.999 opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper? opts['inverse_num'] = 100 # Number of real points to inverse. opts['objective'] = None # Generative model parameters opts["init_std"] = FLAGS.init_std opts["init_bias"] = 0.0 opts['latent_space_distr'] = 'normal' # uniform, normal opts['latent_space_dim'] = FLAGS.zdim opts["gan_epoch_num"] = 100 opts['convolutions'] = True opts['d_num_filters'] = 512 opts['d_num_layers'] = 4 opts['g_num_filters'] = 1024 opts['g_num_layers'] = 4 opts['e_is_random'] = False opts['e_num_filters'] = 1024 opts['e_num_layers'] = 4 opts['g_arch'] = 'dcgan_mod' opts['g_stride1_deconv'] = False opts['g_3x3_conv'] = 0 opts['e_arch'] = 'dcgan' opts['e_3x3_conv'] = 0 opts['conv_filters_dim'] = 5 # --GAN specific: opts['conditional'] = False opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images) opts['unrolling_steps'] = 5 # Used only if unrolled = True # --VAE specific opts['vae'] = FLAGS.vae opts['vae_sigma'] = 0.01 # --POT specific opts['pot'] = FLAGS.pot opts['pot_pz_std'] = 2. opts['pot_lambda'] = FLAGS.pot_lambda opts['adv_c_loss'] = 'conv' opts['vgg_layer'] = 'pool2' opts['adv_c_patches_size'] = 5 opts['adv_c_num_units'] = 32 opts['adv_c_loss_w'] = 1.0 opts['cross_p_w'] = 0.0 opts['diag_p_w'] = 0.0 opts['emb_c_loss_w'] = 1.0 opts['reconstr_w'] = 0.0 opts['z_test'] = 'gan' opts['z_test_corr_w'] = 1.0 opts['z_test_proj_dim'] = 10 # Optimizer parameters opts['optimizer'] = 'adam' # sgd, adam opts["batch_size"] = 100 opts["d_steps"] = 1 opts['d_new_minibatch'] = False opts["g_steps"] = 2 opts['batch_norm'] = True opts['dropout'] = False opts['dropout_keep_prob'] = 0.5 opts['recon_loss'] = 'l2sq' # "manual" or number (float or int) giving the number of epochs to divide # the learning rate by 10 (converted into an exp decay per epoch). opts['decay_schedule'] = 'manual' opts['opt_learning_rate'] = FLAGS.learning_rate opts['opt_d_learning_rate'] = FLAGS.d_learning_rate opts['opt_g_learning_rate'] = FLAGS.g_learning_rate opts["opt_beta1"] = FLAGS.adam_beta1 opts['batch_norm_eps'] = 1e-05 opts['batch_norm_decay'] = 0.9 if opts['e_is_random']: assert opts['latent_space_distr'] == 'normal',\ 'Random encoders currently work only with Gaussian Pz' # Data augmentation opts['data_augm'] = True if opts['verbose']: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s') utils.create_dir(opts['work_dir']) utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir'])) with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text: text.write('Parameters:\n') for key in opts: text.write('%s : %s\n' % (key, opts[key])) data = DataHandler(opts) assert data.num_points >= opts['batch_size'], 'Training set too small' adagan = AdaGan(opts, data) metrics = Metrics() train_size = data.num_points random_idx = np.random.choice(train_size, 4 * 320, replace=False) metrics.make_plots(opts, 0, data.data, data.data[random_idx], adagan._data_weights, prefix='dataset_') for step in range(opts["adagan_steps_total"]): logging.info('Running step {} of AdaGAN'.format(step + 1)) adagan.make_step(opts, data) num_fake = opts['eval_points_num'] logging.debug('Sampling fake points') fake_points = adagan.sample_mixture(num_fake) logging.debug('Sampling more fake points') more_fake_points = adagan.sample_mixture(500) logging.debug('Plotting results') if opts['dataset'] == 'gmm': metrics.make_plots(opts, step, data.data[:500], fake_points[0:100], adagan._data_weights[:500]) logging.debug('Evaluating results') (likelihood, C) = metrics.evaluate(opts, step, data.data[:500], fake_points, more_fake_points, prefix='') else: metrics.make_plots(opts, step, data.data, fake_points[:320], adagan._data_weights) if opts['inverse_metric']: logging.debug('Evaluating results') l2 = np.min(adagan._invert_losses[:step + 1], axis=0) logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2))) res = metrics.evaluate(opts, step, data.data[:500], fake_points, more_fake_points, prefix='') logging.debug("AdaGan finished working!")
batch_size, shuffle=False, num_workers=4, collate_fn=trim_collate) else: train_loader = DataLoader(train_dset, batch_size, shuffle=True, num_workers=4, collate_fn=trim_collate) eval_loader = DataLoader(val_dset, batch_size, shuffle=False, num_workers=4, collate_fn=trim_collate) output_meta_folder = join(args.output, "regat_%s" % args.relation_type) utils.create_dir(output_meta_folder) args.output = output_meta_folder + "/%s_%s_%s_%d" % ( fusion_methods, args.relation_type, args.dataset, args.seed) if exists(args.output) and os.listdir(args.output): raise ValueError("Output directory ({}) already exists and is not " "empty.".format(args.output)) utils.create_dir(args.output) with open(join(args.output, 'hps.json'), 'w') as writer: json.dump(vars(args), writer, indent=4) logger = utils.Logger(join(args.output, 'log.txt')) train(model, train_loader, eval_loader, args, device)
"template_next_version": next_version, "template_reasons": substitutions.get("reasons", REASONS), "template_remove_examples": substitutions.get("remove", ""), "template_new_features": substitutions.get("new_features", ""), } # Do the substitution result = src.safe_substitute(d) try: # 1 January 2020 eol = datetime.datetime.strptime(substitutions["template_eol"], "%d %B %Y") except ValueError: # 2020-01-01 eol = datetime.datetime.strptime(substitutions["template_eol"], "%Y-%m-%d") # EOL in the future? if now < eol: result = result.replace("about time", "soon time") result = result.replace(" reached the ", " reaches the ") # print(result) # Save it outfile = os.path.join(version, "index.html") print(outfile) create_dir(version) with open(outfile, "w") as f: f.write(result)
def __init__(self, num_visible, num_hidden, batch_size, trans_func=tf.nn.sigmoid, num_labels=5, restore_dir=None, name='rbm'): self.num_visible = num_visible self.num_hidden = num_hidden self.num_labels = num_labels self.trans_func = trans_func self.name = name self.weights = self._initialize_weights() # v is used for both cd and generate hidden states self.v = tf.placeholder(tf.float32, [None, num_visible], name='v') # self.h = tf.placeholder(tf.float32, [None, num_hidden], name='h') self.one = tf.placeholder(tf.float32, [None, num_hidden], name='1fe') self.vrand = tf.placeholder(tf.float32, [None, num_visible], name='vrand') self.hrand = tf.placeholder(tf.float32, [None, num_hidden], name='hrand') self.avg_tfe = tf.placeholder(tf.float32, name='avg_tfe') self.avg_vfe = tf.placeholder(tf.float32, name='avg_vfe') self.reconstruct_loss = tf.placeholder(tf.float32, name='rloss') self.lr = tf.placeholder(tf.float32, name='lr') self.batch_size = batch_size # generate hidden state from visible state x # and random distribution hrand self.encode = self.sample_hidden_from_visible(self.v)[0] # generate visible state from hidden state x # and random distribution vrand self.reconstruct = self.sample_visible_from_hidden(self.encode)[0] # training algorithm: constractive divergence self.updates = self._create_cd1() self.loss = self._create_reconstruct_loss() # measure the goodness of the weights self.goodness = self._create_goodness() self.free_energy = self._create_free_energy() time_str = strftime("%b-%d-%Y-%H-%M-%S", localtime()) self.dirname = self.name + '/Run-' + time_str self.train_writer = tf.summary.FileWriter(self.dirname) self._create_summaries() self.merged_summary = tf.summary.merge_all() self.sess = tf.Session() self.saver = tf.train.Saver(self.weights) if restore_dir is not None: self.model_dir = restore_dir self.saver.restore(self.sess, self.model_dir) print('Model restored') else: init = tf.global_variables_initializer() self.model_dir = 'rbm_models/%dby%d' % (num_visible, num_hidden) create_dir('rbm_models') self.sess.run(init) print('RBM Model built and initialized')
def train(model, train_loader, eval_loader, num_epochs, output, eval_each_epoch, tiny_train=False): utils.create_dir(output) optim = torch.optim.Adamax(model.parameters()) logger = utils.Logger(os.path.join(output, 'log.txt')) all_results = [] total_step = 0 ans_embed = np.load("./data/mutant_both_vqacp_v2/answer_embs.npy") + 1e-8 ans_embed = torch.from_numpy(ans_embed).cuda() ans_embed = torch.nn.functional.normalize(ans_embed, dim=1) for epoch in range(num_epochs): total_loss = 0 train_score = 0 t = time.time() # model.train(False) # results = evaluate(model, eval_loader, tiny_train) # eval_score = results["score"] # bound = results["upper_bound"] # print("RANDOM SCORE BEFORE TRAINING", eval_score) # print("UPPER BOUNG", bound) model.train(True) cos = nn.CosineSimilarity() for i, (v, q, a, b, answertypefeats, top_ans_emb) in tqdm(enumerate(train_loader), ncols=100, desc="Epoch %d" % (epoch + 1), total=len(train_loader)): total_step += 1 if tiny_train and total_step == 10: break v = Variable(v).cuda() q = Variable(q).cuda() a = Variable(a).cuda() b = Variable(b).cuda() answertypefeats = Variable(answertypefeats).cuda() top_ans_emb = Variable(top_ans_emb).cuda() gen_embs, pred, loss, all_ans_embs = model(v, None, q, a, b) all_ans_embs = torch.stack([all_ans_embs] * gen_embs.shape[0]) # print("gen_embs.shape BEFORE unsqueeze", gen_embs.shape) # print("all_ans_embs.shape", all_ans_embs.shape) # print(type(gen_embs), type(all_ans_embs)) sys.stdout.flush() if (loss != loss).any(): raise ValueError("NaN loss") ## NCE LOSS positive_dist = cos(gen_embs, top_ans_emb) # shape b,k;b,k-> b gen_embs = torch.cat([gen_embs.unsqueeze(1)] * all_ans_embs.shape[1], dim=1) d_logit = cos(gen_embs, all_ans_embs) # print("gen_embs.shape AFTER unsqueeze", gen_embs.shape) # print("all_ans_embs.shape", all_ans_embs.shape) num = torch.exp(positive_dist).squeeze(-1) den = torch.exp(d_logit).sum(-1) # print("num.shape,den.shape", num.shape,den.shape) # sys.stdout.flush() loss_nce = -1 * torch.log(num / den) loss_nce = loss_nce.mean() loss = loss + loss_nce loss.backward() nn.utils.clip_grad_norm(model.parameters(), 0.25) optim.step() optim.zero_grad() batch_score = compute_score_with_logits(pred, a.data).sum() total_loss += loss.data[0] * v.size(0) train_score += batch_score if tiny_train: L = total_step else: L = len(train_loader.dataset) total_loss /= L train_score = 100 * train_score / L run_eval = eval_each_epoch or (epoch == num_epochs - 1) if run_eval: model.train(False) results = evaluate(model, eval_loader, tiny_train) results["epoch"] = epoch + 1 results["step"] = total_step results["train_loss"] = total_loss results["train_score"] = train_score eval_score = results["score"] bound = results["upper_bound"] all_results.append(results) with open(join(output, "results.json"), "w") as f: json.dump(all_results, f, indent=2) model.train(True) logger.write('epoch %d, time: %.2f' % (epoch + 1, time.time() - t)) logger.write('\ttrain_loss: %.2f, score: %.2f' % (total_loss, train_score)) if run_eval: logger.write('\teval score: %.2f (%.2f)' % (100 * eval_score, 100 * bound)) model_path = os.path.join(output, 'model' + str(epoch) + '.pth') torch.save(model.state_dict(), model_path)
# concate input and label together img_pair = np.concatenate([img_input, img_label], 2) # crop the patch patch_list = crop_patch(img_pair, 100, 100, False) # save the patches into h5py file for patch_idx in range(len(patch_list)): data = patch_list[patch_idx].copy() h5f.create_dataset(str(img_idx) + '_' + str(patch_idx), shape=(200, 200, 8), data=data) h5f.close() if __name__ == "__main__": # generating train/valid/test datasets # dataset_type = 'train' src_input_path = ".../input" src_label_path = ".../label" dst_path = ".../h5py_file" create_dir(dst_path) src_input_files = sorted(glob.glob(src_input_path + "/*.png")) src_label_files = sorted(glob.glob(src_label_path + "/*.png")) print("start dataset generation!") gen_dataset(src_input_files, src_label_files, dst_path)
# plt.colorbar() plt.scatter(clcd[:, 0], clcd[:, 1], marker='s', s=5, alpha=.3, edgecolors='none') plt.xlabel(r'$C_L$') if pos == 'first': plt.ylabel(r'$C_D$') if __name__ == "__main__": results_dir = 'results_clcd' create_dir(results_dir) database = np.load('data/airfoil_interp.npy') N = database.shape[0] ''' Data ''' fname = '{}/clcd_data.npy'.format(results_dir) if os.path.exists(fname): clcd_data = np.load(fname) else: list_cl = [] list_cd = [] for i, airfoil in enumerate(database): if detect_intersect(airfoil): te = (airfoil[0] + airfoil[-1]) / 2 airfoil[0] = airfoil[-1] = te _, cl, cd = evaluate(airfoil, return_CL_CD=True)
def generate_metadata(tf_config, input_audio, labels): """Generates metadata file containing sample rate and thresholds of each keyword""" K.clear_session() sess = tf.Session(config=tf_config) set_session(sess) K.set_learning_phase(0) model_file = join(args.tmpdir, kCheckpointFileName + '.h5') model = load_model(model_file) prediction_probabilities = model.predict(input_audio, batch_size=args.batch_size) label_ids = np.where(labels)[1] sorted_indices = label_ids.argsort() prediction_probabilities = prediction_probabilities[sorted_indices] label_ids = label_ids[sorted_indices] class_range = np.concatenate(([0], np.cumsum(labels.sum(axis=0)))).astype(int) # Metadata num_classes = len(args.keywords_list) thresholds = [None] * num_classes max_range = labels.sum(axis=0).min().astype(int) * (num_classes - 1) # Compute optimal threshold for each class for i in range(num_classes): total = min(class_range[i + 1] - class_range[i], max_range) chosen_indices = np.random.choice( np.arange(class_range[i], class_range[i + 1]), total, replace=False) target_labels = [label_ids[chosen_indices]] predictions = [prediction_probabilities[chosen_indices, i]] split = np.round(total / (num_classes - 1)).astype(int) for j in range(num_classes): if j != i: chosen_indices = np.random.choice( np.arange(class_range[j], class_range[j + 1]), min(split, total), replace=False) total -= split target_labels.append(label_ids[chosen_indices]) predictions.append(prediction_probabilities[chosen_indices, i]) target_labels = np.concatenate(target_labels) predictions = np.concatenate(predictions) thresholds[i] = find_optimal_threshold(target_labels, predictions, i) # Prepare and save the metadata input_tensor = model.inputs[0] input_layer_name = input_tensor.name.split(':')[0].split('/')[0] input_tensor_shape = [1, DataConfig.kWindowLength, DataConfig.kNumMels, DataConfig.kNumFeatures] metadata = OrderedDict( [('<feature_extraction_node_name>', OrderedDict( [('isaac.audio.VoiceCommandFeatureExtraction', OrderedDict( [('sample_rate', DataConfig.kSampleRate), ('fft_length', DataConfig.kFftLength), ('num_mels', DataConfig.kNumMels), ('hop_size', DataConfig.kHopLength), ('window_length', DataConfig.kWindowLength), ('num_classes', num_classes), ('classes', args.keywords_list)])) ])), ('<tensorflow_inference_node_name>', OrderedDict( [('isaac.ml.TensorflowInference', OrderedDict( [('input_tensor_info', [OrderedDict([('ops_name', input_layer_name), ('index', 0), ('dims', input_tensor_shape)]) ]), ('output_tensor_info', [OrderedDict( [('ops_name', kOutputOpName), ('index', 0), ('dims', [1, num_classes])]) ]) ])) ])), ('<command_construction_node_name>', OrderedDict( [('isaac.audio.VoiceCommandConstruction', OrderedDict( [('num_classes', num_classes), ('classes', args.keywords_list), ('thresholds', list(thresholds))])) ])) ]) create_dir(args.model_output_path) with open(join(args.model_output_path, kMetadataFileName), 'w') as f: json.dump(metadata, f, indent=2)
""" Driver function to control what is run and when if this is the main python script being ran. As the project was developed in a jupyter notebook, everything is self-contained in the main file. Any expansion, however, would be able to use the predefined classes and functions for whatever purpose without running anything. """ history, model, eval_X, eval_Y = None, None, None, None if (not args.train and not args.evaluate and not args.predict): print( "Type -h and get a list of possible tasks. You may select multiple." ) else: if (args.train): history, model, eval_X, eval_Y = train_generator(out_file) plot_training_history(history.history, "Training History") if (args.evaluate): history = evaluate_generator(model, save_location, out_file) print(history) if (args.predict): choreograph_dance(model, save_location, out_file) if __name__ == "__main__": reload_packages() utils.create_dir(np_save_dir) save_location = utils.create_dir(logs_save_dir) out_file = open(os.path.join(save_location, "outfile.txt"), "w") main(save_location, out_file) out_file.close() # In[ ]:
""" model = torch.load(MODEL_PATH).to(device) test_iterator1, test_iterator2 = load_test() # Evaluate test loss and accuracy test_loss, test_acc = evaluate_(model, test_iterator2) print("Test Loss: {} | Test Acc: {}%".format( round(test_loss, 2), round(test_acc*100, 2))) if __name__ == "__main__": # Creating necessary folders # ------------------------------------------ utils.create_dir(utils.RESULTS_FOLDER, delete_old=False) utils.create_dir(utils.MODELS_FOLDER, delete_old=False) utils.create_dir("{}{}/".format(utils.RESULTS_FOLDER, MODEL_NAME), False) # Logging # ------------------------------------------ logging.basicConfig( filename="{}{}/{}.log".format(utils.RESULTS_FOLDER, MODEL_NAME, MODEL_NAME), filemode="a", level=logging.DEBUG, format="%(asctime)s - %(levelname)s: %(message)s", datefmt="%Y/%m/%d %H:%M:%S", )
def __init__(self, generator, test_dataset, out_dir, num_img=2): self.num_img = num_img self.generator = generator self.test_dataset = test_dataset self.out_dir = create_dir(out_dir)
def main(config): create_dir(config['result_dir']) output_dir = os.path.join(config['result_dir'], 'labels') create_dir(output_dir) atlas_paths = { 'images': list(np.loadtxt(config['train_ims_file'], dtype='str')), 'labels': list(np.loadtxt(config['train_lbs_file'], dtype='str')) } print('Loading test images', end=' ', flush=True) test_image_paths = list(np.loadtxt(config['test_ims_file'], dtype='str')) print('Ok') print('Loading Elastix registration parameters', end=' ', flush=True) parameter_map_lst = [ sitk.ReadParameterFile(os.path.join(config['param_dir'], f)) for f in sorted(os.listdir(config['param_dir'])) ] print('Ok') print('Building Multi-Atlas Segmentation model', end=' ', flush=True) mas = MultiAtlasSegmentation(atlas_paths, config['atlas_size'], config['image_metric'], config['label_fusion']) print('Ok') print('Building RCA Classifier', end=' ', flush=True) rca = SingleAtlasClassifier() print('Ok') print('Processing test images') dice_scores = [] for i, image_path in enumerate(test_image_paths): print('{}/{} Image filename: {}'.format(i + 1, len(test_image_paths), os.path.basename(image_path))) # Predict segmentation print(' - Predicting segmentation', end=' ', flush=True) label, label_path, atlas_idxs = mas.predict_segmentation( image_path, output_dir, parameter_map_lst) print('Ok') atlas_paths_rca = { 'images': [atlas_paths['images'][i] for i in atlas_idxs], 'labels': [atlas_paths['labels'][i] for i in atlas_idxs] } # Predict accuracy print(' - Predicting accuracy', end=' ', flush=True) rca.set_atlas_path(image_path, label_path) dice = rca.predict_dice(atlas_paths_rca, output_dir, parameter_map_lst) dice_scores.append(dice) print('Ok') # Save results print('Saving results', end=' ', flush=True) with open(os.path.join(config['result_dir'], 'rca_predictions.csv'), 'w') as f: writer = csv.DictWriter(f, fieldnames=['file', 'dice']) writer.writeheader() for i in range(len(test_image_paths)): writer.writerow({ 'file': os.path.basename(test_image_paths[i]), 'dice': dice_scores[i] }) print('Ok')
def train(model, train_loader, eval_loader, num_epochs, output, opt=None, s_epoch=0): lr_default = 1e-3 if eval_loader is not None else 7e-4 lr_decay_step = 2 lr_decay_rate = .25 lr_decay_epochs = range( 10, 20, lr_decay_step) if eval_loader is not None else range( 10, 20, lr_decay_step) gradual_warmup_steps = [ 0.5 * lr_default, 1.0 * lr_default, 1.5 * lr_default, 2.0 * lr_default ] saving_epoch = 3 grad_clip = .25 utils.create_dir(output) optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) \ if opt is None else opt logger = utils.Logger(os.path.join(output, 'log.txt')) best_eval_score = 0 utils.print_model(model, logger) logger.write('optim: adamax lr=%.4f, decay_step=%d, decay_rate=%.2f, grad_clip=%.2f' % \ (lr_default, lr_decay_step, lr_decay_rate, grad_clip)) if os.path.exists('./saved_models/ban/model_epoch16.pth'): checkpoint = torch.load('./saved_models/ban/model_epoch16.pth') model.load_state_dict(checkpoint.get('model_state', checkpoint)) s_epoch = checkpoint.get('epoch') logger.write('reload from %d epoch' % s_epoch) for epoch in range(s_epoch, num_epochs): total_loss = 0 train_score = 0 total_norm = 0 count_norm = 0 t = time.time() N = len(train_loader.dataset) if epoch < len(gradual_warmup_steps): optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch] logger.write('gradual warmup lr: %.4f' % optim.param_groups[0]['lr']) elif epoch in lr_decay_epochs: optim.param_groups[0]['lr'] *= lr_decay_rate logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr']) else: logger.write('lr: %.4f' % optim.param_groups[0]['lr']) for i, (v, b, q, a) in enumerate(train_loader): v = v.cuda() b = b.cuda() q = q.cuda() a = a.cuda() pred, att = model(v, b, q, a) loss = instance_bce_with_logits(pred, a) loss.backward() total_norm += nn.utils.clip_grad_norm_(model.parameters(), grad_clip) count_norm += 1 optim.step() optim.zero_grad() batch_score = compute_score_with_logits(pred, a.data).sum() total_loss += loss.item() * v.size(0) train_score += batch_score.item() total_loss /= N train_score = 100 * train_score / N if None != eval_loader: model.train(False) eval_score, bound, entropy = evaluate(model, eval_loader) model.train(True) logger.write('epoch %d, time: %.2f' % (epoch, time.time() - t)) logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' % (total_loss, total_norm / count_norm, train_score)) if eval_loader is not None: logger.write('\teval score: %.2f (%.2f)' % (100 * eval_score, 100 * bound)) if eval_loader is not None and entropy is not None: info = '' for i in range(entropy.size(0)): info = info + ' %.2f' % entropy[i] logger.write('\tentropy: ' + info) if (eval_loader is not None and eval_score > best_eval_score) or ( eval_loader is None and epoch >= saving_epoch): model_path = os.path.join(output, 'model_epoch%d.pth' % epoch) utils.save_model(model_path, model, epoch, optim) if eval_loader is not None: best_eval_score = eval_score
print('######################################################') start_time = time.time() opt_x, opt_airfoil, opt_perfs = optimize(func, perturb_type, perturb, n_eval) end_time = time.time() opt_x_runs.append(opt_x) opt_airfoil_runs.append(opt_airfoil) opt_perfs_runs.append(opt_perfs) time_runs.append(end_time - start_time) opt_x_runs = np.array(opt_x_runs) opt_airfoil_runs = np.array(opt_airfoil_runs) opt_perfs_runs = np.array(opt_perfs_runs) save_dir = 'results_opt/ffd_ga' create_dir(save_dir) np.save('{}/opt_solution.npy'.format(save_dir), opt_x_runs) np.save('{}/opt_airfoil.npy'.format(save_dir), opt_airfoil_runs) np.save('{}/opt_history.npy'.format(save_dir), opt_perfs_runs) # Plot optimization history mean_perfs_runs = np.mean(opt_perfs_runs, axis=0) plt.figure() plt.plot(np.arange(n_eval + 1, dtype=int), opt_perfs) plt.title('Optimization History') plt.xlabel('Number of Evaluations') plt.ylabel('Optimal CL/CD') # plt.xticks(np.linspace(0, n_eval+1, 5, dtype=int)) plt.savefig('{}/opt_history.svg'.format(save_dir)) plt.close()
def gerberise(manufacturer='default'): """ Generate Gerbers for one or more layers """ # Open the board's SVG svg_in = utils.openBoardSVG() # Get Gerber generation settings gcd = config.brd['gerber'] decimals = gcd['decimals'] digits = gcd['digits'] steps = gcd['steps-per-segment'] length = gcd['min-segment-length'] # Get layer data xpath_regex = "" ns = {'pcbmode':config.cfg['ns']['pcbmode'], 'svg':config.cfg['ns']['svg']} # Save to file base_dir = os.path.join(config.cfg['base-dir'], config.cfg['locations']['build'], 'production') # Create directory if it doesn't exist already utils.create_dir(base_dir) base_name = "%s_rev_%s" % (config.brd['config']['name'], config.brd['config']['rev']) filename_info = config.cfg['manufacturers'][manufacturer]['filenames']['gerbers'] # Process Gerbers for PCB layers and sheets for pcb_layer in utils.getSurfaceLayers(): # Get the SVG layer corresponding to the PCB layer svg_layer = svg_in.find("//svg:g[@pcbmode:pcb-layer='%s']" % (pcb_layer), namespaces=ns) # Get masks (must be placed right after pours) mask_paths = svg_in.findall(".//svg:defs//svg:mask[@pcbmode:pcb-layer='%s']//svg:path" % pcb_layer, namespaces=ns) sheets = ['copper', 'soldermask', 'solderpaste', 'silkscreen'] for sheet in sheets: # Get the SVG layer corresponding to the 'sheet' sheet_layer = svg_layer.find(".//svg:g[@pcbmode:sheet='%s']" % (sheet), namespaces=ns) if sheet == 'copper': mask_paths_to_pass = mask_paths else: mask_paths_to_pass = [] # Create a Gerber object gerber = Gerber(sheet_layer, mask_paths_to_pass, decimals, digits, steps, length) add = '_%s_%s.%s' % (pcb_layer, sheet, filename_info[pcb_layer][sheet].get('ext') or 'ger') filename = os.path.join(base_dir, base_name + add) with open(filename, "wb") as f: for line in gerber.getGerber(): f.write(line) # Process module sheets sheets = ['outline', 'documentation'] for sheet in sheets: # Get the SVG layer corresponding to the 'sheet' sheet_layer = svg_in.find(".//svg:g[@pcbmode:sheet='%s']" % (sheet), namespaces=ns) # Create a Gerber object gerber = Gerber(sheet_layer, [], decimals, digits, steps, length) add = '_%s.%s' % (sheet, filename_info['other'][sheet].get('ext') or 'ger') filename = os.path.join(base_dir, base_name + add) with open(filename, "wb") as f: for line in gerber.getGerber(False): f.write(line) return ['bullshit']
def train(model, train_loader, eval_loader, num_epochs, output, opt=None, s_epoch=0): lr_default = 1e-3 if eval_loader is not None else 7e-4 lr_decay_step = 2 lr_decay_rate = 1 lr_decay_epochs = range( 10, 20, lr_decay_step) if eval_loader is not None else range( 10, 20, lr_decay_step) gradual_warmup_steps = [ 0.5 * lr_default, 1.0 * lr_default, 1.5 * lr_default, 2.0 * lr_default ] saving_epoch = 3 grad_clip = .25 utils.create_dir(output) optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) \ if opt is None else opt logger = utils.Logger(os.path.join(output, 'log.txt')) best_eval_score = 0 utils.print_model(model, logger) logger.write('optim: adamax lr=%.4f, decay_step=%d, decay_rate=%.2f, grad_clip=%.2f' % \ (lr_default, lr_decay_step, lr_decay_rate, grad_clip)) for epoch in range(s_epoch, num_epochs): total_loss = 0 train_score = 0 total_norm = 0 count_norm = 0 t = time.time() N = len(train_loader.dataset) print(N) if epoch < len(gradual_warmup_steps): optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch] logger.write('gradual warmup lr: %.4f' % optim.param_groups[0]['lr']) elif epoch in lr_decay_epochs: optim.param_groups[0]['lr'] = 1e-3 logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr']) else: logger.write('lr: %.4f' % optim.param_groups[0]['lr']) ''' for i, (v, b, q, a,ques,im,g,gender) in enumerate(train_loader): v = v.cuda() b = b.cuda() q = q.cuda() a = a.cuda() visual_pred, att = model(v, b, q, a) #import pdb;pdb.set_trace() gender=gender.squeeze(1) weights=torch.Tensor([2.0,1.0,0.0001]).cuda() #loss = instance_bce_with_logits(visual_pred, g.cuda()) loss=nn.CrossEntropyLoss(weights) loss=loss(visual_pred,gender.cuda()) #import pdb;pdb.set_trace() loss.backward() total_norm += nn.utils.clip_grad_norm_(model.parameters(), grad_clip) count_norm += 1 optim.step() optim.zero_grad() batch_score=torch.eq(visual_pred.argmax(1),gender.cuda()).sum() #batch_score = compute_score_with_logits(visual_pred, g.cuda()).sum() #total_loss += loss.item() * v.size(0) train_score += batch_score.item() #train_score+=batch_score ''' total_loss /= N train_score = 100 * train_score / N if None != eval_loader: model.train(False) eval_score, bound, _ = evaluate(model, eval_loader) model.train(True) logger.write('epoch %d, time: %.2f' % (epoch, time.time() - t)) logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' % (total_loss, total_norm / count_norm, train_score)) logger.write('\teval score: %.2f (%.2f)' % (100 * eval_score, 100 * bound)) model_path = os.path.join(output, 'model_epoch%d.pth' % epoch) utils.save_model(model_path, model, epoch, optim)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataset', type=str, default='gdi', help='name of dataset, gdi or massvis (default: gdi)') parser.add_argument('--dataset_dir', type=str, default='/path/to/datase_dirt', help='dataset directory') parser.add_argument('--fcn_type', type=str, help='FCN type, fcn32 or fcn16 (default: gdi)', default='fcn32', choices=['fcn32', 'fcn16']) parser.add_argument('--overlaid_img_dir', type=str, default='/path/to/overlaid_img_dir', help='output directory path for images with heatpmap overlaid onto input images') parser.add_argument('--pretrained_model', type=str, default='/path/to/pretrained_model', help='pretrained model converted from Caffe models') parser.add_argument('--config', type=int, default=1, choices=configurations.keys(), help='configuration for training where several hyperparameters are defined') parser.add_argument('--log_file', type=str, default='F:/dataset/visimportance/log', help='/path/to/log_file') parser.add_argument('--resume', type=str, default='', help='checkpoint file to be loaded when retraining models') parser.add_argument('--checkpoint_dir', type=str, default='/path/to/checkpoint_dir', help='checkpoint file to be saved in each epoch') parser.add_argument('--eval_only', action='store_true', help='evaluation only') parser.add_argument('--gpu', type=int, default=0, help='GPU id (default: 0)') args = parser.parse_args() utils.create_dir(os.path.join(args.overlaid_img_dir, "train")) utils.create_dir(os.path.join(args.overlaid_img_dir, "valid")) if not args.eval_only: utils.create_dir(args.checkpoint_dir) print(args) gpu = args.gpu cfg = configurations[args.config] log_file = args.log_file resume = args.resume os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu) cuda = torch.cuda.is_available() args.cuda = cuda if args.cuda: print("torch.backends.cudnn.version: {}".format(torch.backends.cudnn.version())) torch.manual_seed(1337) if cuda: torch.cuda.manual_seed(1337) # 1. dataset root = os.path.expanduser(args.dataset_dir) kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {} train_loader = None if not args.eval_only: # training + validation if args.dataset == 'gdi': dt = GDI(root, image_dir="gd_train", imp_dir="gd_imp_train", split='train', transform=True) else: dt = Massvis(root, image_dir="train", imp_dir="train_imp", split='train', transform=True) train_loader = torch.utils.data.DataLoader(dt, batch_size=1, shuffle=True, **kwargs) print("no of images in training", len(train_loader)) if args.dataset == 'gdi': # validation dv = GDI(root, image_dir="gd_val", imp_dir="gd_imp_val", split='valid', transform=True) else: dv = Massvis(root, image_dir="valid", imp_dir="valid_imp", split='valid', transform=True) val_loader = torch.utils.data.DataLoader(dv, batch_size=1, shuffle=False, **kwargs) print("no of images in evaluation", len(val_loader)) # 2. model model = models.FCN32s() if args.fcn_type == 'fcn32' else models.FCN16s() start_epoch = 0 start_iteration = 0 if resume: checkpoint = torch.load(resume) model.load_state_dict(checkpoint['model_state_dict']) start_epoch = checkpoint['epoch'] start_iteration = checkpoint['iteration'] if args.fcn_type == 'fcn32': assert checkpoint['arch'] == 'FCN32s' else: assert checkpoint['arch'] == 'FCN16s' else: if args.fcn_type in ['fcn32', 'fcn16']: model_weight = torch.load(args.pretrained_model) model.load_state_dict(model_weight) if not args.eval_only: model._initialize_weights() else: fcn32s = models.FCN32s() checkpoint = torch.load(args.pretrained_model) fcn32s.load_state_dict(checkpoint['model_state_dict']) model.copy_params_from_fcn32s(fcn32s) model._initialize_weights() if cuda: model = model.cuda() # 3. optimizer optim = torch.optim.SGD( [ {'params': get_parameters(model, bias=False, fcn_type=args.fcn_type)}, {'params': get_parameters(model, bias=True, fcn_type=args.fcn_type), 'lr': cfg['lr'] * 2, 'weight_decay': 0}, ], lr=cfg['lr'], momentum=cfg['momentum'], weight_decay=cfg['weight_decay']) if resume: optim.load_state_dict(checkpoint['optim_state_dict']) # lr_policy: step last_epoch = start_iteration if resume else -1 lr_scheduler = torch.optim.lr_scheduler.StepLR(optim, cfg['step_size'], gamma=cfg['gamma'], last_epoch=last_epoch) trainer = Trainer( cuda=cuda, model=model, optimizer=optim, lr_scheduler=lr_scheduler, train_loader=train_loader, val_loader=val_loader, checkpoint_dir=args.checkpoint_dir, log_file=log_file, max_iter=cfg['max_iteration'], iter_size=cfg['iter_size'], interval_validate=cfg.get('interval_validate', len(train_loader)) if not args.eval_only else 0, overlaid_img_dir=args.overlaid_img_dir, dataset=args.dataset, eval_only=args.eval_only, ) trainer.epoch = start_epoch trainer.iteration = start_iteration if not args.eval_only: trainer.train() else: trainer.validate()