def demux(self):
'''
Demux video, audio, subs
Return an object with the filenames
'''
src_dir_series = os.path.join(self.src_dir_top, self.series)
for r in self._regions():
if r not in self.demux_map:
continue
src_dir = os.path.join(src_dir_series, r)
dest_dir = os.path.join(self.temp_dir, r)
create_dir(dest_dir)
logger.info('Demuxing %s %s %s...', self.series, self.number, r)
self.files[r] = demux(self, src_dir, dest_dir,
self.demux_map[r],
novid=((r == 'R1' or r == 'PIONEER') and
not self.demux_r1_vid),
nosub=(r == 'R2'), sub_only=self.sub_only,
orange_brick=(
r == 'R1' and
self.series == 'DBZ' and
not self.is_special))
if not self.sub_only and 'R2' in self.files:
self.r2_chapters = _load_r2_chapters(
self.files['R2']['chapters'][0], self.series,
self.is_special)
def extract(self):
batch_time = utils.AverageMeter()
self.model.eval()
end = time.time()
for batch_idx, (imgs, target, img_files, class_ids) in tqdm.tqdm(
enumerate(self.val_loader), total=len(self.val_loader),
desc='Extract', ncols=80, leave=False):
gc.collect()
if self.cuda:
imgs = imgs.cuda()
imgs = Variable(imgs, volatile=True)
output = self.model(imgs) # N C H W torch.Size([1, 1, 401, 600])
if self.flatten_feature:
output = output.view(output.size(0), -1)
output = output.data.cpu().numpy()
assert output.shape[0] == len(img_files)
for i, img_file in enumerate(img_files):
base_name = os.path.splitext(img_file)[0]
feature_file = os.path.join(self.feature_dir, base_name + ".npy")
utils.create_dir(os.path.dirname(feature_file))
np.save(feature_file, output[i])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % self.print_freq == 0:
log_str = 'Extract: [{0}/{1}]\tTime: {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(
batch_idx, len(self.val_loader), batch_time=batch_time)
print(log_str)
self.print_log(log_str)
def generate_lgs(inkmls, path):
create_dir(path)
path = path + '/' if not path.endswith('/') else path
for inkml in inkmls:
fname = inkml.src.rstrip('inkml') + "lg"
fname = path + os.path.basename(fname)
open(fname, 'w+').write(inkml.get_lg())
def download_from_url(client_id, url, base_dir, override=False):
"""Download the given playlist"""
downloaded = 0
skipped = 0
errors = 0
# Retrieve playlist data
client = soundcloud.Client(client_id=client_id)
playlist = client.get('/resolve', url=url)
# Create dir
playlist_title = playlist.title
dir = os.path.join(base_dir, playlist_title)
utils.create_dir(dir)
# Download tracks
for trak in playlist.tracks:
try:
#done = song.down(client, track, dir, override)
done = track.download_from_id(client_id, trak['id'], dir, override)
if done: downloaded = downloaded + 1
else: skipped = skipped + 1
except requests.exceptions.HTTPError, err:
if err.response.status_code == 404:
print 'Error: could not download'
errors = errors + 1
else:
raise
def test_one_dir(self):
utils.create_dir("dir1")
response = self.client.request("GET", "/")
(status, reason, body, headers) = response
self.assertEqual(200, status)
body = eval(body)
self.assertTrue(isinstance(body,dict))
body = body["dir"]
self.assertEqual(1, len(body))
self.assertEqual("dir1", body[0]["name"])
self.assertEqual(True, body[0]["is_dir"])
response = self.client.request("GET", "/dir1/")
(status, reason, body, headers) = response
self.assertEqual(200, status)
body = eval(body)
body = body["dir"]
self.assertTrue(isinstance(body,list))
self.assertEqual(0, len(body))
def compute_target(answers_dset, ans2label, name, cache_root='data/cache'):
"""Augment answers_dset with soft score as label
***answers_dset should be preprocessed***
Write result into a cache file
"""
target = []
for ans_entry in answers_dset:
answers = ans_entry['answers']
answer_count = {}
for answer in answers:
answer_ = answer['answer']
answer_count[answer_] = answer_count.get(answer_, 0) + 1
labels = []
scores = []
for answer in answer_count:
if answer not in ans2label:
continue
labels.append(ans2label[answer])
score = get_score(answer_count[answer])
scores.append(score)
target.append({
'question_id': ans_entry['question_id'],
'image_id': ans_entry['image_id'],
'labels': labels,
'scores': scores
})
utils.create_dir(cache_root)
cache_file = os.path.join(cache_root, name+'_target.pkl')
cPickle.dump(target, open(cache_file, 'wb'))
return target
def init_visit_dir(self):
"""Create results and logs directories for this visit."""
visit_name = str(self.instance_num)
self.visit_dir = os.path.join(self.base_dir, visit_name)
ut.create_dir(self.visit_dir)
self.visit_log_dir = os.path.join(self.visit_dir, 'logs')
ut.create_dir(self.visit_log_dir)
def save(config, data):
"""Save data and create backup, creating a new data file if necessary."""
create_dir(os.path.dirname(config['data_path']))
# atomically save by writing to temporary file and moving to destination
try:
# write to temp file
with open(
config['tmp_path'],
'w',
encoding='utf-8',
errors='replace') as f:
for path, weight in data.items():
if is_python3():
f.write(("%s\t%s\n" % (weight, path)))
else:
f.write(unicode(
"%s\t%s\n" % (weight, path)).encode('utf-8'))
f.flush()
os.fsync(f)
except IOError as ex:
print("Error saving autojump data (disk full?)" % ex, file=sys.stderr)
sys.exit(1)
# create backup file if it doesn't exist or is older than BACKUP_THRESHOLD
if not os.path.exists(config['backup_path']) or \
(time() - os.path.getmtime(config['backup_path'])
> BACKUP_THRESHOLD):
move_file(config['data_path'], config['backup_path'])
# move temp_file -> autojump.txt
move_file(config['tmp_path'], config['data_path'])
def add_sim_modules_to_project(tags, sim_dict, user_paths):
#utils.pretty_print_dict(tags)
#utils.pretty_print_dict(sim_dict)
#Get the directory of where to put the sim modules
base_dir = utils.resolve_path(tags["BASE_DIR"])
project_dir = tags["PROJECT_NAME"]
out_dir = os.path.join(base_dir, "sim", "sim_modules")
if not os.path.exists(out_dir):
utils.create_dir(out_dir)
#Find all the file locations
module_filename = utils.find_module_filename(sim_dict["name"], user_paths)
module_filepath = utils.find_rtl_file_location(module_filename, user_paths)
out_file_path = os.path.join(out_dir, module_filename)
#print "copy %s > %s" % (module_filepath, out_file_path)
shutil.copy2(module_filepath, os.path.join(out_dir, out_file_path))
#Get the locations for each of the auxilary files
for f in sim_dict["aux_files"]:
module_path = utils.find_rtl_file_location(f)
out_file_path = os.path.join(out_dir, f)
#print "copy %s > %s" % (module_path, out_file_path)
shutil.copy2(module_path, os.path.join(out_dir, f))
def run_STAR(args):
read1 = ','.join(args.read1)
read2 = ','.join(args.read2)
output_dir = args.outdir + '/' + args.prefix + '/'
utils.create_dir(output_dir)
STAR_args = ['STAR', '--runThreadN', args.cores, '--genomeDir', args.genome_index,
'--outFileNamePrefix ',output_dir , '--outSAMunmapped', 'Within']
if args.outdir is None:
args.outdir = 'STARmapped_%s_out' % time.strftime('%Y-%m-%d:%H-%M-%S')
if args.optional_args is not None:
STAR_args.extend([args.optional_args])
STAR_args.extend(['--readFilesIn',read1, read2 ])
#to make sure we convert all the args to string
STAR_args = map(str,STAR_args)
#run STAR
utils.run_external_command(STAR_args, logger=args.logger)
#expected output sam file
expected_sam_file = output_dir + 'Aligned.out.sam'
#convert sam to bam
bamFile = bamUtils.sam_to_bam(expected_sam_file, delete_sam=True)
def setup_crawl_dirs(self, test_url=TEST_URL):
crawl_name = ut.append_timestamp("crawl")
self.crawl_dir = ut.create_dir(join(cm.TEST_FILES_DIR, crawl_name))
batch_dir = ut.create_dir(join(self.crawl_dir, str(self.batch_num)))
self.site_dir = ut.create_dir(join(batch_dir,
ut.get_filename_from_url(test_url,
self.site_num)))
def write_file(self, location = "", filename=""):
"""write_file
Search through the specified location, if the location doesn't exist then
create the location.
then write out the specified file
Args:
location: the location where the file is to be written
filename: the name of the output file to write
Returns:
Nothing
Raises:
IOError
"""
home = False
location = utils.resolve_path(location)
if not os.path.exists(location):
utils.create_dir(location)
fname = os.path.join(location, filename)
fileout = open(fname, "w")
fileout.write(self.buf)
fileout.close()
return
def __init__(self, api_key = "", output_dir = "", text_list_file = "", *args, **kwargs):
super(GetTextsSpider, self).__init__(*args, **kwargs)
utils.create_dir(output_dir)
if len(api_key) == 0:
raise scrapy.exceptions.CloseSpider("expected 'api_key' to be a string")
if len(output_dir) == 0:
raise scrapy.exceptions.CloseSpider("expected 'output_dir' to be a string")
if len(text_list_file) == 0:
raise scrapy.exceptions.CloseSpider("expected 'text_list_file' to be a string")
self.api_key = api_key
self.output_dir = output_dir
with open(text_list_file, "r") as f:
self.all_texts = json.load(f)['texts']
if utils.is_file_exists(output_dir+"/master_record.json"):
with open(output_dir+"/master_record.json", "r") as f:
json_file = json.load(f)
self.master_record = json_file["master_record"]
self.last_unallocated_number = json_file["last_unallocated_number"]
else:
self.master_record = {text["gid"]: "" for text in self.all_texts}
self.last_unallocated_number = 1
self.text_name_prefix = "Guardian"
self.text_name_length = int(math.floor(math.log(len(self.all_texts))/math.log(10)) + 1)
def main():
colorama.init()
config = load_config_file()
args, wtf = create_args().parse_known_args()
if (wtf):
logger.error('Unknown argument %s', wtf[0])
sys.exit(1)
init_logging(args.verbose)
# don't proceed if paths aren't right/programs missing
pre_check(args, config)
try:
working_dir = config.get(APP_NAME, 'working_dir')
except configparser.Error:
working_dir = None
if working_dir:
if not os.path.isdir(working_dir):
create_dir(working_dir)
tempfile.tempdir = working_dir
tmp_dir = tempfile.mkdtemp()
logger.debug('Episode temp folder: %s', tmp_dir)
atexit.register(delete_temp, tmp_dir)
start, end, special = validate_args(args)
print(WELCOME_MSG)
for ep in range(start, end + 1):
start_time = time.clock()
episode = Episode(ep, config, args, tmp_dir, special)
if not args.no_demux:
episode.demux()
else:
if args.sub_only:
detect_streams(os.path.join(config.get(APP_NAME, 'output_dir'),
args.series,
str(ep if not special else special).zfill(3),
'R1', 'Subtitle.idx'))
if not args.no_retime:
episode.retime_subs()
episode.retime_audio()
if not args.no_demux and args.no_mux:
# move files to destination folder
episode.move_demuxed_files()
if not args.no_mux:
episode.mux()
if args.make_avs:
# only works on files generated with --no-mux
episode.make_avs()
delete_temp(episode.temp_dir)
elapsed = time.clock() - start_time
logger.debug('Elapsed time: %s seconds', elapsed)
logger.info('Finished!')
def create_crawl_dir(self):
"""Create a timestamped crawl."""
ut.create_dir(self.output) # ensure that we've a results dir
crawl_dir_wo_ts = os.path.join(self.output, 'crawl')
crawl_dir = ut.create_dir(ut.append_timestamp(crawl_dir_wo_ts))
crawl_logs_dir = os.path.join(crawl_dir, 'logs')
ut.create_dir(crawl_logs_dir)
return crawl_dir, crawl_logs_dir
def get_profile_photos(browser, username):
save_dir = 'profiles/' + username + '/'
create_dir(save_dir) # need to fix these names
i = 0
for image in browser.find_elements_by_xpath('//div[@id="profile_thumbs"]//img[@src]'):
image_url = image.get_attribute('src')
save_path = save_dir + str(i) + '.webp' # probably a bad idea to assume the img format
download_image(image_url, save_path)
i += 1 # using 'i' is probably not the clean way to name these images
def setup_env(tbb_rec_ver=None):
"""Initialize the tbb directory and import TBB signing keys.
Download recommended TBB version and verify it.
"""
import_tbb_signing_keys()
ut.create_dir(TBB_BASE_DIR)
if not tbb_rec_ver:
tbb_rec_ver = get_recommended_tbb_version()
download_tbb_tarball(tbb_rec_ver)
def test_dir(self):
utils.create_dir("dir1")
response = self.client.request("GET", "/dir1")
(status, reason, body, headers) = response
location = utils.get_header("location", headers)
self.assertEqual(307, status)
self.assertEqual("http://localhost:%s/dir1/" % utils.get_port(), location)
def __init__(self, number, config, args, tmp_dir, special):
series = args.series
self.is_movie = False
if args.movie:
series = 'MOVIES'
number = series_to_movie(args.series, number)
self.is_movie = True
if special:
ep_str = special
self.is_special = True
else:
ep_str = str(number).zfill(pad_zeroes(series))
self.is_special = False
if args.r1_dbox:
frame_data, op_offset = load_frame_data('DBoxZ', ep_str)
else:
frame_data, op_offset = load_frame_data(series, ep_str)
# config stuff
self.pgcdemux = config.get(APP_NAME, 'pgcdemux')
self.vsrip = config.get(APP_NAME, 'vsrip')
self.delaycut = config.get(APP_NAME, 'delaycut')
self.dgindex = config.get(APP_NAME, 'dgindex')
self.mkvmerge = config.get(APP_NAME, 'mkvmerge')
self.restream = config.get(APP_NAME, 'restream')
self.src_dir_top = config.get(APP_NAME, 'source_dir')
self.output_dir = config.get(APP_NAME, 'output_dir')
# special flags
self.is_r1dbox = args.series == 'DBZ' and args.r1_dbox
self.is_pioneer = args.pioneer
self.no_funi = args.no_funi
self.demux_r1_vid = args.r1_vid
self.verbose = args.verbose
# options
self.no_mux = args.no_mux
self.sub_only = args.sub_only
self.temp_dir = os.path.join(tmp_dir, ep_str)
create_dir(self.temp_dir)
self.number = ep_str
self.series = series
self.offsets = _combine_framedata(frame_data, op_offset)
self.pioneer_offsets = None if not self.is_pioneer else load_frame_data(series, str(number+50))[0]
self.r2_chapters = {}
if not args.no_demux:
self.demux_map = load_demux_map(series, ep_str)
if self.is_r1dbox:
self.demux_map['R1_DBOX'] = auto_detect(self)
else:
self.demux_map = {'R1': {'audio': ['en', 'jp']}}
self.files = self._init_files() if args.no_demux else {}
def test_html(self):
file1contents = "file 1 contents"
utils.create_dir("dir1")
utils.write_file("dir1/file1.txt", file1contents)
response = self.client.request("GET", "/dir1/", {"Accept": "text/html"})
(status, reason, body, headers) = response
content_type = utils.get_header("content-type", headers)
self.assertEqual(200, status)
self.assertEqual("text/html", content_type)
def download(client, track, dir, override=False):
"""Download a track using the given SC client"""
title = fix_title(track.title, track.user['username'])
print '"%s"' % title
if not dir: dir = 'mp3'
utils.create_dir(dir)
file_name = utils.build_file_name(dir, title)
if not override and os.path.exists(file_name):
print "File already exists, skipped"
return False
stream_url = client.get(track.stream_url, allow_redirects=False)
urllib.urlretrieve(stream_url.location, file_name)
return True
def edit_ratio_histogram(self):
utils.log('creating edit histogram %s' % self.lang)
f_out = utils.create_dir('results/ratio_histograms')
df = pd.read_csv(self.db_path)
df.page_id = df.page_id.astype(float)
df = df.loc[df['linked_id'] != None]
df.linked_id = df.linked_id.astype(float)
df = self.drop_dups(df)
utils.log('dropped %s duplicates' % len(df.set_index('page_id',drop=False).index.get_duplicates()))
df = df.drop_duplicates(subset='page_id',keep=False)
if self.drop1:
df = df.loc[(df['len'] > 1)]
for r in self.revert:
utils.log('%s %s' % (self.lang,r))
utils.log('%s pages' % len(df))
n0 = df.loc[(df['namespace'] == 0)].set_index('page_id',drop=False)
n1 = df.loc[(df['namespace'] == 1)].set_index('linked_id',drop=False)
utils.log('%s articles' % len(n0))
utils.log('%s talk' % len(n1))
ratio = n0[r].divide(n1[r],axis='index',fill_value=-1).to_frame()
ratio.columns = ['ratio']
ratio.ratio = ratio.ratio.astype(int)
ratio = n0.join(ratio).set_index('page_id')
ratio = ratio.loc[ratio['ratio'] >= 0]
utils.log('%s ratios' % len(ratio))
result = ratio['ratio'].value_counts().to_frame()
result = result.sort_index(ascending=True)
result.columns = ['pages']
result.to_csv('%s/%s_%s.csv' % (f_out,self.lang,r),encoding='utf-8',index_label='edit_ratio')
def _create_dex_PRE(self):
if self._check_classes() and \
(True if self.dx_bin else False) and \
(True if self.project_path else False):
# Create bin folder if doesn't exist
create_dir(os.path.join(self.project_path, 'bin'))
log.info('Generating DEX executable')
if self._need_to_re_compile_java_classes() or \
not self._check_dex():
self._create_dex_task()
else:
log.info('No new/modified classes to re-generate DEX executable')
self._create_dex_POST()
else:
log.warn('Missing compiled classes and/or build tools!')
def __do_instance(self):
for self.job.visit in xrange(self.job.visits):
ut.create_dir(self.job.path)
wl_log.info("*** Visit #%s to %s ***", self.job.visit, self.job.url)
with self.driver.launch():
try:
self.driver.set_page_load_timeout(cm.SOFT_VISIT_TIMEOUT)
except WebDriverException as seto_exc:
wl_log.error("Setting soft timeout %s", seto_exc)
self.__do_visit()
if self.screenshots:
try:
self.driver.get_screenshot_as_file(self.job.png_file)
except WebDriverException:
wl_log.error("Cannot get screenshot.")
sleep(float(self.job.config['pause_between_visits']))
self.post_visit()
def _compile_java_code_PRE(self):
if self._check_sources() and \
(True if self.javac_bin else False) and \
(True if self.project_path else False) and \
(True if self.android_jar else False):
# Create obj folder if doesn't exist
create_dir(os.path.join(self.project_path, 'obj'))
log.info('Compiling java source')
if self._need_to_re_compile_java_sources() or \
not self._check_classes():
self._compile_java_code_task()
else:
log.info('No new/modified sources to re-compile sources')
self._compile_java_code_POST()
else:
log.warn('Missing sources and/or build tools!')
def test_one_file_in_dir(self):
file1contents = "file 1 contents"
utils.create_dir("dir1")
utils.write_file("dir1/file1.txt", file1contents)
response = self.client.request("GET", "/dir1/")
(status, reason, body, headers) = response
self.assertEqual(200, status)
body = eval(body)
body = body["dir"]
self.assertTrue(isinstance(body,list))
self.assertEqual(1, len(body))
self.assertEqual("file1.txt", body[0]["name"])
self.assertEqual(False, body[0]["is_dir"])
def render_flame_file(batch_name, output_path='./output/'):
u"""
Dado un nombre de lote y una ruta de output opcional,
renderiza el archivo flame con flam3 y guarda las imágenes resultates en la ruta
definida por el output y el nombre del lote.
:param batch_name: nombre del lote
:param output_path: directorio de salida para las imágenes
:return: boolean: Indica si la función ha tenido éxito
"""
# Crear la ruta de carpetas para el conjunto de flames que se van a renderizar
current_batch_dir = output_path + batch_name + '/'
create_dir(current_batch_dir)
current_batch_flame_file = current_batch_dir + batch_name + '.flame'
# Renderizar el archivo XML de flames con flam3
try:
flame_file = open(current_batch_flame_file, 'r')
except IOError:
print 'Error: El archivo "' + current_batch_flame_file + '" no existe'
return False
else:
# Establecer las variables de entorno para flam3
# Flam3 utiliza variables de entorno en lugar de parámetros
envy = os.environ.copy()
envy['prefix'] = current_batch_dir
envy['name_enable'] = '0' # Debe ser cero para que la opción 'prefix' funcione
envy['bits'] = '32' # Utilizar 32-bits integer
# Por defecto 33 (32-bits flotante) y se generan
# imágenes de muy baja calidad para 'quality' bajos
# Llamar a flam3 y renderizar todos los flames
process = subprocess.Popen('./flam3/flam3-render.exe', stdin=flame_file,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=envy)
# Imprimir la salida de flam3 para ver que está pasando
for e in iter(lambda: process.stderr.read(1), ''):
sys.stdout.write(e)
return True
def generate_session_pages(self):
self.date_data = get_date_dataset()
self.date_data.reverse()
if self.fast_run:
COUNTER = 0
for leg, sess, num, d, dpub, page_start, page_end in self.date_data:
dateobj = parse_iso_date(d)
session = get_session_from_legsessnum(leg, sess, num)
if not session:
log.warn("File for %s-%s-%s is missing from the transcripts dataset!" % (leg, sess, num))
continue
target_dir = "%s%d/%02d/%02d" % (self.sessions_path, dateobj.year, dateobj.month, dateobj.day)
filename = "%s/index.html" % target_dir
info = get_session_info(leg, sess, num)
create_dir(os.path.join(self.output_dir, target_dir))
if type(session) in (str, unicode):
# sessão em texto simples
context = {'date': dateobj,
'year_number': dateobj.year,
'leg': leg,
'sess': sess,
'num': num,
'text': session,
'monthnames': MESES,
'pdf_url': 'xpto',
'page_name': 'sessoes',
}
if info:
context['session_info'] = info
self.render_template_into_file('session_plaintext.html', filename, context)
elif type(session) in (dict, OrderedDict):
# usar entradas do .json como contexto
session['date'] = parse_iso_date(session['session_date'])
session['monthnames'] = MESES
session['page_name'] = 'sessoes'
self.render_template_into_file('session.html', filename, session)
if self.fast_run:
COUNTER += 1
if COUNTER > self.fast_run_count:
break
def test_simple_in_dir(self):
utils.create_dir("a-dir")
file1contents = "file 1 contents"
utils.write_file("a-dir/file.txt", file1contents)
response = self.client.request("GET", "/a-dir/file.txt")
(status, reason, body, headers) = response
content_length = utils.get_header("content-length", headers)
etag = utils.get_header("etag", headers)
etag_pattern = re.compile(r'^"[^"]+"$')
self.assertEqual(200, status)
self.assertEqual(file1contents, body)
self.assertEqual(len(file1contents), int(content_length))
self.assertTrue(etag != None)
self.assertTrue(etag_pattern.match(etag) != None)
def __init__(self, molID=None, dset=None, run_name=None, tset_path=None):
if molID and dset and not run_name and not tset_path:
if not __class__._run_name or not __class__._tset_path:
msg = 'run_name has to be specified before to instance a'\
' useful class!'
lg.critical(msg)
raise RuntimeError(msg)
self._inout_path = os.path.join(__class__._run_name, dset,
'inout', __class__._inout_id)
self._inout_path = os.path.abspath(self._inout_path)
create_dir(self._inout_path)
self._xyzp = os.path.join(__class__._tset_path, dset,
'geometry', molID.split('.')[1] + '.xyz')
self._inout_inp_path = os.path.join(self._inout_path,
molID + '.inp')
self._inout_out_path = os.path.join(self._inout_path,
molID + '.log')
self.molID = molID
create_dir(config['densities_repo'])
self._wb97x_saves = os.path.join(config['densities_repo'],
self.molID + '.wb97x')
self._ddsc_saves = os.path.join(config['densities_repo'],
self.molID + '.ddsc')
self._sbatch_file = \
os.path.join(config['sbatch_script_prefix'],
self.molID)
elif not molID and not dset and run_name and tset_path:
__class__._run_name = run_name
__class__._tset_path = tset_path
else:
msg = 'You cannot specify run_name, dset_path and molID all'\
' together in Run'
lg.critical(msg)
raise AttributeError(msg)
def compute_coeff(airfoil, reynolds=500000, mach=0, alpha=3, n_iter=200, tmp_dir='tmp'):
create_dir(tmp_dir)
gc.collect()
safe_remove('{}/airfoil.log'.format(tmp_dir))
fname = '{}/airfoil.dat'.format(tmp_dir)
with open(fname, 'wb') as f:
np.savetxt(f, airfoil)
try:
# Has error: Floating point exception (core dumped)
# This is the "empty input file: 'tmp/airfoil.log'" warning in other approaches
child = pexpect.spawn('xfoil')
timeout = 10
child.expect('XFOIL c> ', timeout)
child.sendline('load {}/airfoil.dat'.format(tmp_dir))
child.expect('Enter airfoil name s> ', timeout)
child.sendline('af')
child.expect('XFOIL c> ', timeout)
child.sendline('OPER')
child.expect('.OPERi c> ', timeout)
child.sendline('VISC {}'.format(reynolds))
child.expect('.OPERv c> ', timeout)
child.sendline('ITER {}'.format(n_iter))
child.expect('.OPERv c> ', timeout)
child.sendline('MACH {}'.format(mach))
child.expect('.OPERv c> ', timeout)
child.sendline('PACC')
child.expect('Enter polar save filename OR <return> for no file s> ', timeout)
child.sendline('{}/airfoil.log'.format(tmp_dir))
child.expect('Enter polar dump filename OR <return> for no file s> ', timeout)
child.sendline()
child.expect('.OPERva c> ', timeout)
child.sendline('ALFA {}'.format(alpha))
child.expect('.OPERva c> ', timeout)
child.sendline()
child.expect('XFOIL c> ', timeout)
child.sendline('quit')
child.expect(pexpect.EOF)
child.close()
res = np.loadtxt('{}/airfoil.log'.format(tmp_dir), skiprows=12)
if len(res) == 9:
CL = res[1]
CD = res[2]
else:
CL = np.nan
CD = np.nan
except Exception as ex:
# print(ex)
print('XFoil error!')
CL = np.nan
CD = np.nan
safe_remove(':00.bl')
return CL, CD
:param lines: list of ranges, [article1_start, article1_end, article2_start, article2_end, ...]
:return: vector with binary values, where 1 is on position of the topic (article) change
"""
# dimension of y_ is the index of last article line -1 (-1 because there can be n-1 topic changes,
# where n is number of rows in the prediction matrix)
dim = lines[-1] - 1
y_ = np.zeros((dim, 1))
for i in range(1, len(lines) - 1, 2):
y_[lines[i] - 1] = 1
return y_
def get_next_data(data_names):
for name in data_names:
yield config.get_seg_data(name)
if __name__ == '__main__':
print('Loading the instances')
classifier = load_pickle(config.classifier)
for data in get_next_data(config.data.keys()):
print('Processing ' + data['name'] + ' data')
create_dir(data['dir'])
y_true = line_map_to_y(load_pickle(data['line_map']))
print('Saving the data')
np.save(data['y_true_lm'], y_true)
TF_LOG_PATH = os.path.join(TF_LOG_DIR, 'alphanum', str(TEST_IDX))
KERACT_PATH = os.path.join(KERACT_DIR, 'alphanum', str(TEST_IDX))
MODEL_PATH = os.path.join(MODEL_DIR, 'alphanum')
MODEL_IMG_PATH = os.path.join(MODEL_PATH, f'model_{TEST_IDX:3}.png')
MODEL_STRUCT_PATH = os.path.join(MODEL_PATH, f'model_{TEST_IDX:3}.h5')
DATA_PATH = os.path.join(DATA_DIR, 'alphanum')
TEST_DATA_PATH = os.path.join(DATA_PATH, 'test')
TRAIN_DATA_PATH = os.path.join(DATA_PATH, 'train')
create_clean_dir(TF_LOG_PATH)
create_clean_dir(KERACT_PATH)
create_dir(MODEL_PATH)
model = Sequential()
model.add(
Conv2D(filters=12,
kernel_size=(3, 3),
strides=(1, 1),
padding='valid',
input_shape=(CHAR_IMG_HEIGHT, CHAR_IMG_WIDTH, 3),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), padding='valid'))
model.add(Flatten())
model.add(Dense(512, activation='sigmoid'))
model.add(Dense(256, activation='sigmoid'))
model.add(Dense(NUM_ALPHANUM_CHARS, activation='softmax'))
model.compile(optimizer=SGD(lr=0.05),
from os import listdir, remove, path
from shutil import rmtree
from utils import load_json, call, check_skip, create_dir, extract, move
import json
import re
config = load_json('cdn-config.json')
print '[INFO] Rebuild CDN collection.'
for target in config['targets']:
print '[INFO] Collect %s libraries.' % target
target_dir = target[target.find('/') +
1:] if target.find('/') > 0 else target
create_dir("%s/%s" % (config['directory'], target_dir))
lib_info = call("bower info %s -j --allow-root" % target)
if not lib_info:
print '[ERROR] Cannot collect information about library'
break
versions = [
version for version in json.loads(lib_info)['versions']
if not check_skip(version, config['skipWords'])
]
for version in versions:
target_directory = "%s/%s/%s" % (config['directory'], target_dir,
version)
if not create_dir(target_directory) and listdir(target_directory):
from utils import first_option, create_dir, save_pickle
if __name__ == '__main__':
# Number of vectors in one sequence, input data structure [samples, time_steps, features]
time_steps = 200
if Path(config.lstm_model_1).is_file() and first_option(
'Do you want to continue training the saved model?', 'y', 'n'):
print("Loading new model")
model = load_model(config.lstm_model_1)
else:
print("Building new model")
model = build_model(time_steps, 1)
# Create dir for histories
create_dir(config.hist_dir)
print("Loading the data")
train = config.get_seg_data('train')
held_out = config.get_seg_data('held_out')
X_train_or = np.load(train['y'])
y_train_or = np.load(train['y_true_lm'])
X_held_out = np.load(held_out['y'])
y_held_out = np.load(held_out['y_true_lm'])
print("Computing the distances on test data")
X_held_out = compute_distance(X_held_out, cosine_distances)
# Split the 2D matrix to 3D matrix of dimensions [samples, time_steps, features]
parser = argparse.ArgumentParser(description='Transformer NER')
# parser.add_argument('--corpus-data', type=str, default='../data/auto_only-nav-distance_BOI.txt',
# help='path to corpus data')
parser.add_argument('--save-dir',
type=str,
default='./data_char/',
help='path to save processed data')
parser.add_argument('--onnx-dir',
type=str,
default='./model_onnx/',
help='path to save processed data')
parser.add_argument('--pre-w2v', type=str, default='../data/w2v')
args = parser.parse_args()
create_dir(args.onnx_dir)
pre_w2v = torch.load(args.save_dir + 'pre_w2v')
pre_w2v = torch.Tensor(pre_w2v).to(device)
model_ckpt = torch.load(os.path.join(
args.save_dir, '{}.pyt'.format("Transformer_NER_best")),
map_location=torch.device(device))
config = load_obj(args.save_dir + 'Config.json')
model = Transformer_Mix(config, pre_w2v).to(device)
model.load_state_dict(model_ckpt['model'])
# Initialize the DataLoader
data_loader = DataLoader_test(args.save_dir)
def train_coco(model, train_loader, eval_loader, num_epochs, output,s_epoch=0):
lr_default=0.001
grad_clip = .25
utils.create_dir(output)
lr_decay_step = 2
lr_decay_rate = .5
lr_decay_epochs = range(8, 12, lr_decay_step)
gradual_warmup_steps = [0.5 * lr_default, 1.0 * lr_default,2.0*lr_default]
optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default)
logger = utils.Logger(os.path.join(output, 'log.txt'))
best_eval_score = 0
utils.print_model(model, logger)
for epoch in range(s_epoch, num_epochs):
total_loss = 0
train_score = 0
total_norm = 0
count_norm = 0
t = time.time()
N = len(train_loader.dataset)
if epoch < len(gradual_warmup_steps):
optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch]
logger.write('gradual warmup lr: %.4f' % optim.param_groups[0]['lr'])
elif epoch in lr_decay_epochs:
optim.param_groups[0]['lr'] *= lr_decay_rate
logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr'])
else:
logger.write('lr: %.4f' % optim.param_groups[0]['lr'])
for i, (v, b, q, a,question_id,image_id,types) in enumerate(train_loader):
v = Variable(v).cuda()
b = Variable(b).cuda()
q = Variable(q).cuda()
a = Variable(a).cuda()
pred,att_1,att_2= model(v, b, q, a)
loss = instance_bce_with_logits(pred, a)
loss.backward()
total_norm += nn.utils.clip_grad_norm(model.parameters(), grad_clip)
count_norm += 1
optim.step()
optim.zero_grad()
batch_score = compute_score_with_logits(pred, a.data).sum()
total_loss += loss.data[0] * v.size(0)
train_score += batch_score
total_loss /= N
train_score = 100 * train_score / N
if None != eval_loader:
model.train(False)
eval_score,eval_loss= evaluate(model, eval_loader)
model.train(True)
logger.write('epoch %d, time: %.2f' % (epoch, time.time()-t))
logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' % (total_loss, total_norm/count_norm, train_score))
if eval_loader is not None:
logger.write('\teval score: %.2f,eval loss:%.2f' % (100 * eval_score,eval_loss))
if (eval_loader is not None and eval_score > best_eval_score) or (eval_loader is None and epoch>=0):
model_path = os.path.join(output, 'model_epoch%d.pth' % epoch)
utils.save_model(model_path, model, epoch, optim)
if eval_loader is not None:
best_eval_score = eval_score
def train(args,
model,
question_model,
train_loader,
eval_loader,
s_opt=None,
s_epoch=0):
device = args.device
model = model.to(device)
question_model = question_model.to(device)
# create packet for output
utils.create_dir(args.output)
# for every train, create a packet for saving .pth and .log
run_timestamp = datetime.now().strftime("%Y%b%d-%H%M%S")
ckpt_path = os.path.join(args.output, run_timestamp)
utils.create_dir(ckpt_path)
# create logger
logger = utils.Logger(os.path.join(ckpt_path, 'medVQA.log')).get_logger()
logger.info(">>>The net is:")
logger.info(model)
logger.info(">>>The args is:")
logger.info(args.__repr__())
# Adamax optimizer
optim = torch.optim.Adamax(params=model.parameters())
# Scheduler learning rate
#lr_decay = lr_scheduler.CosineAnnealingLR(optim,T_max=len(train_loader)) # only fit for sgdr
# Loss function
criterion = torch.nn.BCEWithLogitsLoss()
ae_criterion = torch.nn.MSELoss()
best_eval_score = 0
best_epoch = 0
# Epoch passing in training phase
for epoch in range(s_epoch, args.epochs):
total_loss = 0
train_score = 0
number = 0
model.train()
# Predicting and computing score
for i, (v, q, a, answer_type, question_type, phrase_type,
answer_target) in enumerate(train_loader):
#lr_decay.step()
optim.zero_grad()
if args.maml:
v[0] = v[0].reshape(v[0].shape[0], 84, 84).unsqueeze(1)
v[0] = v[0].to(device)
if args.autoencoder:
v[1] = v[1].reshape(v[1].shape[0], 128, 128).unsqueeze(1)
v[1] = v[1].to(device)
if args.other_model:
v = v.to(device)
q = q.to(device)
a = a.to(device)
# MEVF loss computation
if args.autoencoder:
last_output_close, last_output_open, a_close, a_open, decoder = model(
v, q, a, answer_target)
else:
last_output_close, last_output_open, a_close, a_open = model(
v, q, a, answer_target)
preds_close, preds_open = model.classify(last_output_close,
last_output_open)
#loss
loss_close = criterion(preds_close.float(), a_close)
loss_open = criterion(preds_open.float(), a_open)
loss = loss_close + loss_open
if args.autoencoder:
loss_ae = ae_criterion(v[1], decoder)
loss = loss + (loss_ae * args.ae_alpha)
# loss /= answers.size()[0]
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 0.25)
optim.step()
#compute the acc for open and close
batch_close_score = compute_score_with_logits(
preds_close, a_close.data).sum()
batch_open_score = compute_score_with_logits(
preds_open, a_open.data).sum()
total_loss += loss.item()
train_score += batch_close_score + batch_open_score
number += q.shape[0]
total_loss /= len(train_loader)
train_score = 100 * train_score / number
logger.info('-------[Epoch]:{}-------'.format(epoch))
logger.info('[Train] Loss:{:.6f} , Train_Acc:{:.6f}%'.format(
total_loss, train_score))
# Evaluation
if eval_loader is not None:
eval_score = evaluate_classifier(model, question_model,
eval_loader, args, logger)
if eval_score > best_eval_score:
best_eval_score = eval_score
best_epoch = epoch
# Save the best acc epoch
model_path = os.path.join(ckpt_path,
'{}.pth'.format(best_epoch))
utils.save_model(model_path, model, best_epoch, optim)
logger.info('[Result] The best acc is {:.6f}% at epoch {}'.format(
best_eval_score, best_epoch))
def create_rate_subs(lang, mech_name, therm_name=None, last_spec=None):
"""Create rate subroutines from mechanism.
Parameters
----------
lang : {'c', 'cuda', 'fortran', 'matlab'}
Language type.
mech_name : str
Reaction mechanism filename (e.g. 'mech.dat').
therm_name : str, optional
Thermodynamic database filename (e.g. 'therm.dat')
or nothing if info in mechanism file.
last_spec : str, optional
If specified, the species to assign to the last index.
Typically should be N2, Ar, He or another inert bath gas
Returns
-------
None
"""
lang = lang.lower()
if lang not in utils.langs:
print('Error: language needs to be one of: ')
for l in utils.langs:
print(l)
sys.exit()
if lang in ['fortran', 'matlab']:
print('WARNING: Fortran and Matlab support incomplete.')
# create output directory if none exists
build_path = './out/'
utils.create_dir(build_path)
# Interpret reaction mechanism file, depending on Cantera or
# Chemkin format.
if mech_name.endswith(tuple(['.cti', '.xml'])):
[elems, specs, reacs] = mech.read_mech_ct(mech_name)
else:
[elems, specs, reacs] = mech.read_mech(mech_name, therm_name)
# Check to see if the last_spec is specified
if last_spec is not None:
# Find the index if possible
isp = next((i for i, sp in enumerate(specs)
if sp.name.lower() == last_spec.lower().strip()), None)
if isp is None:
print('Warning: User specified last species {} '
'not found in mechanism.'
' Attempting to find a default species.'.format(last_spec))
last_spec = None
else:
last_spec = isp
else:
print('User specified last species not found or not specified. '
'Attempting to find a default species')
if last_spec is None:
wt = chem.get_elem_wt()
#check for N2, Ar, He, etc.
candidates = [('N2', wt['n'] * 2.), ('Ar', wt['ar']), ('He', wt['he'])]
for sp in candidates:
match = next(
(isp for isp, spec in enumerate(specs)
if sp[0].lower() == spec.name.lower() and sp[1] == spec.mw),
None)
if match is not None:
last_spec = match
break
if last_spec is not None:
print('Default last species '
'{} found.'.format(specs[last_spec].name))
if last_spec is None:
print('Warning: Neither a user specified or default last species '
'could be found. Proceeding using the last species in the '
'base mechanism: {}'.format(specs[-1].name))
last_spec = len(specs) - 1
# ordering of species and reactions not changed.
fwd_rxn_mapping = range(len(reacs))
spec_maps = utils.get_species_mappings(len(specs), last_spec)
fwd_spec_mapping, reverse_spec_mapping = spec_maps
#pick up the last_spec and drop it at the end
temp = specs[:]
for i in range(len(specs)):
specs[i] = temp[fwd_spec_mapping[i]]
# Reassign reaction's product / reactant / third body list
# to integer indexes for speed
utils.reassign_species_lists(reacs, specs)
## Now begin writing subroutines
# print reaction rate subroutine
rate.write_rxn_rates(build_path, lang, specs, reacs, fwd_rxn_mapping)
# if third-body/pressure-dependent reactions,
# print modification subroutine
if next((r for r in reacs if (r.thd_body or r.pdep)), None):
rate.write_rxn_pressure_mod(build_path, lang, specs, reacs,
fwd_rxn_mapping)
# write species rates subroutine
rate.write_spec_rates(build_path, lang, specs, reacs, fwd_spec_mapping,
fwd_rxn_mapping)
# write chem_utils subroutines
rate.write_chem_utils(build_path, lang, specs)
# write derivative subroutines
rate.write_derivs(build_path, lang, specs, reacs)
# write mass-mole fraction conversion subroutine
rate.write_mass_mole(build_path, lang, specs)
write_header(build_path, lang, specs)
write_main(build_path, lang, specs)
return 0
#!/usr/bin/env python
import time
import os
import subprocess
import itertools
from utils import SLURM_TEMPLATE, create_dir
SAVE_DIR = "/home/gridsan/apjacob/sweeps"
JOB_SWEEP_NAME = "multitask_fixed_sweep_test"
time = time.strftime("%Y%m%d-%H%M%S")
job_output_dir = os.path.join(SAVE_DIR, f"{JOB_SWEEP_NAME}-{time}")
print(f"Job output directory: {job_output_dir}")
# Make top level directories
create_dir(job_output_dir)
# Sweep params
rules = [3, 7, 12, 14, 21, 28, 80]
rnns = ["zero", "rnn"]
train_modes = ["coach", "executor", "both"]
for rule, rnn, train_mode in itertools.product(rules, rnns, train_modes):
job_name = f"rule-{rule}-rnns-{rnn}-train_mode-{train_mode}"
job_file_name = os.path.join(job_output_dir, f"{job_name}.job")
job_log_file = os.path.join(job_output_dir, f"{job_name}.log")
if rnn == "zero" and train_mode != "executor":
continue
python_command = "python -u /home/gridsan/apjacob/minirts/scripts/self_play/multitask-fixed.py --coach1 rnn500 " \
"--coach2 rnn500 " \
def main():
opts = {}
opts['random_seed'] = 66
opts['dataset'] = 'guitars' # gmm, circle_gmm, mnist, mnist3 ...
opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
opts['unrolling_steps'] = 5 # Used only if unrolled = True
opts['data_dir'] = 'mnist'
opts['trained_model_path'] = 'models'
opts[
'mnist_trained_model_file'] = 'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
opts['gmm_max_val'] = 15.
opts['toy_dataset_size'] = 10000
opts['toy_dataset_dim'] = 2
opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
opts['input_normalize_sym'] = True # Normalize data to [-1, 1]
opts['adagan_steps_total'] = 3
opts['samples_per_component'] = 1000 # 50000
opts['work_dir'] = FLAGS.workdir
opts['is_bagging'] = FLAGS.is_bagging
opts['beta_heur'] = 'uniform' # uniform, constant
opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
opts['beta_constant'] = 0.5
opts['topk_constant'] = 0.5
opts["init_std"] = FLAGS.init_std
opts["init_bias"] = 0.0
opts['latent_space_distr'] = 'normal' # uniform, normal
opts['optimizer'] = 'adam' # sgd, adam
opts["batch_size"] = 64
opts["d_steps"] = 1
opts["g_steps"] = 1
opts["verbose"] = True
opts['tf_run_batch_size'] = 100
opts['gmm_modes_num'] = 5
opts['latent_space_dim'] = FLAGS.zdim
opts["gan_epoch_num"] = 50
opts["mixture_c_epoch_num"] = 5
opts['opt_learning_rate'] = FLAGS.learning_rate
opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
opts["opt_beta1"] = FLAGS.adam_beta1
opts['batch_norm_eps'] = 1e-05
opts['batch_norm_decay'] = 0.9
opts['d_num_filters'] = 64
opts['g_num_filters'] = 64
opts['conv_filters_dim'] = 4
opts["early_stop"] = -1 # set -1 to run normally
opts["plot_every"] = 1 # set -1 to run normally
opts["eval_points_num"] = 1000 # 25600
opts['digit_classification_threshold'] = 0.999
opts['inverse_metric'] = True # Use metric from the Unrolled GAN paper?
opts['inverse_num'] = 64 # Number of real points to inverse.
if opts['verbose']:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s - %(message)s')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
utils.create_dir(opts['work_dir'])
with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
text.write('Parameters:\n')
for key in opts:
text.write('%s : %s\n' % (key, opts[key]))
data = DataHandler(opts)
assert data.num_points >= opts['batch_size'], 'Training set too small'
adagan = AdaGan(opts, data)
metrics = Metrics()
for step in range(opts["adagan_steps_total"]):
logging.info('Running step {} of AdaGAN'.format(step + 1))
adagan.make_step(opts, data)
num_fake = opts['eval_points_num']
logging.debug('Sampling fake points')
fake_points = adagan.sample_mixture(num_fake)
logging.debug('Sampling more fake points')
more_fake_points = adagan.sample_mixture(500)
logging.debug('Plotting results')
if opts['dataset'] == 'gmm':
metrics.make_plots(opts, step, data.data[:500], fake_points[0:100],
adagan._data_weights[:500])
logging.debug('Evaluating results')
(likelihood, C) = metrics.evaluate(opts,
step,
data.data[:500],
fake_points,
more_fake_points,
prefix='')
else:
metrics.make_plots(opts, step, data.data, fake_points[:6 * 16],
adagan._data_weights)
logging.debug('Evaluating results')
if opts['inverse_metric']:
l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
res = metrics.evaluate(opts,
step,
data.data[:500],
fake_points,
more_fake_points,
prefix='')
logging.debug("AdaGan finished working!")
print('# lambda0 = {}'.format(lambda0))
print('# lambda1 = {}'.format(lambda1))
print('# disc_lr = {}'.format(disc_lr))
print('# gen_lr = {}'.format(gen_lr))
print('#################################')
# Read dataset
data_fname = './data/xs_train.npy'
perf_fname = './data/ys_train.npy'
X = np.load(data_fname)
Y = np.load(perf_fname)
N = X.shape[0]
assert N == Y.shape[0]
# Prepare save directory
create_dir('./trained_gan')
save_dir = './trained_gan/{}_{}'.format(lambda0, lambda1)
create_dir(save_dir)
# print('Plotting training samples ...')
# samples = X[np.random.choice(N, size=36, replace=False)]
# plot_samples(None, samples, scale=1.0, scatter=False, lw=1.2, alpha=.7, c='k', fname='{}/samples'.format(save_dir))
# Train
surrogate_dir = './surrogate/trained_surrogate'
model = BezierGAN(latent_dim, noise_dim, X.shape[1], bezier_degree, bounds, lambda0, lambda1)
if args.mode == 'train':
safe_remove(save_dir)
timer = ElapsedTimer()
model.train(X, batch_size=batch_size, train_steps=train_steps, disc_lr=disc_lr, gen_lr=gen_lr,
save_interval=save_interval, directory=save_dir, surrogate_dir=surrogate_dir)
img_degraded_wzwb = process(img_degraded, wb_chosen, ccm)
img_label_wzwb = process(img_unprocessed[pad_length : H+pad_length, pad_length : W+pad_length, :], wb_chosen, ccm)
"""save the degradation results"""
tifffile.imwrite(os.path.join(input_dir, (filename + '.tiff')), (img_degraded_wzwb * 65535).astype(np.uint16))
print('Image is saved in path: %s' % (os.path.join(input_dir, filename)))
tifffile.imwrite(os.path.join(label_dir, (filename + '.tiff')), (img_label_wzwb * 65535).astype(np.uint16))
print('Image is saved in path: %s' % (os.path.join(label_dir, filename)))
if __name__ == "__main__":
# input image path
# label8bit_dir = '~/train_datasets/label_8bit'
label8bit_dir = '~/valid_datasets/label_8bit'
# label raw path
# labelraw_dir = '~/train_datasets/label_rgb'
labelraw_dir = '~/valid_datasets/label_rgb'
create_dir(labelraw_dir)
# output image path
# inputraw_dir = '~/train_datasets/input_rgb'
inputraw_dir = '~/valid_datasets/input_rgb'
create_dir(inputraw_dir)
# kernel path
kernel_path = '~/kernel/kernel.mat'
# interval of the patch, 10pixels
patch_itv = 50
# size of the patch, 100pixels
patch_size = 150
# generate the image list
img_list = sorted(glob.glob(label8bit_dir + '/*.png'))
ccm = np.array([[ 1.93994141, -0.73925781, -0.20068359],
[-0.28857422, 1.59741211, -0.30883789],
def main():
opts = {}
# Utility
opts['random_seed'] = 66
opts['dataset'] = 'cifar10' # gmm, circle_gmm, mnist, mnist3 ...
opts['data_dir'] = 'cifar10'
opts['trained_model_path'] = None #'models'
opts[
'mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
opts['work_dir'] = FLAGS.workdir
opts['ckpt_dir'] = 'checkpoints'
opts["verbose"] = 1
opts['tf_run_batch_size'] = 128
opts["early_stop"] = -1 # set -1 to run normally
opts["plot_every"] = 150
opts["save_every_epoch"] = 10
opts['gmm_max_val'] = 15.
# Datasets
opts['toy_dataset_size'] = 10000
opts['toy_dataset_dim'] = 2
opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
opts['input_normalize_sym'] = False # Normalize data to [-1, 1]
opts['gmm_modes_num'] = 5
# AdaGAN parameters
opts['adagan_steps_total'] = 1
opts['samples_per_component'] = 1000
opts['is_bagging'] = FLAGS.is_bagging
opts['beta_heur'] = 'uniform' # uniform, constant
opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
opts['beta_constant'] = 0.5
opts['topk_constant'] = 0.5
opts["mixture_c_epoch_num"] = 5
opts["eval_points_num"] = 25600
opts['digit_classification_threshold'] = 0.999
opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper?
opts['inverse_num'] = 100 # Number of real points to inverse.
opts['objective'] = None
# Generative model parameters
opts["init_std"] = FLAGS.init_std
opts["init_bias"] = 0.0
opts['latent_space_distr'] = 'normal' # uniform, normal
opts['latent_space_dim'] = FLAGS.zdim
opts["gan_epoch_num"] = 100
opts['convolutions'] = True
opts['d_num_filters'] = 512
opts['d_num_layers'] = 4
opts['g_num_filters'] = 1024
opts['g_num_layers'] = 4
opts['e_is_random'] = False
opts['e_num_filters'] = 1024
opts['e_num_layers'] = 4
opts['g_arch'] = 'dcgan_mod'
opts['g_stride1_deconv'] = False
opts['g_3x3_conv'] = 0
opts['e_arch'] = 'dcgan'
opts['e_3x3_conv'] = 0
opts['conv_filters_dim'] = 5
# --GAN specific:
opts['conditional'] = False
opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
opts['unrolling_steps'] = 5 # Used only if unrolled = True
# --VAE specific
opts['vae'] = FLAGS.vae
opts['vae_sigma'] = 0.01
# --POT specific
opts['pot'] = FLAGS.pot
opts['pot_pz_std'] = 2.
opts['pot_lambda'] = FLAGS.pot_lambda
opts['adv_c_loss'] = 'conv'
opts['vgg_layer'] = 'pool2'
opts['adv_c_patches_size'] = 5
opts['adv_c_num_units'] = 32
opts['adv_c_loss_w'] = 1.0
opts['cross_p_w'] = 0.0
opts['diag_p_w'] = 0.0
opts['emb_c_loss_w'] = 1.0
opts['reconstr_w'] = 0.0
opts['z_test'] = 'gan'
opts['z_test_corr_w'] = 1.0
opts['z_test_proj_dim'] = 10
# Optimizer parameters
opts['optimizer'] = 'adam' # sgd, adam
opts["batch_size"] = 100
opts["d_steps"] = 1
opts['d_new_minibatch'] = False
opts["g_steps"] = 2
opts['batch_norm'] = True
opts['dropout'] = False
opts['dropout_keep_prob'] = 0.5
opts['recon_loss'] = 'l2sq'
# "manual" or number (float or int) giving the number of epochs to divide
# the learning rate by 10 (converted into an exp decay per epoch).
opts['decay_schedule'] = 'manual'
opts['opt_learning_rate'] = FLAGS.learning_rate
opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
opts["opt_beta1"] = FLAGS.adam_beta1
opts['batch_norm_eps'] = 1e-05
opts['batch_norm_decay'] = 0.9
if opts['e_is_random']:
assert opts['latent_space_distr'] == 'normal',\
'Random encoders currently work only with Gaussian Pz'
# Data augmentation
opts['data_augm'] = True
if opts['verbose']:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s - %(message)s')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
utils.create_dir(opts['work_dir'])
utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir']))
with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
text.write('Parameters:\n')
for key in opts:
text.write('%s : %s\n' % (key, opts[key]))
data = DataHandler(opts)
assert data.num_points >= opts['batch_size'], 'Training set too small'
adagan = AdaGan(opts, data)
metrics = Metrics()
train_size = data.num_points
random_idx = np.random.choice(train_size, 4 * 320, replace=False)
metrics.make_plots(opts,
0,
data.data,
data.data[random_idx],
adagan._data_weights,
prefix='dataset_')
for step in range(opts["adagan_steps_total"]):
logging.info('Running step {} of AdaGAN'.format(step + 1))
adagan.make_step(opts, data)
num_fake = opts['eval_points_num']
logging.debug('Sampling fake points')
fake_points = adagan.sample_mixture(num_fake)
logging.debug('Sampling more fake points')
more_fake_points = adagan.sample_mixture(500)
logging.debug('Plotting results')
if opts['dataset'] == 'gmm':
metrics.make_plots(opts, step, data.data[:500], fake_points[0:100],
adagan._data_weights[:500])
logging.debug('Evaluating results')
(likelihood, C) = metrics.evaluate(opts,
step,
data.data[:500],
fake_points,
more_fake_points,
prefix='')
else:
metrics.make_plots(opts, step, data.data, fake_points[:320],
adagan._data_weights)
if opts['inverse_metric']:
logging.debug('Evaluating results')
l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
res = metrics.evaluate(opts,
step,
data.data[:500],
fake_points,
more_fake_points,
prefix='')
logging.debug("AdaGan finished working!")
batch_size,
shuffle=False,
num_workers=4,
collate_fn=trim_collate)
else:
train_loader = DataLoader(train_dset,
batch_size,
shuffle=True,
num_workers=4,
collate_fn=trim_collate)
eval_loader = DataLoader(val_dset,
batch_size,
shuffle=False,
num_workers=4,
collate_fn=trim_collate)
output_meta_folder = join(args.output, "regat_%s" % args.relation_type)
utils.create_dir(output_meta_folder)
args.output = output_meta_folder + "/%s_%s_%s_%d" % (
fusion_methods, args.relation_type, args.dataset, args.seed)
if exists(args.output) and os.listdir(args.output):
raise ValueError("Output directory ({}) already exists and is not "
"empty.".format(args.output))
utils.create_dir(args.output)
with open(join(args.output, 'hps.json'), 'w') as writer:
json.dump(vars(args), writer, indent=4)
logger = utils.Logger(join(args.output, 'log.txt'))
train(model, train_loader, eval_loader, args, device)
"template_next_version": next_version,
"template_reasons": substitutions.get("reasons", REASONS),
"template_remove_examples": substitutions.get("remove", ""),
"template_new_features": substitutions.get("new_features", ""),
}
# Do the substitution
result = src.safe_substitute(d)
try:
# 1 January 2020
eol = datetime.datetime.strptime(substitutions["template_eol"],
"%d %B %Y")
except ValueError:
# 2020-01-01
eol = datetime.datetime.strptime(substitutions["template_eol"],
"%Y-%m-%d")
# EOL in the future?
if now < eol:
result = result.replace("about time", "soon time")
result = result.replace(" reached the ", " reaches the ")
# print(result)
# Save it
outfile = os.path.join(version, "index.html")
print(outfile)
create_dir(version)
with open(outfile, "w") as f:
f.write(result)
def __init__(self,
num_visible,
num_hidden,
batch_size,
trans_func=tf.nn.sigmoid,
num_labels=5,
restore_dir=None,
name='rbm'):
self.num_visible = num_visible
self.num_hidden = num_hidden
self.num_labels = num_labels
self.trans_func = trans_func
self.name = name
self.weights = self._initialize_weights()
# v is used for both cd and generate hidden states
self.v = tf.placeholder(tf.float32, [None, num_visible], name='v')
# self.h = tf.placeholder(tf.float32, [None, num_hidden], name='h')
self.one = tf.placeholder(tf.float32, [None, num_hidden], name='1fe')
self.vrand = tf.placeholder(tf.float32, [None, num_visible],
name='vrand')
self.hrand = tf.placeholder(tf.float32, [None, num_hidden],
name='hrand')
self.avg_tfe = tf.placeholder(tf.float32, name='avg_tfe')
self.avg_vfe = tf.placeholder(tf.float32, name='avg_vfe')
self.reconstruct_loss = tf.placeholder(tf.float32, name='rloss')
self.lr = tf.placeholder(tf.float32, name='lr')
self.batch_size = batch_size
# generate hidden state from visible state x
# and random distribution hrand
self.encode = self.sample_hidden_from_visible(self.v)[0]
# generate visible state from hidden state x
# and random distribution vrand
self.reconstruct = self.sample_visible_from_hidden(self.encode)[0]
# training algorithm: constractive divergence
self.updates = self._create_cd1()
self.loss = self._create_reconstruct_loss()
# measure the goodness of the weights
self.goodness = self._create_goodness()
self.free_energy = self._create_free_energy()
time_str = strftime("%b-%d-%Y-%H-%M-%S", localtime())
self.dirname = self.name + '/Run-' + time_str
self.train_writer = tf.summary.FileWriter(self.dirname)
self._create_summaries()
self.merged_summary = tf.summary.merge_all()
self.sess = tf.Session()
self.saver = tf.train.Saver(self.weights)
if restore_dir is not None:
self.model_dir = restore_dir
self.saver.restore(self.sess, self.model_dir)
print('Model restored')
else:
init = tf.global_variables_initializer()
self.model_dir = 'rbm_models/%dby%d' % (num_visible, num_hidden)
create_dir('rbm_models')
self.sess.run(init)
print('RBM Model built and initialized')
def train(model,
train_loader,
eval_loader,
num_epochs,
output,
eval_each_epoch,
tiny_train=False):
utils.create_dir(output)
optim = torch.optim.Adamax(model.parameters())
logger = utils.Logger(os.path.join(output, 'log.txt'))
all_results = []
total_step = 0
ans_embed = np.load("./data/mutant_both_vqacp_v2/answer_embs.npy") + 1e-8
ans_embed = torch.from_numpy(ans_embed).cuda()
ans_embed = torch.nn.functional.normalize(ans_embed, dim=1)
for epoch in range(num_epochs):
total_loss = 0
train_score = 0
t = time.time()
# model.train(False)
# results = evaluate(model, eval_loader, tiny_train)
# eval_score = results["score"]
# bound = results["upper_bound"]
# print("RANDOM SCORE BEFORE TRAINING", eval_score)
# print("UPPER BOUNG", bound)
model.train(True)
cos = nn.CosineSimilarity()
for i, (v, q, a, b, answertypefeats,
top_ans_emb) in tqdm(enumerate(train_loader),
ncols=100,
desc="Epoch %d" % (epoch + 1),
total=len(train_loader)):
total_step += 1
if tiny_train and total_step == 10:
break
v = Variable(v).cuda()
q = Variable(q).cuda()
a = Variable(a).cuda()
b = Variable(b).cuda()
answertypefeats = Variable(answertypefeats).cuda()
top_ans_emb = Variable(top_ans_emb).cuda()
gen_embs, pred, loss, all_ans_embs = model(v, None, q, a, b)
all_ans_embs = torch.stack([all_ans_embs] * gen_embs.shape[0])
# print("gen_embs.shape BEFORE unsqueeze", gen_embs.shape)
# print("all_ans_embs.shape", all_ans_embs.shape)
# print(type(gen_embs), type(all_ans_embs))
sys.stdout.flush()
if (loss != loss).any():
raise ValueError("NaN loss")
## NCE LOSS
positive_dist = cos(gen_embs, top_ans_emb) # shape b,k;b,k-> b
gen_embs = torch.cat([gen_embs.unsqueeze(1)] *
all_ans_embs.shape[1],
dim=1)
d_logit = cos(gen_embs, all_ans_embs)
# print("gen_embs.shape AFTER unsqueeze", gen_embs.shape)
# print("all_ans_embs.shape", all_ans_embs.shape)
num = torch.exp(positive_dist).squeeze(-1)
den = torch.exp(d_logit).sum(-1)
# print("num.shape,den.shape", num.shape,den.shape)
# sys.stdout.flush()
loss_nce = -1 * torch.log(num / den)
loss_nce = loss_nce.mean()
loss = loss + loss_nce
loss.backward()
nn.utils.clip_grad_norm(model.parameters(), 0.25)
optim.step()
optim.zero_grad()
batch_score = compute_score_with_logits(pred, a.data).sum()
total_loss += loss.data[0] * v.size(0)
train_score += batch_score
if tiny_train:
L = total_step
else:
L = len(train_loader.dataset)
total_loss /= L
train_score = 100 * train_score / L
run_eval = eval_each_epoch or (epoch == num_epochs - 1)
if run_eval:
model.train(False)
results = evaluate(model, eval_loader, tiny_train)
results["epoch"] = epoch + 1
results["step"] = total_step
results["train_loss"] = total_loss
results["train_score"] = train_score
eval_score = results["score"]
bound = results["upper_bound"]
all_results.append(results)
with open(join(output, "results.json"), "w") as f:
json.dump(all_results, f, indent=2)
model.train(True)
logger.write('epoch %d, time: %.2f' % (epoch + 1, time.time() - t))
logger.write('\ttrain_loss: %.2f, score: %.2f' %
(total_loss, train_score))
if run_eval:
logger.write('\teval score: %.2f (%.2f)' %
(100 * eval_score, 100 * bound))
model_path = os.path.join(output, 'model' + str(epoch) + '.pth')
torch.save(model.state_dict(), model_path)
# concate input and label together
img_pair = np.concatenate([img_input, img_label], 2)
# crop the patch
patch_list = crop_patch(img_pair, 100, 100, False)
# save the patches into h5py file
for patch_idx in range(len(patch_list)):
data = patch_list[patch_idx].copy()
h5f.create_dataset(str(img_idx) + '_' + str(patch_idx),
shape=(200, 200, 8),
data=data)
h5f.close()
if __name__ == "__main__":
# generating train/valid/test datasets
# dataset_type = 'train'
src_input_path = ".../input"
src_label_path = ".../label"
dst_path = ".../h5py_file"
create_dir(dst_path)
src_input_files = sorted(glob.glob(src_input_path + "/*.png"))
src_label_files = sorted(glob.glob(src_label_path + "/*.png"))
print("start dataset generation!")
gen_dataset(src_input_files, src_label_files, dst_path)
# plt.colorbar()
plt.scatter(clcd[:, 0],
clcd[:, 1],
marker='s',
s=5,
alpha=.3,
edgecolors='none')
plt.xlabel(r'$C_L$')
if pos == 'first':
plt.ylabel(r'$C_D$')
if __name__ == "__main__":
results_dir = 'results_clcd'
create_dir(results_dir)
database = np.load('data/airfoil_interp.npy')
N = database.shape[0]
''' Data '''
fname = '{}/clcd_data.npy'.format(results_dir)
if os.path.exists(fname):
clcd_data = np.load(fname)
else:
list_cl = []
list_cd = []
for i, airfoil in enumerate(database):
if detect_intersect(airfoil):
te = (airfoil[0] + airfoil[-1]) / 2
airfoil[0] = airfoil[-1] = te
_, cl, cd = evaluate(airfoil, return_CL_CD=True)
def generate_metadata(tf_config, input_audio, labels):
"""Generates metadata file containing sample rate and thresholds of each keyword"""
K.clear_session()
sess = tf.Session(config=tf_config)
set_session(sess)
K.set_learning_phase(0)
model_file = join(args.tmpdir, kCheckpointFileName + '.h5')
model = load_model(model_file)
prediction_probabilities = model.predict(input_audio, batch_size=args.batch_size)
label_ids = np.where(labels)[1]
sorted_indices = label_ids.argsort()
prediction_probabilities = prediction_probabilities[sorted_indices]
label_ids = label_ids[sorted_indices]
class_range = np.concatenate(([0], np.cumsum(labels.sum(axis=0)))).astype(int)
# Metadata
num_classes = len(args.keywords_list)
thresholds = [None] * num_classes
max_range = labels.sum(axis=0).min().astype(int) * (num_classes - 1)
# Compute optimal threshold for each class
for i in range(num_classes):
total = min(class_range[i + 1] - class_range[i], max_range)
chosen_indices = np.random.choice(
np.arange(class_range[i], class_range[i + 1]), total, replace=False)
target_labels = [label_ids[chosen_indices]]
predictions = [prediction_probabilities[chosen_indices, i]]
split = np.round(total / (num_classes - 1)).astype(int)
for j in range(num_classes):
if j != i:
chosen_indices = np.random.choice(
np.arange(class_range[j], class_range[j + 1]), min(split, total), replace=False)
total -= split
target_labels.append(label_ids[chosen_indices])
predictions.append(prediction_probabilities[chosen_indices, i])
target_labels = np.concatenate(target_labels)
predictions = np.concatenate(predictions)
thresholds[i] = find_optimal_threshold(target_labels, predictions, i)
# Prepare and save the metadata
input_tensor = model.inputs[0]
input_layer_name = input_tensor.name.split(':')[0].split('/')[0]
input_tensor_shape = [1, DataConfig.kWindowLength, DataConfig.kNumMels, DataConfig.kNumFeatures]
metadata = OrderedDict(
[('<feature_extraction_node_name>', OrderedDict(
[('isaac.audio.VoiceCommandFeatureExtraction', OrderedDict(
[('sample_rate', DataConfig.kSampleRate),
('fft_length', DataConfig.kFftLength),
('num_mels', DataConfig.kNumMels),
('hop_size', DataConfig.kHopLength),
('window_length', DataConfig.kWindowLength),
('num_classes', num_classes),
('classes', args.keywords_list)]))
])),
('<tensorflow_inference_node_name>', OrderedDict(
[('isaac.ml.TensorflowInference', OrderedDict(
[('input_tensor_info',
[OrderedDict([('ops_name', input_layer_name),
('index', 0),
('dims', input_tensor_shape)])
]),
('output_tensor_info',
[OrderedDict(
[('ops_name', kOutputOpName),
('index', 0),
('dims', [1, num_classes])])
])
]))
])),
('<command_construction_node_name>', OrderedDict(
[('isaac.audio.VoiceCommandConstruction', OrderedDict(
[('num_classes', num_classes),
('classes', args.keywords_list),
('thresholds', list(thresholds))]))
]))
])
create_dir(args.model_output_path)
with open(join(args.model_output_path, kMetadataFileName), 'w') as f:
json.dump(metadata, f, indent=2)
""" Driver function to control what is run and when if this is the main python script being ran.
As the project was developed in a jupyter notebook, everything is self-contained in the main file.
Any expansion, however, would be able to use the predefined classes and functions for whatever purpose without running anything.
"""
history, model, eval_X, eval_Y = None, None, None, None
if (not args.train and not args.evaluate and not args.predict):
print(
"Type -h and get a list of possible tasks. You may select multiple."
)
else:
if (args.train):
history, model, eval_X, eval_Y = train_generator(out_file)
plot_training_history(history.history, "Training History")
if (args.evaluate):
history = evaluate_generator(model, save_location, out_file)
print(history)
if (args.predict):
choreograph_dance(model, save_location, out_file)
if __name__ == "__main__":
reload_packages()
utils.create_dir(np_save_dir)
save_location = utils.create_dir(logs_save_dir)
out_file = open(os.path.join(save_location, "outfile.txt"), "w")
main(save_location, out_file)
out_file.close()
# In[ ]:
"""
model = torch.load(MODEL_PATH).to(device)
test_iterator1, test_iterator2 = load_test()
# Evaluate test loss and accuracy
test_loss, test_acc = evaluate_(model, test_iterator2)
print("Test Loss: {} | Test Acc: {}%".format(
round(test_loss, 2), round(test_acc*100, 2)))
if __name__ == "__main__":
# Creating necessary folders
# ------------------------------------------
utils.create_dir(utils.RESULTS_FOLDER, delete_old=False)
utils.create_dir(utils.MODELS_FOLDER, delete_old=False)
utils.create_dir("{}{}/".format(utils.RESULTS_FOLDER, MODEL_NAME), False)
# Logging
# ------------------------------------------
logging.basicConfig(
filename="{}{}/{}.log".format(utils.RESULTS_FOLDER,
MODEL_NAME, MODEL_NAME),
filemode="a",
level=logging.DEBUG,
format="%(asctime)s - %(levelname)s: %(message)s",
datefmt="%Y/%m/%d %H:%M:%S",
)
def __init__(self, generator, test_dataset, out_dir, num_img=2):
self.num_img = num_img
self.generator = generator
self.test_dataset = test_dataset
self.out_dir = create_dir(out_dir)
def main(config):
create_dir(config['result_dir'])
output_dir = os.path.join(config['result_dir'], 'labels')
create_dir(output_dir)
atlas_paths = {
'images': list(np.loadtxt(config['train_ims_file'], dtype='str')),
'labels': list(np.loadtxt(config['train_lbs_file'], dtype='str'))
}
print('Loading test images', end=' ', flush=True)
test_image_paths = list(np.loadtxt(config['test_ims_file'], dtype='str'))
print('Ok')
print('Loading Elastix registration parameters', end=' ', flush=True)
parameter_map_lst = [
sitk.ReadParameterFile(os.path.join(config['param_dir'], f))
for f in sorted(os.listdir(config['param_dir']))
]
print('Ok')
print('Building Multi-Atlas Segmentation model', end=' ', flush=True)
mas = MultiAtlasSegmentation(atlas_paths, config['atlas_size'],
config['image_metric'],
config['label_fusion'])
print('Ok')
print('Building RCA Classifier', end=' ', flush=True)
rca = SingleAtlasClassifier()
print('Ok')
print('Processing test images')
dice_scores = []
for i, image_path in enumerate(test_image_paths):
print('{}/{} Image filename: {}'.format(i + 1, len(test_image_paths),
os.path.basename(image_path)))
# Predict segmentation
print(' - Predicting segmentation', end=' ', flush=True)
label, label_path, atlas_idxs = mas.predict_segmentation(
image_path, output_dir, parameter_map_lst)
print('Ok')
atlas_paths_rca = {
'images': [atlas_paths['images'][i] for i in atlas_idxs],
'labels': [atlas_paths['labels'][i] for i in atlas_idxs]
}
# Predict accuracy
print(' - Predicting accuracy', end=' ', flush=True)
rca.set_atlas_path(image_path, label_path)
dice = rca.predict_dice(atlas_paths_rca, output_dir, parameter_map_lst)
dice_scores.append(dice)
print('Ok')
# Save results
print('Saving results', end=' ', flush=True)
with open(os.path.join(config['result_dir'], 'rca_predictions.csv'),
'w') as f:
writer = csv.DictWriter(f, fieldnames=['file', 'dice'])
writer.writeheader()
for i in range(len(test_image_paths)):
writer.writerow({
'file': os.path.basename(test_image_paths[i]),
'dice': dice_scores[i]
})
print('Ok')
def train(model,
train_loader,
eval_loader,
num_epochs,
output,
opt=None,
s_epoch=0):
lr_default = 1e-3 if eval_loader is not None else 7e-4
lr_decay_step = 2
lr_decay_rate = .25
lr_decay_epochs = range(
10, 20, lr_decay_step) if eval_loader is not None else range(
10, 20, lr_decay_step)
gradual_warmup_steps = [
0.5 * lr_default, 1.0 * lr_default, 1.5 * lr_default, 2.0 * lr_default
]
saving_epoch = 3
grad_clip = .25
utils.create_dir(output)
optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) \
if opt is None else opt
logger = utils.Logger(os.path.join(output, 'log.txt'))
best_eval_score = 0
utils.print_model(model, logger)
logger.write('optim: adamax lr=%.4f, decay_step=%d, decay_rate=%.2f, grad_clip=%.2f' % \
(lr_default, lr_decay_step, lr_decay_rate, grad_clip))
if os.path.exists('./saved_models/ban/model_epoch16.pth'):
checkpoint = torch.load('./saved_models/ban/model_epoch16.pth')
model.load_state_dict(checkpoint.get('model_state', checkpoint))
s_epoch = checkpoint.get('epoch')
logger.write('reload from %d epoch' % s_epoch)
for epoch in range(s_epoch, num_epochs):
total_loss = 0
train_score = 0
total_norm = 0
count_norm = 0
t = time.time()
N = len(train_loader.dataset)
if epoch < len(gradual_warmup_steps):
optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch]
logger.write('gradual warmup lr: %.4f' %
optim.param_groups[0]['lr'])
elif epoch in lr_decay_epochs:
optim.param_groups[0]['lr'] *= lr_decay_rate
logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr'])
else:
logger.write('lr: %.4f' % optim.param_groups[0]['lr'])
for i, (v, b, q, a) in enumerate(train_loader):
v = v.cuda()
b = b.cuda()
q = q.cuda()
a = a.cuda()
pred, att = model(v, b, q, a)
loss = instance_bce_with_logits(pred, a)
loss.backward()
total_norm += nn.utils.clip_grad_norm_(model.parameters(),
grad_clip)
count_norm += 1
optim.step()
optim.zero_grad()
batch_score = compute_score_with_logits(pred, a.data).sum()
total_loss += loss.item() * v.size(0)
train_score += batch_score.item()
total_loss /= N
train_score = 100 * train_score / N
if None != eval_loader:
model.train(False)
eval_score, bound, entropy = evaluate(model, eval_loader)
model.train(True)
logger.write('epoch %d, time: %.2f' % (epoch, time.time() - t))
logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' %
(total_loss, total_norm / count_norm, train_score))
if eval_loader is not None:
logger.write('\teval score: %.2f (%.2f)' %
(100 * eval_score, 100 * bound))
if eval_loader is not None and entropy is not None:
info = ''
for i in range(entropy.size(0)):
info = info + ' %.2f' % entropy[i]
logger.write('\tentropy: ' + info)
if (eval_loader is not None and eval_score > best_eval_score) or (
eval_loader is None and epoch >= saving_epoch):
model_path = os.path.join(output, 'model_epoch%d.pth' % epoch)
utils.save_model(model_path, model, epoch, optim)
if eval_loader is not None:
best_eval_score = eval_score
print('######################################################')
start_time = time.time()
opt_x, opt_airfoil, opt_perfs = optimize(func, perturb_type, perturb,
n_eval)
end_time = time.time()
opt_x_runs.append(opt_x)
opt_airfoil_runs.append(opt_airfoil)
opt_perfs_runs.append(opt_perfs)
time_runs.append(end_time - start_time)
opt_x_runs = np.array(opt_x_runs)
opt_airfoil_runs = np.array(opt_airfoil_runs)
opt_perfs_runs = np.array(opt_perfs_runs)
save_dir = 'results_opt/ffd_ga'
create_dir(save_dir)
np.save('{}/opt_solution.npy'.format(save_dir), opt_x_runs)
np.save('{}/opt_airfoil.npy'.format(save_dir), opt_airfoil_runs)
np.save('{}/opt_history.npy'.format(save_dir), opt_perfs_runs)
# Plot optimization history
mean_perfs_runs = np.mean(opt_perfs_runs, axis=0)
plt.figure()
plt.plot(np.arange(n_eval + 1, dtype=int), opt_perfs)
plt.title('Optimization History')
plt.xlabel('Number of Evaluations')
plt.ylabel('Optimal CL/CD')
# plt.xticks(np.linspace(0, n_eval+1, 5, dtype=int))
plt.savefig('{}/opt_history.svg'.format(save_dir))
plt.close()
def gerberise(manufacturer='default'):
"""
Generate Gerbers for one or more layers
"""
# Open the board's SVG
svg_in = utils.openBoardSVG()
# Get Gerber generation settings
gcd = config.brd['gerber']
decimals = gcd['decimals']
digits = gcd['digits']
steps = gcd['steps-per-segment']
length = gcd['min-segment-length']
# Get layer data
xpath_regex = ""
ns = {'pcbmode':config.cfg['ns']['pcbmode'],
'svg':config.cfg['ns']['svg']}
# Save to file
base_dir = os.path.join(config.cfg['base-dir'],
config.cfg['locations']['build'],
'production')
# Create directory if it doesn't exist already
utils.create_dir(base_dir)
base_name = "%s_rev_%s" % (config.brd['config']['name'],
config.brd['config']['rev'])
filename_info = config.cfg['manufacturers'][manufacturer]['filenames']['gerbers']
# Process Gerbers for PCB layers and sheets
for pcb_layer in utils.getSurfaceLayers():
# Get the SVG layer corresponding to the PCB layer
svg_layer = svg_in.find("//svg:g[@pcbmode:pcb-layer='%s']" % (pcb_layer),
namespaces=ns)
# Get masks (must be placed right after pours)
mask_paths = svg_in.findall(".//svg:defs//svg:mask[@pcbmode:pcb-layer='%s']//svg:path" % pcb_layer,
namespaces=ns)
sheets = ['copper', 'soldermask', 'solderpaste', 'silkscreen']
for sheet in sheets:
# Get the SVG layer corresponding to the 'sheet'
sheet_layer = svg_layer.find(".//svg:g[@pcbmode:sheet='%s']" % (sheet),
namespaces=ns)
if sheet == 'copper':
mask_paths_to_pass = mask_paths
else:
mask_paths_to_pass = []
# Create a Gerber object
gerber = Gerber(sheet_layer,
mask_paths_to_pass,
decimals,
digits,
steps,
length)
add = '_%s_%s.%s' % (pcb_layer, sheet,
filename_info[pcb_layer][sheet].get('ext') or 'ger')
filename = os.path.join(base_dir, base_name + add)
with open(filename, "wb") as f:
for line in gerber.getGerber():
f.write(line)
# Process module sheets
sheets = ['outline', 'documentation']
for sheet in sheets:
# Get the SVG layer corresponding to the 'sheet'
sheet_layer = svg_in.find(".//svg:g[@pcbmode:sheet='%s']" % (sheet),
namespaces=ns)
# Create a Gerber object
gerber = Gerber(sheet_layer,
[],
decimals,
digits,
steps,
length)
add = '_%s.%s' % (sheet,
filename_info['other'][sheet].get('ext') or 'ger')
filename = os.path.join(base_dir, base_name + add)
with open(filename, "wb") as f:
for line in gerber.getGerber(False):
f.write(line)
return ['bullshit']
def train(model,
train_loader,
eval_loader,
num_epochs,
output,
opt=None,
s_epoch=0):
lr_default = 1e-3 if eval_loader is not None else 7e-4
lr_decay_step = 2
lr_decay_rate = 1
lr_decay_epochs = range(
10, 20, lr_decay_step) if eval_loader is not None else range(
10, 20, lr_decay_step)
gradual_warmup_steps = [
0.5 * lr_default, 1.0 * lr_default, 1.5 * lr_default, 2.0 * lr_default
]
saving_epoch = 3
grad_clip = .25
utils.create_dir(output)
optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) \
if opt is None else opt
logger = utils.Logger(os.path.join(output, 'log.txt'))
best_eval_score = 0
utils.print_model(model, logger)
logger.write('optim: adamax lr=%.4f, decay_step=%d, decay_rate=%.2f, grad_clip=%.2f' % \
(lr_default, lr_decay_step, lr_decay_rate, grad_clip))
for epoch in range(s_epoch, num_epochs):
total_loss = 0
train_score = 0
total_norm = 0
count_norm = 0
t = time.time()
N = len(train_loader.dataset)
print(N)
if epoch < len(gradual_warmup_steps):
optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch]
logger.write('gradual warmup lr: %.4f' %
optim.param_groups[0]['lr'])
elif epoch in lr_decay_epochs:
optim.param_groups[0]['lr'] = 1e-3
logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr'])
else:
logger.write('lr: %.4f' % optim.param_groups[0]['lr'])
'''
for i, (v, b, q, a,ques,im,g,gender) in enumerate(train_loader):
v = v.cuda()
b = b.cuda()
q = q.cuda()
a = a.cuda()
visual_pred, att = model(v, b, q, a)
#import pdb;pdb.set_trace()
gender=gender.squeeze(1)
weights=torch.Tensor([2.0,1.0,0.0001]).cuda()
#loss = instance_bce_with_logits(visual_pred, g.cuda())
loss=nn.CrossEntropyLoss(weights)
loss=loss(visual_pred,gender.cuda())
#import pdb;pdb.set_trace()
loss.backward()
total_norm += nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
count_norm += 1
optim.step()
optim.zero_grad()
batch_score=torch.eq(visual_pred.argmax(1),gender.cuda()).sum()
#batch_score = compute_score_with_logits(visual_pred, g.cuda()).sum()
#total_loss += loss.item() * v.size(0)
train_score += batch_score.item()
#train_score+=batch_score
'''
total_loss /= N
train_score = 100 * train_score / N
if None != eval_loader:
model.train(False)
eval_score, bound, _ = evaluate(model, eval_loader)
model.train(True)
logger.write('epoch %d, time: %.2f' % (epoch, time.time() - t))
logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' %
(total_loss, total_norm / count_norm, train_score))
logger.write('\teval score: %.2f (%.2f)' %
(100 * eval_score, 100 * bound))
model_path = os.path.join(output, 'model_epoch%d.pth' % epoch)
utils.save_model(model_path, model, epoch, optim)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='gdi', help='name of dataset, gdi or massvis (default: gdi)')
parser.add_argument('--dataset_dir', type=str, default='/path/to/datase_dirt', help='dataset directory')
parser.add_argument('--fcn_type', type=str, help='FCN type, fcn32 or fcn16 (default: gdi)', default='fcn32',
choices=['fcn32', 'fcn16'])
parser.add_argument('--overlaid_img_dir', type=str, default='/path/to/overlaid_img_dir',
help='output directory path for images with heatpmap overlaid onto input images')
parser.add_argument('--pretrained_model', type=str, default='/path/to/pretrained_model',
help='pretrained model converted from Caffe models')
parser.add_argument('--config', type=int, default=1, choices=configurations.keys(),
help='configuration for training where several hyperparameters are defined')
parser.add_argument('--log_file', type=str, default='F:/dataset/visimportance/log', help='/path/to/log_file')
parser.add_argument('--resume', type=str, default='',
help='checkpoint file to be loaded when retraining models')
parser.add_argument('--checkpoint_dir', type=str, default='/path/to/checkpoint_dir',
help='checkpoint file to be saved in each epoch')
parser.add_argument('--eval_only', action='store_true', help='evaluation only')
parser.add_argument('--gpu', type=int, default=0, help='GPU id (default: 0)')
args = parser.parse_args()
utils.create_dir(os.path.join(args.overlaid_img_dir, "train"))
utils.create_dir(os.path.join(args.overlaid_img_dir, "valid"))
if not args.eval_only:
utils.create_dir(args.checkpoint_dir)
print(args)
gpu = args.gpu
cfg = configurations[args.config]
log_file = args.log_file
resume = args.resume
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
cuda = torch.cuda.is_available()
args.cuda = cuda
if args.cuda:
print("torch.backends.cudnn.version: {}".format(torch.backends.cudnn.version()))
torch.manual_seed(1337)
if cuda:
torch.cuda.manual_seed(1337)
# 1. dataset
root = os.path.expanduser(args.dataset_dir)
kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {}
train_loader = None
if not args.eval_only: # training + validation
if args.dataset == 'gdi':
dt = GDI(root, image_dir="gd_train", imp_dir="gd_imp_train", split='train', transform=True)
else:
dt = Massvis(root, image_dir="train", imp_dir="train_imp", split='train', transform=True)
train_loader = torch.utils.data.DataLoader(dt, batch_size=1, shuffle=True, **kwargs)
print("no of images in training", len(train_loader))
if args.dataset == 'gdi': # validation
dv = GDI(root, image_dir="gd_val", imp_dir="gd_imp_val", split='valid', transform=True)
else:
dv = Massvis(root, image_dir="valid", imp_dir="valid_imp", split='valid', transform=True)
val_loader = torch.utils.data.DataLoader(dv, batch_size=1, shuffle=False, **kwargs)
print("no of images in evaluation", len(val_loader))
# 2. model
model = models.FCN32s() if args.fcn_type == 'fcn32' else models.FCN16s()
start_epoch = 0
start_iteration = 0
if resume:
checkpoint = torch.load(resume)
model.load_state_dict(checkpoint['model_state_dict'])
start_epoch = checkpoint['epoch']
start_iteration = checkpoint['iteration']
if args.fcn_type == 'fcn32':
assert checkpoint['arch'] == 'FCN32s'
else:
assert checkpoint['arch'] == 'FCN16s'
else:
if args.fcn_type in ['fcn32', 'fcn16']:
model_weight = torch.load(args.pretrained_model)
model.load_state_dict(model_weight)
if not args.eval_only:
model._initialize_weights()
else:
fcn32s = models.FCN32s()
checkpoint = torch.load(args.pretrained_model)
fcn32s.load_state_dict(checkpoint['model_state_dict'])
model.copy_params_from_fcn32s(fcn32s)
model._initialize_weights()
if cuda:
model = model.cuda()
# 3. optimizer
optim = torch.optim.SGD(
[
{'params': get_parameters(model, bias=False, fcn_type=args.fcn_type)},
{'params': get_parameters(model, bias=True, fcn_type=args.fcn_type), 'lr': cfg['lr'] * 2, 'weight_decay': 0},
],
lr=cfg['lr'],
momentum=cfg['momentum'],
weight_decay=cfg['weight_decay'])
if resume:
optim.load_state_dict(checkpoint['optim_state_dict'])
# lr_policy: step
last_epoch = start_iteration if resume else -1
lr_scheduler = torch.optim.lr_scheduler.StepLR(optim, cfg['step_size'], gamma=cfg['gamma'], last_epoch=last_epoch)
trainer = Trainer(
cuda=cuda,
model=model,
optimizer=optim,
lr_scheduler=lr_scheduler,
train_loader=train_loader,
val_loader=val_loader,
checkpoint_dir=args.checkpoint_dir,
log_file=log_file,
max_iter=cfg['max_iteration'],
iter_size=cfg['iter_size'],
interval_validate=cfg.get('interval_validate', len(train_loader)) if not args.eval_only else 0,
overlaid_img_dir=args.overlaid_img_dir,
dataset=args.dataset,
eval_only=args.eval_only,
)
trainer.epoch = start_epoch
trainer.iteration = start_iteration
if not args.eval_only:
trainer.train()
else:
trainer.validate()