def main():
if not os.path.exists(IMAGE_DIR):
os.mkdir(IMAGE_DIR)
empty_dir(IMAGE_DIR)
# p_gtype=2 (old clothes) has a good balance of person and clothe images.
url_base = 'https://zozo.jp/category/tops/?p_gtype=2&pno={page_number}'
num_pages = 500
interval = 30 # seconds
num_cpus = multiprocessing.cpu_count()
for page_number in range(1, num_pages + 1):
print(f'{page_number} / {num_pages}')
# parser image source
url = url_base.format(page_number=page_number)
soup = make_soup(url)
srcs = parse_img_src(soup)
# download images with multiprocessing
with multiprocessing.Pool(num_cpus) as p:
with tqdm(total=len(srcs)) as t:
for _ in p.imap_unordered(download_image, srcs):
t.update(1)
print(f'Waiting for {interval} seconds...')
time.sleep(interval)
def clone(self, printer, indicator, prompter, username):
if username.strip() is '':
printer('Invalid username! Please try again.')
indicator(f'Cloning failed.', False)
return 1
else:
save_current_user(username)
printer(f'Scanning user GitHub account \'{username}\'...')
indicator(f'Scanning GitHub account...', True)
generate_meta_json(username)
printer(f'Fetching repositories of user \'{username}\'...')
indicator(f'Cloning from {username}...', True)
repo_list = get_repo_list(username)
repo_count = len(repo_list)
if repo_count == 0:
printer(f'No repositories found for user \'{username}\'! Please try again.')
indicator(f'Cloning failed.', False)
return 2
printer(f'Found {repo_count} repositories.')
proceed = prompter(
f'Do you want to clone all {repo_count} repositories found for the user \'{username}\'?')
if proceed:
empty_dir('./repos')
for url in repo_list:
printer(f'Cloning repository \'{url}\'')
clone_by_url(url)
printer(f'Cloned {repo_count} repositories successfully!')
indicator(f'Cloning successful!', False)
return 0
else:
printer('Cloning aborted by user. Previously cloned repositories might be removed!')
indicator(f'Cloning aborted.', False)
return -1
def main():
empty_dir(TRAIN_IMAGE_DIR)
df = pd.read_csv(os.path.join(OUTPUT_DIR, 'labels.csv'))
for filename in tqdm(df['filename']):
src = os.path.join(IMAGE_DIR, filename)
dst = os.path.join(TRAIN_IMAGE_DIR, filename)
copyfile(src, dst)
def unzip_with_progress(file_path, path):
empty_dir(path)
zf = zipfile.ZipFile(file_path)
extracted_size = 0
for contain_file in zf.infolist():
extracted_size += contain_file.file_size
zf.extract(contain_file, path)
def run_evaluate(inputdir, outputdir, log):
os.system("python batch2lg.py '%s'" % inputdir)
empty_dir('Results_%s' % outputdir)
os.system("evaluate '%s' '%s'" % (outputdir, inputdir))
os.system("cat 'Results_%sSummary.txt'" % outputdir)
cont = open("Results_%sSummary.txt" % outputdir).read()
cont = "# LOG: " + log + "\nENDLOG\n" + cont
now = str(datetime.datetime.utcnow()).split(".")[0].replace(" ","_")
open("Summary_%s.txt" % now, 'w').write(cont)
def main():
mkdir_if_not_exists(IMAGE_DIR)
empty_dir(IMAGE_DIR)
grab_area = read_grab_area()
crop_areas = find_crop_areas()
for _ in tqdm(range(2 ** 11)):
ss = grab_screen(grab_area)
draw_crop_areas(ss, crop_areas)
for img in crop_images(ss, crop_areas):
save_path = os.path.join(IMAGE_DIR, generate_uuid() + '.png')
cv2.imwrite(save_path, img)
refresh(grab_area)
time.sleep(1)
OUTPUT:
"""
##### ARGUMENTS #####
config = argparser()
assert config.audios_sph_to_transcript_fname is not None
data_dir = os.path.dirname(config.audios_sph_to_transcript_fname)
audios_sph_to_transcript_map = read_jsonl_file(config.audios_sph_to_transcript_fname)
out_wav_dname = os.path.join(data_dir, "segmented-audio-wav")
out_transcript_dname = os.path.join(data_dir, "segmented-transcripts")
for dname in [out_wav_dname, out_transcript_dname]:
empty_dir(dname, allow_data_folder_deletion=True)
if not os.path.exists(dname):
os.makedirs(dname)
##### SOX Transformer #####
sox_tfm = sox.Transformer()
##### RUN #####
for entry in audios_sph_to_transcript_map:
inp_sph_fname = os.path.join(data_dir, entry["audio_sph_file"])
inp_transcript_fname = os.path.join(data_dir, entry["transcript_file"])
filter_criteria = entry["filter_criteria"]
###
transcript_segments = get_transcript_segments(inp_transcript_fname) # list. each elem is (start_time, end_time, text). times are in seconds.
write_segmented_wavs_and_transcripts(data_dir, inp_sph_fname, transcript_segments, out_wav_dname, out_transcript_dname, entry, sox_tfm)
def run(dataset, model, str_optimizer, str_preconditioner, runs, epochs, lr,
weight_decay, early_stopping, logger, momentum, eps, update_freq,
gamma, alpha, hyperparam):
if logger is not None:
if hyperparam:
logger += f"-{hyperparam}{eval(hyperparam)}"
path_logger = os.path.join(path_runs, logger)
print(f"path logger: {path_logger}")
ut.empty_dir(path_logger)
logger = SummaryWriter(log_dir=os.path.join(
path_runs, logger)) if logger is not None else None
val_losses, accs, durations = [], [], []
torch.manual_seed(42)
for i_run in range(runs):
data = dataset[0]
data = data.to(device)
model.to(device).reset_parameters()
if str_preconditioner == 'KFAC':
preconditioner = psgd.KFAC(
model,
eps,
sua=False,
pi=False,
update_freq=update_freq,
alpha=alpha if alpha is not None else 1.,
constraint_norm=False)
else:
preconditioner = None
if str_optimizer == 'Adam':
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
elif str_optimizer == 'SGD':
optimizer = torch.optim.SGD(
model.parameters(),
lr=lr,
momentum=momentum,
)
if torch.cuda.is_available():
torch.cuda.synchronize()
t_start = time.perf_counter()
best_val_loss = float('inf')
test_acc = 0
val_loss_history = []
for epoch in range(1, epochs + 1):
lam = (float(epoch) /
float(epochs))**gamma if gamma is not None else 0.
train(model, optimizer, data, preconditioner, lam)
eval_info = evaluate(model, data)
eval_info['epoch'] = int(epoch)
eval_info['run'] = int(i_run + 1)
eval_info['time'] = time.perf_counter() - t_start
eval_info['eps'] = eps
eval_info['update-freq'] = update_freq
if gamma is not None:
eval_info['gamma'] = gamma
if alpha is not None:
eval_info['alpha'] = alpha
if logger is not None:
for k, v in eval_info.items():
logger.add_scalar(k, v, global_step=epoch)
if eval_info['val loss'] < best_val_loss:
best_val_loss = eval_info['val loss']
test_acc = eval_info['test acc']
val_loss_history.append(eval_info['val loss'])
if early_stopping > 0 and epoch > epochs // 2:
tmp = tensor(val_loss_history[-(early_stopping + 1):-1])
if eval_info['val loss'] > tmp.mean().item():
break
if torch.cuda.is_available():
torch.cuda.synchronize()
t_end = time.perf_counter()
val_losses.append(best_val_loss)
accs.append(test_acc)
durations.append(t_end - t_start)
if logger is not None:
logger.close()
loss, acc, duration = tensor(val_losses), tensor(accs), tensor(durations)
print(
'Val Loss: {:.4f}, Test Accuracy: {:.2f} ± {:.2f}, Duration: {:.3f} \n'
.format(loss.mean().item(), 100 * acc.mean().item(),
100 * acc.std().item(),
duration.mean().item()))
def build(self):
t = time.time()
utils.check_dir('db')
utils.empty_dir('db')
utils.check_dir('wally')
utils.empty_dir('wally')
db = database.Database()
conn = database.Connector(db)
conn.root['conf'] = PersistentDict()
conn.root['conf']['size'] = self.size
conn.root['conf']['tick'] = 0
conn.root['conf']['mutation'] = self.mutation
conn.root['conf']['capacity'] = self.capacity
conn.root['rookies'] = {}
# nada
conn.root['nada'] = (matplotlib.image.imread('meteo/nada.png') == 1)
# meteo
conn.root['meteo'] = numpy.zeros((10, self.size, self.size, 3), dtype=numpy.float)
for i in xrange(10):
sol = matplotlib.image.imread('meteo/sol_%i.png' %i) * self.meteo_max
lluvia = matplotlib.image.imread('meteo/lluvia_%i.png' %i) * self.meteo_max
pool = matplotlib.image.imread('meteo/pool_%i.png' %i) * self.meteo_max
for x in xrange(self.size):
for y in xrange(self.size):
conn.root['meteo'][i,x,y,0] = sol[x,y]
conn.root['meteo'][i,x,y,1] = lluvia[x,y]
conn.root['meteo'][i,x,y,2] = pool[x,y]
# wally
conn.root['wally'] = numpy.zeros((self.size, self.size, self.capacity, 12), dtype=numpy.int32)
p_gen = 1
p_start = 0
p_nrg = 1000
f_gen = 0
f_max = 100
f_num = 15
f_nrg = 1000
f_sex = 3000
f_luz = 50
f_agua = 50
f_pool = 50
f_meta = 3000
#[(self.size/4, self.size/4),
#(self.size/4, 3*self.size/4),
#(3*self.size/4, self.size/4),
#(3*self.size/4, 3*self.size/4)]
adanes = [(14, 70), (38, 70), (64, 70), (82, 70),
(14, 30), (38, 30), (64, 30), (82, 30)]
for pos in adanes:
conn.root['wally'][pos[0],pos[1]][0] = [p_gen,p_start,p_nrg,f_gen,f_max,f_num, \
f_nrg,f_sex,f_luz,f_agua,f_pool,f_meta]
print '\nBuild completed in %f secs\n' % (time.time()-t)
conn.close()
db.close()