def rename_exercise_files(self, down_dir):
# get downloaded temporary file
tmp_fname = ''
wait_time = 3 # check download dir every wait_time seconds
wait_time_max = 10 * 60 # give up after wait_time_max seconds
nretry = 0
nretry_max = wait_time_max / wait_time
while True:
tmp_flist = [
self.cache_dir + '/' + x for x in os.listdir(self.cache_dir)
if x.endswith('.zip')
]
if len(tmp_flist) != 0:
tmp_flist.sort(key=lambda x: os.path.getmtime(x), reverse=True)
tmp_fname = tmp_flist[0]
break
utils.wait(3)
nretry += 1
if nretry == nretry_max:
utils.print_message('*ERROR*: download time out. retry later!')
utils.clean_dir(self.cache_dir)
sys.exit(1)
os.makedirs(down_dir, exist_ok=True)
new_fname = '{}/{}'.format(down_dir, tmp_fname.split('/')[-1])
os.rename(tmp_fname, new_fname)
utils.print_message(
'save downloaded exercise files as {}'.format(new_fname))
return new_fname
def test():
album = Album(directory='tests/images', clustering_method='som')
build_dir = os.path.join('build', "tests/vocabularySOM")
if not os.path.exists(build_dir):
os.makedirs(build_dir)
clean_dir(build_dir)
font = cv2.FONT_HERSHEY_COMPLEX_SMALL
for image in album.images:
logger.debug('Started processing output for %s' % image.filename)
grayed = image.grayed
height, width = grayed.shape
outfile = os.path.join(build_dir, os.path.basename(image.filename))
for word in image.words:
if word.noise:
logger.debug('Skipped noisy word %d' % word.value)
continue
cv2.putText(
grayed,
'%d' % word.value,
(int(word.pt[0]), int(word.pt[1])),
font, 0.5, (255, 0, 0), 1, cv2.CV_AA)
cv2.imwrite(outfile, grayed)
logger.debug('Finished processing output for %s' % image.filename)
recall_and_precision(album.images, SimpleCounting.distance)
def get_evolver_and_sketch(source_image,
output_folder,
num_triangles,
save_frequency,
color_type,
start_from=None,
continue_run=False,
randomized=False):
im = Image.open(source_image)
im = im.convert(mode=color_type)
assert not (start_from and continue_run
), "you should only use start_from or continue_run, not both"
sketch = None
e = None
if continue_run:
# find the latest file in the folder
file_names = glob.glob(
os.path.join(output_folder, "intermediate_???.txt"))
best = -1
for filename in file_names:
num = int(filename[-7:-4])
if num > best:
best = num
if best >= 0:
filename = os.path.join(output_folder,
"intermediate_%03d.txt" % best)
module_logger.info(
"Restarting evolution based on found file 'intermediate_%03d.txt'",
best)
sketch = Sketch.read(filename)
e = Evolver(im,
output_folder,
num_triangles,
save_frequency,
color_type=color_type,
save_index=best + 1)
# Preferred is to load from the auto-save, but in case it died while saving the above will still work
filename = os.path.join(output_folder, "auto_save.txt")
if os.path.exists(filename):
sketch = Sketch.read(filename)
if sketch is None:
utils.clean_dir(output_folder)
if start_from:
sketch = Sketch.read(start_from)
else:
if randomized:
seeder = RandomSeeder(im, output_folder, num_triangles,
color_type)
else:
seeder = Seeder(im, output_folder, num_triangles, color_type)
sketch = seeder.run()
e = Evolver(im,
output_folder,
num_triangles,
save_frequency,
color_type=color_type)
return e, sketch
def cleanup(self):
Logger.pinfo('Initializing clean-up...', start='\n')
# Cleans all the mess up
clean_dirs = [self.build_dir, self.install_dir]
for path in clean_dirs:
utils.clean_dir(path)
Logger.pinfo('Clean-up completed succesfully', start='\n')
def align_audio(payload: PayLoad):
try:
load_audio(payload.bucket_id, payload.sub_dir, payload.file_name)
except ClientError:
raise HTTPException(status_code=404, detail="Item not found")
prepare_text(payload.text)
sync_map = force_align()
clean_dir()
response = Response(alignment=sync_map, file_name=payload.file_name)
return response
def get_evolver_and_sketch(
source_image,
output_folder,
num_triangles,
save_frequency,
color_type,
start_from=None,
continue_run=False,
randomized=False,
):
im = Image.open(source_image)
im = im.convert(mode=color_type)
assert not (start_from and continue_run), "you should only use start_from or continue_run, not both"
sketch = None
e = None
if continue_run:
# find the latest file in the folder
file_names = glob.glob(os.path.join(output_folder, "intermediate_???.txt"))
best = -1
for filename in file_names:
num = int(filename[-7:-4])
if num > best:
best = num
if best >= 0:
filename = os.path.join(output_folder, "intermediate_%03d.txt" % best)
module_logger.info("Restarting evolution based on found file 'intermediate_%03d.txt'", best)
sketch = Sketch.read(filename)
e = Evolver(im, output_folder, num_triangles, save_frequency, color_type=color_type, save_index=best + 1)
# Preferred is to load from the auto-save, but in case it died while saving the above will still work
filename = os.path.join(output_folder, "auto_save.txt")
if os.path.exists(filename):
sketch = Sketch.read(filename)
if sketch is None:
utils.clean_dir(output_folder)
if start_from:
sketch = Sketch.read(start_from)
else:
if randomized:
seeder = RandomSeeder(im, output_folder, num_triangles, color_type)
else:
seeder = Seeder(im, output_folder, num_triangles, color_type)
sketch = seeder.run()
e = Evolver(im, output_folder, num_triangles, save_frequency, color_type=color_type)
return e, sketch
def get_trajectory_images(trajectories, states, path):
try:
os.makedirs(path)
except:
clean_dir(path, file_type=".png")
trajectory_idx = 0
for trajectory in trajectories:
counter = 0
for state in trajectory.states:
trajectory_str = str(
trajectory_idx
) if trajectory_idx > 9 else "0" + str(trajectory_idx)
counter_str = str(counter) if counter > 9 else "0" + str(counter)
time_date = "_".join(str(datetime.datetime.now()).split())
img_name = "_".join([trajectory_str, counter_str, time_date])
counter += 1
states[state].save_image(path, img_name)
trajectory_idx += 1
def test():
build_dir = os.path.join('build', "tests/segmentation-otsu")
if not os.path.exists(build_dir):
os.makedirs(build_dir)
clean_dir(build_dir)
images = recursive_list_dir('tests/images')
for image in images:
logger.info('Processing %s' % image)
original = cv2.imread(image)
outfile = os.path.join(build_dir, os.path.basename(image))
cv2.imwrite(outfile, original)
(_, markers, count) = seg_otsu_watershed(original)
cv2.imwrite(outfile.replace('.jpg', '-segments.jpg'), markers)
logger.info('{filename} has {segments} segments'.format(
filename=image,
segments=count
))
def __init__(self, num_pads, num_frogs, num_iterations, wpath):
self.pads_dict = self.initialize_pads_dict(num_pads, num_frogs)
self.transition_matrix = self.initialize_transition_matrix(num_pads)
self.write_path = wpath
self.print_initial_stage(
num_frogs,
wpath) # Print initial stage of frogs/lilypads, pre-jumpsn
# makes wpath folder if not existing. deletes latex_tables.txt if existing
utils.clean_dir(
wpath
) # since we append to the bottom of the file instead of overwriting
net_flow = []
mse = []
current_distribution = self.get_pads_distribution_dict(self.pads_dict)
for i in range(num_iterations):
print('After {} jumps:'.format(i + 1))
prev_distribution = current_distribution
self.increment_time()
current_distribution = self.get_pads_distribution_dict(
self.pads_dict)
net_flow.append(
self.get_net_flow(current_distribution, prev_distribution, i,
num_pads, num_frogs))
mse.append(
utils.mean_squared_error(current_distribution,
prev_distribution, num_pads,
num_frogs))
utils.pretty_print_dict(current_distribution)
utils.save_histogram_image(current_distribution, i + 1, num_frogs,
wpath)
utils.save_distribution_table(current_distribution, i + 1,
num_frogs, wpath, i + 1)
print('\n')
e_vals = sorted(utils.get_evals(self.transition_matrix), reverse=True)
utils.print_and_write_results(net_flow, mse, e_vals, wpath)
import utils
import os
if __name__ == '__main__':
# initialize directory structure
os.makedirs(r'data/eval/positive', exist_ok=True)
os.makedirs(r'data/eval/negative', exist_ok=True)
os.makedirs(r'data/train/positive', exist_ok=True)
os.makedirs(r'data/train/negative', exist_ok=True)
os.makedirs(r'data/eval/positive', exist_ok=True)
os.makedirs(r'records/eval', exist_ok=True)
os.makedirs(r'records/train', exist_ok=True)
os.makedirs(r'rnn_seq_model', exist_ok=True)
os.makedirs(r'rnn_cls_model', exist_ok=True)
os.makedirs(r'rnn_seq_logs', exist_ok=True)
os.makedirs(r'rnn_cls_logs', exist_ok=True)
utils.clean_dir('records')
utils.clean_dir('data')
utils.clean_dir('rnn_seq_model')
utils.clean_dir('rnn_cls_model')
utils.clean_dir('rnn_seq_logs')
utils.clean_dir('rnn_cls_logs')
# create directory
fname = fname.replace("?", "_")
fname = fname.replace("!", "_")
fname = fname.replace("'", "_")
print fname
return fname + ".html"
def crawl_url(seed): # returns list of crawled links
crawled = []
content = urlutils.get_page(seed)
crawled = get_target_urls(content)
return crawled
# create a directory with the name of the URL if it does not exist
# if it exists, clean the files from the directory
dir_name = urlutils.get_stem_url(URL1)
dir_name = PREFIX+dir_name
if not os.path.exists(dir_name):
os.mkdir(dir_name)
utils.clean_dir(dir_name)
#crawl urls
crawled = crawl_url(URL1)
fout = open(dir_name+"//_url_list.txt",'w')
utils.print_list(crawled, fout)
fout.close()
########################################################################################################################
if mode == 'train':
print('\ntrain\n')
# included by Dom
# To calculate the number of training examples per class
find_all_files(train_im_dir, name_file_tot, saved_data_file_path)
# end of included by Dom
########################################## slicing ################################################
if not dont_perform_slicing: # if slicing were already done previously, you might want to skip it
utils.clean_dir(train_slice_dir)
print('slicing files ...')
# openCV can be heavier to use depending on your setup, you have the option to use PIL instead
if not dont_use_openCV:
slice_train(train_im_dir, train_slice_dir, sliceWidth=slice_width, sliceHeight=slice_width,
overlap=slice_overlap, Pobj=Pobj, Pimage=Pimage)
else:
slice_PIL.slice_train(train_im_dir, train_slice_dir, sliceWidth=slice_width, sliceHeight=slice_width,
overlap=slice_overlap, Pobj=Pobj, Pimage=Pimage)
if not dont_use_openCV:
utils.list_files(train_slice_dir, '.png', train_tmp_dir, 'train') # slices are listed to be used by darknet
else:
utils.list_files(train_slice_dir, '.jpg', train_tmp_dir, 'train')
def set_seq_id(self,val):
self.seq_id = val
return self
def build(self):
return TFEncoder(self)
if __name__ == '__main__':
utils.clean_dir(r'records')
encoder = TFEncoder.Builder().\
set_src_path(r'data').\
set_dst_path(r'records').\
set_max_steps(600).\
build()
result = encoder.encode()
# result = encoder.xencode(0,(2000, 5500))
# utils.clean_dir(r'records')
# result = encoder.encode()
#
# print(result)
def clean(self):
clean_dir(self.sample_path)
clean_dir(self.stream_path)
clean_dir(self.region_path)
import os.path as osp
from datetime import datetime, timedelta
from tzwhere import tzwhere
if len(sys.argv) != 3:
print 'Error: python %s firename year' % sys.argv[0]
sys.exit(1)
firename = sys.argv[1]
year = sys.argv[2]
if year == str(datetime.today().year):
year = 'current_year'
dst_in = 'perim_orig'
dst_out = 'perim'
clean_dir(dst_in)
clean_dir(dst_out)
baseurl = 'https://rmgsc.cr.usgs.gov/outgoing/GeoMAC/'
url = osp.join(baseurl, year + '_fire_data/KMLS/')
r = requests.get(url, stream=True)
content = r.content
plist = re.findall('([a-z\d\s-]+%s[\d\s-]+.kml)' % firename, content,
re.IGNORECASE)
for p in plist:
get_url(osp.join(url, p), osp.join(dst_in, p))
files = glob.glob(osp.join(dst_in, '*.kml'))
print 'Transforming KML files to UTC from %s to %s' % (dst_in, dst_out)
for k, file in enumerate(files):
f = open(file, "r")
f_str = ''.join(f.readlines())
def train_cifar10():
logging.info("Args = %s", args)
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
global_step = tf.Variable(initial_value=0, trainable=False, dtype=tf.int32)
epoch = tf.Variable(initial_value=0, trainable=False, dtype=tf.int32)
best_acc_top1 = tf.Variable(initial_value=0.0,
trainable=False,
dtype=tf.float32)
################################################ model setup #######################################################
train_ds, test_ds = utils.load_cifar10(args.batch_size, args.cutout_size)
total_steps = int(np.ceil(50000 / args.batch_size)) * args.epochs
model = NASNetworkCIFAR(classes=10,
reduce_distance=args.cells,
num_nodes=args.nodes,
channels=args.channels,
keep_prob=args.keep_prob,
drop_path_keep_prob=args.drop_path_keep_prob,
use_aux_head=args.use_aux_head,
steps=total_steps,
arch=args.arch)
temp_ = tf.random.uniform((64, 32, 32, 3),
minval=0,
maxval=1,
dtype=tf.float32)
temp_ = model(temp_, step=1, training=True)
model.summary()
model_size = utils.count_parameters_in_MB(model)
print("param size = {} MB".format(model_size))
logging.info("param size = %fMB", model_size)
criterion = keras.losses.CategoricalCrossentropy(from_logits=True)
learning_rate = keras.experimental.CosineDecay(
initial_learning_rate=args.initial_lr,
decay_steps=total_steps,
alpha=0.0001)
# learning_rate = keras.optimizers.schedules.ExponentialDecay(
# initial_learning_rate=args.initial_lr, decay_steps=total_steps, decay_rate=0.99, staircase=False, name=None
# )
optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)
########################################## restore checkpoint ######################################################
if args.train_from_scratch:
utils.clean_dir(args.model_dir)
checkpoint_path = os.path.join(args.model_dir, 'checkpoints')
ckpt = tf.train.Checkpoint(model=model,
optimizer=optimizer,
global_step=global_step,
epoch=epoch,
best_acc_top1=best_acc_top1)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_path,
max_to_keep=3)
# if a checkpoint exists, restore the latest checkpoint.
if ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint)
print('Latest checkpoint restored!!')
############################################# training process #####################################################
acc_train_result = []
loss_train_result = []
acc_test_result = []
loss_test_result = []
while epoch.numpy() < args.epochs:
print('epoch {} lr {}'.format(epoch.numpy(),
optimizer._decayed_lr(tf.float32)))
train_acc, train_loss, step = train(train_ds,
model,
optimizer,
global_step,
criterion,
classes=10)
test_acc, test_loss = valid(test_ds, model, criterion, classes=10)
acc_train_result.append(train_acc)
loss_train_result.append(train_loss)
acc_test_result.append(test_acc)
loss_test_result.append(test_loss)
logging.info('epoch %d lr %e', epoch.numpy(),
optimizer._decayed_lr(tf.float32))
logging.info(acc_train_result)
logging.info(loss_train_result)
logging.info(acc_test_result)
logging.info(loss_test_result)
is_best = False
if test_acc > best_acc_top1:
best_acc_top1 = test_acc
is_best = True
epoch.assign_add(1)
if (epoch.numpy() + 1) % 1 == 0:
ckpt_save_path = ckpt_manager.save()
print('Saving checkpoint for epoch {} at {}'.format(
epoch.numpy() + 1, ckpt_save_path))
if is_best:
pass
utils.plot_single_list(acc_train_result,
x_label='epochs',
y_label='acc',
file_name='acc_train')
utils.plot_single_list(loss_train_result,
x_label='epochs',
y_label='loss',
file_name='loss_train')
utils.plot_single_list(acc_test_result,
x_label='epochs',
y_label='acc',
file_name='acc_test')
utils.plot_single_list(loss_test_result,
x_label='epochs',
y_label='loss',
file_name='loss_test')
model_dir=cfg['checkpoint_dir'],
name='{}_step_{}'.format(cfg['netd'] + cfg['data']['dataset'], epoch))
if __name__ == '__main__':
# Load the config file
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', default='./configs/pix2pix_jorder_rain100l.yaml')
args = parser.parse_args()
with open(args.config) as fp:
cfg = yaml.load(fp)
if cfg['resume']:
clean_dir(cfg['checkpoint_dir'])
print(cfg)
# Setup the log
run_id = random.randint(1, 100000)
logdir = os.path.join(cfg['checkpoint_dir'], os.path.basename(args.config)[:-4] + str(run_id))
ensure_dir(logdir)
print("RUNDIR: {}".format(logdir))
shutil.copy(args.config, logdir)
logger = get_logger(logdir)
logger.info("Let the games begin")
# Setup the Visualizer
if cfg['vis']['use']:
vis = Visualizer(cfg['vis']['env'])
import utils
import os
if __name__ == '__main__':
# initialize directory structure
os.makedirs(r'records/eval',exist_ok=True)
os.makedirs(r'records/train',exist_ok=True)
os.makedirs(r'model',exist_ok=True)
os.makedirs(r'logs/train',exist_ok=True)
os.makedirs(r'logs/eval', exist_ok=True)
utils.clean_dir('records')
utils.clean_dir('logs')
utils.clean_dir('model')
def download_tools(self):
# Make sure we have the directories required
self.make_toolchain_dirs()
Logger.pinfo(f'Downloading tarballs...', start='\n')
paths = [self.gcc_file, self.binutils_file]
urls = [self.gcc_url, self.binutils_url]
tools = dict(zip(paths, urls))
for path, url in tools.items():
path_ext = f'{path}.tar.xz'
download_path = self.downloads_dir / path_ext
# Get the cross compile toolchain path
extract_path = self.source_dir / path
if extract_path.exists():
# if everything exists, don't validate and re-download sources
Logger.pwarn(
f'[{download_path.name}] already exists at [{self.source_dir}]'
)
elif not download_path.exists():
# if they do not exist, just download them
with open(download_path, 'wb') as file:
response = requests.get(url)
# Get length and stuff like this for the progress bar
length = response.headers.get('content-length')
utils.print_progress(0,
int(length),
prefix='Progress:',
suffix='Complete',
length=50)
if length is None:
Logger.pwarn(
'Could not retrieve length of data stream from response!'
)
file.write(response.content)
else:
dl = 0
for data in response.iter_content(chunk_size=4096):
dl += len(data)
utils.print_progress(dl,
int(length),
prefix='Progress:',
suffix='Complete',
length=50)
file.write(data)
Logger.pdebug(
f'Succesfully downloaded [{download_path.name}]\n')
else:
# Make sure the existing archives have identical hashes
# For some reason md5_compare takes a really long time...
Logger.pwarn(f'[{download_path.name}] already present!')
Logger.pinfo(
f'Checking if files are identical... (this may take a while)'
)
result = self.md5_compare(download_path)
if result is False:
Logger.perror(
f'Files have different hashes! Cleaning downloads directory and re-running the script!'
)
utils.clean_dir(self.downloads_dir)
self.download_tools()
