def get_cells_contours(self, precision):
pb = ProgressBar(total=100,
decimals=2,
prefix="getting cells contours",
length=terminal_size.get_terminal_size()[0] - 40,
fill='#',
zfill='-')
contours = {}
l = 1
step = 100 / len(self.struct)
for i in self.struct.values():
x0, x1 = i[0][0], i[0][1]
y0, y1 = i[1][0], i[1][1]
bottom_dots = self.section_divide((x0, y0), (x1, y0), precision)
right_dots = self.section_divide((x1, y0), (x1, y1), precision)
top_dots = self.section_divide((x1, y1), (x0, y1), precision)
left_dots = self.section_divide((x0, y1), (x0, y0), precision)
inner_dots = []
for k in bottom_dots:
for j in left_dots:
inner_dots.append((k[0], j[1]))
contours[str(l)] = [(x0, y0)] + bottom_dots + [
(x1, y0)
] + right_dots + [(x1, y1)] + top_dots + [
(x0, y1)
] + left_dots + inner_dots
pb.print_progress_bar(step * l)
l += 1
self.contours = contours
def get_symbolic_image(self, precision, f, g):
self.get_cells_contours(precision)
self.get_cells_images(f, g)
self.get_cells_intersects()
pb = ProgressBar(total=100,
decimals=2,
prefix="building graph",
length=terminal_size.get_terminal_size()[0] - 45,
fill='#',
zfill='-')
step = 50 / len(self.struct)
m = 1
G = nx.DiGraph()
for i in self.struct:
G.add_node(i)
pb.print_progress_bar(m * step)
m += 1
step = 50 / len(self.intersects)
m = 1
for i in self.intersects:
for j in self.intersects[i]:
G.add_edge(i, j)
pb.print_progress_bar(m * step)
m += 1
self.symbolic_image = G
self.clear_table()
def save_test_data(fnames, bboxes):
src_folder = 'cars_test'
dst_folder = 'data/test'
num_samples = len(fnames)
pb = ProgressBar(total=100,
prefix='Save test data',
suffix='',
decimals=3,
length=50,
fill='=')
for i in range(num_samples):
fname = fnames[i]
(x1, y1, x2, y2) = bboxes[i]
src_path = os.path.join(src_folder, fname)
src_image = cv.imread(src_path)
height, width = src_image.shape[:2]
# margins of 16 pixels
margin = 16
x1 = max(0, x1 - margin)
y1 = max(0, y1 - margin)
x2 = min(x2 + margin, width)
y2 = min(y2 + margin, height)
# print(fname)
pb.print_progress_bar((i + 1) * 100 / num_samples)
dst_path = os.path.join(dst_folder, fname)
crop_image = src_image[y1:y2, x1:x2]
dst_img = cv.resize(src=crop_image, dsize=(img_height, img_width))
cv.imwrite(dst_path, dst_img)
def test_images(epoch):
def write_image(img, filename):
out_img = img.cpu().detach().numpy()
out_img *= 255.0
out_img = out_img.clip(0, 255)
out_img = np.uint8(out_img)
writer.add_image(filename, out_img, epoch)
with torch.no_grad():
num_minibatch = len(testing_full_data_loader)
pg = ProgressBar(num_minibatch, 'Test %d Images'%num_minibatch, length=50)
model.eval()
if criterion.discriminator is not None:
criterion.discriminator.eval()
for i,batch in enumerate(testing_full_data_loader):
pg.print_progress_bar(i)
input, flow = batch[0].to(device), batch[1].to(device)
B, _, Cin, H, W = input.shape
Hhigh = H * opt.upscale_factor
Whigh = W * opt.upscale_factor
Cout = output_channels
if opt.deferredShading:
channel_mask = [1, 2, 3] #normal
else:
channel_mask = [0, 1, 2] #rgb
previous_output = None
for j in range(dataset_data.num_frames):
# prepare input
if j == 0 or opt.disableTemporal:
previous_warped = torch.zeros(B, Cout, Hhigh, Whigh,
dtype=input.dtype,
device=device)
else:
previous_warped = models.VideoTools.warp_upscale(previous_output, flow[:, j-1, :, :, :], opt.upscale_factor)
previous_warped_flattened = models.VideoTools.flatten_high(previous_warped, opt.upscale_factor)
single_input = torch.cat((
input[:,j,:,:,:],
previous_warped_flattened),
dim=1)
# write warped previous frame
write_image(previous_warped[0, channel_mask], 'image%03d/frame%03d_warped' % (i, j))
# run generator and cost
prediction, residual = model(single_input)
prediction = torch.clamp(prediction, 0, 1)
# write prediction image
write_image(prediction[0, channel_mask], 'image%03d/frame%03d_prediction' % (i, j))
# write residual image
if residual is not None:
write_image(residual[0, channel_mask], 'image%03d/frame%03d_residual' % (i, j))
# write shaded image if network runs in deferredShading mode
if opt.deferredShading:
shaded_image = shading(prediction)
write_image(shaded_image[0], 'image%03d/frame%03d_shaded' % (i, j))
# save output for next frame
previous_output = prediction
pg.print_progress_bar(num_minibatch)
print("Test images sent to Tensorboard for visualization")
def ransac_error_optimized(size, calculation_model_func):
print("Training using RANSAC")
pb = ProgressBar(total=ITERATIONS)
lowest_error = 9999999999
i = 0
while i < ITERATIONS:
samples = choice_random(size)
model = calculation_model_func(samples)
if model is None:
i -= 1
continue
consensus = 0
iter_error = 0
for points in neighbors:
error = calculate_error(model, points)
if error < lowest_error:
consensus += 1
iter_error += error
if lowest_error > iter_error:
lowest_error = iter_error
best_model = model
i += 1
pb.print_progress_bar(i)
return best_model
def _print_progress(self, event):
"""Print a progress based on the information on a GeneralEvent."""
if event.indeterminate:
# indeterminate, no progress to be shown
return
prefix = """
--> {phase_name}
{current_phase}/{target}""".format(
phase_name=event.phase_name,
current_phase=min(event.total_phases, event.current_phase + 1),
target=event.total_phases,
)
elapsed = datetime.now() - self._start_time
statuses = self._get_legends()
suffix = """{runtime}
{statuses}""".format(statuses=statuses,
runtime=format_running_time(elapsed.seconds))
pb = ProgressBar(
total=100,
prefix=prefix,
suffix=suffix,
decimals=0,
length=self.bar_length,
fill=self.filled_bar_char,
zfill=self.empty_bar_char,
file=self._out,
)
pb.print_progress_bar(event.progress * 100)
def save_to_csv(pro, now = '20190605', year = '20181231', index_file='data/sse_ms20.txt'):
"""
Save data to a file in csv format. The index file can be sse50.txt, myshare.txt
or sse_dividend.txt.
"""
print(index_file)
sse50 = [x.rstrip()+'.SH' for x in open(index_file)]
i=0
pb = ProgressBar(total=100,prefix='Downloading', suffix='', decimals=3, length=50, fill='*', zfill='-')
num = len(sse50)
for code in sse50:
if i == 0:
data = get_data(pro, code, year, now)
data.columns=['代码', '最新价', '市盈率', '市净率', '总市值(亿元)', '主营业务收入(亿元)', '净利润(亿元)', '每股收益', '利润率', '利润增长率(%)']
i = i + 1
pb.print_progress_bar((i/num)*100)
else:
new = get_data(pro, code, year, now)
new.columns=['代码', '最新价', '市盈率', '市净率', '总市值(亿元)', '主营业务收入(亿元)', '净利润(亿元)', '每股收益', '利润率', '利润增长率(%)']
data.loc[i] = new.loc[0]
i = i + 1
pb.print_progress_bar((i/num)*100)
# print(new)
data_file_name = 'build/' + msu.file_base_name(index_file) + '.csv'
print('Saving to '+data_file_name)
data.to_csv(data_file_name)
return data
def ransac_error_calc_optimized(size, calculation_model_func):
print("Training using RANSAC")
pb = ProgressBar(total=ITERATIONS)
best_score = 0
i = 0
while i < ITERATIONS:
samples = choice_random(size)
model = calculation_model_func(samples)
if model is None:
i -= 1
continue
consensus = 0
b = 0
for points in neighbors:
error = calculate_error(model, points)
if error < MAX_ERROR:
consensus += 1
if consensus + (len(neighbors) - b) < best_score:
break
b += 1
if consensus > best_score:
best_score = consensus
best_model = model
i += 1
pb.print_progress_bar(i)
return best_model
def build_trie_with_terms_and_definitions(data_path: str, resources_path: str):
data = read_data_file(f"{resources_path}/json/{data_path}", 'json')
total = len(data)
print(f"\n\tSize of word document: {total}\n\n")
trie = create_root_node()
pb = ProgressBar(total=total,
prefix='Start',
suffix='Complete',
decimals=2,
length=50)
current = 0
for word in data:
add_word_to_trie(trie, word, data[word]['definition'],
data[word]['count'])
current += 1
pb.print_progress_bar(current)
print('''\tFinished inserting words into tree \n \
\tCreating and saving pickle...
''')
output_file = f"{resources_path}/pickles/korean_pickle.pkl"
with open(f"{output_file}", 'wb') as file:
pickle.dump(trie, file)
print("\tDone!\n")
def main():
all_users = []
completed_count = 0
page = 1
try:
with open('seafarers.json', 'r') as file:
data = json.load(file)
page = int(data['completed_page']) + 1
total_count = int(data['total_count'])
all_users = data['data']
except FileNotFoundError:
total_count = get_total_count()
pb = ProgressBar(total=total_count,prefix='Here', suffix='Now', decimals=3, length=50, fill='\u25A0', zfill='-')
pb.print_progress_bar(len(all_users))
while len(all_users) < total_count:
users, total_count = get_page(page)
all_users.extend(users)
with open('seafarers.json', 'w') as file:
json.dump({
'completed_count': len(all_users),
'completed_page': page,
'total_count': total_count if total_count else page,
'data': all_users,
}, file, indent=4)
pb.print_progress_bar(len(all_users))
page += 1
def process_sentiment_news():
from analytics.SentimentAnalyser import SentimentAnalyser
from time import time
stime = time()
sa = SentimentAnalyser(True)
d = DB()
x = d.get_news({"polarity": -2})
res = sa.get_polarity([i['text'] for i in x])
r = len(res)
print("Started for", r, "posts")
pb = ProgressBar(total=r - 1,
prefix='Processed',
decimals=3,
length=100,
fill='=',
zfill='-')
for i in range(r):
d.update_news(
x[i]['_id'], {
'polarity': float(res[i]),
'query': '-'.join(x[i]['query'].lower().split())
})
pb.print_progress_bar(i)
print("Processing finished in", time() - stime)
def zipit(self, folders, dst):
pb = ProgressBar(total=100,
prefix='Zip Progress',
suffix='Completed',
decimals=2,
length=50,
fill='>',
zfill='-')
count = 0
progress = 0
self.logger.info("Counting Files...")
zip_file = zipfile.ZipFile(dst, 'w', zipfile.ZIP_DEFLATED)
for folder in folders:
for dirpath, dirnames, filenames in os.walk(folder):
for filename in filenames:
count += 1
self.logger.info("%s files found to be zipped" % count)
for folder in folders:
for dirpath, dirnames, filenames in os.walk(folder):
for filename in filenames:
zip_file.write(
os.path.join(dirpath, filename),
os.path.relpath(os.path.join(dirpath, filename),
os.path.join(folders[0], '../..')))
progress += 1
pb.print_progress_bar((progress / count) * 100)
zip_file.close()
def main():
"""Load data, display data"""
args = arg_parser_setup()
clubhouse_apikey = get_clubhouse_apikey_or_exit()
ch_api = ClubHouseAPI(clubhouse_apikey)
cycle_logic = CycleLogic()
if args.googlesheets:
cycle_logic.enable_google_sheets_output(
sheet_id=SHEET_ID,
scopes=SCOPES,
service_account_file=GOOGLE_SERVICE_ACCOUNT_FILE
)
members = ch_api.get_active_members()
progress_bar = ProgressBar(total=len(members))
for i, member in enumerate(members):
progress_bar.print_progress_bar(i+1)
cycle_logic.add_member(member)
stories = ch_api.stories_by_mention(member['profile']['mention_name'])
for ch_story in stories:
story = Story(load_from_dict=ch_story, owner_id=member['id'])
cycle_logic.add_story(story)
print('\n\n')
print(cycle_logic.tabulate_result(debug_member=args.debugmember))
def get_dividend_data(pro, stock_index, index_file, channel):
"""
Save data to a file in csv format.
"""
i = 0
pb = ProgressBar(total=100,
prefix='Downloading',
suffix='',
decimals=3,
length=50,
fill='*',
zfill='-')
num = len(stock_index)
for code in stock_index:
if i == 0:
df_div = get_div(pro, code, channel)
i = i + 1
pb.print_progress_bar((i / num) * 100)
else:
df_new = get_div(pro, code, channel)
df_div.loc[code] = df_new.loc[code]
i = i + 1
pb.print_progress_bar((i / num) * 100)
# print(new)
return df_div
def clear_table(self):
pb = ProgressBar(total=100,
decimals=2,
prefix="clearing table",
length=terminal_size.get_terminal_size()[0] - 40,
fill='#',
zfill='-')
connected = tuple(nx.strongly_connected_components(
self.symbolic_image))
remember = set()
rest = {}
for i in connected:
if len(i) > 1:
remember.update(i)
# else:
# try: nx.find_cycle(self.symbolic_image, i)
# except nx.exception.NetworkXNoCycle: pass
# else: remember.update(i)
step = 100 / len(self.struct)
k = 1
m = 1
for i in self.struct:
if i in remember:
rest[str(k)] = self.struct[i]
k += 1
pb.print_progress_bar(m * step)
m += 1
self.struct = rest
def predict(img_dir, model):
img_files = []
for root, dirs, files in os.walk(img_dir, topdown=False):
for name in files:
img_files.append(os.path.join(root, name))
img_files = sorted(img_files)
y_pred = []
y_test = []
pb = ProgressBar(total=100,
prefix='Predict data',
suffix='',
decimals=3,
length=50,
fill='=')
for img_path in img_files:
# print(img_path)
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
preds = model.predict(x[None, :, :, :])
decoded = decode_predictions(preds, top=1)
pred_label = decoded[0][0][0]
# print(pred_label)
y_pred.append(pred_label)
tokens = img_path.split('/')
class_id = int(tokens[-2])
# print(str(class_id))
y_test.append(class_id)
pb.print_progress_bar(len(y_pred) * 100 / len(img_files))
return y_pred, y_test
def compute_class_prior(do_plot=False):
categories_folder = 'data/instance-level_human_parsing/Training/Category_ids'
names = [
f for f in os.listdir(categories_folder) if f.lower().endswith('.png')
]
num_samples = len(names)
prior_prob = np.zeros(num_classes)
pb = ProgressBar(total=num_samples,
prefix='Compute class prior',
suffix='',
decimals=3,
length=50,
fill='=')
for i in range(num_samples):
name = names[i]
filename = os.path.join(categories_folder, name)
category = np.ravel(cv.imread(filename, 0))
counts = np.bincount(category)
idxs = np.nonzero(counts)[0]
prior_prob[idxs] += counts[idxs]
pb.print_progress_bar(i + 1)
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Save
np.save(os.path.join(data_dir, "prior_prob.npy"), prior_prob)
if do_plot:
plt.hist(prior_prob, bins=100)
plt.yscale("log")
plt.show()
def do_composite_test():
num_bgs = 20
with open('Combined_Dataset/Test_set/test_bg_names.txt') as f:
bg_files = f.read().splitlines()
with open('Combined_Dataset/Test_set/test_fg_names.txt') as f:
fg_files = f.read().splitlines()
# a_files = os.listdir(a_path)
num_samples = len(fg_files) * num_bgs
pb = ProgressBar(total=100,
prefix='Compose test images',
suffix='',
decimals=3,
length=50,
fill='=')
start = time.time()
bcount = 0
for fcount in range(len(fg_files)):
im_name = fg_files[fcount]
for i in range(num_bgs):
bg_name = bg_files[bcount]
process(im_name, bg_name, fcount, bcount)
bcount += 1
pb.print_progress_bar(bcount * 100.0 / num_samples)
end = time.time()
elapsed = end - start
print('elapsed: {} seconds'.format(elapsed))
def get_yf_data(stock_index, index_file):
print(stock_index)
i = 0
pb = ProgressBar(total=100,
prefix='Downloading',
suffix='',
decimals=3,
length=50,
fill='*',
zfill='-')
num = len(stock_index)
ydata = {}
for stock in stock_index:
ticker = yf.Ticker(stock)
ydata[ticker.info['symbol']] = ticker.info
i = i + 1
pb.print_progress_bar((i / num) * 100)
ydata_frame = pd.DataFrame(ydata).T[[
'longName', 'regularMarketPreviousClose', 'fiftyTwoWeekHigh',
'fiftyTwoWeekLow', 'forwardPE', 'marketCap'
]]
ydata_frame.columns = ['名称', '最新价', '52周最高', '52周最低', '市盈率', '总市值']
ydata_frame['总市值'] = ydata_frame['总市值'] / 100000000
return ydata_frame
def extract(usage, package, image_path, json_path):
filename = 'data/{}.zip'.format(package)
print('Extracting {}...'.format(filename))
with zipfile.ZipFile(filename, 'r') as zip_ref:
zip_ref.extractall('data')
pb = ProgressBar(total=100,
prefix='Save {} data'.format(usage),
suffix='',
decimals=3,
length=50,
fill='=')
if not os.path.exists('data/{}'.format(usage)):
os.makedirs('data/{}'.format(usage))
with open('data/{}/{}'.format(package, json_path)) as json_data:
data = json.load(json_data)
num_samples = len(data)
print("num_samples: " + str(num_samples))
for i in range(num_samples):
item = data[i]
image_name = item['image_id']
label_id = item['label_id']
src_folder = 'data/{}/{}'.format(package, image_path)
src_path = os.path.join(src_folder, image_name)
dst_folder = 'data/{}'.format(usage)
label = "%02d" % (int(label_id), )
dst_path = os.path.join(dst_folder, label)
if not os.path.exists(dst_path):
os.makedirs(dst_path)
dst_path = os.path.join(dst_path, image_name)
src_image = cv.imread(src_path)
dst_image = cv.resize(src_image, (img_height, img_width),
cv.INTER_CUBIC)
cv.imwrite(dst_path, dst_image)
pb.print_progress_bar((i + 1) * 100 / num_samples)
def main():
with open(INPUT_FILE) as file:
data = json.load(file)
total = len(data)
print(f"\n\tSize of word document: {total}\n\n")
trie = create_root_node()
pb = ProgressBar(total=total,
prefix='Start',
suffix='Complete',
decimals=2,
length=50)
current = 0
for word in data:
add_word_to_trie(trie, word, data[word])
current += 1
pb.print_progress_bar(current)
print('''\tFinished inserting words into tree \n \
\tCreating and saving pickle...
''')
with open(f"{OUTPUT_FILE}", 'wb') as file:
pickle.dump(trie, file)
print("\tDone!\n")
def save_test_data(fnames, bboxes):
"""
Saves test data in a separate folder
Args:
fnames: The filenames of the files to save in the seperate folder.
bboxes: Bounding boxes
Returns:
"""
src_folder = "cars_test"
dst_folder = "data/test"
num_samples = len(fnames)
pb = ProgressBar(total=100, prefix="Save test data", suffix="", decimals=3, length=50, fill="=")
for i in range(num_samples):
fname = fnames[i]
(x1, y1, x2, y2) = bboxes[i]
src_path = os.path.join(src_folder, fname)
src_image = cv.imread(src_path)
height, width = src_image.shape[:2]
# margins of 16 pixels
margin = 16
x1 = max(0, x1 - margin)
y1 = max(0, y1 - margin)
x2 = min(x2 + margin, width)
y2 = min(y2 + margin, height)
pb.print_progress_bar((i + 1) * 100 / num_samples)
dst_path = os.path.join(dst_folder, fname)
cv.imwrite(dst_path, src_image)
def test_digits(model, digits, labels, ensemble_size, reshape_fun):
steps_results = {'c_error': {}, 'entropy': {}}
dnum = 200
pb = ProgressBar(total=100,
prefix='Sim trial progress',
length=25,
fill='=',
zfill='_')
for i in range(1, 101):
dnoice = salt_and_pepper(digits, i * dnum)
d = utils.normalize_data(reshape_fun(dnoice))
entropy = ann.test_model(model, [d] * ensemble_size,
labels,
metric='entropy')
c_error = ann.test_model(model, [d] * ensemble_size,
labels,
metric='c_error')
steps_results['entropy'][i] = entropy
steps_results['c_error'][i] = c_error
pb.print_progress_bar(i)
return steps_results
def _print_progress(self, event):
"""Print a progress based on the information on a GeneralEvent."""
if event.indeterminate:
# indeterminate, no progress to be shown
return
prefix = """
--> {phase_name}
{current_phase}/{target}""".format(
phase_name=event.phase_name,
current_phase=min(event.total_phases, event.current_phase + 1),
target=event.total_phases,
)
statuses = ""
legends = self._get_legends(event.sim_states)
for state in event.sim_states:
statuses += " {}\n".format(legends[state])
suffix = """{runtime}
{statuses}""".format(statuses=statuses,
runtime=format_running_time(event.runtime))
pb = ProgressBar(total=100,
prefix=prefix,
suffix=suffix,
decimals=0,
length=self.bar_length,
fill=self.filled_bar_char,
zfill=self.empty_bar_char,
file=self._out)
pb.print_progress_bar(event.progress * 100)
def normalize_background(cls, stack, radius=None):
"""
Normalize background to mean 0 and std 1.
Estimate the mean and std of each image in the stack using pixels
outside radius r (pixels), and normalize the image such that the
background has mean 0 and std 1. Each image in the stack is corrected
separately.
:param radius: radius for normalization (default is half the side of image)
Example:
normalized_stack = cryo_normalize_background(stack,55);
"""
validate_square_projections(stack)
num_images = stack.shape[0] # assuming C-contiguous array
side = stack.shape[1]
if radius is None:
radius = np.floor(side / 2)
# find indices of backgruond pixels in the images
ctr = (side + 1) / 2
x_axis, y_axis = np.meshgrid(range(1, side + 1), range(1, side + 1))
radiisq = (x_axis.flatten() - ctr)**2 + (y_axis.flatten() - ctr)**2
background_pixels_idx = radiisq > radius * radius
if AspireConfig.verbosity == 1:
pb = ProgressBar(total=100,
prefix='normalizing background',
suffix='completed',
decimals=0,
length=100,
fill='%')
else:
pb = None
normalized_stack = np.ones(stack.shape)
for i in range(num_images):
if pb:
pb.print_progress_bar((i + 1) / num_images * 100)
proj = stack[i, :, :]
background_pixels = proj.flatten() * background_pixels_idx
background_pixels = background_pixels[background_pixels != 0]
# compute mean and standard deviation of background pixels
proj_mean = np.mean(background_pixels)
std = np.std(background_pixels, ddof=1)
# normalize the projections
if std < 1.0e-5:
logger.warning(
f'Variance of background of image {i} is too small (std={std}). '
'Cannot normalize!')
normalized_stack[i, :, :] = (proj - proj_mean) / std
return normalized_stack
def test_images(epoch):
def write_image(img, filename):
out_img = img.cpu().detach().numpy()
out_img *= 255.0
out_img = out_img.clip(0, 255)
out_img = np.uint8(out_img)
writer.add_image(filename, out_img, epoch)
with torch.no_grad():
num_minibatch = len(testing_full_data_loader)
pg = ProgressBar(num_minibatch,
'Test %d Images' % num_minibatch,
length=50)
model.eval()
if criterion.has_discriminator:
criterion.discr_eval()
for i, batch in enumerate(testing_full_data_loader):
pg.print_progress_bar(i)
input = batch[0].to(device)
B, _, Cin, H, W = input.shape
Hhigh = H * opt.upscale_factor
Whigh = W * opt.upscale_factor
Cout = output_channels
channel_mask = [0, 1, 2] #RGB
previous_output = None
for j in range(dataset_data.num_frames):
# prepare input
if j == 0 or opt.disableTemporal:
previous_warped = initialImage(input[:, 0, :, :, :], Cout,
opt.initialImage, False,
opt.upscale_factor)
else:
previous_warped = models.VideoTools.warp_upscale(
previous_output,
flow[:, j - 1, :, :, :],
opt.upscale_factor,
special_mask=True)
previous_warped_flattened = models.VideoTools.flatten_high(
previous_warped, opt.upscale_factor)
single_input = torch.cat(
(input[:, j, :, :, :], previous_warped_flattened), dim=1)
# run generator and cost
prediction, residual = model(single_input)
# write prediction image
write_image(prediction[0, channel_mask],
'image%03d/frame%03d_prediction' % (i, j))
# write residual image
if residual is not None:
write_image(residual[0, channel_mask],
'image%03d/frame%03d_residual' % (i, j))
# save output for next frame
previous_output = prediction
pg.print_progress_bar(num_minibatch)
print("Test images sent to Tensorboard for visualization")
def percentage(now, total):
value = (now * 100) / total
pb = ProgressBar(total=100,
prefix='Here',
decimals=2,
length=50,
fill='#',
zfill='-')
pb.print_progress_bar(value)
def main():
net = load_convnet()
get_features = K.function(
[net.layers[0].input, K.learning_phase()],
[net.get_layer("flatten_2").output])
base_path = 'data/train'
paths = find_paths(base_path)
# print paths
dict_features = {}
pb = ProgressBar(total=100,
prefix='Extraindo features train...',
suffix='',
decimals=3,
length=50,
fill='=')
i = 0
for image_path in paths:
pb.print_progress_bar((i + 1) * 100 / len(paths))
# print 'image_path: ', image_path
im_data = cv2.imread(image_path)
fname = image_path[16:]
# print 'fname: ', fname
features = get_features([im_data[np.newaxis, ...], 0])[0]
# print 'features: ', features
dict_features[fname] = features
i = i + 1
np.savez_compressed("cars_train.npz", **dict_features)
base_path = 'data/test'
paths = find_paths(base_path)
dict_features = {}
pb = ProgressBar(total=100,
prefix='Extraindo features test...',
suffix='',
decimals=3,
length=50,
fill='=')
i = 0
for image_path in paths:
pb.print_progress_bar((i + 1) * 100 / len(paths))
# print 'image_path: ', image_path
im_data = cv2.imread(image_path)
fname = image_path[15:]
# print 'fname: ', fname
features = get_features([im_data[np.newaxis, ...], 0])[0]
# print 'features: ', features
dict_features[fname] = features
i = i + 1
np.savez_compressed("cars_test.npz", **dict_features)
def analyzeAllDomains(self):
pb = ProgressBar(total=int(len(self.domains)-1),prefix='Domain analysis in progress', suffix='', decimals=3, length=50, fill='X', zfill='-')
data = self.getFrames()
for idx,domain in enumerate(self.getDomains()):
topic = data[domain]
arr = topic.to_numpy()
self.analyzeByDomain(domain, arr)
self.calculateGlobalData(arr)
pb.print_progress_bar(idx)
self.serializeResultsDict()
def import_data(data_directory, size=(50, 50), max_samples=0):
# Define X is features, y is label.
X = []
y = []
# Define data-set directory.
dir_labels = data_directory
# We use the folder name as a label name.
labels = os.listdir(dir_labels)
# Search all folder in data-set directory.
for i, label in enumerate(labels):
# Get all image name in each folder.
dir_images = dir_labels + os.sep + label
image_names = os.listdir(dir_images)
# Create a new progress bar.
progress_bar = ProgressBar(total=min(len(image_names), max_samples),
prefix="Label(%s):" % label,
suffix="%d/%d" % (i + 1, len(labels)),
length=50)
# Search all image in each folder.
n = 0
for j, image_name in enumerate(image_names):
# Read image from file.
filename = dir_images + os.sep + image_name
image = Image.imread(filename)
image = cv2.resize(image, size)
# Add into X, y.
X.append(image)
y.append(i)
# Update progress bar.
progress_bar.print_progress_bar(j + 1)
n += 1
if n >= max_samples:
break
# Shuffle data.
data = list(zip(X, y))
random.shuffle(data)
X, y = zip(*data)
# Final X, y.
X = np.array(X)
y = np.array(y)
return X, y, labels
