def read_tiff(filename, bands=None, xBSize=5000, yBSize=5000):
'''import'''
import gdal
from tqdm import tqdm_gui
'''program'''
ds = gdal.Open(filename)
gdal.UseExceptions()
nrow = ds.RasterYSize
ncol = ds.RasterXSize
if bands == None:
bands = range(ds.RasterCount)
data = np.zeros((nrow, ncol, len(bands)))
for b in bands:
band = ds.GetRasterBand(b + 1)
for i in tqdm_gui(range(0, nrow, yBSize),
desc="Channel %d/%d" % (b, len(bands) - 1),
leave=False):
if i + yBSize < nrow:
numRows = yBSize
else:
numRows = nrow - i
for j in range(0, ncol, xBSize):
if j + xBSize < ncol:
numCols = xBSize
else:
numCols = ncol - j
data[i:(i + numRows), j:(j + numCols),
b] = band.ReadAsArray(j, i, numCols, numRows)
return data.astype(np.uint8)
def main():
activator = stats.Activator(desired_stats)
try:
progress_bar = tqdm_gui(desc='Computing statistics',
total=os.path.getsize(dataset_path),
position=0,
leave=False)
with open(dataset_path) as dataset:
order = 1
line = None
for row in dataset:
line = Line(
order,
row.rstrip('\n').split(
maxsplit=-1 if row[0] == 'H' else 2))
for stat in activator.active_stats[line.cmd]:
stat(line)
execute_line_no_loads(line)
order += 1
progress_bar.update(len(row))
for end_func in activator.end_dataset:
end_func(line)
progress_bar.close()
except OSError:
print('Could not open dataset.')
print(dataset_path)
progress_bar.close()
return
for writer in activator.console_output:
print(f'{writer.__self__.__class__.__name__}\n{writer()}\n')
for graph in activator.graph_output:
graph()
def main():
print('Welcome to unofficial dataset generator for key-value 2019 contest.')
total_lines = valid_input(prompt='Enter dataset size: ', key_formatter=int)
dataset: data.Dataset = valid_choice(data.datasets, heading='Choose dataset rules')
data.generate_cmds(dataset)
data.create_zones(total_lines, dataset)
progress_bar = tqdm_gui(
desc='Generating commands',
total=total_lines,
position=0,
leave=False
)
order = 1
for idx, end_of_zone in enumerate(dataset.zones):
Implementation.cyc_cmd = dataset.cycles_cmds[idx]
for i in range(order, end_of_zone + 1):
generated.append(row := add_line())
execute_line_no_loads(Line(order, row.split(maxsplit=-1 if row[0] == 'H' else 2)))
progress_bar.update()
order += 1
progress_bar.close()
print(
f'\n{total_lines} lines successfully generated.', '\nPreview:\n',
*list(generated[:PREVIEW]),
'...' if total_lines > PREVIEW else '',
sep='\n'
)
file_dialog(generated, prompt='Do you want to save dataset?', start_dir=start_at, ext='txt')
print('Program will now exit.')
def main():
file_info = [f'{idx + 1}: {item}' for idx, item in enumerate(sources)]
print(*file_info, sep='\n', end='\n\n')
progress_bar = None
try:
max_size = max(map(os.path.getsize, sources))
progress_bar = tqdm_gui(desc='Finding differences in files',
total=max_size,
position=0,
leave=False)
print(end=(newline := '\n'))
def add_label(self):
result_down = self.check_files()
if self.set_quarter == 1:
self.startday = '%s-04-25' % self.set_year
self.endday = '%s-07-25' % self.set_year
elif self.set_quarter == 2:
self.startday = '%s-07-25' % self.set_year
self.endday = '%s-10-25' % self.set_year
elif self.set_quarter == 3:
self.startday = '%s-10-25' % self.set_year
self.endday = '%s-01-25' % (self.set_year + 1)
elif self.set_quarter == 4:
self.startday = '%s-01-25' % (self.set_year + 1)
self.endday = '%s-04-25' % (self.set_year + 1)
else:
print('季度输入错误')
if result_down == True:
data = pd.read_csv(
'../report/base_information/perfprmance_%s-%s.csv' %
(self.set_year, self.set_quarter),
encoding='gbk')
data.code = data.code.apply(lambda x: '%06d' % x)
data['earnings'] = NaN
data['fluctuate'] = NaN
for item, code_ in tqdm.tqdm_gui(enumerate(data.code)):
print(item, code_, self.startday, self.endday)
try:
df = ts.get_k_data(code_, self.startday, self.endday)
price = df.close
price_f = price.iloc[0]
price_e = price.iloc[-1]
price_max = price.max()
price_min = price.min()
return_r = (price_e - price_f) / price_f * 100 # 收入
p_range = (price_max - price_min) / price_min * 100 #波动
data.ix[item, ['earnings']] = return_r
data.ix[item, ['fluctuate']] = p_range
print(price_f, price_e, price_max, price_min)
print(return_r.round(-1), p_range.round(-1))
except:
pass
data.to_csv(
'../report/base_information/perfprmance_earn_%s-%s.csv' %
(self.set_year, self.set_quarter),
encoding='gbk',
index=False)
else:
pass
def concat_csv(path_dir):
file_list = os.listdir(path_dir) # path에 존재하는 파일 읽기
file_list.sort() # 파일 이름순서로 정렬
for i in tqdm_gui(range(len(file_list)), desc='Merging progress'):
file = file_list[i]
path = path_dir + '{}'.format(file)
df_temp = pd.read_csv(path, sep=',', encoding="euc-kr")
if i == 0:
df = df_temp.copy()
else:
df = pd.concat([df, df_temp], axis=0, sort=False)
return df
def get_all_magic():
CodeList = list_input('all')
i = 2
df = get_magic(CodeList.code[i * 50])
df = df.sort_index(ascending=False)
for code_ in tqdm.tqdm_gui(CodeList.code[i * 50 + 1:i * 50 + 50]):
time.sleep(3)
print(code_)
mg = get_magic(code_)
mg.name = code_
df = pd.concat([df, mg], axis=1)
df.to_csv('..\\report\\magic_%s.csv' % CodeList.code[i * 50])
return df
def end_of_dataset(self, last_line):
total_keys, keys_one_mention, keys_set_after_del = 0, 0, 0
distances_medians, distances_lengths = [], []
progress_bar = tqdm_gui(desc='Computing medians',
total=len(self.records) + 4,
position=0,
leave=False)
for key in self.records:
record = self.records[key]
total_keys += 1
distances_lengths.append(len(record.distances) + 1)
if record.set_after_del:
keys_set_after_del += 1
if record.distances:
distances_medians.append(median(record.distances))
else:
keys_one_mention += 1
progress_bar.update()
def _median(source: list):
m = median(source) if source else 0
progress_bar.update()
return m
self.wtab = Ptw(['Subject', 'Value'], aligns='cR')
self.results = {
'Total number of keys':
total_keys,
'Keys seen once only':
keys_one_mention,
'Keys set after removing':
keys_set_after_del,
'Median of distances':
_median(distances_medians),
'Median of number of cmds per key':
_median(distances_lengths),
'Median of number of keys per hashload':
_median(self.H_list_lengths),
'Median of number of unique keys per hashload':
_median(self.H_set_lengths)
}
for subject in self.results:
self.wtab.write_raw([subject, self.results[subject]])
self.wtab.add_raw_to_table()
progress_bar.close()
def main():
try:
progress_bar = tqdm_gui(desc='Solving dataset',
total=os.path.getsize(dataset_path),
position=0,
leave=False)
with open(dataset_path) as dataset:
line = None
for row in dataset:
line = Line(
None,
row.rstrip('\n').split(
maxsplit=-1 if row[0] == 'H' else 2))
if (result := execute_line_return_result(line)):
generated.append(result)
progress_bar.update(len(row))
progress_bar.close()
def run(self, graph=False, verbose=True):
results = [0] * len(self.setup.teams)
num_mafia = self.setup.countAlignment(self.setup.mafia_type) + 1
num_town = self.setup.countAlignment(self.setup.town_type) + 1
town_results_by_day = [[0 for x in range(num_mafia)]
for x in range(num_town)]
maf_results_by_day = [[0 for x in range(num_mafia)]
for x in range(num_town)]
self.combined_results_by_day = [[0 for x in range(num_mafia)]
for x in range(num_town)]
last_day_count = [0] * (len(self.setup.players) + 1)
for i in tqdm_gui(range(self.num_iterations)):
sim = MafiaGame(setup=self.setup, verbose=verbose)
sim.play()
for [town, maf] in sim.alignment_numbers_history:
if (sim.winner.name == "Town"):
town_results_by_day[town][maf] += 1
else:
maf_results_by_day[town][maf] += 1
results[[x.name
for x in self.setup.teams].index(sim.winner.name)] += 1
last_day_count[sum(sim.alignment_numbers_history[-1])] += 1
# Results
for i, alignment in enumerate(self.setup.teams):
print(f"{alignment.name}: {results[i]/self.num_iterations}")
for i, num in enumerate(last_day_count):
if (num != 0):
print(f"{i}: {num/self.num_iterations}")
for i in range(len(town_results_by_day)):
for j in range(len(town_results_by_day[i])):
t = town_results_by_day[i][j]
m = maf_results_by_day[i][j]
if t + m == 0:
print("----", end=" ")
self.combined_results_by_day[i][j] = -1
else:
self.combined_results_by_day[i][j] = t / (t + m)
print("%3.2f" % (self.combined_results_by_day[i][j]),
end=" ")
print("")
if (graph):
self.drawGraph()
def func2():
input_list = sorted(
glob('./../datasets/autocolorization/illustrations_resized/*.*'))
check_folder(
'./../datasets/autocolorization/illustrations_resized_256/original/')
check_folder(
'./../datasets/autocolorization/illustrations_resized_256/xdog/')
for e, data in tqdm_gui(enumerate(input_list)):
image = misc.imread(data, mode='RGB')
image = resize_and_crop(image, 256)
misc.imsave(
'./../datasets/autocolorization/illustrations_resized_256/original/'
+ str(e) + '.png', np.array(image))
image = xdog(image, sigma_list=[0.3, 0.4, 0.5])
misc.imsave(
'./../datasets/autocolorization/illustrations_resized_256/xdog/' +
str(e) + '.png', np.array(image))
def add_google_matrix():
distance_km_list = []
time_min_list = []
id_list = []
df_ulykke = pd.read_csv('./csv/komplett2010.csv',
'r',
delimiter=',',
encoding='iso-8859-1')
df_ulykke = df_ulykke[[
'Ulykkes id', 'latitude', 'longitude', 's_id', 's_latitude',
's_longitude'
]]
for i in tqdm_gui(range(len(df_ulykke))):
try:
acc_id = df_ulykke['Ulykkes id'][i]
lat = df_ulykke['latitude'][i]
long = df_ulykke['longitude'][i]
origin = f'{lat},{long}'
s_lat = df_ulykke['s_latitude'][i]
s_long = df_ulykke['s_longitude'][i]
destination = f'{s_lat},{s_long}'
except:
pass
try:
get = get_google_dist_matrix(origin, destination)
except:
get = [0, 0]
distance_km = get[0]
time_min = get[1]
distance_km_list.append(distance_km)
time_min_list.append(time_min)
id_list.append(acc_id)
print(f'Added info for ulykke {acc_id}')
google_distance = pd.DataFrame({
'sv_acc_id': id_list,
'road_km': distance_km_list,
'time_min': time_min_list
})
google_distance.to_csv('google_distance_matrix.csv')
print('I made the file, now let me rest -.- ')
def get_progress_bar(progress_bar_type, description, total, unit):
"""Construct a tqdm progress bar object, if tqdm is ."""
if tqdm is None:
if progress_bar_type is not None:
warnings.warn(_NO_TQDM_ERROR, UserWarning, stacklevel=3)
return None
try:
if progress_bar_type == "tqdm":
return tqdm.tqdm(desc=description, total=total, unit=unit)
elif progress_bar_type == "tqdm_notebook":
return tqdm.tqdm_notebook(desc=description, total=total, unit=unit)
elif progress_bar_type == "tqdm_gui":
return tqdm.tqdm_gui(desc=description, total=total, unit=unit)
except (KeyError, TypeError):
# Protect ourselves from any tqdm errors. In case of
# unexpected tqdm behavior, just fall back to showing
# no progress bar.
warnings.warn(_NO_TQDM_ERROR, UserWarning, stacklevel=3)
return None
def __download_data(self, show_progress=False):
url = URL + self.lang + ".tar.gz"
print("Downloading a new data set in " + VALID_LANGUAGES[self.lang] +
" to " + self.path)
print("from " + url + "...")
try:
if show_progress:
from tqdm import tqdm_gui
with open(self.archive, "wb") as file:
for data in tqdm_gui(
requests.get(url, stream=True).iter_content()):
file.write(data)
else:
from tqdm import tqdm
with open(self.archive, "wb") as file:
for data in tqdm(
requests.get(url, stream=True).iter_content()):
file.write(data)
except KeyboardInterrupt:
print("Interrupted by user.")
self.__del_archive()
sys.exit(1)
def user_similarity_analysis(names, uids, filename):
doc_path = './data/all_doc_list.pkl'
if os.path.exists(doc_path):
with open(doc_path, 'rb') as f:
all_doc_list = pickle.load(f)
else:
all_doc_list = []
for name, uid in tqdm_gui(list(zip(names, uids))[len(all_doc_list):]):
print('获取弹幕：{}'.format(name))
user = User(uid)
user.get_danmaku()
print('弹幕长度：{}'.format(len(user.danmaku_list)))
user.danmaku_list.extract_keywords(500)
doc = user.danmaku_list.tags
# doc = [word for sentence in user.danmaku_list for word in jieba.cut(sentence)]
all_doc_list.append(doc)
with open(doc_path, 'wb') as f:
pickle.dump(all_doc_list, f)
global dictionary, corpus
dictionary = corpora.Dictionary(all_doc_list)
corpus = [dictionary.doc2bow(doc) for doc in all_doc_list]
similarity_matrix = np.zeros((len(names), len(names)))
dataset = np.zeros((len(names), len(dictionary.keys())))
for idx, name in enumerate(names):
doc_test_list = all_doc_list[idx]
sim_row, data_row = calc_similarities(doc_test_list)
similarity_matrix[idx, :] = sim_row
dataset[idx, :] = data_row
sim_frame = pd.DataFrame(similarity_matrix, index=names, columns=names)
sim_frame.to_csv('data/{}.csv'.format(filename), encoding='utf8')
df = pd.DataFrame(dataset, index=names)
df.to_csv('data/{}_dataset.csv'.format(filename), encoding='utf8')
return sim_frame, df
def _draw_rtree_nodes(graph, tree: RTreeBase, include_images):
num_plots = len(list(tree.get_nodes())) + len(list(tree.get_leaf_entries()))
with tqdm_gui(total=num_plots, desc="Drawing R-Tree", unit="node") as pbar:
for level, nodes in enumerate(tree.get_levels()):
subgraph = pydot.Subgraph(rank='same')
graph.add_subgraph(subgraph)
for node in nodes:
img = None
if include_images:
img = tempfile.mkstemp(prefix='node_', suffix='.png')[1]
highlight_node = node if not node.is_root else None
plot_rtree(tree, filename=img, show=False, highlight_node=highlight_node)
subgraph.add_node(_rtree_node_to_pydot(node, img))
pbar.update()
leaf_subgraph = pydot.Subgraph(rank='same')
graph.add_subgraph(leaf_subgraph)
for entry in tree.get_leaf_entries():
img = None
if include_images:
img = tempfile.mkstemp(prefix='entry_', suffix='.png')[1]
plot_rtree(tree, filename=img, show=False, highlight_entry=entry)
leaf_subgraph.add_node(_rtree_leaf_entry_to_pydot(entry, img))
pbar.update()
def tag_similarity_analysis(keywords, filename, tids_1=None):
doc_path = './data/all_doc_list.pkl'
if os.path.exists(doc_path):
with open(doc_path, 'rb') as f:
all_doc_list = pickle.load(f)
else:
all_doc_list = []
for keyword in tqdm_gui(keywords[len(all_doc_list):]):
danmaku_list = search_and_get_danmaku(keyword, tids_1)
danmaku_list.extract_keywords(500)
doc = danmaku_list.tags
all_doc_list.append(doc)
with open(doc_path, 'wb') as f:
pickle.dump(all_doc_list, f)
global dictionary, corpus
dictionary = corpora.Dictionary(all_doc_list)
corpus = [dictionary.doc2bow(doc) for doc in all_doc_list]
similarity_matrix = np.zeros((len(keywords), len(keywords)))
dataset = np.zeros((len(keywords), len(dictionary.keys())))
for idx, name in enumerate(keywords):
doc_test_list = all_doc_list[idx]
sim_row, data_row = calc_similarities(doc_test_list)
similarity_matrix[idx, :] = sim_row
dataset[idx, :] = data_row
sim_frame = pd.DataFrame(similarity_matrix,
index=keywords,
columns=keywords)
sim_frame.to_csv('data/{}.csv'.format(filename), encoding='utf8')
df = pd.DataFrame(dataset, index=keywords)
df.to_csv('data/{}_dataset.csv'.format(filename), encoding='utf8')
return sim_frame, df
def retrain(self, steps):
inp, out = self.build_full('QQQ')
inp = np.reshape(inp[:, :, :, 0], newshape=[-1, 30, 5, 1])
out = np.reshape(out, [-1, 5, 5, 1])
meta = '{}/{}_model.ckpt.meta'.format(self.direc, self.symbol.lower())
ckpt = '{}/{}_model.ckpt'.format(self.direc, self.symbol.lower())
with tf.Session() as sess:
saver = tf.train.import_meta_graph(meta)
saver.restore(sess, ckpt)
for i in tqdm_gui(range(int(steps))):
if i % 50 == 0:
train_accuracy = sess.run("Mean:0",
feed_dict={
"input_:0": inp,
"exp_out:0": out
})
ac = sess.run("Sub:0",
feed_dict={
"input_:0": inp,
"exp_out:0": out
})
print(
'========================SUMMARY REPORT============================='
)
print('step %d, train loss: %g' % (i, train_accuracy))
print('Validation accuracy {}%'.format(str(ac)))
# print('Estimated Time Remaining = ' + str(round((20000-i)*(timer/60)/60,2)) + ' Hours')
print(
'==================================================================='
)
sess.run("Adam", feed_dict={"input_:0": inp, "exp_out:0": out})
saver.save(sess, ckpt)
with open(self.direc + "/time.txt", 'w') as f:
f.write('%s' % (self.get_time()))
print(self.get_time())
from tqdm import tqdm, tqdm_gui
import time
for i in tqdm_gui(range(1000), gui=True, disable=False):
time.sleep(0.1)
def aug_function(inpath, outputpath, incsvfilepath, outcsvfilepath, augtype,
t_factor, r_factor, s_factor):
#print("\n")
#print(augtype)
#print("\n")
file1 = open(incsvfilepath,
'r') # 2. open file containing bounding box coordinates
lines = file1.readlines()
randlist = lines # selecting k random images to augment
for i in tqdm_gui(randlist):
x = i.strip().split(",")
#print("\nx: ")
#print(x)
#print("\n")
mylist = [
int(float(x[1])),
int(float(x[2])),
int(float(x[3])),
int(float(x[4]))
] #extract coordinates from file
cord = []
for i in mylist:
a = int(i)
cord.append(a)
a = x[0] # image filename
imageFolderPath = inpath # 3. input image folder path
loc = os.path.join(imageFolderPath, a) #location of image
# # Brightness and Contrast
if (augtype == 'Brightness and Contrast'):
image = cv.imread(loc)
alpha = random.triangular(1, 2)
beta = random.randint(20, 50)
result = cv.addWeighted(image, alpha,
np.zeros(image.shape, image.dtype), 0,
beta)
oname = "brightness_" + a
csvr = [oname] + mylist
with open(outcsvfilepath, 'a',
newline='') as file: # 4. write to csv file
writer = csv.writer(file)
writer.writerow(csvr)
#print(mylist)
#print(oname)
drawRectangle(result, outputpath, cord, output_name=oname)
else:
img_class = utils.Image(path=loc) # Create image class
img = img_class.getImage()
coord = cord
if (augtype == 'Scale'):
# # Test Scaling
output = img_class.transform('scale', coord, s_factor)
oname = "scaled_" + a # name of output image
ocord = output[1] # augmented coordinates
csvr = [oname
] + output[1] # line to be written to output csv file
with open(outcsvfilepath, 'a',
newline='') as file: # 4. write to csv file
writer = csv.writer(file)
writer.writerow(csvr)
drawRectangle(output[0],
outputpath,
output[1],
output_name=oname) # for saving output image
elif (augtype == 'Rotate'):
#print("\nrotate")
##Test Rotation
output = img_class.transform('rotate', coord, r_factor)
oname = "rotated_" + a
# ##print("ONAME",oname)
# ##print("New Bounding Boxes rotation: ", output[1])
ocord = output[1]
csvr = [oname] + output[1]
# ##print("ocord",ocord)
with open(outcsvfilepath, 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow(csvr)
drawRectangle(output[0],
outputpath,
output[1],
output_name=oname)
#drawRectangle(img,outputpath, coord, output_name = "output_original.jpeg")
elif (augtype == 'Translate'):
#print("\ntranslate")
# # Test Translation - Horizontal
output = img_class.transform('translate', coord, t_factor)
oname = "translated_" + a # name of output image
ocord = output[1] # augmented coordinates
csvr = [oname
] + output[1] # line to be written to output csv file
with open(outcsvfilepath, 'a',
newline='') as file: #write to csv file
writer = csv.writer(file)
writer.writerow(csvr)
drawRectangle(output[0],
outputpath,
output[1],
output_name=oname) # for saving output image
elif (augtype == 'Flip'):
#print("\nflip")
# # Test Flipping
output = img_class.transform('flip', coord)
oname = "flipped_" + a # name of output image
ocord = ' '.join(str(e)
for e in output[1]) # augmented coordinates
csvr = oname + ' ' + ocord # line to be written to output csv file
listcsvr = csvr.split(" ")
#print("\noutcsvfilepath: ")
#print(outcsvfilepath)
#print("\ncsvr: ")
#print(csvr)
with open(outcsvfilepath, 'a',
newline='') as file: #write to csv file
writer = csv.writer(file)
writer.writerow(listcsvr)
drawRectangle(output[0],
outputpath,
output[1],
output_name=oname) # for saving output image
elif (augtype == 'Shear'):
# # Test Shear
#print("\nshear")
output = img_class.transform('shear', coord)
oname = "shear_" + a # name of output image
ocord = output[1] # augmented coordinates
csvr = [oname
] + output[1] # line to be written to output csv file
with open(outcsvfilepath, 'a',
newline='') as file: #write to csv file
writer = csv.writer(file)
writer.writerow(csvr)
drawRectangle(output[0],
outputpath,
output[1],
output_name=oname) # for saving output image
elif (augtype == 'Saturation'):
#print("\nhsv")
HSV_output_name = "Saturated_" + a
img_HSV, bboxes_HSV = utils.RandomHSV(hue=None,
saturation=100,
brightness=None)(
img.copy(),
cord.copy())
drawRectangle(
img_HSV,
outputpath,
bboxes_HSV,
output_name=HSV_output_name) # for saving output image
csvr = [HSV_output_name
] + bboxes_HSV # line to be written to output csv file
with open(outcsvfilepath, 'a',
newline='') as file: #write to csv file
writer = csv.writer(file)
writer.writerow(csvr)
sum(n.performance) / len(n.performance) * 100)
plt.pause(.05)
lastUpdate = time.time()
pass
pass
pass
plt.show()
# %%
scorecard = []
testData = pd.read_csv("https://pjreddie.com/media/files/mnist_test.csv",
header=None)
ai = tqdm_gui(total=1)
for index, row in tqdm_gui(testData.iterrows(), total=testData[0].size):
corectLabel = int(row[0])
inputs = (numpy.asfarray(row[1:]) / 255.0 * 0.99) + 0.01
outputs = n.query(inputs)
label = numpy.argmax(outputs)
if (label == corectLabel):
scorecard.append(1)
else:
scorecard.append(0)
pass
from tqdm import tqdm_gui
import tqdm
from random import random, randint
from time import sleep
for i in tqdm_gui(range(0, 50)):
#tqdm_gui.set_description()
sleep(.03)
##t = tqdm.__init__(self, iterable=None, desc=None, total=None, leave=True, file=None, ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, position=None, postfix=None, unit_divisor=1000, gui=False, **kwargs)
para['kindex'] = GPdc.MAT52
para['mprior'] = sp.array([0.]+[-1.]*d)
para['sprior'] = sp.array([1.]*(d+1))
para['s'] = 1e-9
para['ninit'] = 10
#para['maxf'] = 2500
para['volper'] = 1e-6
para['DH_SAMPLES'] = 8
para['DM_SAMPLES'] = 8
para['DM_SUPPORT'] = 1200
para['DM_SLICELCBPARA'] = 1.
para['SUPPORT_MODE'] = [ESutils.SUPPORT_SLICELCB,ESutils.SUPPORT_SLICEPM]
OP = OPTutils.PESFS(ojf,lb,ub,para,initstate=copy.deepcopy(initstate))
#[OP.X,OP.Y,OP.S,OP.D,OP.R,OP.C,OP.T,OP.Tr,OP.Ymin,OP.Xmin,OP.Yreg,OP.Rreg] = copy.deepcopy(initstate)
for i in tqdm_gui(xrange(runn),gui=True):
state = [OP.X,OP.Y,OP.S,OP.D,OP.R,OP.C,OP.T,OP.Tr,OP.Ymin]
try:
pass
#OP.step()
except:
import pickle
pickle.dump(state,open('state.p','wb'))
raise
OE.step()
O.step()
#OL.step()
plt.close(f)
plt.plot(progress_range, durations)
#plt.plot(progress_range)
#plt.plot(durations)
plt.show()
count=0
for i in progress_range:
# here do something long at each iteration
time.sleep(durations[count])
count += 1
pbar.update(i) #this adds a little symbol at each iteration
pbar.finish()
print
print("Using tqdm")
from tqdm import tqdm
count=0
for i in tqdm(progress_range):
time.sleep(durations[count])
count += 1
print("Using tqdm_gui")
from tqdm import tqdm_gui
count=0
for i in tqdm_gui(progress_range):
time.sleep(durations[count])
count += 1
print(
colored(
'Before using please make sure your csv file has a "Name" Column to get names',
'red'))
university = input("Enter Your University Name: ")
acronym = input("Enter Your University Acronym: ")
eventname = input("Enter event name: ")
leadname = input("Your Name: ")
currentdate = datetime.date(datetime.now())
fname = 'certificates/'
if os.path.exists(fname):
shutil.rmtree(fname)
os.mkdir(fname)
for names in tqdm_gui(file['Name']):
image = Image.new('RGB', (1000, 900), (255, 255, 255))
draw = ImageDraw.Draw(image)
font_path = './Almondita.ttf'
fontdev = ImageFont.truetype('arial.ttf', size=35)
fontcert = ImageFont.truetype('arial.ttf', size=55)
fontname = ImageFont.truetype('arial.ttf', size=35)
signature = ImageFont.truetype(font_path, 150)
colordev = 'rgb(128, 128, 128)'
colorcert = 'rgb(89, 89, 89)'
colorname = 'rgb(77, 148, 255)'
dsc_logo = Image.open('logo.jpg')
dsc_logo = dsc_logo.resize((75, 75))
img.save('Albedomap_grey_nov.png')
FILENAME = 'Albedomap_grey_nov.png' #image can be in gif jpeg or png format
im = Image.open(FILENAME).convert('RGB')
pix = im.load()
# Define longitude of ascending node and loop statements
k, k_final, dk = -180, 180, 1
longrun = False
zeroiteration = True
firstiteration = False
seconditeration = False
#loop to obtain all data
while dogleg == True:
warnings.simplefilter("ignore")
for i in tqdm_gui(range(388)):
time.sleep(0.0001)
#intial values
xtab = []
ytab = []
x2tab = []
y2tab = []
albedotab = []
ttab = []
lattab = []
difftab = []
#Several runs over range of possible k at different precisions
if k > k_final and zeroiteration == True:
zeroiteration = False
"2010", "2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018",
"2019"
]
target_month = [
"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"
]
target_date = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", \
"11", "12", "13", "14", "15", "16", "17", "18", "19", "20", \
"21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"]
# Target url
url_format = "http://www.airportal.go.kr/servlet/aips.mobile.MobileRbHanCTL?cmd=c_getList&index=0&count=500&depArr=D¤t_date={search_date}&tm={search_hour}&airport=ICN"
# headers = {'content-type': 'application/json;charset=utf-8'}
# ScrabpingMain Loop
for p_year in tqdm_gui(target_year, desc='target_year'):
for p_month in target_month:
# 종료 조건 추가 '2019년 9월'까지 데이터 수집
if p_year == "2019" and p_month == "10": break
last_day = calendar.monthrange(int(p_year), int(p_month))[1] # 30
for p_date in target_date:
# 해당월의 마지막 날짜 체크
exception_tf = False
if last_day < (int(p_date)): break
search_date = p_year + p_month + p_date
if search_date in empty_list:
print("search_date{} is no data".format(search_date))
continue
if search_date in file_list:
from tqdm import tqdm_gui
import random
from time import time
for _ in tqdm_gui(range(10**8)):
val = random.randint(0, 100)
L.append(val)
for _ in tqdm_gui(range(10**6)):
pos = random.randint(0, len(L)) ## insertion avant
val = random.randint(0, 100)
L.insert(pos, val)
## Astuce
L = []
debut = time()
for i in range(10**3):
L.insert(0, i)
fin = time()
duree = fin - debut
print("Duree Insert: ", duree)
L = []
debut = time()
for i in range(10**3):
L.append(i)
L.reverse()
fin = time()
duree = fin - debut
print("Duree Append+Reverse: ", duree)
def train_network(stock, years, steps, direct, scale=True):
bd = BuildDataset(symbol=stock, years=years, scale=scale)
inp, out = bd.build_full('QQQ')
inp=np.reshape(inp[:,:,:,0], newshape=[-1, 30,5,1])
out = np.reshape(out, [-1, 5,5,1])
print("Input data size:" + str(inp.shape))
print("Output data size:" + str(out.shape))
inp, inp_v, y, y_v = train_test_split(inp, out, test_size=.10)
print(inp_v.shape)
def next_batch(num, data, labels):
'''
Return a total of `num` random samples and labels.
'''
idx = np.arange(0, len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = [data[i] for i in idx]
labels_shuffle = [labels[i] for i in idx]
return np.asarray(data_shuffle), np.asarray(labels_shuffle)
def weight_var(shape):
init = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(init)
def bias_var(shape):
init = tf.constant(0.0, shape=shape)
return tf.Variable(init)
def conv2d (x, W, stride):
return tf.nn.conv2d(x, W, strides=[1, stride, stride, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
def Conv_Layer(input_, shape, name, str=1, pool=False):
with tf.variable_scope(name):
with tf. name_scope('Weights'):
w = weight_var(shape)
with tf.name_scope('Bias'):
b = bias_var([shape[3]])
h_conv = tf.nn.relu(conv2d(input_, w, stride=str)+ b)
if pool == True:
return max_pool_2x2(h_conv)
return h_conv
with tf.device("/gpu:0"):
x_ = tf.placeholder(tf.float32, shape=[None, 30, 5, 1], name='input_')
y_exp = tf.placeholder(tf.float32, shape=[None, 5, 5, 1], name='exp_out')
#Conv 1
cnv1 = Conv_Layer(x_, [3, 3, 1, 6], name="conv1")
#Conv 2
cnv2 = Conv_Layer(cnv1, [3, 3, 6, 6], name="conv2")
with tf.name_scope("max_pool"):
cnv2 = max_pool_2x2(cnv2)
cnv22 = Conv_Layer(cnv2, [3, 3, 6, 16], name="conv22")
cnv23 = Conv_Layer(cnv22, [3, 3, 16, 32], name="conv23")
#Conv 3
cnv3 = Conv_Layer(cnv23, [3, 3, 32, 32], name="conv3")
#Conv 4
cnv4 = Conv_Layer(cnv3, [3, 3, 32, 32], name="conv4")
with tf.name_scope("max_pool"):
cnv4 = max_pool_2x2(cnv4)
#Fully connected layers
with tf.name_scope("fully_connected1"):
w_1 = weight_var([8*2*32, 1000])
b1 = bias_var([1000])
c4_flat = tf.reshape(cnv4, [-1, 8*2*32])
fc1 = tf.nn.relu(tf.matmul(c4_flat, w_1) + b1)
with tf.name_scope("fully_connected2"):
w_2 = weight_var([1000, 800])
b2 = bias_var([800])
fc2 = tf.nn.relu(tf.matmul(fc1, w_2) + b2)
with tf.name_scope("fully_connected3"):
w_3 = weight_var([800, 400])
b3 = bias_var([400])
fc3 = tf.nn.relu(tf.matmul(fc2, w_3) + b3)
with tf.name_scope("Up_conv1"):
fc3 = tf.reshape(fc3, shape=[-1, 20, 20, 1])
#d_cnv1 = tf.layers.conv2d_transpose(fc3, filters=16 , kernel_size = 2, strides=2)
cnvu1 = Conv_Layer(fc3, shape=[3, 3, 1, 6], name="dconv1", pool=True)
with tf.name_scope("Up_conv2"):
cnvu2 = Conv_Layer(cnvu1, shape=[3, 3, 6, 16], name="dconv1", pool=False)
cnvu2 = Conv_Layer(cnvu2, shape=[3, 3, 16, 6], name="dconv2", pool=True)
with tf.name_scope("Up_conv3"):
#d_cnv2 = tf.layers.conv2d_transpose(cnvu1, filters=6, kernel_size=1, strides=1)
logits = Conv_Layer(cnvu2, shape=[3, 3, 6, 1], name="dconv2")
tf.identity(logits, name="y_out")
#real_c = tf.summary.image("Exp", y_exp, max_outputs=2)
#pred_c = tf.summary.image("pred", logits, max_outputs=2)
loss = tf.reduce_mean(tf.losses.mean_squared_error(labels=y_exp, predictions=logits))
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
acc = tf.reduce_mean(tf.losses.absolute_difference(labels=y_exp,predictions=logits))
acc = tf.cast(tf.subtract(tf.constant(100, dtype=tf.float32),tf.multiply(acc, tf.constant(100, dtype=tf.float32))), tf.float32)
train_c = tf.summary.scalar("Train_loss", loss)
val_c = tf.summary.scalar('Val_loss', loss)
Accuracy = tf.cast(loss, tf.float32)
print(acc)
print(Accuracy)
saver = tf.train.Saver(tf.all_variables())
# TF SESSION
config =tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth =True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
timer = 0
train_start =time()
write = tf.summary.FileWriter('/home/ian/Quant', sess.graph)
j= 0
loss = 0
patience = 0
for i in tqdm_gui(range(int(steps))):
time_start = time()
#batch = next_batch(len(), inp, y)
if i% 5==0:
lo = acc.eval(feed_dict={x_: inp_v, y_exp: y_v})
if lo-loss < .0005:
patience +=1
else:
patience=0
loss = lo
#print(patience)
if patience == 5:
break
if i % 50 == 0:
train_accuracy = Accuracy.eval(feed_dict={x_: inp, y_exp: y})
ac = acc.eval(feed_dict={x_: inp_v, y_exp: y_v})
tc = train_c.eval(feed_dict={x_: inp, y_exp: y})
vc = val_c.eval(feed_dict={x_: inp_v, y_exp: y_v})
#image1, image2 = sess.run([real_c, pred_c],feed_dict={x_: batch[0][:10], y_exp: batch[1][:10]})
#p = int(100 * np.random.rand())
#print(bd.rescale(logits.eval(feed_dict={x_: inp_v[:100]})[p]).astype(np.int))
#print(bd.rescale(y_v[p]).astype(np.int))
#write.add_summary(image1, i)
#write.add_summary(image2, i)
write.add_summary(tc, i)
write.add_summary(vc, i)
print('========================SUMMARY REPORT=============================')
print('step %d, train loss: %g' % (i,train_accuracy))
print('Validation accuracy {}%'.format(str(ac)))
#print('Estimated Time Remaining = ' + str(round((20000-i)*(timer/60)/60,2)) + ' Hours')
print('===================================================================')
train_step.run(feed_dict={x_: inp, y_exp: y})
time_stop = time()
timer = time_stop - time_start
#print('Step: ' + str(i) + ' Epoch Time: ' + str(round(timer,2)) + ' Secs.' + ' Time elapsed: ' +
#str(round((time_stop-train_start)/60, 2)) + ' Mins.' + str(round((i/80000) *100, 1)) + " % complete")
directory = direct
if not os.path.exists(directory):
os.makedirs(directory)
s_path = saver.save(sess, "{}/{}_model.ckpt".format(directory, stock.lower()))
print("model saved in {}".format(s_path))
with open(directory+"/time.txt", 'w') as f:
f.write('%s' % (bd.get_time()))
print(bd.get_time())
def train(model, train_data, valid_data, optim, device, opt, start_i):
if opt.log:
log_train_file = opt.log + '/train.log'
log_valid_file = opt.log + '/valid.log'
print(
'[INFO] Training performance will be written to {} and {}'.format(
log_train_file, log_valid_file))
# check log file exists or not
if not (os.path.exists(opt.log)):
os.mkdir(opt.log)
if not (os.path.exists(log_train_file)
and os.path.exists(log_valid_file)):
with open(log_train_file,
'w') as log_tf, open(log_valid_file, 'w') as log_vf:
log_tf.write('epoch,loss\n')
log_vf.write('epoch,loss\n')
train_loss_list = []
valid_loss_list = []
for epoch_i in tqdm_gui(range(start_i, opt.epoch)):
print('[INFO] Epoch: {}'.format(epoch_i))
# train process
start = time.time()
train_loss = train_epoch(model, train_data, optim, device, opt)
print('\t- (Training) loss: {:8.5f}, elapse: {:3.3f}'.format(
train_loss, (time.time() - start) / 60))
train_loss_list += [train_loss] # record each train loss
# valid process
start = time.time()
valid_loss = valid_epoch(model, valid_data, device, opt)
print('\t- (Validation) loss: {:8.5f}, elapse: {:3.3f}'.format(
valid_loss, (time.time() - start) / 60))
valid_loss_list += [valid_loss] # record each valid loss
# record train and valid log files
if log_train_file and log_valid_file:
with open(log_train_file,
'a') as log_tf, open(log_valid_file, 'a') as log_vf:
log_tf.write('{},{:8.5f}\n'.format(epoch_i, train_loss))
log_vf.write('{},{:8.5f}\n'.format(epoch_i, valid_loss))
# to save trained model
model_state_dict = model.state_dict()
checkpoint = {
'model': model_state_dict,
'setting': opt,
'epoch': epoch_i
}
if not (os.path.exists(opt.chkpt)):
os.mkdir(opt.chkpt)
if opt.save_mode == 'best':
model_name = '{}/eye_model.chkpt'.format(opt.chkpt)
if train_loss <= min(train_loss_list):
torch.save(checkpoint, model_name)
print('\t[INFO] The checkpoint has been updated ({}).'.format(
opt.save_mode))
elif opt.save_mode == 'interval':
if (epoch_i % opt.save_interval) == 0 and epoch_i != 0:
model_name = '{}/{}_{:0.3f}.chkpt'.format(
opt.chkpt, epoch_i, train_loss)
torch.save(checkpoint, model_name)
print('\t[INFO] The checkpoint has been updated ({}).'.format(
opt.save_mode))
elif opt.save_mode == 'best_and_interval':
model_name = '{}/eye_model.chkpt'.format(opt.chkpt)
if train_loss <= min(train_loss_list):
torch.save(checkpoint, model_name)
print('\t[INFO] The best has been updated ({}).'.format(
opt.save_mode))
if (epoch_i % opt.save_interval) == 0 and epoch_i != 0:
model_name = '{}/{}_{:0.3f}.chkpt'.format(
opt.chkpt, epoch_i, train_loss)
torch.save(checkpoint, model_name)
print('\t[INFO] The checkpoint has been saved ({}).'.format(
opt.save_mode))
# save last trained model
if epoch_i == (opt.epoch - 1):
model_name = '{}/{}_{:0.3f}.chkpt'.format(opt.chkpt, epoch_i,
train_loss)
torch.save(checkpoint, model_name)
print('\t[INFO] The last checkpoint has been saved.')
variants = CaptureOne.selected_variants()
for variant in variants:
images.append(variant.parent_image.get())
else:
raise ValueError(
"Don't know what variants/images to use, please specify --all, --collection or --selected"
)
if arguments["--progress"]:
image_iterator = tqdm.tqdm(images,
unit="Image",
unit_scale=False,
leave=True,
position=0)
elif arguments["--progress-gui"]:
image_iterator = tqdm.tqdm_gui(images, unit="Image", unit_scale=False)
else:
image_iterator = images
for img_ae_obj in image_iterator:
image = C1Image(img_ae_obj)
matched_image = None
if image.photo_name_size_key() in new_location_files:
matched_image = new_location_files[image.photo_name_size_key()]
log = None
if arguments["--progress"]:
log = tqdm.tqdm.write
else:
log = print
internationalsatlist, internationalsunlist = [], [], [], [], [],[], []
url = url_format.format(api_key=api_key, page_no=page_no)
response = requests.get(url, headers=headers)
html = response.text
soup = BeautifulSoup(html, 'html.parser')
rescode = response.status_code
# 제대로 데이터가 수신됐는지 확인하는 코드 성공시 200
if (rescode == 200):
soup = BeautifulSoup(html, 'html.parser')
total_count = int(soup.find("totalcount").text)
else : print("search_error")
# for page_no in tqdm_notebook(range(total_count), desc = 'page'):
for page_no in tqdm_gui(range(total_count), desc='page'):
print(page_no)
url = url_format.format(api_key=api_key, page_no=page_no+1)
response = requests.get(url, headers=headers)
html = response.text
soup = BeautifulSoup(html, 'html.parser')
rescode = response.status_code
# 제대로 데이터가 수신됐는지 확인하는 코드 성공시 200
if (rescode == 200):
soup = BeautifulSoup(html, 'html.parser')
airlinekorean = soup.find_all("airlinekorean")
airport = soup.find_all("airport")
