def main():
time_elapsed = int(time.time())
args = argparser()
copyfile(args.cdb, args.cdb + ".backup") #make a backup of the cards.cdb
con = sql_connection(args.cdb)
cursor = con.cursor()
cursor.execute("SELECT id FROM texts")
req = cursor.fetchall()
prog = pyprog.ProgressBar("", "", len(req), complete_symbol="█", not_complete_symbol="-")
prog2 = pyprog.ProgressIndicatorFraction("", "", len(req))
prog.update()
prog2.update()
i = 0
for row in req:
i += 1
prog.set_stat(i)
prog2.set_stat(i)
prog.update()
prog2.update()
id = add_zero(row[0])
page = search_card(id, args.yugipedia)
info = get_info(id, page, args)
if info is not None:
print(info)
cursor.execute("UPDATE texts SET name = (?), desc = (?) WHERE id = (?)", (info['name'], info['description'], row[0]))
con.commit()
else:
missing(id)
time_elapsed = int(time.time()) - time_elapsed
print()
print("Script ended in: " + str(time_elapsed))
con.close()
def progresso(self, x):
prog = pyprog.ProgressBar("\tImportando Hosts ", "[OK]", x)
prog.update()
for i in range(x):
sleep(0.1)
prog.set_stat(i + 1)
prog.update()
prog.end()
def predict_pattern(test_val_flag, aton_iteration, execute_file, execute_notes,
num_frames):
read_file_loader(execute_file, 1) # Load test/val set
read_annotation(execute_notes, 1) if test_val_flag == 0 else None
test_dataset = Dataset('dataset/test', True,
False if test_val_flag == 0 else True,
num_frames) # Create Test/Unlabel dataset
model_dir = os.path.join('models/strong_model',
str(aton_iteration)) # Directory to the models
weights_path = os.path.join(model_dir, 'best_weights.mat')
model_path = os.path.join(model_dir,
'best_model.json') # Path to trained model
model = load_model(model_path)
load_weights(model, weights_path)
scores = []
gt = []
text = 'Executing Strong Model Free Dataset...' if test_val_flag == 0 else 'Predict Pattern Strong Model...'
print(text)
prog = pyprog.ProgressBar("",
"",
total=len(test_dataset),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
for iv in range(len(test_dataset)):
inputs = test_dataset[iv]
predictions = model.predict_on_batch(
np.array(inputs[0:1] if test_val_flag == 0 else inputs)
) # Get anomaly prediction for each of 16 frame segment.
gt.append(inputs[1]) if test_val_flag == 0 else None
scores.append(predictions[0][0])
prog.set_stat(iv)
prog.update()
prog.end()
AUC = roc_auc_score(np.array(gt),
np.array(scores)) if test_val_flag == 0 else None
print(AUC) if test_val_flag == 0 else None
return (np.array(gt) if test_val_flag == 0 else None), (np.array(scores))
def __init__(self, total, bar_length, complete_symbol):
self.__state = 0
self.__prog = pyprog.ProgressBar(
" ",
" ",
total=total,
bar_length=bar_length,
complete_symbol=complete_symbol,
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
def execute_free(aton_iteration, free_file, features, pred_gap):
read_file_loader(free_file, 0)
model_dir = os.path.join(
'models/weak_model', str(aton_iteration)
) # Model_dir is the folder where we have placed our trained weights
weights_path = os.path.join(model_dir, 'best_weights.mat')
model_path = os.path.join(model_dir,
'best_model.json') # Path to trained model
model = load_model(model_path)
load_weights(model, weights_path)
print('Executing Weak Model Free Dataset...')
prog = pyprog.ProgressBar("",
"",
total=len(files_train),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
scores = []
for i, video in enumerate(files_train):
inputs = load_dataset_One_Video_Features(
video[0]) # 32 segments features for one testing video
predictions = model.predict_on_batch(
inputs) # Get anomaly prediction for each of 32 video segments.
preds_video = np.concatenate(
([[preds[0]] * pred_gap for preds in predictions]),
axis=0) # Reshape 32 segments to 480 frames
reshape_preds = preds_video.reshape(-1, features).mean(
axis=1
) # Reshape 480 to 30, minimum multiple common between 30 and 32 é 15
scores = np.concatenate(
(scores,
reshape_preds), axis=0) if len(scores) > 0 else reshape_preds
prog.set_stat(i + 1)
prog.update()
prog.end()
return scores
def old_dictionaryOffoundPMID(pmIDList,
pathURL='/home/gvashisth/thesis/processed/'):
# '/media/gaurav/Elements/Thesis/data/MMO/CompleteData/files/processed/'): #
"""
:param pmIDList:
:param pathURL:
:return: A dictionary containing all the pmID found in the pmIDList, with key as pmID
"""
mylogger.info("finding the values for all the PMID in pmIDList")
prog = pyprog.ProgressBar("",
" Done",
len(pmIDList),
complete_symbol="█",
not_complete_symbol="-")
prog.update()
count = 0
foundPMIDDict = dict()
for dicts in cf.getAllfiles(pathURL):
# bar.next()
if (len(foundPMIDDict) != len(pmIDList)):
try:
# bar.next()
with open(pathURL + dicts, 'rb') as fin:
dictipickle = pickle.load(fin)
for ids in pmIDList:
if ids in dictipickle.keys():
count = count + 1
# print('%d : percent done',int(totale_element/count))
prog.set_stat(count)
prog.update()
# print('\n')
if ids not in foundPMIDDict.keys():
foundPMIDDict[ids] = dictipickle[ids]
except Exception as e:
mylogger.error('dictionaryOffoundPMID| dicts: %s| %s' %
(dicts, e))
else:
prog.end()
break
return foundPMIDDict
def execute(path_loader, model_dir, frames_per_feature, output):
read_file_loader(path_loader)
test_dataset = Dataset() # Create Test dataset
weights_path = os.path.join(model_dir, 'best_weights.mat')
model_path = os.path.join(model_dir,
'best_model.json') # Path to trained model
model = load_model(model_path)
load_weights(model, weights_path)
scores = []
print('Predicting...')
prog = pyprog.ProgressBar("",
"",
total=len(test_dataset),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
for iv in range(len(test_dataset)):
inputs = [test_dataset[iv]]
predictions = model.predict_on_batch(np.array(
inputs)) # Get anomaly prediction for each of 32 video segments.
preds_video = np.concatenate(
([[preds[0]] * frames_per_feature for preds in predictions]),
axis=0)
scores = np.concatenate(
(scores, preds_video), axis=0) if len(scores) > 0 else preds_video
prog.set_stat(iv)
prog.update()
prog.end()
savemat(output[:-4] + '.mat', {'scores': np.array(scores)})
return np.array(scores)
def main():
if len(sys.argv) < 2 or any(map(lambda el: el == '--help', sys.argv[1:])):
print "Program shreds the file passed as first argument. If --delete is also defined, the file will be also get deleted"
print "----------------------------------------------------------------------------------------------------------------"
print "Syntax: $ python erase.py <file-to-delete> --delete(optional) "
return
delete = any(map(lambda el: el == '--delete', sys.argv[1:]))
if os.path.exists(sys.argv[1]):
sbytes = getSize(sys.argv[1])
print "File path: " + os.path.realpath(sys.argv[1])
print "File size: " + str(sbytes) + "bytes"
if delete:
print "\033[31mMarked for DELETION\033[0m"
else:
print "The file doesn't exist! "
return
if not click.confirm('Do you want to continue?', default=False):
print "Quitting..."
return
f = open(str(sys.argv[1]), 'rb+')
print "-----------------------------------------------------------------"
print "Beggin shreding..."
prog = pyprog.ProgressBar(" ", "", sbytes)
prog.update()
for x in range(0, sbytes):
f.seek(x)
prog.set_stat(x + 1)
prog.update()
f.write(b"\x0a")
f.close()
if delete:
print "\nMarked for deletion, deleting..."
print "Finished"
os.remove(sys.argv[1])
else:
print "\nFinished"
def test(test_val_flag, aton_iteration, test_file, test_notes, num_features):
results = 'FINAL/' if test_val_flag else 'VAL/'
read_file_loader(test_file, 1) # Load test/val set
read_annotation(test_notes, 1) # Load test/val annotations
test_dataset = Dataset('dataset/test', True, False,
num_features) # Create test dataset
Results_Path = os.path.join('results/strong', results, str(aton_iteration))
model_dir = os.path.join('models/strong_model',
str(aton_iteration)) # Directory to the models
weights_path = os.path.join(model_dir, 'best_weights.mat')
model_path = os.path.join(model_dir,
'best_model.json') # Path to trained model
if not os.path.exists(Results_Path):
os.makedirs(Results_Path)
model = load_model(model_path)
load_weights(model, weights_path)
scores = []
gt = []
print('Testing Strong Classifier...')
prog = pyprog.ProgressBar("",
"",
total=len(test_dataset),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
for iv in range(len(test_dataset)):
inputs = test_dataset[iv]
predictions = model.predict_on_batch(
np.array(inputs[0:1]
)) # Get anomaly prediction for each of 16 frame segment.
gt.append(inputs[1])
scores.append(predictions[0][0])
prog.set_stat(iv)
prog.update()
prog.end()
AUC = roc_auc_score(np.array(gt), np.array(scores))
fpr, tpr, thresholds = roc_curve(np.array(gt), np.array(scores))
print(AUC)
savemat(
os.path.join(Results_Path, 'eval_AUC_' + str(aton_iteration) + '.mat'),
{
'AUC': AUC,
'X': fpr,
'Y': tpr,
'scores': scores,
'gt': gt
})
return np.array(gt), np.array(scores)
def train(path_loader, min_iterations, aton_iteration, val_file, val_notes,
num_features, batchsize, save_best):
read_file_loader(path_loader, 0) # Load train set
read_file_loader(val_file, 1) # Load val set
read_annotation(val_notes, 1) # Load val annotation
model = create_model()
adagrad = Adagrad(lr=0.01, epsilon=1e-08)
model.compile(loss='binary_crossentropy', optimizer=adagrad)
Results_Path = os.path.join(
'results/strong/VAL',
str(aton_iteration)) # Directory to save val stats in training
output_dir = os.path.join(
'models/strong_model',
str(aton_iteration)) # Directory to save models and checkpoints
model_path = os.path.join(output_dir, 'model.json')
best_model_path = os.path.join(output_dir, 'best_model.json')
best_weights_path = os.path.join(output_dir, 'best_weights.mat')
if not os.path.exists(Results_Path):
os.makedirs(Results_Path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
train_dataset = Dataset('dataset/train', False, False,
num_features) # Create train dataset
val_dataset = Dataset('dataset/val', True, False,
num_features) # Create val dataset
loss_graph = []
full_batch_loss = []
total_iterations = 0
bestAUC = 0
previousAUC = [0]
print('Train dataset: ' + str(len(train_dataset)))
plt.ion()
print('Training Strong Classifier...')
prog = pyprog.ProgressBar("",
" AUC - " + str(round(previousAUC[-1], 4)) + '%',
total=min_iterations,
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
while total_iterations != min_iterations:
inputs, targets, stats = load_dataset_train_batch(
train_dataset,
batchsize) # Load normal and abnormal video C3D features
batch_loss = model.train_on_batch(inputs, targets)
full_batch_loss.append(float(batch_loss))
statistics.stats_batch(full_batch_loss, aton_iteration)
loss_graph = np.hstack((loss_graph, batch_loss))
total_iterations += 1
if total_iterations % 20 == 0:
iteration_path = output_dir + 'Iterations_graph_' + str(
total_iterations) + '.mat'
savemat(iteration_path,
dict(loss_graph=loss_graph)) # Loss checkpoint
previousAUC.append(
auc(model, val_dataset, total_iterations,
aton_iteration)) # Validation results
if previousAUC[-1] > bestAUC and save_best:
save_model(model, best_model_path,
best_weights_path) # Best model checkpoint
bestAUC = previousAUC[-1]
weights_path = output_dir + 'weightsStrong_' + str(
total_iterations) + '.mat'
save_model(model, model_path, weights_path) # Model checkpoint
prog.set_suffix(" AUC - " + str(round(previousAUC[-1], 4)) +
'% | Best AUC - ' + str(round(bestAUC, 4)) + '%')
prog.set_stat(total_iterations)
prog.update()
prog.end()
plt.ioff()
save_model(
model, best_model_path, best_weights_path
) if not save_best else None # Save last as best if the best was not kept
def demo(path_C3D, path_frames, model_dir, frames_per_feature, fps, output,
norm_file):
global minmax
with open(norm_file, encoding='utf-8') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
minmax.append(np.array(row).astype(float))
result = execute(path_C3D, model_dir, frames_per_feature, output)
left, width = 0.15, 0.75
height_1, height_2 = 0.5, 0.35
bottom_1, bottom_2 = 0.13 + height_2, 0.1
rect_1 = [left, bottom_1, width, height_1]
rect_2 = [left, bottom_2, width, height_2]
fig = plt.figure()
axs0 = plt.axes(rect_1)
axs0.set_yticks([])
axs0.set_xticks([])
axs1 = plt.axes(rect_2)
axs1.set_ylim(-0.05, 1.05)
axs1.grid(True)
axs1.set_ylabel('Prediction')
axs1.set_xlabel('Frame')
impng = []
show_result = []
y = []
last_variance = []
print('Plotting demo...')
prog = pyprog.ProgressBar("",
"",
total=len(result),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
frames = [
os.path.join(path_frames, frame)
for frame in humanSort(os.listdir(path_frames))
]
for i in range(len(result)):
im1 = axs0.imshow(plt.imread(frames[i]), animated=True, aspect='auto')
variance = [im1]
if i % 16 == 0:
show_result.append(result[i])
y.append(i)
last_variance = []
tmp, = axs1.plot(y,
show_result,
color='black',
linewidth=2,
alpha=0.7)
last_variance.append(tmp)
impng.append(variance + last_variance)
prog.set_stat(i)
prog.update()
prog.end()
print('Writing video demo...')
# Set up formatting for the movie files
Writer = animation.writers['ffmpeg']
writer = Writer(fps=fps, metadata=dict(artist='Me'),
bitrate=4000) # Set bitrate higher if needed
ani1 = animation.ArtistAnimation(fig, impng, interval=fps)
ani1.save(output, writer=writer)
plt.show()
async def check_progress_for_dl(aria2, gid, event, previous_message):
# g_id = event.reply_to_message.from_user.id
try:
file = aria2.get_download(gid)
complete = file.is_complete
is_file = file.seeder
if not complete:
if not file.error_message:
msg = ""
# sometimes, this weird https://t.me/c/1220993104/392975
# error creeps up
# TODO: temporary workaround
downloading_dir_name = "N/A"
try:
# another derp -_-
# https://t.me/c/1220993104/423318
downloading_dir_name = str(file.name)
except:
pass
#
prog = pyprog.ProgressBar(" ", " ", total=100, bar_length=15, complete_symbol="●", not_complete_symbol="○", wrap_bar_prefix=" 〖", wrap_bar_suffix="〗 ", progress_explain="", progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
old_stdout = sys.stdout
new_stdout = io.StringIO()
sys.stdout = new_stdout
p = file.progress_string()
l = len(p)
p=p[0:l-1]
a = float(p)
prog.set_stat(a)
prog.update()
output = new_stdout.getvalue()
sys.stdout = old_stdout
prg = output[3:len(output)]
i = 0
i = int(i)
STR = int(os.environ.get("STR", 30))
msg = f"╭──────── ⌊ 📥 <b>Downloading</b> ⌉ \n"
msg += "│"+"\n├"+f"{prg}\n" +"│"
msg += f"\n├<b>FileName</b> 📚: "
while(len(downloading_dir_name)>0):
st = downloading_dir_name[0:STR]
if(i==0):
msg += f"{downloading_dir_name[0:STR-15]}"
downloading_dir_name = downloading_dir_name[STR-15:len(downloading_dir_name)]
i = 1
else:
msg += f"\n│{st}"
downloading_dir_name = downloading_dir_name[STR:len(downloading_dir_name)]
msg += f"\n├<b>Speed</b> 🚀 : <code>{file.download_speed_string()} </code>"
msg += f"\n├<b>Total Size</b> 🗂 : <code>{file.total_length_string()}</code>"
if is_file is None :
msg += f"\n├<b>Connections</b> 📬 : <code>{file.connections}</code>"
else :
msg += f"\n├<b>Info</b> 📄 : <code>[ P : {file.connections} || S : {file.num_seeders} ]</code>"
# msg += f"\n<b>Status</b> : <code>{file.status}</code>"
msg += f"\n├<b>ETA</b> ⏳ : <code>{file.eta_string()}</code>" +"\n│"
msg += "\n╰─── ⌊ ⚡️ using engine aria2 ⌉"
inline_keyboard = []
ikeyboard = []
ikeyboard.append(
InlineKeyboardButton(
"Cancel 🚫", callback_data=(f"cancel {gid}").encode("UTF-8")
)
)
inline_keyboard.append(ikeyboard)
reply_markup = InlineKeyboardMarkup(inline_keyboard)
if msg != previous_message:
if not file.has_failed:
try:
await event.edit(msg, reply_markup=reply_markup)
except FloodWait as e_e:
LOGGER.warning(f"Trying to sleep for {e_e}")
time.sleep(e_e.x)
except MessageNotModified as e_p:
LOGGER.info(e_p)
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
previous_message = msg
else:
LOGGER.info(
f"Cancelling downloading of {file.name} may be due to slow torrent"
)
await event.edit(
f"Download cancelled :\n<code>{file.name}</code>\n\n #MetaDataError"
)
file.remove(force=True, files=True)
return False
else:
msg = file.error_message
LOGGER.info(msg)
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await event.edit(f"`{msg}`")
return False
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await check_progress_for_dl(aria2, gid, event, previous_message)
else:
LOGGER.info(
f"Downloaded Successfully: `{file.name} ({file.total_length_string()})` 🤒"
)
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await event.edit(
f"Downloaded Successfully: `{file.name} ({file.total_length_string()})` 🤒"
)
return True
except aria2p.client.ClientException:
await event.edit(
f"Download cancelled :\n<code>{file.name} ({file.total_length_string()})</code>"
)
except MessageNotModified as ep:
LOGGER.info(ep)
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await check_progress_for_dl(aria2, gid, event, previous_message)
except FloodWait as e:
LOGGER.info(e)
time.sleep(e.x)
except RecursionError:
file.remove(force=True, files=True)
await event.edit(
"Download Auto Canceled :\n\n"
"Your Torrent/Link is Dead.".format(file.name)
)
return False
except Exception as e:
LOGGER.info(str(e))
if "not found" in str(e) or "'file'" in str(e):
await event.edit(
f"Download cancelled :\n<code>{file.name} ({file.total_length_string()})</code>"
)
return False
else:
LOGGER.info(str(e))
await event.edit(
"<u>error</u> :\n<code>{}</code> \n\n#error".format(str(e))
)
return False
user.default_profile_image, quote_avg, retweet_avg, reply_avg,
favorite_avg, user_mentions_avg, symbols_avg, urls_avg, hashtags_avg,
media_avg, avg_num_tweets, tweet_regularity)
return ret_obj
client = MongoClient(
"mongodb+srv://twitter:[email protected]/test")
db = client.twitterdb
with open('real_users.txt', 'r') as bots:
count = 0
bar = pyprog.ProgressBar("\t\t",
" ",
total=2001,
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog = pyprog.ProgressIndicatorFraction(" ", " ", 2001)
bar.update()
prog.update()
for i, line in enumerate(bots.readlines()):
count += 1
bar.set_stat(i + 1)
bar.update()
prog.set_stat(i + 1)
prog.update()
# user = twitter.get_user("MogleTanner")
try:
def test(test_val_flag, aton_iteration, test_file, test_notes, pred_gap):
results = 'FINAL' if test_val_flag else 'VAL'
read_file_loader(test_file, 1) # Load test/val set
read_annotation(test_notes, 1) # Load test/val annotations
Results_Path = os.path.join('results/weak', results, str(aton_iteration))
model_dir = os.path.join('models/weak_model',
str(aton_iteration)) # Directory to the models
weights_path = os.path.join(model_dir, 'best_weights.mat')
model_path = os.path.join(model_dir,
'best_model.json') # Path to trained model
if not os.path.exists(Results_Path):
os.makedirs(Results_Path)
model = load_model(model_path)
load_weights(model, weights_path)
print('Testing Weak Classifier...')
prog = pyprog.ProgressBar("",
"",
total=len(files_test),
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
scores = []
for i, (video, target) in enumerate(files_test):
inputs = load_dataset_One_Video_Features(
video) # 32 segment features for one testing video
predictions = model.predict_on_batch(
inputs) # Get anomaly prediction for each of 32 video segments.
preds_video = np.concatenate(
([[preds[0]] * pred_gap for preds in predictions]),
axis=0) # Reshape 32 segments to 480 frames
scores = np.concatenate(
(scores, preds_video), axis=0) if len(scores) > 0 else preds_video
prog.set_stat(i + 1)
prog.update()
prog.end()
AUC = roc_auc_score(np.array(notes_test), np.array(scores))
fpr, tpr, thresholds = roc_curve(np.array(notes_test), np.array(scores))
print(AUC)
savemat(
os.path.join(Results_Path, 'eval_AUC_' + str(aton_iteration) + '.mat'),
{
'AUC': AUC,
'X': fpr,
'Y': tpr,
'scores': scores,
'gt': notes_test
})
return np.array(notes_test), np.array(scores)
def train(path_loader, min_iterations, aton_iteration, val_file, val_notes,
batchsize, tot_segments, pred_gap, save_best):
global epoch, batch_obj, segmt_obj
read_file_loader(path_loader, 0) # Load train set
read_file_loader(val_file, 1) # Load val set
read_annotation(val_notes, 1) # Load val annotation
epoch = tf.Variable(
0, dtype=tf.float32
) # Variable to control the weights in the loss function
batch_obj = batchsize # Variable of the loss function
segmt_obj = tot_segments # Variable of the loss function
model = create_model()
adagrad = Adagrad(lr=0.01, epsilon=1e-08)
model.compile(loss=custom_objective, optimizer=adagrad)
Results_Path = os.path.join(
'results/weak/VAL',
str(aton_iteration)) # Directory to save val stats in training
output_dir = os.path.join(
'models/weak_model',
str(aton_iteration)) # Directory to save models and checkpoints
model_path = os.path.join(output_dir, 'model.json')
best_model_path = os.path.join(output_dir, 'best_model.json')
best_weights_path = os.path.join(output_dir, 'best_weights.mat')
if not os.path.exists(Results_Path):
os.makedirs(Results_Path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
abnormalPath = [file[0] for file in files_train if int(file[1]) == 1]
normalPath = [file[0] for file in files_train if int(file[1]) == 0]
loss_graph = []
full_batch_loss = []
total_iterations = 0
bestAUC = 0
previousAUC = [0]
plt.ion()
print('Training Weak Classifier...')
prog = pyprog.ProgressBar("",
" AUC - " + str(round(previousAUC[-1], 4)) + '%',
total=min_iterations,
bar_length=50,
complete_symbol="=",
not_complete_symbol=" ",
wrap_bar_prefix=" [",
wrap_bar_suffix="] ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
prog.update()
while total_iterations != min_iterations:
inputs, targets = load_dataset_Train_batch(
abnormalPath, normalPath, batchsize, tot_segments
) # Load normal and abnormal bags with fixed temporal segments of C3D features
batch_loss = model.train_on_batch(inputs, targets)
full_batch_loss.append(float(batch_loss))
statistics.stats_batch(full_batch_loss, aton_iteration)
loss_graph = np.hstack((loss_graph, batch_loss))
if total_iterations % 20 == 0:
iteration_path = output_dir + 'Iterations_graph_' + str(
total_iterations) + '.mat'
savemat(iteration_path,
dict(loss_graph=loss_graph)) # Loss checkpoint
previousAUC.append(
auc(model, total_iterations, aton_iteration,
pred_gap)) # Validation results
if previousAUC[-1] > bestAUC and save_best: # Best model checkpoint
bestAUC = previousAUC[-1]
save_model(model, best_model_path, best_weights_path)
weights_path = output_dir + 'weightsWeak_' + str(
total_iterations) + '.mat'
save_model(model, model_path, weights_path) # Model checkpoint
prog.set_suffix(" AUC - " + str(round(previousAUC[-1], 4)) +
'% | Best AUC - ' + str(round(bestAUC, 4)) + '%')
total_iterations += 1
epoch.assign_add(tf.Variable(1,
dtype=tf.float32)) # Update loss variable
prog.set_stat(total_iterations)
prog.update()
prog.end()
plt.ioff()
save_model(
model, best_model_path, best_weights_path
) if not save_best else None # Save last as best if the best was not kept
def search_mhweb_offline(search_string):
'''
Function:
Input crash_string
Output: set of TR ID
'''
search_string = create_python_regex_string(search_string)
print("Regex:", search_string)
#print ("-"*30)
start = timer()
#search trong 1 file nho, de test cho nhanh
#mhwebfilepath = 'data.txt'
# dua vao dong nay de biet dang search file nao, vi co 2 file
print(">>> searching", mhwebfilepath, "...")
thefile = open(mhwebfilepath, encoding="ISO-8859-1")
result_found = 0
#dung cach nay cho ket qua tuc thi, tiet kiem bo nho
#bien nay de luu vi tri line chua ket qua tim
result_lines = []
for lineno, line in enumerate(thefile):
#m = re.search(search_string, line.strip(), re.IGNORECASE)
#print de biet toi line nao
if lineno % 6800000 == 0:
#cho nay se tim cach fix sau
total_line = 37463606
print('File Searching Progress:', int(100 * (lineno / total_line)),
'%')
m = re.match(search_string, line.strip())
if m:
result_found += 1
result_lines.append(lineno)
#turning log
if result_found >= 30:
thefile.close()
break
#find TR_ID
print(">>> Matching log found, start matching TR number...")
tr_id_results = set()
if len(result_lines) == 0:
print(">>> No matching TR found.")
else:
#update trang thai cho no dep mat
prog = pyprog.ProgressBar(" ", "", len(result_lines))
for i, x in enumerate(result_lines):
#print ("processing...", x)
prog.set_stat(i + 1)
prog.update()
while x > 1:
#check_line= get_line(x-1, mhwebfilepath)
#mot lan check 100 line, check line from x - 1 to x-101
check_lines = get_line(x - 1, mhwebfilepath)
found_result = False
for item in check_lines.keys():
pat = r'<value>([a-zA-Z]{2}[0-9]{5})</value></column>'
m = re.match(pat, check_lines[item])
if m:
#no need to print this, only print progress update bar
#print (x, check_lines[item].strip(), m.group(1))
tr_id_results.add(m.group(1))
found_result = True
#tim co ket qua thi thoat ngay
break
if found_result:
break
x = x - 1
print('-' * 30)
print(">>> TR match found:", tr_id_results)
print('-' * 30)
end = timer()
duration = end - start
print(">>> Duration", duration)
return tr_id_results
def dictionaryOffoundPMID(
pmdID_year_list,
pathURL='/home/gvashisth/thesis/correct_MMO/0/'
): #'/media/gaurav/Seagate Expansion Drive/Gaurav/MMO_final/'): #
# '/media/gaurav/Elements/Thesis/data/MMO/CompleteData/files/processed/'): #
"""
this func takes a list of (pmDLISt, year_list publish) and then for for each year it find the corresponding year interval from medlineDB_index
:param pmIDList: list of pmID in a file
:param pathURL: path of the processed MMO .bin files
:return: A dictionary containing all the pmID found in the pmIDList, with key as pmID
"""
pmIDList, year_list = zip(*pmdID_year_list)
mylogger.info("finding the values for all the PMID in pmIDList")
prog = pyprog.ProgressBar("",
" Done",
len(pmIDList),
complete_symbol="█",
not_complete_symbol="-")
prog.update()
medlineDB_index = cf.loadDictionary('medlineDB_index.bin')
count = 0
foundPMIDDict = dict()
for yr in sorted(set(year_list)):
# print(yr)
for lower_year_limit in list(medlineDB_index.keys()):
# print(type(lower_year_limit))
if int(yr) >= lower_year_limit:
uppr_year_limit_list = medlineDB_index[lower_year_limit].keys()
for year_limit in list(uppr_year_limit_list):
if int(yr) <= year_limit:
# print(year_limit)
# print(medlineDB_index[lower_year_limit][year_limit])
for dicts in medlineDB_index[lower_year_limit][
year_limit]:
# bar.next()
if (len(foundPMIDDict) != len(pmIDList)):
try:
# bar.next()
with open(pathURL + dicts, 'rb') as fin:
dictipickle = pickle.load(fin)
for ids in pmIDList:
if ids in dictipickle.keys():
# print('\n')
if ids not in foundPMIDDict.keys(
):
foundPMIDDict[
ids] = dictipickle[ids]
count = count + 1
# print('%d : percent done',int(totale_element/count))
prog.set_stat(count)
prog.update()
except Exception as e:
mylogger.error(
'dictionaryOffoundPMID| dicts: %s| %s'
% (dicts, e))
else:
prog.end()
break
return foundPMIDDict
async def check_progress_for_dl(aria2, gid, event, previous_message):
try:
file = aria2.get_download(gid)
complete = file.is_complete
is_file = file.seeder
if not complete:
if not file.error_message:
msg = ""
# sometimes, this weird https://t.me/c/1220993104/392975
# error creeps up
# TODO: temporary workaround
downloading_dir_name = "N/A"
try:
# another derp -_-
# https://t.me/c/1220993104/423318
downloading_dir_name = str(file.name)
except:
pass
prog = pyprog.ProgressBar(
" ",
" ",
total=100,
bar_length=15,
complete_symbol="●",
not_complete_symbol="○",
wrap_bar_prefix=" 〖",
wrap_bar_suffix="〗 ",
progress_explain="",
progress_loc=pyprog.ProgressBar.PROGRESS_LOC_END)
old_stdout = sys.stdout
new_stdout = io.StringIO()
sys.stdout = new_stdout
p = file.progress_string()
l = len(p)
p = p[0:l - 1]
a = float(p)
prog.set_stat(a)
prog.update()
output = new_stdout.getvalue()
sys.stdout = old_stdout
prg = output[3:len(output)]
i = 0
i = int(i)
STR = int(os.environ.get("STR", 30))
msg = f"<b>╭──────── ⌊ 📥 Downloading ⌉ /b>\n<b>│</b>\n<b>├ {prog} {file.progress_string()}</b>"
msg += f"\n<b>┝ File Name </b> : "
while (len(downloading_dir_name) > 0):
st = downloading_dir_name[0:STR]
if (i == 0):
msg += f"{downloading_dir_name[0:STR-15]}"
downloading_dir_name = downloading_dir_name[
STR - 15:len(downloading_dir_name)]
i = 1
else:
msg += f"\n│{st}"
downloading_dir_name = downloading_dir_name[
STR:len(downloading_dir_name)]
# msg += f"<b>╭──────── ⌊ 📥 Downloading ⌉ </b>\n<b>│</b>\n<b>├ Downloaded : {file.progress_string()}</b>\n<b>│</b>\n<b>├ Name :</b> <code>{downloading_dir_name}</code>"
msg += f"\n<b>┝ Speed : {file.download_speed_string()} </b>"
msg += f"\n<b>┝ Total Size : {file.total_length_string()}</b>"
if is_file is None:
msg += f"\n<b>┝ Connections : {file.connections}</b>"
else:
msg += f"\n<b>┝ Info : [ P :- {file.connections} || S : {file.num_seeders} ]</b>"
msg += f"\n<b>┝ Status : {file.status}</b>"
msg += f"\n<b>┝ ETA : {file.eta_string()}</b>"
msg += f"\n<b>│</b>\n<b>╰── ⌊ 🌟 @TeluguMoviesDL 🌟 ⌉</b>"
inline_keyboard = []
ikeyboard = []
ikeyboard.append(
InlineKeyboardButton(
"✘ Cancel ✘",
callback_data=(f"cancel {gid}").encode("UTF-8")))
inline_keyboard.append(ikeyboard)
reply_markup = InlineKeyboardMarkup(inline_keyboard)
#msg += reply_markup
LOGGER.info(msg)
if msg != previous_message:
await event.edit(msg, reply_markup=reply_markup)
previous_message = msg
else:
msg = file.error_message
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await event.edit(f"`{msg}`")
return False
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await check_progress_for_dl(aria2, gid, event, previous_message)
else:
await asyncio.sleep(EDIT_SLEEP_TIME_OUT)
await event.edit(
f"<b>File Downloaded Successfully ✅</b>\n\n<b>File Name :</b> `{file.name}` 🤒"
)
return True
except aria2p.client.ClientException:
pass
except MessageNotModified:
pass
except RecursionError:
file.remove(force=True)
await event.edit("Download Auto Canceled :\n\n"
"Your Torrent/Link is Dead.".format(file.name))
return False
except Exception as e:
LOGGER.info(str(e))
if " not found" in str(e) or "'file'" in str(e):
await event.edit("Download Canceled :\n<code>{}</code>".format(
file.name))
return False
else:
LOGGER.info(str(e))
await event.edit(
"<u>error</u> :\n<code>{}</code> \n\n#error".format(str(e)))
return False
def new_crash_check(pandas_dataset):
'''
crash_type = 'du' or 'ru'
'''
#filename = 'ENDC_DU_Crash_Alarm_PKG_Daily_Report_20200422.xlsx'
print ("-"*40)
print (version)
#print_logo(filename)
#parse trmapping.txt and get trmapping dict
tr_dict = get_tr_mapping_dict()[0]
tr_mapping_version = get_tr_mapping_dict()[1]
#in version mau do cho de nhin, bien nay thay doi hang ngay, can remark cho ro
print("TR Mapping Version: ", colored(tr_mapping_version,'red'))
list_known_sig = tr_dict.keys()
sw_filter_list = set([
'CXP9024418/12_R55B26', 'CXP9024418/12_R57C74', 'CXP9024418/12_R60B28' , 'CXP9024418/12_R62B21', 'CXP9024418/12_R62B41' ,'CXP9024418/12_R62B45' ,
'CXP9024418/6_R80F30' , 'CXP9024418/6_R80F51',
'CXP9024418/6_R85C59(gNB)' , 'CXP9024418/6_R85C103(gNB)' , 'CXP9024418/15_R4B28' , 'CXP9024418/15_R5B21'
])
#parsing swmapping.txt to get sw_dict
sw_dict = get_sw_mapping_dict()
#print (sw_dict)
#will remove later
#dataset = pandas.read_csv(filename,encoding = "ISO-8859-1",low_memory=False)
dataset = pandas_dataset
#data = pandas.read_csv(myfile, encoding='utf-8', quotechar='"', delimiter=',')
print("Data dimention:", dataset.shape)
print("First 5 row and Last 5 ROW:")
print(dataset)
#parsing csv sheet
crash_details = dataset['Crash Details']
#because header of DU sheet and RU sheet if different, so that have to use this if to check
if "TRMapping" in dataset.columns:
tr_mappings = dataset['TRMapping']
if "TR Mapping" in dataset.columns:
tr_mappings = dataset['TR Mapping']
upgrade_pkg = dataset['UP']
sites_list = dataset['Site Name']
if 'Date Time(KST)' in dataset.columns
crash_daytimes = dataset['Date Time(KST)']
else
crash_daytimes = dataset['KST Time']
len1 = len(crash_details)
#chi tap trung xu ly new crash
crash_detail = ""
signatures = []
count_new = 0
for i in range(len1):
crash_detail = crash_details[i]
#dung check_new_crash(crash_details[i]) de check crash theo file trmapping.txt
if "New Crash" in tr_mappings[i] and check_new_crash(crash_detail):
count_new += 1
##check lai thang ong noi nay
#signature = extract_crash_signature(crash_detail)
signature = extract_crash_signature(crash_detail, True)
signatures.append(signature)
dict_signatures = removeduplicatedandcount(signatures)
#20 la toi uu, can tim cac ket qua thap hon 10 nua
max_sign_check = 20
#chi xu ly top 30 new crash
print ('*'*20)
#print ("TOP 30 CRASH:")
j = 0
#https://stackoverflow.com/questions/10865483/print-results-in-mysql-format-with-python
x = PrettyTable(["No.", "Crash signature", "Count","Site", "Latest Crash PKG" , "Last crash time", "Last Crash Node" ])
x.align["Crash signature"] = "l"
x.align["Latest Crash PKG"] = "l"
x.align["SW"] = "l"
#filtered table
x2 = PrettyTable(["No.", "Crash signature", "Crash", "Site","Latest Crash PKG" , "Last crash time KST", "Last Crash Node", "Match TR" ])
x2.align["Crash signature"] = "l"
x2.align["Latest Crash PKG"] = "l"
x2.align["SW"] = "l"
#filtered table, 2
x3 = PrettyTable(["No.", "Crash signature", "Crash", "Site","Latest Crash PKG" , "Last crash time KST", "Last Crash Node", "Match TR" ])
x3.align["Crash signature"] = "l"
x3.align["Latest Crash PKG"] = "l"
x3.align["SW"] = "l"
filtered_crashs = []
# Create Object
prog = pyprog.ProgressBar(" ", "", max_sign_check)
#print ("Parsing data ...[No of Site/Crash, SW PKG, Matching TR...]")
#start to count time
start = timer()
parsing_result_list = []
print ("Total signature found:", len(dict_signatures) )
print ("Parsing data ...[No of Site/Crash, SW PKG, Matching TR...]")
prog.update()
for i in range(max_sign_check):
sign = dict_signatures[i][0]
count_sign = dict_signatures[i][1]
sw_list = set()
temp_site_list = []
last_crash_node_day = find_last_crash_day(sign, crash_details, crash_daytimes, sites_list,tr_mappings)
last_crash_day = last_crash_node_day[0]
last_crash_node = last_crash_node_day[1]
delta_day = now - last_crash_day
delta_day = delta_day.days
#find sw list of sign
for i1 in range(len1):
if sign in crash_details[i1]:
if upgrade_pkg[i1] not in sw_list and "New Crash" in tr_mappings[i1] and check_new_crash(crash_details[i1]) and not isinstance(upgrade_pkg[i1], float) :
sw_list.add(upgrade_pkg[i1])
if(sites_list[i1] not in temp_site_list) and "New Crash" in tr_mappings[i1]:
temp_site_list.append(sites_list[i1])
count_sites = len(temp_site_list)
#tim tap hop sw moi mat
last_sw_list = find_newest_pkg(sw_list)
#chuyen doi package string to COMMON PKG ID like MTR20.05
last_sw_list_ID = []
for item in last_sw_list:
last_sw_list_ID.append(lookup_pkg_id(item,sw_dict))
if sign != "" and sign != "Extra: terminated by signal 1":
parsing_result_list.append((i, sign, count_sign,count_sites, sw_list, last_crash_day, last_crash_node))
x.add_row([i, sign[:40], count_sign, count_sites , last_sw_list_ID,last_crash_day,last_crash_node])
# Set current status
prog.set_stat(i + 1)
# Update Progress Bar
prog.update()
#filter crash
if sign != "" and sign != "Extra: terminated by signal 1" and 'CMC supervision.nIllegal access by port' not in sign and count_sign >= 10 and count_sites >= 2 and delta_day <= 3 and check_if_crash_in_new_sw(last_sw_list, sw_filter_list):
#find matching TR ID
print ("\n>>> Matching TR ID for new crash signature:", colored(sign, 'red'))
#chay cham qua, tam thoi turn off
match_tr = skt_5g_cd_tool.search_mhweb_offline(sign)
#de print cho dep, khong phai print set()
if len(match_tr) == 0 : match_tr = ""
##test filter crash with pandas
#last_crash_time_kst_pd, last_crash_node_pd, last_crash_detail_pd, last_crash_pkg_pd = parsing_excel_by_pandas.filter_last_crash(sign,pandas_dataset)
#x3.add_row([i, sign[:30],count_sign,count_sites, last_crash_pkg_pd,last_crash_time_kst_pd,last_crash_node_pd, match_tr])
#dung PKG ID trong filter table, cho no ngan gon
x2.add_row([i, sign[:30],count_sign,count_sites, last_sw_list_ID,last_crash_day,last_crash_node, match_tr])
#doi voi detail crash, thi chi can sign full, sw full, con lai ko can, cho don gian
filtered_crashs.append([i, sign, "No. Crash: "+str(count_sign), "No. Site: "+str(count_sites), last_sw_list])
print ("\n")
print ("TOP 30 CRASH:")
for item in parsing_result_list:
#(i, sign, count_sign,count_sites, sw_list, last_crash_day, last_crash_node)
#color print
print (item[0],"|", colored(item[1], 'green'),"|", item[2],"|", colored(item[3], 'red'), "|", colored(item[4], 'cyan'), "|", colored(item[5], 'yellow'), "|", item[6])
#colored(item[1], 'cyan')
print ("SUMMARY TABLE:")
print(x)
print ("FILTER CRITERIA:")
print ('1.', "CRASH COUNT >= 10, COUNT SITE >= 2, LAST CRASH DAY <= 3 DAYS AGO")
print('2.', 'Filter PKG')
for pkg in sw_filter_list:
pkg_id = lookup_pkg_id(pkg,sw_dict)
print(f'{pkg:<25} {pkg_id:<20} ')
print("\n\n")
print('*'*15, "FILTERD CRASH", '*'*15)
#print ("Now: ", now)
print(x2)
print ("\n>>> Detail of crash signature:\n")
for item in filtered_crashs:
print (item)
end = timer()
print("\n>>> Duration ", end - start)