def alarm():
sql = "select room,hostname,ip,cluster,aralm,date,count(counts) from openfalcon_roominfo group by room;"
room = "select name from openfalcon_roomname where name not in(select room from openfalcon_roominfo);"
sqlinfo = db.select_count_table(sql)
room_name = db.select_room_table(room)
open('./json_alarm', 'w')
json = open('./json_alarm', 'a')
json.write('[')
for row in sqlinfo:
row1 = ''.join(row[0]).encode('utf-8')
row7 = ''.join(str(row[6]))
json.write("{")
json.write('"name":' + '"' + row1.strip() + '",')
json.write('"value":' + '"' + row7 + '"')
json.write("},")
for name in room_name:
name_row = ''.join(name).encode('utf-8')
json.write("{")
json.write('"name":' + '"' + name_row.strip() + '",')
json.write('"value":' + '"0"')
json.write("},")
json.write("]")
json.close()
'''得到最终结果'''
info = open('./json_alarm', 'r')
rows = info.readlines()
rows_info = ''.join(rows)
result = rows_info.replace(',]', ']')
#print result
rs = open('/systemfalcon/systemfalcon/static/alarm/result_alarm', 'w')
rs.write(result)
rs.close()
async def lgo(client, message):
engine = message.Engine
pablo = await edit_or_reply(message, engine.get_string("PROCESSING"))
if not message.reply_to_message:
await pablo.edit(
engine.get_string("NEEDS_REPLY").format("Animated Sticker"))
return
if not message.reply_to_message.sticker:
await pablo.edit(
engine.get_string("NEEDS_REPLY").format("Animated Sticker"))
return
if message.reply_to_message.sticker.mime_type != "application/x-tgsticker":
await pablo.edit(
engine.get_string("NEEDS_REPLY").format("Animated Sticker"))
return
lol = await message.reply_to_message.download("tgs.tgs")
cmdo = f"lottie_convert.py {lol} json.json"
await runcmd(cmdo)
if not os.path.exists('json.json'):
await pablo.edit(
engine.get_string("NEEDS_REPLY").format("Animated Sticker"))
os.remove("tgs.tgs")
return
json = open("json.json", "r")
jsn = json.read()
json.close()
jsn = (jsn.replace("[1]", "[2]").replace("[2]", "[3]").replace(
"[3]", "[4]").replace("[4]", "[5]").replace("[5]", "[6]"))
open("json.json", "w").write(jsn)
await runcmd(f"lottie_convert.py json.json tgs.tgs")
await client.send_sticker(message.chat.id, "tgs.tgs")
os.remove("json.json")
os.remove(lol)
os.remove("tgs.tgs")
await pablo.delete()
async def lgo(client, message):
pablo = await edit_or_reply(message, "`Processing...`")
if not message.reply_to_message:
await pablo.edit("Reply to A Animated Sticker...")
return
if not message.reply_to_message.sticker:
await pablo.edit("Reply to A Animated Sticker...")
return
if message.reply_to_message.sticker.mime_type != "application/x-tgsticker":
await pablo.edit("`Reply to A Animated Sticker...`")
return
lol = await message.reply_to_message.download("tgs.tgs")
cmdo = f"lottie_convert.py {lol} json.json"
await runcmd(cmdo)
if not os.path.exists('json.json'):
await pablo.edit("`Invalid Tgs Sticker I Suppose.`")
os.remove("tgs.tgs")
return
json = open("json.json", "r")
jsn = json.read()
json.close()
jsn = (
jsn.replace("[1]", "[2]")
.replace("[2]", "[3]")
.replace("[3]", "[4]")
.replace("[4]", "[5]")
.replace("[5]", "[6]")
)
open("json.json", "w").write(jsn)
await runcmd(f"lottie_convert.py json.json tgs.tgs")
await client.send_sticker(message.chat.id, "tgs.tgs")
os.remove("json.json")
os.remove(lol)
os.remove("tgs.tgs")
await pablo.delete()
def alarm_class():
"""告警分类图表"""
sql = "select metric,hostname,ip,cluster,aralm,date,count(counts) from openfalcon_roominfo group by metric;"
sqlinfo = db.select_count_table(sql)
open('./json_class', 'w')
json = open('./json_class', 'a')
json.write('[')
for row in sqlinfo:
row1 = ''.join(row[0]).encode('utf-8')
row7 = ''.join(str(row[6]))
json.write("{")
json.write('"name":' + '"' + row1 + '",')
json.write('"value":' + '"' + row7 + '"')
json.write("},")
json.write("]")
json.close()
'''得到最终结果'''
info = open('./json_class', 'r')
rows = info.readlines()
rows_info = ''.join(rows)
result = rows_info.replace(',]', ']')
#print result
rs = open('/systemfalcon/systemfalcon/static/alarm/result_class', 'w')
rs.write(result)
rs.close()
def guardar_nivel(ruta, datos):
lista = []
archivo = open(ruta, 'r')
lista = json.load(archivo)
json.close()
lista.append(datos)
archivo = open(ruta, 'w')
json.dump(lista, archivo)
json.close()
def start_crawler():
logging.info("start_crawler")
process_root('../resources/sites.ini')
postings_file = '../data/job_postings_' + datetime.now().strftime(
"%y-%m-%d_%H.%M") + '.json'
json = open(postings_file, 'w+', encoding='utf-8')
json.write('[\n')
process_work_queue(json)
json.write(']')
json.close()
remove_last_comma(postings_file)
copyfile(postings_file, '../data/job_postings.json')
def obtener_listado(ruta, nivel_seleccionado):
archivo_niveles = open(ruta, 'r')
lista_niveles = json.loads(archivo_niveles)
json.close()
if nivel_seleccionado is 'Nivel facil':
dicc_niveles = lista_niveles[
'nivel_facil'] #se guarda en el dicc a los puntajes del modo facil
ventana_puntajes('Nivel facil', dicc_niveles)
if nivel_seleccionado is 'Nivel medio':
dicc_niveles = lista_niveles[
'nivel_medio'] #se guarda en el dicc a los puntajes del modo facil
ventana_puntajes('Nivel medio', dicc_niveles)
if nivel_seleccionado is 'Nivel dificil':
dicc_niveles = lista_niveles[
'nivel_dificil'] #se guarda en el dicc a los puntajes del modo facil
ventana_puntajes('Nivel dificil', dicc_niveles)
def validateConfigJson(file):
if os.path.isfile(file) == False:
print "You did not pass a valid path for the config.json. Exiting \n"
sys.exit(4)
json = open(file, 'r')
config = json.load(json)
if not config['NDK']:
print "NDK field is empty. Exiting \n"
sys.exit(4)
if not config['SDK']:
print "SDK field is empty. Exiting \n"
sys.exit(4)
json.close()
def FileOper(etl, data, type):
path = etl.FilePath;
filetype = path.split('.')[-1].lower();
encode = 'utf-8' if etl.EncodingType == 'UTF8' else 'ascii';
if filetype in ['csv', 'txt']:
import csv
file = open(etl.FilePath, type, encoding=encode);
sp = ',' if filetype == 'csv' else '\t';
if type == 'r':
reader = csv.DictReader(file, delimiter=sp)
for r in reader:
yield r;
else:
writer = csv.DictWriter(file, delimiter=sp)
start = False;
for r in data:
if not start:
field = r.keys;
writer.fieldnames = field;
writer.writerow(dict(zip(field, field)))
start = True;
writer.writer(r)
yield r;
file.close();
elif filetype == 'xlsx':
pass;
elif filetype == 'xml' and type == 'r':
tree = ET.parse(path);
root = tree.getroot();
root = root.findall('Doc');
for etool in root:
p = {r: etool.attrib[r] for r in etool.attrib};
yield p;
elif filetype == 'xml' and type == 'w':
pass;
elif filetype == 'json':
if type == 'r':
items = json.load(open(path, encoding=encode));
for r in items:
yield r;
else:
json.open(path);
for r in data:
json.write(r)
yield r;
json.close()
json.dump([r for r in data], open(path, type, encode));
def read():
# os.system('python join.py hosts')
file = open('./temp', 'r')
file1 = open('./hosts', 'r')
file2 = open('./datetime', 'r')
open('./json', 'w')
json = open('./json', 'a')
json.write('{ "告警":[')
while 1:
json.write("{")
line = file.readline().strip()
line1 = file1.readline().strip()
line2 = file2.readline().strip()
# print '"host":'+'"'+line1.strip(), line.strip(),line2.strip()+'"'
json.write('"room":' + '"' + line1.strip() + line.strip() +
line2.strip() + '"')
json.write("},")
if not line and not line1 and not line2:
break
pass
json.write("]}")
json.close()
def writeToJSON(str, fileName, fileType, outputPath):
json = open(outputPath + fileName + "_" + fileType + ".json", "w")
json.write(str)
json.close()
stock_data = web.DataReader("%s.KS" %st[0],'yahoo',start,end)
if selection == str(2):
print("너냐")
stock_data = web.DataReader("%s.KQ" %st[0],'yahoo',start,end)
print("너냐!")
print("1번마")
stock_data.loc[:,'Company'] = CompanyName
stock_data.loc[:,'StockCode'] = StockCode
stock_data.loc[:,'Date'] = stock_data.index.astype('str')
json = open('DAILY_STOCK_DATA.json','a')
#StockData.append(stock_data)
#json.dumps(stock_data, ensure_ascii=False, sort_keys=False, separators=(',', ':')).encode('utf-8')
# force_ascii=False for korean
print("2번마")
json.write(stock_data.to_json(orient='records',force_ascii=False))
json.close()
print("3번마")
engine = create_engine('mysql+mysqlconnector://ADMIN:ADMIN@localhost:3306/stock', echo=False)
print("4번마")
stock_data.to_sql(name='stock', con=engine, if_exists = 'append', index=False)
print(stock_data)
print("------------------------------------")
except:
print("예외입니다잉")
pass
def remove_last_comma(postings_file):
lines = open(postings_file, 'r+').readlines()
lines[-1] = lines[-1].replace(',', '')
json = open(postings_file, 'w+', encoding='utf-8')
json.writelines(lines)
json.close()
def leer_Nivel(ruta):
archivo = open(ruta, 'r')
lista = json.load(archivo)
json.close()
return lista
def sum():
import wikipedia
from fuzzywuzzy import process
from fuzzywuzzy import fuzz
import copy
import numpy as np
def superImportant(topic):
content = wikipedia.summary(topic)
pageObj = wikipedia.WikipediaPage(topic)
hyperLinks1 = []
nonhyperLinks = content.split()
for words2 in pageObj.links:
if words2 in content:
hyperLinks1.append(words2)
contentWords = list(set(copy.deepcopy(hyperLinks1) + nonhyperLinks))
fails = 0
for words in hyperLinks1:
try:
content = wikipedia.summary(words)
pageObj = wikipedia.WikipediaPage(words)
count = 0
for words2 in pageObj.links:
if words2 in content:
contentWords.append(words2)
count += 1
if count >= 1 * len(hyperLinks1):
break
except:
fails += 1
contentWords = set((" ".join(contentWords)).split())
return contentWords
def changePriorities(dic, mapWords):
frequencies = []
for k, v in dic.items():
frequencies.append(v)
X = copy.deepcopy(np.percentile(np.array(frequencies), 95))
misMatches = 0
for words in mapWords:
if words in dic.keys():
dic[words] = X
elif words.lower() in dic.keys():
dic[words.lower()] = X
else:
misMatches += 1
return dic, misMatches
# JUST RUN THIS CELL
import copy
import nltk
import numpy as np
import string
from sklearn.feature_extraction.text import CountVectorizer
from nltk.corpus import stopwords
from fuzzywuzzy import process
from fuzzywuzzy import fuzz
from scipy import stats
import matplotlib.pyplot as plt
nltk.download('wordnet')
nltk.download('stopwords')
def chunk(sourceFile, wordsPerLine=None, endLineAt=None):
fi = open(sourceFile, "r+")
text = fi.read()
text = text.replace("\n", "")
if wordsPerLine != None:
text = text.split()
for words in range(1, len(text) + 1):
if words % 3 == 0:
text[words - 1] = text[words - 1] + "\n"
fi.seek(0)
fi.write(" ".join(text))
if endLineAt != None:
for words in endLineAt:
text = text.split(words)
text = "\n".join(text)
fi.seek(0)
fi.write(text)
fi.close()
return
def getKey(D, val):
for key, value in D.items():
if val == value:
return key
return -1
def completeFiltering(singleStringTxt,
multiLineTxt,
limitOnFreq,
limitOnDataW=10000):
wholeText = singleStringTxt
cleansed = wholeText.split()[:limitOnDataW]
table = str.maketrans("", "", string.punctuation)
cleansed = [w.translate(table) for w in cleansed]
patched = " ".join(cleansed)
cleansed = patched.split()
cleansed = [
words for words in cleansed
if not words.lower() in stopwords.words()
]
cleansedTxt = " ".join(cleansed)
wholeText = [cleansedTxt]
lineWiseText = multiLineTxt
# list of text documents
# create the transform
vectorizer1 = CountVectorizer()
vectorizer2 = CountVectorizer()
# tokenize and build vocab
vectorizer1.fit(wholeText)
vectorizer2.fit(lineWiseText)
# summarize
wToInd1 = vectorizer1.vocabulary_
wToInd2 = vectorizer2.vocabulary_
# encode document
vector1 = vectorizer1.transform(wholeText)
vector2 = vectorizer2.transform(lineWiseText)
# summarize encoded vector
v1 = vector1.toarray()
v2 = vector2.toarray()
finalCount = np.sum(v1, axis=0, keepdims=False)
countDict1 = dict()
countDict2 = dict()
priorities2 = dict()
for ind in range(len(finalCount)):
if finalCount[ind] >= limitOnFreq:
countDict1[getKey(wToInd1, ind)] = finalCount[ind]
for lines in range(v2.shape[0]):
countDict = dict()
for ind in range(v2.shape[1]):
if v2[lines][ind] >= limitOnFreq:
countDict[getKey(wToInd2, ind)] = v2[lines][ind]
priorities = sorted(countDict, key=countDict.get, reverse=True)
countDict2[str(lines + 1)] = countDict
priorities2[str(lines + 1)] = priorities
contentWords = superImportant("Apple Inc")
countDict1, misMatch = changePriorities(countDict1, contentWords)
print("These many got mismatched : ", misMatch)
priorities1 = sorted(countDict1, key=countDict1.get, reverse=True)
return priorities1, priorities2, countDict1, countDict2
def fuzzyWayCondense(fileSource,
priorities1,
priorities2,
prioritiesMap1,
prioritiesMap2,
limitOnLines=3,
limitOnDataL=100,
method="Frequency",
printLineScores=False):
if method == "Frequency":
priorities = priorities1
prioritiesMap = prioritiesMap1
elif method == "TF-IDF":
priorities = priorities1
prioritiesMap = prioritiesMap1
prioritiesMapext = prioritiesMap2
includeTFIDF = np.zeros((limitOnDataL, len(priorities)))
fi = open(fileSource, "r")
include = np.zeros((limitOnDataL, len(priorities)))
wholeLines = fi.readlines()[:limitOnDataL]
maintain = dict()
for lines in range(1, limitOnDataL + 1):
maintain[str(lines)] = []
fi.close()
for words in priorities:
options = process.extract(words, wholeLines, limit=limitOnDataL)
for line, score in options:
if (words in line.split()) and method == "Frequency":
maintain[str(wholeLines.index(line) + 1)].append(words)
include[wholeLines.index(line)][priorities.index(
words)] = score * prioritiesMap[words]
elif (words in line.split()) and method == "TF-IDF":
maintain[str(wholeLines.index(line) + 1)].append(words)
includeTFIDF[wholeLines.index(line)][priorities.index(words)] = \
prioritiesMapext[str(wholeLines.index(line) + 1)][words] * prioritiesMap[words]
if method == "TF-IDF":
includeTFIDF = list(np.sum(includeTFIDF, axis=0))
for words in priorities:
options = process.extract(words,
wholeLines,
limit=limitOnDataL)
for line, score in options:
if (words in line.split()):
include[wholeLines.index(line)][priorities.index(
words
)] = score * includeTFIDF[priorities.index(words)]
for lines in range(1, limitOnDataL + 1):
maintain[str(lines)] = set(maintain[str(lines)])
include = list(np.sum(include, axis=1))
includeTemp = np.array(copy.deepcopy(include))
if printLineScores == True:
print("\nThe Scores of the Sentences from 1 to", limitOnDataL,
" are as follows \n", include)
print("\nThe Key Words Per Line for all the lines are : \n",
maintain)
condensedLines = []
condensedLinesIndices = []
if limitOnLines != "NormSTDPick":
includeTemp = (np.sort(includeTemp))[::-1]
for i in range(limitOnLines):
condensedLines.append(wholeLines[include.index(
includeTemp[i])])
condensedLinesIndices.append(include.index(includeTemp[i]) + 1)
include[include.index(includeTemp[i])] = -1
else:
includeTemp = np.array([(value >= np.percentile(includeTemp, 50))
for value in includeTemp]).astype(int)
includeTemp = np.reshape(np.argwhere(includeTemp), (-1, )) + 1
condensedLines = [wholeLines[i - 1] for i in includeTemp]
condensedLinesIndices = includeTemp
condensedText = " ".join(condensedLines)
return condensedText, condensedLines, condensedLinesIndices
import json
T1path = "final1.txt" # "/content/drive/My Drive/transcript1"
T2path = "final2.txt" # "/content/drive/My Drive/transcript2"
T3path = "final3.txt" # "/content/drive/My Drive/transcript3"
T4path = "final4.txt" # "/content/drive/My Drive/transcript4"
Tpath = "1.txt" # "/content/drive/My Drive/entireTranscript.txt"
transcript = ""
tpt1 = open(T1path, "r")
T1 = tpt1.read()
T1dict = eval(T1)
tpt1.close()
tpt2 = open(T2path, "r")
T2 = tpt2.read()
T2dict = eval(T2)
tpt2.close()
tpt3 = open(T3path, "r")
T3 = tpt3.read()
T3dict = eval(T3)
# T3dict = json.loads(T3dict.decode("utf-8"))
tpt3.close()
tpt4 = open(T4path, "r")
T4 = tpt4.read()
T4dict = eval(T4)
# T4dict = json.loads(T4dict.decode("utf-8"))
tpt4.close()
transcript += T1dict["response"]["transcript"]
transcript += T2dict["response"]["transcript"]
transcript += T3dict["response"]["transcript"]
transcript += T4dict["response"]["transcript"]
tpt = open(Tpath, "w")
tpt.write(transcript)
tpt.close()
# tpt = open(Tpath,"r")
Tdicts = [T1dict, T2dict, T3dict, T4dict]
# print(tpt.read())
# RUN THIS CELL WITH SPECIFIED PATH TO LOAD ALL THE TEXT FILE AS STRING INTO "wholeText" AND TEXT FILE AS LINES INTO "lineWiseText"
# path = "/content/drive/My Drive/TedTranscript.txt" #Path of the file from the drive
path = Tpath # "/content/drive/My Drive/entireTranscript.txt"
chunk(path, endLineAt=[".", "?"])
fi = open(path, "r")
wholeText = fi.read()
fi.seek(0)
totalWords = len((fi.read()).split())
fi.seek(0)
totalLines = len(fi.readlines())
fi.seek(0)
lineWiseText = fi.readlines()
fi.close()
# print("Total Lines present in the Source File is : ", totalLines)
# print("Total Words present in the source File is : ", totalWords)
# "completeFiltering" func takes "wholeText", "lineWiseText", "limitOnFreq" (let this be unchanged), "limitOnDataW" (this equals the "totalWords" in above cell)
# "completeFiltering" func returns priorities1,2 and countDict1,2 which are used more for internal purposes so I'm hiding these outputs
priorities1, priorities2, countDict1, countDict2 = completeFiltering(
wholeText, lineWiseText, limitOnFreq=1, limitOnDataW=totalWords)
# print("\nTop Prior Words of 10K Words data : ", priorities1)
# print("\nTop Prior Words of every Line data : ", priorities2)
# "fuzzyWayCondense" func takes "path", "priorities1,2", "countDict1,2", "limitOnLines" (this is can be anything <= "totalLines"), "limitonDataL"(this equals the "totalLines" in above cell), "method" (let it be unchanged), "printLineScores"(let it be False setting it to True just prints scores which are of no use to u)
# "fuzzyWayCondense" func returns "condensedText"(optional use to u), "condensedLines"(optional use to u just gives list of line strings) ,"condensedLinesIndices1(u might need this)"
condensedText, condensedLines, condensedLinesIndices1 = fuzzyWayCondense(
path,
priorities1,
priorities2,
countDict1,
countDict2,
limitOnLines="NormSTDPick",
limitOnDataL=totalLines,
method="TF-IDF",
printLineScores=False)
print("\nThis is the TF-IDF Way : \n")
print(
"\nThe Original Lines which made thorugh the filtering process are the line numbers : \n",
condensedLinesIndices1)
# print("\nOverall the condensed Text : \n", condensedText)
# "fuzzyWayCondense" func takes "path", "priorities1,2", "countDict1,2", "limitOnLines" (this is can be anything <= "totalLines"), "limitonDataL"(this equals the "totalLines" in above cell), "method" (let it be unchanged), "printLineScores"(let it be False setting it to True just prints scores which are of no use to u)
# "fuzzyWayCondense" func returns "condensedText"(optional use to u), "condensedLines"(optional use to u just gives list of line strings) ,"condensedLinesIndices2(u might need this)"
# condensedText, condensedLines, condensedLinesIndices2 = fuzzyWayCondense(path,priorities1, priorities2,countDict1, countDict2, limitOnLines="NormSTDPick", limitOnDataL = totalLines, method = "Frequency", printLineScores=False)
# print("\nThis is the Frequency Way : \n")
# print("\nThe Original Lines which made thorugh the filtering process are the line numbers : \n", condensedLinesIndices2)
# print("\nOverall the condensed Text : \n", condensedText)
# "finalSet" gives union of results of both methods
# finalSet = set(condensedLinesIndices1).union(set(condensedLinesIndices2))
finalSet = set(condensedLinesIndices1)
print("\nConsider these lines as Important : ", finalSet)
print("\nPercentage of Condensation of initial Text is : {:.4f}%".format(
((totalLines - len(finalSet)) / totalLines) * 100))
Jsonpath = "2.txt" # "/content/drive/My Drive/entireJasonObj.txt"
correction = 0
for transcripts in range(4):
for times in Tdicts[transcripts]["response"]["words"]:
times["start"] = times["start"] + correction
times["end"] = times["end"] + correction
mark = times["end"]
correction = mark
jsonObjs = {"response": dict()}
jsonObjs["response"]["words"] = T1dict["response"]["words"] + T2dict[
"response"]["words"] + T3dict["response"]["words"] + T4dict[
"response"]["words"]
jsonObjs = str(jsonObjs)
json = open(Jsonpath, "w")
json.write(jsonObjs)
json.close()
pathr = Jsonpath # "/content/drive/My Drive/tedxtJasonObject" # PATH TO THE TEXT FILE CONTAINING JASON OBJECT (DICTIONARY OF REQUEST ID, RESPONSE, TIMESTAMPS OF WORDS ....)
fi = open(pathr, "r")
fullDataDict = eval(fi.read())
def convert(seconds):
seconds = seconds % (24 * 3600)
hour = seconds // 3600
seconds %= 3600
minutes = seconds // 60
seconds %= 60
return "%d:%02d:%02d" % (hour, minutes, seconds)
wholeWords = wholeText.split()
wholeLines = lineWiseText
prevWordsCount = dict()
wds = 0
for sen in wholeLines:
prevWordsCount[str(wholeLines.index(sen) + 1)] = wds
wds += len(sen.split())
timeStamps = dict()
for line in finalSet:
try:
startInd = prevWordsCount[str(line)]
endInd = prevWordsCount[str(line + 1)] - 1
# USE "convert" if u need the output time stamps to be in HH:MM:SS if not remove "convert"
timeRange = (fullDataDict["response"]["words"][startInd]["start"],
fullDataDict["response"]["words"][endInd]["end"])
timeStamps[int(line)] = timeRange
count = 1
except:
print("")
c = 0
Stamps = []
for line in sorted(timeStamps):
c += 1
Stamps.append(timeStamps[line])
print(Stamps)
V = []
clip = VideoFileClip("apple.mp4")
for t in range(0, len(Stamps)):
a = Stamps[t][0]
b = Stamps[t][1]
clipx = clip.subclip(a, b)
V.append(clipx)
# x concatinating both the clips
final = concatenate_videoclips(V)
final.write_videofile("apple.avi", codec='rawvideo')
