def alarm(): sql = "select room,hostname,ip,cluster,aralm,date,count(counts) from openfalcon_roominfo group by room;" room = "select name from openfalcon_roomname where name not in(select room from openfalcon_roominfo);" sqlinfo = db.select_count_table(sql) room_name = db.select_room_table(room) open('./json_alarm', 'w') json = open('./json_alarm', 'a') json.write('[') for row in sqlinfo: row1 = ''.join(row[0]).encode('utf-8') row7 = ''.join(str(row[6])) json.write("{") json.write('"name":' + '"' + row1.strip() + '",') json.write('"value":' + '"' + row7 + '"') json.write("},") for name in room_name: name_row = ''.join(name).encode('utf-8') json.write("{") json.write('"name":' + '"' + name_row.strip() + '",') json.write('"value":' + '"0"') json.write("},") json.write("]") json.close() '''得到最终结果''' info = open('./json_alarm', 'r') rows = info.readlines() rows_info = ''.join(rows) result = rows_info.replace(',]', ']') #print result rs = open('/systemfalcon/systemfalcon/static/alarm/result_alarm', 'w') rs.write(result) rs.close()
async def lgo(client, message): engine = message.Engine pablo = await edit_or_reply(message, engine.get_string("PROCESSING")) if not message.reply_to_message: await pablo.edit( engine.get_string("NEEDS_REPLY").format("Animated Sticker")) return if not message.reply_to_message.sticker: await pablo.edit( engine.get_string("NEEDS_REPLY").format("Animated Sticker")) return if message.reply_to_message.sticker.mime_type != "application/x-tgsticker": await pablo.edit( engine.get_string("NEEDS_REPLY").format("Animated Sticker")) return lol = await message.reply_to_message.download("tgs.tgs") cmdo = f"lottie_convert.py {lol} json.json" await runcmd(cmdo) if not os.path.exists('json.json'): await pablo.edit( engine.get_string("NEEDS_REPLY").format("Animated Sticker")) os.remove("tgs.tgs") return json = open("json.json", "r") jsn = json.read() json.close() jsn = (jsn.replace("[1]", "[2]").replace("[2]", "[3]").replace( "[3]", "[4]").replace("[4]", "[5]").replace("[5]", "[6]")) open("json.json", "w").write(jsn) await runcmd(f"lottie_convert.py json.json tgs.tgs") await client.send_sticker(message.chat.id, "tgs.tgs") os.remove("json.json") os.remove(lol) os.remove("tgs.tgs") await pablo.delete()
async def lgo(client, message): pablo = await edit_or_reply(message, "`Processing...`") if not message.reply_to_message: await pablo.edit("Reply to A Animated Sticker...") return if not message.reply_to_message.sticker: await pablo.edit("Reply to A Animated Sticker...") return if message.reply_to_message.sticker.mime_type != "application/x-tgsticker": await pablo.edit("`Reply to A Animated Sticker...`") return lol = await message.reply_to_message.download("tgs.tgs") cmdo = f"lottie_convert.py {lol} json.json" await runcmd(cmdo) if not os.path.exists('json.json'): await pablo.edit("`Invalid Tgs Sticker I Suppose.`") os.remove("tgs.tgs") return json = open("json.json", "r") jsn = json.read() json.close() jsn = ( jsn.replace("[1]", "[2]") .replace("[2]", "[3]") .replace("[3]", "[4]") .replace("[4]", "[5]") .replace("[5]", "[6]") ) open("json.json", "w").write(jsn) await runcmd(f"lottie_convert.py json.json tgs.tgs") await client.send_sticker(message.chat.id, "tgs.tgs") os.remove("json.json") os.remove(lol) os.remove("tgs.tgs") await pablo.delete()
def alarm_class(): """告警分类图表""" sql = "select metric,hostname,ip,cluster,aralm,date,count(counts) from openfalcon_roominfo group by metric;" sqlinfo = db.select_count_table(sql) open('./json_class', 'w') json = open('./json_class', 'a') json.write('[') for row in sqlinfo: row1 = ''.join(row[0]).encode('utf-8') row7 = ''.join(str(row[6])) json.write("{") json.write('"name":' + '"' + row1 + '",') json.write('"value":' + '"' + row7 + '"') json.write("},") json.write("]") json.close() '''得到最终结果''' info = open('./json_class', 'r') rows = info.readlines() rows_info = ''.join(rows) result = rows_info.replace(',]', ']') #print result rs = open('/systemfalcon/systemfalcon/static/alarm/result_class', 'w') rs.write(result) rs.close()
def guardar_nivel(ruta, datos): lista = [] archivo = open(ruta, 'r') lista = json.load(archivo) json.close() lista.append(datos) archivo = open(ruta, 'w') json.dump(lista, archivo) json.close()
def start_crawler(): logging.info("start_crawler") process_root('../resources/sites.ini') postings_file = '../data/job_postings_' + datetime.now().strftime( "%y-%m-%d_%H.%M") + '.json' json = open(postings_file, 'w+', encoding='utf-8') json.write('[\n') process_work_queue(json) json.write(']') json.close() remove_last_comma(postings_file) copyfile(postings_file, '../data/job_postings.json')
def obtener_listado(ruta, nivel_seleccionado): archivo_niveles = open(ruta, 'r') lista_niveles = json.loads(archivo_niveles) json.close() if nivel_seleccionado is 'Nivel facil': dicc_niveles = lista_niveles[ 'nivel_facil'] #se guarda en el dicc a los puntajes del modo facil ventana_puntajes('Nivel facil', dicc_niveles) if nivel_seleccionado is 'Nivel medio': dicc_niveles = lista_niveles[ 'nivel_medio'] #se guarda en el dicc a los puntajes del modo facil ventana_puntajes('Nivel medio', dicc_niveles) if nivel_seleccionado is 'Nivel dificil': dicc_niveles = lista_niveles[ 'nivel_dificil'] #se guarda en el dicc a los puntajes del modo facil ventana_puntajes('Nivel dificil', dicc_niveles)
def validateConfigJson(file): if os.path.isfile(file) == False: print "You did not pass a valid path for the config.json. Exiting \n" sys.exit(4) json = open(file, 'r') config = json.load(json) if not config['NDK']: print "NDK field is empty. Exiting \n" sys.exit(4) if not config['SDK']: print "SDK field is empty. Exiting \n" sys.exit(4) json.close()
def FileOper(etl, data, type): path = etl.FilePath; filetype = path.split('.')[-1].lower(); encode = 'utf-8' if etl.EncodingType == 'UTF8' else 'ascii'; if filetype in ['csv', 'txt']: import csv file = open(etl.FilePath, type, encoding=encode); sp = ',' if filetype == 'csv' else '\t'; if type == 'r': reader = csv.DictReader(file, delimiter=sp) for r in reader: yield r; else: writer = csv.DictWriter(file, delimiter=sp) start = False; for r in data: if not start: field = r.keys; writer.fieldnames = field; writer.writerow(dict(zip(field, field))) start = True; writer.writer(r) yield r; file.close(); elif filetype == 'xlsx': pass; elif filetype == 'xml' and type == 'r': tree = ET.parse(path); root = tree.getroot(); root = root.findall('Doc'); for etool in root: p = {r: etool.attrib[r] for r in etool.attrib}; yield p; elif filetype == 'xml' and type == 'w': pass; elif filetype == 'json': if type == 'r': items = json.load(open(path, encoding=encode)); for r in items: yield r; else: json.open(path); for r in data: json.write(r) yield r; json.close() json.dump([r for r in data], open(path, type, encode));
def read(): # os.system('python join.py hosts') file = open('./temp', 'r') file1 = open('./hosts', 'r') file2 = open('./datetime', 'r') open('./json', 'w') json = open('./json', 'a') json.write('{ "告警":[') while 1: json.write("{") line = file.readline().strip() line1 = file1.readline().strip() line2 = file2.readline().strip() # print '"host":'+'"'+line1.strip(), line.strip(),line2.strip()+'"' json.write('"room":' + '"' + line1.strip() + line.strip() + line2.strip() + '"') json.write("},") if not line and not line1 and not line2: break pass json.write("]}") json.close()
def writeToJSON(str, fileName, fileType, outputPath): json = open(outputPath + fileName + "_" + fileType + ".json", "w") json.write(str) json.close()
stock_data = web.DataReader("%s.KS" %st[0],'yahoo',start,end) if selection == str(2): print("너냐") stock_data = web.DataReader("%s.KQ" %st[0],'yahoo',start,end) print("너냐!") print("1번마") stock_data.loc[:,'Company'] = CompanyName stock_data.loc[:,'StockCode'] = StockCode stock_data.loc[:,'Date'] = stock_data.index.astype('str') json = open('DAILY_STOCK_DATA.json','a') #StockData.append(stock_data) #json.dumps(stock_data, ensure_ascii=False, sort_keys=False, separators=(',', ':')).encode('utf-8') # force_ascii=False for korean print("2번마") json.write(stock_data.to_json(orient='records',force_ascii=False)) json.close() print("3번마") engine = create_engine('mysql+mysqlconnector://ADMIN:ADMIN@localhost:3306/stock', echo=False) print("4번마") stock_data.to_sql(name='stock', con=engine, if_exists = 'append', index=False) print(stock_data) print("------------------------------------") except: print("예외입니다잉") pass
def remove_last_comma(postings_file): lines = open(postings_file, 'r+').readlines() lines[-1] = lines[-1].replace(',', '') json = open(postings_file, 'w+', encoding='utf-8') json.writelines(lines) json.close()
def leer_Nivel(ruta): archivo = open(ruta, 'r') lista = json.load(archivo) json.close() return lista
def sum(): import wikipedia from fuzzywuzzy import process from fuzzywuzzy import fuzz import copy import numpy as np def superImportant(topic): content = wikipedia.summary(topic) pageObj = wikipedia.WikipediaPage(topic) hyperLinks1 = [] nonhyperLinks = content.split() for words2 in pageObj.links: if words2 in content: hyperLinks1.append(words2) contentWords = list(set(copy.deepcopy(hyperLinks1) + nonhyperLinks)) fails = 0 for words in hyperLinks1: try: content = wikipedia.summary(words) pageObj = wikipedia.WikipediaPage(words) count = 0 for words2 in pageObj.links: if words2 in content: contentWords.append(words2) count += 1 if count >= 1 * len(hyperLinks1): break except: fails += 1 contentWords = set((" ".join(contentWords)).split()) return contentWords def changePriorities(dic, mapWords): frequencies = [] for k, v in dic.items(): frequencies.append(v) X = copy.deepcopy(np.percentile(np.array(frequencies), 95)) misMatches = 0 for words in mapWords: if words in dic.keys(): dic[words] = X elif words.lower() in dic.keys(): dic[words.lower()] = X else: misMatches += 1 return dic, misMatches # JUST RUN THIS CELL import copy import nltk import numpy as np import string from sklearn.feature_extraction.text import CountVectorizer from nltk.corpus import stopwords from fuzzywuzzy import process from fuzzywuzzy import fuzz from scipy import stats import matplotlib.pyplot as plt nltk.download('wordnet') nltk.download('stopwords') def chunk(sourceFile, wordsPerLine=None, endLineAt=None): fi = open(sourceFile, "r+") text = fi.read() text = text.replace("\n", "") if wordsPerLine != None: text = text.split() for words in range(1, len(text) + 1): if words % 3 == 0: text[words - 1] = text[words - 1] + "\n" fi.seek(0) fi.write(" ".join(text)) if endLineAt != None: for words in endLineAt: text = text.split(words) text = "\n".join(text) fi.seek(0) fi.write(text) fi.close() return def getKey(D, val): for key, value in D.items(): if val == value: return key return -1 def completeFiltering(singleStringTxt, multiLineTxt, limitOnFreq, limitOnDataW=10000): wholeText = singleStringTxt cleansed = wholeText.split()[:limitOnDataW] table = str.maketrans("", "", string.punctuation) cleansed = [w.translate(table) for w in cleansed] patched = " ".join(cleansed) cleansed = patched.split() cleansed = [ words for words in cleansed if not words.lower() in stopwords.words() ] cleansedTxt = " ".join(cleansed) wholeText = [cleansedTxt] lineWiseText = multiLineTxt # list of text documents # create the transform vectorizer1 = CountVectorizer() vectorizer2 = CountVectorizer() # tokenize and build vocab vectorizer1.fit(wholeText) vectorizer2.fit(lineWiseText) # summarize wToInd1 = vectorizer1.vocabulary_ wToInd2 = vectorizer2.vocabulary_ # encode document vector1 = vectorizer1.transform(wholeText) vector2 = vectorizer2.transform(lineWiseText) # summarize encoded vector v1 = vector1.toarray() v2 = vector2.toarray() finalCount = np.sum(v1, axis=0, keepdims=False) countDict1 = dict() countDict2 = dict() priorities2 = dict() for ind in range(len(finalCount)): if finalCount[ind] >= limitOnFreq: countDict1[getKey(wToInd1, ind)] = finalCount[ind] for lines in range(v2.shape[0]): countDict = dict() for ind in range(v2.shape[1]): if v2[lines][ind] >= limitOnFreq: countDict[getKey(wToInd2, ind)] = v2[lines][ind] priorities = sorted(countDict, key=countDict.get, reverse=True) countDict2[str(lines + 1)] = countDict priorities2[str(lines + 1)] = priorities contentWords = superImportant("Apple Inc") countDict1, misMatch = changePriorities(countDict1, contentWords) print("These many got mismatched : ", misMatch) priorities1 = sorted(countDict1, key=countDict1.get, reverse=True) return priorities1, priorities2, countDict1, countDict2 def fuzzyWayCondense(fileSource, priorities1, priorities2, prioritiesMap1, prioritiesMap2, limitOnLines=3, limitOnDataL=100, method="Frequency", printLineScores=False): if method == "Frequency": priorities = priorities1 prioritiesMap = prioritiesMap1 elif method == "TF-IDF": priorities = priorities1 prioritiesMap = prioritiesMap1 prioritiesMapext = prioritiesMap2 includeTFIDF = np.zeros((limitOnDataL, len(priorities))) fi = open(fileSource, "r") include = np.zeros((limitOnDataL, len(priorities))) wholeLines = fi.readlines()[:limitOnDataL] maintain = dict() for lines in range(1, limitOnDataL + 1): maintain[str(lines)] = [] fi.close() for words in priorities: options = process.extract(words, wholeLines, limit=limitOnDataL) for line, score in options: if (words in line.split()) and method == "Frequency": maintain[str(wholeLines.index(line) + 1)].append(words) include[wholeLines.index(line)][priorities.index( words)] = score * prioritiesMap[words] elif (words in line.split()) and method == "TF-IDF": maintain[str(wholeLines.index(line) + 1)].append(words) includeTFIDF[wholeLines.index(line)][priorities.index(words)] = \ prioritiesMapext[str(wholeLines.index(line) + 1)][words] * prioritiesMap[words] if method == "TF-IDF": includeTFIDF = list(np.sum(includeTFIDF, axis=0)) for words in priorities: options = process.extract(words, wholeLines, limit=limitOnDataL) for line, score in options: if (words in line.split()): include[wholeLines.index(line)][priorities.index( words )] = score * includeTFIDF[priorities.index(words)] for lines in range(1, limitOnDataL + 1): maintain[str(lines)] = set(maintain[str(lines)]) include = list(np.sum(include, axis=1)) includeTemp = np.array(copy.deepcopy(include)) if printLineScores == True: print("\nThe Scores of the Sentences from 1 to", limitOnDataL, " are as follows \n", include) print("\nThe Key Words Per Line for all the lines are : \n", maintain) condensedLines = [] condensedLinesIndices = [] if limitOnLines != "NormSTDPick": includeTemp = (np.sort(includeTemp))[::-1] for i in range(limitOnLines): condensedLines.append(wholeLines[include.index( includeTemp[i])]) condensedLinesIndices.append(include.index(includeTemp[i]) + 1) include[include.index(includeTemp[i])] = -1 else: includeTemp = np.array([(value >= np.percentile(includeTemp, 50)) for value in includeTemp]).astype(int) includeTemp = np.reshape(np.argwhere(includeTemp), (-1, )) + 1 condensedLines = [wholeLines[i - 1] for i in includeTemp] condensedLinesIndices = includeTemp condensedText = " ".join(condensedLines) return condensedText, condensedLines, condensedLinesIndices import json T1path = "final1.txt" # "/content/drive/My Drive/transcript1" T2path = "final2.txt" # "/content/drive/My Drive/transcript2" T3path = "final3.txt" # "/content/drive/My Drive/transcript3" T4path = "final4.txt" # "/content/drive/My Drive/transcript4" Tpath = "1.txt" # "/content/drive/My Drive/entireTranscript.txt" transcript = "" tpt1 = open(T1path, "r") T1 = tpt1.read() T1dict = eval(T1) tpt1.close() tpt2 = open(T2path, "r") T2 = tpt2.read() T2dict = eval(T2) tpt2.close() tpt3 = open(T3path, "r") T3 = tpt3.read() T3dict = eval(T3) # T3dict = json.loads(T3dict.decode("utf-8")) tpt3.close() tpt4 = open(T4path, "r") T4 = tpt4.read() T4dict = eval(T4) # T4dict = json.loads(T4dict.decode("utf-8")) tpt4.close() transcript += T1dict["response"]["transcript"] transcript += T2dict["response"]["transcript"] transcript += T3dict["response"]["transcript"] transcript += T4dict["response"]["transcript"] tpt = open(Tpath, "w") tpt.write(transcript) tpt.close() # tpt = open(Tpath,"r") Tdicts = [T1dict, T2dict, T3dict, T4dict] # print(tpt.read()) # RUN THIS CELL WITH SPECIFIED PATH TO LOAD ALL THE TEXT FILE AS STRING INTO "wholeText" AND TEXT FILE AS LINES INTO "lineWiseText" # path = "/content/drive/My Drive/TedTranscript.txt" #Path of the file from the drive path = Tpath # "/content/drive/My Drive/entireTranscript.txt" chunk(path, endLineAt=[".", "?"]) fi = open(path, "r") wholeText = fi.read() fi.seek(0) totalWords = len((fi.read()).split()) fi.seek(0) totalLines = len(fi.readlines()) fi.seek(0) lineWiseText = fi.readlines() fi.close() # print("Total Lines present in the Source File is : ", totalLines) # print("Total Words present in the source File is : ", totalWords) # "completeFiltering" func takes "wholeText", "lineWiseText", "limitOnFreq" (let this be unchanged), "limitOnDataW" (this equals the "totalWords" in above cell) # "completeFiltering" func returns priorities1,2 and countDict1,2 which are used more for internal purposes so I'm hiding these outputs priorities1, priorities2, countDict1, countDict2 = completeFiltering( wholeText, lineWiseText, limitOnFreq=1, limitOnDataW=totalWords) # print("\nTop Prior Words of 10K Words data : ", priorities1) # print("\nTop Prior Words of every Line data : ", priorities2) # "fuzzyWayCondense" func takes "path", "priorities1,2", "countDict1,2", "limitOnLines" (this is can be anything <= "totalLines"), "limitonDataL"(this equals the "totalLines" in above cell), "method" (let it be unchanged), "printLineScores"(let it be False setting it to True just prints scores which are of no use to u) # "fuzzyWayCondense" func returns "condensedText"(optional use to u), "condensedLines"(optional use to u just gives list of line strings) ,"condensedLinesIndices1(u might need this)" condensedText, condensedLines, condensedLinesIndices1 = fuzzyWayCondense( path, priorities1, priorities2, countDict1, countDict2, limitOnLines="NormSTDPick", limitOnDataL=totalLines, method="TF-IDF", printLineScores=False) print("\nThis is the TF-IDF Way : \n") print( "\nThe Original Lines which made thorugh the filtering process are the line numbers : \n", condensedLinesIndices1) # print("\nOverall the condensed Text : \n", condensedText) # "fuzzyWayCondense" func takes "path", "priorities1,2", "countDict1,2", "limitOnLines" (this is can be anything <= "totalLines"), "limitonDataL"(this equals the "totalLines" in above cell), "method" (let it be unchanged), "printLineScores"(let it be False setting it to True just prints scores which are of no use to u) # "fuzzyWayCondense" func returns "condensedText"(optional use to u), "condensedLines"(optional use to u just gives list of line strings) ,"condensedLinesIndices2(u might need this)" # condensedText, condensedLines, condensedLinesIndices2 = fuzzyWayCondense(path,priorities1, priorities2,countDict1, countDict2, limitOnLines="NormSTDPick", limitOnDataL = totalLines, method = "Frequency", printLineScores=False) # print("\nThis is the Frequency Way : \n") # print("\nThe Original Lines which made thorugh the filtering process are the line numbers : \n", condensedLinesIndices2) # print("\nOverall the condensed Text : \n", condensedText) # "finalSet" gives union of results of both methods # finalSet = set(condensedLinesIndices1).union(set(condensedLinesIndices2)) finalSet = set(condensedLinesIndices1) print("\nConsider these lines as Important : ", finalSet) print("\nPercentage of Condensation of initial Text is : {:.4f}%".format( ((totalLines - len(finalSet)) / totalLines) * 100)) Jsonpath = "2.txt" # "/content/drive/My Drive/entireJasonObj.txt" correction = 0 for transcripts in range(4): for times in Tdicts[transcripts]["response"]["words"]: times["start"] = times["start"] + correction times["end"] = times["end"] + correction mark = times["end"] correction = mark jsonObjs = {"response": dict()} jsonObjs["response"]["words"] = T1dict["response"]["words"] + T2dict[ "response"]["words"] + T3dict["response"]["words"] + T4dict[ "response"]["words"] jsonObjs = str(jsonObjs) json = open(Jsonpath, "w") json.write(jsonObjs) json.close() pathr = Jsonpath # "/content/drive/My Drive/tedxtJasonObject" # PATH TO THE TEXT FILE CONTAINING JASON OBJECT (DICTIONARY OF REQUEST ID, RESPONSE, TIMESTAMPS OF WORDS ....) fi = open(pathr, "r") fullDataDict = eval(fi.read()) def convert(seconds): seconds = seconds % (24 * 3600) hour = seconds // 3600 seconds %= 3600 minutes = seconds // 60 seconds %= 60 return "%d:%02d:%02d" % (hour, minutes, seconds) wholeWords = wholeText.split() wholeLines = lineWiseText prevWordsCount = dict() wds = 0 for sen in wholeLines: prevWordsCount[str(wholeLines.index(sen) + 1)] = wds wds += len(sen.split()) timeStamps = dict() for line in finalSet: try: startInd = prevWordsCount[str(line)] endInd = prevWordsCount[str(line + 1)] - 1 # USE "convert" if u need the output time stamps to be in HH:MM:SS if not remove "convert" timeRange = (fullDataDict["response"]["words"][startInd]["start"], fullDataDict["response"]["words"][endInd]["end"]) timeStamps[int(line)] = timeRange count = 1 except: print("") c = 0 Stamps = [] for line in sorted(timeStamps): c += 1 Stamps.append(timeStamps[line]) print(Stamps) V = [] clip = VideoFileClip("apple.mp4") for t in range(0, len(Stamps)): a = Stamps[t][0] b = Stamps[t][1] clipx = clip.subclip(a, b) V.append(clipx) # x concatinating both the clips final = concatenate_videoclips(V) final.write_videofile("apple.avi", codec='rawvideo')