def get_testcaseID(self, file_name):
try:
file_config = "%s/orig.config.yaml" %file_name
if not utilities.is_file_valid(file_config):
print "File %s does not exist" %(file_config)
else:
config_dict = utilities.read_yaml(file_config)
return config_dict.get('testcase_id')
except Exception:
print "Error reporting result"
return None
def check_summary(self, file_summary):
status = False
if not utilities.is_file_valid(file_summary):
print "File %s does not exist" %(file_summary)
else:
summary_dict = utilities.read_yaml(file_summary)
#set ceph version
if not self.ceph_version:
self.ceph_version = summary_dict.get('version')
if summary_dict.get('success'):
status = True
return status
def __init__(self, targetfile, bat_type=None, set_number=None,
report_result=False, runlist_path=None, log_path=None,
nuke=False, mail=False, no_poweroff=False):
self.targetfile = targetfile
self.bat = bat_type
self.set_number = set_number
self.report_result = report_result
self.runlist = runlist_path
self.log_path = log_path
self.nuke = nuke
self.runlist_use = []
self.result = None
self.mail = mail
self.no_poweroff = no_poweroff
self.execution_time = 0
#get all the targets from target files
self.target_nodes = utilities.read_yaml(self.targetfile).get('targets')
self.target_nodes = self.target_nodes.keys()
self.ceph_version = None
self.tls_obj = None
if self.report_result:
self.tls_obj = testlinklib.Testlink()
import gensim.corpora as corpora
from gensim.models import CoherenceModel
from gensim.models import ldamodel, LsiModel
from joblib import dump, load
from utilities import read_yaml
import pyLDAvis.gensim
import pandas as pd
param = read_yaml("conf/best_params.yaml", "opt_topic")
print(param)
lda_model = load("model/lda_model.pkl")
reviews_sw_removed = load("model/reviews.pkl")
id2word = corpora.Dictionary(reviews_sw_removed)
texts = reviews_sw_removed
corpus = [id2word.doc2bow(text) for text in texts]
# print('\nCoherence Score: ', coherence_lda)
#print("\nPerplexity: ", lda_model.log_perplexity(corpus))
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
pyLDAvis.save_html(vis, "report/lda.html")
# print(lda_model.print_topics())
x = lda_model.show_topics(formatted=False)
topics_words = [(tp[0], [wd[0] for wd in tp[1]]) for tp in x]
df_topics = pd.DataFrame(topics_words, columns=['Topic ID', 'Topics'])
df_topics.to_csv("report/topics.csv", index=False)
id2word=id2word,
num_topics=i,
random_state=42,
passes=10,
per_word_topics=True,
)
coherence_model_lda = CoherenceModel(
model=lda_modeli,
texts=reviews_sw_removed,
dictionary=id2word,
coherence="c_v",
)
coherence_lda = coherence_model_lda.get_coherence()
print("topis no ", i, "Coherence Score: ", coherence_lda)
if max_topic_no == i:
max_score = coherence_lda
dump(lda_modeli, 'model/lda_model.pkl')
else:
if coherence_lda > max_score:
max_topic_no = i
max_score = coherence_lda
dump(lda_modeli, 'model/lda_model.pkl')
dump_yaml({"opt_topic": max_topic_no}, "conf/best_params.yaml")
if __name__ == "__main__":
start = read_yaml("params.yaml", "startTopic")
end = read_yaml("params.yaml", "endTopic")
main(start, end)
import os
import os.path
import utilities
import subprocess
from subprocess import Popen, PIPE
import re
from datetime import datetime
import multiprocessing
path = os.path.abspath('./rsync.yaml')
configs = utilities.read_yaml(path)
iterv = configs['Number_of_Folders']
fiterv = configs['Number_of_Files']
Email_ID = configs['Email_ID']
DB_IP = configs['DB_IP']
spath=configs['Source_Path']
dpath=configs['Destination_Path']
def rsync_copy(srcpath,dpath):
print "Rsync started for folder : ", srcpath
logfile = re.sub(r'[\W/]','_',srcpath) #print "logfile" , logfile
final_logile = datetime.now().strftime('rsync'+logfile+'_%H_%M_%d_%m_%Y.log')
flog="/root/log/"+final_logile #print flog
fh = open(flog,"w")
fh.write("#########################################\n")
process = Popen(['/usr/bin/rsync','-arP','--progress',srcpath,dpath], stdout=PIPE) #, stderr=PIPE)
#print "=====================================================\n"
while process.poll() is None:
#output = process.stdout.readline()
output = process.communicate()[0]
