def getRadiusDensity():
hashMap = label()
hashMapRadiusDensity = {}
for poi in hashMap:
numberRequests = len(hashMap[poi]['arrayPoints'])
radius = hashMap[poi]['farestDist']
area = calculateArea(radius)
density = numberRequests / area
tempObj = {'radius': radius, 'density': density}
hashMapRadiusDensity[poi] = tempObj
return hashMapRadiusDensity
def getAvgDistanceStdForEachPOI():
hashMap = label()
hashMapPOI = convertArrayPoiToHashMap()
hashMapDistance = {}
for poi in hashMap:
point2dPoi = hashMapPOI[poi]
arrayPoint = hashMap[poi]['arrayPoints']
res = []
for point in arrayPoint:
dist = distance(point2dPoi,point)
res.append(dist)
averageDist = np.mean(res)
std = np.std(res)
obj = {'averageDist': averageDist, 'standardDeviation': std}
hashMapDistance[poi] = obj
return hashMapDistance
# join the databses
DB = pd.concat([DB1_sub, DB2_sub])
del DB1_sub, DB2_sub, List1, List2
#clean up
DB['date'] = pd.to_datetime(DB['date'])
DB['month'] = DB['date'].dt.month
#remove <>
DB['E. Coli'] = DB['E. Coli'].astype(str).str.replace('<', '')
DB['E. Coli'] = DB['E. Coli'].astype(str).str.replace('>', '')
DB['E. Coli'] = DB['E. Coli'].astype(str).str.replace('?', '')
DB['E. Coli'] = DB['E. Coli'].astype(float)
#Label
label(DB)
List0 = list(DB['Watershed'].unique())
List1 = [
"ANA01", "ANA02", "ANA03", "ANA04", "ANA05", "ANA06", "ANA07", "ANA08",
"ANA09", "ANA10", "ANA11", "ANA12", "ANA13", "ANA14", "ANA15", "ANA16",
"ANA28", "ANA17", "ANA18", "ANA19", "ANA20", "ANA21", "ANA21 ", "ANA22",
"ANA23", "ANA24", "ANA25", "ANA26", "ANA27", "ANA29", "ANA30"
]
List2 = [
"PMS01", "PMS02", "PMS03", "PMS05", "PMS07", "PMS08", "PMS09", "PMS10",
"PMS11", "PMS12", "PMS13", "PMS16", "PMS18", "PMS21", "PMS21 ", "PMS23",
"PMS25", "PMS27", "PMS29", "PMS31", "PMS33", "PMS35", "PMS37", "PMS39",
"PMS41", "PMS44", "PMS46", "PMS48", "PMS51", "PMS52"
]
List3 = ["RCR12", "RCR01", "RCR04", "RCR07", "RCR09"] # Rock Creek
def Batch_S5(folderPath, batch_Id):
import time
Start_time = time.time()
stage = "UNIQ_S5"
path = folderPath
print(path)
#batchId = batch_Id
path_split = path.split("_")
print(path_split, 'this is the split path')
docx = "_".join(path_split[-3:-1]) + ".docx"
file_path = path.replace("\\", "/") + "/tx1.xml"
print(file_path, 'this is the file path')
file = file_path.split('/')[-2].split("_")
print(file, 'this is the file path for error rectification')
jid = file[-3]
aid = file[-2]
itemid = jid + "_" + aid + "_" + stage
time1 = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
counter = 1
error_count = 0
uid = socket.gethostname()
exists = os.path.isfile(path.replace("\\", "/") + "/UNIQ_S5.xml")
jss_path = path.replace("\\", "/") + "/" + jid + "-jss.xml"
order_path = path.replace("\\", "/") + "/" + jid + "_" + aid + "_order.xml"
mss = path + "/" + docx
tp = 0
fp = 0
tn = 0
fn = 0
copy2(
path.replace("\\", "/") + "/tx1.xml.log.xml",
path.replace("\\", "/") + "/UNIQ_S5.xml")
log_path = path.replace("\\", "/") + "/UNIQ_S5.xml"
Master_list = [
itemid, uid, jid, aid, stage, time1, error_count, counter, time1,
time.time() - Start_time
]
insert_db_master(Master_list)
def log_transaction_db(obj, error_count, tp, tn, fp, fn):
append_log(obj, log_path)
tempdesc = ""
ss = ''
if 'error' in obj[0]:
ss = obj[0][-1]
error_count += len(obj)
tn += 1
else:
tp += 1
for sub in obj:
if "error" in sub:
#if(tempdesc==""):
# tn+=1
#if(tempdesc!="" and sub[-1]!=tempdesc):
# tn+=1
if (obj.index(sub) != 0):
ss = ss + ';' + sub[-1]
#tempdesc=sub[-1]
#error_count+=1
if 'error' in obj[0]:
trans_list = [itemid, obj[0][1], obj[0][0], ss, len(ss.split(';'))]
insert_db_trans(trans_list)
return error_count, tp, tn, fp, fn
obj = Brf037a(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'second one', 'Vinay')
obj = Rule_17(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'third one', 'vipin')
obj = rule_15(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'fourth one', 'vipin')
obj = Rule_08(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'sixth one', 'preeti')
obj = Rule_93(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'eight one', 'preeti')
obj = Rule_91(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'ninth one', 'preeti')
obj = AUN009A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'eleventh one', 'shani')
obj = ack003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'twelveth one', 'shani')
obj = Rule_108(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'thirteenth one', 'preeti')
obj = article_history(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'fourteenth one', 'shani')
obj = rule_59(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'fifteenth one', 'vipin')
obj = highlights(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'sixteenth one', 'shani')
obj = rule_87(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'seventeenth one', 'preeti')
obj = Rule_89(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, 'eighteenth one', 'preeti')
obj = rule_HIS003A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '19th one', 'shani')
obj = rule_73(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '20th one', 'vipin')
obj = rule_66(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '21th one', 'vipin')
obj = rule_71(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '22th one', 'vipin')
obj = rule_67(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '23th one', 'vipin')
obj = aug001(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '24th one', 'shani')
obj = BRX011(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '25th one', 'rachit')
obj = Brx042(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '26th one', 'kumar')
obj = brx039(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '27th one', 'kumar')
obj = rule31(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '28th one', 'aditya')
obj = rule_afn003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '29th one', 'shani')
obj = brx029b(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '30th one', 'kumar')
#obj = Table(file_path)
#error_count,tp,tn,fp,fn,=log_transaction_db(obj,error_count,tp,tn,fp,fn)
#print(obj,'31th one','shani')
obj = BRX007(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '31th one', 'aditya')
obj = BRX006(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '32th one', 'aditya')
obj = Table(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '33th one', 'shani')
obj = BRX018(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '34th one', 'rachit')
obj = BRX005A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '35th one', 'aditya')
obj = brx030(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '36th one', 'kumar')
obj = brx041(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '37th one', 'kumar')
obj = BRX020A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '38th one', 'rachit')
obj = brx032(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
## append_log(obj,log_path)
## for sub in obj:
## if "error" in sub:
## error_count+=1
## trans_list=[jid+"_"+aid+"_"+"S5", itemid, sub[1], sub[0], sub[-1], counter]
## insert_db_trans(trans_list)
print(obj, '39th one', 'kumar')
obj = BRX001a(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '40th one', 'aditya')
obj = BRX019(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '41th one', 'rachit')
obj = BRX024(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '42th one', 'rachit')
obj = BRF091B(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '43th one', 'vipin')
obj = BRX005B_35(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '44th one', 'kumar')
obj = aeu003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '45th one', 'kumar')
obj = BRX020B(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '46th one', 'rachit')
obj = AFO002A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '47th one', 'preeti')
obj = BRX005B_36(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '48th and 49th one', 'aditya')
obj = tab018(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '50th ', 'kumar')
obj = XsTB701(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '51th', 'kumar')
obj = XsTB702(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '52th', 'kumar')
obj = BRX022(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '53th', 'rachit')
obj = Rule_114(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '54th', 'preeti')
obj = rule132(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '55th', 'rachit')
obj = rule40(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '56th', 'aditya')
obj = Rule_113(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '57th', 'preeti')
obj = rule41(file_path, mss, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '58th', 'aditya')
obj = table_header(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '59th', 'shani')
obj = rule_80(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '60th', 'vipin')
obj = rule_72(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '61th', 'vipin')
obj = rule_78_138(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '62th', 'vipin')
obj = Rule_111(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '63th', 'preeti')
obj = Rule_96(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '64th', 'preeti')
obj = label(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '65th', 'shani')
obj = keywords(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '66th', 'shani')
obj = Rule_109(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '67th', 'preeti')
obj = Authors_Email(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '68th', 'Vinay')
obj = Rule139_140(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '69th', 'aditya')
obj = corr_check(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '70th', 'shani')
obj = rule_75(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '71th', 'vipin')
obj = rule_69(file_path, mss, jss_path)
#print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$",obj)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '72th', 'vipin')
obj = rule120(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '73th', 'aditya')
obj = rule_70(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '74th', 'vipin')
obj = rule_79(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '75th', 'vipin')
obj = rule_16(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '76th', 'vipin')
obj = rule_83(jss_path, file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '77th', 'vipin')
obj = tab021(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '78th', 'kumar')
obj = tab012(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '79th', 'kumar')
obj = XsFG601(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '80th', 'kumar')
obj = XsFM101(file_path, jss_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '81th', 'kumar')
obj = rule135(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '82th', 'rachit')
obj = rule144(mss, file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '83th', 'rachit')
# obj = rule130(file_path,json_path)
# append_log(obj,log_path)
# if "error" in obj:
# error_count+=1
# trans_list=[jid+"_"+aid+"_"+"S5", itemid, obj[2], obj[0], obj[-2], counter]
# insert_db_trans(trans_list)
# print(obj,'84th','rachit')
obj = Rule_7(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '84th', 'preeti')
obj = rule116(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '85th', 'aditya')
obj = Rule_149(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '86th', 'preeti')
obj = Tail(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '87th', 'shani')
obj = Rule_128(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '88th', 'preeti')
obj = CHE015_11(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '89th', 'rachit')
obj = fund(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(obj, error_count, tp, tn,
fp, fn)
print(obj, '90th', 'vinay')
updated_Master_list = [
itemid, error_count,
time.time() - Start_time, tp, tn, fp, fn, batch_Id, uid
]
update_db_master(updated_Master_list)
return
def Batch_S5(folderPath, batch_Id):
try:
import time
Start_time = time.time()
#stage = "UNIQ_S5"
stage = config["STAGE"]["stage"]
print(stage, 'stage?' * 50)
path = folderPath
print(path)
#batchId = batch_Id
path_split = path.split("_")
print(path_split, 'this is the split path')
print(path_split[-1])
if path_split[-1] == '110':
print('its in 110 block of batch' * 5)
docx = "_".join(path_split[-3:-1]) + ".docx"
file_path = path.replace("\\", "/") + "/tx1.xml"
print(file_path, 'this is the file path')
file = file_path.split('/')[-2].split("_")
print(file, 'this is the file path for error rectification')
jid = file[-3]
aid = file[-2]
itemid = jid + "_" + aid + "_" + stage
logging.info(
' **-------Logger start from here for {}-------** '.format(
itemid))
time1 = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
counter = 1
error_count = 0
uid = socket.gethostname()
exists = os.path.isfile(path.replace("\\", "/") + "/UNIQ_S5.xml")
jss_path = path.replace("\\", "/") + "/" + jid + "-jss.xml"
order_path = path.replace(
"\\", "/") + "/" + jid + "_" + aid + "_order.xml"
mss = path.replace("\\", "/") + "/" + jid + "_" + aid + ".docx"
tp = 0
fp = 0
tn = 0
fn = 0
else:
print('its not in 110 block of batch' * 5)
docx = "_".join(path_split[-2:]) + ".docx"
file_path = path.replace("\\", "/") + "/tx1.xml"
print(file_path, 'this is the file path')
file = file_path.split('/')[-2].split("_")
print(file, 'this is the file path for error rectification')
jid = file[-2]
aid = file[-1]
itemid = jid + "_" + aid + "_" + stage
logging.info(
' **-------Logger start from here for {}-------** '.format(
itemid))
time1 = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
counter = 1
error_count = 0
uid = socket.gethostname()
exists = os.path.isfile(path.replace("\\", "/") + "/UNIQ_S5.xml")
jss_path = path.replace("\\", "/") + "/" + jid + "-jss.xml"
order_path = path.replace(
"\\", "/") + "/" + jid + "_" + aid + "_order.xml"
mss = path.replace("\\", "/") + "/" + jid + "_" + aid + ".docx"
#mss = path+"/"+docx
#print('local path for mss2',mss2)
print('path for gen docx ', docx)
print('path for mss ', mss)
print('path for order ', order_path)
print('path for jss xml ', jss_path)
tp = 0
fp = 0
tn = 0
fn = 0
#def Batch_S5(folderPath,batch_Id):
# import time
# Start_time = time.time()
#
# stage = "UNIQ_S5"
# path=folderPath
# print(path)
# #batchId = batch_Id
# #path_split = path.split("_")
# path_split = path.split("\\")[-1].split("_")
# print(path_split,'this is the split path')
# #docx = "_".join(path_split[-3:-1])+".docx"
# docx = "_".join(path_split[-2:])+".docx"
# #copy2(path.replace("\\","/")+"/tx1.xml.log.xml",path.replace("\\","/")+"/UNIQ_S5.xml")
# file_path = path.replace("\\","/") + "/tx1.xml"
# print(file_path,'this is the file path')
# file = file_path.split('/')[-2].split("_")
# print(file,'this is the file path for error rectification')
# jid = file[-2]
# aid = file[-1]
# itemid = jid+"_"+aid+"_"+stage
# time1 = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
# counter = 1
# error_count = 0
# uid = socket.gethostname()
# exists = os.path.isfile(path.replace("\\","/")+"/UNIQ_S5.xml")
# jss_path = path.replace("\\","/")+"/"+jid+"-jss.xml"
# order_path = path.replace("\\","/")+"/"+jid+"_"+aid+"_order.xml"
# mss = path+"/"+docx
# tp=0
# fp=0
# tn=0
# fn=0
#print("starting path",path)
dest = r'E:\Desktop_files\TESTED FILES'
copy2(
path.replace("\\", "/") + "/tx1.xml.log.xml",
dest.replace("\\", "/") + "/temp_UNIQ_S5.xml")
#copy2(path+"\\tx1.xml.log.xml",path+"\\UNIQ_S5.xml")
#print("path",path)
log_path = dest.replace("\\", "/") + "/temp_UNIQ_S5.xml"
Master_list = [
itemid, uid, jid, aid, stage, time1, error_count, counter, time1,
time.time() - Start_time
]
insert_db_master(Master_list)
def log_transaction_db(obj, error_count, tp, tn, fp, fn):
append_log(obj, log_path)
tempdesc = ""
ss = ''
if 'error' in obj[0]:
ss = obj[0][-1]
error_count += len(obj)
tn += 1
else:
tp += 1
for sub in obj:
if "error" in sub:
#if(tempdesc==""):
# tn+=1
#if(tempdesc!="" and sub[-1]!=tempdesc):
# tn+=1
if (obj.index(sub) != 0):
ss = ss + ';' + sub[-1]
#tempdesc=sub[-1]
#error_count+=1
if 'error' in obj[0]:
trans_list = [
itemid, obj[0][1], obj[0][0], ss,
len(ss.split(';'))
]
insert_db_trans(trans_list)
return error_count, tp, tn, fp, fn
obj = Brf037a(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'second one', 'Vinay')
obj = Rule_17(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'third one', 'vipin')
obj = rule_15(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'fourth one', 'vipin')
obj = Rule_08(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'sixth one', 'preeti')
obj = Rule_93(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'eight one', 'preeti')
obj = Rule_91(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'ninth one', 'preeti')
obj = AUN009A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'eleventh one', 'shani')
obj = ack003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'twelveth one', 'shani')
obj = Rule_108(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'thirteenth one', 'preeti')
obj = article_history(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'fourteenth one', 'Vipin')
obj = rule_59(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'fifteenth one', 'vipin')
obj = highlights(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'sixteenth one', 'shani')
obj = rule_87(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'seventeenth one', 'preeti')
obj = Rule_89(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, 'eighteenth one', 'preeti')
obj = rule_HIS003A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '19th one', 'shani')
obj = rule_73(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '20th one', 'vipin')
obj = rule_66(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '21th one', 'vipin')
obj = rule_71(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '22th one', 'vipin')
obj = rule_67(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '23th one', 'vipin')
obj = aug001(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '24th one', 'shani')
obj = BRX011(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '25th one', 'rachit')
obj = Brx042(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '26th one', 'kumar')
obj = brx039(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '27th one', 'kumar')
obj = rule31(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '28th one', 'aditya')
obj = rule_afn003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '29th one', 'shani')
obj = brx029b(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '30th one', 'kumar')
#obj = Table(file_path)
#error_count,tp,tn,fp,fn,=log_transaction_db(obj,error_count,tp,tn,fp,fn)
#print(obj,'31th one','shani')
obj = BRX007(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '31th one', 'aditya')
obj = BRX006(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '32th one', 'aditya')
obj = Table(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '33th one', 'shani')
obj = BRX018(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '34th one', 'rachit')
obj = BRX005A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '35th one', 'aditya')
obj = brx030(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '36th one', 'kumar')
obj = brx041(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '37th one', 'kumar')
obj = BRX020A(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '38th one', 'rachit')
obj = brx032(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
## append_log(obj,log_path)
## for sub in obj:
## if "error" in sub:
## error_count+=1
## trans_list=[jid+"_"+aid+"_"+"S5", itemid, sub[1], sub[0], sub[-1], counter]
## insert_db_trans(trans_list)
print(obj, '39th one', 'kumar')
obj = BRX001a(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '40th one', 'aditya')
obj = BRX019(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '41th one', 'rachit')
obj = BRX024(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '42th one', 'rachit')
obj = BRF091B(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '43th one', 'vipin')
obj = BRX005B_35(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '44th one', 'vipin')
obj = aeu003(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '45th one', 'kumar')
obj = BRX020B(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '46th one', 'rachit')
obj = AFO002A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '47th one', 'preeti')
obj = BRX005B_36(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '48th and 49th one', 'vipin')
obj = tab018(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '50th ', 'kumar')
obj = XsTB701(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '51th', 'kumar')
obj = XsTB702(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '52th', 'kumar')
obj = BRX022(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '53th', 'rachit')
obj = Rule_114(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '54th', 'preeti')
obj = rule132(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '55th', 'rachit')
obj = rule40(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '56th', 'aditya')
obj = Rule_113(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '57th', 'preeti')
obj = rule41(file_path, mss, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '58th', 'aditya')
obj = table_header(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '59th', 'shani')
obj = rule_80(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '60th', 'vipin')
obj = rule_72(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '61th', 'vipin')
obj = rule_78_138(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '62th', 'vipin')
obj = Rule_111(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '63th', 'preeti')
obj = Rule_96(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '64th', 'preeti')
obj = label(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '65th', 'shani')
obj = keywords(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '66th', 'shani')
obj = Rule_109(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '67th', 'preeti')
#obj = Authors_Email(file_path, mss)
#error_count,tp,tn,fp,fn,=log_transaction_db(obj,error_count,tp,tn,fp,fn)
#print(obj,'68th','Vinay')
obj = Rule139_140(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '69th', 'aditya')
#obj = corr_check(file_path)
#error_count,tp,tn,fp,fn,=log_transaction_db(obj,error_count,tp,tn,fp,fn)
#print(obj,'70th','shani') commented shani acl003 (151 UFA)
obj = rule_75(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '71th', 'vipin')
obj = rule_69(file_path, mss, jss_path)
#print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$",obj)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '72th', 'vipin')
obj = rule120(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '73th', 'aditya')
obj = rule_70(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '74th', 'vipin')
obj = rule_79(file_path, jss_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '75th', 'vipin')
obj = rule_16(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '76th', 'vipin')
obj = rule_83(jss_path, file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '77th', 'vipin')
obj = tab021(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '78th', 'kumar')
obj = tab012(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '79th', 'kumar')
obj = XsFG601(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '80th', 'kumar')
obj = XsFM101(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '81th', 'vipin')
obj = rule135(file_path, order_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '82th', 'rachit')
# obj = rule144(mss, file_path)
# error_count,tp,tn,fp,fn,=log_transaction_db(obj,error_count,tp,tn,fp,fn)
# print(obj,'83th','rachit')
# obj = rule130(file_path,json_path)
# append_log(obj,log_path)
# if "error" in obj:
# error_count+=1
# trans_list=[jid+"_"+aid+"_"+"S5", itemid, obj[2], obj[0], obj[-2], counter]
# insert_db_trans(trans_list)
# print(obj,'84th','rachit')
obj = Rule_7(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '84th', 'preeti')
obj = rule116(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '85th', 'aditya')
obj = Rule_149(file_path, mss)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '86th', 'preeti')
obj = Tail(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '87th', 'shani')
obj = Rule_128(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '88th', 'preeti')
obj = CHE015_11(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '89th', 'rachit')
obj = fund(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '90th', 'vinay')
obj = AEU001A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '91th', 'vipin')
obj = BRX010_42(jss_path, file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '92th', 'vipin')
obj = CHE_045(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '93th', 'manisha')
obj = FUN_002(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '94th', 'manisha')
obj = FUN_002(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '94th', 'manisha')
obj = CHE011(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '95th', 'Mayank')
obj = Ellip(file_path, jss_path)
#print(obj)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '96th', 'Mayank')
obj = BRF033_(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '97th', 'Mayank')
obj = BRF037A(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '98th', 'Mayank')
obj = DQO002(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '99th', 'Mayank')
obj = CHE011_tri(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '100th', 'Mayank')
obj = APP001(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '101st', 'preeti')
obj = abs004(file_path)
error_count, tp, tn, fp, fn, = log_transaction_db(
obj, error_count, tp, tn, fp, fn)
print(obj, '102nd', 'preeti')
copy2(
dest.replace("\\", "/") + "/temp_UNIQ_S5.xml",
path.replace("\\", "/") + "/UNIQ_S5.xml")
os.remove(dest.replace("\\", "/") + "/temp_UNIQ_S5.xml")
updated_Master_list = [
itemid, error_count,
time.time() - Start_time, tp, tn, fp, fn, batch_Id, uid
]
update_db_master(updated_Master_list)
#logging.info('\n\n')
logging.info(
' **-------Logger End from here for {}-------** '.format(
itemid))
logging.info('\n\n')
logging.info('\n\n')
except Exception as e:
logging.info('==' * 50)
logging.exception('Got exception on main handler-----------')
logging.info('\n\n')
logging.shutdown()
return