def connection(con_name=""):
data = Json_evaluation.readJSON(__guidePath__, __connectionFile__)
if data[con_name]["connType"] == "MySql":
try:
con = pymysql.connect(data[con_name]["server"],
data[con_name]["user"],
data[con_name]["password"],
data[con_name]["database"],
charset=data[con_name]["charset"],
use_unicode=data[con_name]["use_unicode"],
port=data[con_name]["port"])
except Exception as e:
log("Error_sqlConn_mySQLCon@" + e)
return con
elif data[con_name]["connType"] == "MS Sql Server":
data = Json_evaluation.readJSON(__guidePath__, __connectionFile__)
try:
con = pymssql.connect(data[con_name]["server"],
data[con_name]["user"],
data[con_name]["password"],
data[con_name]["database"],
port=data[con_name]["port"])
except Exception as e:
log("Error_sqlConn_msSQLCon@" + e)
return con
def deleteJson(fileName, key):
try:
if fileName == 'Jobs':
Json_evaluation.removeJson(key=key,
filename=__jobFile__,
path=__path__)
return "Job deleted Successfully!!"
elif fileName == 'Connections':
Json_evaluation.removeJson(key=key,
filename=__connectionFile__,
path=__path__)
return "Connection deleted Successfully!!"
elif fileName == 'Parameters':
Json_evaluation.removeJson(key=key,
filename=__parameterFile__,
path=__path__)
return "Parameter deleted Successfully!!"
elif fileName == 'Email':
Json_evaluation.removeJson(key=key,
filename=__emailFile__,
path=__path__)
return "Email SMTP deleted Successfully!!"
elif fileName == 'Step':
Json_evaluation.removeJson(key=key,
filename=__stepsFile__,
path=__path__)
return key + " Step deleted Successfully!!"
except Exception as e:
return str(e), 500
def setJob(job_dict,
isNew=0,
path=__guidePath__,
histoyPath=__historyPath__,
logPath=__logPath__):
try:
Json_evaluation.updateJson(job_dict, __jobFile__, path=path)
if isNew == 1:
for x in job_dict.keys():
Json_evaluation.writeText(
filename=str(x) + ".txt",
path=histoyPath,
data=Generic.prefixTimeStamp("Job Created!!"))
log("Adding " + str(job_dict.keys()) + " Job", path=logPath)
else:
log("Update made on " + str(job_dict.keys()) + " Job",
path=logPath)
for x in job_dict.keys():
History.addJobHistory(filename=str(x + ".txt"),
path=histoyPath,
data=Generic.prefixTimeStamp(
str("Job has been updated!!")))
pass
except Exception as e:
log("Error_Jobs_setJob@" + str(e), path=logPath)
def addSync(sync_dict):
try:
Json_evaluation.updateJson(dict=sync_dict,filename=__syncFile__)
log("Adding "+str(sync_dict.keys())+" Parameter ")
pass
except Exception as e:
log("Error_Sync_addSync@"+str(e))
def addSmpt(smpt_dict, path=__guidePath__, logPath=__logPath__):
try:
Json_evaluation.updateJson(dict=smpt_dict,
filename=__emailFile__,
path=path)
log("Adding " + str(smpt_dict.keys()) + " smpt ", path=logPath)
pass
except Exception as e:
log("Error_Email_addSmpt@" + str(e), path=logPath)
def addStep(jobName, step_dict, path=__guidePath__, logPath=__logPath__):
try:
Json_evaluation.updateJson(step_dict, __stepsFile__, path=path)
log("Adding " + str(step_dict.keys()) + " step under job " +
jobName,
path=logPath)
pass
except Exception as e:
log("Error_Jobs_addStep@" + str(e))
def updateParamValue(paramName="Willdo", value=""):
try:
data = Json_evaluation.getJsonByKey(key=paramName,
filename=__parameterFile__)
data["paramValue"] = value
data = {paramName: data}
Json_evaluation.updateJson(dict=data, filename=__parameterFile__)
except Exception as e:
log("Error_Parameter_updateParamValue@" + str(e))
def addParam(param_dict, path=__guidePath__, logPath=__logPath__):
try:
Json_evaluation.updateJson(dict=param_dict,
filename=__parameterFile__,
path=path)
log("Adding " + str(param_dict.keys()) + " Parameter ",
path=logPath)
pass
except Exception as e:
log("Error_Parameter_addParam@" + str(e))
def getSchedulerDetail():
try:
remoteDict = Json_evaluation.getJsonByKey(filename=__syncFile__,
key="susServer",
path=__path__)
Sync.getSchedulerStatus(remoteDict, path=__path__, logPath=__logPath__)
return jsonify(
Json_evaluation.readJSON(path=__path__,
filename=__schedulerTimeStampFile__))
except Exception as e:
return str(e), 500
def getSchedulerStatus(remote_dict,path=__guidePath__,logPath=__logPath__):
try:
#dateTime=Generic.getDateTime()
status="OFF"
#print(len(Sync.remoteSearchSchedulerCall(remote_dict,logPath=logPath)))
if len(Sync.remoteSearchSchedulerCall(remote_dict,logPath=logPath))>1:
status="ON"
#print(status)
Json_evaluation.updateJson({"schedulerStatus":status},filename=__schedulerTimeStampFile__,path=path)
except Exception as e:
log("Error_setSchedulerTimeStap"+str(e),path=logPath)
def lastRunUpdate(jobName):
try:
data = Json_evaluation.getJsonByKey(filename=__jobFile__,
key=jobName)
data = Job.updateNextRun(data)
data["lastRunDate"] = Generic.getDate()
n_data = {}
n_data[jobName] = data #Next scheduled time update
Json_evaluation.updateJson(dict=n_data, filename=__jobFile__)
log("Job " + jobName + " last Run Updated!!")
except Exception as e:
log("Error_Jobs_lastRunUpdate@" + str(e))
def setSchedulerTimeStap():
try:
dateTime = Generic.currentTimestamp()
Json_evaluation.updateJson(
{
"schedulerTimeStamp": str(dateTime),
"schedulerStatus": "ON"
},
filename=__schedulerTimeStampFile__)
log("Scheduler Restarted at " + str(dateTime))
except Exception as e:
log("Error_setSchedulerTimeStap" + str(e))
def insertConnection(con_dict, path=__guidePath__, logPath=__logPath__):
try:
Json_evaluation.updateJson(con_dict,
filename=__connectionFile__,
path=path)
#data=Json_evaluation.readJSON(filename=__connectionFile__,path=path)
#data = {"Con_1":{"server":"94.156.144.217","user":"******","password":"******","database":"hotels","charset":"utf8","port":3306,"use_unicode":"True"}}
#data.update(con_dict)
#Json_evaluation.writeJSON(data=data,filename=__connectionFile__,path=path)
log("Adding " + str(con_dict.keys()) + " Connetions", path=logPath)
pass
except Exception as e:
print("Error_insertConnection_insertConnection@" + str(e))
def getInbuiltParam(stepName, path=__guidePath__):
try:
stepData = Json_evaluation.getJsonByKey(key=stepName,
filename=__stepsFile__,
path=path)
if str(stepData["stepType"]) in "1,4":
print(stepData["stepType"])
stepStr = stepData["statement"]
index = stepStr.find("@")
inbulitParaData = []
while index != -1:
tempStr = stepStr[index + 1:]
end = str(tempStr).find("@")
inbulitParaData.insert(0,
stepStr[index + 1:index + end + 1])
stepStr = stepStr[index + end + 2:]
index = stepStr.find("@")
return inbulitParaData
except Exception as e:
log("Error_getInbuitParam@" + str(e))
def getJobQueue():
#return ""
try:
return jsonify(
Json_evaluation.readJSON(path=__path__, filename=__jobQueue__))
except Exception as e:
return str(e), 500
def getConList():
try:
return jsonify(
Json_evaluation.readJSON(path=__path__,
filename=__connectionFile__))
except Exception as e:
return str(e), 500
def getParamList():
try:
return jsonify(
Json_evaluation.readJSON(path=__path__,
filename=__parameterFile__))
except Exception as e:
return str(e), 500
def getJson(file, key):
try:
#return ""
#if file=='Jobs':
if file == 'history':
return jsonify(
History.readJobHistory(path=__historyPath__, jobName=key))
elif file == 'Log':
return jsonify(
Json_evaluation.readFile(path=__logPath__ + "/log.txt"))
else:
return jsonify(
Json_evaluation.getJsonByKey(key=str(key),
path=__path__,
filename=file + ".json"))
except Exception as e:
return str(e), 500
def prepareJobQueue():
try:
todayList = {}
data = Job.evaluateJob()
#print(data)
curDate = Generic.strToDate(Generic.getDate())
curTime = Generic.strToTime(Generic.getTime())
#print(curTime)
for key in data.keys():
timeKeyword = ""
lastRunDate = data[key]["lastRunDate"]
if data[key]["lastRunDate"] == "-1":
timeKeyword = "scheuledTime"
else:
timeKeyword = "nextRunTime"
if Generic.strToDate(
data[key]
["startDate"]) <= curDate and Generic.strToDate(
data[key]["endDate"]) >= curDate and (
data[key]["isActive"] == 1
or data[key]["isActive"] == "on") and (
lastRunDate == "-1" or Job.isJobEligibleToRun(
int(data[key]["interval"]), curDate,
curTime, data[key][timeKeyword])
== 1) and Generic.strToTime(
data[key][timeKeyword]) >= curTime:
todayList[key] = {
"scheduledTime":
data[key][timeKeyword],
"remainingSec":
Generic.timeDiff(Generic.getTime(),
data[key][timeKeyword]),
"nextRunDate":
"-" if timeKeyword == "scheuledTime" else
data[key]['nextRunDate'],
"nextRunTime":
"-" if timeKeyword == "scheuledTime" else
data[key]['nextRunTime']
}
#print(todayList)
Json_evaluation.writeJSON(data=todayList, filename=__jobQueue__)
log("Job Queue Prepared....")
except Exception as e:
log("Error_Jobs_prepareJobQueue@" + str(e))
def getParamByJob(jobName, path=__guidePath__):
try:
data = Json_evaluation.readJSON(filename=__parameterFile__,
path=path)
params = []
for key in data.keys():
if data[key]["jobName"] == jobName:
params.insert(0, key)
return params
except Exception as e:
log("Error_Parameter_getParamByJob@" + str(e))
def sendEmail(step_data):
#context = ssl.create_default_context()
# Try to log in to server and send email
try:
data = Json_evaluation.getJsonByKey(
filename=__emailFile__, key=step_data["emailProfilerName"])
#print(step_data[data["attachmentFilePath"]])
server = smtplib.SMTP(data["emailServer"], int(data["emailPort"]),
None, 30)
#import traceback
#traceback.print_exc()
msg = MIMEMultipart()
msg['From'] = data["emailUser"]
msg['To'] = step_data["receiver_emails"]
msg['Subject'] = "Willdo-Automated generated email!!"
body = str(step_data["emailMsg"])
msg.attach(MIMEText(body, 'html'))
attachment = ""
#for attachPath in Generic.removeLastSperator(step_data["attachmentFilePath"]).split('|'):
attachPath = step_data["attachmentFilePath"]
part = MIMEBase('application', 'octet-stream')
try:
attachment = open(attachPath, "rb")
except Exception as ee:
log("Attachment block" + str(ee))
if attachment != "":
#print(attachPath)
part.set_payload(attachment.read())
attachment.close()
part.add_header(
'Content-Disposition', "attachment; filename= %s" %
Generic.getFileNameFromPath(attachPath))
encoders.encode_base64(part)
msg.attach(part)
text = msg.as_string()
server.ehlo() # Can be omitted
server.starttls() # Secure the connection
server.ehlo() # Can be omitted
server.login(data["emailUser"], data["emailPassword"])
server.sendmail(data["emailUser"],
str(step_data["receiver_emails"]).split(','), text)
except smtplib.SMTPException as e:
log("Error_Email_sendEmail@SMTPException_" +
str(e)) # Didn't make an instance.
except smtplib.socket.error:
log("Error_Email_sendEmail@Socket_" + str(e))
finally:
server.quit()
def validateJobName(jobName):
try:
data = Json_evaluation.getJsonByKey(filename=__jobFile__,
key=jobName)
if data != -1:
return 0
return 1
#log("Job "+jobName+" last Run Updated!!")
except Exception as e:
log("Error_Jobs_validateJobName@" + str(e))
def getStepsByJob(job_name, path=__guidePath__):
try:
#print(job_name)
steps = []
data = Json_evaluation.readJSON(filename=__stepsFile__, path=path)
for key in data.keys():
if str(key.split('|')[0]).strip() == str(job_name).strip():
steps.insert(int(data[key]["stepNo"]), key)
return steps
except Exception as e:
log("Error_Jobs_getStepsByJob@" + str(e))
def validateConName(conName):
try:
data = Json_evaluation.getJsonByKey(filename=__connectionFile__,
key=conName)
if int(data) != -1:
return 0
return 1
#log("Job "+jobName+" last Run Updated!!")
except Exception as e:
log("Error_insertConnection_validateJobName@" + str(e))
return 0
def evaluateJob(path=__guidePath__):
try:
#step_data=Json_evaluation.readJSON(filename=__stepsFile__)
return Json_evaluation.readJSON(filename=__jobFile__, path=path)
#print(job_data["conName"])
#con_data=Json_evaluation.readJSON(filename=__connectionFile__)
#sjc_data={**step_data,**job_data,**con_data}
#print(sjc_data)
pass
except Exception as e:
log("Error_Job_evaluateJob@" + str(e))
def getParamByStep(data):
try:
paramData = {}
if len(data["parameter"]) > 0:
params = data["parameter"].split('|')
paramFlag = 1
if len(params) > 1:
for key in params:
paramDataTemp = Json_evaluation.getJsonByKey(
filename=__parameterFile__, key=key[:-1])
paramData[key] = paramDataTemp
else:
paramDataTemp = Json_evaluation.getJsonByKey(
filename=__parameterFile__, key=params[0][:-1])
paramData = {params[0]: paramDataTemp}
return paramData, params
except Exception as e:
log("Error_Parameter_getParamByStep@" + str(e))
def getParamOptions(stepName, path=__guidePath__, logPath=__logPath__):
try:
#data=Json_evaluation.getJsonByKey(key=stepName,filename=__stepsFile__,path=path)
optionData = {}
paramOptionData = Json_evaluation.readJSON(
filename=__paramOptionFile__, path=path)
stepData = Json_evaluation.getJsonByKey(key=stepName,
filename=__stepsFile__,
path=path)
stepType = stepData['stepType']
for paramKey in paramOptionData.keys():
if str(stepType) in str(paramOptionData[paramKey]['stepType']):
optionData[paramOptionData[paramKey]['name']] = paramKey
return optionData
except Exception as e:
log("Error_Parameter_getParamOptions@" + str(e))
return "-1"
async def check_pending_notifications():
try:
#con_dict={"con_4":{"server": "94.156.144.217", "user": "******", "password": "******", "database": "hotels", "charset": "utf8", "port": 3306, "use_unicode": "True","connType":"mysql"}}
#insertConnection(con_dict)
#job={"JOB3":{"conName":"","jobStatement":"","interval":0,"scheuledTime":"","description":"","type":"","startDate":"","endDate":"17/04/2019","isActive":1,"lastRunDate":"-1"}}
#Job.setJob(job)
#step={"step1":{"jobName":"","stepNo":1,"statement":"","interval":"","stepType":"urlDownload","isActive":1,"parameter":"p1=2|p2=5|p3='ddsd'"}}
#addStep("JOB1",step)
#print()
#getResult("CALL `getEmailTemplate`();")
#ExecuteStep.executeStep("Willdo|step1")
#Url.callApi()
Job.prepareJobQueue()
#Job.getNearestJob()
#Job.executeJob(Job.getNearestJob())
#nearestJob=Job.getNearestJob() #Disabled in v3.0
#nearestJobName=Json_evaluation.getJsonByKey(filename=__jobQueue__,key=nearestJob) #Disabled in v3.0
readyQueue = Json_evaluation.readJSON(filename=__jobQueue__)
for nearestJobName in readyQueue.keys():
if readyQueue[nearestJobName]['remainingSec'] == 0:
jobThread = threading.Thread(target=executeJob,
args=(
nearestJobName,
readyQueue,
))
jobThread.daemon = True
jobThread.start()
#print(data)
#print(data["remainingSec"])
#Job.lastRunUpdate(nearestJob)
#filename=""
#if email_template[0][8] == 1:
# filename = getFilename(email_template[0][11])
#send_report(filename)
pass
except Exception as e:
print("#Warning: No Job is Pending." + str(e))
def getNearestJob(): #Disabled in v3.0
try:
data = Json_evaluation.readJSON(filename=__jobQueue__)
#print(data)
jobCache = ""
curDate = Generic.strToDate(Generic.getDate())
lesserTime = -1
for key in data.keys():
if lesserTime == -1 and Generic.strToTime(
data[key]["scheduledTime"]) >= Generic.strToTime(
Generic.getTime()):
lesserTime = data[key]["remainingSec"]
jobCache = key
if lesserTime > int(
data[key]["remainingSec"]) and Generic.strToTime(
data[key]["scheduledTime"]) >= Generic.strToTime(
Generic.getTime()):
lesserTime = data[key]["remainingSec"]
jobCache = key
return jobCache
except Exception as e:
log("Error_Jobs_getNearestJob@" + str(e))
def submitForm(formName, masterName):
if request.method == 'POST':
result_dict = request.form.to_dict(flat=True)
for k in result_dict.keys():
data = k.replace("'", "\"")
data = json.loads(data)
masterKey = data[masterName]
data = {masterKey: data}
try:
if formName == 'Jobs':
Job.setJob(data,
isNew=1,
path=__path__,
histoyPath=__historyPath__,
logPath=__logPath__)
return "Job Added Successfully!!"
elif formName == 'JobUpdate':
Job.setJob(data,
isNew=0,
path=__path__,
histoyPath=__historyPath__,
logPath=__logPath__)
return "Job updated Successfully!!"
elif formName == 'newCon':
insertConnection(data, path=__path__, logPath=__logPath__)
return "Connection Created Successfully!!"
elif formName == 'updateCon':
insertConnection(data, path=__path__, logPath=__logPath__)
return "Connection updated Successfully!!"
elif formName == 'newParam':
Parameter.addParam(data, path=__path__, logPath=__logPath__)
return "Parameter created Successfully!!"
elif formName == 'updateParam':
Parameter.addParam(data, path=__path__, logPath=__logPath__)
return "Parameter updated Successfully!!"
elif formName == 'newEmail':
Email.addSmpt(data, path=__path__, logPath=__logPath__)
return "Email created Successfully!!"
elif formName == 'updateEmail':
Email.addSmpt(data, path=__path__, logPath=__logPath__)
return "Email updated Successfully!!"
elif formName == 'newStep':
jobName = data[masterKey]["jobName"]
data = {
str(data[masterKey]["jobName"] + "|" + data[masterKey]['stepName']):
data[masterKey]
}
Job.addStep(str(jobName),
data,
path=__path__,
logPath=__logPath__)
return "Step Created Successfully under " + jobName + " !!"
elif formName == 'manageStep':
jobName = data[masterKey]["jobName"]
data = {
str(data[masterKey]["jobName"] + "|" + data[masterKey]['stepName']):
data[masterKey]
}
#print(data)
Job.addStep(str(jobName),
data,
path=__path__,
logPath=__logPath__)
return "Step Updated Successfully under " + jobName + " !!"
elif formName == 'assigneParam':
stepData = Json_evaluation.getJsonByKey(key=masterKey,
filename=__stepsFile__,
path=__path__)
print(stepData)
if str(stepData["parameter"]) == "":
stepData["parameter"] = str(
data[masterKey]['paramName']) + str(
data[masterKey]['paramOption'])
else:
stepData["parameter"] += "|" + str(
data[masterKey]['paramName']) + str(
data[masterKey]['paramOption'])
stepData = {masterKey: stepData}
Json_evaluation.updateJson(dict=stepData,
filename=__stepsFile__,
path=__path__)
return "Parameter Assined Successfully!!"
except Exception as e:
print(str(e))
return "<h2>Success</h2>"