def main(command, server_address, server_username, server_pass):
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=server_address,
username=server_username,
password=server_pass)
session = ssh.get_transport().open_session()
session.set_combine_stderr(True)
session.get_pty()
session.exec_command("sudo bash -c \"" + command + "\"")
stdin = session.makefile('wb', -1)
stdout = session.makefile('rb', -1)
stdin.write(server_pass + '\n')
stdin.flush()
#print(stdout.read().decode("utf-8"))
except Exception as e:
print("The following error has occurred during your requested process")
print(e.message)
emailer.raise_error(
"EDW encountered an error at {}".format(datetime.datetime.now()),
"The ETL YUM Cleaner encountered the following error: {}.".format(
e.message))
finally:
if ssh:
session.close()
ssh.close()
print('\n yum clean all and rm -rf /var/cache/yum run successfully.\n')
def main(command, server_address, server_username, server_pass):
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=server_address,
username=server_username,
password=server_pass)
session = ssh.get_transport().open_session()
session.set_combine_stderr(True)
session.get_pty()
session.exec_command(command)
stdin = session.makefile('wb', -1)
stdout = session.makefile('rb', -1)
out = stdout.read().decode("utf-8")
if len(out) > 0:
session.exec_command(restart)
ts = datetime.datetime.now()
print('Restart Required at %s') % ts
emailer.raise_error(
"EDW required a restart at {}".format(ts),
"The Autorestart job found the EDW server to be inactive and send the command for a restart as of {}."
.format(ts))
else:
ts = datetime.datetime.now()
print('Server Active as of %s') % ts
except Exception as e:
print("The following error has occurred during your requested process")
print(e.message)
emailer.raise_error(
"EDW Autorestart ERROR",
"The Autorestart job experienced the following error: {}".format(
e.message))
finally:
if ssh:
session.close()
ssh.close()
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname='80.88.187.155',
username='******',
password=keyring.get_password("talend", "benemen"))
stdin, stdout, stderr = ssh_client.exec_command(
'cd /etl/python_jobs && python error_log_monthly_copier.py')
today = datetime.today()
string = "Error Log Successfully stored and reset for {}-{}".format(
int(today.month) - 1, today.year)
print(string)
except Exception as e:
print("Error occurred:")
print(e)
finally:
if ssh_client:
ssh_client.close()
if __name__ == "__main__":
try:
with time_limit(600):
main()
except TimeoutException as e:
print("Log_Monthly_Backup time has exceeded the safety limit.")
emailer.raise_error(
"Log_Monthly_Backup ERROR",
"The Log_Monthly_Backup execution time has exceeded the 10 minute safety limit"
)
"EDW required a restart at {}".format(ts),
"The Autorestart job found the EDW server to be inactive and send the command for a restart as of {}."
.format(ts))
else:
ts = datetime.datetime.now()
print('Server Active as of %s') % ts
except Exception as e:
print("The following error has occurred during your requested process")
print(e.message)
emailer.raise_error(
"EDW Autorestart ERROR",
"The Autorestart job experienced the following error: {}".format(
e.message))
finally:
if ssh:
session.close()
ssh.close()
if __name__ == '__main__':
try:
with time_limit(180):
main(command, server_address, server_username, server_pass)
except TimeoutException as e:
print("Execution time has exceeded the safety limit.")
emailer.raise_error(
"EDW Autorestart ERROR",
"The Autorestart job execution time has exceeded the safety limit")
import time_limit
from time_limit import *
import emailer
ssh_client= None
def main():
try:
ssh_client=paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname='80.88.187.155',username='******',password=keyring.get_password("talend", "benemen"))
stdin, stdout, stderr = ssh_client.exec_command("/usr/bin/bash -lc '/etl/cronjobs/60minute_jobs.sh' 1>/dev/null 2>/etl/logs/60minute_jobs.txt")
out = stdout.read()
print(out)
except Exception as e:
print('The following error has occurred during the XRM_Jobs dkron process')
print(e.message)
finally:
if ssh_client:
ssh_client.close()
print('\n60Minute_Jobs complete\n')
if __name__ == '__main__':
try:
with time_limit(3720):
main()
except TimeoutException as e:
print("60minute_jobs time has exceeded the safety limit.")
emailer.raise_error("60minute_jobs ERROR","The 60minute_jobs execution time has exceeded the 62 minute safety limit")
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname='80.88.187.155',
username='******',
password='******')
stdin, stdout, stderr = ssh_client.exec_command(
"/usr/bin/bash -lc '/etl/cronjobs/daily_jobs.sh' 1>/dev/null 2>/etl/logs/daily_jobs.txt"
)
out = stdout.read()
print(out)
except Exception as e:
print(
'The following error has occurred during the XRM_Jobs dkron process'
)
print(e.message)
finally:
if ssh_client:
ssh_client.close()
print('\nDaily_Jobs complete\n')
if __name__ == '__main__':
try:
with time_limit(600):
main()
except TimeoutException as e:
print("daily_jobs time has exceeded the safety limit.")
emailer.raise_error(
"daily_jobs ERROR",
"The daily_jobs execution time has exceeded the 10 minute safety limit"
)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname='80.88.187.155',
username='******',
password='******')
stdin, stdout, stderr = ssh_client.exec_command(
"/usr/bin/bash -lc '/etl/cronjobs/weekly_push_jobs.sh' 1>/dev/null 2>/etl/logs/weekly_push_jobs.txt"
)
out = stdout.read()
print(out)
except Exception as e:
print(
'The following error has occurred during the XRM_Jobs dkron process'
)
print(e.message)
finally:
if ssh_client:
ssh_client.close()
print('\nWeekly_Push_Jobs complete\n')
if __name__ == '__main__':
try:
with time_limit(600):
main()
except TimeoutException as e:
print("weekly_push_jobs time has exceeded the safety limit.")
emailer.raise_error(
"weekly_push_jobs ERROR",
"The weekly_push_jobs execution time has exceeded the 10 minute safety limit"
)
import sshgit_etl_development
import sshgit_etl_BeneDW
import sshgit_QlikSense_BeneReports_development
import sshgit_QlikSense_BeneReports_production
import sshgit_esb_beneesb
import git_dkron
import sys
sys.path.append("/home/benemenadmin/PyResources") # Directory for Python classes
import time_limit
from time_limit import *
import emailer
def main():
git_BAP.main()
git_dkron.main()
sshgit_talend_jobs.main()
sshgit_etl_development.main()
sshgit_etl_BeneDW.main()
sshgit_QlikSense_BeneReports_development.main()
sshgit_QlikSense_BeneReports_production.main()
sshgit_esb_beneesb.main()
if __name__ == '__main__':
try:
with time_limit(300):
main()
except TimeoutException as e:
print("Auto Gitlab Uploader time has exceeded the safety limit.")
emailer.raise_error("Auto Gitlab Uploader ERROR","Auto Gitlab Uploader execution time has exceeded the 5 minute safety limit")
session.get_pty()
session.exec_command("sudo bash -c \"" + command + "\"")
stdin = session.makefile('wb', -1)
stdout = session.makefile('rb', -1)
stdin.write(server_pass + '\n')
stdin.flush()
#print(stdout.read().decode("utf-8"))
except Exception as e:
print("The following error has occurred during your requested process")
print(e.message)
emailer.raise_error(
"EDW encountered an error at {}".format(datetime.datetime.now()),
"The ETL YUM Cleaner encountered the following error: {}.".format(
e.message))
finally:
if ssh:
session.close()
ssh.close()
print('\n yum clean all and rm -rf /var/cache/yum run successfully.\n')
if __name__ == '__main__':
try:
with time_limit(180):
main(command, server_address, server_username, server_pass)
except TimeoutException as e:
print("Execution time has exceeded the safety limit.")
emailer.raise_error(
"ETL/YUM Cleaner ERROR",
"The ETL YUM Cleaner job execution time has exceeded the safety limit"
)
if __name__ == "__main__":
failed = multiprocessing.Queue()
counter = Value('i', 0) #JOB COUNTER
start_time = datetime.datetime.utcnow() #START TIME
task_id = uuid.uuid4()
#GREENPLUM CONNECTION AND TIMESTAMP
gp = GP_Connect()
t = Timestamp_Update()
#CREATES LIST FROM SQL TABLE AND DETERMINES # OF JOBS
paths = gp.main(
command=
"SELECT id, full_reload FROM benereports.v_app_reloads ORDER BY template_name,short_name;"
)
total_jobs = len(paths)
try:
with time_limit(
898
): #Set at 2 seconds less than your desired time as it will take approx that long to start the time limit
main(task_id)
except TimeoutException as e:
print("Timed out!")
subject = "Qlik App Reloader long execution time"
msg = "The Qlik App Reloader has exceeded the 15 minute execution time safety limit."
emailer.raise_error(subject, msg)
end_time = datetime.datetime.utcnow()
log_table(total_jobs, end_time, task_id)
