def ReplaceOldWithNewFile(orig_file='', new_temp_file=''):
"""
Compare original file and the new temp file ( contents and permissions).
If they are the same, just remove the temp version. ( maybe not needed, handled in calling function)
If they are different, backup original and then replace teh orig_file with the new_temp_file
Return code values:
0: No changes made
1: Changes made
"""
## Well I've broken this by moving it to another module.
## I need to read up on logging, and logging config files
# https://docs.python.org/3.8/howto/logging.html#configuring-logging
import os
import time
import shutil
# If file exists,
try:
type(logMessage) ## check if logMessage is already set up
except:
from general_functions import LogMessage
logMessage = LogMessage() # test just using default log file
if os.path.exists(orig_file):
import filecmp
#content_matches=filecmp.cmp(orig_file,new_temp_file)
#permission_matches=os.stat(orig_file).st_mode == os.stat(new_temp_file).st_mode
#user_matches=os.stat(orig_file).st_uid == os.stat(new_temp_file).st_uid
#group_matches=os.stat(orig_file).st_gid == os.stat(new_temp_file).st_gid
orig_permission = os.stat(orig_file).st_mode
new_permission = os.stat(new_temp_file).st_mode
orig_user = os.stat(orig_file).st_uid
new_user = os.stat(new_temp_file).st_uid
orig_group = os.stat(orig_file).st_gid
new_group = os.stat(new_temp_file).st_gid
content_matches = filecmp.cmp(orig_file, new_temp_file)
permission_matches = orig_permission == new_permission
user_matches = orig_user == new_user
group_matches = orig_group == new_group
logMessage.info('Checking file ' + orig_file + '. content_matches:' +
str(content_matches) + '; permission_matches:' +
str(permission_matches) + '; user_matches:' +
str(user_matches) + '; group_matches:' +
str(group_matches))
if content_matches and permission_matches and user_matches and group_matches:
logMessage.info(orig_file + ' is unchanged.')
os.remove(new_temp_file)
return 0
else:
logMessage.info(orig_file + 'Permission: ' + str(orig_permission) +
', owner: ' + str(orig_user) + ', group :' +
str(orig_group))
logMessage.info(new_temp_file + 'Permission: ' +
str(new_permission) + ', owner: ' + str(new_user) +
', group :' + str(new_group))
# backup the original file
t = time.localtime()
backupfile = orig_file + time.strftime('%Y%m%d_%H%M%S', t)
shutil.copyfile(orig_file, backupfile)
else:
logMessage.info(orig_file + ' - does not exist. Creating new file.')
## Only got to here if does not match (ie new or different)
logMessage.info(orig_file + ' - has been amended. ( to match ' +
new_temp_file + ' )')
#shutil.copyfile(new_temp_file, orig_file)
shutil.move(new_temp_file, orig_file)
## On some test servers this doesn't end up with correct ownership and permissions. So adding this to get round it
try:
os.chown(orig_file, new_user, new_group)
except PermissionError:
logMessage.error('Unable to set ownership on ' + orig_file +
'. Trying to change to chown ' + new_user + ':' +
new_group())
try:
os.chmod(orig_file, new_permission)
except:
logMessage.error('Unable to set permissions on ' + orig_file +
'. Trying to change to chmod ' + orig_file + ' ' +
new_permission)
return 1
def main():
## Main line
## This script has been coded with 3.7.
import sys
if sys.version_info < (3, 7):
sys.exit("\nThis script requires python 3.7 or higher.\n")
args = HandleInputParameters()
#args = parser.parse_args()
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
global logMessage
logMessage = LogMessage()
setLogFile(args.logfile)
logMessage.info("Starting the compile script.")
url = 'https://' + args.serviceTierHostName + ':' + args.isPort + '/ibm/iis/api/dscdesignerapi'
p = {
"api": "exportAssets",
"engineName": args.hostName,
"domain": args.serviceTierHostName,
"projectName": args.projectName,
"replace": 'yes',
"includeDesign": 'yes',
"remoteFilePath":
'/var/git/test1_repo/code_sheff/DataStageJobsTest1/dstage1/SteTest/RunCommand4/RunCommand4.isx',
"apiVersion": "3.0"
}
user = args.user
password = args.password
# Do as multiple requests in one session, so only needs to authenticate once.
import requests
from requests.auth import HTTPBasicAuth
#response = requests.get('https://127.0.0.1:9443/ibm/iis/api/dscdesignerapi', params={"api": "compileDSJob", "jobName": "Job_1", "projectName" : args.projectName , "hostName" : args.hostName , "getFullOutput" : "False", "apiVersion" : "3.0"}, auth = HTTPBasicAuth(user,password),verify=False )
#jobs=['Job_1','Job_2','Job_3']
jobs = args.job_list
with requests.Session() as s:
ssl_warned_already = False
import warnings
import urllib3
#urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#warnings.filterwarnings('error',"InsecureRequestWarning")
warnings.filterwarnings(
'error'
) # Not sure how to specify just the warning we're interested in.
for job in jobs:
#p["jobName"] = job
try:
response = s.get(
url,
params=p,
auth=HTTPBasicAuth(user, password),
verify=False) #Need to look into sorting out the ssl here.
except urllib3.exceptions.InsecureRequestWarning as e:
# Log a message, then Switch warnings off for this warning, and get the response. This might mean we're actually doing the compile 2x here but anyway, that's fine for now.
# Need to switch warnings back on after
if ssl_warned_already == False:
logMessage.warning(str(e))
ssl_warned_already = True
warnings.filterwarnings('default')
urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning)
response = s.get(
url,
params=p,
auth=HTTPBasicAuth(user, password),
verify=False) #Need to look into sorting out the ssl here.
except Exception as e:
logMessage.error(
'Exception occured. Exception is :' + str(e) +
'. Check if url ( hostname and port) is correct.')
break
if response.status_code != 200:
if response.status_code == 401:
logMessage.error(
'Unauthorized for url : ' + url +
'. Check you have entered username and password correctly.'
)
break
if response.status_code == 404:
logMessage.error(
'status code 404 probably means something. for url : '
+ url +
'. Check you have entered username and password correctly.'
)
break
if response.json()['succeeded'] == True:
logMessage.info(job + ' imported ok.')
else:
logMessage.warning(
job + ' import FAILED.' +
str(response.json()['failureMessage']) +
' ( If "Null RID" then the combination of hostName,project and jobname does not exist.) '
)
warnings.filterwarnings('default')
def main(arrgv=None):
## Main line
## This script has been coded with 3.7., but I stuck im a workaround for 3.6
import sys
if sys.version_info < (3,6):
sys.exit("\nThis script requires python 3.6 or higher.\n")
## The test ... If I can't read this and understand what's going on ...then it needs re-writing
## Any function should fit on 1 screen ( ish) ( ideally)
## Each function should be easily described so we know what it does
import os
# Get input paramaters
parser=HandleInputParameters()
global args
args = parser.parse_args()
#args = parser.parse_known_args()
#import MyLogging.mylogging
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
global logMessage
logMessage=LogMessage()
setLogFile(args.logfile)
#Check input params
errorfound=False
#if not os.path.exists(args.template_dsparam):
# errorfound=True
# logMessage.info('Template dsenv file not found at: ' + args.template_dsparam)
if not os.path.exists(args.install_base):
logMessage.warning('Install base does not existat: ' + args.install_base)
if errorfound:
sys.exit("\nInput parameter failed validation. See previous messages.\n")
if args.modified_since_timestamp is None:
logMessage.info('No value for --since-timestamp . So will export all components')
logMessage.info('logfile is :' + args.logfile)
logMessage.info('install_base is :' + args.install_base )
if not os.geteuid() == 0 :
sys.exit("\nOnly root can run this script\n")
## Main Code
#from datastage_functions import GetListOfComponentsRecentlyModified
import general_functions
#import datastage_functions
from datastage_functions import GetListOfComponentsRecentlyModified
components_list=GetListOfComponentsRecentlyModified(modified_since=args.modified_since_timestamp)
#from datetime import datetime
import datetime
components_list_test={
('HN01', 'dstage1'): [
('JobDefn', 'ExternalSource', datetime.datetime(2021, 1, 7, 17, 21, 34, 685000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand2', datetime.datetime(2021, 4, 6, 12, 0, 48, 589000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand3', datetime.datetime(2021, 4, 8, 16, 4, 20, 516000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand4', datetime.datetime(2021, 4, 9, 15, 11, 28, 998000, tzinfo=datetime.timezone.utc), '\\\\SteTest'),
('JobDefn', 'RunCommand5', datetime.datetime(2021, 4, 15, 16, 28, 9, 708000, tzinfo=datetime.timezone.utc), '\\\\SteTest\\\\SubFolder1')]
}
#components_list=components_list_test
print(components_list)
def ExportComponentList(export_base_dir='/var/repo_base/default', components_list={}):
import os
import re
for project_namespace_tuple, job_details in components_list.items():
engine_host=project_namespace_tuple[0]
project=project_namespace_tuple[1]
logMessage.info('Exporting Jobs from ' + engine_host + ':/' + project )
for job_info in job_details:
logMessage.info('Exporting Jobs ' + job_info[3] )
#Create path for archive file
component_name=job_info[1]
category=re.sub(r"\\\\",os.sep,job_info[3]) # Convert to using os.sep ('/')
category_path=re.sub(r"^"+os.sep,'',category) # Remove the leading '/'
archive_dir_path=os.path.join(export_base_dir,project,category_path,component_name)
if ( not(os.path.exists(archive_dir_path))):
os.makedirs(archive_dir_path,0o755)
archive_file=component_name + '.isx'
archive_path=os.path.join(archive_dir_path, archive_file)
# Export the job
import subprocess
ds_pattern=engine_host + '/' + project + '/' + '*' + '/'+ component_name + '.*'
command_to_run_old='/iis/01/InformationServer/Clients/istools/cli/istool.sh export -archive ' + archive_path + ' -up -ds "hn01/dstage1/*/'+component_name + '.*" -u isadmin -p default;'
command_to_run='/iis/01/InformationServer/Clients/istools/cli/istool.sh export -archive ' + archive_path + ' -up -ds "' +ds_pattern + '" -u isadmin -p default;'
## Annoying
import sys
if sys.version_info >= (3,7):
result = subprocess.run([command_to_run] , capture_output=True, shell=True, encoding="UTF-8")
else:
result = subprocess.run([command_to_run] , shell=True, encoding="UTF-8", stdout=subprocess.PIPE)
if result.returncode != 0:
logMessage.warning('Export of ' + component_name + ' ended with return code ' + str(result.returncode) + '. stdout : ' + result.stdout + '. stderr: ' + result.stderr )
else:
logMessage.debug('Export of ' + component_name + ' ended with return code ' + str(result.returncode) + '. stdout : ' + result.stdout + '. stderr: ' + str(result.stderr) )
logMessage.info('Export to ' + archive_path + ' finished .')
ExportComponentList(export_base_dir='/var/git/test1_repo/code_sheff/DataStageJobsTest1', components_list=components_list)
logMessage.info('Done')