def main():
## Main line
## This script has been coded with 3.7.
import sys
if sys.version_info < (3, 7):
sys.exit("\nThis script requires python 3.7 or higher.\n")
args = HandleInputParameters()
#args = parser.parse_args()
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
global logMessage
logMessage = LogMessage()
setLogFile(args.logfile)
logMessage.info("Starting the compile script.")
url = 'https://' + args.serviceTierHostName + ':' + args.isPort + '/ibm/iis/api/dscdesignerapi'
p = {
"api": "exportAssets",
"engineName": args.hostName,
"domain": args.serviceTierHostName,
"projectName": args.projectName,
"replace": 'yes',
"includeDesign": 'yes',
"remoteFilePath":
'/var/git/test1_repo/code_sheff/DataStageJobsTest1/dstage1/SteTest/RunCommand4/RunCommand4.isx',
"apiVersion": "3.0"
}
user = args.user
password = args.password
# Do as multiple requests in one session, so only needs to authenticate once.
import requests
from requests.auth import HTTPBasicAuth
#response = requests.get('https://127.0.0.1:9443/ibm/iis/api/dscdesignerapi', params={"api": "compileDSJob", "jobName": "Job_1", "projectName" : args.projectName , "hostName" : args.hostName , "getFullOutput" : "False", "apiVersion" : "3.0"}, auth = HTTPBasicAuth(user,password),verify=False )
#jobs=['Job_1','Job_2','Job_3']
jobs = args.job_list
with requests.Session() as s:
ssl_warned_already = False
import warnings
import urllib3
#urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#warnings.filterwarnings('error',"InsecureRequestWarning")
warnings.filterwarnings(
'error'
) # Not sure how to specify just the warning we're interested in.
for job in jobs:
#p["jobName"] = job
try:
response = s.get(
url,
params=p,
auth=HTTPBasicAuth(user, password),
verify=False) #Need to look into sorting out the ssl here.
except urllib3.exceptions.InsecureRequestWarning as e:
# Log a message, then Switch warnings off for this warning, and get the response. This might mean we're actually doing the compile 2x here but anyway, that's fine for now.
# Need to switch warnings back on after
if ssl_warned_already == False:
logMessage.warning(str(e))
ssl_warned_already = True
warnings.filterwarnings('default')
urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning)
response = s.get(
url,
params=p,
auth=HTTPBasicAuth(user, password),
verify=False) #Need to look into sorting out the ssl here.
except Exception as e:
logMessage.error(
'Exception occured. Exception is :' + str(e) +
'. Check if url ( hostname and port) is correct.')
break
if response.status_code != 200:
if response.status_code == 401:
logMessage.error(
'Unauthorized for url : ' + url +
'. Check you have entered username and password correctly.'
)
break
if response.status_code == 404:
logMessage.error(
'status code 404 probably means something. for url : '
+ url +
'. Check you have entered username and password correctly.'
)
break
if response.json()['succeeded'] == True:
logMessage.info(job + ' imported ok.')
else:
logMessage.warning(
job + ' import FAILED.' +
str(response.json()['failureMessage']) +
' ( If "Null RID" then the combination of hostName,project and jobname does not exist.) '
)
warnings.filterwarnings('default')
#!/usr/bin/env python3
#print('Hello')
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
import os
global logMessage
logMessage = LogMessage()
#test = getLogFile()
if getLogFile() is None:
setLogFile(
os.path.join(os.sep, 'tmp', 'logs', 'stetest_datastage_functions.log'))
def GetValueFromVersionXML(
VersionXMLPath='/iis/test/InformationServer/Version.xml',
VariableName='datastage.user.name'):
"""
return value for 'VariableName' in Version.xml file
"""
# Define versionXMLValue as global, so that we can cache the results
global versionXMLValue
try:
versionXMLValue[VariableName]
except (NameError, KeyError):
versionXMLValue = GetValuesFromVersionXML(
VersionXMLPath=VersionXMLPath)
#!/usr/bin/env python3
"""
This will be used to store functions that are generally of use in our scripting.
This can be used to help give a standard behaviour to all our scripts.
"""
#import MyLogging.mylogging
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
import os
global logMessage
logMessage = LogMessage()
#test = getLogFile()
if getLogFile() is None:
setLogFile(
os.path.join(os.sep, 'tmp', 'logs', 'stetest_general_functions.log'))
def MakeALogFileName():
import inspect
import os
current_stack = inspect.stack()
for frame in current_stack:
if frame[3] == 'main':
source_file = frame[1]
break
if source_file is None:
top_level_program = 'default'
else:
def main(arrgv=None):
## Main line
## This script has been coded with 3.7., but I stuck im a workaround for 3.6
import sys
if sys.version_info < (3,6):
sys.exit("\nThis script requires python 3.6 or higher.\n")
## The test ... If I can't read this and understand what's going on ...then it needs re-writing
## Any function should fit on 1 screen ( ish) ( ideally)
## Each function should be easily described so we know what it does
import os
# Get input paramaters
parser=HandleInputParameters()
global args
args = parser.parse_args()
#args = parser.parse_known_args()
#import MyLogging.mylogging
from MyLogging.mylogging import getLogFile, setLogFile, LogMessage
global logMessage
logMessage=LogMessage()
setLogFile(args.logfile)
#Check input params
errorfound=False
#if not os.path.exists(args.template_dsparam):
# errorfound=True
# logMessage.info('Template dsenv file not found at: ' + args.template_dsparam)
if not os.path.exists(args.install_base):
logMessage.warning('Install base does not existat: ' + args.install_base)
if errorfound:
sys.exit("\nInput parameter failed validation. See previous messages.\n")
if args.modified_since_timestamp is None:
logMessage.info('No value for --since-timestamp . So will export all components')
logMessage.info('logfile is :' + args.logfile)
logMessage.info('install_base is :' + args.install_base )
if not os.geteuid() == 0 :
sys.exit("\nOnly root can run this script\n")
## Main Code
#from datastage_functions import GetListOfComponentsRecentlyModified
import general_functions
#import datastage_functions
from datastage_functions import GetListOfComponentsRecentlyModified
components_list=GetListOfComponentsRecentlyModified(modified_since=args.modified_since_timestamp)
#from datetime import datetime
import datetime
components_list_test={
('HN01', 'dstage1'): [
('JobDefn', 'ExternalSource', datetime.datetime(2021, 1, 7, 17, 21, 34, 685000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand2', datetime.datetime(2021, 4, 6, 12, 0, 48, 589000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand3', datetime.datetime(2021, 4, 8, 16, 4, 20, 516000, tzinfo=datetime.timezone.utc), '\\\\Jobs'),
('JobDefn', 'RunCommand4', datetime.datetime(2021, 4, 9, 15, 11, 28, 998000, tzinfo=datetime.timezone.utc), '\\\\SteTest'),
('JobDefn', 'RunCommand5', datetime.datetime(2021, 4, 15, 16, 28, 9, 708000, tzinfo=datetime.timezone.utc), '\\\\SteTest\\\\SubFolder1')]
}
#components_list=components_list_test
print(components_list)
def ExportComponentList(export_base_dir='/var/repo_base/default', components_list={}):
import os
import re
for project_namespace_tuple, job_details in components_list.items():
engine_host=project_namespace_tuple[0]
project=project_namespace_tuple[1]
logMessage.info('Exporting Jobs from ' + engine_host + ':/' + project )
for job_info in job_details:
logMessage.info('Exporting Jobs ' + job_info[3] )
#Create path for archive file
component_name=job_info[1]
category=re.sub(r"\\\\",os.sep,job_info[3]) # Convert to using os.sep ('/')
category_path=re.sub(r"^"+os.sep,'',category) # Remove the leading '/'
archive_dir_path=os.path.join(export_base_dir,project,category_path,component_name)
if ( not(os.path.exists(archive_dir_path))):
os.makedirs(archive_dir_path,0o755)
archive_file=component_name + '.isx'
archive_path=os.path.join(archive_dir_path, archive_file)
# Export the job
import subprocess
ds_pattern=engine_host + '/' + project + '/' + '*' + '/'+ component_name + '.*'
command_to_run_old='/iis/01/InformationServer/Clients/istools/cli/istool.sh export -archive ' + archive_path + ' -up -ds "hn01/dstage1/*/'+component_name + '.*" -u isadmin -p default;'
command_to_run='/iis/01/InformationServer/Clients/istools/cli/istool.sh export -archive ' + archive_path + ' -up -ds "' +ds_pattern + '" -u isadmin -p default;'
## Annoying
import sys
if sys.version_info >= (3,7):
result = subprocess.run([command_to_run] , capture_output=True, shell=True, encoding="UTF-8")
else:
result = subprocess.run([command_to_run] , shell=True, encoding="UTF-8", stdout=subprocess.PIPE)
if result.returncode != 0:
logMessage.warning('Export of ' + component_name + ' ended with return code ' + str(result.returncode) + '. stdout : ' + result.stdout + '. stderr: ' + result.stderr )
else:
logMessage.debug('Export of ' + component_name + ' ended with return code ' + str(result.returncode) + '. stdout : ' + result.stdout + '. stderr: ' + str(result.stderr) )
logMessage.info('Export to ' + archive_path + ' finished .')
ExportComponentList(export_base_dir='/var/git/test1_repo/code_sheff/DataStageJobsTest1', components_list=components_list)
logMessage.info('Done')