def loadOraConnectionData( inputFilePath = None):
"""read connection data from json input file and return a list of dictionaires to the caller
NO CONNECTIONs are actually opened to the database! Also password must be acquired by
other means.
"""
if inputFilePath == None:
moduleSelfDir = os.path.dirname( inspect.getfile(inspect.currentframe()) )
inputFilePath = os.path.join ( moduleSelfDir, './ora_connection_cfg.json' )
if not os.path.exists( inputFilePath ):
_errorExit( "File %s does not seem to exist!" % ( inputFilePath ) )
conns = []
jData = json.load( open(inputFilePath, "r") )
_dbx( len( jData ) )
connRecs = jData[ "connectData" ]
# __errorExit( "fixme: json structure chagned!" )
for connDict in connRecs : # .keys():
_dbx( connDict )
nickname= connDict[ 'nickname' ]
host= connDict[ 'host' ]
port= connDict[ 'port' ]
service= connDict[ 'service' ]
user= connDict[ 'user' ]
conn = NicknamedOraConnection( nickname= nickname, host=host, port= port, service= service, username= user )
_dbx( str( conn ) )
conns.append( conn )
_dbx( len( conns ) )
return conns
def getOraPassword ( oraUser, oraPasswordEnvVar, batchMode, envSuffix= "" ):
"""Prompt for Oracle password if it is not found from environment variable.
Password entered will be hidden.
oraPasswordEnvVar defines the OS environment variable name with which this method tries
to extract password. However, it is not uncommon that DEV environments may have the same
username and password for an account that is uses to read the data dictionary, while
higher environment like PROD and Pre-production has specific password for this user.
Therefore if env_suffix is specified, this method first thing will string oraPasswordEnvVar and envSuffix
to get the password for that environment (=envSuffix). If such a variable is not found, it will
try to use the one given by oraPasswordEnvVar
fixme: batchMode is currently ignored!
"""
passwordEnv= None; hiddenPassword= ""
envVarToUse = oraPasswordEnvVar + envSuffix
if envVarToUse not in os.environ:
envVarToUse = oraPasswordEnvVar
if envVarToUse in os.environ:
passwordEnv= os.environ[ envVarToUse ]
if platform.system() == "Windows":
if envVarToUse == None:
_errorExit('INFO: Password could be passed as environment variable %s however it is not set.' % ( envVarToUse) )
pwToUse = passwordEnv
else:
pwToUse = getpass.getpass('Enter password for Oracle user %s at %s or simple ENTER (The input will be hidden if supported by the OS platform)' % ( oraUser, envSuffix ) )
if pwToUse == "" :
pwToUse= passwordEnv
return pwToUse
def action_devTest(jsonFile):
if not os.path.exists(jsonFile):
_errorExit("File %s does not seem to exist!" % (jsonFile))
jStr = open(jsonFile, "r").read()
jData = json.loads(jStr)
list = jData["filePaths"]
filePaths = []
for elem in list: # .keys():
_dbx(filePaths)
filePath = elem['path']
filePaths.append(filePath)
return filePaths
def getEnvList(csv):
""" convert CSV string in expected format to list of environment code
ensuring that the code is valid
"""
retVal = []
for ix, envCode in enumerate(objectCsv.split(",")):
_dbx("%d. env %s " % (ix, envCode))
if envCode not in g_mapDbNameOfEnvCode.keys():
_errorExit("envCode %s is not mapped!" % (envCode))
retVal.append(envCode)
return retVal
def extractTouchedScripts(commitA, commitB="HEAD"):
""" extract scripts which have been modfified or added between 2 commits
"""
global g_filesToExcludeFromInstall
args = ["git", "diff", "--name-only", commitA, commitB]
outFh, tmpOutFile = tempfile.mkstemp()
_dbx("using %s to capture output from git diff \nunix-style: %s" %
(tmpOutFile, dosPath2Unix(tmpOutFile)))
# outFh = open( tmpOutFile, "w" )
subprocess.run(args, stdout=outFh)
# _errorExit("test exit")
gitOutLines = open(tmpOutFile, "r").readlines()
if len(gitOutLines) == 0:
_errorExit("No lines found in git diff output file %s" % (tmpOutFile))
scriptsSet = set()
for line in gitOutLines:
if "we used git diff --name-status" == "but then there are issues with renames":
# _dbx( line )
match = re.search("^([ADM])\s+(.*)$", line)
if match == None:
raise ValueError(
"git diff returned line with unexpected content: %s" %
line)
else:
staCode, script = match.groups(1)[0:2]
#_dbx( staCode)# ; _dbx( script )
if staCode in "AM":
scriptsSet.add(script)
elif staCode == "D":
scriptsSet.discard(script)
else:
doExclude = False
for blackLine in g_filesToExcludeFromInstall:
if line.strip() == blackLine.strip():
doExclude = True
break
if not doExclude:
scriptsSet.add(line)
_dbx(len(scriptsSet))
return list(scriptsSet)
def getGitCurrBranchName():
args = ["git", "branch"]
outFh, tmpOutFile = tempfile.mkstemp()
_dbx("using %s to capture output from git branch \nunix-style: %s" %
(tmpOutFile, tmpOutFile))
# outFh = open( tmpOutFile, "w" )
subprocess.run(args, stdout=outFh)
# _errorExit("test exit")
gitOutLines = open(tmpOutFile, "r").readlines()
if len(gitOutLines) == 0:
_errorExit("No lines found in git branch output file %s" %
(tmpOutFile))
branchName = None
for line in gitOutLines:
if line.startswith('*'):
branchName = line.split()[1]
return branchName
def genUnixDiff(oldPath, newPath, recursive=False):
"""Calls the unix diff command and returns its output to the calling function
bomb out if any error was detected but only displayed upto 10 lines of the stderr
"""
diffCmdArgsUnix = ['diff', '-b', oldPath, newPath]
if recursive: diffCmdArgsUnix.insert(1, '-r')
# for a in diffCmdArgsUnix: _dbx( a ); _errorExit( 'test' )
proc = subprocess.Popen(diffCmdArgsUnix,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
unixOutMsgs, errMsgs = proc.communicate()
if len(errMsgs) > 0: # got error, return immediately
_errorExit('got error from diff. Only first 10 lines are shown:\n%s ' %
'\n'.join(errMsgs[0:10]))
_dbx(len(unixOutMsgs))
return unixOutMsgs
def getObjectList(objectCsv):
""" convert CSV string in expected format to list of DBObject instances
"""
retVal = []
for ix, obj in enumerate(objectCsv.split(",")):
_dbx("%d. obj %s " % (ix, obj))
tokens = obj.split(".")
if len(tokens) != 3:
_errorExit(
"object %s does not conform to expected format of schema.name.type_code"
% (obj))
(owner, name, typeCode) = tokens[:]
_dbx("owner %s typ %s" % (obj, typeCode))
typeOfficial = oraUtils.g_mapFileExtDbmsMetaDataTypeToad[typeCode]
_dbx("typeOfficial %s" % (typeOfficial))
retVal.append(
oraUtils.DBObject(name=name,
owner=owner,
type=typeOfficial,
fileExt=typeCode))
return retVal
def main():
argParserResult = parseCmdLine()
setDebug(argParserResult.debug)
if argParserResult.action == 'dbs':
action_dbs(envCsv=argParserResult.environments,
objCsv=argParserResult.objects)
elif argParserResult.action == 'extract':
action_extractScripts(objCsv=argParserResult.objects,
envCsv=argParserResult.environments)
# elif argParserResult.action == 'grepInst':
# action_grepInst( baseLocation= argParserResult.baseLocation, inputFilePaths = argParserResult.inputFilePaths )
elif argParserResult.action == 'os':
action_os(
inputFilePathsCsv=argParserResult.inputFilePaths,
branchName=argParserResult.featureName,
inputPathsFromJsonFile=argParserResult.inputPathsFromJsonFile)
elif argParserResult.action == 'devTest':
action_devTest(jsonFile=argParserResult.inputPathsFromJsonFile)
else:
_errorExit("action %s is not yet implemented" %
(argParserResult.action))
def plsqlTokenize(inpLines):
lnCnt = len(inpLines)
_dbx(lnCnt)
lineNo = 0
nodeStack = TokenStack(name='plsqlTokenize')
curTreeId = None
tokBuf = ""
interceptBufferLines = []
(interceptStartLineNo, interceptStartColNo) = (-1, -1)
# just for clarity. First reference is when we hit block_comment_begin
# match for alphanumString OR dblQuotedAlphanumString OR assignment OR singleEqual OR doubleEqual OR dotOperator
# match macros
# match block_comment_begin OR lineComment
# match single quote, assignment operator
# match comparison operators, named param operator
# match arithmetric operators, left or right parenthesis, comma, semicolon
# match Q notation begin in various flavours
eng = re.compile(
"""^(\s*)(\$\$plsql_unit|\$\$plsql_line|q\{"|[\$#a-z0-9_]+|"[\$#a-z0-9_]+"|:=|>=|<=|>|<|!=|=>|=|/\*|--|\|\||\.\.|\.|%|\(|\)|\+|-|\*|/|,|;|@)(\s*)""",
re.IGNORECASE)
curSta = FsmState.start
for line in inpLines[:29999]:
colNo = 1
lineNo += 1
lnBuf = line
eoLine = False
if None != re.search('^(\s*)$', line): # match empty line
pass # _dbx( 'Line %d is empty' % lineNo )
else:
pass # _dbx( "line %d len: %d. Line content >>>>>>>>>%s" % ( lineNo, len( line ), line.rstrip("\n") ) )
i = 0
# do we need eoLine indeed or can we just bump colNo accordingly?
while (i < 999 and colNo < len(line) and not eoLine
): # process line with safety belt against infinite loop
i += 1
#_dbx( "Ln/col %d/%d curSta: '%s'" % ( lineNo, colNo, curSta ) )
if curSta == FsmState.find_block_comment_end:
m = re.search('^(.*)(\*/)', lnBuf) # math end of block comment
if m == None:
#_dbx( "need to cache block comment" )
interceptBufferLines.append(lnBuf)
eoLine = True
continue
else: # found end of block comment
interceptBufferLines.append(m.group(1) + m.group(2))
_dbx("group1>>%s, group2>>%s, lnBuf>>>>%s" %
(m.group(1), m.group(2), lnBuf))
lenUptoStarSlash = len(m.group(1)) + len(m.group(2))
_dbx("lenUptoStarSlash:%d" % (lenUptoStarSlash))
colNo += lenUptoStarSlash
_dbx("found block comment end at col %d" % colNo)
lnBuf = lnBuf[lenUptoStarSlash:]
_dbx("stuff at comment is >>>%s" % (lnBuf.rstrip("\n")))
curSta = FsmState.start
node = TokenNode(text="".join(interceptBufferLines),
type=TokenType.block_comment_begin,
staAtCreation=curSta,
lineNo=interceptStartLineNo,
colNo=interceptStartColNo,
parentId=curTreeId)
nodeStack.push(node)
continue # while not EOL
elif curSta == FsmState.in_single_quoted_literal:
#_dbx( "scanning for end single quote in >>> %s " % lnBuf )
endOfLitFound, partOfLit = scanEndOfSQLiteral(lnBuf)
if not endOfLitFound: # line break is part of string literal
interceptBufferLines.append(lnBuf)
eoLine = True # line is done
else: # found end of literal in line, possibly with rest not belonging to literal
curSta = FsmState.start
interceptBufferLines.append(partOfLit)
literalText = "".join(interceptBufferLines)
node = TokenNode(
text=literalText,
type=TokenType.single_quoted_literal_begin,
staAtCreation=curSta,
lineNo=interceptStartLineNo,
colNo=interceptStartColNo,
parentId=curTreeId)
nodeStack.push(node)
colNo += len(partOfLit)
lnBuf = line[colNo - 1:]
#_dbx( "lnBuf>>>%s" % lnBuf )
continue
elif curSta == FsmState.in_q_notation_begin:
#_dbx( "scanning for end q notation literal in >>> %s " % lnBuf )
endOfLitFound, partOfLit = scanEndOfQNotationLiteral(
q2and3, lnBuf)
if not endOfLitFound: # line break is part of string literal
interceptBufferLines.append(lnBuf)
eoLine = True # line is done
else: # found end of literal in line, possibly with rest not belonging to literal
curSta = FsmState.start
interceptBufferLines.append(partOfLit)
literalText = "".join(interceptBufferLines)
node = TokenNode(
text=literalText,
type=TokenType.single_quoted_literal_begin,
staAtCreation=curSta,
lineNo=interceptStartLineNo,
colNo=interceptStartColNo,
parentId=curTreeId)
nodeStack.push(node)
colNo += len(partOfLit)
lnBuf = line[colNo - 1:]
# _dbx( "lnBuf>>>%s" % lnBuf )
continue
m = re.search('^(\s*)$', lnBuf) # match empty line
if m != None:
eoLine = True
if eoLine:
continue
# process other types of token
m = eng.match(lnBuf) # _dbx( type( m ) )
#_dbx( 'lnBuf being parsed >>>>>> %s' % lnBuf.rstrip("\n") )
if m == None:
# the special scan for single quoted literal is no longer needed since we can use the triple single quotes!
m = re.search("^(\s*)(')", lnBuf) # match single quote
if m != None: # found single quote
# stateStack.push( curSta, curTreeId )
curSta = FsmState.in_single_quoted_literal
interceptBufferLines = []
(interceptStartLineNo, interceptStartColNo) = (lineNo,
colNo)
interceptBufferLines.append(m.group(2))
colNo += len(m.group(1) + m.group(2))
lnBuf = line[colNo - 1:]
#_dbx( colNo )
continue # we must skip the fine-grained FSM
else:
_infoTs(
"Rest of line %d could not be tokenized. Line content follows \n%s"
% (lineNo, lnBuf))
return nodeStack
else:
# second re group i.e. token
tok = m.group(2)
# third re group i.e. optional whitespaces
#if len( m.group(3) ) > 0: # found trailing whitespaces
colNo += len(m.group(1)) + len(m.group(2)) + len(m.group(3))
# _dbx( "colNo: %d" % colNo )
#_dbx( "Ln/col %d/%d raw tok: '%s'" % ( lineNo, colNo, tok ) )
lnBuf = line[colNo - 1:]
# _dbx( "rest of line: %s" % lnBuf.rstrip("\n") )
tokTyp, normed = gettokentype(tok)
#_dbx( "tokTyp: %s normed: '%s'" % ( tokTyp, normed ) )
if tokTyp == TokenType.block_comment_begin:
if curSta == FsmState.find_block_comment_end:
_errorExit("Encountered tokTyp %s while in state %s!" %
(tokTyp, curSta))
else: # found block_comment if the middle of somewhere, switch the parser to specifically search for end of comment
curSta = FsmState.find_block_comment_end
interceptBufferLines = []
(interceptStartLineNo, interceptStartColNo) = (lineNo,
colNo)
interceptBufferLines.append(tok)
#_dbx( "we must skip the fine-grained FSM ")
continue # we must skip the fine-grained FSM
elif tokTyp == TokenType.single_line_comment_begin: # found double minus
#_dbx( foo )
if curSta == FsmState.find_block_comment_end:
_errorExit("Encountered tokTyp %s while in state %s!" %
(tokTyp, curSta))
else: # not in wrong status, just push line comment node, no change of state
node = TokenNode(
text=normed + lnBuf.rstrip("\n"),
type=TokenType.single_line_comment_begin,
staAtCreation=curSta,
lineNo=lineNo,
colNo=colNo - len(normed),
parentId=curTreeId)
nodeStack.push(node)
eoLine = True
continue
elif tokTyp == TokenType.q_notation_begin:
#_dbx( foo )
if curSta == FsmState.in_q_notation_begin:
_errorExit("Encountered tokTyp %s while in state %s!" %
(tokTyp, curSta))
else: # not in wrong status, just push line comment node, no change of state
# stateStack.push( curSta, curTreeId )
q2and3 = normed[
1:
3] # should be the open bracket and single or double quote, in any order
_dbx("normed>>>%s lnBuf>>> %s" % (normed, lnBuf))
curSta = FsmState.in_q_notation_begin
interceptBufferLines = []
(interceptStartLineNo, interceptStartColNo) = (lineNo,
colNo)
interceptBufferLines.append(m.group(1) + m.group(2))
continue # we must skip the fine-grained FSM
else:
pass #_dbx( "lineNo/colNo: %d/%d lnBuf >>>%s" % ( lineNo, colNo, lnBuf ))
#
#
node = TokenNode(text=normed,
type=tokTyp,
staAtCreation=None,
lineNo=lineNo,
colNo=colNo - len(normed),
parentId=curTreeId)
nodeStack.push(node)
return nodeStack # plsqlTokenize
def fsmMain(preTokStack, startStatus=FsmState.start):
""" make ASSUMPTION that comments tokens are in a different stack and in the main stack
we only have non-comment tokens. when later on significant tokens have been linked properpy
we will have another pass to insert the comment tokens based on lineNo/ColNo
"""
retTokStack = TokenStack(name="fsmMain")
stateStack = StateStack(name="main_state")
preTokStackLen = len(preTokStack.arr)
curTreeId = None
curSta = startStatus
thenComesFromStack = StateStack(name="thenComesFrom")
while preTokStack.peekAhead() != None:
curTokens = preTokStack.popComplexAware()
tokId, normed, tokTyp = (curTokens[0].id, curTokens[0].text,
curTokens[0].type)
# lineNo, colNo = (curTokens[0].lineNo, curTokens[0].colNo )
_dbx("curSta %s curTokens len %d, 1st id:%s type:%s >>>%s" %
(curSta, len(curTokens), tokId, tokTyp, normed))
if curSta == FsmState.start:
if tokTyp == TokenType.relevant_keyword and normed == "CREATE":
stateStack.push(curSta, curTreeId)
newSta = FsmState.in_compilation_unit_header
curTokens[0].state = staAtCreation = newSta
newTreeId = curTokens[0].id
else:
_errorExit("Unknown token id %s type %s in state %s " %
(tokId, tokTyp, curSta))
elif curSta == FsmState.in_compilation_unit_header:
if tokTyp == TokenType.relevant_keyword and normed == "AS":
_dbx(foo)
newSta = FsmState.in_declaration
curTokens[0].state = staAtCreation = newSta
newTreeId = curTokens[0].id
elif tokTyp == TokenType.semicolon: # forward declaration of function/procedure
_dbx(foo)
newSta, newTreeId = stateStack.pop()
else:
_dbx("other token type %s in state %s " % (tokTyp, curSta))
elif curSta == FsmState.in_declaration:
if tokTyp == TokenType.relevant_keyword and normed in ["BEGIN"]:
_dbx(foo)
newSta = FsmState.in_body
newTreeId = curTokens[0].id
else:
_dbx("other token type %s in state %s " % (tokTyp, curSta))
newSta = FsmState.started_declaration_entry
newTreeId = curTokens[0].id
elif curSta == FsmState.started_declaration_entry:
if tokTyp == TokenType.semicolon:
_dbx(foo)
newSta = FsmState.in_declaration
else:
_dbx("other token type %s in state %s " % (tokTyp, curSta))
elif curSta == FsmState.in_body:
if tokTyp in [TokenType.aggEndIdentSemic, TokenType.aggEndSemic]:
_dbx(foo)
newSta, newTreeId = stateStack.pop()
elif normed == "BEGIN":
_dbx(foo)
newSta = FsmState.in_body
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
elif normed in ['IF', 'WHILE']:
newSta = FsmState.expect_bool_expression
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
if normed == "IF":
thenComesFromStack.push(FsmState.if_or_case_statement_open,
None)
elif normed == "CASE" and preTokStack.peekAhead().text == "WHEN":
_dbx(foo)
for nextTok in preTokStack.popComplexAware():
curTokens.append(nextTok)
newSta = FsmState.expect_bool_expression
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
thenComesFromStack.push(FsmState.if_or_case_statement_open,
None)
elif normed == "CASE" and preTokStack.peekAhead().text != "WHEN":
# here we must not pop the peeked token, it must go thru normal FSM
newSta = FsmState.expect_expression
newTreeId = curTokens[0].id
#no pop expected! stateStack.push( curSta, curTreeId)
thenComesFromStack.push(FsmState.if_or_case_statement_open)
elif normed in ['DECLARE']:
newSta = FsmState.in_declaration
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
else:
_dbx("other token type %s in state %s " % (tokTyp, curSta))
stateStack.push(curSta, curTreeId)
newSta = FsmState.expect_expression
newTreeId = curTokens[0].id
elif curSta in [FsmState.expect_expression]:
if tokTyp in [
TokenType.semicolon, TokenType.aggEndSemic,
TokenType.aggEndIfSemic, TokenType.aggEndCaseSemic,
TokenType.aggEndLoopSemic
]:
_dbx(foo)
newSta, newTreeId = stateStack.pop()
elif normed in ['THEN']: # this is for "CASE WHEN .. THEN .."
_dbx(foo)
peekThenComesFrom = thenComesFromStack.peek()[
0] # we dont care about the parentId
if peekThenComesFrom == FsmState.case_bool_expression_open:
newSta = FsmState.in_body
newTreeId = curTokens[0].id
else:
_errorExit(
"Found THEN at %s without opening CASE token in thenComesFromStack"
% tokId)
thenComesFromStack.pop() # ignore return values
elif normed == "ELSE":
_dbx(foo)
newSta = FsmState.in_body
newTreeId = curTokens[0].id
elif normed == "CASE" and preTokStack.peekAhead().text == "WHEN":
_dbx(foo)
for nextTok in preTokStack.popComplexAware():
curTokens.append(nextTok)
newSta = FsmState.expect_bool_expression
newTreeId = curTokens[0].id
#do not expect pop! stateStack.push( curSta, curTreeId)
thenComesFromStack.push(FsmState.case_bool_expression_open,
None)
elif normed == "CASE" and preTokStack.peekAhead().text != "WHEN":
_dbx(foo)
# here we must not pop the peeked token, it must go thru normal FSM
newSta = FsmState.expect_expression
newTreeId = curTokens[0].id
# stateStack.push( curSta, curTreeId)
thenComesFromStack.push(FsmState.case_bool_expression_open,
None)
elif normed in ['LOOP']:
# this is for "FOR rec IN ( select * from xyz ) LOOP or similar constructs"
newSta = FsmState.in_body
elif tokTyp == TokenType.left_bracket:
_dbx(foo)
stateStack.push(curSta, curTreeId)
newTreeId = curTokens[0].id
elif tokTyp == TokenType.right_bracket:
newSta, newTreeId = stateStack.pop()
else:
_dbx("other token type %s in state %s " % (tokTyp, curSta))
elif curSta in [FsmState.expect_bool_expression]:
_dbx(foo)
if tokTyp in [TokenType.aggEndSemic
]: # this is for "CASE ... END;"
_dbx(foo)
newSta, newTreeId = stateStack.pop()
elif normed in [
'THEN'
]: # this is for "IF x THEN .. ELSE " or "WHILE y LOOP" or "CASE WHEN .. THEN .."
_dbx(foo)
peekThenComesFrom = thenComesFromStack.peek()[
0] # we dont care about the parentId
if peekThenComesFrom == FsmState.if_or_case_statement_open:
newSta = FsmState.in_body
newTreeId = curTokens[0].id
elif peekThenComesFrom == FsmState.case_bool_expression_open:
newSta = FsmState.expect_expression
newTreeId = curTokens[0].id
else:
_errorExit("No matching OPENER for THEN at %s" % tokId)
thenComesFromStack.pop() # ignore return values
elif normed in [
'ELSE', 'ELSIF', 'LOOP'
]: # this is for "IF x THEN .. ELSE " or "WHILE y LOOP" or "CASE WHEN .. THEN .."
_dbx(foo)
newSta = FsmState.in_body
newTreeId = curTokens[0].id
elif tokTyp == TokenType.left_bracket:
_dbx(foo)
stateStack.push(curSta, curTreeId)
newTreeId = curTokens[0].id
elif tokTyp == TokenType.right_bracket:
newSta, newTreeId = stateStack.pop()
elif normed == "CASE" and preTokStack.peekAhead().text == "WHEN":
for nextTok in preTokStack.popComplexAware():
curTokens.append(nextTok)
newSta = FsmState.expect_bool_expression
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
elif normed == "CASE" and preTokStack.peekAhead().text != "WHEN":
# here we must not pop the peeked token, it must go thru normal FSM
newSta = FsmState.expect_expression
newTreeId = curTokens[0].id
stateStack.push(curSta, curTreeId)
else:
_errorExit("No handler for state %s with input %s " %
(curSta, tokTyp))
for ix, curTok in enumerate(curTokens):
# _dbx( "ix: %d type %s" % (ix, type( curTok)) )
newNode = TokenNode(text=curTok.text,
type=curTok.type,
staAtCreation=curSta,
lineNo=curTok.lineNo,
colNo=curTok.colNo,
parentId=curTreeId)
retTokStack.push(newNode)
#_dbx( "ret stack len %d" % (len( retTokStack.arr ) ) )
_dbx("cur sta %s new sta %s" % (curSta, newSta))
curSta, curTreeId = newSta, newTreeId
_infoTs("final sta was %s" % (newSta))
if preTokStackLen != len(retTokStack.arr):
_errorExit(
"OOPPS preTokStackLen is %d and len( retTokStack.arr ) is %d" %
(preTokStackLen, len(retTokStack.arr)))
return retTokStack
#
# editing mark
#
return retTokStack
def createSchemataInstallScripts(sqlScriptTemplatePath,
baseCommit,
lastCommit,
featureName=None,
fileSufix=None,
storeReleaseMetadata=True):
""" Create SQL and BAT install scripts for the schemata with deployable scripts
Deployable scripts are: 1) file is not at top level of the schema
and 2) extension is not blacklisted
"""
global g_internalSepator, g_listIndexedBySchemaType, g_schemataFound
insertSqlStmtTemplate = """
--------------------------------------------------------------------------------
-- Store software release information: at this position we also record attempted
-- deployment
--------------------------------------------------------------------------------
DECLARE
lv_rel_id NUMBER;
BEGIN
SELECT basic_data.APPL_RELEASE_SQ.nextval INTO lv_rel_id FROM dual;
INSERT INTO basic_data.t_applied_releases( id, release_no, creation_dt ) VALUES( lv_rel_id, q'[{featureName}]', sysdate );
INSERT INTO basic_data.t_applied_files( id, release_id, filename )
SELECT appl_files_sq.nextval, lv_rel_id, q'[{basenameSqlScript}, git-branch: {featureName}, baseline-commit:{baselineCommit}, last-commit:{lastCommit}]'
FROM dual;
COMMIT;
END;
/
"""
suffixUsed = "-" + fileSufix if fileSufix else ""
sentinelPatPrefix = "REM place_here_scripts_for:"
fh = open(sqlScriptTemplatePath, mode="r")
inpTemplateLines = fh.readlines()
_dbx("got %d lines from template" % (len(inpTemplateLines)))
scriptTemplateText = "".join(inpTemplateLines)
tmpDir = tempfile.mkdtemp()
_infoTs("install scripts will be placed under %s" % (tmpDir))
batchScriptTemplate = """
SET NLS_LANG=GERMAN_GERMANY.WE8MSWIN1252
SQLPLUS /nolog @{sqlScriptBaseName}
"""
readmeContentHeader = """
Order to run install scripts:
"""
readmeContentFooter = """
All processes in groups xxx, yyy must be stopped
"""
batchScripts = []
for schema in g_schemataFound:
_dbx("schema %s\n" % (schema))
script4Schema = scriptTemplateText
script4Schema = script4Schema.replace("<TARGET_SCHEMA>", schema)
for schemaType in g_listIndexedBySchemaType.keys():
if schemaType.startswith(schema):
typeOnly = schemaType.split(g_internalSepator)[1]
if typeOnly.upper() not in [
'.SQL'
]: # dirty fix to filter out top-level sql script
sentinelPattern = "%s%s" % (sentinelPatPrefix,
typeOnly.upper())
_dbx("schemaType %s, sentinel %s" %
(schemaType, sentinelPattern))
listOfScripts = g_listIndexedBySchemaType[schemaType]
_dbx("cnt scripts %s" % (len(listOfScripts)))
# aggregate scripts of schemaType into one string
stringToAppend = "\n".join(listOfScripts)
_dbx(stringToAppend)
found = script4Schema.find(sentinelPattern)
if found > 0:
_dbx("found pattern")
script4Schema = script4Schema.replace(
sentinelPattern,
"\n%s\n%s" % (sentinelPattern, stringToAppend))
else:
_errorExit("Sentinel '%s' not found in template!" %
(sentinelPattern)) # , isWarning = True
# print( script4Schema )
# now remove the sentinel's
tempScript = script4Schema
newLines = []
for line in tempScript.split("\n"):
if not line.startswith(sentinelPatPrefix):
newLines.append(line)
script4Schema = "\n".join(newLines)
schemaDir = os.path.join(tmpDir, schema)
os.mkdir(schemaDir)
basenameSqlScript = "install_%s%s.sql" % (schema, suffixUsed)
_dbx(basenameSqlScript)
scriptPathSql = os.path.join(schemaDir, basenameSqlScript)
if storeReleaseMetadata:
# for INSERT of applied release information
insertSqlStmt = insertSqlStmtTemplate.format( featureName= featureName \
, baselineCommit= baseCommit, lastCommit= lastCommit, basenameSqlScript= basenameSqlScript )
_dbx(insertSqlStmt)
script4Schema = script4Schema.format(
placeHolderStoreReleaseMetadata=insertSqlStmt,
baselineCommit=baseCommit,
featureName=featureName)
else:
script4Schema = script4Schema.format(
placeHolderStoreReleaseMetadata="",
baselineCommit=baseCommit,
featureName=featureName)
batScriptBaseName = "install_%s%s.bat" % (schema, suffixUsed)
scriptPathBat = os.path.join(schemaDir, batScriptBaseName)
fh = open(scriptPathSql, mode="w")
fh.write(script4Schema)
fh.close()
sqlScriptBaseName = os.path.basename(scriptPathSql)
batchScriptContent = batchScriptTemplate.format(
sqlScriptBaseName=sqlScriptBaseName)
fh = open(scriptPathBat, mode="w")
fh.write(batchScriptContent)
fh.close()
_infoTs("output SQL script unix-style >>>>> %s \nDOS style >>>: %s" %
(dosPath2Unix(scriptPathSql), scriptPathSql))
_infoTs("output BAT script unix-style >>>>> %s \nDOS style >>>: %s" %
(dosPath2Unix(scriptPathBat), scriptPathBat))
#_errorExit( "in test - stopped after 1 schema!! " );
batchScripts.append(os.path.basename(scriptPathBat))
# create install readme
readmeFile = os.path.join(tmpDir, "install%s-readme.txt" % (suffixUsed))
items = []
for batchScript in batchScripts:
items.append("?. %s" % (batchScript))
_dbx(batchScript)
itemsText = "\n".join(items)
readmeText = "%s\n%s\n%s\n" % (readmeContentHeader, itemsText,
readmeContentFooter)
fh = open(readmeFile, mode="w")
fh.write(readmeText)
fh.close()
_infoTs("readme file unix-style >>>>> %s \nDOS style >>>: %s" %
(dosPath2Unix(readmeFile), readmeFile))
def parseCmdLine():
import argparse
parser = argparse.ArgumentParser()
# lowercase shortkeys
parser.add_argument(
'-a',
'--action',
choices=['dbs', 'extract', 'os', 'twoRepos', 'devTest'],
help=
"""dbs: input are from databases, os: input is comma separated file paths
twoRepos: provide the root location of 2 git repos on the local PC. This program will cd to the root location and extract the branch name. Input file paths are extracted from --inputPathsFromJsonFile, attribute inputFilePaths
""",
required=True)
parser.add_argument('-b',
'--baseLocation',
help='base location of input files')
parser.add_argument(
'-e',
'--environments',
help='comma separated list of environment codes, e.g prod, uat2, gt2',
required=False)
parser.add_argument(
'-f',
'--featureName',
help='branch or feature name, will be used to qualify the file name')
parser.add_argument('-I',
'--inputFilePaths',
help='comma separated input file paths',
required=False)
parser.add_argument('-j',
'--inputPathsFromJsonFile',
help='json file containing various input data',
default=".\\scripts_for_quickDiff.json")
parser.add_argument(
'-o',
'--objects',
help=
'comma separated list of objects, e.g: process.sk_process_control.pks',
required=False)
parser.add_argument('--debug',
help='print debugging messages',
required=False,
action='store_true')
parser.add_argument('--no-debug',
help='do not print debugging messages',
dest='debug',
action='store_false')
result = parser.parse_args()
if result.featureName == None:
result.featureName = getGitCurrBranchName()
if result.featureName == None:
result.featureName = g_defaultBranchName
if result.action == 'dbs':
if result.environments == None or result.objects == None:
_errorExit("Action '%s' requires both env codes ans object list" %
(result.action))
elif result.action == 'extract':
#if result.connectQuad == None or result.objects == None: _errorExit( "Action '%s' require connectQuad and objects" % (result.action ) )
if result.environments == None or result.objects == None:
_errorExit("Action '%s' requires both env codes ans object list" %
(result.action))
elif result.action == 'os':
if result.inputFilePaths == None and result.inputPathsFromJsonFile == None:
_errorExit(
"Action '%s' require inputFilePaths or inputPathsFromJsonFile"
% (result.action))
elif result.action == 'devTest':
if result.inputPathsFromJsonFile == None:
_errorExit("Action '%s' require inputPathsFromJsonFile" %
(result.action))
return result
