def searchmodule(self, application, key, file):
# try :
#Print.info("file scan module :"+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('moduleexport', before='', element='module.*', after='')
greferences += [
reference for reference in rf.find_references_in_file(file)
]
for ref in greferences:
try:
if ref.value.find('module') is not -1:
if ref.value.find('=') is not -1:
response = ref.value.replace("=", '')
resp = response.replace("module", '')
cleanvalue = re.sub('[^0-9a-zA-Z.]+', ' ', resp)
fsharpmoduleObjReferences = application.search_objects(
category='fsharp_module',
name=cleanvalue,
load_properties=False)
if fsharpmoduleObjReferences is not None:
for o in fsharpmoduleObjReferences:
self.scan_fs(application, key, o, file)
except AttributeError:
logging.info("error search module sub")
def parsegosuproperty(self, gosufile, file):
try :
#Print.info("file scan class :"+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('propertyexportp', before='', element='property.*', after='\{')
#rf.add_pattern('constructexportprot', before='protected', element='construct.*', after='\{')
#rf.add_pattern('constructexportprivate', before='private', element='construct.*', after='\{')
greferences += [reference for reference in rf.find_references_in_file(gosufile)]
for ref in greferences:
try:
if ref.value.find('property') is not -1:
response = ref.value
resp = response.replace("property", '')
self.counter = self.counter+1
gsobj = cast.analysers.CustomObject()
cleanvalue= re.sub('[^0-9a-zA-Z.]+', ' ',resp)
gsobj.set_name(str(cleanvalue))
self.classname =str(resp.strip())
gsobj.set_type('gosu_property')
gsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
gsobj.set_guid(str(Path(file.get_path()).name)+str(self.counter))
gsobj.save()
bookmark = cast.analysers.Bookmark(file, 1,1,-1,-1)
gsobj.save_position(bookmark)
#Print.info("Save object property: "+cleanvalue)
#self.parentOBJtwo =gsobj
cast.analysers.create_link('callLink', self.parentOBJtwo, gsobj, bookmark)
#Print.info("link created: "+cleanvalue)
except ValueError:
Print.info ("error loading gosu2 property")
except:
return
def parsegosupackage(self, gosufile, file):
try :
#Print.info("file scan package :"+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('packageexport', before='', element = 'package.*', after='')
greferences += [reference for reference in rf.find_references_in_file(gosufile)]
for ref in greferences:
try:
if ref.value.find('package') is not -1:
response = ref.value.replace("package", '"')
gsobj = cast.analysers.CustomObject()
gsobj.set_name(str(response.strip()))
self.package =str(response.strip())
gsobj.set_type('gosu_package')
gsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
gsobj.set_guid(self.fielPath+str(Path(file.get_path()).name)+str(random.randint(1, 200))+str(random.randint(1, 200)))
gsobj.save()
self.parentOBJ = gsobj
#Print.info("Save object package:"+response.strip())
except ValueError:
Print.info ("error loading gosu2 package")
except:
return
def parsegosuuses(self, gosufile, file):
try :
#Print.info("file scan uses "+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('usesexport', before='', element = 'uses.*', after='')
greferences += [reference for reference in rf.find_references_in_file(gosufile)]
for ref in greferences:
try:
if ref.value.find('uses') is not -1:
response = ref.value.replace("uses", '')
gsobj = cast.analysers.CustomObject()
cleanvalue= re.sub('[^0-9a-zA-Z.]+', '', response)
gsobj.set_name(str(response.strip()))
gsobj.set_type('gosu_uses')
gsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
gsobj.set_guid(response+str(Path(file.get_path()).name)+str(random.randint(1, 200))+str(random.randint(1, 200)))
gsobj.save()
bookmark = cast.analysers.Bookmark(file, 1,1,-1,-1)
gsobj.save_position(bookmark)
#Print.info("Save object uses: "+cleanvalue)
cast.analysers.create_link('callLink', self.parentOBJ,gsobj, bookmark)
#Print.info("link created: "+cleanvalue)
except ValueError:
Print.info ("error loading gosu2 uses")
except:
return
def end_application(self, application):
logging.info('Creating links to iBatis objects...')
for namedQuery in application.search_objects(category='CAST_SQL_NamedQuery', load_properties=True):
logging.debug('Processing Name query %s' % namedQuery.get_name())
iBatisLinks.create(application, namedQuery, 'iBatisProperties.class')
iBatisLinks.create(application, namedQuery, 'iBatisProperties.parameterClass')
iBatisLinks.create(application, namedQuery, 'iBatisProperties.resultClass')
iBatisLinks.create(application, namedQuery, 'iBatisProperties.listClass')
iBatisLinks.create(application, namedQuery, 'iBatisProperties.resultType')
MyBatisLinks.create(application, namedQuery)
#GREP in DotNet/Java, looking for iBatis objects
logging.info('Creating dynamic links to iBatis objects...')
rf = ReferenceFinder()
rf.add_pattern('iBatis', before='["\.]', element='[A-Za-z0-9_\-]+', after='"')
references = []
for o in application.get_files(['CAST_DotNet_CSharpFile', 'JV_FILE']):
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
references += [reference for reference in rf.find_references_in_file(o)]
for reference in references:
for namedQuery in application.objects().has_type('CAST_SQL_NamedQuery'):
if reference.value == namedQuery.get_name():
logging.debug('Reference to %s found in %s' % (namedQuery.get_name(), reference.object.get_name()))
link = create_link('callLink', reference.object, namedQuery, reference.bookmark)
link.mark_as_not_sure()
def scan_fs(self, application, errorkey, program, file):
# one RF for multiples patterns
if len(self.errorlinestr) > 0:
searchpattern = re.escape(self.errorlinestr)
rfCall = ReferenceFinder()
rfCall.add_pattern(('srcline'),
before='',
element=searchpattern,
after='') # requires application_1_4_7 or above
# search all patterns in current program
try:
references = [
reference for reference in rfCall.find_references_in_file(
self.currentsrcfile)
]
except FileNotFoundError:
logging.warning("Wrong file or file path:" +
str(self.currentsrcfile))
else:
#try: # for debugging and traversing the results
for reference in references:
if reference.pattern_name == 'srcline':
reference.bookmark.file = file
#logging.info(reference.bookmark)
#logging.debug("DONE: reference found: >"+errorkey +str(reference))
program.save_violation(
'fsharplint_CustomMetrics_' + errorkey + "." +
errorkey, reference.bookmark)
logging.info("violation saved: >" +
'fsharplint_CustomMetrics.' + errorkey +
" line:::" + self.errorlinestr)
def end_application(self, application):
logging.info("GDPR Running code at the end of an Application")
application.declare_property_ownership(
'GDPRViolation_CustomMetrics.GDPRJAVAViolation', ['JV_FILE'])
str_FindList = []
filename = self.find_csv_fullpath(application)
#logging.info("Filename--> " + str(filename))
with self.my_open_source_file(filename) as csvfile:
index = filename.rfind('\\')
self.name = filename[index + 1:]
mydictreader = csv.reader(
csvfile
) # wo fieldnames : line #1 i used to get the column names
for row in mydictreader:
if row[0] not in str_FindList:
str_FindList.append(row[0])
files = application.get_files(['JV_FILE'])
rfCall = ReferenceFinder()
for strval in str_FindList:
rfCall.add_pattern(strval, before="", element=strval, after="")
#references = []
gjvf = list(
application.search_objects(category='JV_FILE',
load_properties=True))
#logging.info("file JVF from search--" + str(gjvf))
#jvFile_list = list(application.search_objects(category='JV_FILE', load_properties= True))
#print("Lenght of jvlist %s" + str(gjvf))
for fle in files:
#logging.info("file value--" + str(fle.name))
if (fle.get_path().endswith('.java')):
references = [
reference
for reference in rfCall.find_references_in_file(fle)
]
for reference in references:
sfpath = fle.name
#print("file path%s" + str(sfpath))
jvf = filter(lambda x: (x.name == sfpath), gjvf)
#print("Lenght of jvlist %s" + str(jvf))
try:
if jvf != None:
for j in jvf:
#print("Lenght of jvlist---" + str(j.name))
print("references.bookmark---" +
str(reference.bookmark))
#j.save_violation('GDPR_CustomMetrics.GDPR_violation', reference.bookmark)
j.save_violation(
'GDPRViolation_CustomMetrics.GDPRJAVAViolation',
reference.bookmark)
logging.info("Violation saved for file --" +
str(j.name))
break
except Exception as e:
print("Error in violation---" + str(e) + '------' +
str(j.name))
def CFPatternLinking(self, application):
''' Link ColdFusion objects based on pattern.
See links element in ColdFusionLanguagePattern.xml. InvokeWebMethodAndComponentLinking already provides the support of links from cfinvoke objects.
'''
logging.info('Creating links based on patterns')
patterns = {
'methodCall': (r'methodcall\s*=\s*"(\w+)', 'callLink'),
'dotNameCall': (r'\.+(\w+)', 'callLink'),
'slashNameCall':
(r'/([a-zA-Z_\x7f-\xff][\w.\x7f-\xff]*)', 'callLink')
}
application_objects = [
application_object for application_object in application.objects()
]
prefixes = {
'CFComponent': 'cfcomponent.',
'CFMail': 'cfmail.',
'CFQuery': 'cfquery.',
'CFInvoke': 'cfinvoke.',
'CFFile': 'cffile.',
'CFFTP': 'cfftp.',
'CFHttp': 'cfhttp.',
'CFFunction': 'cffunction.',
'CFWebService': 'cfwebservice.'
}
linksReferenceFinder = ReferenceFinder()
for pn, pd in patterns.items():
linksReferenceFinder.add_pattern(pn, '', pd[0], '')
for f in application.get_files():
logging.info('Searching for references in %s', f.get_path())
for reference in linksReferenceFinder.find_references_in_file(f):
# Using logging.info for inestigations
# logging.debug('Reference found: %s', reference.value)
logging.info('Reference found: %s', reference.value)
callee_name = re.sub(patterns[reference.pattern_name][0],
r'\1', reference.value)
# Using logging.info for inestigations
# logging.debug('Callee name: %s', callee_name)
logging.info('Callee name: %s', callee_name)
for application_object in application_objects:
if application_object.get_type() in prefixes:
calle_name_with_prefix = prefixes[
application_object.get_type()] + callee_name
else:
callee_name_with_prefix = callee_name
if application_object.get_name(
) == callee_name_with_prefix:
logging.info('Creating link between %s(%s) and %s(%s)',
reference.object.get_fullname(),
reference.object.get_type(),
application_object.get_fullname(),
application_object.get_type())
cast.application.create_link(
patterns[reference.pattern_name][1],
reference.object, application_object,
reference.bookmark)
def Call_to_spring_Batch_Job(self, application):
springBatch_access = ReferenceFinder()
springBatch_access.add_pattern(
"springBatchCall",
before="",
element="\(Job\)[ ]*context\.getBean\(\"[a-zA-Z0-9_-]+",
after="")
nb_links = 0
for o in application.get_files(['JV_FILE']):
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
for reference in springBatch_access.find_references_in_file(o):
#logging.debug("Reference " + reference.value)
springBatchJob_name = reference.value.split("\"")[1]
#logging.debug("searching " + springBatchJob_name)
if springBatchJob_name in self.springBatchJob:
springBatchJob_target = self.springBatchJob[
springBatchJob_name]
create_link('callLink', reference.object,
springBatchJob_target, reference.bookmark)
logging.debug("Creating link between " +
str(reference.object) + " and " +
str(springBatchJob_target))
nb_links += 1
else:
logging.debug("Spring Batch Job not found [" +
str(springBatchJob_name) + "]")
# Begin Specific for CPP 2017 project
if 'processFlux' in str(springBatchJob_name):
logging.debug(" !!!!!! Spring Batch Job name dynamic ")
for springBatchJob_target_processFlux_name in self.springBatchJob:
if 'processFlux' in springBatchJob_target_processFlux_name:
springBatchJob_target_processFlux = self.springBatchJob[
springBatchJob_target_processFlux_name]
create_link('callLink', reference.object,
springBatchJob_target_processFlux,
reference.bookmark)
logging.debug(
"Creating link between " +
str(reference.object) + " and " +
str(springBatchJob_target_processFlux))
nb_links += 1
# end Specific for CPP 2017 project
logging.debug(
"Nb of links created between Java classes and Spring Batch Job : "
+ str(nb_links))
self.global_nb_links += nb_links
def lookForGeneratedCodingInIncludes(self, application):
logging.info('**** lookForGeneratedCodingInIncludes')
generated_coding_access = ReferenceFinder()
generated_coding_access.add_pattern(
"GeneratedCodingInclude",
before="",
element="THIS FILE IS GENERATED BY THE FUNCTION LIBRARY",
after="")
#THIS FILE IS GENERATED BY THE FUNCTION LIBRARY
for o in application.get_files(
): # restriction to ABAP class would be better
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
if not o.get_path().lower().endswith('.abap'):
continue
for reference in generated_coding_access.find_references_in_file(
o):
logging.info("Reference " + reference.value)
# manipulate the reference pattern found
searched_generated_coding = reference.value
sourcefilepath = o.get_path()
logging.info("found generated coding [" +
searched_generated_coding + "] in file [" +
sourcefilepath + "]")
# reading source file
with open(sourcefilepath) as f:
for line in f:
if line.strip().startswith('INCLUDE_NAME'):
include_name = line.split(' ')[1].split('.')[0]
logging.info(
"include name [" + include_name +
"] in file [" + sourcefilepath +
"] contain generated coding : will be set as external"
)
if include_name not in self.abapIncludesExcluded:
includeObject = self.abapIncludes[include_name]
includeObject.set_as_external()
self.abapIncludesExcluded[
include_name] = includeObject
self.nbAbapIncludeExcluded += 1
else:
logging.info("include name [" + include_name +
"] in file [" + sourcefilepath +
"] was already set as external")
def parsefsharpmodule(self, fsharpfile, file):
try:
#Print.info("file scan module :"+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('moduleexport',
before='',
element='module.*',
after='')
greferences += [
reference
for reference in rf.find_references_in_file(fsharpfile)
]
for ref in greferences:
try:
if ref.value.find('module') is not -1:
if ref.value.find('=') is not -1:
response = ref.value.replace("=", '')
resp = response.replace("module", '')
self.counter = self.counter + 1
fsobj = cast.analysers.CustomObject()
cleanvalue = re.sub('[^0-9a-zA-Z.]+', ' ', resp)
fsobj.set_name(str(cleanvalue))
self.modulename = str(resp.strip())
fsobj.set_type('fsharp_module')
fsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
fsobj.set_guid(
str(Path(file.get_path()).name) +
str(self.counter))
fsobj.save()
bookmark = cast.analysers.Bookmark(
file, 1, 1, -1, -1)
fsobj.save_position(bookmark)
setpropertyparent.add_propertynamespace(
fsobj, self.namespace)
#Print.info("Save object module: "+cleanvalue)
self.parentOBJtwo = fsobj
cast.analysers.create_link('callLink',
self.parentOBJ, fsobj,
bookmark)
#Print.info("link created: "+cleanvalue)
except ValueError:
Print.info("error loading fsharp2 module")
except:
return
def setprop(self, application, file, sobjname, rulename):
# one RF for multiples patterns
rfCall = ReferenceFinder()
rfCall.add_pattern(('srcline'),
before='',
element=self.sregex,
after='') # requires application_1_4_7 or above
# search all patterns in current program
try:
# self.propvalue =[]
getvalue = ""
cntprop = 0
references = [
reference for reference in rfCall.find_references_in_file(file)
]
for reference in references:
reference.bookmark.file = file
cntprop = cntprop + 1
#self.propvalue.append(str(reference.value)+" "+str(reference.bookmark))
file.save_violation(
'dboraclemigration_CustomMetrics.' + rulename,
reference.bookmark)
logging.debug("violation saved: >" +
'dboraclemigration_CustomMetrics.' + rulename +
" line:::" + str(reference.value) +
str(reference.bookmark))
#break
# file.save_property('dboraclemigrationScript.'+sobjname, reference.value+" "+str(reference.bookmark) )
# logging.info("property saved: >" +'dboraclemigrationScript.'+sobjname +" "+str(reference.bookmark)+ ' '+ str(reference.value))
getvalue = str(cntprop)
#logging.debug("Value of list-->"+ str(getvalue))
if cntprop > 0:
file.save_property('dboraclemigrationScript.' + sobjname,
getvalue)
logging.debug("property saved: --->" +
'dboraclemigrationScript.' + sobjname + " " +
getvalue)
#
except Exception as e:
logging.info(
": error saving property violation on properties : %s",
str(e))
return
def lookForGeneratedCodingInFunctions(self, application):
logging.info('**** lookForGeneratedCodingInFunctions')
generated_coding_access = ReferenceFinder()
generated_coding_access.add_pattern("GeneratedCodingFunction",
before="",
element="Generated Coding",
after="")
#***** Generated Coding, do not modify *****
for o in application.get_files(
): # restriction to ABAP class would be better
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
if not o.get_path().lower().endswith('.abap'):
continue
for reference in generated_coding_access.find_references_in_file(
o):
logging.info("Reference " + reference.value)
# manipulate the reference pattern found
searched_generated_coding = reference.value
sourcefilepath = o.get_path()
logging.info("found generated coding [" +
searched_generated_coding + "] in file [" +
sourcefilepath + "]")
# reading source file
with open(sourcefilepath) as f:
for line in f:
if line.strip().startswith('FUNCTION'):
function_name = line.split(' ')[1].split('.')[0]
logging.info(
"function name [" + function_name +
"] in file [" + sourcefilepath +
"] contain generated coding : will be set as external"
)
functionObject = self.abapFunctions[function_name]
functionObject.set_as_external()
self.nbAbapFunctionExcluded += 1
def parsefsharpnamespace(self, fsharpfile, file):
try:
#Print.info("file scan namespace :"+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('namespaceexport',
before='',
element='namespace.*',
after='')
greferences += [
reference
for reference in rf.find_references_in_file(fsharpfile)
]
for ref in greferences:
try:
if ref.value.find('namespace') is not -1:
response = ref.value.replace("namespace", '')
fsobj = cast.analysers.CustomObject()
self.counter = self.counter + 1
cleanvalue = re.sub('[^0-9a-zA-Z.]+', ' ', response)
fsobj.set_name(str(cleanvalue))
self.namespace = str(response.strip())
fsobj.set_type('fsharp_namespace')
fsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
fsobj.set_guid(
str(Path(file.get_path()).name) +
str(self.counter))
fsobj.save()
self.parentOBJ = fsobj
bookmark = cast.analysers.Bookmark(file, 1, 1, -1, -1)
fsobj.save_position(bookmark)
self.parentOBJ = fsobj
#Print.info("Save object namespace: "+cleanvalue )
except ValueError:
Print.info("error loading fsharp2 namespace")
except:
return
def parsefsharpopen(self, fsharpfile, file):
try:
Print.info("file scan open " + Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('openexport', before='', element='open.*', after='')
greferences += [
reference
for reference in rf.find_references_in_file(fsharpfile)
]
for ref in greferences:
try:
if ref.value.find('open') is not -1:
response = ref.value.replace("open", '')
fsobj = cast.analysers.CustomObject()
cleanvalue = re.sub('[^0-9a-zA-Z.]+', '', response)
fsobj.set_name(str(response.strip()))
fsobj.set_type('fsharp_open')
fsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
fsobj.set_guid(response +
str(Path(file.get_path()).name) +
str(random.randint(1, 200)) +
str(random.randint(1, 200)))
fsobj.save()
bookmark = cast.analysers.Bookmark(file, 1, 1, -1, -1)
fsobj.save_position(bookmark)
setpropertyparent.add_propertynamespace(
fsobj, self.namespace)
#Print.info("Save object open: "+cleanvalue)
cast.analysers.create_link('callLink', self.parentOBJ,
fsobj, bookmark)
#Print.info("link created: "+cleanvalue)
except ValueError:
Print.info("error loading fsharp2 open")
except:
return
def parsemqsuses(self, mqsfile, file, objname, bref, aref, refkey):
try:
#Print.info("file scan uses "+Path(file.get_path()).name)
rf = ReferenceFinder()
greferences = []
rf.add_pattern('usesexport',
before=bref,
element=refkey,
after=aref)
greferences += [
reference for reference in rf.find_references_in_file(mqsfile)
]
for ref in greferences:
try:
cleankey = re.sub('[^0-9a-zA-Z]+', '', refkey)
if ref.value.find(cleankey) is not -1:
response = ref.value.replace(cleankey, '')
gsobj = cast.analysers.CustomObject()
cleanvalue = re.sub('[^0-9a-zA-Z.]+', '', response)
gsobj.set_name(str(response.strip()))
gsobj.set_type(objname)
gsobj.set_parent(file)
parentFile = file.get_position().get_file()
self.fielPath = parentFile.get_fullname()
gsobj.set_guid(response +
str(Path(file.get_path()).name) +
str(random.randint(1, 200)) +
str(random.randint(1, 200)))
gsobj.save()
bookmark = cast.analysers.Bookmark(file, 1, 1, -1, -1)
gsobj.save_position(bookmark)
Print.info(file.get_path() + " Save object " +
objname + " :" + cleanvalue)
except ValueError:
Print.info("error loading mqs2 uses")
except:
return
def setpropjavasql(self, application, file, sobjname, rulename):
# one RF for multiples patterns
rfCall = ReferenceFinder()
rfCall.add_pattern(('srcline'),
before='',
element=self.sregex,
after='') # requires application_1_4_7 or above
# search all patterns in current program
try:
self.propvalue = []
self.uniqueobjlist = []
cntj = 0
references = [
reference for reference in rfCall.find_references_in_file(file)
]
for reference in references:
reference.bookmark.file = file
linenb = int(str(reference.bookmark).split(',')[2])
#logging.debug( str(reference.bookmark).split(',')[2])
#logging.debug("Specific object:"+ str(file.find_most_specific_object(linenb, 1)))
obj = file.find_most_specific_object(linenb, 1)
cntj = cntj + 1
self.uniqueobjlist.append(sobjname + "cast" + str(obj))
obj.save_violation(
'dboraclemigration_CustomMetrics.' + rulename,
reference.bookmark)
#logging.debug("violation saved: >" +'dboraclemigration_CustomMetrics.'+rulename+" line:::"+str(reference.value)+str(reference.bookmark))
#break
# file.save_property('dboraclemigrationScript.'+sobjname, reference.value+" "+str(reference.bookmark) )
# logging.info("property saved: >" +'dboraclemigrationScript.'+sobjname +" "+str(reference.bookmark)+ ' '+ str(reference.value))
#logging.info('unique' + str(self.uniqueobjlist))
self.unique(self.uniqueobjlist, application)
#
except Exception as e:
logging.info(": error saving property violation : %s", str(e))
return
def end_application(self, application):
logging.info(
"GenericRefLink : Running extension code at the end of an application"
)
# list all files saved
try:
#fpath="C:\ProgramData\CAST\CAST\Extensions\com.castsoftware.labs.GenRefLink.1.0.0\Generic.xml"
fpath = self.Castpath.get_drive(
) + "ProgramData\CAST\CAST\Extensions\com.castsoftware.labs.GenericRefLink.1.1.0\Generic.xml"
logging.info(str(fpath))
if (os.path.isfile(fpath)):
tree = ET.parse(fpath, ET.XMLParser(encoding="UTF-8"))
root = tree.getroot()
cnt = 0
for group in root.findall('Search'):
sbefore = group.find('RegexPatternBefore').text
safter = group.find('RegexPatternAfter').text
sregex = group.find('RegexPattern').text
sfileext = group.find('RefFileExtension').text
sfilecategory = group.find('RefFileCastCatergory').text
nb_links = 0
links = []
files = []
rf = ReferenceFinder()
cnt = cnt + 1
if sbefore is None:
sbefore = ""
if safter is None:
safter = ""
if sfileext is not None:
sfileext = sfileext.replace(".", "")
logging.debug(
str(sbefore) + "---" + str(sregex) + "---" +
str(safter))
rf.add_pattern('Search' + sfileext + str(cnt),
before=sbefore,
element=sregex,
after=safter)
# list all files saved
try:
fileCount = sum(
1 for x in application.get_files([sfilecategory]))
files = application.get_files([sfilecategory])
self.findpattern(application, files, group, links, rf)
except Exception as e:
logging.error(": Error getting source file set : %s",
str(e))
# 3. Create the links
for link in links:
logging.info("Creating link between " + str(link[1]) +
" and " + str(link[2]))
create_link(*link)
nb_links = nb_links + 1
links = []
logging.info("Nb of links created " + str(nb_links))
except Exception as e:
logging.error(": Error Generic ref link extension set : %s",
str(e))
def links_from_JSP_to_Tiles(self, application):
nb_links = 0
nb_links2 = 0
nb_links3 = 0
nb_notfound = 0
logging.info(
"==> solves the following problem : Missing links between JSP and Tiles"
)
# 1. search all references in all files
logging.info("Scanning jsp and jspx files for calls to Tiles tags")
# 2. scan each JSP file
# we search a pattern
jsp_access = ReferenceFinder()
jsp_access.add_pattern(
"Mapping",
before="tiles:insertAttribute",
element=
"[ \n\r\t]+name[ \n\r\t]*\=[ \n\r\t]*[A-Za-z0-9\=_\-\.\" ]+",
after="")
links = []
# iterate all objects of the application
for o in application.get_files(['CAST_Web_File']):
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
if (not o.get_path().lower().endswith('.jsp')) and (not o.get_path(
).lower().endswith('.jspx')): # check if JSP file
continue
#logging.debug("JSP name = [" + o.get_path() + "]")
jsp_webapp = self.webapp_container(o)
#logging.debug("JSP webapp = [" + jsp_webapp + "]")
for reference in jsp_access.find_references_in_file(o):
#logging.debug("Reference [" + reference.value + "]")
# manipulate the reference pattern found
if not 'name=\"' in reference.value:
continue
searched_tiles_tag_name = reference.value.split("\"")[1]
#logging.debug("searching [" + searched_tiles_tag_name + "]")
#logging.debug('searching [' + jsp_webapp + '#' + searched_tiles_tag_name.replace("\\", "/") + ']')
# Searching in TilesDefinition
if (jsp_webapp + "#" + searched_tiles_tag_name.replace(
"\\", "/")) in self.tilesDefWebapp:
target = self.tilesDefWebapp[
jsp_webapp + "#" +
searched_tiles_tag_name.replace("\\", "/")]
#logging.debug("target tilesDef = [" + str(target) + "]")
create_link('callLink', o, target)
nb_links += 1
# Searching in Put-attribute
elif (jsp_webapp + "#" + searched_tiles_tag_name.replace(
"\\", "/")) in self.tilesPutAttWebapp:
target = self.tilesPutAttWebapp[
jsp_webapp + "#" +
searched_tiles_tag_name.replace("\\", "/")]
#logging.debug("target put-attribute = [" + str(target) + "]")
create_link('callLink', o, target)
nb_links2 += 1
else:
logging.debug(
"target not a tilesDef neither of put-attribute in the same webapp as the JSP = ["
+ str(searched_tiles_tag_name) + "] search was [" +
jsp_webapp + "#" +
searched_tiles_tag_name.replace("\\", "/") + "]")
if searched_tiles_tag_name in self.tilesPutAtt:
target = self.tilesPutAtt[searched_tiles_tag_name]
logging.debug(
"creating a less targeted link : target tilesDef = ["
+ str(target) + "]")
create_link('callLink', o, target)
nb_links3 += 1
else:
logging.debug(
"no link created between the JSP and the tilesDef or put-attribute"
)
nb_notfound += 1
# 3. Create the links
for link in links:
logging.debug("Creating link between " + str(link[1]) + " and " +
str(link[2]))
create_link(*link)
logging.debug(
"Nb of links created between JSP and Tiles Definition tags : " +
str(nb_links))
logging.debug(
"Nb of links created between JSP and Tiles put-attribute tags : " +
str(nb_links2))
logging.debug(
"Nb of links created between JSP and Tiles put-attribute tags without targeting the webapp: "
+ str(nb_links3))
logging.debug(
"Nb of links not created between JSP and Tiles Definition or put-attribute tags : "
+ str(nb_notfound))
self.global_nb_links = self.global_nb_links + nb_links + nb_links2 + nb_links3
def scan_phpfile(self, application, phpfile):
logging.debug("INIT scan_phpfile > " + str(phpfile.name))
# one RF for multiples patterns
rfCall = ReferenceFinder()
# Be careful, the order here is important !!!!!!
# lines comments in PHP files
rfCall.add_pattern('CSCOMMENTEDline',
before='',
element=r'^[\t ]*//.*$|^[\t ]*#.*$',
after='') # requires application_1_4_7 or above
# Mapping class
# example : @ORM\Table(name="affaire_r"
#/**
# * AffaireR
# * @ORM\Table(name="affaire_r")
# * @ORM\Entity(repositoryClass="NatachaBundle\Repository\AffaireRRepository")
# * @ORM\Entity
# * @ORM\Table(name="affaire_r")
#*/
rexTableAnnotation = '@ORM\\\\Table\(name=[\'"]([A-Za-z0-9\-_]+)[\'"]'
rfCall.add_pattern('DoctrineTableAnnotation',
before='',
element=rexTableAnnotation,
after='')
# Repository class
#rexRepositoryClass = '@ORM\\\\Entity\(repositoryClass="[A-Za-z0-9\-_\\\\]([A-Za-z0-9\-_]+)"'
rexRepositoryClass = '@ORM\\\\Entity\(repositoryClass=[\'"][A-Za-z0-9\-_\\\\]+[\\\\]([A-Za-z0-9\-_]+)[\'"]'
rfCall.add_pattern('DoctrineRepositoryClass',
before='',
element=rexRepositoryClass,
after='')
# Class from methods to Classes
# looking for class name after a namespace
# examples :
# -> LeftJoin("NatachaBundle:AffaireC",'ac','WITH','a.id=ac.ic'
# . "JOIN AppBundle:AuditLog audit "
# -> getRepository("AppTdexBundle:TCasier");
# $commune = $em-> getRepository('AppBundle:Commune')->getAutocomplete();
# ->from('NatachaBundle:AffaireC'
rexClass = '[\'"].*[A-Za-z0-9\-_]+[:]([A-Za-z0-9\-_]+)'
# examples :
# ->join('u.AffaireR', 'ar', 'WITH', 'a.id = ar.id')
rexClass = rexClass + '|' + '\([\'"][A-Za-z0-9]+[.]([A-Za-z0-9_-]+)'
# examples :
# ->from('AffaireR'
# ->join('AffaireR'
# ->leftJoin('AffaireR'
# ->innerJoin('AffaireR'
rexClass = rexClass + '|' + '([fF][rR][oO][mM]|[jJ][oO][iI][nN])\([\'"]([A-Za-z0-9]+)[\'"]'
rfCall.add_pattern('MethodToClassLink',
before='',
element=rexClass,
after='')
# PHP to symfony services
# examples :
rexphpToSymfonyService = '[gG][eE][tT]\(["\']([A-Za-z0-9_\.-]+)["\']'
rfCall.add_pattern('PhpToSymfonyService',
before='',
element=rexphpToSymfonyService,
after='')
try:
references = [
reference
for reference in rfCall.find_references_in_file(phpfile)
]
except FileNotFoundError:
logging.warning("Wrong file or file path, from Vn-1 or previous " +
str(phpfile))
else:
# for debugging and traversing the results
for reference in references:
#logging.debug(" DONE: reference found: >" +str(reference))
# identify the boolmark and parent object
bk_line_code = reference.bookmark.begin_line
most_specific_object = phpfile.find_most_specific_object(
bk_line_code, 1)
# Pattern 1 - Looking for the @ORM\Table Doctrine annotation
if reference.pattern_name == 'DoctrineTableAnnotation':
logging.debug("\t\t DoctrineTableAnnotation>" +
reference.value)
m0 = re.search(rexTableAnnotation, reference.value)
if m0:
tablename = m0.group(1)
try:
tableObject = self.tables[tablename]
try:
# get the class name
parentObject = self.mappingSectionClass[
most_specific_object.get_fullname()]
except KeyError:
#if there is no php class in the php section we keep the section as parent object
parentObject = most_specific_object
create_link("useLink", parentObject, tableObject,
reference.bookmark)
self.nbLinksDoctrineTableAnnotation += 1
except KeyError:
logging.warning(
"\t\t Couldn't find table in local schema : "
+ tablename)
# Pattern 2 - Looking for the repository class
if reference.pattern_name == 'DoctrineRepositoryClass':
logging.debug("\t\t DoctrineRepositoryClass>" +
reference.value)
m0 = re.search(rexRepositoryClass, reference.value)
if m0:
repositoryClassName = m0.group(1)
try:
repositoryClassObject = self.phpClassesByName[
repositoryClassName]
try:
# get the class name for this section
parentObject = self.mappingSectionClass[
most_specific_object.get_fullname()]
except KeyError:
#if there is no php class in the php section we keep the section as parent object
parentObject = most_specific_object
create_link("useLink", parentObject,
repositoryClassObject,
reference.bookmark)
self.nbLinksDoctrineRepositoryClass += 1
except KeyError:
logging.warning(
"\t\t Couldn't find phpClass in local schema (DoctrineRepositoryClass) : "
+ repositoryClassName)
# Pattern 3 - Looking for method with namespace to class link to create
if reference.pattern_name == 'MethodToClassLink':
logging.debug("\t\t MethodToClassLink>" + reference.value)
m0 = re.search(rexClass, reference.value)
if m0:
classname = m0.group(1)
if classname == None: classname = m0.group(2)
if classname == None: classname = m0.group(4)
logging.debug("\t\t classname>" + classname)
try:
classObject = self.phpClassesByName[classname]
parentObject = most_specific_object
create_link("useLink", parentObject, classObject,
reference.bookmark)
self.nbLinksMethodToClass += 1
except KeyError:
logging.warning(
"\t\t Couldn't find phpClass in local schema (MethodToClassLink) : "
+ classname)
if reference.pattern_name == 'PhpToSymfonyService':
logging.debug("\t\t PhpToSymfonyService>" +
reference.value)
m0 = re.search(rexphpToSymfonyService, reference.value)
if m0:
servicename = m0.group(1)
try:
serviceObject = self.phpSymfonyServices[
servicename]
parentObject = most_specific_object
create_link("callLink", parentObject,
serviceObject, reference.bookmark)
self.nbLinksPHPToSymfonyService += 1
except KeyError:
logging.warning(
"\t\t Couldn't find symfony service in local schema (PhpToSymfonyService) : "
+ servicename)
def scan_yamlfile(self, application, yfile):
logging.debug("INIT scan_yamlfile > " + str(yfile.name))
bContainsService = False
# one RF for multiples patterns
rfCall = ReferenceFinder()
# to make sure this file contains Symfony services
rfCall.add_pattern('containsService',
before='',
element='services:',
after='')
# to collect the Symfony service name
rexSymfonyServiceName = '^[ ][ ][ ][ ]([A-Za-z0-9_\.-]+)[:]'
rfCall.add_pattern('SymfonyServiceName',
before='',
element=rexSymfonyServiceName,
after='')
# to collect the service clas name
rexSymfonyServiceClassName = '^[ ][ ][ ][ ][ ][ ][ ][ ][cC][lL][aA][sS][sS][:][\t ]*[A-Za-z0-9\-_\\\\]+[\\\\]([A-Za-z0-9\-_]+)'
rfCall.add_pattern('SymfonyServiceClassName',
before='',
element=rexSymfonyServiceClassName,
after='')
try:
references = [
reference
for reference in rfCall.find_references_in_file(yfile)
]
except FileNotFoundError:
logging.warning("Wrong file or file path, from Vn-1 or previous " +
str(yfile))
else:
# for debugging and traversing the results
for reference in references:
#logging.debug(" DONE: reference found: >" +str(reference))
# identify the boolmark and parent object
bk_line_code = reference.bookmark.begin_line
most_specific_object = yfile.find_most_specific_object(
bk_line_code, 1)
# Service name
if reference.pattern_name == 'containsService':
logging.debug("\t\t containsService>" + reference.value)
bContainsService = True
self.nbYAMLFileWithSymfonyServices += 1
# Service name
if bContainsService and reference.pattern_name == 'SymfonyServiceName':
logging.debug("\t\t SymfonyServiceName>" + str(reference))
m0 = re.search(rexSymfonyServiceName, reference.value)
if m0:
self.currentSymfonyServiceName = m0.group(1)
# Service class name
if bContainsService and reference.pattern_name == 'SymfonyServiceClassName':
logging.debug("\t\t\t SymfonyServiceClassName>" +
str(reference))
logging.debug("\t\t\t\t self.currentSymfonyServiceName>" +
self.currentSymfonyServiceName)
m0 = re.search(rexSymfonyServiceClassName, reference.value)
if m0:
symfonyServiceClassName = m0.group(1)
try:
oclass = self.phpClassesByName[
symfonyServiceClassName]
osymfonyservice = self.phpSymfonyServices[
self.currentSymfonyServiceName]
logging.debug(
"\t\t\t\t creating link betwween SymfonyService "
+ osymfonyservice.get_fullname() +
" and class " + oclass.get_fullname())
create_link("useLink", osymfonyservice, oclass,
reference.bookmark)
self.nbLinksSymfonyServiceToServiceClass += 1
except KeyError:
logging.warning('Not able to find class ' +
symfonyServiceClassName)
def links_through_ActionMapping_SpringMVC(self, application):
nb_links = 0
logging.info(
"==> Solving the following problem : Missing links between Portlet (JSP) and Java Methods"
)
# 1. search all references in all files
logging.info(
"Scanning Portlets for calls to Java Methods through Annotation ActionMapping (Spring MVC)"
)
Java_Methods = {}
for Java_Method in application.search_objects(category='JV_METHOD',
load_properties=True):
#logging.debug("Java Method [" + str(Java_Method) + "]")
for Annotation in Java_Method.get_property(
"CAST_Java_AnnotationMetrics.Annotation"):
if str(Annotation).startswith("@ActionMapping"):
#logging.debug("Java Method get Property Annotation StartsWith[" + Annotation + "]")
if '"' in str(Annotation):
Target = str(Annotation).split("\"")[1]
#logging.debug("Java Method [" + str(Java_Method) + "] Target[" + Target + "]")
Java_Methods[Target] = Java_Method
# 2. scan each JSP file
# we search a pattern
portlet_and_method_access = ReferenceFinder()
#portlet_and_method_access.add_pattern("ActionMapping", before="", element="<portlet:actionURL name=\"[a-zA-Z0-9_-]+", after="")
portlet_and_method_access.add_pattern("ActionMapping",
before="<portlet:actionURL",
element="[A-Za-z0-9\=_\-\" ]+",
after="")
links = []
for o in application.get_files(['CAST_Web_File']):
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
for reference in portlet_and_method_access.find_references_in_file(
o):
#logging.debug("Reference " + reference.value)
# manipulate the reference pattern found
if not 'name=\"' in reference.value:
continue
searched_java_method_name = reference.value.split("\"")[1]
#logging.debug("searching " + searched_java_method_name)
try:
Java_Method = Java_Methods[searched_java_method_name]
links.append(('callLink', reference.object, Java_Method,
reference.bookmark))
except:
pass
# 3. Create the links
for link in links:
logging.debug("Creating link between " + str(link[1]) + " and " +
str(link[2]))
create_link(*link)
nb_links = nb_links + 1
logging.debug("Nb of links created " + str(nb_links))
self.global_nb_links = self.global_nb_links + nb_links
def links_through_RequestMapping_SpringMVC(self, application):
nb_links = 0
logging.info(
"==> Solving the following problem : Missing links between Java Methods and Portlet (JSP) response "
)
# 1. search all references in all files
logging.info(
"Scanning Java Methods for calls to Portlet through Annotation RequestMapping (Spring MVC)"
)
Java_Methods = {}
Request_Mappings = {}
# annotation RequestMapping
request_mapping = next(
application.get_objects_by_name(name="RequestMapping",
external=True))
#logging.debug("Annotation request Mapping [" + str(request_mapping) + "]")
# all method with a link to annotation
for link in application.links().has_callee([
request_mapping
]).has_caller(application.objects().is_executable()).load_positions():
JSPRedirection = False
#logging.debug("caller (Method calling the RequestMapping Annotation) [" + str(link.get_caller()) + "]")
Java_Methods = application.search_objects(
name=link.get_caller().get_name(), load_properties=True)
for Java_Method in Java_Methods:
#logging.debug("Java Method =[" + str(Java_Method) + "]")
for Annotation in Java_Method.get_property(
"CAST_Java_AnnotationMetrics.Annotation"):
if str(Annotation).startswith(
"@RequestMapping(params=\"action="):
AnnotationAction = str(Annotation).split(
"\"")[1].split("=")[1]
#logging.debug("Annotation Action =[" + AnnotationAction +"]")
# caller code :
code = link.get_caller().get_positions()[0].get_code()
#logging.debug(code)
#Search in code the following
#return \"[A-Za-z0-9_-]+\"
#JSP with same name can be part of different packages
main_package = link.get_caller().get_fullname().split(".")[4]
#logging.debug("Main package [" + main_package + "] + caller name = [" + link.get_caller().get_fullname() + "]")
# Return all words beginning with character 'a', as an iterator yielding match objects
it = re.finditer("return \"[A-Za-z0-9_\-\/]+\"", code)
for match in it:
#logging.debug("match = [" + format(match.group()) + "]")
if "/" in format(match.group()):
redirectJSP = format(
match.group()).split("/")[1].split("\"")[0] + ".jsp"
if "/" not in format(match.group()):
redirectJSP = format(match.group()).split("\"")[1] + ".jsp"
#logging.debug("Redirect JSP [" + redirectJSP + "]")
redirectJSP_iter = application.get_objects_by_name(
name=redirectJSP)
for redirectJSP_object in redirectJSP_iter:
#logging.debug("Redirect JSP [" + redirectJSP + "] + [" + str(redirectJSP_object) + "]")
#logging.debug("Redirect JSP full name[" + redirectJSP_object.get_fullname() + "]")
if main_package in redirectJSP_object.get_fullname():
#logging.debug("Main package found for JSP [" + main_package +"]")
Request_Mappings[
str(main_package +
AnnotationAction)] = redirectJSP_object
JSPRedirection = True
if JSPRedirection == False:
#logging.debug(code)
pass
# 2. scan each Java method
# we search a pattern
request_mapping_access = ReferenceFinder()
request_mapping_access.add_pattern(
"RequestMapping",
before="",
element=
"response\.setRenderParameter\(\"action\",[ ]+\"[a-zA-Z0-9_-]+",
after="")
links = []
for o in application.get_files(['JV_FILE']):
# check if file is analyzed source code, or if it generated (Unknown)
if not o.get_path():
continue
for reference in request_mapping_access.find_references_in_file(o):
#logging.debug("Reference " + reference.value)
# manipulate the reference pattern found
searched_request_mapping = reference.value.split("\"")[3]
#logging.debug("searching searched_request_mapping [" + searched_request_mapping + "]")
searched_package = o.get_path().split("\\src\\")[1].split(
"\\")[0]
#logging.debug("o.get_path() [" + o.get_path() + "]")
#logging.debug("searched package [" + searched_package + "]")
try:
JSP_Redirect = Request_Mappings[str(
searched_package + searched_request_mapping)]
links.append(('callLink', reference.object, JSP_Redirect,
reference.bookmark))
pass
except:
pass
# 3. Create the links
for link in links:
logging.debug("Creating link between " + str(link[1]) + " and " +
str(link[2]))
create_link(*link)
nb_links = nb_links + 1
logging.debug("Nb of links created " + str(nb_links))
self.global_nb_links = self.global_nb_links + nb_links