def parse_output(target, is_archive, foss_file, ninka_file, opts):
"""
Parses the FOSSology and Ninka scan output, and places the results in a
unified file (for our internal use).
The file format is archive_name;file_name;FOSSology_output;ninka_output
"""
verbose = False
if opts:
if opts == "-v":
verbose = True
if not os.path.isfile(foss_file):
sys.stderr.write("ERROR: " + foss_file + " not found")
exit(1)
elif not os.path.isfile(ninka_file):
sys.stderr.write("ERROR: " + ninka_file + " not found")
exit(1)
else:
out_name = get_file_from_absolute_path(target)
if verbose:
print("Creating combined output file")
combined_out = paths.SCANNER_OUTPUT_PATH + "/"
combined_out += out_name + ".dual_out.txt"
check_file(combined_out)
if not is_archive:
archive_name = "NONE"
else:
archive_name = out_name
foss = open(foss_file, 'r')
ninka = open(ninka_file, 'r')
output_file = open(combined_out, 'w')
foss_lines = str(foss.read()).split("\n")
ninka_lines = str(ninka.read()).split("\n")
#If not equal, go with the shortest one, will put in later
if len(foss_lines) == len(ninka_lines):
for i in range(0, len(foss_lines) - 1):
foss_out = foss_parser(foss_lines[i])
ninka_out = ninka_parser(ninka_lines[i])
if foss_out[0].strip() == ninka_out[0].strip():
output = archive_name + ";"
output += foss_out[0].strip() + ";"
output += foss_out[1] + ";"
output += ninka_out[1] + "\n"
output_file.write(output)
foss.close()
ninka.close()
output_file.close()
if verbose:
print("Output file complete")
def parse_output(target, is_archive, foss_file, ninka_file):
if not os.path.isfile(foss_file):
print("ERROR: " + foss_file + " not found")
exit(1)
elif not os.path.isfile(ninka_file):
print("ERROR: " + ninka_file + " not found")
exit(1)
else:
out_name = get_file_from_absolute_path(target)
print("Creating combined output file")
combined_out = paths.SCANNER_OUTPUT_PATH + "/"
combined_out += out_name + ".dual_out.txt"
check_file(combined_out)
if not is_archive:
archive_name = "NONE"
else:
archive_name = out_name
foss = open(foss_file, "r")
ninka = open(ninka_file, "r")
output_file = open(combined_out, "w")
l1 = str(foss.read()).split("\n")
l2 = str(ninka.read()).split("\n")
# If not equal, go with the shortest one, will put in later
if len(l1) == len(l2):
for i in range(0, len(l1) - 1):
foss_out = foss_parser(l1[i])
ninka_out = ninka_parser(l2[i])
if foss_out[0].strip() == ninka_out[0].strip():
output = archive_name + ";"
output += foss_out[0].strip() + ";"
output += foss_out[1] + ";"
output += ninka_out[1] + "\n"
output_file.write(output)
foss.close()
ninka.close()
output_file.close()
print("Output file complete")
def populateFileInfo(self,scanOption):
'''Runs the two scanners and parses their output into the fileInfo object'''
''' Get File Type'''
mime = MimeTypes()
self.fileType = mime.guess_type(self.filePath)[0]
if self.fileType == None:
self.fileType = 'Unknown'
'''Check to see if file is cached.'''
cached = self.isCached()
'''If it isn't cached, run scans, else get file from database.'''
if cached == -1:
if scanOption == 'fossology':
'''Run fossology'''
'''Fossology doesn't return an exit code of 0 so we must always catch the output.'''
try:
fossOutput = subprocess.check_output([settings.FOSSOLOGY_PATH,
self.filePath])
except OSError as ose:
print "Error running FOSSology nomos, check your path to nomos in settings.py"
except Exception as e:
fossOutput = str(e.output)
'''Parse outputs'''
(fileName, fossLicense) = output_parser.foss_parser(fossOutput)
self.licenseInfoInFile.append(fossLicense)
self.licenseComments = "#FOSSology "
self.licenseComments += fossLicense
else:
'''Scan to find licenses'''
'''Run Ninka'''
ninkaOutput = subprocess.check_output(
[settings.NINKA_PATH, self.filePath],
preexec_fn=lambda: signal(SIGPIPE, SIG_DFL)
)
'''Run fossology'''
'''Fossology doesn't return an exit code of 0 so we must always catch the output.'''
try:
fossOutput = subprocess.check_output([settings.FOSSOLOGY_PATH,
self.filePath])
except OSError as ose:
print "Error running FOSSology nomos, check your path to nomos in settings.py"
except Exception as e:
fossOutput = str(e.output)
'''Parse outputs'''
(fileName, ninkaLicense) = output_parser.ninka_parser(ninkaOutput)
(fileName, fossLicense) = output_parser.foss_parser(fossOutput)
'''Get extracted text from ninka "senttok" file'''
try:
with open(self.filePath + ".senttok", 'r') as f:
for line in f:
if ninkaLicense in line:
line_tok = line.split(';')
self.extractedText += line_tok[3] + "\n"
self.extractedText += line_tok[4]
except Exception as e:
'''Do nothing, we just wont have extracted text for this license.'''
'''License merging logic.'''
fossLicense = fossLicense.upper().strip()
ninkaLicense = ninkaLicense.upper().strip()
match = output_parser.lic_compare(fossLicense, ninkaLicense)
if match and fossLicense != 'ERROR':
self.licenseInfoInFile.append(fossLicense)
elif match and fossLicense == 'ERROR':
self.licenseInfoInFile.append(ninkaLicense)
elif not match and fossLicense == 'UNKNOWN':
self.licenseInfoInFile.append(ninkaLicense)
else:
self.licenseInfoInFile.append("NO ASSERTION")
self.licenseComments = "#FOSSology "
self.licenseComments += fossLicense
self.licenseComments += " #Ninka "
self.licenseComments += ninkaLicense
else:
with MySQLdb.connect(host=settings.database_host,
user=settings.database_user,
passwd=settings.database_pass,
db=settings.database_name) as dbCursor:
self.getChecksum()
self.getFileInfoFromChecksum( dbCursor)
def populateFileInfo(self, scanOption):
'''Runs the two scanners and parses their output into the fileInfo object'''
''' Get File Type'''
mime = MimeTypes()
self.fileType = mime.guess_type(self.filePath)[0]
if self.fileType == None:
self.fileType = 'Unknown'
'''Check to see if file is cached.'''
cached = self.isCached()
'''If it isn't cached, run scans, else get file from database.'''
if cached == -1:
if scanOption == 'fossology':
'''Run fossology'''
'''Fossology doesn't return an exit code of 0 so we must always catch the output.'''
try:
fossOutput = subprocess.check_output(
[settings.FOSSOLOGY_PATH, self.filePath])
except OSError as ose:
print "Error running FOSSology nomos, check your path to nomos in settings.py"
except Exception as e:
fossOutput = str(e.output)
'''Parse outputs'''
(fileName, fossLicense) = output_parser.foss_parser(fossOutput)
self.licenseInfoInFile.append(fossLicense)
self.licenseComments = "#FOSSology "
self.licenseComments += fossLicense
else:
'''Scan to find licenses'''
'''Run Ninka'''
ninkaOutput = subprocess.check_output(
[settings.NINKA_PATH, self.filePath],
preexec_fn=lambda: signal(SIGPIPE, SIG_DFL))
'''Run fossology'''
'''Fossology doesn't return an exit code of 0 so we must always catch the output.'''
try:
fossOutput = subprocess.check_output(
[settings.FOSSOLOGY_PATH, self.filePath])
except OSError as ose:
print "Error running FOSSology nomos, check your path to nomos in settings.py"
except Exception as e:
fossOutput = str(e.output)
'''Parse outputs'''
(fileName,
ninkaLicense) = output_parser.ninka_parser(ninkaOutput)
(fileName, fossLicense) = output_parser.foss_parser(fossOutput)
'''Get extracted text from ninka "senttok" file'''
try:
with open(self.filePath + ".senttok", 'r') as f:
for line in f:
if ninkaLicense in line:
line_tok = line.split(';')
self.extractedText += line_tok[3] + "\n"
self.extractedText += line_tok[4]
except Exception as e:
'''Do nothing, we just wont have extracted text for this license.'''
'''License merging logic.'''
fossLicense = fossLicense.upper().strip()
ninkaLicense = ninkaLicense.upper().strip()
match = output_parser.lic_compare(fossLicense, ninkaLicense)
if match and fossLicense != 'ERROR':
self.licenseInfoInFile.append(fossLicense)
elif match and fossLicense == 'ERROR':
self.licenseInfoInFile.append(ninkaLicense)
elif not match and fossLicense == 'UNKNOWN':
self.licenseInfoInFile.append(ninkaLicense)
else:
self.licenseInfoInFile.append("NO ASSERTION")
self.licenseComments = "#FOSSology "
self.licenseComments += fossLicense
self.licenseComments += " #Ninka "
self.licenseComments += ninkaLicense
else:
with MySQLdb.connect(host=settings.database_host,
user=settings.database_user,
passwd=settings.database_pass,
db=settings.database_name) as dbCursor:
self.getChecksum()
self.getFileInfoFromChecksum(dbCursor)
