def parse_output(target, is_archive, foss_file, ninka_file, opts): """ Parses the FOSSology and Ninka scan output, and places the results in a unified file (for our internal use). The file format is archive_name;file_name;FOSSology_output;ninka_output """ verbose = False if opts: if opts == "-v": verbose = True if not os.path.isfile(foss_file): sys.stderr.write("ERROR: " + foss_file + " not found") exit(1) elif not os.path.isfile(ninka_file): sys.stderr.write("ERROR: " + ninka_file + " not found") exit(1) else: out_name = get_file_from_absolute_path(target) if verbose: print("Creating combined output file") combined_out = paths.SCANNER_OUTPUT_PATH + "/" combined_out += out_name + ".dual_out.txt" check_file(combined_out) if not is_archive: archive_name = "NONE" else: archive_name = out_name foss = open(foss_file, 'r') ninka = open(ninka_file, 'r') output_file = open(combined_out, 'w') foss_lines = str(foss.read()).split("\n") ninka_lines = str(ninka.read()).split("\n") #If not equal, go with the shortest one, will put in later if len(foss_lines) == len(ninka_lines): for i in range(0, len(foss_lines) - 1): foss_out = foss_parser(foss_lines[i]) ninka_out = ninka_parser(ninka_lines[i]) if foss_out[0].strip() == ninka_out[0].strip(): output = archive_name + ";" output += foss_out[0].strip() + ";" output += foss_out[1] + ";" output += ninka_out[1] + "\n" output_file.write(output) foss.close() ninka.close() output_file.close() if verbose: print("Output file complete")
def parse_output(target, is_archive, foss_file, ninka_file): if not os.path.isfile(foss_file): print("ERROR: " + foss_file + " not found") exit(1) elif not os.path.isfile(ninka_file): print("ERROR: " + ninka_file + " not found") exit(1) else: out_name = get_file_from_absolute_path(target) print("Creating combined output file") combined_out = paths.SCANNER_OUTPUT_PATH + "/" combined_out += out_name + ".dual_out.txt" check_file(combined_out) if not is_archive: archive_name = "NONE" else: archive_name = out_name foss = open(foss_file, "r") ninka = open(ninka_file, "r") output_file = open(combined_out, "w") l1 = str(foss.read()).split("\n") l2 = str(ninka.read()).split("\n") # If not equal, go with the shortest one, will put in later if len(l1) == len(l2): for i in range(0, len(l1) - 1): foss_out = foss_parser(l1[i]) ninka_out = ninka_parser(l2[i]) if foss_out[0].strip() == ninka_out[0].strip(): output = archive_name + ";" output += foss_out[0].strip() + ";" output += foss_out[1] + ";" output += ninka_out[1] + "\n" output_file.write(output) foss.close() ninka.close() output_file.close() print("Output file complete")
def populateFileInfo(self,scanOption): '''Runs the two scanners and parses their output into the fileInfo object''' ''' Get File Type''' mime = MimeTypes() self.fileType = mime.guess_type(self.filePath)[0] if self.fileType == None: self.fileType = 'Unknown' '''Check to see if file is cached.''' cached = self.isCached() '''If it isn't cached, run scans, else get file from database.''' if cached == -1: if scanOption == 'fossology': '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output([settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, fossLicense) = output_parser.foss_parser(fossOutput) self.licenseInfoInFile.append(fossLicense) self.licenseComments = "#FOSSology " self.licenseComments += fossLicense else: '''Scan to find licenses''' '''Run Ninka''' ninkaOutput = subprocess.check_output( [settings.NINKA_PATH, self.filePath], preexec_fn=lambda: signal(SIGPIPE, SIG_DFL) ) '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output([settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, ninkaLicense) = output_parser.ninka_parser(ninkaOutput) (fileName, fossLicense) = output_parser.foss_parser(fossOutput) '''Get extracted text from ninka "senttok" file''' try: with open(self.filePath + ".senttok", 'r') as f: for line in f: if ninkaLicense in line: line_tok = line.split(';') self.extractedText += line_tok[3] + "\n" self.extractedText += line_tok[4] except Exception as e: '''Do nothing, we just wont have extracted text for this license.''' '''License merging logic.''' fossLicense = fossLicense.upper().strip() ninkaLicense = ninkaLicense.upper().strip() match = output_parser.lic_compare(fossLicense, ninkaLicense) if match and fossLicense != 'ERROR': self.licenseInfoInFile.append(fossLicense) elif match and fossLicense == 'ERROR': self.licenseInfoInFile.append(ninkaLicense) elif not match and fossLicense == 'UNKNOWN': self.licenseInfoInFile.append(ninkaLicense) else: self.licenseInfoInFile.append("NO ASSERTION") self.licenseComments = "#FOSSology " self.licenseComments += fossLicense self.licenseComments += " #Ninka " self.licenseComments += ninkaLicense else: with MySQLdb.connect(host=settings.database_host, user=settings.database_user, passwd=settings.database_pass, db=settings.database_name) as dbCursor: self.getChecksum() self.getFileInfoFromChecksum( dbCursor)
def populateFileInfo(self, scanOption): '''Runs the two scanners and parses their output into the fileInfo object''' ''' Get File Type''' mime = MimeTypes() self.fileType = mime.guess_type(self.filePath)[0] if self.fileType == None: self.fileType = 'Unknown' '''Check to see if file is cached.''' cached = self.isCached() '''If it isn't cached, run scans, else get file from database.''' if cached == -1: if scanOption == 'fossology': '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output( [settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, fossLicense) = output_parser.foss_parser(fossOutput) self.licenseInfoInFile.append(fossLicense) self.licenseComments = "#FOSSology " self.licenseComments += fossLicense else: '''Scan to find licenses''' '''Run Ninka''' ninkaOutput = subprocess.check_output( [settings.NINKA_PATH, self.filePath], preexec_fn=lambda: signal(SIGPIPE, SIG_DFL)) '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output( [settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, ninkaLicense) = output_parser.ninka_parser(ninkaOutput) (fileName, fossLicense) = output_parser.foss_parser(fossOutput) '''Get extracted text from ninka "senttok" file''' try: with open(self.filePath + ".senttok", 'r') as f: for line in f: if ninkaLicense in line: line_tok = line.split(';') self.extractedText += line_tok[3] + "\n" self.extractedText += line_tok[4] except Exception as e: '''Do nothing, we just wont have extracted text for this license.''' '''License merging logic.''' fossLicense = fossLicense.upper().strip() ninkaLicense = ninkaLicense.upper().strip() match = output_parser.lic_compare(fossLicense, ninkaLicense) if match and fossLicense != 'ERROR': self.licenseInfoInFile.append(fossLicense) elif match and fossLicense == 'ERROR': self.licenseInfoInFile.append(ninkaLicense) elif not match and fossLicense == 'UNKNOWN': self.licenseInfoInFile.append(ninkaLicense) else: self.licenseInfoInFile.append("NO ASSERTION") self.licenseComments = "#FOSSology " self.licenseComments += fossLicense self.licenseComments += " #Ninka " self.licenseComments += ninkaLicense else: with MySQLdb.connect(host=settings.database_host, user=settings.database_user, passwd=settings.database_pass, db=settings.database_name) as dbCursor: self.getChecksum() self.getFileInfoFromChecksum(dbCursor)