def mergeLibrariesByPaths(pathFirstLibary, pathSecondLibrary): rootFirst = supermod.libraryType() rootFirst = supermod.parse(str(pathFirstLibary), True) rootSecond = supermod.libraryType() rootSecond = supermod.parse(str(pathSecondLibrary), True) logger.info("Merging two libraries process: STARTED") mergeLibraries(rootFirst, rootSecond) logger.info("Merging two libraries process: FINISHED") xmlFileName = str(Path.cwd() / "outFiles" / "mergedLibs" / pathFirstLibary.name) exportLib2XML(xmlFileName, rootFirst) xsd = str(Path.cwd() / "inputFiles" / "XSD_Schema" / "library.xsd") logger.info("XSD Validation of XML processes: STARTED") if xmlValidator(xmlFileName, xsd): print("XSD Schema validation: Success") logger.info( "XSD Validation of XML processes: FINISHED - with result: SUCCESS") else: print("XSD Schema validation: Failed") logger.error( "ERROR: XSD Validation of XML processes: FINISHED - with result: FAILED") logger.info("% Threat mitigation validation process: STARTED") print("\nTesting of Threat mitigations:\n") libMitigationTest(xmlFileName,[]) logger.info("% Threat mitigation validation process: FINISHED") return "The last version (merged) of the library is located in the path %s.\n"%xmlFileName
def readInfoFromXml(lib_path, columns): data = list() root = sl.parse(str(lib_path), silence=True) for component in root.get_components().get_component(): sizeWeaks = len(component.get_weaknesses().get_weakness()) sizeControls = len(component.get_controls().get_control()) sizeUseCases = len(component.get_usecases().get_usecase()) sizeThreats = 0 for usecase in component.get_usecases().get_usecase(): sizeThreats += len(usecase.get_threats().get_threat()) data.append([ root.get_name(), component.get_name(), sizeUseCases, sizeThreats, sizeWeaks, sizeControls ]) data = pd.DataFrame(data, columns=columns) array = [[ root.get_name(), "TOTAL:", data[columns[2]].sum(), data[columns[3]].sum(), data[columns[4]].sum(), data[columns[5]].sum() ]] dfm = pd.DataFrame(array, columns=columns) data = data.append(dfm) return data
def getAllDataFromLibrary(lib_path): root = sl.parse(str(lib_path), silence=True) componentDefinitions = root.get_componentDefinitions( ).get_componentDefinition() categoryComponents = root.get_categoryComponents().get_categoryComponent() supportedstandards = root.get_supportedStandards().get_supportedStandard() riskPatterns = root.get_components().get_component() rules = root.get_rules().get_rule() threats = list() useCases = list() weaknesses = list() controls = list() for riskPattern in riskPatterns: contls = riskPattern.get_controls().get_control() for contl in contls: controls.append(contl) weaks = riskPattern.get_weaknesses().get_weakness() for weak in weaks: weaknesses.append(weak) uses = riskPattern.get_usecases().get_usecase() for use in uses: useCases.append(use) thrs = use.get_threats().get_threat() for thr in thrs: threats.append(thr) return componentDefinitions, categoryComponents, supportedstandards, rules, riskPatterns
def getControlsList(pathFile, controlsList): root = sl.parse(str(Path.cwd().parents[0] / "libraries" / pathFile), silence=True) components = root.get_components() supportedStandards = root.get_supportedStandards() for component in components.get_component(): controls = component.get_controls() for control in controls.get_control(): controlName = control.get_name() controlRef = control.get_ref() controlDesc = control.get_desc() standards = control.get_standards().get_standard() controlStandardName = "" controlStandardRef = "" if len(standards) > 0: for standard in standards: for supportedStandard in supportedStandards.get_supportedStandard( ): if standard.get_supportedStandardRef( ) == supportedStandard.get_ref(): controlStandardName = supportedStandard.get_name() controlStandardRef = standard.get_ref() controlsList.append([ controlName, controlRef, controlDesc, controlStandardName, controlStandardRef ]) return controlsList
def libMitigationTestASVSv4(lib_path, exceptions, forAsvs4=False): text = "" errors = list() errorsIntegrity = list() root = sl.parse(str(lib_path), silence=True) components = root.get_components() libraryRef = root.get_ref() for component in components.get_component(): asvs4Controls = list() for control in component.get_controls().get_control(): asvs4 = False asvs3 = False for standard in control.get_standards().get_standard(): if standard.get_supportedStandardRef()[0:12] == 'owasp-asvs4-': asvs4 = True if standard.get_supportedStandardRef()[0:11].lower( ) == 'owasp-asvs-': asvs3 = True if asvs4: asvs4Controls.append(control.get_ref()) if not asvs3 and not asvs4: asvs4Controls.append(control.get_ref()) for usecase in component.get_usecases().get_usecase(): for threat in usecase.get_threats().get_threat(): totalMitigation = 0 for control in threat.get_controls().get_control(): if control.get_ref() in asvs4Controls: totalMitigation = totalMitigation + int( control.get_mitigation()) #We also check we have the same value for control ref and control mitigation under the weaknesses tree for weakness in threat.get_weaknesses().get_weakness(): for contrl in weakness.get_controls().get_control( ): if control.get_ref() == contrl.get_ref( ) and control.get_mitigation( ) != contrl.get_mitigation(): errorsIntegrity.append(control.get_ref()) if [component.get_ref(), threat.get_ref()] in exceptions: totalMitigation = 100 if totalMitigation < 100 and totalMitigation > 0: print("Component Id: %s --> Threat: %s == Mitigation: %s" % (component.get_ref(), threat.get_ref(), totalMitigation)) errors.append(threat.get_ref()) if totalMitigation > 140: print("Component Id: %s --> Threat: %s == Mitigation: %s" % (component.get_ref(), threat.get_ref(), totalMitigation)) errors.append(threat.get_ref()) text = "" if len(errorsIntegrity) != 0: text += "Threat mitigation integrity fails in the following countermeasures: %s\n" % str( errorsIntegrity).replace("[", "").replace("]", "") if len(errors) != 0: text += "Threat mitigation fails in the following threats: %s\n" % str( errors).replace("[", "").replace("]", "") return text
def updateLibrary(xmlFilePath, xmlFilePathDestination): rootClass = sl.libraryType() fileName = xmlFilePath[xmlFilePath.rfind("/") + 1:xmlFilePath.rfind(".xml")] f = open( str(Path.cwd() / "src" / "resources") / 'updateDemolibProperties.yaml') libProperties = yaml.safe_load(f) f.close() if fileName in libProperties: rootClass = sl.parse(xmlFilePath, True) logger.info( "Component Definitions and Categories creation process for the library '%s' is STARTED." % fileName) updateCategoryComponents(rootClass, libProperties[fileName]) updateComponentDefinitions(rootClass, libProperties[fileName]) logger.info( "Component Definitions and Categories creation process for the library '%s' is FINISHED." % fileName) logger.info("Rules deletion process for the library '%s' is STARTED." % fileName) removeRules(rootClass, libProperties[fileName]) logger.info( "Rules deletion process for the library '%s' is FINISHED." % fileName) logger.info("Rules creation process for the library '%s' is STARTED." % fileName) createRules(rootClass, libProperties[fileName]) logger.info( "Rules creation process for the library '%s' is FINISHED." % fileName) exportLib2XML(xmlFilePathDestination, rootClass)
def convertExcelToXml(excelFilePath): rules, errorsRules = getRulesFromExcel(excelFilePath) #generateDataFrameHTMLReport(rules) (productProperties,componentDefinitions, supportedStandards, errors)=getPropertiesFromExcel(excelFilePath) if errors == "": xmlFileName, errors = generateXmlFromXlsxFile(excelFilePath, productProperties,componentDefinitions, supportedStandards, rules, errors) libraryName = supermod.parse(str(xmlFileName), silence=True).get_name() errors+="Library '%s' was converted to XML file successfully and the output file is in the path '%s'.\n"%(libraryName, xmlFileName) return errorsRules+errors
def convertRulesFromExcelToXML(excelFilePath): rules, errorsRules = getRulesFromExcelRulesEditor(excelFilePath) #generateDataFrameHTMLReport(rules) (productProperties,componentDefinitions, supportedStandards) = getSimplifiedPropertiesFromExcel(excelFilePath) xmlFileName = generateXmlFromRulesXlsxFile(excelFilePath, productProperties,componentDefinitions, supportedStandards, rules) libraryName = supermod.parse(str(xmlFileName), silence=True).get_name() okMessage = "Rules in Excel Utility '%s' were converted to XML file successfully and the output file is in the path '%s'.\n"%(excelFilePath, xmlFileName) logger.info("Excel Rules converted successfully to XML") return errorsRules+okMessage
def uploadLibraryByApi(pathLibrary, urlServer, apiToken): libRef=sl.parse(str(pathLibrary), silence=True).get_ref() responsePost=postLibrary(urlServer, libRef, apiToken, pathLibrary) if(responsePost.status_code == 201): text="The library %s was uploaded well.\n" % (libRef) logger.info(text) return text else: text="The library %s was not uploaded well, the api call has got the next response: %s.\n" % (libRef, responsePost.text) logger.error(text) return text
def removeLibraryByApi(pathLibrary, urlServer, apiToken): libRef=sl.parse(str(pathLibrary), silence=True).get_ref() responseGet= getLibrary(urlServer, libRef, apiToken) if responseGet.status_code == 200: response=removeLibrary(urlServer, libRef, apiToken) if response.status_code == 204: logger.info("The library %s was removed well with the status code: %s" % (libRef, response.status_code)) else: logger.error("The library %s was not removed well, the api call has got the next response: %s" % (libRef, response.text)) else: logger.error("The Get libraries call failed for the library %s and with the response: %s" % (libRef, responseGet.text))
def addReferencesToLibrariesByExcelFile(excel_path, replaceLibs, fileName): results = "" fileName = Path(fileName).name.replace(".xlsx", "") path_libs = Path.cwd() / "libraries" outlibs_path = Path.cwd() / "outFiles" / "outputLibs" sheet_name = "Mapping References" dfm = pd.read_excel(str(excel_path), sheet_name=sheet_name, header=0) dfm = extractAllReferencesForAllControls(dfm) dfm.sort_values(by="Control Id", inplace=True) dfm = dfm.drop_duplicates() controls = list() for lib in os.listdir(str(path_libs)): if lib.endswith(".xml"): changed = False root = sl.parse(str(path_libs / lib), silence=True) for component in root.get_components().get_component(): for control in component.get_controls().get_control(): found = False refNames, refUrls = getReferencesByControlId( control.get_ref(), dfm) for referenceName, referenceUrl in zip(refNames, refUrls): for reference in control.get_references( ).get_reference(): if referenceName == reference.get_name(): found = True if not found: control.get_references().add_reference( sl.referenceType.factory(name=referenceName, url=referenceUrl)) changed = True if not found: controls.append( [control.get_ref(), control.get_name()]) if found: controls.append( [control.get_ref(), control.get_name()]) if changed: if replaceLibs: path = path_libs / lib else: path = outlibs_path / lib root.set_revision(int(root.get_revision()) + 1) exportLib2XML(str(path), root) results += "Library '%s' generated with the new references in the path: %s.\n" % ( root.get_name(), str(path)) results += showMappingReport(dfm, controls, fileName) return results
def getControlsByPatterns(patterns): data = list() path_libs = Path.cwd() / "libraries" for lib in os.listdir(str(path_libs)): if lib.endswith(".xml"): root = sl.parse(str(path_libs / lib), silence=True) for component in root.get_components().get_component(): if component.get_ref() in patterns: for control in component.get_controls().get_control(): data.append([component.get_ref(), control.get_ref()]) data = pd.DataFrame(data, columns=['Component Id', 'Control Id']) data.sort_values("Control Id", inplace=True) return data
def getProductDataFromXml(xmlPath): table = list() # Main headers for the Excel table.append( ["Risk Pattern", "", "", "Use case", "", "" "Threat", "", "", "", "Weakness", "", "", "", "Countermeasure", "", "", "", "", ""]) table.append( ["Id", "Name", "Desc", "Id", "Name", "Desc", "Id", "Name", "Desc", "References", "Id", "Name", "Desc", "References", "Id", "Name", "Desc", "Test steps", "References", "Standards" ]) # We get all nodes of the xml file rootClass = supermod.parse(xmlPath, silence=True) # we get all important data from the xml file and transform it in a list table.extend(getAllDataFromRiskPatterns(rootClass)) # These are the column names of the DataFrame columns = ["Risk Pattern Id", "Risk Pattern Name", "Risk Pattern Desc", "Use case Id", "Use case Name", "Use case Desc", "Threat Id", "Threat Name", "Threat Desc", "Threat Refs", "Weakness Id", "Weakness Name", "Weakness Desc", "Weakness Refs", "Countermeasure Id", "Countermeasure Name", "Countermeasure Desc", "Countermeasure Test steps", "Countermeasure Refs", "Countermeasure standards"] dfm = pd.DataFrame(table, columns=columns) tableInfo = list() tableInfo.append(["Trust Zone", "Component", "Question answered"]) root = etree.parse(xmlPath).getroot() trustzones = dict() for trustzone in root.find("trustZones").iter("trustZone"): trustzones[trustzone.attrib['ref']] = trustzone.attrib['name'] for component in root.iter("component"): for trustZone in component.iter("trustZone"): tz = trustZone.attrib["ref"] compName = component.attrib["name"] tz = trustzones[tz] comp = compName for question in component.iter("question"): tableInfo.append([tz, comp, question.attrib["ref"]]) tz = "" comp = "" columnsQuestions = ["Trust Zone", "Component", "Question answered"] dfmQuestions = pd.DataFrame(tableInfo, columns=columnsQuestions) return dfm, dfmQuestions
def generateHtmlFromLibrariesAndStandard(standard_path, arrayLibrariesPaths): controls=list() librariesNames="" std=pd.read_csv(standard_path, sep='|') std=pd.DataFrame(std.values, columns=['Category', 'Ref', 'Req.', 'Included']) standardName=std['Ref'][0] standardRef=std['Category'][0] standards=std.iloc[1:] for lib_path in arrayLibrariesPaths: librariesNames+=lib_path.name.replace(".xml","")+", " root=sl.parse(str(lib_path), silence=True) controls=getControlsList(lib_path,controls) controlsbyStandard=controlsByStantard(controls, standardName) librariesNames=librariesNames[0:-2] outputPath = generateHtml(standards, standardName, controlsbyStandard, librariesNames) return "Html file for the standard %s was generated in the path '%s'."%(standard_path.name, outputPath), outputPath
def getDataFromXml(xmlPath): table=list() # Main headers for the Excel table.append(["Risk Pattern", "", "", "Use case", "", "", "Threat", "", "", "", "Weakness", "", "", "", "Countermeasure", "", "", "", "", ""]) table.append(["Id", "Name", "Desc", "Id", "Name", "Desc", "Id", "Name", "Desc", "References", "Id", "Name", "Desc", "References", "Id", "Name", "Desc", "Test steps", "References", "Standards" ]) # We get all nodes of the xml file rootClass=supermod.parse(xmlPath, silence=True) # we get all important data from the xml file and transform it in a list table.extend(getAllDataFromRiskPatterns(rootClass)) # These are the column names of the DataFrame columns=["Risk Pattern Id", "Risk Pattern Name", "Risk Pattern Desc", "Use case Id", "Use case Name", "Use case Desc", "Threat Id", "Threat Name", "Threat Desc", "Threat Refs", "Weakness Id", "Weakness Name", "Weakness Desc", "Weakness Refs", "Countermeasure Id", "Countermeasure Name", "Countermeasure Desc", "Countermeasure Test steps", "Countermeasure Refs", "Countermeasure standards"] dfm=pd.DataFrame(table, columns=columns) # We get the library details, component definitions and supported standards libraryDetails=getLibraryDetails(rootClass) componentDefinitionDetails = getComponentDefinitionDetails(rootClass) supportedStandardDetails = getSupportedStandardDetails(rootClass) # We convert all library information to dataframes dfmLibrary=pd.DataFrame(libraryDetails, columns=["General", "Values"]) dfmComponentDefinitions=pd.DataFrame(componentDefinitionDetails, columns=["Component Definition Name", "Component Definition Ref", "Component Definition Desc", "Category Name", "Category Ref", "Risk Patterns"]) dfmSupStandard=pd.DataFrame(supportedStandardDetails, columns=["Supported Standard Name", "Supported Standard Ref"]) # We join the thre dataframes of library details in one dataframe dfmInfo = pd.concat([dfmLibrary, dfmComponentDefinitions, dfmSupStandard], axis=1, sort=False) rules = getRulesData(rootClass) dfmRules=pd.DataFrame(rules, columns=[ "Rule Name", "Module", "Generated by GUI", "Condition Name", "Condition Value", "Condition Field", "Action Name", "Action Value", "Action Project" ]) return dfm, dfmInfo, dfmRules
def libMitigationTest(lib_path, exceptions): text = "" errors = list() errorsIntegrity = list() root = sl.parse(str(lib_path), silence=True) components = root.get_components() libraryRef = root.get_ref() for component in components.get_component(): for usecase in component.get_usecases().get_usecase(): for threat in usecase.get_threats().get_threat(): totalMitigation = 0 for control in threat.get_controls().get_control(): #print("Controls mitigation: " + control.attrib['mitigation']) totalMitigation = totalMitigation + int( control.get_mitigation()) #We also check we have the same value for control ref and control mitigation under the weaknesses tree for weakness in threat.get_weaknesses().get_weakness(): for contrl in weakness.get_controls().get_control(): if control.get_ref() == contrl.get_ref( ) and control.get_mitigation( ) != contrl.get_mitigation(): errorsIntegrity.append(control.get_ref()) if [component.get_ref(), threat.get_ref()] in exceptions: totalMitigation = 100 if totalMitigation != 100: errors.append(threat.get_ref()) text = "" if len(errorsIntegrity) != 0: text += "Threat mitigation integrity fails in the following countermeasures: %s\n" % str( errorsIntegrity).replace("[", "").replace("]", "") if len(errors) != 0: text += "Threat mitigation fails in the following threats: %s\n" % str( errors).replace("[", "").replace("]", "") return text
def generateHTML(xml_path, html_path): code = "<!DOCTYPE html><html><head><title>Risk Patterns for " + xml_path.name + "</title><link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css\" integrity=\"sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm\" crossorigin=\"anonymous\">" code = code + "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\" integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\"></script>" code = code + "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js\" integrity=\"sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49\" crossorigin=\"anonymous\"></script>" code = code + "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js\" integrity=\"sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy\" crossorigin=\"anonymous\"></script></head><body>" code += "<center><h2>Risk patterns for " + xml_path.name + "<h2></center>" rootClass = supermod.parse(str(xml_path), True) cont = 0 components = rootClass.get_components() code += "<div class=\"accordion\" id=\"components\">" codeComponent = "" for component in components.get_component(): weaknesses = component.get_weaknesses() controls = component.get_controls() usecases = component.get_usecases() codeUseCase = "<div class=\"accordion\" id=\"usecases\">" for usecase in usecases.get_usecase(): threats = usecase.get_threats() codeThreat = "<div class=\"accordion\" id=\"threats\">" for threat in threats.get_threat(): weaknessesThreat = threat.get_weaknesses() codeWeakness = "<div class=\"accordion\" id=\"weaknesses\">" # Weaknesses and controls within addedControlsRefs = [] for weaknessT in weaknessesThreat.get_weakness(): for weakness in weaknesses.get_weakness(): if weaknessT.get_ref() == weakness.get_ref(): weakName = weakness.get_name() weakRef = weakness.get_ref() weakDesc = weakness.get_desc() controlsWeak = weaknessT.get_controls() codeControl = "<div class=\"accordion\" id=\"controls\">" for controlW in controlsWeak.get_control(): for control in controls.get_control(): if controlW.get_ref() == control.get_ref(): addedControlsRefs.append(control.get_ref()) codeControl += includeCard( "controls", "Countermeasure", "", control.get_name(), control.get_ref(), control.get_desc(), cont) cont = cont + 1 codeControl += "</div>" codeWeakness += includeCard("weaknesses", "Weakness", codeControl, weakName, weakRef, weakDesc, cont) cont = cont + 1 codeWeakness += "</div>" codeWeakness += "<div class=\"accordion\" id=\"controlsWithoutWeaknesses\">" # Controls without weaknesses for control in threat.get_controls().get_control(): for controlNo in controls.get_control(): if control.get_ref() == controlNo.get_ref(): if control.get_ref() not in addedControlsRefs: codeWeakness += includeCard( "controlsWithoutWeaknesses", "Countermeasure", "", controlNo.get_name(), controlNo.get_ref(), controlNo.get_desc(), cont) cont = cont + 1 codeWeakness += "</div>" codeThreat += includeCard("threats", "Threat", codeWeakness, threat.get_name(), threat.get_ref(), threat.get_desc(), cont) cont = cont + 1 codeThreat += "</div>" codeUseCase += includeCard("usecases", "Use case", codeThreat, usecase.get_name(), usecase.get_ref(), usecase.get_desc(), cont) cont = cont + 1 codeUseCase += "</div>" codeComponent += includeCard("components", "Risk pattern", codeUseCase, component.get_name(), component.get_ref(), component.get_desc(), cont) cont = cont + 1 codeComponent += "</div>" code += codeComponent code += "</body></html>" fileHtml = open(str(html_path), "w") fileHtml.write(code) fileHtml.close() print("HTML file was generated in the path '%s'." % html_path)
def searchControls(library_path, supportedStandard_name, standard_file_path): rootObj = sl.parse(str(library_path), silence=True) supportedStandards = rootObj.get_supportedStandards() components = rootObj.get_components().get_component() supportedStandardCreated = False dfm = pd.read_csv(str(standard_file_path), sep="|") dfm.columns = [ 'Standard ASVS', "Ref ASVS", "Supported Standard Name", "Supported Standard Ref", "Standard Ref" ] for index, row in dfm.iterrows(): asvs_supportedStandardRef = row.get("Standard ASVS") asvs_ref = str(row.get("Ref ASVS")) supportedStandard_name = row.get("Supported Standard Name") supportedStandardRef = row.get("Supported Standard Ref") standardRef = row.get("Standard Ref") for component in components: controls = component.get_controls().get_control() for control in controls: standards = control.get_standards() for standard in standards.get_standard(): if standard.get_supportedStandardRef( ) == asvs_supportedStandardRef: if standard.get_ref() == asvs_ref: alreadyExist = False for stard in standards.get_standard(): if stard.get_supportedStandardRef( ) == supportedStandardRef and stard.get_ref( ) == standardRef: alreadyExist = True #construir standard y añadirlo if alreadyExist == False: standards.add_standard( createStandard(supportedStandardRef, standardRef)) supportedStandardCreated = True supportedStandardFound = False for supportedStandard in supportedStandards.get_supportedStandard(): if supportedStandard.get_ref() == supportedStandardRef: supportedStandardFound = True if supportedStandardFound == False and supportedStandardCreated == True: supportedStandards.add_supportedStandard( sl.supportedStandardType.factory(ref=supportedStandardRef, name=supportedStandard_name)) rootObj.set_revision(int(rootObj.get_revision()) + 1) output_path = Path.cwd() / "outFiles" / "libraries" / library_path.name exportLib2XML(str(output_path), rootObj) text = "SuportedStandard was added for the library and saved in the new created file '%s'\n" % output_path else: if supportedStandardCreated == True: rootObj.set_revision(int(rootObj.get_revision()) + 1) output_path = Path.cwd( ) / "outFiles" / "libraries" / library_path.name exportLib2XML(str(output_path), rootObj) text = "SuportedStandard was updated for the library and saved in the new created file '%s'\n" % output_path else: text = "SuportedStandard was not necessary to create for the library '%s'\n" % library_path.name print(text) return text