def main():
from lib.utils.mngFiles import getFile_Source2Dict
from lib.utils.mngRmtObjs import getRmt_data
from lib.utils.mngSources_r01 import clsDataSource
dst_creds = {
"URI": "http://172.16.167.147/taxii-discovery-service",
"usrName": "admin",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
src_data = clsDataSource(iID=82)
src_data.chnkSize = 250
src_data.dstCreds = dst_creds
src_data.filePath = os.path.dirname(os.path.abspath(__file__)) + '/'
src_data.from_dict(
getFile_Source2Dict('../../data/db_sourceList.json', src_data.ID))
sndMSG('---[ Start: Src_' + str(src_data.ID) + ' ]---', 'INFO', 'main()')
# // Get Source data from Producer's Site
# sndMSG('---< NOT UPDATING >--- ','INFO')
# getRmt_data(src_data)
# ---[ Extract / Transform / Load ]---
dictObj = extract_asDict(src_data, process_NewDataOnly=True)
return 0
def adptrLoad_asXML(srcData, data, isChunk=False):
from lib.conns.curlTAXII import sndTAXII
bFlag = False
import pprint
tmpObj = adptrTransform_Dict2Obj(srcData, data)
if not tmpObj: return (None)
#locSTIXFile = 'STIX_' + srcData.fileName.split('.')[0] + '.xml'
locSTIXFile = 'STIX.xml'
with open(locSTIXFile, "w") as outfile:
outfile.write(tmpObj.to_xml())
outfile.close()
sNAMEFUNC = 'adptrLoad_asXML()'
sndMSG('Called...', 'INFO', sNAMEFUNC)
taxiiMsg = sndTAXII(srcData.dstCreds, tmpObj.to_xml(), True)
if str(taxiiMsg[:1000]).find('SUCCESS') != -1:
sndMSG('Pkg Sent Successfully...', 'INFO', sNAMEFUNC)
bFlag = True
else:
sndMSG('Pkg was not Sent...', 'ERROR', sNAMEFUNC)
sndMSG(taxiiMsg)
bFlag = False
return (bFlag)
def extract_asDict(src_data, process_NewDataOnly=True):
sndMSG('Called...', 'INFO', 'extract_as_dict()')
assert isinstance(src_data, object)
# // Get Local Data - Used to Remove duplicates
lcl_data = getDB_local(src_data)
from lib.utils import feedparser
items = feedparser.parse(src_data.srcCreds['URI'])
print len(items['entries'])
#print items['entries'][0]
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(items['entries'][0])
sys.exit(0)
def connector_wQue(que, conn_cred,stixData,bTAXIIWrp=True,sTrg=None):
import Queue
sendMSG("Trying to send to TAXII API : " + conn_cred['taxiiURL']);
#print "Attemping to delete file: " + str(sTrg)
if bTAXIIWrp == True:
stixData = addTAXIIWrapper(stixData);
lstTmp = sTrg.split("/");
sFileName = lstTmp[(len(lstTmp)-1)]
sHdr = genHeader_TAXII(stixData,isSSL=False)
bFlag = _connecterTAXII(conn_cred,sHdr,stixData,sFileName,isSSL=False)
#que.put(connTAXII_api(conn_cred,sHdr,stixData,sFileName,isSSL=False));
#s = que.get()
#print s
#if not sTrg == None:
if bFlag == True and not sTrg == None:
import os
sndMSG("Attemping to delete file: " + sTrg)
os.remove(sTrg)
#sendMSG("NOTE Connector is temp diabled : ")
return(0);
def sndTAXII(conn_cred,stixData,bTAXIIWrp=True,sTrg=None):
sFuncName = 'sndTAXII'
sTxt = "Trying to send to TAXII API : " + conn_cred['URI']
sndMSG(sTxt,'INFO',sys._getframe().f_code.co_name);
if bTAXIIWrp == True:
stixData = addTAXIIWrapper(stixData);
sURL = conn_cred['URI']
if 'https' in sURL:
isSSL = True
else:
isSSL = False
sFileName = ''
sHdr = genHeader_TAXII(stixData,isSSL)
sMsg = _connecterTAXII(conn_cred,sHdr,stixData,sFileName)
# if bFlag == True and not sTrg == None:
# import os
# sndMSG("Attemping to delete file: " + sTrg)
# os.remove(sTrg)
return(sMsg);
def adptrExtract_asDict(srcData, isUpdateNewDataOnly=True):
sNAMEFUNC = 'adptrExtract_asDict()'
sndMSG('Called...', 'INFO', sNAMEFUNC)
if not srcData: return (None) #srcData can not be empty
### Generally required for all Adpters
from lib.utils.mngDateTime import getUTCTime
### Get Localy Data - Used to Remove duplicates
data = getDB_local(srcData)
srcDict = cnvt_csv2dict(srcData)
localData = data
from lib.utils.mngFiles import getFile_JSON2Dict
newData = getFile_JSON2Dict('db_pattern.json')
### From srcDict look for duplicated data
for item in srcDict:
sKey = srcDict[item]['phish_id']
if localData['data'].has_key(sKey):
#TODO: Check if hash is diffrent,
# if so, created updated STIX Doc
# Use smae GUID as original
pass
else:
localData['data'][sKey] = {'src': srcDict[item]}
localData['data'][sKey]['meta'] = {
'md5': genHash_md5(srcDict[item])
}
localData['data'][sKey]['meta'].update({'dateDL': getUTCTime()})
newData['data'][sKey] = localData['data'][sKey]
if isUpdateNewDataOnly == False:
newData = localData
if len(newData['data']):
sTxt = "Found " + str(len(newData['data'])) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = {}
setDB_local(localData, srcData)
return (newData)
def main():
from lib.utils.mngFiles import getFile_Source2Dict
from lib.utils.mngSources_r01 import clsDataSource
dstCreds = {
"URI": "http://www.hailataxii.com/taxii-discovery-service",
"usrName": "phishtank_com",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
dstCreds = {
"URI": "http://172.16.167.147/taxii-discovery-service",
"usrName": "admin",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
srcData = clsDataSource(iID=28)
srcData.chnkSize = 250
srcData.pkgTitle = "PhishEmail URL "
srcData.pkgDscrpt = "List of embed URI found in suspected Phishing Emails "
srcData.pkgLink = "http://www.phishtank.com/"
srcData.from_dict(
getFile_Source2Dict('../../data/db_sourceList.json', srcData.ID))
srcData.dstCreds = dstCreds
srcData.filePath = os.path.dirname(os.path.abspath(__file__)) + '/'
sndMSG('---[ Start: Src_' + str(srcData.ID) + ' ]---', 'INFO')
#sndMSG('---< NOT UPDATING >--- ','INFO')
getRmt_data(srcData)
### Extract / Transform / Load
dictObj = adptrExtract_asDict(srcData, True)
subDict = {'data': {}, 'source': {}}
if len(dictObj):
iSize_Total = len(dictObj['data'])
iSize_Sent = 0
for sKey in dictObj['data']:
subDict['data'][sKey] = dictObj['data'][sKey]
if len(subDict['data']) > srcData.chnkSize:
iSize_Sent += len(subDict['data'])
subDict['source'].update(dictObj['source'])
bFlag = adptrLoad_asXML(srcData, subDict)
sndMSG('Sent: ' + str(iSize_Sent) + " of " + str(iSize_Total),
'INFO')
subDict = None
subDict = {'data': {}, 'source': {}}
if len(subDict):
iSize_Sent += len(subDict['data'])
subDict['source'].update(dictObj['source'])
bFlag = adptrLoad_asXML(srcData, subDict)
sndMSG('Sent: ' + str(iSize_Sent) + " of " + str(iSize_Total),
'INFO')
sndMSG('---[ Finsh: Src_' + str(srcData.ID) + ' ]---', 'INFO')
return (0)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
### Parse Source File in to a Dictionary Object
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = '\n'
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
srcData.pkgTitle = "SNORT Rule by Emergingthreats | Block Botnet Command and Control"
srcData.pkgDscrpt = "Emerging Threats Botnet Command and Control drop rules. These are generated from the EXCELLENT work done by the Shadowserver team and the abuse.ch folks. All Volunteers, we're grateful for their dedication! http://www.shadowserver.org; https://spyeyetracker.abuse.ch; https://palevotracker.abuse.ch; https://zeustracker.abuse.ch. More information available at www.emergingthreats.net"
srcData.pkgLink = "http://rules.emergingthreats.net/blockrules/emerging-botcc.portgrouped.rules"
for col in srcDict:
# {0: u'alert tcp $HOME_NET any -> 50.116.1.225 22 (msg:"ET CNC Shadowserver Reported CnC Server Port 22 Group 1"; flags:S; reference:url,doc.emergingthreats.net/bin/view/Main/BotCC; reference:url,www.shadowserver.org; threshold: type limit, track by_src, seconds 360, count 1; classtype:trojan-activity; flowbits:set,ET.Evil; flowbits:set,ET.BotccIP; sid:2405000; rev:3570;)'}
sKey = srcDict[col][0]
strTmp = sKey.split("(")
tmpList = strTmp[0].split(" ")
ipProt = None
if tmpList[1]:
ipProt = tmpList[1]
ipList = None
if tmpList[5]:
if "[" in tmpList[5]:
tmpList[5] = tmpList[5][1:-1]
ipList = tmpList[5].split(",")
ipPort = None
if tmpList[6]:
ipPort = tmpList[6]
attrList = strTmp[1].split(";")[:-1]
tmpDict = {}
for i in range(len(attrList)):
attrList[i] = cleanString(attrList[i])
tmpKey = attrList[i].split(':')[0]
tmpVal = attrList[i].split(':')[1]
if tmpKey in tmpDict:
tmpDict[tmpKey] += "|" + tmpVal
else:
tmpDict.update({tmpKey: tmpVal})
dictAttrib = tmpDict
dictAttrib.update({
'ipAddrList': ipList,
'rule': sKey,
'ipPort': ipPort,
'ipProt': ipProt
})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['URI']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
obsURI.title = 'URI: ' + sURI
sDscrpt = 'URI: ' + sURI + " | "
sDscrpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
#Parser File Hash
sHash = data[sKey]['attrib']['hash']
if len(sHash) > 0:
objFile = File()
sFileName = data[sKey]['attrib']['fileName']
if len(sFileName) > 0:
objFile.file_name = sFileName
objFile.file_format = sFileName.split('.')[1]
objFile.add_hash(Hash(sHash, exact=True))
obsFile = Observable(objFile)
objFile = None
obsFile.sighting_count = 1
obsFile.title = 'File: ' + sFileName
sDscrpt = 'FileName: ' + sFileName + " | "
sDscrpt += "FileHash: " + sHash + " | "
obsFile.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsFile)
obsFile = None
objIndicator.add_indicator_type("File Hash Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Old Title / Description Generator
#objIndicator.title = data[sKey]['attrib']['title'];
#objIndicator.description = "<![CDATA[" + data[sKey]['attrib']['dscrpt'] + "]]>";
### Generate Indicator Title based on availbe data
sTitle = 'ZeuS Tracker (' + data[sKey]['attrib'][
'status'] + ')| ' + data[sKey]['attrib']['title']
if len(sAddr) > 0:
sAddLine = "This IP address has been identified as malicious"
if len(sDomain) > 0:
sAddLine = "This domain has been identified as malicious"
if len(sAddLine) > 0:
sTitle = sTitle + " | " + sAddLine
if len(srcObj.Domain) > 0:
sTitle = sTitle + " by " + srcObj.Domain
else:
sTitle = sTitle + "."
if len(sTitle) > 0:
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if len(sAddr) > 0:
sAddLine = "This IP address " + sAddr
if len(sDomain) > 0:
sAddLine = "This domain " + sDomain
if len(sAddr) > 0 and len(sDomain) > 0:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt = sDscrpt + sAddLine
sDscrpt = sDscrpt + " has been identified as malicious"
if len(srcObj.Domain) > 0:
sDscrpt = sDscrpt + " by " + srcObj.Domain
else:
sDscrpt = sDscrpt + "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + data[
sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_name("ZeuS")
objMalware.add_name("Zbot")
objMalware.add_name("Zeus")
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = "Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system"
objMalware.description = "Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))"
objTTP = TTP(title="ZeuS")
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
oObsSrcData = genObsSrcData(srcObj, data[sKey])
### Parsing IP Address
sAddr = data[sKey]['attrib']['ip']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_destination = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.type = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.type = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
oObsSrcData.sighting_count = 1
obsAddr.observable_source.append(oObsSrcData)
sTitle = 'IP: ' + sAddr
obsAddr.title = sTitle
sDscpt = 'ipv4-addr' + ': ' + sAddr + " | "
sDscpt += "is_destination: True | "
if data[sKey]['attrib']['first']:
sDscpt += "firstSeen: " + data[sKey]['attrib']['first'] + " | "
if data[sKey]['attrib']['last']:
sDscpt += "lastSeen: " + data[sKey]['attrib']['last'] + " | "
obsAddr.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
### Parsing Network Address
sAddr = data[sKey]['attrib']['inetnum']
if sAddr:
objAddr = Address()
objAddr.is_destination = True
sAddrNet = sAddr.split("-")
objAddr.address_value = sAddrNet[0].strip(
) + "##comma##" + sAddrNet[1].strip()
objAddr.address_value.condition = 'InclusiveBetween'
objAddr.category = 'ipv4-net'
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
oObsSrcData.sighting_count = 1
obsAddr.observable_source.append(oObsSrcData)
sTitle = 'NETWORK_range: ' + sAddr
obsAddr.title = sTitle
sDscpt = 'ipv4-net' + ': ' + sAddr + " | "
if data[sKey]['attrib']['netname']:
sDscpt += 'netName' + ': ' + data[sKey]['attrib'][
'netname'] + " | "
obsAddr.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
### Parsing Email Address
sEMAIL = data[sKey]['attrib']['email']
if sEMAIL:
objEmail = EmailAddress()
#objEmail.is_source = True
objEmail.address_value = sEMAIL
objEmail.address_value.condition = 'Equals'
objEmail.category = 'e-mail'
obsEmail = Observable(objEmail)
objEmail = None
obsEmail.sighting_count = 1
oObsSrcData.sighting_count = 1
if len(data[sKey]['attrib']['source']) > 0:
oObsSrcData.name = data[sKey]['attrib']['source']
obsEmail.observable_source.append(oObsSrcData)
sTitle = 'REGISTRAR_email: ' + sEMAIL
obsEmail.title = sTitle
sDscrpt = 'REGISTRAR_email: ' + sEMAIL
if data[sKey]['attrib']['descr']:
sDscrpt += " | REGISTRAR_name: " + data[sKey]['attrib'][
'descr'] + " | "
obsEmail.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsEmail)
obsEmail = None
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
objDomain.is_destination = True
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
oObsSrcData.sighting_count = 1
obsDomain.observable_source.append(oObsSrcData)
obsDomain.title = 'Domain: ' + sDomain
sDscpt = 'Domain: ' + sDomain + " | "
sDscpt += "isDestination: True | "
if data[sKey]['attrib']['first']:
sDscpt += "firstSeen: " + data[sKey]['attrib']['first'] + " | "
if data[sKey]['attrib']['last']:
sDscpt += "lastSeen: " + data[sKey]['attrib']['last'] + " | "
#if
obsDomain.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['url']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
oObsSrcData.sighting_count = 1
obsURI.observable_source.append(oObsSrcData)
obsURI.title = 'URI: ' + sURI
sDscpt = 'URI: ' + sURI + " | "
sDscpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
sDscrpt = None
sCntry_code = None
sCntry_name = None
sRgstra_email = None
sRgstra_name = None
# add Phishing Email Target
# add Phishing email Details phishtank_ID
if data[sKey]['attrib']['country']:
sCntry_code = data[sKey]['attrib']['country']
if sCntry_code in dictCC2CN:
sCntry_name = dictCC2CN[sCntry_code]
if data[sKey]['attrib']['email'] > 0:
sRgstra_email = data[sKey]['attrib']['email']
if data[sKey]['attrib']['descr']:
sRgstra_name = data[sKey]['attrib']['descr']
sDscrpt = " clean-mx.de has identified this "
if isIPv4(data[sKey]['attrib']['domain']):
sDscrpt += "ip address " + data[sKey]['attrib']['domain'] + " "
else:
sDscrpt += "domain " + data[sKey]['attrib']['domain'] + " "
sDscrpt += "as malicious "
if data[sKey]['attrib']['target']:
sDscrpt += "and uses phishing email(s) targeting " + data[sKey][
'attrib']['target'] + " users with "
else:
sDscrpt += "and sent out "
sDscrpt += "email containg this url <-Do Not Connect-> {" + data[sKey][
'attrib']['url'] + "} <-Do Not Connect-> link. "
if data[sKey]['attrib']['phishtank']:
sDscrpt += "For more detail on the specific phisihing email use this phishtank ID [" + data[
sKey]['attrib']['phishtank'] + "]. "
if sCntry_code:
sDscrpt += " This url appears to originated in " + sCntry_code
if sCntry_name:
sDscrpt += " (" + sCntry_name + ")"
if sCntry_code and (sRgstra_email or sRgstra_name):
sDscrpt += " and is "
if sRgstra_email:
sDscrpt += "register to " + sRgstra_email
if sRgstra_email and sRgstra_name:
sDscrpt += " of " + sRgstra_name
elif sRgstra_name:
sDscrpt += "register to " + sRgstra_name
sDscrpt += "."
if sCntry_code or sRgstra_email or sRgstra_name:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
sTitle = 'Phishing ID:' + sKey + " "
if data[sKey]['attrib']["target"]:
sTitle += "Target: " + data[sKey]['attrib']["target"] + " "
if data[sKey]['attrib']["url"]:
sTitle += "URL: " + data[sKey]['attrib']["url"] + " "
objIndicator.title = sTitle
### Add Generated observable to Indicator
objIndicator.add_indicator_type("IP Watchlist")
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['first'])
objIndicator.set_received_time(data[sKey]['dateDL'])
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
#sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + srcObj.srcTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### Parse Source File in to a Dictionary Object
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = '\t'
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
newData = {}
for col in srcDict:
# {0: u'', 1: u'20161231', 2: u'38zu.cn', 3: u'attackpage', 4: u'safebrowsing.google.com', 5: u'20140703', 6: u'20140302', 7: u'20130325', 8: u'20120426', 9: u'20110715', 10: u'relisted'}
if len(srcDict[col]) < 6:
continue
else:
sKey = srcDict[col][2]
lstDateVF = []
for idx in range(5, len(srcDict[col])):
if len(srcDict[col][idx]) > 0 and isNumber(srcDict[col][idx][1]):
sDateVF = srcDict[col][idx]
try:
dSrt = datetime.strptime(sDateVF, "%Y%m%d")
sDateVF = dSrt.strftime("%Y-%m-%dT%H:%M:%SZ")
lstDateVF.append(sDateVF)
except:
pass
#nextvalidation domain type original_reference-why_it_was_listed dateverified
dictAttrib = {
"domain": cleanString(srcDict[col][2]),
"type": cleanString(srcDict[col][3]),
"ref": cleanString(srcDict[col][4]),
"lstDateVF": lstDateVF
}
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_src2Dict(src_data, isUpdateNewDataOnly):
namefunc = 'adptr_src2Dict()'
stxt = "Called... "
sndMSG(stxt, 'INFO', namefunc)
### Input Check
if src_data is None:
# TODO: Needs error msg: Missing srcData Object
return False
locDataFile = 'db_' + src_data.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
# srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
# print "------< NOT UPDATING >------"
if not getRmt_File(src_data.srcCreds,
src_data.filePath + src_data.fileName) == True:
# if no source data is found, this script will exit
return False
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
trimFile_btwn(src_data.filePath + src_data.fileName,
'<?xml version="1.0" encoding="ISO-8859-1" ?>', '</rss>')
srcDict = cnvt_XML2Dict(src_data.filePath + src_data.fileName)
### DEBUG CODE ####
###################
src_data.pkgTitle = srcDict['rss']['channel']['title']
src_data.pkgDscrpt = srcDict['rss']['channel']['description']
src_data.pkgLink = srcDict['rss']['channel']['link']
newData = {}
for col in srcDict['rss']['channel']['item']:
sKey = col['guid']
sCol = col['title']
sDateVF = sCol.split('(')[1]
sDateVF = sDateVF[0:-1]
try:
dSrt = datetime.strptime(sDateVF, "%Y-%m-%d %H:%M:%S")
sDateVF = dSrt.strftime("%Y-%m-%dT%H:%M:%SZ")
except:
sDateVF = None
sDomain = None
sIPAddr = cleanString(sCol.split('(')[0])
if not isIPv4(sIPAddr):
sDomain = sIPAddr
sIPAddr = None
sCol = col['description']
lstAttrib = sCol.split(',')
dictAttrib = {
"dateVF": sDateVF,
"title": cleanString(col['title']),
"link": cleanString(col['link']),
"dscrpt": cleanString(col['description']),
"ipAddr": sIPAddr,
"domain": sDomain,
}
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
# TODO:Check If Exist Element's inactive status changed
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if not isUpdateNewDataOnly:
newData = dstData
if len(newData) > 0:
stxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(stxt, 'INFO', namefunc)
else:
stxt = "Found no new data"
sndMSG(stxt, 'INFO', namefunc)
newData = False
return newData
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
### Input Check
if srcObj is None or data is None:
# TODO: Needs error msg: Missing srcData Object
return False
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if sAddr:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if sDomain:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
objIndicator.add_indicator_type("Domain Watchlist")
# Parser File Hash
# sHash = data[sKey]['attrib']['hash'];
# if len(sHash) > 0:
# objFile = File()
# sFileName = data[sKey]['attrib']['fileName']
# if len(sFileName) > 0:
# objFile.file_name = sFileName
# objFile.file_format = sFileName.split('.')[1]
# objFile.add_hash(Hash(sHash, exact=True))
# obsFile = Observable(objFile)
# objFile = None;
# obsFile.sighting_count = 1
# obsFile.title = 'File: ' + sFileName
# sDscrpt = 'FileName: ' + sFileName + " | "
# sDscrpt += "FileHash: " + sHash + " | "
# obsFile.description = "<![CDATA[" + sDscrpt + "]]>"
# listOBS.append(obsFile)
# obsFile = None;
# objIndicator.add_indicator_type("File Hash Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(srcObj.Domain) > 0:
objIndicator.set_producer_identity(srcObj.Domain)
if data[sKey]['attrib']['dateVF']:
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Old Title / Description Generator
#objIndicator.title = data[sKey]['attrib']['title'];
#objIndicator.description = "<![CDATA[" + data[sKey]['attrib']['dscrpt'] + "]]>";
### Generate Indicator Title based on availbe data
sTitle = 'Feodo Tracker: '
if sAddr:
sAddLine = "This IP address has been identified as malicious"
if sDomain:
sAddLine = "This domain has been identified as malicious"
if len(sAddLine) > 0:
sTitle += " | " + sAddLine
if len(srcObj.Domain) > 0:
sTitle += " by " + srcObj.Domain
else:
sTitle += "."
if len(sTitle) > 0:
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if sAddr:
sAddLine = "This IP address " + sAddr
if sDomain:
sAddLine = "This domain " + sDomain
if sAddr and sDomain:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt += sAddLine
sDscrpt += " has been identified as malicious"
if len(srcObj.Domain) > 0:
sDscrpt += " by " + srcObj.Domain
else:
sDscrpt += "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + \
data[sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_name("Cridex")
objMalware.add_name("Bugat")
objMalware.add_name("Dridex")
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = "Feodo (also known as Cridex or Bugat) is a Trojan used to commit ebanking fraud and steal sensitive information from the victims computer, such as credit card details or credentials"
sDscrpt = "Feodo (also known as Cridex or Bugat) is a Trojan used to commit ebanking fraud and steal sensitive information from the victims computer, such as credit card details or credentials. At the moment, Feodo Tracker is tracking four versions of Feodo, and they are labeled by Feodo Tracker as version A, version B, version C and version D:\n"
sDscrpt += "\n"
sDscrpt += " Version A: Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.\n"
sDscrpt += " Version B: Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.\n"
sDscrpt += " Version C: Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.\n"
sDscrpt += " Version D: Successor of Cridex. This version is also known as Dridex\n"
objMalware.description = "<![CDATA[" + sDscrpt + "]]>"
objTTP = TTP(title="Feodo")
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return stix_package
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
oObsSrcData = genObsSrcData(srcObj, data[sKey])
### Parsing IP Address
sAddr = sKey
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'InclusiveBetween'
objAddr.category = 'ipv4-net'
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = int(data[sKey]['attrib']['Attacks'])
oObsSrcData.sighting_count = int(data[sKey]['attrib']['Attacks'])
obsAddr.observable_source.append(oObsSrcData)
sTitle = 'NETWORK_range: ' + sAddr
obsAddr.title = sTitle
sDscpt = 'ipv4-net' + ': ' + sAddr + " | "
sDscpt += "is_source: True | "
sDscpt += "Attack_Count: " + data[sKey]['attrib']['Attacks'] + " | "
sDscpt += "Attack_DateRange: " + data[sKey]['attrib'][
'dateRange'] + " | "
obsAddr.description = sDscpt
listOBS.append(obsAddr)
obsAddr = None
### Parsing Registrar Information
if data[sKey]['attrib']['email']:
objEmail = EmailAddress()
objEmail.address_value = data[sKey]['attrib']['email']
objEmail.address_value.condition = 'Equals'
objEmail.category = 'e-mail'
objWhoisReg = WhoisRegistrar()
if len(data[sKey]['attrib']['Name']) > 1:
objWhoisReg.name = data[sKey]['attrib']['Name']
objWhoisReg.email_address = objEmail
objEmail = None
objWhois = WhoisEntry()
objWhois.registrar_info = objWhoisReg
obsWhois = Observable(objWhois)
#print obsWhois.id_
objWhois = None
obsWhois.sighting_count = 1
sTitle = 'REGISTRAR_email: ' + data[sKey]['attrib']['email']
if len(data[sKey]['attrib']['Name']) > 0:
sTitle += " | REGISTRAR_name: " + data[sKey]['attrib'][
'Name'] + " | "
obsWhois.title = sTitle
obsWhois.description = sTitle
listOBS.append(obsWhois)
obsWhois = None
sDscrpt = None
sCntry_code = None
sCntry_name = None
sRgstra_email = None
sRgstra_name = None
if len(data[sKey]['attrib']['Country']) > 0:
sCntry_code = data[sKey]['attrib']['Country']
if sCntry_code in dictCC2CN:
sCntry_name = dictCC2CN[sCntry_code]
if 'email' in data[sKey]['attrib']:
sRgstra_email = data[sKey]['attrib']['email']
if len(data[sKey]['attrib']['Name']) > 0:
sRgstra_name = data[sKey]['attrib']['Name']
sDscrpt = "This IP block appears to have "
if sCntry_code:
sDscrpt += "originated in " + sCntry_code
if sCntry_name:
sDscrpt += "(" + sCntry_name + ")"
if sCntry_code and (sRgstra_email or sRgstra_name):
sDscrpt += " and is "
if sRgstra_email:
sDscrpt += "register to " + sRgstra_email
if sRgstra_email and sRgstra_name:
sDscrpt += " of " + sRgstra_name
elif sRgstra_name:
sDscrpt += "register to " + sRgstra_name
sDscrpt += "."
if sCntry_code or sRgstra_email or sRgstra_name:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
objIndicator.title = sAddr.replace('##comma##',
' - ') + " | " + srcObj.pkgTitle
### Add Generated observable to Indicator
objIndicator.add_indicator_type("IP Watchlist")
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
#sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + srcObj.srcTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### Parse Source File in to a Dictionary Object
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = ','
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
newData = {}
#metaData =
for col in srcDict:
# {0: u'', 1: u'20161231', 2: u'38zu.cn', 3: u'attackpage', 4: u'safebrowsing.google.com', 5: u'20140703', 6: u'20140302', 7: u'20130325', 8: u'20120426', 9: u'20110715', 10: u'relisted'}
if len(srcDict[col]) < 1:
continue
sKey = srcDict[col]['IP Address']
dictAttrib = {}
dictAttrib['Flags'] = {}
dictAttrib['Ports'] = {}
for item in srcDict[col]:
if 'Flag' in item:
if srcDict[col][item].isdigit():
dictAttrib['Flags'].update({item: int(srcDict[col][item])})
else:
dictAttrib['Flagss'].update({item: None})
elif 'Port' in item:
if srcDict[col][item].isdigit():
dictAttrib['Ports'].update({item: int(srcDict[col][item])})
else:
dictAttrib['Ports'].update({item: None})
elif 'Uptime' in item:
if srcDict[col][item].isdigit():
dictAttrib.update({item: int(srcDict[col][item])})
else:
dictAttrib.update({item: None})
elif 'Bandwidth' in item:
if srcDict[col][item].isdigit():
dictAttrib.update({item: int(srcDict[col][item])})
else:
dictAttrib.update({item: None})
else:
dictAttrib.update({item: srcDict[col][item]})
if dictAttrib['Hostname'] == dictAttrib['IP Address']:
dictAttrib['Hostname'] = None
# tmpHash = hashlib.md5(str(dictAttrib)).hexdigest()
if sKey in dstData:
dstData[sKey]['meta']['cnt'] += 1
dstData[sKey]['meta']['dateDL'] = getUTCTime()
# if not tmpHash == dstData[sKey]['meta']['attribHash']:
# dstData[sKey]['meta']['hasChanged'] = True
# print '---< Found Change >--- ' + sKey
# dstData[sKey]['meta']['attribHash'] = tmpHash
else:
dstData[sKey] = {}
dstData[sKey]['meta'] = {
'cnt': 1,
'dateDL': getUTCTime(),
'IDs': {},
'hasChanged': False,
'attribHash': 0x0
}
#dstData[sKey]['meta']['attribHash'] = tmpHash
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
#print "------< No Remote Data >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = '\t'
#oDialect.header = True
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
srcData.pkgTitle = "DShield.org Recommended Block List "
srcData.pkgDscrpt = "This list summarizes the top 20 attacking class C (/24) subnets over the last three days. The number of 'attacks' indicates the number of targets reporting scans from this subnet."
srcData.pkgLink = "http://feeds.dshield.org/block.txt"
sDateVF = None
s3daysAgo = None
try:
sDateVF = getFile_lineByValue(
srcData.filePath + srcData.fileName,
"updated:")[0].split("updated:")[1].strip()
sDateVF = datetime.strptime(sDateVF, "%a %b %d %H:%M:%S %Y %Z")
s3daysAgo = sDateVF + timedelta(days=-3)
if sDateVF:
sDateVF = sDateVF.strftime("%Y-%m-%dT%H:%M:%SZ")
s3daysAgo = s3daysAgo.strftime("%Y-%m-%dT%H:%M:%SZ")
srcData.pkgDscrpt = srcData.pkgDscrpt.replace(
'last three days.',
('last three days (' + s3daysAgo + " - " + sDateVF + ')'))
except:
pass
for col in srcDict:
if 'End' in srcDict[col]:
sKey = srcDict[col]['Start'] + "##comma##" + srcDict[col]['End']
else:
continue
dictAttrib = srcDict[col]
if sDateVF:
dictAttrib.update({"dateVF": str(sDateVF)})
if s3daysAgo:
dictAttrib.update(
{"dateRange": str(s3daysAgo) + " - " + str(sDateVF)})
if 'noemail' in srcDict[col]['email']:
dictAttrib.update({"email": None})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
# if 'indicator' in data[sKey]['meta']['IDs']:
# objIndicator.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({objIndicator.id_:'indicator'})
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['IP Address']
if sAddr:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
# if 'address"' in data[sKey]['meta']['IDs']:
# obsAddr.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({objIndicator.id_:'address'})
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['Hostname']
if sDomain:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
# if 'domain' in data[sKey]['meta']['IDs']:
# obsDomain.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({obsDomain.id_:'domain'})
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
### Parsing Port Number
sPortList = data[sKey]['attrib']['Ports']
for item in sPortList:
if sPortList[item]:
objPort = Port()
sPort = sPortList[item]
objPort.port_value = int(sPort)
objPort.port_value.condition = 'Equals'
objPort.layer4_protocol = 'TCP'
obsPort = Observable(objPort)
objPort = None
obsPort.sighting_count = 1
obsPort.title = 'Port: ' + str(sPort)
sDscrpt = 'PortNumber: ' + str(sPort) + " | "
sDscrpt += "Protocol: TCP | "
obsPort.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsPort)
### Add Generated observable to Indicator
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
infoSrc = InformationSource(identity=Identity(name=srcObj.Domain))
#infoSrc.add_contributing_source(data[sKey]['attrib']['ref'])
objIndicator.producer = infoSrc
# if data[sKey]['attrib']['lstDateVF']:
# objIndicator.set_produced_time(data[sKey]['attrib']['lstDateVF'][0]);
objIndicator.set_received_time(data[sKey]['meta']['dateDL'])
### Generate Indicator Title based on availbe data
lstContainng = []
lstIs = []
sTitle = ' This'
if data[sKey]['attrib']['Hostname']:
sTitle += ' domain ' + data[sKey]['attrib']['Hostname']
else:
sTitle += ' ipAddress ' + sKey
sTitle += ' has been identified as a TOR network "Exit Point" router'
objIndicator.title = sTitle
### Generate Indicator Description based on availbe data
sDscrpt = ' torstatus.blutmagie.de has identified this'
if data[sKey]['attrib']['Hostname']:
sDscrpt += ' domain ' + data[sKey]['attrib']['Hostname']
else:
sDscrpt += ' ipAddress ' + sKey
# sDscrpt += ' with a router name of "' + data[sKey]['attrib']['Router Name'] + '"'
# if data[sKey]['attrib']['Ports']['ORPort']:
# sDscrpt += ' using ORPort: ' + str(data[sKey]['attrib']['Ports']['ORPort'])
# if data[sKey]['attrib']['Ports']['DirPort']:
# sDscrpt += ' and DirPort: ' + str(data[sKey]['attrib']['Ports']['DirPort'])
sDscrpt += ' as a TOR network "Exit Point" router'
if data[sKey]['attrib']['Country Code']:
sCntry_code = data[sKey]['attrib']['Country Code']
if sCntry_code in dictCC2CN:
sCntry_name = dictCC2CN[sCntry_code]
sDscrpt += ', which appears to be located in ' + sCntry_name
sDscrpt += '. \n\n RawData: ' + str(data[sKey]['attrib'])
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
# objMalware = MalwareInstance()
# objMalware.add_type("Remote Access Trojan")
# ttpTitle = data[sKey]['attrib']['type']
# objTTP = TTP(title=ttpTitle)
# objTTP.behavior = Behavior()
# objTTP.behavior.add_malware_instance(objMalware)
# objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
# locDataFile = 'db_' + srcObj.fileName.split('.')[0] + '.json'
# sndFile_Dict2JSON(data,locDataFile);
# data = None
return (stix_package)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
### Add Generated observable to Indicator
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
infoSrc = InformationSource(identity=Identity(name=srcObj.Domain))
infoSrc.add_contributing_source(data[sKey]['attrib']['ref'])
if len(srcObj.Domain) > 0:
objIndicator.producer = infoSrc
if data[sKey]['attrib']['lstDateVF']:
objIndicator.set_produced_time(
data[sKey]['attrib']['lstDateVF'][0])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Generate Indicator Title based on availbe data
lstContainng = []
lstIs = []
sTitle = 'This domain ' + data[sKey]['attrib'][
'domain'] + ' has been identified as malicious'
if len(data[sKey]['attrib']['ref']):
sTitle += ' by ' + data[sKey]['attrib']['ref']
if len(data[sKey]['attrib']['type']) > 0:
sTitle += ', via this vector [' + data[sKey]['attrib'][
'type'] + '].'
else:
sTitle += '.'
objIndicator.title = sTitle
### Generate Indicator Description based on availbe data
sDscrpt = 'Lehigh.edu site has added this domain ' + data[sKey][
'attrib']['domain']
sDscrpt += ' to recommend block list.'
sDscrpt += ' This site has been identified as malicious'
sDscrpt += ' by ' + data[sKey]['attrib']['ref']
sDscrpt += ' and was still attive on the following dates ' + str(
data[sKey]['attrib']['lstDateVF']) + "."
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_type("Remote Access Trojan")
ttpTitle = data[sKey]['attrib']['type']
objTTP = TTP(title=ttpTitle)
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_destination = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = objAddr.CAT_IPV4
elif isIPv6(sAddr):
objAddr.category = objAddr.CAT_IPV6
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isDestination: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['title']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
obsURI.title = 'URI: ' + sURI
sDscrpt = 'URI: ' + sURI + " | "
sDscrpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Generate Indicator Title based on availbe data
sTitle = 'C2C Site: ' + sURI
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if len(sAddr) > 0:
sAddLine = "This IP address " + sAddr
if len(sDomain) > 0:
sAddLine = "This domain " + sDomain
if len(sAddr) > 0 and len(sDomain) > 0:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt += sAddLine
sDscrpt = sDscrpt + " has been identified as a command and control site for " + data[
sKey]['attrib']['dscrpt'] + ' malware'
if len(srcObj.Domain) > 0:
sDscrpt = sDscrpt + " by " + srcObj.Domain
else:
sDscrpt = sDscrpt + "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + data[
sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
pass
#Parse TTP
sType = data[sKey]['attrib']['dscrpt']
if len(sType) > 0:
objMalware = MalwareInstance()
objMalware.add_name(sType)
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = ""
objMalware.description = ""
objTTP = TTP(title=sType)
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
#print "------< Not Updating >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
srcDict = cnvt_XML2Dict(srcData.filePath + srcData.fileName)
srcData.pkgTitle = "Clean MX Phishing URL Block List "
srcData.pkgDscrpt = ""
srcData.pkgLink = "http://support.clean-mx.de/clean-mx/phishing.php"
for item in srcDict['output']['entries']['entry']:
sKey = item['id']
if item['first'] == "0":
item['first'] = None
else:
item['first'] = datetime.fromtimestamp(int(
item['first'])).strftime('%Y-%m-%dT%H:%M:%SZ')
if item['last'] == "0":
item['last'] = None
else:
item['last'] = datetime.fromtimestamp(int(
item['last'])).strftime('%Y-%m-%dT%H:%M:%SZ')
dictAttrib = item
lstNS = []
for i in range(1, 5):
if dictAttrib['ns' + str(i)]:
lstNS.append(dictAttrib['ns' + str(i)])
dictAttrib.update({"nsList": lstNS})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptrTransform_Dict2Obj(srcData, data=None):
sNAMEFUNC = 'adptrTransform_Dict2Obj'
sndMSG('Called...', 'INFO', sNAMEFUNC)
if not srcData: return (None) #srcData can not be empty
from stix.core import STIXPackage
from stix.common import InformationSource, Identity
from stix.data_marking import Marking
from stix.ttp import TTP
### Build Package
objMarkingSpecification = genObject_MarkingSpecification(data)
objMarkingSpecification.controlled_structure = "//node()"
objHdr = genData_STIXHeader(data)
objHdr.handling = Marking(objMarkingSpecification)
objHdr.information_source = InformationSource(identity=Identity(
name=srcData.pkgLink))
objTTP = genObject_TTP(data)
objTTP.information_source = InformationSource(identity=Identity(
name=srcData.pkgLink))
objPkg = STIXPackage()
objPkg.stix_header = objHdr
objPkg.add_ttp(objTTP)
for sKey in data['data']:
obsList = []
### Build Observables
obsURI = genObject_URI(data['data'][sKey]['src'])
try:
obsURI.id_ = data['data'][sKey]['meta']['uri']
except:
data['data'][sKey]['meta'].update({'uri': obsURI.id_})
### Srt: Stupid test to make sure URL be output via STIX.to_xml()
try:
testPkg = STIXPackage()
testPkg.add_observable(obsURI)
testPkg.to_xml()
except:
sNAMEFUNC = 'adptrTransform_Dict2Obj'
sndMSG('Error Parsing URL for this key: [' + sKey + ']', 'INFO',
sNAMEFUNC)
testPkg = None
continue
### End: Stupid test
objPkg.add_observable(obsURI)
obsList.append(genRefObs(obsURI))
### Build Indicators
objInd = genObject_Indicator(data['data'][sKey]['src'])
try:
obsURI.id_ = data['data'][sKey]['meta']['ind']
except:
data['data'][sKey]['meta'].update({'ind': objInd.id_})
objInd.producer = InformationSource(identity=Identity(
name=srcData.pkgLink))
objInd.observables = obsList
objInd.indicator_types = ["URL Watchlist"]
objInd.observable_composition_operator = "OR"
objInd.set_received_time(data['data'][sKey]['meta']['dateDL'])
try:
objInd.set_produced_time(
data['data'][sKey]['src']['verification_time'])
except:
pass
if not data['data'][sKey]['src']['target'] == 'Other':
from stix.ttp import TTP
objVictimTargeting = genData_VictimTargeting(
data['data'][sKey]['src'])
if obsURI:
objVictimTargeting.targeted_technical_details = genRefObs(
obsURI)
objTTP_vic = TTP()
objTTP_vic.title = "Targeting: " + data['data'][sKey]['src'][
'target']
objTTP_vic.victim_targeting = objVictimTargeting
objInd.add_indicated_ttp(objTTP_vic)
objInd.add_indicated_ttp(TTP(idref=objTTP.id_))
objPkg.add_indicator(objInd)
#updateDB_local(data,srcData)
return (objPkg)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
trimFile_btwn(srcData.filePath + srcData.fileName,
'<?xml version="1.0" encoding="ISO-8859-1" ?>', '</rss>')
srcDict = cnvt_XML2Dict(srcData.filePath + srcData.fileName)
srcData.pkgTitle = srcDict['rss']['channel']['title']
srcData.pkgDscrpt = srcDict['rss']['channel']['description']
srcData.pkgLink = srcDict['rss']['channel']['link']
for col in srcDict['rss']['channel']['item']:
sKey = col['guid']
sCol = col['title']
sDateVF = sCol.split('(')[1]
sDateVF = sDateVF[0:-1]
dSrt = datetime.strptime(sDateVF, "%Y-%m-%d")
sDateVF = dSrt.strftime("%Y-%m-%dT%H:%M:%SZ")
sCol = col['description']
lstAttrib = sCol.split(',')
sURI = lstAttrib[0][4:]
dictAttrib = {
"dateVF": cleanString(sDateVF),
"URI": cleanString(sURI),
"status": cleanString(lstAttrib[1].split(':')[1]),
"version": cleanString(lstAttrib[2].split(':')[1]),
"hash": cleanString(lstAttrib[3].split(':')[1]),
"title": cleanString(col['title']),
"link": cleanString(col['link']),
"dscrpt": cleanString(col['description']),
"fileName": "",
"ipAddr": "",
"domain": ""
}
if len(sURI) > 0:
tmpList = sURI.split("/")
if len(tmpList) > 1:
idx = len(tmpList) - 1
dictAttrib.update({"fileName": cleanString(tmpList[idx])})
if tmpList[2][0:1].isdigit():
dictAttrib.update({"ipAddr": cleanString(tmpList[2])})
else:
dictAttrib.update({"domain": cleanString(tmpList[2])})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
dstData[sKey]['status'] = dictAttrib['status']
#TODO:Check If Exist Element's inactive status changed
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
for sAddr in data[sKey]['attrib']['ipAddrList']:
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_destination = True
objAddr.address_value = sAddr
#objAddr.address_value.operator = 'Equals'
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isDestination: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
### Parsing Port Number
sPort = data[sKey]['attrib']['ipPort']
if len(sPort) > 0:
objPort = Port()
objPort.port_value = int(sPort)
objPort.port_value.condition = 'Equals'
sProtocol = data[sKey]['attrib']['ipProt']
if len(sProtocol) > 0:
objPort.layer4_protocol = sProtocol.upper()
obsPort = Observable(objPort)
objPort = None
obsPort.sighting_count = 1
obsPort.title = 'Port: ' + sPort
sDscrpt = 'PortNumber' + ': ' + sPort + " | "
sDscrpt += "Protocol: " + sProtocol.upper() + " | "
obsPort.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsPort)
### Add Generated observable to Indicator
objIndicator.add_indicator_type("IP Watchlist")
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
from stix.extensions.test_mechanism.snort_test_mechanism import SnortTestMechanism
from stix.common import InformationSource, Identity
testMech = SnortTestMechanism()
testMech.rules = [data[sKey]['attrib']['rule']]
testMech.efficacy = "Unknown"
infoSrc = InformationSource(identity=Identity(name=srcObj.Domain))
infoSrc.add_contributing_source("http://www.shadowserver.org")
infoSrc.add_contributing_source("https://spyeyetracker.abuse.ch")
infoSrc.add_contributing_source("https://palevotracker.abuse.ch")
infoSrc.add_contributing_source("https://zeustracker.abuse.ch")
testMech.producer = infoSrc
lstRef = data[sKey]['attrib']['reference'].split('|')
testMech.producer.references = lstRef
objIndicator.test_mechanisms = [testMech]
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
#objIndicator.set_produced_time(data[sKey]['attrib']['dateVF']);
objIndicator.set_received_time(data[sKey]['dateDL'])
### Title / Description Generator
objIndicator.set_received_time(data[sKey]['dateDL'])
sTitle = "sid:" + data[sKey]['attrib']['sid'] + " | "
sTitle += data[sKey]['attrib']['msg'] + " | "
sTitle += "rev:" + data[sKey]['attrib']['rev']
objIndicator.title = sTitle
sDscrpt = "SNORT Rule by Emergingthreats | " + data[sKey]['attrib'][
'rule']
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
nameList = data[sKey]['attrib']['flowbits']
if len(nameList) > 0:
nameList = nameList.split("|")
for sName in nameList:
sName = sName.split(",")[1]
objMalware.add_name(sName)
#objMalware.add_type("Remote Access Trojan")
objMalware.short_description = data[sKey]['attrib']['msg']
ttpTitle = data[sKey]['attrib']['classtype'] + " | " + data[sKey][
'attrib']['msg']
objTTP = TTP(title=ttpTitle)
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
