def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
trimFile_btwn(srcData.filePath + srcData.fileName,
'<?xml version="1.0" encoding="ISO-8859-1" ?>', '</rss>')
srcDict = cnvt_XML2Dict(srcData.filePath + srcData.fileName)
srcData.pkgTitle = srcDict['rss']['channel']['title']
srcData.pkgDscrpt = srcDict['rss']['channel']['description']
srcData.pkgLink = srcDict['rss']['channel']['link']
for col in srcDict['rss']['channel']['item']:
sKey = col['guid']
sCol = col['title']
sDateVF = sCol.split('(')[1]
sDateVF = sDateVF[0:-1]
dSrt = datetime.strptime(sDateVF, "%Y-%m-%d")
sDateVF = dSrt.strftime("%Y-%m-%dT%H:%M:%SZ")
sCol = col['description']
lstAttrib = sCol.split(',')
sURI = lstAttrib[0][4:]
dictAttrib = {
"dateVF": cleanString(sDateVF),
"URI": cleanString(sURI),
"status": cleanString(lstAttrib[1].split(':')[1]),
"version": cleanString(lstAttrib[2].split(':')[1]),
"hash": cleanString(lstAttrib[3].split(':')[1]),
"title": cleanString(col['title']),
"link": cleanString(col['link']),
"dscrpt": cleanString(col['description']),
"fileName": "",
"ipAddr": "",
"domain": ""
}
if len(sURI) > 0:
tmpList = sURI.split("/")
if len(tmpList) > 1:
idx = len(tmpList) - 1
dictAttrib.update({"fileName": cleanString(tmpList[idx])})
if tmpList[2][0:1].isdigit():
dictAttrib.update({"ipAddr": cleanString(tmpList[2])})
else:
dictAttrib.update({"domain": cleanString(tmpList[2])})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
dstData[sKey]['status'] = dictAttrib['status']
#TODO:Check If Exist Element's inactive status changed
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def main():
sSOURCEID = 'src_36'
### Setup for running as Main and template for use of adptr function
tmpJSON = getFile_JSON2Dict('../../data/openSourceList.json')
tmpDict = None
for sKey in tmpJSON:
if tmpJSON[sKey]['srcIndex'] == sSOURCEID:
tmpDict = tmpJSON[sKey]
### Without a valide Source Meta data this function will exit
if tmpDict == None:
retrun(False)
### This infomation is only require if you wish upload this data
### to a TAXII Server
dstCreds = {
"URI": "http://www.hailataxii.com/taxii-discovery-service",
"usrName": "lehigh_edu",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
dstCreds = {
"URI": "http://172.16.167.147/taxii-discovery-service",
"usrName": "admin",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
### The adpter function requires clsDataSource object populated
### with a minimum of data
srcData = clsDataSource(isDebugOn=True)
srcData.from_dict(tmpDict)
srcData.chnkSize = 250
# This version does not make use of the chucking capability
srcData.dstCreds = dstCreds
srcData.filePath = os.path.dirname(os.path.abspath(__file__)) + '/'
srcData.pkgTitle = "Domain Block list by MalwareDomain from Lehigh University"
srcData.pkgDscrpt = "A list of domains that are known to be used to propagate malware are listed in Bind and Windows zone files. The domains are loaded onto an internal DNS server. When a computer requests a URL or file from one of these domains, a fake reply is sent, thus preventing many malware installs from occuring"
srcData.pkgLink = "http://malwaredomains.lehigh.edu/files/"
print "------< NOT UPDATING >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
### Extract(src2Dict) Transform(dict2STIX) Load(sndTAXII)
dictObj = adptr_src2Dict(srcData, True)
if not dictObj == False:
iCnt = 0
tmpDict = {}
for sKey in dictObj:
if not dictObj[sKey] == None:
tmpDict[sKey] = dictObj[sKey]
if iCnt == srcData.chnkSize:
stixObj = adptr_dict2STIX(srcData, tmpDict)
tmpDict = {}
iCnt = 0
if not stixObj == False:
taxiiMsg = sndTAXII(srcData.dstCreds, stixObj.to_xml(),
True)
iCnt += 1
return (0)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
#print "------< Not Updating >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
srcDict = cnvt_XML2Dict(srcData.filePath + srcData.fileName)
srcData.pkgTitle = "Clean MX Phishing URL Block List "
srcData.pkgDscrpt = ""
srcData.pkgLink = "http://support.clean-mx.de/clean-mx/phishing.php"
for item in srcDict['output']['entries']['entry']:
sKey = item['id']
if item['first'] == "0":
item['first'] = None
else:
item['first'] = datetime.fromtimestamp(int(
item['first'])).strftime('%Y-%m-%dT%H:%M:%SZ')
if item['last'] == "0":
item['last'] = None
else:
item['last'] = datetime.fromtimestamp(int(
item['last'])).strftime('%Y-%m-%dT%H:%M:%SZ')
dictAttrib = item
lstNS = []
for i in range(1, 5):
if dictAttrib['ns' + str(i)]:
lstNS.append(dictAttrib['ns' + str(i)])
dictAttrib.update({"nsList": lstNS})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
#print "------< No Remote Data >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = '\t'
#oDialect.header = True
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
srcData.pkgTitle = "DShield.org Recommended Block List "
srcData.pkgDscrpt = "This list summarizes the top 20 attacking class C (/24) subnets over the last three days. The number of 'attacks' indicates the number of targets reporting scans from this subnet."
srcData.pkgLink = "http://feeds.dshield.org/block.txt"
sDateVF = None
s3daysAgo = None
try:
sDateVF = getFile_lineByValue(
srcData.filePath + srcData.fileName,
"updated:")[0].split("updated:")[1].strip()
sDateVF = datetime.strptime(sDateVF, "%a %b %d %H:%M:%S %Y %Z")
s3daysAgo = sDateVF + timedelta(days=-3)
if sDateVF:
sDateVF = sDateVF.strftime("%Y-%m-%dT%H:%M:%SZ")
s3daysAgo = s3daysAgo.strftime("%Y-%m-%dT%H:%M:%SZ")
srcData.pkgDscrpt = srcData.pkgDscrpt.replace(
'last three days.',
('last three days (' + s3daysAgo + " - " + sDateVF + ')'))
except:
pass
for col in srcDict:
if 'End' in srcDict[col]:
sKey = srcDict[col]['Start'] + "##comma##" + srcDict[col]['End']
else:
continue
dictAttrib = srcDict[col]
if sDateVF:
dictAttrib.update({"dateVF": str(sDateVF)})
if s3daysAgo:
dictAttrib.update(
{"dateRange": str(s3daysAgo) + " - " + str(sDateVF)})
if 'noemail' in srcDict[col]['email']:
dictAttrib.update({"email": None})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)
def getRmt_data(srcData):
from lib.utils.mngRmtObjs import getRmt_File
getRmt_File(srcData.srcCreds, srcData.filePath + srcData.fileName)
return ()
def main():
sSOURCEID = 'src_83'
### Setup for running as Main and template for use of adptr function
tmpJSON = getFile_JSON2Dict('../../data/openSourceList.json')
tmpDict = None
if tmpJSON:
for sKey in tmpJSON:
if tmpJSON[sKey]['srcIndex'] == sSOURCEID:
tmpDict = tmpJSON[sKey]
else:
return (0)
### Without a valide Source Meta data this function will exit
if tmpDict == None:
retrun(False)
### This infomation is only require if you wish upload this data
### to a TAXII Server
dstCreds = {
"URI": "http://www.hailataxii.com/taxii-discovery-service",
"usrName": "blutmagie_de",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
dstCreds = {
"URI": "http://172.16.167.147/taxii-discovery-service",
"usrName": "admin",
"usrPass": "******",
"crtName": "",
"crtPass": ""
}
### The adpter function requires clsDataSource object populated
### with a minimum of data
srcData = clsDataSource(isDebugOn=True)
srcData.from_dict(tmpDict)
srcData.chnkSize = 250
# This version does not make use of the chucking capability
srcData.dstCreds = dstCreds
srcData.filePath = os.path.dirname(os.path.abspath(__file__)) + '/'
srcData.pkgTitle = "Tor 'Exit Point' router IP/Host list"
srcData.pkgDscrpt = "torstatus.blutmagie.de idenitifes the following IP/Host as Tor network 'Exit Point' routers"
srcData.pkgLink = "http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv"
#print "------< NOT UPDATING >------"
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
### Extract(src2Dict) Transform(dict2STIX) Load(sndTAXII)
dictObj = adptr_src2Dict(srcData, True)
if not dictObj == False:
iCnt = 0
tmpDict = {}
if len(dictObj) > srcData.chnkSize:
for sKey in dictObj:
if not dictObj[sKey] == None:
tmpDict[sKey] = dictObj[sKey]
if iCnt == srcData.chnkSize:
stixObj = adptr_dict2STIX(srcData, tmpDict)
tmpDict = {}
iCnt = 0
if not stixObj == False:
taxiiMsg = sndTAXII(srcData.dstCreds, stixObj.to_xml(),
True)
iCnt += 1
else:
stixObj = adptr_dict2STIX(srcData, dictObj)
if not stixObj == False:
taxiiMsg = sndTAXII(srcData.dstCreds, stixObj.to_xml(), True)
def adptr_src2Dict(src_data, isUpdateNewDataOnly):
namefunc = 'adptr_src2Dict()'
stxt = "Called... "
sndMSG(stxt, 'INFO', namefunc)
### Input Check
if src_data is None:
# TODO: Needs error msg: Missing srcData Object
return False
locDataFile = 'db_' + src_data.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
# srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
# print "------< NOT UPDATING >------"
if not getRmt_File(src_data.srcCreds,
src_data.filePath + src_data.fileName) == True:
# if no source data is found, this script will exit
return False
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
trimFile_btwn(src_data.filePath + src_data.fileName,
'<?xml version="1.0" encoding="ISO-8859-1" ?>', '</rss>')
srcDict = cnvt_XML2Dict(src_data.filePath + src_data.fileName)
### DEBUG CODE ####
###################
src_data.pkgTitle = srcDict['rss']['channel']['title']
src_data.pkgDscrpt = srcDict['rss']['channel']['description']
src_data.pkgLink = srcDict['rss']['channel']['link']
newData = {}
for col in srcDict['rss']['channel']['item']:
sKey = col['guid']
sCol = col['title']
sDateVF = sCol.split('(')[1]
sDateVF = sDateVF[0:-1]
try:
dSrt = datetime.strptime(sDateVF, "%Y-%m-%d %H:%M:%S")
sDateVF = dSrt.strftime("%Y-%m-%dT%H:%M:%SZ")
except:
sDateVF = None
sDomain = None
sIPAddr = cleanString(sCol.split('(')[0])
if not isIPv4(sIPAddr):
sDomain = sIPAddr
sIPAddr = None
sCol = col['description']
lstAttrib = sCol.split(',')
dictAttrib = {
"dateVF": sDateVF,
"title": cleanString(col['title']),
"link": cleanString(col['link']),
"dscrpt": cleanString(col['description']),
"ipAddr": sIPAddr,
"domain": sDomain,
}
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
# TODO:Check If Exist Element's inactive status changed
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if not isUpdateNewDataOnly:
newData = dstData
if len(newData) > 0:
stxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(stxt, 'INFO', namefunc)
else:
stxt = "Found no new data"
sndMSG(stxt, 'INFO', namefunc)
newData = False
return newData
def adptr_src2Dict(srcData, isUpdateNewDataOnly):
sNAMEFUNC = 'adptr_src2Dict()'
sTxt = "Called... "
sndMSG(sTxt, 'INFO', sNAMEFUNC)
### Input Check
if srcData == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
sName = srcData.fileName
locDataFile = 'db_' + srcData.fileName.split('.')[0] + '.json'
### fetch from Source location for newest version
#srcData.getSrcData(); #TODO: This function in the clsDataSource is not completed
# so this getRmt_File is used until class is completed
if not getRmt_File(srcData.srcCreds,
srcData.filePath + srcData.fileName) == True:
# if no source data is found, this script will exit
return (False)
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
### Here the code become specific (unique) this data source
### in time I hope to refactor out as much unique as possible
### Parse Source File in to a Dictionary Object
dstData = getFile_JSON2Dict(locDataFile)
if not dstData:
dstData = {}
newData = {}
oDialect = clsCSVDialect()
oDialect.from_dict(srcData.parsearg)
oDialect.delimiter = '\n'
srcDict = cnvt_CSV2Dict(srcData.filePath + srcData.fileName,
dialect=oDialect)
srcData.pkgTitle = "SNORT Rule by Emergingthreats | Block Botnet Command and Control"
srcData.pkgDscrpt = "Emerging Threats Botnet Command and Control drop rules. These are generated from the EXCELLENT work done by the Shadowserver team and the abuse.ch folks. All Volunteers, we're grateful for their dedication! http://www.shadowserver.org; https://spyeyetracker.abuse.ch; https://palevotracker.abuse.ch; https://zeustracker.abuse.ch. More information available at www.emergingthreats.net"
srcData.pkgLink = "http://rules.emergingthreats.net/blockrules/emerging-botcc.portgrouped.rules"
for col in srcDict:
# {0: u'alert tcp $HOME_NET any -> 50.116.1.225 22 (msg:"ET CNC Shadowserver Reported CnC Server Port 22 Group 1"; flags:S; reference:url,doc.emergingthreats.net/bin/view/Main/BotCC; reference:url,www.shadowserver.org; threshold: type limit, track by_src, seconds 360, count 1; classtype:trojan-activity; flowbits:set,ET.Evil; flowbits:set,ET.BotccIP; sid:2405000; rev:3570;)'}
sKey = srcDict[col][0]
strTmp = sKey.split("(")
tmpList = strTmp[0].split(" ")
ipProt = None
if tmpList[1]:
ipProt = tmpList[1]
ipList = None
if tmpList[5]:
if "[" in tmpList[5]:
tmpList[5] = tmpList[5][1:-1]
ipList = tmpList[5].split(",")
ipPort = None
if tmpList[6]:
ipPort = tmpList[6]
attrList = strTmp[1].split(";")[:-1]
tmpDict = {}
for i in range(len(attrList)):
attrList[i] = cleanString(attrList[i])
tmpKey = attrList[i].split(':')[0]
tmpVal = attrList[i].split(':')[1]
if tmpKey in tmpDict:
tmpDict[tmpKey] += "|" + tmpVal
else:
tmpDict.update({tmpKey: tmpVal})
dictAttrib = tmpDict
dictAttrib.update({
'ipAddrList': ipList,
'rule': sKey,
'ipPort': ipPort,
'ipProt': ipProt
})
if sKey in dstData:
dstData[sKey]['cnt'] += 1
dstData[sKey]['dateDL'] = getUTCTime()
else:
### Add new Data to local Database
dstData[sKey] = {'cnt': 1, 'dateDL': getUTCTime()}
dstData[sKey]['attrib'] = dictAttrib
### Generate list of new data only for STIX output
newData[sKey] = dstData[sKey]
sndFile_Dict2JSON(dstData, locDataFile)
if isUpdateNewDataOnly == False:
newData = dstData
if len(newData) > 0:
sTxt = "Found " + str(len(newData)) + " new data elements"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
else:
sTxt = "Found no new data"
sndMSG(sTxt, 'INFO', sNAMEFUNC)
newData = False
return (newData)