def checkMISPHashEventExist(self, filename):
try:
LibIoC_DK.debugging(
"Checking the MISP hash event existence: %s" % (filename),
main._DEBUG_, main._LOGGING_, main.hFile)
response = self.misp_connection.search_all(filename)
except requests.exceptions.HTTPError:
LibIoC_DK.debugging("HTTPError occurred", main._DEBUG_,
main._LOGGING_, main.hFile)
return False
if (main.is_py2 and not response.has_key('response')) or (
main.is_py3 and 'response' not in response):
#if not response.has_key('response'):
LibIoC_DK.debugging("The MISP hash event NOT exist", main._DEBUG_,
main._LOGGING_, main.hFile)
return False
for r in response['response']:
if r['Event']['info'] == filename:
LibIoC_DK.debugging("The MISP hash event ALREADY exists",
main._DEBUG_, main._LOGGING_, main.hFile)
return True
for attr in r['Event']['Attribute']:
if attr['category'] == 'Payload installation':
if attr['value'] == filename:
LibIoC_DK.debugging(
"The hash value ALREADY stored as an attribute of : %s"
% (r['Event']['info']), main._DEBUG_,
main._LOGGING_, main.hFile)
return True
return False
def processFile(misp, f, config_value):
# Adding MISP by APT report-wise manner: one MISP event is consisted by one API report.
if _MISP_EVENT_GENERATION_TYPE == 1:
# check if the event data is already in MISP server
if misp.checkEventExist(LibIoC_DK.getFileName(f)) == True:
return
LibIoC_DK.debugging("Processing file: %s"%(LibIoC_DK.getFileName(f)), _DEBUG_, _LOGGING_, hFile)
# extract IoCs in the report file
ioc = parse_ioc(f)
if ioc is not None and len(ioc) > 0:
date = getFileDate(f, config_value)
misp.createMISPEvent(ioc, f, date)
if _STAT_:
ioc_stat.finalizeTempStatistics()
ioc_stat.reset()
ioc_stat.saveIoCStatistics()
# Adding MISP by malware-wise manner. MISP events are consisted by each malware hashes in APT report
# if further analysis results could be retrieved from malware repository,
# otherwise rest of IoCs are stored in one MISP event under the event name as the title of the report.
elif _MISP_EVENT_GENERATION_TYPE == 2:
if misp.checkMISPReportEventExist(LibIoC_DK.getFileName(f)):
return
nonHash_IoC = []
hash_IoC = []
ioc = parse_ioc(f)
if ioc is not None and len(ioc) > 0:
for attr in ioc:
# if the IoC is a hash value and the analysis result of the hash could be retrieved,
# create a MISP event of the hash value
if attr[0] == 'MD5' or attr[0] == 'SHA1' or attr[0] == 'SHA256':
if _STAT_:
ioc_stat.increaseHashInReport()
# if the IoC is not a hash value or the analysis result of a hash could not be retrieved,
# keep the IoC to store under the MISP event of the API report
else:
nonHash_IoC.append(('report',attr[0],attr[1]))
# add hash IoCs in each MISP events
if (_MODE_==2 or _MODE_==3) and len(hash_IoC) > 0:
for attr in hash_IoC:
misp.createMISPEvent(attr[2], f)
# add non-hash IoCs in one MISP event
if (_MODE_==1 or _MODE_==3) and len(nonHash_IoC) > 0:
date = getFileDate(f, config_value)
misp.createMISPEvent(nonHash_IoC+hash_IoC, f, date)
if _STAT_:
ioc_stat.finalizeTempStatistics()
ioc_stat.reset()
ioc_stat.saveIoCStatistics()
return
def getMISPEvent_of_month(self, year, month):
retval = []
from_date = ''
to_date = ''
if month < 10:
from_date = year + "-0%d-01" % (month)
to_date = year + "-0%d-31" % (month)
else:
from_date = year + "-%d-01" % (month)
to_date = year + "-%d-31" % (month)
try:
ee = self.misp.misp_connection.search(date_from=from_date,
date_to=to_date)['response']
except Exception as e:
print('error misp_connection.search (month=%s, year=%s)' %
(month, year))
return
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
continue
else:
retval.append(e)
hash_event = []
for r in retval:
for yy in range(2008, 3000):
for mm in range(1, 13):
if mm < 10:
from_date = str(yy) + "-0%d-01" % (mm)
to_date = str(yy) + "-0%d-31" % (mm)
else:
from_date = str(yy) + "-%d-01" % (mm)
to_date = str(yy) + "-%d-31" % (mm)
try:
ee = self.misp.misp_connection.search(
values=r['Event']['info'],
category="Other",
type_attribute="comment",
date_from=from_date,
date_to=to_date)['response']
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
hash_event.append(e)
except Exception as e:
print('error %s / %s (month=%d, year=%d)' %
(str(e), r['Event']['info'], mm, yy))
continue
return retval + hash_event
return
def getMalwareInfo(self, _hash, misp_event=False, misp=False):
LibIoC_DK.debugging("Getting the malware info: %s" %(_hash) , main._DEBUG_, main._LOGGING_, main.hFile)
result1 = self.getMalwareStaticInfo(_hash, misp_event, )
if result1:
result2 = self.getMalwareBehaviorInfo(_hash, misp_event, misp)
if misp_event and misp:
return result1 or result2
else:
if result2:
return result1 + result2
else:
return result1
else:
LibIoC_DK.debugging("NO the malware info found: %s" %(_hash), main._DEBUG_, main._LOGGING_, main.hFile)
return False
def getEventIoCStatistics(self, event_list):
if event_list == None:
return
s = PostStat()
for e in event_list:
if LibIoC_DK.isHash(e['Event']['info']):
s.add_hash_events()
else:
s.add_report_events()
for attr in e['Event']['Attribute']:
if 'md5' in attr['type'] or 'sha1' in attr[
'type'] or 'sha256' in attr['type']:
s.add_hash()
elif 'ip' in attr['type']:
s.add_IP()
elif 'url' in attr['type']:
s.add_URL()
elif 'email' in attr['type']:
s.add_email()
elif 'vulnerability' in attr['type']:
s.add_cve()
elif 'filename' in attr['type']:
s.add_filename()
elif 'pdb' in attr['type']:
s.add_pdb()
elif 'text' in attr['type']:
if 'TimeStamp' in attr['comment']:
s.add_date()
elif 'digital sign serial number' in attr['comment']:
s.add_codesign()
else:
s.add_string()
return s.stat
def createMISPEvent(self, val, filename='', _date=''):
if type(val) == tuple and LibIoC_DK.isHash(val[2]):
if not self.checkMISPHashEventExist(val[2]):
if self.createMISPEventFromHash(val[2], filename):
return True
else:
LibIoC_DK.debugging("Failed to create the MISP hash event",
main._DEBUG_, main._LOGGING_,
main.hFile)
return False
else:
eid = self.getMISPEventID(val[2])
e = self.misp_connection.get_event(eid)
self.misp_connection.add_named_attribute(
e,
category='Other',
type_value='comment',
value=LibIoC_DK.getFileName(
filename)) # this will work as the ground truth of IoC
return True
elif type(val) == list:
if self.createMISPEventFromReport(val, filename, _date):
return True
LibIoC_DK.debugging(
"Failed to create the MISP event: val(%s), filename(%s)" %
(val, LibIoC_DK.getFileName(filename)), main._DEBUG_,
main._LOGGING_, main.hFile)
return False
def getMISPEventID(self, filename):
try:
response = self.misp_connection.search_all(filename)
except requests.exceptions.HTTPError:
LibIoC_DK.debugging("[getMISPEventID] HTTPError occurred",
main._DEBUG_, main._LOGGING_, main.hFile)
return False
if not response.has_key('response'):
return False
for r in response['response']:
if r['Event']['info'] == filename:
return r['Event']['id']
if LibIoC_DK.isHash(filename):
for attr in r['Event']['Attribute']:
if attr['category'] == 'Payload installation':
if attr['value'] == filename:
return r['Event']['id']
return False
def checkAttribute(self, attr, eid=None):
if eid == None:
try:
response = self.misp_connection.search(attr)
except requests.exceptions.HTTPError:
LibIoC_DK.debugging(
"[checkAttribute] HTTPError while searching " + attr,
main._DEBUG_, main._LOGGING_, main.hFile)
return False
except:
return False
if response.has_key('response'):
retval = []
for e in response['response']:
found = False
for i in range(len(e['Event']['Attribute'])):
if e['Event']['Attribute'][i]['value'] == attr.lower():
retval.append(e['Event']['id'])
found = True
break
if found:
break
if found:
break
return retval
else:
return False
else:
if type(eid) != 'str':
eid = str(eid)
response = self.misp_connection.get(eid)
if not response.has_key('Event'):
return False
if len(response) > 0 and len(response['Event']['Attribute']) > 0:
for val in response['Event']['Attribute']:
for i in val:
if i['value'] == attr.lower():
return True
return False
def getMalwareCollectedDate(self, _hash):
_params = {
'api_key': self.key,
'hash': _hash,
'env_type': 2
}
response = None
try:
LibIoC_DK.debugging("Getting the malware collection date: %s"%(_hash), main._DEBUG_, main._LOGGING_, main.hFile)
response = requests.get('https://private.api.malwares.com/v3/file/mwsinfo', params=_params)
self.query_counter = self.query_counter + 1
except:
print('getMalwareCollectedDate exception!')
return False
try:
summary = response.json()
except:
LibIoC_DK.debugging("Exception occurred during decoding response", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if summary['result_code'] <= 0 and summary['result_code'] != 2:
LibIoC_DK.debugging("Error returned while getting the malware collection date", main._DEBUG_, main._LOGGING_, main.hFile)
return False
return summary['first_seen'].encode('utf-8')[0:10]
def downloadMalware(self, _hash, save=False):
_params = {
'api_key': self.key,
'hash': _hash,
}
try:
LibIoC_DK.debugging("Downloading the malware: %s"%(_hash), main._DEBUG_, main._LOGGING_, main.hFile)
response = requests.get('https://private.api.malwares.com/v3/file/download', params=_params)
self.query_counter = self.query_counter + 1
except:
print('downloadMalware exception!')
return False
if response == None or response == '':
LibIoC_DK.debugging("NO response returned while downloading the malware", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if response.status_code < 0 or 'Invalid type of hash' in response.content:
LibIoC_DK.debugging("Error returned while downloading the malware", main._DEBUG_, main._LOGGING_, main.hFile)
return False
downloaded_file = response.content
# Save Malware File
if save:
f = open(_hash, 'w')
f.write(downloaded_file)
f.close()
self.unzipMalware(_hash)
return True
return downloaded_file
def getMISPEvent_of_Year(self, year):
retval = []
ee = self.misp.misp_connection.search(date_from=year + "-01-01",
date_to=year +
"-12-31")['response']
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
continue
else:
retval.append(e)
hash_event = []
for r in retval:
print(r['Event']['info'] + '\n')
ee = self.misp.misp_connection.search(
values=r['Event']['info'],
category="Other",
type_attribute="comment")['response']
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
hash_event.append(e)
return retval + hash_event
def waitToNextMidnight_UTC(self):
LibIoC_DK.debugging("current time : %s UTC" % (str(datetime.utcnow())), main._DEBUG_, main._LOGGING_, main.hFile)
LibIoC_DK.debugging("Wait to tomorrow 00:00 am. UTC", main._DEBUG_, main._LOGGING_, main.hFile)
stop_d = datetime.utcnow().day
while True:
time.sleep(600)
if datetime.utcnow().day > stop_d:
break
LibIoC_DK.debugging("Waked up!!!!", main._DEBUG_, main._LOGGING_, main.hFile)
return
def checkMISPReportEventExist(self, filename):
try:
LibIoC_DK.debugging(
"Checking the MISP file event existence: %s" % (filename),
main._DEBUG_, main._LOGGING_, main.hFile)
response = self.misp_connection.search_all(filename)
except requests.exceptions.HTTPError:
LibIoC_DK.debugging("HTTPError while querying " + filename,
main._DEBUG_, main._LOGGING_, main.hFile)
return False
if (main.is_py2 and not response.has_key('response')) or (
main.is_py3 and 'response' not in response):
LibIoC_DK.debugging("The MISP file event NOT exist", main._DEBUG_,
main._LOGGING_, main.hFile)
return False
for r in response['response']:
if r['Event']['info'] == filename:
LibIoC_DK.debugging("The MISP file event ALREADY exists",
main._DEBUG_, main._LOGGING_, main.hFile)
return True
return False
def exportXML_Date(self, year, event_type, output_filename):
#import xml.etree.ElementTree as ET
from lxml import etree as ET
root = ET.Element('CTIMinerDataset')
report_event = []
for mm in range(1, 13):
if mm < 10:
from_date = str(year) + "-0%d-01" % (mm)
to_date = str(year) + "-0%d-31" % (mm)
else:
from_date = str(year) + "-%d-01" % (mm)
to_date = str(year) + "-%d-31" % (mm)
try:
ee = self.misp_connection.search(date_from=from_date,
date_to=to_date)['response']
except Exception as e:
print('error %s / (year=%d, month=%d)' % (str(e), year, mm))
continue
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
continue
else:
if event_type == 'report':
root = self.addEventDataToElementTree(root, e)
elif event_type == 'malware':
report_event.append(e)
if event_type == 'report':
tree = ET.ElementTree(root)
f = open(output_filename, 'wb')
f.write(ET.tostring(tree, pretty_print=True))
f.close()
return
for r in report_event:
for yy in range(2008, 3000):
for mm in range(1, 13):
try:
if mm < 10:
from_date = str(yy) + "-0%d-01" % (mm)
to_date = str(yy) + "-0%d-31" % (mm)
else:
from_date = str(yy) + "-%d-01" % (mm)
to_date = str(yy) + "-%d-31" % (mm)
ee = self.misp_connection.search(
values=r['Event']['info'],
category="Other",
type_attribute="comment",
date_from=from_date,
date_to=to_date)['response']
for e in ee:
if LibIoC_DK.isHash(e['Event']['info']):
root = self.addEventDataToElementTree(root, e)
except Exception as e:
print('error %s / %s (month=%d, year=%d)' %
(str(e), r['Event']['info'], mm, yy))
continue
tree = ET.ElementTree(root)
f = open(output_filename, 'wb')
f.write(ET.tostring(tree, pretty_print=True))
f.close()
return
def processFileList(misp, file_names, config_value):
processed_list_root = openFileProcessingResult()
# Adding MISP by APT report-wise manner: one MISP event is consisted by one API report.
if _MISP_EVENT_GENERATION_TYPE == 1:
for f in file_names:
# check if the event data is already in MISP server
if misp.checkEventExist(LibIoC_DK.getFileName(f)) == True:
LibIoC_DK.debugging("The MISP file event ALREADY exists: %s" %(LibIoC_DK.getFileName(f)), _DEBUG_, _LOGGING_, hFile)
continue
# extract IoCs in the report file
ioc = parse_ioc(f)
if ioc is not None and len(ioc) > 0:
date = getFileDate(f, config_value)
misp.createMISPEvent(ioc, f, date)
if _STAT_:
ioc_stat.finalizeTempStatistics()
ioc_stat.reset()
ioc_stat.saveIoCStatistics()
# Adding MISP by malware-wise manner. MISP events are consisted by each malware hashes in APT report
# if further analysis results could be retrieved from malware repository,
# otherwise rest of IoCs are stored in one MISP event under the event name as the title of the report.
elif _MISP_EVENT_GENERATION_TYPE == 2:
for f in file_names:
if isFileProcessedBefore(processed_list_root, LibIoC_DK.getFileName(f)) or misp.checkMISPReportEventExist(LibIoC_DK.getFileName(f)):
continue
nonHash_IoC = []
hash_IoC = []
LibIoC_DK.debugging("Parsing IoC...", _DEBUG_, _LOGGING_, hFile)
ioc = parse_ioc(f)
if ioc is not None and len(ioc) > 0:
for attr in ioc:
# if the IoC is a hash value and the analysis result of the hash could be retrieved,
# create a MISP event of the hash value
if attr[0] == 'MD5' or attr[0] == 'SHA1' or attr[0] == 'SHA256':
hash_IoC.append(('report',attr[0],attr[1]))
if _STAT_:
ioc_stat.increaseHashInReport()
# if the IoC is not a hash value or the analysis result of a hash could not be retrieved,
# keep the IoC to store under the MISP event of the API report
else:
nonHash_IoC.append(('report',attr[0],attr[1]))
# add hash IoCs in each MISP events
if (_MODE_==2 or _MODE_==3) and len(hash_IoC) > 0:
for attr in hash_IoC:
if misp.createMISPEvent(attr, f) and _STAT_:
misp.ioc_stat.increaseAnalyzedHash()
else:
nonHash_IoC.append(attr)
# add non-hash IoCs in one MISP event
if (_MODE_==1 or _MODE_==3) and len(nonHash_IoC) > 0:
date = getFileDate(f, config_value)
if misp.createMISPEvent(nonHash_IoC, f, date):
addFileProcessingResult(processed_list_root, LibIoC_DK.getFileName(f), 'success')
else:
addFileProcessingResult(processed_list_root, LibIoC_DK.getFileName(f), 'fault')
if _STAT_:
ioc_stat.finalizeTempStatistics()
ioc_stat.reset()
ioc_stat.saveIoCStatistics()
else:
LibIoC_DK.debugging("NO IoC found in %s" %(LibIoC_DK.getFileName(f)), _DEBUG_, _LOGGING_, hFile)
addFileProcessingResult(processed_list_root, LibIoC_DK.getFileName(f), 'noIoC')
saveFileProcessingResult(processed_list_root)
return
def createMISPEventFromHash(self, _hash, filename, additional_hash=False):
LibIoC_DK.debugging("Creating the MISP hash event: %s" % (_hash),
main._DEBUG_, main._LOGGING_, main.hFile)
_hash = _hash.lower()
# if the MISP event for the hash value already exists, stop the further process.
if self.checkMISPHashEventExist(_hash):
LibIoC_DK.debugging("The MISP hash event ALREADY exists",
main._DEBUG_, main._LOGGING_, main.hFile)
return False
result = self.malware_repo_connector.getMalwareInfo(_hash)
if not result:
return False
event = self.misp_connection.new_event(0, 1, 2, _hash)
self.misp_connection.add_named_attribute(
event,
category='Other',
type_value='comment',
value=LibIoC_DK.getFileName(
filename)) # this will work as the ground truth of IoCs
# the first three attributes in the result is md5, sha1, and sha256 representation malware hash, so manually store them in the event.
md5 = result.pop(0)[2]
sha1 = result.pop(0)[2]
sha256 = result.pop(0)[2]
self.misp_connection.add_hashes(event,
category='Payload installation',
md5=md5)
if main._STAT_:
self.ioc_stat.addCategory2('hash')
self.misp_connection.add_hashes(event,
category='Payload installation',
sha1=sha1)
if main._STAT_:
self.ioc_stat.addCategory2('hash')
self.misp_connection.add_hashes(event,
category='Payload installation',
sha256=sha256)
if main._STAT_:
self.ioc_stat.addCategory2('hash')
if main._DOWNLOAD_MALWARE_:
sample_path = self.config[
'SampleRoot'] + '/' + LibIoC_DK.getReportPublicationYear(
filename)
if not os.path.exists(sample_path + '/' + _hash):
malware_buffer = self.malware_repo_connector.downloadMalware(
sha256, False)
if malware_buffer:
if not os.path.exists(sample_path):
os.makedirs(sample_path)
f = open(sample_path + '/' + _hash, 'wb')
f.write(malware_buffer)
f.close()
extracted = self.malware_repo_connector.unzipMalware(
sample_path + '/' + _hash, sample_path)
if extracted.lower() != _hash.lower():
os.remove(sample_path + '/' + _hash)
_hash = extracted
header_info = LibIoC_DK.getMalwareHeaderInfo(sample_path +
'/' + _hash)
if header_info is not None:
self.addMalwareHeaderInfo(header_info, event)
if main._STAT_ and self.ioc_stat.report_name != filename:
self.ioc_stat.setReportBuffer(filename)
if main._PARALLELIZE_ATTRIB_ADDITION_:
# Parallelized Version
print('[Hash] Parallelized attribute storing...')
joblib.Parallel(joblib.cpu_count())(
delayed(addAttribute)(self, event, attr) for attr in result)
else:
# Sequential Version
for attr in result:
self.addAttribute(event, attr, filename)
LibIoC_DK.debugging("The MISP hash event created", main._DEBUG_,
main._LOGGING_, main.hFile)
return True
def addAttribute(self, event, attr, filepath):
LibIoC_DK.debugging(
"Adding Attribute: filename(%s), attribute(%s)" %
(LibIoC_DK.getFileName(filepath), attr), main._DEBUG_,
main._LOGGING_, main.hFile)
return addAttribute(self, event, attr, filepath)
def createMISPEventFromReport(self, ioc, filename, _date):
LibIoC_DK.debugging(
"Creating the MISP report event: %s" %
(LibIoC_DK.getFileName(filename)), main._DEBUG_, main._LOGGING_,
main.hFile)
'''
# if the MISP event for the filename already exists, add ioc in the event
eid = self.getMISPEventID(LibIoC_DK.getFileName(filename))
if eid:
event = self.misp_connection.get_event(eid)
for attr in ioc:
self.addAttribute(event, attr)
return False
'''
if self.checkMISPReportEventExist(filename):
LibIoC_DK.debugging("The MISP report event ALREADY exists",
main._DEBUG_, main._LOGGING_, main.hFile)
return False
try:
if _date is not None:
#_date = datetime.datetime.strptime(_date, "%m/%d/%Y").strftime("%Y-%m-%d")
_date = _date.split("/")
if int(_date[0]) > 12:
_date = datetime.date(int(_date[0]), int(_date[1]),
int(_date[2])).isoformat()
else:
_date = datetime.date(int(_date[2]), int(_date[0]),
int(_date[1])).isoformat()
except Exception as e:
print e
event = self.misp_connection.new_event(0,
1,
2,
LibIoC_DK.getFileName(filename),
date=_date)
self.misp_connection.add_named_attribute(
event,
category='Other',
type_value='comment',
value=LibIoC_DK.getFileName(
filename)) # this will work as the ground truth of IoC
if main._STAT_ and self.ioc_stat.report_name != filename:
self.ioc_stat.setReportBuffer(filename)
if main._PARALLELIZE_ATTRIB_ADDITION_:
# Parallelized Version
print('[Report] Parallelized attribute storing...')
attr_added = joblib.Parallel(joblib.cpu_count())(
delayed(addAttribute)(self, event, attr, filename)
for attr in ioc)
else:
# Sequential Version
attr_added = False
for attr in ioc:
attr_added = self.addAttribute(event, attr,
filename) or attr_added
if (type(attr_added) is bool and not attr_added):
self.misp_connection.delete_event(event['Event']['id'])
LibIoC_DK.debugging(
"NO attribute added for the report: %s" %
(LibIoC_DK.getFileName(filename)), main._DEBUG_,
main._LOGGING_, main.hFile)
return False
if type(attr_added) is list:
if not (True in attr_added):
self.misp_connection.delete_event(event['Event']['id'])
LibIoC_DK.debugging(
"NO attribute added for the report: %s" %
(LibIoC_DK.getFileName(filename)), main._DEBUG_,
main._LOGGING_, main.hFile)
return False
LibIoC_DK.debugging("The MISP report event created", main._DEBUG_,
main._LOGGING_, main.hFile)
return True
def getMalwareStaticInfo(self, _hash, misp_event=False, misp=False):
add_result_to_system = False
if misp_event and misp:
add_result_to_system = True
retval = []
_params = {
'api_key': self.key,
'hash': _hash
}
response = None
static = None
if self.getAIScore(_hash) < 60:
return False
while True:
try:
LibIoC_DK.debugging("Getting the malware static info", main._DEBUG_, main._LOGGING_, main.hFile)
response = requests.get('https://private.api.malwares.com/v3/file/staticinfo', params=_params)
self.query_counter = self.query_counter + 1
except:
LibIoC_DK.debugging("Exception occurred during the malware static info request", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if response == None or response == '':
LibIoC_DK.debugging("NO response returned as the malware static info", main._DEBUG_, main._LOGGING_, main.hFile)
return False
try:
static = response.json()
except:
LibIoC_DK.debugging("Exception occurred during decoding response", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if static['result_code'] == 1: # Data exists
break
elif static['result_code'] == 2: # Analyzing
print("[Static Info] Analyzing ("+_hash+")")
return False
elif static['result_code'] == -14: # Over the daily request limit
LibIoC_DK.debugging("Over the daily request limit", main._DEBUG_, main._LOGGING_, main.hFile)
LibIoC_DK.debugging("Accumulated the number of query: %d" %(self.query_counter), main._DEBUG_, main._LOGGING_, main.hFile)
self.waitToNextMidnight_UTC()
self.query_counter = 0
elif static['result_code'] <= 0:
LibIoC_DK.debugging("Error returned while getting the malware static info", main._DEBUG_, main._LOGGING_, main.hFile)
return False
# add all different hash representations of the malware
if 'md5' in static:
static_input = ('static','md5',static['md5'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
if 'sha1' in static:
static_input = ('static','sha1',static['sha1'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
if 'sha256' in static:
static_input = ('static','sha256',static['sha256'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
# add string information in the malware
if 'strings_result' in static and static['strings_result'] is not None:
prev_offset = 0
for _str in static['strings_result']['strings']:
# The concatenated attribute the right after previous one without null string, we ignore it.
offset_distance = int(_str['offset'],16) - prev_offset
prev_offset = int(_str['offset'],16)
charset = _str['charset']
if charset == 'ascii':
charset_len = 1
elif charset == 'unicode':
charset_len = 2
if offset_distance <= len(_str['strings'])*charset_len:
continue
if _str['pattern'] == 'url':
# since the returned string results does not always contain clear ioc, filter the outputs.
ioc = LibIoC_DK.filterRegularIoC(_str['strings'], _str['pattern'])
if ioc is not None:
for _ioc in ioc:
static_input = ('static','url',_ioc)
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
elif _str['pattern'] == 'ip':
# since the returned string results does not always contain clear ioc, filter the outputs.
ioc = LibIoC_DK.filterRegularIoC(_str['strings'], _str['pattern'])
if ioc is not None:
for _ioc in ioc:
if _ioc.endswith('0.0'):
continue
static_input = ('static','ip',_ioc)
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
elif _str['pattern'] == 'path':
if LibIoC_DK.isFileofInterest(_str['strings']):
static_input = ('static','filepath',_str['strings'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
# We found the almost the hash attribute returned as static analysis is errornous and meaningless,
# since it is very unusual to insert hashes with in the malware code.
elif _str['pattern'] == 'hash':
None
#
# # since the returned string results does not always contain clear ioc, filter the outputs.
# ioc = LibIoC_DK.filterRegularIoC(_str['strings'], _str['pattern'])
# if ioc is not None:
# # this condition filters irrelevant strings look like hash values
# if len(ioc) > 1:
# remove_val = []
# for i in range(len(ioc)-1):
# if remove_val.count(ioc[i]) != 0:
# continue
# for j in range(i+1,len(ioc)):
# tmp1 = ioc[i].encode('utf-8')
# tmp2 = ioc[j].encode('utf-8')
# if tmp1 in tmp2:
# if remove_val.count(ioc[i]) == 0:
# remove_val.append(ioc[i])
# if remove_val.count(ioc[j]) == 0:
# remove_val.append(ioc[j])
# for r in range(len(remove_val)):
# while ioc.count(remove_val[r]) != 0:
# ioc.remove(remove_val[r])
#
# for _ioc in ioc:
# _ioc = _ioc.encode('utf-8')
# if len(_ioc) == 32:
# static_input = ('static','md5',_ioc)
# retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
# elif len(_ioc) == 40:
# static_input = ('static','sha1',_ioc)
# retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
# elif len(_ioc) == 64:
# static_input = ('static','sha256',_ioc)
# retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
elif _str['pattern'] == 'email':
static_input = ('static','email',_str['strings'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
else:
if _str['strings']:
static_input = ('static','string',_str['strings'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
# store digital signature information of the malware if available
if 'peinfo' in static:
# if 'internal_name' in static['peinfo']['file_info']:
# static_input = ('static','filename/internal_name',static['peinfo']['file_info']['internal_name'])
# retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
if 'original_filename' in static['peinfo']['file_info'] and static['peinfo']['file_info']['original_filename']:
static_input = ('static','filename/original_filename',static['peinfo']['file_info']['original_filename'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
if static['peinfo']['signcheck']['verified'] != 'Unsigned':
for s in static['peinfo']['signcheck']['signers_details']:
if '00:00:00:00:00' in s['serial_number']:
continue
static_input = ('static','signcheck/digital sign serial number',s['serial_number'])
retval = self.addExtractedInfo(static_input, add_result_to_system, retval, misp, misp_event)
if type(retval) and len(retval) > 0:
LibIoC_DK.debugging("The malware static info returned", main._DEBUG_, main._LOGGING_, main.hFile)
return retval
else:
LibIoC_DK.debugging("The malware static info NOT returned", main._DEBUG_, main._LOGGING_, main.hFile)
return False
def isFileofInterest(self, target):
return LibIoC_DK.isFileofInterest(target)
def getMalwareBehaviorInfo(self, _hash, misp_event=False, misp=False):
add_result_to_system = False
if misp_event and misp:
add_result_to_system = True
retval = []
_params = {
'api_key': self.key,
'hash': _hash,
'env_type': 2
}
response = None
behavior = None
while True:
try:
LibIoC_DK.debugging("Getting the malware behavior info", main._DEBUG_, main._LOGGING_, main.hFile)
response = requests.get('https://private.api.malwares.com/v3/file/behaviorinfo', params=_params)
self.query_counter = self.query_counter + 1
except:
LibIoC_DK.debugging("Exception occurred during the malware behavior info request", main._DEBUG_, main._LOGGING_, main.hFile)
return False
try:
behavior = response.json()
except:
LibIoC_DK.debugging("Exception occurred during decoding response", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if behavior['result_code'] == 1: # Data exists
break
elif behavior['result_code'] == 2: # Analyzing
LibIoC_DK.debugging("Analyzing the malware behavior info", main._DEBUG_, main._LOGGING_, main.hFile)
return False
elif behavior['result_code'] == -14: # Over the daily request limit
LibIoC_DK.debugging("Over the daily request limit", main._DEBUG_, main._LOGGING_, main.hFile)
LibIoC_DK.debugging("Accumulated the number of query: %d" %(self.query_counter), main._DEBUG_, main._LOGGING_, main.hFile)
self.waitToNextMidnight_UTC()
self.query_counter = 0
elif behavior['result_code'] <= 0:
LibIoC_DK.debugging("Error returned while getting the malware behavior info", main._DEBUG_, main._LOGGING_, main.hFile)
return False
# add all different malware hash representations
if 'md5' in behavior:
behavior_input = ('behavior','md5',behavior['md5'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
if 'sha1' in behavior:
behavior_input = ('behavior','sha1',behavior['sha1'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
if 'sha256' in behavior:
behavior_input = ('behavior','sha256',behavior['sha256'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
# add behavior analysis results
if 'behavior' in behavior:
for bb in behavior['behavior']:
if 'file_write_event' in bb and bb['file_write_event'] is not None:
for _filename in bb['file_write_event']:
if LibIoC_DK.isFileofInterest(_filename['filename']):
behavior_input = ('behavior','filename/file_write_event',_filename['filename'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
if 'service_create_event' in bb and bb['service_create_event'] is not None:
for service in bb['service_create_event']:
if service['displayname']:
behavior_input = ('behavior','string/service display name',service['displayname'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
if 'process_create_event' in bb and bb['process_create_event'] is not None:
for process in bb['process_create_event']:
if 'sha256' in process and process['sha256'].lower() != behavior['sha256'].lower():
if self.isNormalProcess(self.normal_process_list_root, process['sha256'].lower()):
continue
ai_score = self.getAIScore(process['sha256'])
if ai_score >= 70:
behavior_input = ('behavior','sha256',process['sha256'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
else:
LibIoC_DK.debugging("Normal process", main._DEBUG_, main._LOGGING_, main.hFile)
self.addNormalProcessList(self.normal_process_list_root, process['newprocessname'], process['sha256'].lower(), str(ai_score))
self.saveNormalProcessList(self.normal_process_list_root)
# add network analysis results
if 'network' in behavior and behavior['network'] is not None:
for flow in behavior['network']['flow']:
flow_type = flow['protocol']
if flow['application']:
flow_type += '_'+flow['application']
if flow['application'] == 'DNS' and 'detail_info' in flow:
for detail in flow['detail_info']:
if detail['dns_msg_type'] == 'QUERY':
behavior_input = ('behavior','host/'+flow_type,detail['query_name'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
elif detail['dns_msg_type'] == 'ANSWER_IPV4':
behavior_input = ('behavior','dest_ip/'+flow_type,detail['ipv4_address'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
elif 'protocol' in flow and (flow['protocol'] == 'TCP' or flow['protocol']=='UDP' or flow['protocol']=='IGMP' or flow['protocol']=='IGMP'):
if flow['dest_ip'] != '<MWS_GUEST_IP>': # malwares.com virtual machine IP
behavior_input = ('behavior','dest_ip/'+flow_type,flow['dest_ip'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
elif flow['src_ip'] != '<MWS_GUEST_IP>': # malwares.com virtual machine IP
behavior_input = ('behavior','src_ip/'+flow_type,flow['src_ip'])
retval = self.addExtractedInfo(behavior_input, add_result_to_system, retval, misp, misp_event)
if type(retval)==list and len(retval) > 0:
LibIoC_DK.debugging("The malware behavior info returned", main._DEBUG_, main._LOGGING_, main.hFile)
return retval
else:
LibIoC_DK.debugging("The malware behavior info NOT returned", main._DEBUG_, main._LOGGING_, main.hFile)
False
def addAttribute(connector, event, attr, filepath):
# if attr[0], the attribute type, is written in large character, the attr comes from the ioc parser,
# otherwise from the malware repository.
attribute_info = attr[1].split('/')
attribute_type = attribute_info[0].lower()
if len(attribute_info) == 2:
attribute_comment = attribute_info[1]
else:
attribute_comment = ''
attr_lower_case = attr[2].lower().replace('[.]', '.')
attribute_added = False
event_name, source_name = connector.getEventFileName(event=event)
if not event_name:
return False
try:
event_name = event_name.encode('utf-8')
source_name = source_name.encode('utf-8')
except:
return False
# if the attribute is already stored in the MISP event, skip it
tmp_id = connector.checkAttribute(attr=attr[2])
sharedIoC = False
if tmp_id:
for curr_id in tmp_id:
if str(event['Event']['id']) == curr_id:
return False
else:
if LibIoC_DK.isHash(event_name):
for a in event['Event']['Attribute']:
if a['category'] == 'Other' and a[
'value'] == connector.misp_connection.get_event(
curr_id)['Event']['info']:
if main._STAT_:
connector.ioc_stat.addCategory3(
attr_lower_case,
connector.ioc_stat.convertIoCType(
attribute_type)) # category 3
sharedIoC = True
elif LibIoC_DK.isHash(
connector.misp_connection.get_event(curr_id)['Event']
['info']):
e = connector.misp_connection.get_event(curr_id)
for a in e['Event']['Attribute']:
if a['category'] == 'Other' and a['value'] == event[
'Event']['info']:
if main._STAT_:
connector.ioc_stat.addCategory3(
attr_lower_case,
connector.ioc_stat.convertIoCType(
attribute_type)) # category 3
sharedIoC = True
if sharedIoC:
break
if attr[0] == 'report' or attr[0] == 'static':
attribute_category = 'External analysis'
elif attr[0] == 'behavior':
attribute_category = 'Artifacts dropped'
if main._MODE_ == 1 or main._MODE_ == 3:
if attribute_type == 'host': # or attribute_type == 'url':
connector.misp_connection.add_url(event,
attr[2],
comment=attribute_comment)
attribute_added = True
elif attribute_type == 'ip' or attribute_type == 'src_ip':
connector.misp_connection.add_ipsrc(event,
attr[2],
comment=attribute_comment)
attribute_added = True
elif attribute_type == 'dest_ip':
connector.misp_connection.add_ipdst(event,
attr[2],
comment=attribute_comment)
attribute_added = True
elif attribute_type == 'email':
connector.misp_connection.add_email_src(event,
attr[2],
comment=attribute_comment)
attribute_added = True
elif attribute_type == 'cve':
connector.misp_connection.add_named_attribute(
event,
category=attribute_category,
type_value='vulnerability',
value=attr[2],
comment=attribute_comment)
attribute_added = True
# elif attribute_type == 'registry':
# connector.misp_connection.add_named_attribute(event, category=attribute_category, type_value='regkey', value=attr[2], comment=attribute_comment)
# attribute_added = True
elif attribute_type == 'filename':
if LibIoC_DK.isFileofInterest(attr[2]):
connector.misp_connection.add_named_attribute(
event,
category=attribute_category,
type_value='filename',
value=attr[2],
comment=attribute_comment)
attribute_added = True
# elif attr[0] == 'Filepath':
# connector.misp_connection.add_named_attribute(event, category='External analysis', type_value='text', value=attr[1], comment='file path')
# attribute_added = True
elif attribute_type == 'pdb':
connector.misp_connection.add_named_attribute(
event,
category='Artifacts dropped',
type_value='pdb',
value=attr[2],
comment=attribute_comment)
attribute_added = True
elif attribute_type == 'signcheck':
connector.misp_connection.add_named_attribute(
event,
category=attribute_category,
type_value='text',
value=attr[2],
comment=attribute_comment)
attribute_added = True
# If a malware hash is found, check if it is already stored in MISP as an attribute.
# If MISP has already stored it, just add it as an attribute of the current event.
# Otherwise, create new MISP event if the analysis result could be found from malware repository.
# If no the analysis result found from the repository, just add it as an attribute of the current event.
elif attribute_type == 'md5' or attribute_type == 'sha1' or attribute_type == 'sha256':
if connector.checkAttribute(attr_lower_case):
if attribute_type == 'md5':
connector.misp_connection.add_hashes(
event,
category=attribute_category,
md5=attr_lower_case,
comment=attribute_comment)
elif attribute_type == 'sha1':
connector.misp_connection.add_hashes(
event,
category=attribute_category,
sha1=attr_lower_case,
comment=attribute_comment)
elif attribute_type == 'sha256':
connector.misp_connection.add_hashes(
event,
attribute_category,
sha256=attr_lower_case,
comment=attribute_comment)
attribute_added = True
elif LibIoC_DK.isHash(
event['Event']
['info']) and connector.createMISPEventFromHash(
attr_lower_case, filepath) and main._STAT_:
connector.ioc_stat.increaseAnalyzedAdditionalHash()
elif attribute_type == 'string':
connector.misp_connection.add_named_attribute(
event,
category=attribute_category,
type_value='text',
value=attr[2],
comment=attribute_comment)
attribute_added = True
# IoC statistics calculation.
if main._STAT_:
if not sharedIoC: # and attribute_added:
if LibIoC_DK.isHash(event_name):
if attribute_added:
if connector.ioc_stat.checkIoCInReport(
attr[2]) or connector.ioc_stat.checkIoCInReport(
attr_lower_case):
connector.ioc_stat.addCategory2(
connector.ioc_stat.convertIoCType(
attribute_type)) # category 2
else:
connector.ioc_stat.addCategory4(
connector.ioc_stat.convertIoCType(
attribute_type)) # category 4
else:
if attribute_added:
connector.ioc_stat.addCategory1(
connector.ioc_stat.convertIoCType(
attribute_type)) # category 1
else:
connector.ioc_stat.addcategory5(
connector.ioc_stat.convertIoCType(
attribute_type)) # category 5
return attribute_added
def getAIScore(self, _hash):
_params = {
'api_key': self.key,
'hash': _hash}
response = None
ai_score = None
while True:
try:
LibIoC_DK.debugging("Getting the malware MAX score", main._DEBUG_, main._LOGGING_, main.hFile)
response = requests.get('https://private.api.malwares.com/v3/file/mwsinfo', params=_params)
self.query_counter = self.query_counter + 1
except:
LibIoC_DK.debugging("Exception occurred during the malware MAX score request", main._DEBUG_, main._LOGGING_, main.hFile)
return False
try:
ai_score = response.json()
except:
LibIoC_DK.debugging("Exception occurred during decoding response", main._DEBUG_, main._LOGGING_, main.hFile)
return False
if ai_score['result_code'] == 1: # Data exists
break
elif ai_score['result_code'] == 2: # Analyzing
LibIoC_DK.debugging("Analyzing the malware MAX score", main._DEBUG_, main._LOGGING_, main.hFile)
return False
elif ai_score['result_code'] == -14: # Over the daily request limit
LibIoC_DK.debugging("Over the daily request limit", main._DEBUG_, main._LOGGING_, main.hFile)
LibIoC_DK.debugging("Accumulated the number of query: %d" %(self.query_counter), main._DEBUG_, main._LOGGING_, main.hFile)
self.waitToNextMidnight_UTC()
self.query_counter = 0
elif ai_score['result_code'] <= 0:
LibIoC_DK.debugging("Error returned while getting the malware MAX score", main._DEBUG_, main._LOGGING_, main.hFile)
return False
return ai_score['ai_score']
def addFileProcessingResult(root, fName, result):
ET.SubElement(root, 'file', {'title':fName.decode('utf-8'),'result':result})
return
def saveFileProcessingResult(root):
tree = ET.ElementTree(root)
tree.write(fileProcessingResult_file)
return
'''
The main routine
'''
if __name__ == '__main__':
if _DEBUG_:
LibIoC_DK.debugging("**********************************************************", _DEBUG_, _LOGGING_, hFile)
LibIoC_DK.debugging("******System Execution at %s******"%(str(datetime.now())), _DEBUG_, _LOGGING_, hFile)
LibIoC_DK.debugging("**********************************************************", _DEBUG_, _LOGGING_, hFile)
# Load configuration file
config_value = getConfig(config_file)
file_names = getFileName(config_value['ReportRoot'])
misp = MISPConnector.MISPConnector(config_value, ioc_stat)
if _PARALLELIZE_FILE_PROCESSING_:
joblib.Parallel(joblib.cpu_count())(delayed(processFile)(misp, f, config_value) for f in file_names)
else:
processFileList(misp, file_names, config_value)
if _DEBUG_:
LibIoC_DK.debugging("All file processing FINISHED!!!", _DEBUG_, _LOGGING_, hFile)