def create_misp_tags(misp_api):
"""
Check if all tags exist in the MISP instance and if one is not, create it.
:param misp_api: MISP Object API.
:return: Tags created.
:rtype: list
"""
print(str(timezone.now()) + " - " + 'Generate MISP Tags')
print('-----------------------------')
required_tags = [
'Watcher', 'Impersonation', 'Malicious Domain', 'Typosquatting',
'TLP:Amber'
]
tag_list = list()
tags = misp_api.tags(pythonify=True)
tags_names = list()
for tag in tags:
tags_names.append(tag.name)
for tag in required_tags:
t = MISPTag()
t.name = tag
t.org_id = 1
if tag not in tags_names:
print(str(timezone.now()) + " - " + "Create tag: ", tag)
misp_api.add_tag(t)
tag_list.append(t)
return tag_list
def create_misp_tags(misp_api):
"""
Check if all tags exist in the MISP instance and if one is not, create it.
:param misp_api: MISP Object API.
:return: Tags created.
:rtype: list
"""
print(str(timezone.now()) + " - " + 'Generate MISP Tags')
print('-----------------------------')
required_tags = settings.MISP_TAGS
tag_list = list()
tags = misp_api.tags(pythonify=True)
tags_names = list()
for tag in tags:
tags_names.append(tag.name)
for tag in required_tags:
t = MISPTag()
t.name = tag
t.org_id = 1
if tag not in tags_names:
print(str(timezone.now()) + " - " + "Create tag: ", tag)
misp_api.add_tag(t)
tag_list.append(t)
return tag_list
def _check_tag(self, target_tag):
if self.misp == None:
self._connect()
if self._registered_tags == None:
self._get_tags()
for tag_info in self._registered_tags:
if tag_info.get('name', '') == target_tag:
return True
self.debug_print('new tag: {}'.format(target_tag))
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
tmp = MISPTag()
tmp.from_dict(name=target_tag)
self.misp.add_tag(tmp)
self._get_tags()
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('add new tag retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return False
def create_tag(self, name: str, exportable: bool, reserved: bool):
tag = MISPTag()
tag.name = name
tag.exportable = exportable
if reserved:
tag.org_id = self.host_org.id
self.create_or_update_tag(tag)
def add_tag_filter_sync(self, server_sync: MISPServer, name: str):
# Add tag to limit push
tag = MISPTag()
tag.name = name
tag.exportable = False
tag.org_id = self.host_org.id
tag = self.site_admin_connector.add_tag(tag)
if not isinstance(tag, MISPTag):
for t in self.site_admin_connector.tags():
if t.name == name:
tag = t
break
else:
raise Exception('Unable to find tag')
# Set limit on sync config
filter_tag_push = {
"tags": {
'OR': [tag.id],
'NOT': []
},
'orgs': {
'OR': [],
'NOT': []
}
}
# filter_tag_pull = {"tags": {'OR': [], 'NOT': []}, 'orgs': {'OR': [], 'NOT': []}}
server_sync.push_rules = json.dumps(filter_tag_push)
# server.pull_rules = json.dumps(filter_tag_pull)
server_sync = self.site_admin_connector.update_server(server_sync)
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {"name": tag_name}
if hex_color:
tag_properties["colour"] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def generate_misp_tags(self) -> List[MISPTag]:
tagList = []
tags = [
{
'name': 'AutoGenerated',
'colour': '#00ace6',
'exportable': True
},
{
'name': 'HoneytrapEvent',
'colour': '#581845',
'exportable': True
},
{
'name': 'ModSecurity',
'colour': '#a04000',
'exportable': True
},
]
try:
for item in tags:
tag = MISPTag()
for key in item:
tag[key] = item[key]
self.misp.add_tag(tag, pythonify=True)
tagList.append(tag)
log.info("MISP tags generated")
return tagList
except PyMISPError as e:
raise RuntimeError("Failed to initialise MISP tags")
def get_set_all_tags(iUUID):
ret_list = []
session = Session()
try:
# query = "SELECT * FROM mp_tags WHERE uuid = ?"
tmp_dict = session.query(md.Tag). \
filter(md.Tag.uuid == iUUID). \
all()
tag_dict = md.TagSchema(many=True).dump(tmp_dict)
# BUILD RETURN LIST
for tag in tag_dict:
tmpGalaxy = tag["galaxy"]
tmpTagVal = tag["tag"]
tmpTypeTag = tag["type"]
newTag = MISPTag()
# POTENTIAL tmpTypeTag Values
# "ACTOR"
# "COUNTRY_SPONSOR"
# "GALAXY"
# "GALAXY_SYNONIM"
# "ISO_COUNTRY"
# "MALWARE"
# "TARGETS"
# "TYPE_OF_INCIDENT"
# "VICTIMS"
if "tlp:" in tmpTagVal:
newTag.name = tmpTagVal
elif tmpTypeTag == "ACTOR":
newTag.name = "Actor: " + tmpTagVal
newTag.colour = gv._ACTOR_TAG
elif tmpTypeTag == "COUNTRY_SPONSOR":
newTag.name = "Sponsor: " + tmpTagVal
newTag.colour = gv._COUNTRY_SPONSOR_TAG
elif tmpTypeTag == "GALAXY":
if tmpGalaxy != "":
newTag.name = "misp-galaxy:" + tmpGalaxy + "=\"" + tmpTagVal + "\""
else:
newTag.name = "Synonym: " + tmpTagVal
newTag.colour = gv._GALAXY_SYNONIM_TAG
elif tmpTypeTag == "GALAXY_SYNONIM":
newTag.name = "Synonym: " + tmpTagVal
newTag.colour = gv._GALAXY_SYNONIM_TAG
elif tmpTypeTag == "ISO_COUNTRY":
newTag.name = "Country ISO: " + tmpTagVal
newTag.colour = gv._ISO_COUNTRY_TAG
elif tmpTypeTag == "MALWARE":
newTag.name = "Malware: " + tmpTagVal
newTag.colour = gv._MALWARE_TAG
elif tmpTypeTag == "TARGETS":
newTag.name = "Target: " + tmpTagVal
newTag.colour = gv._TARGETS_TAG
elif tmpTypeTag == "TYPE_OF_INCIDENT":
newTag.name = "Type of Incident: " + tmpTagVal
newTag.colour = gv._TYPE_OF_INCIDENT_TAG
elif tmpTypeTag == "VICTIMS":
newTag.name = "Victim: " + tmpTagVal
newTag.colour = gv._VICTIMS_TAG
else:
newTag.name = tmpTagVal
newTag.colour = gv._OTHER_TAG
ret_list.append(newTag)
return ret_list
except Exception as error:
print("f(x) get_set_all_tags: DATABASE ERROR: {}".format(error))
sys.exit(error)
finally:
session.close()
def tag_ker(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:KER'
return tag
def tag_mlo(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:MLO'
return tag
import logging, requests, bs4, re, shelve, os, sys
from pymisp import PyMISP, MISPEvent, MISPTag
from keys import *
from iocparser import IOCParser
logging.basicConfig(level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s")
logging.info("Starting the Pastebin scraper.")
# Setting the directory path:
absolutePath = os.path.abspath(__file__)
path = os.path.dirname(absolutePath)
parsedPaste = []
usernames = []
toScrape = []
tagosint = MISPTag()
tagosint.name = 'paste_osint'
os.chdir(path)
try:
shelfFile = shelve.open('knownpastes')
knownPastes = shelfFile['knownPastes']
shelfFile.close()
logging.debug(
"Imported a grand total of {} previously imported pastes.".format(
len(knownPastes)))
except:
logging.warning("Failed to import known pastes.")
knownPastes = []
f = open('usernames.conf')
def tag_stp(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:STP'
return tag
def tag_bcs(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:BCS'
return tag
def tag_ptb(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:PTB'
return tag
def tag_alt(self) -> MISPTag:
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:ALT'
return tag
def tag_nzd(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:NZD'
return tag
def createFamily(iUUID, iUpdate=False):
try:
# fUNCTION SETUP
# -----------------------------------------------
myUUID = iUUID
myLinks = []
myTags = []
myMeta = []
myCommonName = ""
# ATTRIBUTES COMMON FIELDS
# -----------------------------------------------
attributeToIDS = 0 # 0 false : 1 true
attributeComment = ""
attribDisableCorrelation = 1 # 0 false : 1 true
# MISP SETUP
# -----------------------------------------------
event = pm.MISPEvent()
event.uuid = myUUID
# GET UUID METADATA FROM PARENT CHILD TABLE
# -----------------------------------------------
iPC_META = db.get_parent_child_data(iUUID=myUUID)
parentuuid = iPC_META["parentuuid"]
event.extends_uuid = parentuuid
# -----------------------------------------------
# REFERENCES/URLS
myLinks = db.get_links(myUUID)
for link in myLinks:
attributeType = "link"
attributeCategory = "Internal reference"
if gv._DEBUG:
print(
"f(x) createFamily: LINK: \nCATEGORY: {} \nTYPE: {} \nVALUE: {} \nTO_IDS: {} \nCOMMENT: {}\nDISABLE CORRELATION: {} \
".format(attributeCategory, attributeType, link["url"],
attributeToIDS, attributeComment,
attribDisableCorrelation))
event.add_attribute(attributeType,
link["url"],
comment=attributeComment,
category=attributeCategory,
to_ids=attributeToIDS,
disable_correlation=attribDisableCorrelation)
# GET TAGS
myTags = db.get_set_all_tags(myUUID)
event.tags = myTags
if gv._DEBUG:
print("f(x) createFamily: TAGS")
print(*myTags, sep="\n")
# GET META FOR ACTOR (USE COMMON NAME AS INCIDENT NAME)
myMeta = db.get_family_meta(iUUID=myUUID)
if gv._DEBUG:
print("f(x) createFamily: META")
print(json.dumps(myMeta, indent=4))
# USED AS INCIDENT NAME
myCommonName = myMeta["commonname"]
event.info = myCommonName
print("f(x) createFamily: MALWARE NAME: {}".format(myCommonName))
# USED AS A TEXT ATTRIBUTE
myDescription = myMeta["description"]
if myDescription != "":
attributeType = "text"
attributeCategory = "Internal reference"
if gv._DEBUG:
print(
"f(x) createFamily: CREATING FAMILY COMMENT: \nCATEGORY: {} \nTYPE: {} \nVALUE: {} \nCOMMENT: {} \nDISABLE CORRELATION: {} \
".format(attributeCategory, attributeType, myDescription,
attributeToIDS, attributeComment,
attribDisableCorrelation))
event.add_attribute(attributeType,
myDescription,
comment=attributeComment,
category=attributeCategory,
to_ids=attributeToIDS,
disable_correlation=attribDisableCorrelation)
# MARK SOURCE OF INFORMATION
attributeType = "link"
attributeCategory = "Internal reference"
attributeComment = "DATA FROM MALPEDIA."
if gv._DEBUG:
print(
"f(x) createFamily: ATTRIBUTION LINK: \nCATEGORY: {} \nTYPE: {} \nVALUE: {} \nTO_IDS: {} \nCOMMENT: {} \nDISABLE CORRELATION: {} \
".format(attributeCategory, attributeType,
gv._MALPEDIA_URL, attributeToIDS,
attributeComment, attribDisableCorrelation))
event.add_attribute(attributeType,
gv._MALPEDIA_URL,
comment=attributeComment,
category=attributeCategory,
to_ids=attributeToIDS,
disable_correlation=attribDisableCorrelation)
# YARA
# ADD OBJECTS
# -----------------------------------------------
# YARA
iYara = db.get_yara_rules(myUUID)
tlp = ""
yaraAbsPath = ""
for yara in iYara:
tagList = []
newTag = MISPTag()
tlp = yara["tlp"]
yaraAbsPath = yara["path_to_yara"]
tlpTag = "tlp:" + tlp.split("_")[1]
newTag.name = tlpTag
tagList.append(newTag)
yaraUUID = yara["attribute_uuid"]
yaraContents = ""
with open(yaraAbsPath, 'r') as yaraIn:
yaraContents = yaraIn.read()
yaraIn.close()
misp_object = pm.tools.GenericObjectGenerator("yara")
misp_object.comment = tlpTag
misp_object.uuid = yaraUUID
subAttribute = misp_object.add_attribute("yara", yaraContents)
subAttribute.disable_correlation = True
subAttribute.to_ids = False
subAttribute.comment = tlpTag
subAttribute.tags = tagList
event.add_object(misp_object)
if gv._DEBUG:
print("f(x) createFamily: YARA")
print(*iYara, sep="\n")
gv._THREAD_LIST.append(
executor.submit(pushToMISP, event, iUpdate, gv._MISP_URL,
gv._MISP_KEY, gv._MISP_VERIFYCERT, gv._DEBUG))
# pushToMISP(event, iUpdate, gv._MISP_URL, gv._MISP_KEY, gv._MISP_VERIFYCERT, gv._DEBUG)
except Exception as e:
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("f(x) createFamily: {}: {}: {}".format(exc_type, fname,
exc_tb.tb_lineno))
sys.exit(e)
def misp_add_tag(self, event, a_value):
# Create Tag object in MISP
misp_tag = MISPTag()
if a_value: misp_tag.name = a_value
event.add_tag(misp_tag)
