def domainlist_indicator(domains=[]):
domainlist = Indicator()
domainlist.add_indicator_type("Domain Watchlist")
for d in domains:
domain = DomainName()
domain.value = d
domainlist.add_observable(domain)
return domainlist
def hostchars_indicator(hostnames=[]):
hostchars = Indicator()
hostchars.add_indicator_type("Host Characteristics")
for h in hostnames:
hostname = Hostname()
hostname.hostname_value = h
hostchars.add_observable(hostname)
return hostchars
def iplist_indicator(ips=[]):
iplist = Indicator()
iplist.add_indicator_type("IP Watchlist")
for i in ips:
address = Address()
address.address_value = i
#address.category="ipv4-addr"
iplist.add_observable(address)
return iplist
def hash_indicator(hashes=[]):
hashlist = Indicator()
hashlist.add_indicator_type("File Hash Watchlist")
for h in hashes:
file = File()
hash = Hash(
hash_value = h,
#type_ = "MD5",
)
file.add_hash(hash)
hashlist.add_observable(file)
return hashlist
def create_ip_indicator(self, ip_indicator):
indicator = Indicator()
indicator.title = 'IP address of site hosting malware'
indicator.add_indicator_type('IP Watchlist')
addr = Address(address_value=ip_indicator, category=Address.CAT_IPV4)
addr.condition = 'Equals'
indicator.add_observable(addr)
return indicator
def create_host_indicator(self, host_indicator):
indicator = Indicator()
indicator.title = 'Hostname of site hosting malware'
indicator.add_indicator_type('Domain Watchlist')
host = Hostname()
host.value = host_indicator
host.condition = 'Equals'
indicator.add_observable(host)
return indicator
def create_url_indicator(self, url_indicator):
indicator = Indicator()
indicator.title = 'URL of site hosting malware'
indicator.add_indicator_type('URL Watchlist')
url = URI()
url.value = url_indicator
url.type_ = URI.TYPE_URL
url.condition = 'Equals'
indicator.add_observable(url)
return indicator
def create_ip_indicator(self, ip_indicator):
indicator = Indicator()
indicator.title = 'IP address of site hosting malware'
indicator.add_indicator_type('IP Watchlist')
addr = Address(address_value=ip_indicator, category=Address.CAT_IPV4)
addr.condition = 'Equals'
indicator.add_observable(addr)
return indicator
def get_indicator_from_object(object_, title, user_timezone):
# Observableを作成する
observable = Observable()
observable.object_ = object_
# observable,description,titleを設定する
indicator = Indicator()
indicator.timestamp = datetime.datetime.now(tz=user_timezone)
indicator.title = title
indicator.description = title
indicator.observable = observable
return indicator
def create_host_indicator(self, host_indicator):
indicator = Indicator()
indicator.title = 'Hostname of site hosting malware'
indicator.add_indicator_type('Domain Watchlist')
host = Hostname()
host.value = host_indicator
host.condition = 'Equals'
indicator.add_observable(host)
return indicator
def create_url_indicator(self, url_indicator):
indicator = Indicator()
indicator.title = 'URL of site hosting malware'
indicator.add_indicator_type('URL Watchlist')
url = URI()
url.value = url_indicator
url.type_ = URI.TYPE_URL
url.condition = 'Equals'
indicator.add_observable(url)
return indicator
def _export_from_json_to_xml(json1):
import re
from stix.indicator.indicator import Indicator
from cybox.core import observable
from cybox.common import Hash
from cybox.objects.file_object import File
indicatorIns = Indicator()
indicDesc = "Additional fields: "
if testMode:
print "------------New Indicator: Start------------"
print "--------------check dictionary ----------------"
if json1["relatedid_restriction"]: print json1["relatedid_restriction"]
if json1["source"]: print json1["source"]
if "ardig" in json1 and json1["ardig"]: print str(json1["ardig"])
if "address" in json1 and json1["address"]:print json1["address"]
print "--------------check dictionary:END----------------"
#need to be set before setting times producer attributes
indicatorIns.set_producer_identity("None")
strAddress = ""
strAsn = ""
strAsnDesc = ""
strRir = ""
strCc = ""
strPrefix = ""
strRdata = ""
strDescShort = ""
strMalware = ""
strProtocol = ""
strPortList = ""
for item in json1.keys():
if fieldsList.get(item) is None:
log_string = log_string + "New field: " + str(json1.get(item)) + "\n"
if json1.get("whois") is not None:
indicDesc = indicDesc + "whois = " + str(json1["whois"]) + "; "
if json1.get("relatedid_restriction") is not None:
indicDesc = indicDesc + "relatedid_restriction = " + str(json1["relatedid_restriction"]) + "; "
if json1.get("source") is not None:
indicatorIns.set_producer_identity(json1["source"])
if json1.get("contact") is not None:
indicDesc = indicDesc + "contact = " + json1["contact"] + "; "
if json1.get("purpose") is not None:
indicDesc = indicDesc + "purpose = " + str(json1["purpose"]) + "; "
if json1.get("asn") is not None:
strAsn = json1["asn"]
if json1.get("asn_desc") is not None:
strAsnDesc = json1["asn_desc"]
if json1.get("rir") is not None:
strRir = json1["rir"]
if json1.get("cc") is not None:
strCc = json1["cc"]
if json1.get("rdata") is not None:
strRdata = json1["rdata"]
if json1.get("prefix") is not None:
strPrefix = json1["prefix"]
if json1.get("alternativeid") is not None:
indicDesc = indicDesc + "alternativeid = " + str(json1["alternativeid"]) + "; "
if json1.get("detecttime") is not None:
indicatorIns.set_produced_time(json1["detecttime"])
if json1.get("address") is not None:
strAddress = json1["address"]
if json1.get("alternativeid_restriction") is not None:
indicDesc = indicDesc + "alternativeid_restriction = " + str(json1["alternativeid_restriction"]) + "; "
if json1.get("id") is not None:
indicatorIns.id_=json1["id"]
if json1.get("guid") is not None:
indicDesc = indicDesc + "guid = " + str(json1["guid"]) + "; "
if json1.get("severity") is not None:
indicDesc = indicDesc + "severity = " + str(json1["severity"]) + "; "
if json1.get("assessment") is not None:
indicDesc = "Assessment: " + str(json1["assessment"]) + ". " + indicDesc
if json1.get("description") is not None:
descList = str(json1["description"]).rsplit()
if len(descList) > 1:
strDescShort = descList[len(descList)-1]
else:
strDescShort = str(json1["description"])
indicDesc = indicDesc + "description = " + str(json1["description"]) + "; "
if json1.get("relatedid") is not None:
indicDesc = indicDesc + "relatedid = " + str(json1["relatedid"]) + "; "
if json1.get("reporttime") is not None:
indicatorIns.set_received_time(json1["reporttime"])
if json1.get("confidence") is not None:
indicDesc = indicDesc + "confidence = " + json1["confidence"] + "; "
if json1.get("restriction") is not None:
indicDesc = indicDesc + "restriction = " + json1["restriction"] + "; "
if json1.get("malware_hash") is not None:
strMalware =json1["malware_hash"]
if json1.get("protocol") is not None:
strProtocol = str(json1["protocol"])
if json1.get("portlist") is not None:
strPortList = str(json1["portlist"])
#Address
#build address param
addressParam = {'strAddress':strAddress,'strDescShort':strDescShort,'strAsn':strAsn,'strAsnDesc':strAsnDesc,
'strRir':strRir,'strCc':strCc,'strPrefix':strPrefix,'strRdata':strRdata,
'strProtocol':strProtocol,'strPortList':strPortList,'indicDesc':indicDesc,}
if testMode:
print "Address: " + addressParam["strAddress"]
print "strDescShort: " + addressParam["strDescShort"]
if strAddress:
#address
indicDesc = _build_adderss_obj(addressParam,indicatorIns)
#rdata onlly
if strAddress is None and strRdata:
matchRez = re.match("""((25[0-5]|2[0-4]\d|1\d\d|[1-9]\d|\d)\.){3}
(25[0-5]|2[0-4]\d|1\d\d|[1-9]\d|\d)""", strRdata)
if matchRez:
#address = helper.create_ipv4_observable(strRdata)
indicatorIns.add_observable(observable.Observable(Address(strRdata,'ipv4-addr')))
else:
indicatorIns.add_observable(observable.Observable(URI(strRdata,'Domain Name')))
if testMode:
print "It's rdata only"
#malware
if strMalware:
malFile = File()
hash_ = Hash(strMalware)
malFile.add_hash(hash_)
malware = observable.Observable(malFile)
indicatorIns.add_observable(malware)
if testMode:
print "It's malware_hash"
if indicDesc:
indicatorIns.description = indicDesc
return indicatorIns
def transform(addsec_data):
#
# Parse the Addition Security protobuf object, which contains a STIX report representation
#
as_report = addsec_cti_pb2.Report()
as_report.ParseFromString(addsec_data)
#
# Create a new STIX package & report container
#
stix_package = STIXPackage()
stix_package.stix_header = STIXHeader()
stix_package.stix_header.description = "Addition Security Report"
stix_report = Report()
#
# Addition Security includes various identification information re: the entity of the report.
# We are going to convert it into three CybOX objects: Product, Device, and Custom
#
cybox_product = Product()
cybox_product.product = "MobileAwareness"
cybox_product.vendor = "Addition Security"
cybox_device = Device()
cybox_device.device_type = "Mobile Device"
cybox_custom_sourceapp = Custom()
cybox_custom_sourceapp.custom_name = "addsec:sourceApplication"
cybox_custom_sourceapp.custom_properties = CustomProperties()
p = Property()
p.name = "organizationId"
p.value = as_report.organizationId.encode(
'hex') # NOTE: this is binary bytes
cybox_custom_sourceapp.custom_properties.append(p)
p = Property()
p.name = "application"
p.value = as_report.applicationId # NOTE: bundleId/packageId of hosting app
cybox_custom_sourceapp.custom_properties.append(p)
p = Property()
p.name = "instanceId"
p.value = as_report.systemId.encode('hex') # NOTE: this is binary bytes
cybox_custom_sourceapp.custom_properties.append(p)
stix_report.add_observable(cybox_product)
stix_report.add_observable(cybox_device)
stix_report.add_observable(cybox_custom_sourceapp)
#
# Enumerate the Addition Security reported sightings
#
for as_sighting in as_report.observations:
#
# Addition Security lets customers transit custom messages over the reporting channel; these
# messages show up as a "Customer Message" indicator with string-based payload. Since these
# messages are both proprietary in nature and potentially unrelated to STIX, we are going to
# filter them out from this processing.
#
if as_sighting.observationType == 8: continue # 8: CustomerData
#
# Sightings are used to report device information as well; let's expel device-related
# sightings and re-route their data into the CybOX device object (instead of including
# as an indicator w/ sighting)
#
if as_sighting.testId == 1 or as_sighting.testId == 2: #
addsec_to_cybox_device(cybox_device, as_sighting)
continue
# Ditto for reported product information as well
if as_sighting.testId == 8: # 8: SDKVersionInfo
addsec_to_cybox_product(cybox_product, as_sighting)
continue
#
# Compose a STIX-appropriate indicator value from the Addition Security indicator ID & SubID
#
indicator_id = "addsec:asma-%d-%d" % (as_sighting.testId,
as_sighting.testSubId)
stix_indicator = Indicator(id_=indicator_id)
stix_indicator.title = addsec_title_lookup(as_sighting.testId,
as_sighting.testSubId)
#
# Create a sighting for this indicator
#
stix_sighting = Sighting()
stix_indicator.sightings = stix_sighting
stix_sighting.timestamp = datetime.datetime.fromtimestamp(
as_sighting.timestamp)
if as_sighting.confidence > 0:
stix_sighting.confidence = addsec_to_stix_confidence(
as_sighting.confidence)
#
# Enumerate the observables for this sighting
#
for as_observable in as_sighting.datas:
cybox_obj = addsec_to_cybox(as_observable.dataType,
as_observable.data)
if not cybox_obj is None:
stix_sighting.related_observables.append(
RelatedObservable(Observable(cybox_obj)))
#
# Finally, add this indicator (w/ sightings & related observables) to the top level report
#
stix_report.add_indicator(stix_indicator)
#
# Finalize the STIX report and output the XML
#
stix_package.reports = stix_report
return stix_package.to_xml()
doc = xmltodict.parse(res.read())
# Create the STIX Package
package = STIXPackage()
# Create the STIX Header and add a description.
header = STIXHeader()
#header.title = "SANS ISC Top-100 Malicious IP Addresses"
#header.description = "Source: " + url
package.stix_header = header
for entry in doc['topips']['ipaddress']:
bytes = entry['source'].split('.')
indicator = Indicator()
indicator.title = "SANS ISC Malicious IP"
indicator.add_indicator_type("IP Watchlist")
ip = Address()
ip.address_value = "%d.%d.%d.%d" % (int(bytes[0]), int(bytes[1]), int(bytes[2]) , int(bytes[3]))
ip.category = 'ipv4-addr'
ip.condition = 'Equals'
indicator.add_observable(ip)
package.add_indicator(indicator)
print(package.to_xml())
if __name__ == '__main__':
main()
res = urllib2.urlopen(req)
except urllib2.HTTPError, e:
print "HTTPError: " + str(e.code)
except urllib2.URLError, e:
print "URLError: " + str(e.reason)
doc = xmltodict.parse(res.read())
# Create the STIX Package
package = STIXPackage()
# Create the STIX Header and add a description.
header = STIXHeader()
#header.title = "SANS ISC Top-100 Malicious IP Addresses"
#header.description = "Source: " + url
package.stix_header = header
for entry in doc['topips']['ipaddress']:
bytes = entry['source'].split('.')
indicator = Indicator()
indicator.title = "SANS ISC Malicious IP"
indicator.add_indicator_type("IP Watchlist")
ip = Address()
ip.address_value = "%d.%d.%d.%d" % (int(bytes[0]), int(
bytes[1]), int(bytes[2]), int(bytes[3]))
ip.category = 'ipv4-addr'
ip.condition = 'Equals'
indicator.add_observable(ip)
package.add_indicator(indicator)
print(package.to_xml())
if __name__ == '__main__':
main()
def generate_stix_file(input_file, list_type, delimiter, list_name, tc_name, tmp_dir, validate, verbose):
# observable limit per generated stix file
OBSERVABLES_PER_STIX_FILE = 3000
if verbose:
logging.info("=====================")
logging.info("== GENERATING STIX ==")
logging.info("=====================")
# download or open input file
if validators.url(input_file):
res = requests.get(input_file)
items = res.text.split(delimiter)
else:
# exit if input file doesn't exist
if not os.path.isfile(input_file):
logging.error("Supplied input file '{}' doesn't exist".format(input_file))
sys.exit("Error: Supplied input file '{}' doesn't exist".format(input_file))
else:
with open(input_file, 'r') as f:
items = f.read().split(delimiter)
logging.info("Successfully parsed input file at {}".format(input_file))
# slice input into batches
for batch_num, index in enumerate(range(0, len(items), OBSERVABLES_PER_STIX_FILE), 1):
# slice handles out of bounds indices gracefully
batch_items = items[index:index + OBSERVABLES_PER_STIX_FILE]
# create the STIX Package
package = STIXPackage()
# create the STIX Header and add a description
header = STIXHeader()
package.stix_header = header
reporttime = datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M:%S %Z')
# create indicator for each item in the batch
for item in batch_items:
item = item.strip()
# basic filtering of empty items and comments
if not item or item.startswith(('#', '//', '--')):
continue
if list_type == 'ip':
indicator_obj = Address()
# attempt to parse as an ip address
try:
parsed_ip = ipaddress.ip_address(item)
if parsed_ip.version == 4:
indicator_obj.category = Address.CAT_IPV4
elif parsed_ip.version == 6:
indicator_obj.category = Address.CAT_IPV6
else:
logging.warning("Unknown IP Address version type: {} - skipping".format(parsed_ip.version))
continue
except ValueError:
# if ip address parsing fails then attempt to parse as an ip network
try:
parsed_ip = ipaddress.ip_network(item, strict=False)
indicator_obj.category = Address.CAT_CIDR
except ValueError:
logging.warning("IP Address {} is neither an IPv4, IPv6, nor CIDR - skipping".format(item))
continue
indicator_obj.address_value = str(parsed_ip)
indicator_obj.condition = "Equals"
indicator_type = "IP Watchlist"
# customizable components below
indicator_title = "IP: {}"
indicator_description = "IP {} reported from {}"
elif list_type == 'domain':
# validate domain
if validate and not validators.domain(item):
logging.warning("Invalid domain: {} - skipping".format(item))
continue
indicator_obj = DomainName()
indicator_obj.value = item
indicator_type = "Domain Watchlist"
# customizable components below
indicator_title = "Domain: {}"
indicator_description = "Domain {} reported from {}"
elif list_type == 'url':
# validate url
if validate and not validators.url(item):
logging.warning("Invalid url: {} - skipping".format(item))
continue
indicator_obj = URI()
indicator_obj.value = item
indicator_obj.type_ = URI.TYPE_URL
indicator_obj.condition = "Equals"
indicator_type = "URL Watchlist"
# customizable components below
indicator_title = "URL: {}"
indicator_description = "URL {} reported from {}"
else:
# invalid input type
logging.error("invalid input type encountered")
raise Exception('Error: invalid input type encountered')
# create a basic Indicator object from the item
indicator = Indicator()
indicator.title = indicator_title.format(str(item))
indicator.description = indicator_description.format(str(item), list_name)
indicator.add_indicator_type(indicator_type)
indicator.set_producer_identity(list_name)
indicator.set_produced_time(str(reporttime))
indicator.add_observable(indicator_obj)
# add the indicator to the stix package
package.add_indicator(indicator)
# save each batch in a separate stix file with the filename ending ..._part_N.stix
collection_filename = "{}_part_{}.stix".format(strip_non_alphanum(tc_name), batch_num)
with open(os.path.join(tmp_dir, collection_filename), 'wb') as f:
f.write(package.to_xml())
logging.info("Successfully created stix file {}".format(collection_filename))
# clear cybox cache to prevent an Out of Memory error
# https://cybox.readthedocs.io/en/stable/api/cybox/core/object.html#cybox.core.object.Object
cache_clear()
return
