def transform(data, new_only=True): """ transform - The transforms are source specific. Source: http://www.malwaredomainlist.com/hostslist/mdl.xml data - must be source xml converted to a dictionary :param data: :param new_only: :return: """ # Input validation if not isinstance(data, dict): return False work = [] history = db('local_file', 'history', ADPTR_SRC_ID) value2key = db('local_file', 'value_to_key', 'values') items = data.get('rss', {}).get('channel', {}).get('item') if items: for item in items: guid = item.get('guid', {}).get('#text') if guid: # Check to see if this item has been process before # if not, add to work if guid in history: if not new_only: work.append(item) else: work.append(item) db('local_file', 'history', ADPTR_SRC_ID, {guid: { 'date': str(datetime.now()) }}) if work: ### Generate STIXPackage and STIXHeader set_ns_stix(ADPTR_NS_STIX) set_ns_cybox(ADPTR_NS_CYBOX) STIXPackage._version = ADPTR_VER_STIX pkg = STIXPackage() src_info, value2key = gen_info_src({}, 'www.malwaredomainlist.com', value2key) hdr = STIXHeader() hdr.title = data.get('rss', {}).get('channel', {}).get('title') hdr.description = data.get('rss', {}).get('channel', {}).get('description') hdr.information_source = src_info pkg.stix_header = hdr for item in work: key = item.get('guid', {}).get('#text') # Decompose data description tmp = [x.strip() for x in item.get('description').split(',')] decomp = {} for x in tmp: k, v = x.split(':') decomp.update({k.strip(): v.strip()}) # Generate STIX Indicator ind, history = gen_indicator(item, key, history) ind.producer = src_info ind.short_description = 'MDL RefID: %s | %s' % ( key, decomp.get('Description')) # Decompose host host = decomp.get('Host') uri = None file_ = None if '/' in host: host, uri = host.split('/', 1) # TODO: parse out file Name if host: # Generate Cybox HostName obj = Hostname() obj.is_domain_name = True obj.naming_system = 'DNS' obj.hostname_value = host ob, value2key = gen_CyboxOb(obj, host, value2key) ob.title = 'HostName: %s' % obj.hostname_value ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) if uri: # Generate Cybox URI obj = URI() obj.type_ = URI.TYPE_URL url = AnyURI('%s/%s' % (host, uri)) obj.value = url ob, value2key = gen_CyboxOb(obj, url, value2key) ob.title = 'URL: %s' % url ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) if file_: obj = File() ip = decomp.get('IP address') if ip: obj_ip = Address() if isIPv4(ip): obj_ip.category = Address.CAT_IPV4 elif isIPv6(ip): obj_ip.category = Address.CAT_IPV6 else: break obj_ip.is_source = True obj_ip.address_value = ip # if obj_host: # obj_ip.add_related(obj_host, # ObjectRelationship.TERM_RESOLVED_TO, # inline=False) ob = CyboxOb(obj_ip) ob.title = 'IP: %s' % ip ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) asn = decomp.get('ASN') if asn: obj_asn = Address() obj_asn.category = Address.CAT_ASN obj_asn.address_value = asn # if obj_host: # obj_asn.add_related(obj_host, # ObjectRelationship.TERM_CONNECTED_TO, # inline=False) # if obj_ip: # obj_asn.add_related(obj_ip, # ObjectRelationship.TERM_CONNECTED_TO, # inline=False) ob = CyboxOb(obj_asn) ob.title = 'ASN: %s' % ip ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) pkg.add_indicator(ind) db('local_file', 'value_to_key', 'values', value2key) db('local_file', 'history', ADPTR_SRC_ID, history) return pkg