def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() mimetype = PST._get_p_encoding() if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() c.validdomain(rtype=['A'], extended=True) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() mimetype = PST._get_p_encoding() if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() c.validdomain(rtype=['A'], extended=True) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4 : print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger publisher.port = 6380 # Script is the default channel used for the modules. publisher.channel = 'Script' # Section name in bin/packages/modules.cfg config_section = 'Phone' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Run Phone module") # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue # Do something with the message from the queue search_phone(message)
channel = 'creditcard_categ' # Source: http://www.richardsramblings.com/regex/credit-card-numbers/ cards = [ r'\b4\d{3}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16-digit VISA, with separators r'\b5[1-5]\d{2}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16 digits MasterCard r'\b6(?:011|22(?:(?=[\ \-]?(?:2[6-9]|[3-9]))|[2-8]|9(?=[\ \-]?(?:[01]|2[0-5])))|4[4-9]\d|5\d\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Discover Card r'\b35(?:2[89]|[3-8]\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Japan Credit Bureau (JCB) r'\b3[47]\d\d(?:[\ \-]?)\d{6}(?:[\ \-]?)\d{5}\b', # American Express r'\b(?:5[0678]\d\d|6304|6390|67\d\d)\d{8,15}\b', # Maestro ] regex = re.compile('|'.join(cards)) while True: message = p.get_from_set() if message is not None: filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() all_cards = re.findall(regex, content) if len(all_cards) > 0: print 'All matching', all_cards creditcard_set = set([]) for card in all_cards: clean_card = re.sub('[^0-9]', '', card) if lib_refine.is_luhn_valid(clean_card): print clean_card, 'is valid' creditcard_set.add(clean_card)
categories = [ 'CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'ApiKey' ] tmp_dict = {} for filename in categories: bname = os.path.basename(filename) tmp_dict[bname] = [] with open(os.path.join(args.d, filename), 'r') as f: patterns = [r'%s' % (re.escape(s.strip())) for s in f] tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE) prec_filename = None while True: filename = p.get_from_set() if filename is None: publisher.debug("Script Categ is Idling 10s") print('Sleeping') time.sleep(10) continue paste = Paste.Paste(filename) content = paste.get_p_content() for categ, pattern in tmp_dict.items(): found = set(re.findall(pattern, content)) if len(found) >= matchingThreshold: msg = '{} {}'.format(paste.p_rel_path, len(found)) print(msg, categ)
schema = Schema(title=TEXT(stored=True), path=ID(stored=True, unique=True), content=TEXT) if not os.path.exists(indexpath): os.mkdir(indexpath) if not exists_in(indexpath): ix = create_in(indexpath, schema) else: ix = open_dir(indexpath) # LOGGING # publisher.info("ZMQ Indexer is Running") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script Indexer is idling 1s") time.sleep(1) continue docpath = message.split(" ", -1)[-1] paste = PST.get_p_content() print "Indexing :", docpath if indexertype == "whoosh": indexwriter = ix.writer() indexwriter.update_document( title=unicode(docpath, errors='ignore'), path=unicode(docpath, errors='ignore'),
publisher.channel = "Script" config_section = "Release" p = Process(config_section) max_execution_time = p.config.getint("Curve", "max_execution_time") publisher.info("Release scripts to find release names") movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+" tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+" xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+" regexs = [movie, tv, xxx] regex = '|'.join(regexs) while True: signal.alarm(max_execution_time) filepath = p.get_from_set() if filepath is None: publisher.debug("Script Release is Idling 10s") print('Sleeping') time.sleep(10) continue paste = Paste.Paste(filepath) content = paste.get_p_content() #signal.alarm(max_execution_time) try: releases = set(re.findall(regex, content)) if len(releases) == 0: continue
'max_execution_time': default_max_execution_time, 'tag': 'infoleak:automatic-detection="dash-address"', } } if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" config_section = 'Bitcoin' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Run Cryptocurrency module ") # Endless loop getting messages from the input queue while True: # Get one message from the input queue item_id = p.get_from_set() if item_id is None: publisher.debug( "{} queue is empty, waiting".format(config_section)) time.sleep(1) continue # Do something with the message from the queue item_content = Item.get_item_content(item_id) search_crytocurrency(item_id, item_content)
# FUNCTIONS # publisher.info("Script Categ started") categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential'] tmp_dict = {} for filename in categories: bname = os.path.basename(filename) tmp_dict[bname] = [] with open(os.path.join(args.d, filename), 'r') as f: patterns = [r'%s' % re.escape(s.strip()) for s in f] tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE) prec_filename = None while True: filename = p.get_from_set() if filename is None: publisher.debug("Script Categ is Idling 10s") print 'Sleeping' time.sleep(10) continue paste = Paste.Paste(filename) content = paste.get_p_content() for categ, pattern in tmp_dict.items(): found = set(re.findall(pattern, content)) if len(found) > 0: msg = '{} {}'.format(paste.p_path, len(found)) print msg, categ p.populate_set_out(msg, categ)
def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: item_id = p.get_from_set() if item_id is None: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue item_content = item_basic.get_item_content(item_id) mimetype = item_basic.get_item_mimetype(item_id) item_basename = item_basic.get_basename(item_id) item_source = item_basic.get_source(item_id) item_date = item_basic.get_item_date(item_id) if mimetype.split('/')[0] == "text": c.text(rawtext=item_content) c.potentialdomain() c.validdomain(passive_dns=True, extended=False) print(c.vdomain) if c.vdomain and d4.is_passive_dns_enabled(): for dns_record in c.vdomain: p.populate_set_out(dns_record) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning( f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}" ) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning( f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}" ) except IOError: print("CRC Checksum Failed on :", item_id) publisher.error( f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed" )
# # TODO: lauch me in core screen # # TODO: check if already launched in core screen if __name__ == '__main__': publisher.port = 6380 publisher.channel = "Script" config_section = 'D4_client' p = Process(config_section) publisher.info("""D4_client is Running""") last_refresh = time.time() d4_client = d4.create_d4_client() while True: if last_refresh < d4.get_config_last_update_time(): d4_client = d4.create_d4_client() last_refresh = time.time() print('D4 Client: config updated') dns_record = p.get_from_set() if dns_record is None: publisher.debug("Script D4_client is idling 1s") time.sleep(1) continue if d4_client: # Send DNS Record to D4Server d4_client.send_manual_data(dns_record)
class AbstractModule(ABC): """ Abstract Module class """ def __init__(self, module_name=None, queue_name=None, logger_channel='Script'): """ Init Module module_name: str; set the module name if different from the instance ClassName queue_name: str; set the queue name if different from the instance ClassName logger_channel: str; set the logger channel name, 'Script' by default """ # Module name if provided else instance className self.module_name = module_name if module_name else self._module_name() # Module name if provided else instance className self.queue_name = queue_name if queue_name else self._module_name() # Init Redis Logger self.redis_logger = publisher # Port of the redis instance used by pubsublogger self.redis_logger.port = 6380 # Channel name to publish logs # # TODO: refactor logging # If provided could be a namespaced channel like script:<ModuleName> self.redis_logger.channel = logger_channel # Run module endlessly self.proceed = True # Waiting time in secondes between two proccessed messages self.pending_seconds = 10 # Setup the I/O queues self.process = Process(self.queue_name) def get_message(self): """ Get message from the Redis Queue (QueueIn) Input message can change between modules ex: '<item id>' """ return self.process.get_from_set() def send_message_to_queue(self, message, queue_name=None): """ Send message to queue :param message: message to send in queue :param queue_name: queue or module name ex: send_to_queue(item_id, 'Global') """ self.process.populate_set_out(message, queue_name) def run(self): """ Run Module endless process """ # Endless loop processing messages from the input queue while self.proceed: # Get one message (ex:item id) from the Redis Queue (QueueIn) message = self.get_message() if message: try: # Module processing with the message from the queue self.compute(message) except Exception as err: trace = traceback.format_tb(err.__traceback__) self.redis_logger.critical(f"Error in module {self.module_name}: {err}") self.redis_logger.critical(f"Module {self.module_name} input message: {message}") self.redis_logger.critical(trace) print() print(f"ERROR: {err}") print(f'MESSAGE: {message}') print('TRACEBACK:') for line in trace: print(line) else: self.computeNone() # Wait before next process self.redis_logger.debug(f"{self.module_name}, waiting for new message, Idling {self.pending_seconds}s") time.sleep(self.pending_seconds) def _module_name(self): """ Returns the instance class name (ie. the Module Name) """ return self.__class__.__name__ @abstractmethod def compute(self, message): """ Main method of the Module to implement """ pass def computeNone(self): """ Method of the Module when there is no message """ pass
class AbstractModule(ABC): """ Abstract Module class """ def __init__(self, module_name=None, queue_name=None): """ Init Module module_name: str; set the module name if different from the instance ClassName """ # Module name if provided else instance className self.module_name = module_name if module_name else self._module_name() # Module name if provided else instance className self.queue_name = queue_name if queue_name else self._module_name() # Init Redis Logger self.redis_logger = publisher # Port of the redis instance used by pubsublogger self.redis_logger.port = 6380 # Channel name to publish logs self.redis_logger.channel = 'Script' # TODO modify generic channel Script to a namespaced channel like: # publish module logs to script:<ModuleName> channel # self.redis_logger.channel = 'script:%s'%(self.module_name) # Run module endlessly self.proceed = True # Waiting time in secondes between two proccessed messages self.pending_seconds = 10 # Setup the I/O queues self.process = Process(self.queue_name) def run(self): """ Run Module endless process """ # Endless loop processing messages from the input queue while self.proceed: # Get one message (paste) from the QueueIn (copy of Redis_Global publish) message = self.process.get_from_set() if message is None: self.computeNone() # Wait before next process self.redis_logger.debug( f"{self.module_name}, waiting for new message, Idling {self.pending_seconds}s" ) time.sleep(self.pending_seconds) continue try: # Module processing with the message from the queue self.compute(message) except Exception as err: self.redis_logger.critical( f"Error in module {self.module_name}: {err}") def _module_name(self): """ Returns the instance class name (ie. the Module Name) """ return self.__class__.__name__ @abstractmethod def compute(self, message): """ Main method of the Module to implement """ pass def computeNone(self): """ Method of the Module when there is no message """ pass