def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="",
nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc, PST.p_path))
except IOError:
print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
except IOError:
print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4 :
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Phone'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Phone module")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
search_phone(message)
channel = 'creditcard_categ'
# Source: http://www.richardsramblings.com/regex/credit-card-numbers/
cards = [
r'\b4\d{3}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16-digit VISA, with separators
r'\b5[1-5]\d{2}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16 digits MasterCard
r'\b6(?:011|22(?:(?=[\ \-]?(?:2[6-9]|[3-9]))|[2-8]|9(?=[\ \-]?(?:[01]|2[0-5])))|4[4-9]\d|5\d\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Discover Card
r'\b35(?:2[89]|[3-8]\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Japan Credit Bureau (JCB)
r'\b3[47]\d\d(?:[\ \-]?)\d{6}(?:[\ \-]?)\d{5}\b', # American Express
r'\b(?:5[0678]\d\d|6304|6390|67\d\d)\d{8,15}\b', # Maestro
]
regex = re.compile('|'.join(cards))
while True:
message = p.get_from_set()
if message is not None:
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
all_cards = re.findall(regex, content)
if len(all_cards) > 0:
print 'All matching', all_cards
creditcard_set = set([])
for card in all_cards:
clean_card = re.sub('[^0-9]', '', card)
if lib_refine.is_luhn_valid(clean_card):
print clean_card, 'is valid'
creditcard_set.add(clean_card)
categories = [
'CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'ApiKey'
]
tmp_dict = {}
for filename in categories:
bname = os.path.basename(filename)
tmp_dict[bname] = []
with open(os.path.join(args.d, filename), 'r') as f:
patterns = [r'%s' % (re.escape(s.strip())) for s in f]
tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE)
prec_filename = None
while True:
filename = p.get_from_set()
if filename is None:
publisher.debug("Script Categ is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
paste = Paste.Paste(filename)
content = paste.get_p_content()
for categ, pattern in tmp_dict.items():
found = set(re.findall(pattern, content))
if len(found) >= matchingThreshold:
msg = '{} {}'.format(paste.p_rel_path, len(found))
print(msg, categ)
schema = Schema(title=TEXT(stored=True), path=ID(stored=True,
unique=True),
content=TEXT)
if not os.path.exists(indexpath):
os.mkdir(indexpath)
if not exists_in(indexpath):
ix = create_in(indexpath, schema)
else:
ix = open_dir(indexpath)
# LOGGING #
publisher.info("ZMQ Indexer is Running")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script Indexer is idling 1s")
time.sleep(1)
continue
docpath = message.split(" ", -1)[-1]
paste = PST.get_p_content()
print "Indexing :", docpath
if indexertype == "whoosh":
indexwriter = ix.writer()
indexwriter.update_document(
title=unicode(docpath, errors='ignore'),
path=unicode(docpath, errors='ignore'),
publisher.channel = "Script"
config_section = "Release"
p = Process(config_section)
max_execution_time = p.config.getint("Curve", "max_execution_time")
publisher.info("Release scripts to find release names")
movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
regexs = [movie, tv, xxx]
regex = '|'.join(regexs)
while True:
signal.alarm(max_execution_time)
filepath = p.get_from_set()
if filepath is None:
publisher.debug("Script Release is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
paste = Paste.Paste(filepath)
content = paste.get_p_content()
#signal.alarm(max_execution_time)
try:
releases = set(re.findall(regex, content))
if len(releases) == 0:
continue
'max_execution_time': default_max_execution_time,
'tag': 'infoleak:automatic-detection="dash-address"',
}
}
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'Bitcoin'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Cryptocurrency module ")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
item_id = p.get_from_set()
if item_id is None:
publisher.debug(
"{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
item_content = Item.get_item_content(item_id)
search_crytocurrency(item_id, item_content)
# FUNCTIONS #
publisher.info("Script Categ started")
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
tmp_dict = {}
for filename in categories:
bname = os.path.basename(filename)
tmp_dict[bname] = []
with open(os.path.join(args.d, filename), 'r') as f:
patterns = [r'%s' % re.escape(s.strip()) for s in f]
tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE)
prec_filename = None
while True:
filename = p.get_from_set()
if filename is None:
publisher.debug("Script Categ is Idling 10s")
print 'Sleeping'
time.sleep(10)
continue
paste = Paste.Paste(filename)
content = paste.get_p_content()
for categ, pattern in tmp_dict.items():
found = set(re.findall(pattern, content))
if len(found) > 0:
msg = '{} {}'.format(paste.p_path, len(found))
print msg, categ
p.populate_set_out(msg, categ)
def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="",
nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
item_id = p.get_from_set()
if item_id is None:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
item_content = item_basic.get_item_content(item_id)
mimetype = item_basic.get_item_mimetype(item_id)
item_basename = item_basic.get_basename(item_id)
item_source = item_basic.get_source(item_id)
item_date = item_basic.get_item_date(item_id)
if mimetype.split('/')[0] == "text":
c.text(rawtext=item_content)
c.potentialdomain()
c.validdomain(passive_dns=True, extended=False)
print(c.vdomain)
if c.vdomain and d4.is_passive_dns_enabled():
for dns_record in c.vdomain:
p.populate_set_out(dns_record)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning(
f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}"
)
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning(
f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}"
)
except IOError:
print("CRC Checksum Failed on :", item_id)
publisher.error(
f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed"
)
publisher.channel = "Script"
config_section = "Release"
p = Process(config_section)
max_execution_time = p.config.getint("Curve", "max_execution_time")
publisher.info("Release scripts to find release names")
movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
regexs = [movie, tv, xxx]
regex = '|'.join(regexs)
while True:
signal.alarm(max_execution_time)
filepath = p.get_from_set()
if filepath is None:
publisher.debug("Script Release is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
paste = Paste.Paste(filepath)
content = paste.get_p_content()
#signal.alarm(max_execution_time)
try:
releases = set(re.findall(regex, content))
if len(releases) == 0:
continue
# # TODO: lauch me in core screen
# # TODO: check if already launched in core screen
if __name__ == '__main__':
publisher.port = 6380
publisher.channel = "Script"
config_section = 'D4_client'
p = Process(config_section)
publisher.info("""D4_client is Running""")
last_refresh = time.time()
d4_client = d4.create_d4_client()
while True:
if last_refresh < d4.get_config_last_update_time():
d4_client = d4.create_d4_client()
last_refresh = time.time()
print('D4 Client: config updated')
dns_record = p.get_from_set()
if dns_record is None:
publisher.debug("Script D4_client is idling 1s")
time.sleep(1)
continue
if d4_client:
# Send DNS Record to D4Server
d4_client.send_manual_data(dns_record)
class AbstractModule(ABC):
"""
Abstract Module class
"""
def __init__(self, module_name=None, queue_name=None, logger_channel='Script'):
"""
Init Module
module_name: str; set the module name if different from the instance ClassName
queue_name: str; set the queue name if different from the instance ClassName
logger_channel: str; set the logger channel name, 'Script' by default
"""
# Module name if provided else instance className
self.module_name = module_name if module_name else self._module_name()
# Module name if provided else instance className
self.queue_name = queue_name if queue_name else self._module_name()
# Init Redis Logger
self.redis_logger = publisher
# Port of the redis instance used by pubsublogger
self.redis_logger.port = 6380
# Channel name to publish logs
# # TODO: refactor logging
# If provided could be a namespaced channel like script:<ModuleName>
self.redis_logger.channel = logger_channel
# Run module endlessly
self.proceed = True
# Waiting time in secondes between two proccessed messages
self.pending_seconds = 10
# Setup the I/O queues
self.process = Process(self.queue_name)
def get_message(self):
"""
Get message from the Redis Queue (QueueIn)
Input message can change between modules
ex: '<item id>'
"""
return self.process.get_from_set()
def send_message_to_queue(self, message, queue_name=None):
"""
Send message to queue
:param message: message to send in queue
:param queue_name: queue or module name
ex: send_to_queue(item_id, 'Global')
"""
self.process.populate_set_out(message, queue_name)
def run(self):
"""
Run Module endless process
"""
# Endless loop processing messages from the input queue
while self.proceed:
# Get one message (ex:item id) from the Redis Queue (QueueIn)
message = self.get_message()
if message:
try:
# Module processing with the message from the queue
self.compute(message)
except Exception as err:
trace = traceback.format_tb(err.__traceback__)
self.redis_logger.critical(f"Error in module {self.module_name}: {err}")
self.redis_logger.critical(f"Module {self.module_name} input message: {message}")
self.redis_logger.critical(trace)
print()
print(f"ERROR: {err}")
print(f'MESSAGE: {message}')
print('TRACEBACK:')
for line in trace:
print(line)
else:
self.computeNone()
# Wait before next process
self.redis_logger.debug(f"{self.module_name}, waiting for new message, Idling {self.pending_seconds}s")
time.sleep(self.pending_seconds)
def _module_name(self):
"""
Returns the instance class name (ie. the Module Name)
"""
return self.__class__.__name__
@abstractmethod
def compute(self, message):
"""
Main method of the Module to implement
"""
pass
def computeNone(self):
"""
Method of the Module when there is no message
"""
pass
class AbstractModule(ABC):
"""
Abstract Module class
"""
def __init__(self, module_name=None, queue_name=None):
"""
Init Module
module_name: str; set the module name if different from the instance ClassName
"""
# Module name if provided else instance className
self.module_name = module_name if module_name else self._module_name()
# Module name if provided else instance className
self.queue_name = queue_name if queue_name else self._module_name()
# Init Redis Logger
self.redis_logger = publisher
# Port of the redis instance used by pubsublogger
self.redis_logger.port = 6380
# Channel name to publish logs
self.redis_logger.channel = 'Script'
# TODO modify generic channel Script to a namespaced channel like:
# publish module logs to script:<ModuleName> channel
# self.redis_logger.channel = 'script:%s'%(self.module_name)
# Run module endlessly
self.proceed = True
# Waiting time in secondes between two proccessed messages
self.pending_seconds = 10
# Setup the I/O queues
self.process = Process(self.queue_name)
def run(self):
"""
Run Module endless process
"""
# Endless loop processing messages from the input queue
while self.proceed:
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)
message = self.process.get_from_set()
if message is None:
self.computeNone()
# Wait before next process
self.redis_logger.debug(
f"{self.module_name}, waiting for new message, Idling {self.pending_seconds}s"
)
time.sleep(self.pending_seconds)
continue
try:
# Module processing with the message from the queue
self.compute(message)
except Exception as err:
self.redis_logger.critical(
f"Error in module {self.module_name}: {err}")
def _module_name(self):
"""
Returns the instance class name (ie. the Module Name)
"""
return self.__class__.__name__
@abstractmethod
def compute(self, message):
"""
Main method of the Module to implement
"""
pass
def computeNone(self):
"""
Method of the Module when there is no message
"""
pass
