def demarrer_scanner(hWaitStop=None, batch=None):
loggingiocscan.info('Starting an IOC scanner instance : ' + threadname)
print ''
print '\tPlease log in to launch scan'
print ''
username = raw_input('Username: '******'Password: '******''
# Get user
u = session.query(User).filter_by(username = username).first()
# No user or bad password
if not u or hashPassword(password) != u.password:
loggingiocscan.critical('Username or password incorrect, stopping the initialization, press a key...')
raw_input()
return
# Get KEY and decrypt MASTER_KEY
keyFromPassword = crypto.keyFromText(password, base64.b64decode(u.b64_kdf_salt))
MASTER_KEY = crypto.decrypt(u.encrypted_master_key, keyFromPassword)
mk_cksum = session.query(GlobalConfig).filter_by(key = 'master_key_checksum').first()
# No checksum in config ???
if not mk_cksum:
loggingiocscan.critical('Database is broken, please create a new one, stopping the initialization...')
del MASTER_KEY
raw_input()
return
# Someone has been playing with the database !
if checksum(MASTER_KEY)!=mk_cksum.value:
loggingiocscan.critical('MASTER_KEY may have been altered, stopping the initialization...')
del MASTER_KEY
raw_input()
return
loggingiocscan.info('Login successful !')
# INITIALIZATION
# TODO : initialise all IOCs in DB, then link them to CP
all_xmliocs = session.query(XMLIOC).order_by(XMLIOC.name.asc())
all_cp = session.query(ConfigurationProfile).order_by(ConfigurationProfile.name.asc())
ioc_by_cp = {}
for cp in all_cp:
if cp.ioc_list == '':
loggingiocscan.warning('No IOC defined for profile "%s"' % cp.name)
continue
ioc_by_cp[cp.id] = []
for e in cp.ioc_list.split(','):
ioc_by_cp[cp.id].append(int(e))
tree_by_ioc = {}
# Retrieves evaluators for current mode
FLAT_MODE = (IOC_MODE == 'flat')
allowedElements = {}
evaluatorList = ioc_modules.flatEvaluatorList if FLAT_MODE else ioc_modules.logicEvaluatorList
for name, classname in evaluatorList.items():
allowedElements[name] = classname.evalList
# Parse XML Ioc into IOC trees according to what we can do
for xmlioc in all_xmliocs:
content = base64.b64decode(xmlioc.xml_content)
oip = openiocparser.OpenIOCParser(content, allowedElements, FLAT_MODE, fromString=True)
oip.parse()
iocTree = oip.getTree()
# Trees may be stripped from non valid elements
if iocTree is not None:
tree_by_ioc[xmlioc.id] = {'name':xmlioc.name, 'tree':iocTree}
# Each configuration profile has a set of trees
tree_by_cp = {cpid: {i:tree_by_ioc[i] for i in ioclist} for (cpid, ioclist) in ioc_by_cp.items()}
halt = False
tache = None
batchquery = None
# Batch filtering
if batch is not None:
loggingiocscan.info('Filtering for batch "%s"' % batch)
batchquery = session.query(Batch).filter( Batch.name == batch).first()
if batchquery is None:
loggingiocscan.error('Unknown batch "%s" ...' % batch)
halt = True
# LAUNCH
# Main loop
while not halt:
try:
# Get targets to be scanned
# and that are not currently being scanned
# or that don't have any retry left
queue = session.query(Task).filter_by(iocscanned=False, reserved_ioc=False, reserved_hash=False).filter(Task.retries_left_ioc > 0)
# Batch filtering
if batchquery is not None:
queue = queue.filter_by(batch_id = batchquery.id)
taille_queue = queue.count()
# Compute the time after which targets are still recovering from last scan
# Gets target which last retry is NULL or before that time
limite_a_reessayer = datetime.datetime.now() - datetime.timedelta(0, SECONDES_ENTRE_TENTATIVES)
a_scanner = queue.filter(or_(Task.last_retry_ioc <= limite_a_reessayer, Task.last_retry_ioc == None))
taille_a_scanner = a_scanner.count()
# Reads this list
while taille_a_scanner > 0:
# Max priority
priorite_max = a_scanner.order_by(Task.priority_ioc.desc()).first().priority_ioc
taches_priorite_max = a_scanner.filter(Task.priority_ioc==priorite_max)
nbre_taches_priorite_max = taches_priorite_max.count()
if BASE_DE_DONNEES_QUEUE.startswith('sqlite'):
tache = taches_priorite_max.order_by(func.random()).first()
else:
tache = taches_priorite_max.order_by(func.newid()).first()
# Mutex on the task
tache.reserved_ioc = True
tache.date_debut = datetime.datetime.now()
session.commit()
loggingiocscan.debug('===============================================================================')
loggingiocscan.debug('Wake up, there is work to do !')
loggingiocscan.info('Queue size : ' + str(taille_queue) + ', including ' + str(taille_a_scanner) + ' to scan, including ' + str(nbre_taches_priorite_max) + ' at top priority (' + str(priorite_max) + ')')
loggingiocscan.debug(' --------------------------------')
loggingiocscan.info(' Starting IOC Scan')
loggingiocscan.info(' Target : ' + str(tache.ip))
loggingiocscan.debug(' --------------------------------')
# Recover Windows Credential and Configuration Profile from Batch
batch = session.query(Batch).filter_by(id = tache.batch_id).first()
wc = session.query(WindowsCredential).filter_by(id = batch.windows_credential_id).first()
cp = session.query(ConfigurationProfile).filter_by(id = batch.configuration_profile_id).first()
if not wc:
raise Exception('WindowsCredential %d does not exist' % tache.windows_credential_id)
if not cp:
raise Exception('ConfigurationProfile %d does not exist' % tache.configuration_profile_id)
# Decrypt password using MASTER_KEY and create target object
targetPassword = crypto.decrypt(wc.encrypted_password, MASTER_KEY)
targetObject = {'ip': tache.ip,
'login': wc.login,
'password': targetPassword,
'domain': wc.domain,
}
# If high confidentiality is enabled, create local directory if needed
if cp.host_confidential:
loggingiocscan.info('"High confidentiality" mode enabled')
testdir = os.path.join(IOC_COMPONENT_ROOT, IOC_CONFIDENTIAL_DIRECTORY)
if not os.path.isdir(testdir):
loggingiocscan.info('Creating confidential directory %s' % testdir)
os.makedirs(testdir)
# Let the scan begin
if cp.id in tree_by_cp.keys():
try:
resultats_scan = scan(targetObject, tree_by_cp[cp.id], cp.host_confidential)
except remotecmd.ShutdownException:
loggingiocscan.warning('Shutdown exception: %s, %s' % (repr(e), str(e)))
else:
loggingiocscan.warning('No IOC to scan (profile=%s)' % cp.name)
resultats_scan = {}
analyse(resultats_scan, tache)
# Update queue size
taille_a_scanner = a_scanner.count()
if halt:
# Stop signal encountered
break
if halt:
loggingiocscan.info('Stopping IOC scanner : ' + threadname)
break
loggingiocscan.debug('(IOC scanner sleeping for ' + str(SLEEP) + ' seconds...)' \
+ (' (' + str(taille_queue) + ' waiting)' if taille_queue > 0 else ''))
time.sleep(SLEEP)
except KeyboardInterrupt:
halt = True
except Exception, e:
exc_type, exc_obj, exc_tb = sys.exc_info()
loggingiocscan.error('Exception caught:')
for line in traceback.format_exc(exc_tb).splitlines():
loggingiocscan.error(line)
# Cancel changes and unreserve task
session.rollback()
if tache is not None:
tache.reserved_ioc = False
tache.retries_left_ioc = max(0,tache.retries_left_ioc - 1)
session.commit()
def demarrer_scanner(hWaitStop=None, batch=None):
logginghashscan.info("Starting an Hash scanner instance : " + threadname)
print ""
print "\tPlease log in to launch scan"
print ""
username = raw_input("Username: "******"Password: "******""
# Get user
u = session.query(User).filter_by(username=username).first()
# No user or bad password
if not u or hashPassword(password) != u.password:
logginghashscan.critical("Username or password incorrect, shutting down...")
raw_input()
sys.exit(1)
# Get KEY and decrypt MASTER_KEY
keyFromPassword = crypto.keyFromText(password, base64.b64decode(u.b64_kdf_salt))
MASTER_KEY = crypto.decrypt(u.encrypted_master_key, keyFromPassword)
mk_cksum = session.query(GlobalConfig).filter_by(key="master_key_checksum").first()
# No checksum in config ???
if not mk_cksum:
logginghashscan.critical("Database is broken, please create a new one !")
del MASTER_KEY
raw_input()
sys.exit(1)
# Someone has been playing with the database !
if checksum(MASTER_KEY) != mk_cksum.value:
logginghashscan.critical("MASTER_KEY may have been altered")
del MASTER_KEY
raw_input()
sys.exit(1)
logginghashscan.info("Login successful !")
# INITIALIZATION
# TODO : initialise all IOCs in DB, then link them to CP
all_xmliocs = session.query(XMLIOC).order_by(XMLIOC.name.asc())
all_cp = session.query(ConfigurationProfile).order_by(ConfigurationProfile.name.asc())
ioc_by_cp = {cp.id: [int(e) for e in cp.ioc_list.split(",")] for cp in all_cp}
tree_by_ioc = {}
# Retrieves evaluators for current mode
FLAT_MODE = IOC_MODE == "flat"
allowedElements = {}
evaluatorList = hash_modules.flatEvaluatorList if FLAT_MODE else hash_modules.logicEvaluatorList
print evaluatorList
for name, classname in evaluatorList.items():
allowedElements[name] = classname.evalList
# Parse XML Ioc into IOC trees according to what we can do
for xmlioc in all_xmliocs:
content = base64.b64decode(xmlioc.xml_content)
oip = openiocparser.OpenIOCParser(content, allowedElements, FLAT_MODE, fromString=True)
oip.parse()
iocTree = oip.getTree()
# Trees may be stripped from non valid elements
if iocTree is not None:
tree_by_ioc[xmlioc.id] = {"name": xmlioc.name, "tree": iocTree}
# Each configuration profile has a set of trees
tree_by_cp = {cpid: {i: tree_by_ioc[i] for i in ioclist} for (cpid, ioclist) in ioc_by_cp.items()}
halt = False
tache = None
batchquery = None
# Batch filtering
if batch is not None:
logginghashscan.info('Filtering for batch "%s"' % batch)
batchquery = session.query(Batch).filter(Batch.name == batch).first()
if batchquery is None:
logginghashscan.error('Unknown batch "%s" ...' % batch)
halt = True
# LAUNCH
# Main loop
while not halt:
try:
# Get targets to be scanned
# and that are not currently being scanned
# or that don't have any retry left
queue = (
session.query(Task)
.filter_by(hashscanned=False, reserved_ioc=False, reserved_hash=False)
.filter(Task.retries_left_hash > 0)
)
# Batch filtering
if batchquery is not None:
queue = queue.filter_by(batch_id=batchquery.id)
taille_queue = queue.count()
# Compute the time after which targets are still recovering from last scan
# Gets target which last retry is NULL or before that time
limite_a_reessayer = datetime.datetime.now() - datetime.timedelta(0, SECONDES_ENTRE_TENTATIVES)
a_scanner = queue.filter(or_(Task.last_retry_hash <= limite_a_reessayer, Task.last_retry_hash == None))
taille_a_scanner = a_scanner.count()
# Reads this list
while taille_a_scanner > 0:
# Max priority
priorite_max = a_scanner.order_by(Task.priority_hash.desc()).first().priority_hash
taches_priorite_max = a_scanner.filter(Task.priority_hash == priorite_max)
nbre_taches_priorite_max = taches_priorite_max.count()
if BASE_DE_DONNEES_QUEUE.startswith("sqlite"):
tache = taches_priorite_max.order_by(func.random()).first()
else:
tache = taches_priorite_max.order_by(func.newid()).first()
# Mutex on the task
tache.reserved_hash = True
tache.date_debut = datetime.datetime.now()
session.commit()
logginghashscan.debug("===============================================================================")
logginghashscan.debug("Wake up, there is work to do !")
logginghashscan.info(
"Queue size : "
+ str(taille_queue)
+ ", including "
+ str(taille_a_scanner)
+ " to scan, including "
+ str(nbre_taches_priorite_max)
+ " at top priority ("
+ str(priorite_max)
+ ")"
)
logginghashscan.debug(" --------------------------------")
logginghashscan.info(" Starting Hash Scan")
logginghashscan.info(" Target : " + str(tache.ip))
logginghashscan.debug(" --------------------------------")
# Recover Windows Credential and Configuration Profile from Batch
batch = session.query(Batch).filter_by(id=tache.batch_id).first()
wc = session.query(WindowsCredential).filter_by(id=batch.windows_credential_id).first()
cp = session.query(ConfigurationProfile).filter_by(id=batch.configuration_profile_id).first()
if not wc:
raise Exception("WindowsCredential %d does not exist" % tache.windows_credential_id)
if not cp:
raise Exception("ConfigurationProfile %d does not exist" % tache.configuration_profile_id)
# Decrypt password using MASTER_KEY and create target object
targetPassword = crypto.decrypt(wc.encrypted_password, MASTER_KEY)
targetObject = {"ip": tache.ip, "login": wc.login, "password": targetPassword, "domain": wc.domain}
# If high confidentiality is enabled, create local directory if needed
if cp.host_confidential:
logginghashscan.info('"High confidentiality" mode enabled')
testdir = os.path.join(IOC_COMPONENT_ROOT, IOC_CONFIDENTIAL_DIRECTORY)
if not os.path.isdir(testdir):
logginghashscan.info("Creating confidential directory %s" % testdir)
os.makedirs(testdir)
# Let the scan begin
resultats_scan = scan(targetObject, tree_by_cp[cp.id], cp.host_confidential)
# Analyze the results
analyse(resultats_scan, tache)
# Update queue size
taille_a_scanner = a_scanner.count()
try:
# If launched as a service (probably removed soon, TODO)
halt = win32event.WaitForSingleObject(hWaitStop, 2000) == win32event.WAIT_OBJECT_0
except:
pass
if halt:
# Stop signal encountered
break
if halt:
logginghashscan.info("Stopping Hash scanner : " + threadname)
break
logginghashscan.debug(
"(Hash scanner sleeping for "
+ str(SLEEP)
+ " seconds...)"
+ (" (" + str(taille_queue) + " waiting)" if taille_queue > 0 else "")
)
time.sleep(SLEEP)
except KeyboardInterrupt:
halt = True
except Exception, e:
halt = True
logginghashscan.error("Exception caught : %s, %s, %s" % (repr(e), str(e.message), str(e)))
# Cancel changes and unreserve task
session.rollback()
if tache is not None:
tache.reserved_hash = False
tache.retries_left_hash = max(0, tache.retries_left_hash - 1)
session.commit()
def demarrer_scanner(hWaitStop=None, batch=None):
loggingiocscan.info('Starting an IOC scanner instance : ' + threadname)
print ''
print '\tPlease log in to launch scan'
print ''
username = raw_input('Username: '******'Password: '******''
# Get user
u = session.query(User).filter_by(username = username).first()
# No user or bad password
if not u or hashPassword(password) != u.password:
loggingiocscan.critical('Username or password incorrect, stopping the initialization...')
raw_input()
return
# Get KEY and decrypt MASTER_KEY
keyFromPassword = crypto.keyFromText(password, base64.b64decode(u.b64_kdf_salt))
MASTER_KEY = crypto.decrypt(u.encrypted_master_key, keyFromPassword)
mk_cksum = session.query(GlobalConfig).filter_by(key = 'master_key_checksum').first()
# No checksum in config ???
if not mk_cksum:
loggingiocscan.critical('Database is broken, please create a new one, stopping the initialization...')
del MASTER_KEY
raw_input()
return
# Someone has been playing with the database !
if checksum(MASTER_KEY)!=mk_cksum.value:
loggingiocscan.critical('MASTER_KEY may have been altered, stopping the initialization...')
del MASTER_KEY
raw_input()
return
loggingiocscan.info('Login successful !')
# INITIALIZATION
# TODO : initialise all IOCs in DB, then link them to CP
all_xmliocs = session.query(XMLIOC).order_by(XMLIOC.name.asc())
all_cp = session.query(ConfigurationProfile).order_by(ConfigurationProfile.name.asc())
ioc_by_cp = {}
for cp in all_cp:
if cp.ioc_list == '':
loggingiocscan.warning('No IOC defined for profile "%s"' % cp.name)
continue
ioc_by_cp[cp.id] = []
for e in cp.ioc_list.split(','):
ioc_by_cp[cp.id].append(int(e))
tree_by_ioc = {}
# Retrieves evaluators for current mode
FLAT_MODE = (IOC_MODE == 'flat')
allowedElements = {}
evaluatorList = ioc_modules.flatEvaluatorList if FLAT_MODE else ioc_modules.logicEvaluatorList
for name, classname in evaluatorList.items():
allowedElements[name] = classname.evalList
# Parse XML Ioc into IOC trees according to what we can do
for xmlioc in all_xmliocs:
content = base64.b64decode(xmlioc.xml_content)
oip = openiocparser.OpenIOCParser(content, allowedElements, FLAT_MODE, fromString=True)
oip.parse()
iocTree = oip.getTree()
# Trees may be stripped from non valid elements
if iocTree is not None:
tree_by_ioc[xmlioc.id] = {'name':xmlioc.name, 'tree':iocTree}
# Each configuration profile has a set of trees
tree_by_cp = {cpid: {i:tree_by_ioc[i] for i in ioclist} for (cpid, ioclist) in ioc_by_cp.items()}
halt = False
tache = None
batchquery = None
# Batch filtering
if batch is not None:
loggingiocscan.info('Filtering for batch "%s"' % batch)
batchquery = session.query(Batch).filter( Batch.name == batch).first()
if batchquery is None:
loggingiocscan.error('Unknown batch "%s" ...' % batch)
halt = True
# LAUNCH
# Main loop
while not halt:
try:
# Get targets to be scanned
# and that are not currently being scanned
# or that don't have any retry left
queue = session.query(Task).filter_by(iocscanned=False, reserved_ioc=False, reserved_hash=False).filter(Task.retries_left_ioc > 0)
# Batch filtering
if batchquery is not None:
queue = queue.filter_by(batch_id = batchquery.id)
taille_queue = queue.count()
# Compute the time after which targets are still recovering from last scan
# Gets target which last retry is NULL or before that time
limite_a_reessayer = datetime.datetime.now() - datetime.timedelta(0, SECONDES_ENTRE_TENTATIVES)
a_scanner = queue.filter(or_(Task.last_retry_ioc <= limite_a_reessayer, Task.last_retry_ioc == None))
taille_a_scanner = a_scanner.count()
# Reads this list
while taille_a_scanner > 0:
# Max priority
priorite_max = a_scanner.order_by(Task.priority_ioc.desc()).first().priority_ioc
taches_priorite_max = a_scanner.filter(Task.priority_ioc==priorite_max)
nbre_taches_priorite_max = taches_priorite_max.count()
if BASE_DE_DONNEES_QUEUE.startswith('sqlite'):
tache = taches_priorite_max.order_by(func.random()).first()
else:
tache = taches_priorite_max.order_by(func.newid()).first()
# Mutex on the task
tache.reserved_ioc = True
tache.date_debut = datetime.datetime.now()
session.commit()
loggingiocscan.debug('===============================================================================')
loggingiocscan.debug('Wake up, there is work to do !')
loggingiocscan.info('Queue size : ' + str(taille_queue) + ', including ' + str(taille_a_scanner) + ' to scan, including ' + str(nbre_taches_priorite_max) + ' at top priority (' + str(priorite_max) + ')')
loggingiocscan.debug(' --------------------------------')
loggingiocscan.info(' Starting IOC Scan')
loggingiocscan.info(' Target : ' + str(tache.ip))
loggingiocscan.debug(' --------------------------------')
# Recover Windows Credential and Configuration Profile from Batch
batch = session.query(Batch).filter_by(id = tache.batch_id).first()
wc = session.query(WindowsCredential).filter_by(id = batch.windows_credential_id).first()
cp = session.query(ConfigurationProfile).filter_by(id = batch.configuration_profile_id).first()
if not wc:
raise Exception('WindowsCredential %d does not exist' % tache.windows_credential_id)
if not cp:
raise Exception('ConfigurationProfile %d does not exist' % tache.configuration_profile_id)
# Decrypt password using MASTER_KEY and create target object
targetPassword = crypto.decrypt(wc.encrypted_password, MASTER_KEY)
targetObject = {'ip': tache.ip,
'login': wc.login,
'password': targetPassword,
'domain': wc.domain,
}
# If high confidentiality is enabled, create local directory if needed
if cp.host_confidential:
loggingiocscan.info('"High confidentiality" mode enabled')
testdir = os.path.join(IOC_COMPONENT_ROOT, IOC_CONFIDENTIAL_DIRECTORY)
if not os.path.isdir(testdir):
loggingiocscan.info('Creating confidential directory %s' % testdir)
os.makedirs(testdir)
# Let the scan begin
if cp.id in tree_by_cp.keys():
resultats_scan = scan(targetObject, tree_by_cp[cp.id], cp.host_confidential)
else:
loggingiocscan.warning('No IOC to scan (profile=%s)' % cp.name)
resultats_scan = {}
analyse(resultats_scan, tache)
# Update queue size
taille_a_scanner = a_scanner.count()
try:
# If launched as a service (probably removed soon, TODO)
halt = (win32event.WaitForSingleObject(hWaitStop, 2000) == win32event.WAIT_OBJECT_0)
except:
pass
if halt:
# Stop signal encountered
break
if halt:
loggingiocscan.info('Stopping IOC scanner : ' + threadname)
break
loggingiocscan.debug('(IOC scanner sleeping for ' + str(SLEEP) + ' seconds...)' \
+ (' (' + str(taille_queue) + ' waiting)' if taille_queue > 0 else ''))
time.sleep(SLEEP)
except KeyboardInterrupt:
halt = True
except Exception, e:
loggingiocscan.error('Exception caught : %s, %s, %s' % (repr(e), str(e.message), str(e)))
# Cancel changes and unreserve task
session.rollback()
if tache is not None:
tache.reserved_ioc = False
tache.retries_left_ioc = max(0,tache.retries_left_ioc - 1)
session.commit()