def __init__(self):
super(ScanOnConfig, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
self._scanner_config = DetectorConfiguration.ConfigurationLoader()
self._context = '{}'
self._source_data = {}
self._importer = Importer(self._ioc.message())
self._duplification_filter = SourceDeDuplify(self._ioc.message())
def __init__(self, logger, collection='source_dedup'):
# type: (Logging.Logger) -> None
self._logger = logger
self.collection_name = collection
self._config = Configuration()
try:
self._mongo_connection = MongoConnection()
self._client = self._mongo_connection.client()
self._db = self._client.get_database(
name=os.getenv('GREASE_MONGO_DB', 'grease'),
write_concern=pymongo.WriteConcern(w=0))
self._collection = self._db.get_collection(
name=self.collection_name)
self._dedup = True
except ServerSelectionTimeoutError:
self._mongo_connection = None
self._client = None
self._db = None
self._collection = None
self._dedup = False
def mock_data(self):
# type: () -> list
conf = Configuration()
matches = []
final = []
for root, dirnames, filenames in os.walk(conf.opt_dir):
for filename in fnmatch.filter(
filenames,
'*.mock.{0}.json'.format(self.__class__.__name__)):
matches.append(os.path.join(root, filename))
for doc in matches:
with open(doc) as current_file:
content = current_file.read().replace('\r\n', '')
try:
for res in json.loads(content):
final.append(res)
except ValueError:
continue
return final
class test_logging(TestCase):
_config = Configuration()
def test_is_object_LogFile(self):
log = Logging.Logger()
self.assertTrue(log.get_logger(), Logger)
del log
def test_log_get_messages(self):
log = Logging.Logger()
log.debug('random message')
self.assertEqual(1, len(log.get_messages()))
del log
def test_empty_logger(self):
log = Logging.Logger()
log.debug('random message')
log.get_messages_dump()
self.assertEqual(0, len(log.get_messages()))
del log
def test_log_file_write_line(self):
log = Logging.Logger()
# test for directory
self.assertTrue(self._config.grease_dir + self._config.fs_Separator)
# test log file exists
self.assertTrue(self._config.grease_dir + self._config.fs_Separator +
"grease.log")
# test line count
original = sum(
1 for line in open(self._config.grease_dir +
self._config.fs_Separator + "grease.log"))
log.debug("I'm a test message")
final = sum(1
for line in open(self._config.grease_dir +
self._config.fs_Separator + "grease.log"))
self.assertEqual(original + 1, final)
def __init__(self):
super(Detector, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
self._excelsior_config = DetectorConfiguration.ConfigurationLoader()
self._importer = Importer(self._ioc.message())
class Detector(GreaseDaemonCommand):
def __init__(self):
super(Detector, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
self._excelsior_config = DetectorConfiguration.ConfigurationLoader()
self._importer = Importer(self._ioc.message())
def __del__(self):
super(Detector, self).__del__()
self._sql.get_session().close()
def execute(self, context='{}'):
# first lets see if we have some stuff to parse
result = self._sql.get_session().query(SourceData, JobServers)\
.filter(JobServers.id == SourceData.detection_server)\
.filter(JobServers.id == self._config.node_db_id())\
.filter(SourceData.detection_start_time == None)\
.filter(SourceData.detection_end_time == None)\
.filter(SourceData.detection_complete == False)\
.limit(15)\
.all()
if not result:
self._ioc.message().debug(
"No sources scheduled for detection on this node", True)
return True
else:
# Now lets loop through
self._ioc.message().debug(
"TOTAL SOURCE DOCUMENTS RETURNED: [{0}]".format(len(result)),
True)
for source in result:
# first claiming them as ours then parsing them
self._take_ownership(source.SourceData.id)
# now lets parse the sources
self._ioc.message().debug(
"PROCESSING SOURCE ID: [{0}]".format(source.SourceData.id),
True)
parsed_source = self._parse_source(
source.SourceData.source_data,
self._excelsior_config.get_scanner_config(
source.SourceData.scanner))
self._complete(source.SourceData.id)
# now lets assign this parsed source out
if self._schedule_scheduling(source.SourceData.id,
parsed_source):
self._ioc.message().info(
"Successfully Scheduled Parsed Source ID: [" +
str(source.SourceData.id) + "]")
continue
else:
self._reverse(source.SourceData.id)
self._ioc.message().error(
"Failed To Schedule Parsed Source ID: [" +
str(source.SourceData.id) + "]")
continue
return True
def _take_ownership(self, source_file_id):
# type: (int) -> None
stmt = update(SourceData)\
.where(SourceData.id == source_file_id)\
.values(detection_start_time=datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"TAKING OWNERSHIP OF SOURCE ID: [{0}]".format(source_file_id),
True)
def _complete(self, source_file_id):
# type: (int) -> None
stmt = update(SourceData)\
.where(SourceData.id == source_file_id)\
.values(detection_complete=True, detection_end_time=datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"COMPLETING SOURCE ID: [{0}]".format(source_file_id), True)
def _reverse(self, source_file_id):
stmt = update(SourceData)\
.where(SourceData.id == source_file_id)\
.values(detection_start_time=None, detection_end_time=None, detection_complete=None)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"SOURCE FILE FAILED SCHEDULING REVERSING FILE: [{0}]".format(
source_file_id), True)
def _parse_source(self, sources, rule_set):
# type: (dict, list) -> dict
# lets parse the source
# now lets get the data out of it
# now lets loop through those results
# first lets setup some state tracking stuff
final_source_data = defaultdict()
index = 0
total_records = len(sources)
for source in sources:
self._ioc.message().debug(
"PROCESSING OBJECT [{0}] OF [{1}] INTERNAL STRUCTURE: [{2}]".
format(str(index + 1), str(total_records), str(source)), True)
# first lets initialize all the source data into the object
final_source_data[index] = source
# now lets create our rule_processing key
final_source_data[index]['rule_processing'] = defaultdict()
# now lets loop through the rules we want to parse this source with
for rule in rule_set:
self._ioc.message().debug(
"PROCESSING RULE [{0}]".format(str(rule['name'])), True)
# first lets make the rule processing result key
final_source_data[index]['rule_processing'][
rule['name']] = defaultdict()
# next lets compute each rule in the rule processor
# Yes another for loop but I promise this will be fast
# look at some point we have to iterate over data
for detector, detector_config in rule['logic'].iteritems():
# lets try to new up this detector
detector_instance = self._importer.load(
'tgt_grease_enterprise.Detectors', detector, True)
if isinstance(detector_instance, Detectors.BaseDetector):
self._ioc.message().debug(
"PROCESSING RULE [{0}] LOGICAL BLOCK [{1}]".format(
str(rule['name']), str(detector)), True)
# we have a valid detector to parse with
# lets compute the source with the detector config
detector_instance.param_compute(
source, detector_config)
# lets observe the result
result = detector_instance.get_result()
if result['result']:
# we passed the rule lets set that in the final source data
self._ioc.message().debug(
"PROCESSING RULE [{0}] LOGICAL BLOCK [{1}] PASSED"
.format(str(rule['name']),
str(detector)), True)
final_source_data[index]['rule_processing'][
rule['name']]['status'] = True
self._ioc.message().debug(
"RULE [{0}] PASSED THUS FAR".format(
str(rule['name'])), True)
result.pop('result')
if 'parameters' not in final_source_data[index][
'rule_processing'][rule['name']]:
final_source_data[index]['rule_processing'][
rule['name']]['parameters'] = result
else:
final_source_data[index]['rule_processing'][
rule['name']]['parameters'].update(result)
if 'constants' in rule:
final_source_data[index]['rule_processing'][
rule['name']]['parameters'][
'constants'] = rule['constants']
# del out the instance
del detector_instance
# route on
continue
else:
# rule failed
final_source_data[index]['rule_processing'][
rule['name']]['status'] = False
self._ioc.message().debug(
"PROCESSING RULE [{0}] LOGICAL BLOCK [{1}] FAILED :: SOURCE FAILS RULE SET"
.format(str(rule['name']),
str(detector)), True)
# del out the instance
del detector_instance
# route on
break
else:
# we got an invalid detector and it couldn't be found
self._ioc.message().error("Invalid Detector: [" +
str(detector) + "]",
hipchat=True)
del detector_instance
break
# finally lets convert back to normal dict for the rule
final_source_data[index]['rule_processing'][
rule['name']] = dict(final_source_data[index]
['rule_processing'][rule['name']])
# now lets convert the rule_processing back to a normal array
final_source_data[index]['rule_processing'] = dict(
final_source_data[index]['rule_processing'])
self._ioc.message().debug(
"FINAL SOURCE RULE PROCESSING STRUCTURE: [{0}]".format(
str(final_source_data[index]['rule_processing'])), True)
self._ioc.message().debug(
"PROCESSING OBJECT [{0}] OF [{1}] COMPLETE".format(
str(index + 1), str(total_records)), True)
# finally increment our pointer
index += 1
# return final for usage elsewhere
return final_source_data
def _schedule_scheduling(self, source_id, updated_source):
# type: (dict, str) -> bool
# first lets get applicable servers to run detectors
# lets only get the least assigned server so we can round robin
result = self._sql.get_session()\
.query(JobServers)\
.filter(JobServers.scheduler == True)\
.filter(JobServers.active == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if not result:
self._ioc.message().error(
"Failed to find active scheduling server!::Dropping Scan",
hipchat=True)
return False
else:
server = result.id
# Now lets update the sources for the determined server to work
stmt = update(SourceData)\
.where(SourceData.id == source_id)\
.values(scheduling_server=server, source_data=updated_source)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
# finally lets ensure we account for the fact our server is going to do
# that job and increment the assignment counter
stmt = update(JobServers).where(JobServers.id == server).values(
jobs_assigned=result.jobs_assigned + 1)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"DETECTION FOR SOURCE COMPLETE::SCHEDULED TO SERVER [{0}]".
format(server), True)
return True
class Section31(GreaseDaemonCommand):
def __init__(self):
super(Section31, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
def __del__(self):
super(Section31, self).__del__()
self._sql.get_session().close()
def execute(self, context='{}'):
# first lets get the entire farm
self._ioc.message().debug("Fetching Current Farm Status", True)
farm = self._get_farm_status()
# now lets check each server from the last known state
self._ioc.message().debug(
"Examining each server. Current servers: [{0}]".format(len(farm)),
True)
for server in farm:
if self._server_alive(server):
# server seems to be alive keep going
self._ioc.message().debug("Server is Alive::Skipping", True)
continue
else:
self._ioc.message().debug(
"Server is not alive! Attempting to claim", True)
# server is not alive, we need to migrate work from it
# first lets declare ourselves as the doctor
self._declare_doctor(server[0])
# now lets sleep to ensure we don't stop on other doctors
time.sleep(10)
# okay lets ensure we are still the doctor
if self._am_i_the_doctor(server[0]):
# time to reassign
self._ioc.message().debug(
"I am still the doctor, preparing to cull", True)
self._cull_server(server[0])
else:
# during our slumber someone else decided to work on this server let them have it
self._ioc.message().debug(
"Doctor previously declared. Returning", True)
continue
return True
def _get_farm_status(self):
# type: () -> list
sql = """
WITH on_demand AS (
SELECT
jq.host_name,
count(jq.id) AS total
FROM
job_queue jq
WHERE
(
(
jq.in_progress = FALSE AND
jq.completed = FALSE
) OR
jq.in_progress = TRUE
)
GROUP BY jq.host_name
),
completed_on_demand AS (
SELECT
jt.server_id as host_name,
count(jt.id) AS total
FROM
job_servers js
LEFT OUTER JOIN job_telemetry jt ON (jt.server_id = js.id)
GROUP BY jt.server_id
),
completed_daemon AS (
SELECT
js.id as host_name,
count(jtd.id) AS total
FROM
job_servers js
LEFT OUTER JOIN job_telemetry_daemon jtd ON (jtd.server_id = js.id)
GROUP BY js.id
),
sources AS (
SELECT
js.id as host_name,
count(sf.id) AS total
FROM
source_data sf
INNER JOIN job_servers js ON (js.id = sf.detection_server)
WHERE
(
(
sf.detection_start_time IS NOT NULL AND
sf.detection_end_time IS NOT NULL
) OR
(
sf.detection_start_time IS NOT NULL AND
sf.detection_end_time IS NULL
)
)
GROUP BY js.id
),
schedules AS (
SELECT
js.id as host_name,
count(sq.id) AS total
FROM
source_data sq
INNER JOIN job_servers js ON (js.id = sq.scheduling_server)
WHERE
(
(
sq.scheduling_start_time IS NOT NULL AND
sq.scheduling_end_time IS NOT NULL
) OR
(
sq.scheduling_start_time IS NOT NULL AND
sq.scheduling_end_time IS NULL
)
)
GROUP BY js.id
)
SELECT
js.id AS job_server_id,
js.host_name,
coalesce(od.total, 0) AS on_demand_total,
coalesce((cod.total + coda.total), 0) AS completed_total,
coalesce(s.total, 0) AS sources,
coalesce(sch.total, 0) AS schedules,
(coalesce(od.total, 0) || '-' || coalesce((cod.total + coda.total), 0) || '-' || coalesce(s.total, 0) || '-' ||
coalesce(sch.total, 0)) AS str_hash
FROM
job_servers js
LEFT OUTER JOIN on_demand od ON (od.host_name = js.id)
LEFT OUTER JOIN completed_on_demand cod ON (cod.host_name = js.id)
LEFT OUTER JOIN completed_daemon coda ON (coda.host_name = js.id)
LEFT OUTER JOIN sources s ON (s.host_name = js.id)
LEFT OUTER JOIN schedules sch ON (sch.host_name = js.id)
WHERE
js.active IS TRUE
ORDER BY js.id
"""
return list(self._sql.get_engine().execute(sql).fetchall())
def _declare_doctor(self, server_id):
# type: (int) -> None
# first lets check to ensure someone hasn't already claimed this beast
result = self._sql.get_session().query(ServerHealth)\
.filter(ServerHealth.server == server_id)\
.first()
if result and result.doctor != '':
self._ioc.message().error(
"SERVER DOCTOR ALREADY DECLARED FOR [{0}]".format(server_id))
return
stmt = update(ServerHealth).where(
ServerHealth.server == server_id).values(
docter=self._config.node_db_id())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
def _am_i_the_doctor(self, server_id):
# type: (int) -> bool
result = self._sql.get_session().query(ServerHealth)\
.filter(ServerHealth.doctor == self._config.node_db_id())\
.filter(ServerHealth.server == server_id)\
.first()
if result:
return True
else:
return False
def _server_alive(self, server):
# type: (list) -> bool
self._ioc.message().debug(
"data received to be processed [{0}]".format(str(server)), True)
result = self._sql.get_session().query(ServerHealth)\
.filter(ServerHealth.server == server[0])\
.first()
if not result:
self._ioc.message().debug("Server not in health::registering",
True)
# if this is a server not checked into health before
self._register_in_health(server)
result = self._sql.get_session().query(ServerHealth)\
.filter(ServerHealth.server == server[0])\
.first()
# now return to regular logic
self._ioc.message().debug("Inspecting last check timestamp", True)
if result.check_time < (datetime.datetime.now(result.check_time.tzinfo)
- datetime.timedelta(hours=6)):
# server status is old
# lets check the hash though
if result.job_hash == server[6]:
# hash hasn't changed time for d-com
return False
else:
# hash is old but changed, lets update and return True
self._update_hash(server[0], server[6])
return True
else:
# results aren't 12 hrs old yet
if result.job_hash == server[6]:
# log but do nothing so we know bad things will happen if we continue
if result.check_time < (
datetime.datetime.now(result.check_time.tzinfo) -
datetime.timedelta(minutes=10)):
self._ioc.message().warning(
"Server hash has not changed since last poll ID:[{0}] hash:[{1}]"
.format(server[0], server[6]))
else:
# if it changed lets update so we see the current hash
self._update_hash(server[0], server[6])
return True
def _register_in_health(self, server):
newServer = ServerHealth(server=server[0],
job_hash=server[6],
check_time=datetime.datetime.utcnow())
self._sql.get_session().add(newServer)
self._sql.get_session().commit()
def _update_hash(self, server_id, hash_str):
# type: (str, str) -> None
stmt = update(ServerHealth)\
.where(ServerHealth.server == server_id)\
.values(job_hash=hash_str, check_time=datetime.datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
def _deactivate_server(self, server_id):
# type: (int) -> None
stmt = update(JobServers)\
.where(JobServers.id == server_id)\
.values(active=False)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
def _get_new_execution_server(self, server_id):
# type: (int) -> JobServers
# first get current env
old_server = self._sql.get_session().query(JobServers)\
.filter(JobServers.id == server_id)\
.first()
new_server = self._sql.get_session().query(JobServers)\
.filter(JobServers.execution_environment == old_server.execution_environment)\
.filter(JobServers.active == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if new_server:
# we have a server to reassign to
return new_server
else:
# there are no other execution servers for that environment
self._ioc.message().error(
"No other valid execution environment for jobs assigned to server [{0}]"
.format(server_id),
hipchat=True)
return False
def _cull_server(self, server_id):
# type: (int) -> None
self._ioc.message().warning(
"DEACTIVATING SERVER [{0}]".format(server_id), hipchat=True)
# first lets deactivate the job server
self._deactivate_server(server_id)
# next we need to check for any jobs scheduled to that instance and reassign them
# first lets see if there is another available server
server = self._get_new_execution_server(server_id)
if server:
# get the number of jobs being reassigned
result = self._sql.get_session().query(JobQueue) \
.filter(or_(and_(JobQueue.in_progress == False, JobQueue.completed == False), JobQueue.in_progress == True)) \
.filter(JobQueue.host_name == server_id)\
.all()
if result:
jobs = []
for job in result:
jobs.append(job.id)
# we have jobs to move
# step one get the list of jobs to reassign
job_id_group = tuple(jobs)
self._reassign_on_demand_jobs(server_id, server.id,
job_id_group)
else:
self._ioc.message().info(
"No jobs to reassign for server [{0}] being decommissioned"
.format(server_id))
else:
# no other valid server could be found for on-demand work
self._ioc.message().error(
"Failed reassigning jobs for server [{0}]".format(server_id))
# next lets move over any source detection work
result = self._sql.get_session().query(SourceData)\
.filter(SourceData.detection_server == server_id)\
.filter(SourceData.detection_start_time == None)\
.filter(SourceData.detection_end_time == None)\
.filter(SourceData.detection_complete == False)\
.all()
jobs = []
for job in result:
jobs.append(job.id)
source_file_jobs = tuple(jobs)
if len(source_file_jobs):
# we have sources to reassign parsing for
self._reassign_sources(source_file_jobs, server_id)
else:
# no source documents to reassign
self._ioc.message().info(
"No source parsing to reassign for server [{0}]".format(
server_id))
# finally lets worry about schedules to be reassigned
result = self._sql.get_session().query(SourceData) \
.filter(SourceData.scheduling_server == server_id) \
.filter(SourceData.detection_start_time != None)\
.filter(SourceData.detection_end_time != None)\
.filter(SourceData.detection_complete == True)\
.filter(SourceData.scheduling_start_time == None)\
.filter(SourceData.scheduling_end_time == None)\
.filter(SourceData.scheduling_complete == False)\
.all()
jobs = []
for job in result:
jobs.append(job.id)
schedules = tuple(jobs)
if len(schedules):
# we have sources to reassign parsing for
self._reassign_schedules(schedules, server_id)
else:
# no source documents to reassign
self._ioc.message().info(
"No schedules to reassign for server [{0}]".format(server_id))
def _reassign_on_demand_jobs(self, server_id, new_server, job_list):
# type: (int, int, tuple) -> None
# now lets do the update
if not job_list:
job_list = (-1, )
stmt = update(JobQueue)\
.where(JobQueue.id.in_(job_list))\
.values(host_name=new_server)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
# now lets update the totals on the job servers in question
self._update_job_assignment_totals(server_id, new_server,
len(job_list))
self._ioc.message().info(
"JOB ID SET [{0}] MOVED FROM SERVER [{1}] TO SERVER [{2}]".format(
job_list, server_id, new_server))
def _reassign_sources(self, source_file_ids, origin_server):
# type: (tuple, int) -> None
# first lets find our applicable detector server
result = self._sql.get_session().query(JobServers)\
.filter(JobServers.active == True)\
.filter(JobServers.detector == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if result:
new_server = result.id
reassign_total = len(source_file_ids)
# Actual reassignment of sources
stmt = update(SourceData)\
.where(SourceData.id.in_(source_file_ids))\
.values(detection_server=new_server)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._update_job_assignment_totals(origin_server, new_server,
reassign_total)
else:
# oh crap we have no job detector servers left alive
# Me IRL Right now:
# https://68.media.tumblr.com/e59c51080e14cee56ce93416cf8055c8/tumblr_mzncp3nYXh1syplf0o1_250.gif
self._ioc.message().error(
"NO AVAILABLE DETECTOR SERVER CANNOT REASSIGN JOBS [{0}] FROM SERVER [{1}]"
.format(source_file_ids, origin_server),
hipchat=True)
# bail because we have no one to reassign to
return
def _reassign_schedules(self, schedule_ids, origin_server):
# type: (tuple, int) -> None
# first lets find our applicable scheduling server
result = self._sql.get_session().query(JobServers)\
.filter(JobServers.active == True)\
.filter(JobServers.scheduler == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if result:
new_server = result.id
reassign_total = len(schedule_ids)
# Actual reassignment of sources
stmt = update(SourceData)\
.where(SourceData.id.in_(schedule_ids))\
.values(scheduling_server=new_server)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._update_job_assignment_totals(origin_server, new_server,
reassign_total)
else:
# oh crap we have no job scheduler servers left alive
# Me IRL Right now: https://media.giphy.com/media/HUkOv6BNWc1HO/giphy.gif
self._ioc.message().error(
"NO AVAILABLE SCHEDULER SERVER CANNOT REASSIGN JOBS [{0}] FROM SERVER [{1}]"
.format(schedule_ids, origin_server),
hipchat=True)
# bail because we have no one to reassign to
return
def _update_job_assignment_totals(self, old_server, new_server, job_count):
# type: (int, int, int) -> None
# first lets deduct from the bad server
stmt1 = update(JobServers)\
.where(id=old_server)\
.values(jobs_assigned=JobServers.jobs_assigned - job_count)
stmt2 = update(JobServers)\
.where(id=new_server)\
.values(jobs_assigned=JobServers.jobs_assigned + job_count)
self._sql.get_session().execute(stmt1)
self._sql.get_session().execute(stmt2)
self._sql.get_session().commit()
def __init__(self):
super(Section31, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
def __init__(self, router):
super(UnixDaemon, self).__init__(router)
self._config = Configuration()
# self._pidfile = '/tmp/grease/grease.pid'
self._pidfile = self._config.grease_dir + self._config.fs_Separator + 'grease.pid'
self._pid = os.getpid()
class SourceDeDuplify(object):
def __init__(self, logger, collection='source_dedup'):
# type: (Logging.Logger) -> None
self._logger = logger
self.collection_name = collection
self._config = Configuration()
try:
self._mongo_connection = MongoConnection()
self._client = self._mongo_connection.client()
self._db = self._client.get_database(
name=os.getenv('GREASE_MONGO_DB', 'grease'),
write_concern=pymongo.WriteConcern(w=0))
self._collection = self._db.get_collection(
name=self.collection_name)
self._dedup = True
except ServerSelectionTimeoutError:
self._mongo_connection = None
self._client = None
self._db = None
self._collection = None
self._dedup = False
def __del__(self):
self._client.close()
def create_unique_source(self,
source_name,
field_set,
source,
strength=None):
# type: (str, list, list) -> list
if not self._dedup:
self._logger.error(
"Failed To connect to MongoDB de-duplication is off",
hipchat=True)
return source
# we could connect lets process
if not isinstance(source, list):
self._logger.error(
'DeDuplification did not receive list returning empty',
hipchat=True)
return []
self._logger.debug(
"STARTING DEDUPLICATION::TOTAL OBJECTS TO PROCESS [" +
str(len(source)) + "]")
# now comes James' version of machine learning. I call it "Blue Collar Machine Learning"
source_pointer = 0
source_max = len(source)
threads = []
final = []
while source_pointer < source_max:
# Ensure we aren't swamping the system
cpu = cpu_percent(interval=.1)
mem = virtual_memory().percent
if \
cpu >= int(self._config.get('GREASE_THREAD_MAX', '85')) \
or mem >= int(self._config.get('GREASE_THREAD_MAX', '85')):
self._logger.info("sleeping due to high memory consumption",
verbose=True)
# remove variables
del cpu
del mem
continue
# ensure we don't open too many connections
self._logger.debug(
"Current Running DeDuplication Threads [{0}]".format(
len(threads)),
verbose=True)
threads_final = []
for thread in threads:
if thread.isAlive():
threads_final.append(thread)
threads = threads_final
self._logger.debug(
"Remaining Running DeDuplication Threads [{0}]".format(
len(threads)),
verbose=True)
if len(threads) >= int(self._config.get("GREASE_DEDUP_POOL",
"150")):
self._logger.debug("Waiting for DeDup Threads to Complete",
verbose=True)
continue
if not isinstance(source[source_pointer], dict):
self._logger.warning(
'DeDuplification Received NON-DICT from source: [{0}] Type: [{1}] got: [{2}]'
.format(source_name, str(type(source[source_pointer])),
str(source[source_pointer])))
source_pointer += 1
continue
if source_pointer is 0:
pid_num = 1
else:
pid_num = source_pointer
proc = threading.Thread(
target=self.process_obj,
args=(
self.collection_name,
self._logger,
source_name,
source_max,
source_pointer,
field_set,
source[source_pointer],
final,
strength,
),
name="GREASE DEDUPLICATION THREAD [{0}/{1}]".format(
pid_num, source_max))
proc.daemon = True
proc.start()
threads.append(proc)
source_pointer += 1
self._logger.debug(
"All source objects have been threaded for processing",
verbose=True)
while len(threads) > 0:
self._logger.debug("Current DeDuplication Threads [{0}]".format(
len(threads)),
verbose=True)
threads_final = []
for thread in threads:
if thread.isAlive():
threads_final.append(thread)
threads = threads_final
self._logger.debug("Remaining DeDuplication Threads [{0}]".format(
len(threads)),
verbose=True)
# create the auto del index if it doesnt already exist
self._collection.create_index([('expiry', 1),
('expireAfterSeconds', 1)])
self._collection.create_index([('max_expiry', 1),
('expireAfterSeconds', 1)])
if len(final) == 1:
self._logger.debug("DEDUPLICATION COMPLETE::REMAINING OBJECTS [1]")
return final
else:
remaining = len(final) - 1
self._logger.debug(
"DEDUPLICATION COMPLETE::REMAINING OBJECTS [{0}]".format(
remaining))
return final
@staticmethod
def process_obj(collection_name,
logger,
source_name,
source_max,
source_pointer,
field_set,
source_obj,
final,
strength=None):
# first thing try to find the object level hash
mongo_connection = MongoConnection()
client = mongo_connection.client()
db = client.get_database(name=os.getenv('GREASE_MONGO_DB', 'grease'),
write_concern=pymongo.WriteConcern(w=0))
collection = db.get_collection(name=collection_name)
hash_obj = collection.find_one(
{'hash': SourceDeDuplify.generate_hash(source_obj)})
if not hash_obj:
logger.debug(
"Failed To Locate Type1 Match, Performing Type2 Search Match",
True)
# Globally unique hash for request
# create a completely new document hash and all the field set hashes
collection.insert_one({
'expiry':
SourceDeDuplify.generate_expiry_time(),
'max_expiry':
SourceDeDuplify.generate_max_expiry_time(),
'source':
str(source_name),
'score':
1,
'hash':
SourceDeDuplify.generate_hash(source_obj),
'type':
1
})
# Next start field level processing
# first check if our fields are limited
if len(field_set) < 1:
# All fields need to be considered for de-dup
fields = source_obj.keys()
else:
# only registered fields
fields = field_set
# now lets get the composite score
composite_score = SourceDeDuplify.get_field_score(
collection, logger, source_name, source_obj, fields)
if source_pointer is 0:
compo_spot = 1
else:
compo_spot = source_pointer
logger.debug("DEDUPLICATION COMPOSITE SCORE [" + str(compo_spot) +
"/" + str(source_max) + "]: " + str(composite_score))
# now lets observe to see if we have a 'unique' source
if strength is None:
composite_score_limit = float(
os.getenv('GREASE_DEDUP_SCORE', 85))
else:
if isinstance(strength, int) or isinstance(strength, float):
logger.debug("Global DeDuplication strength override",
verbose=True)
composite_score_limit = float(strength)
else:
composite_score_limit = float(
os.getenv('GREASE_DEDUP_SCORE', 85))
if composite_score < composite_score_limit:
# look at that its time to add it to the final list
logger.debug("Type2 ruled Unique adding to final result", True)
final.append(source_obj)
else:
# we have a duplicate source document
# increase the counter and expiry and move on (DROP)
logger.debug("Type1 match found, dropping", True)
if 'max_expiry' in hash_obj:
update_statement = {
"$set": {
'score': int(hash_obj['score']) + 1,
'expiry': SourceDeDuplify.generate_expiry_time()
}
}
else:
update_statement = {
"$set": {
'score': int(hash_obj['score']) + 1,
'expiry': SourceDeDuplify.generate_expiry_time(),
'max_expiry':
SourceDeDuplify.generate_max_expiry_time()
}
}
collection.update_one({'_id': hash_obj['_id']}, update_statement)
mongo_connection.client().close()
@staticmethod
def get_field_score(collection, logger, source_name, document, field_set):
# type: (Collection, Logging.Logger, str, dict, list) -> float
# This function does a field by field assessment of a source
# Woo is what I said too
# first lets create our array of field scores
score_list = []
for field in field_set:
if field in document:
# unicode check because idk even man + Sha256 hash if too long
if not document[field]:
document[field] = ''
if isinstance(document[field], unicode):
check_document = {
'source': str(source_name),
'field': str(field),
'value': document[field].encode('utf-8')
}
else:
check_document = {
'source': str(source_name),
'field': str(field),
'value': str(document[field]).encode('utf-8')
}
# lets only observe fields in our list
field_obj = collection.find_one(
{'hash': SourceDeDuplify.generate_hash(check_document)})
if not field_obj:
logger.debug(
"Failed To Locate Type2 Match, Performing Type2 Search",
True)
# globally unique field->value pair
# alright lets start the search for all fields of its name
field_probability_list = []
for record in collection.find({'source': str(source_name), 'field': field})\
.sort('score', pymongo.ASCENDING) \
.limit(int(os.getenv('GREASE_DEDUP_INSPECTION_STRENGTH', 100))):
# Now we are looping through our fields
if SourceDeDuplify.compare_strings(
record['value'],
check_document['value']) > .95:
# we found a VERY strong match
score_list.append(
1 - SourceDeDuplify.compare_strings(
record['value'], check_document['value']))
# leave the for loop since we found a highly probable match
break
else:
field_probability_list.append(
SourceDeDuplify.compare_strings(
record['value'], check_document['value']))
logger.debug(
"Failed To Location Type1 Match, Performing Type2 Search",
True)
# lets record it in the 'brain' to remember it
check_document['hash'] = SourceDeDuplify.generate_hash(
check_document)
check_document['score'] = 1
check_document[
'expiry'] = SourceDeDuplify.generate_expiry_time()
check_document[
'max_expiry'] = SourceDeDuplify.generate_max_expiry_time(
)
check_document['type'] = 2
collection.insert_one(check_document)
# now lets either choose the highest probable match we found or 0 being a completely globally
# unique value (No matches found in the above loop
if len(field_probability_list) > 0:
probability_average = float(
sum(field_probability_list) /
len(field_probability_list))
logger.debug(
"Field Probability Average: source [{0}] field [{1}] is [{2}]"
.format(source_name, field,
probability_average), True)
score_list.append(probability_average)
else:
score_list.append(100)
else:
logger.debug("Found Type2 Match! Dropping", True)
# exact match we can bug out
if 'max_expiry' in field_obj:
update_statement = {
"$set": {
'score': int(field_obj['score']) + 1,
'expiry':
SourceDeDuplify.generate_expiry_time()
}
}
else:
update_statement = {
"$set": {
'score':
int(field_obj['score']) + 1,
'expiry':
SourceDeDuplify.generate_expiry_time(),
'max_expiry':
SourceDeDuplify.generate_max_expiry_time()
}
}
collection.update_one({'_id': field_obj['_id']},
update_statement)
score_list.append(0)
# finally return our aggregate field score
if len(score_list) is 0:
return 0
return sum(score_list) / float(len(score_list))
@staticmethod
def generate_hash(document):
# type: (dict) -> str
return hashlib.sha256(json.dumps(document)).hexdigest()
@staticmethod
def generate_expiry_time():
return datetime.datetime.utcnow() + datetime.timedelta(
hours=int(os.getenv('GREASE_DEDUP_TIMEOUT', 12)))
@staticmethod
def generate_max_expiry_time():
return datetime.datetime.utcnow() + datetime.timedelta(
weeks=int(os.getenv('GREASE_DEDUP_MAX_TIMEOUT', 1)))
@staticmethod
def compare_strings(constant, variable):
# type: (str, str) -> float
return SequenceMatcher(constant, variable).ratio()
class DaemonRouter(GreaseRouter.Router):
"""
Daemon process routing for GREASE
"""
__author__ = "James E. Bell Jr"
__version__ = "1.0"
_config = Configuration()
_runs = 0
_throttle_tick = 0
_job_completed_queue = []
_current_real_second = datetime.now().second
_current_run_second = datetime.now().second
_log = Logger()
_process = None
_importer = Importer(_log)
_job_metadata = {'normal': 0, 'persistent': 0}
_alchemyConnection = SQLAlchemyConnection(_config)
_ioc = Grease()
_ContextMgr = []
def __init__(self):
GreaseRouter.Router.__init__(self)
if len(self._config.identity) <= 0:
# ensure we won't run without proper registration
print("ERR::Unregistered to Database!")
self._log.error("Registration not found")
sys.exit(1)
@staticmethod
def entry_point():
"""
Application Entry point
:return: void
"""
router = DaemonRouter()
router.gateway()
def gateway(self):
if len(self.args) >= 1:
if self._config.op_name == 'nt':
self.set_process(Daemon.WindowsService(sys.argv, self))
else:
self.set_process(Daemon.UnixDaemon(self))
if self.args[0] == 'start':
self._log.debug("Starting Daemon")
self.get_process().start()
elif self.args[0] == 'restart':
self._log.debug("Restarting Daemon")
self.get_process().restart()
elif self.args[0] == 'stop':
self._log.debug("Stopping Daemon")
self.get_process().stop()
else:
self.bad_exit(
"Invalid Command To Daemon expected [start,stop,restart]",
2)
else:
self.bad_exit(
"Command not given to daemon! Expected: [start,stop,restart]",
1)
def main(self):
"""
Main Daemon Method
:return: void
"""
# Job Execution
self._log.debug("PROCESS STARTUP", True)
# initial rc value
rc = "Garbage"
# All programs are just loops
if self._config.get('GREASE_EXECUTE_LINEAR'):
self._log.debug("LINEAR EXECUTION MODE DETECTED")
while True:
# Windows Signal Catching
if self._config.op_name == 'nt':
if not rc != win32event.WAIT_OBJECT_0:
self._log.debug(
"Windows Kill Signal Detected! Closing GREASE")
break
# Continue Processing
# Process Throttling
if self._config.get('GREASE_THROTTLE'):
if int(self.get_throttle_tick()) > int(
str(self._config.get('GREASE_THROTTLE'))):
# prevent more than 1000 loops per second by default
# check time
self.log_message_once_a_second("Throttle reached", -11)
self.have_we_moved_forward_in_time()
continue
# Job Processing
if self._config.get('GREASE_EXECUTE_LINEAR'):
self.process_queue_standard()
else:
self.process_queue_threaded()
# Final Clean Up
self.inc_runs()
self.inc_throttle_tick()
self.have_we_moved_forward_in_time()
# After all this check for new windows services
if os.name == 'nt':
# Block .5ms to listen for exit sig
rc = win32event.WaitForSingleObject(
self.get_process().hWaitStop, 50)
def log_message_once_a_second(self, message, queue_id):
# type: (str, int) -> bool
# have we moved forward since the last second
if self.have_we_moved_forward_in_time():
# if we have we can log since the log does not have a record for this second
self._log.debug(message)
# We also ensure we record we already logged for zero jobs to process this second
self.add_job_to_completed_queue(queue_id)
return True
else:
# we have not moved forward in time
# have we logged for this second
if not self.has_job_run(queue_id):
# We have not logged for this second so lets do that now
self._log.debug(message)
# record that we logged for this second
self.add_job_to_completed_queue(queue_id)
return True
else:
return False
def process_queue_standard(self):
# type: () -> bool
job_queue = self.get_assigned_jobs()
if len(job_queue) is 0:
self.log_message_once_a_second(
"Total Jobs To Process: [0] Current Runs: [{0}]".format(
self.get_runs()), -1)
else:
# We have some jobs to process
if self._job_metadata['normal'] > 0:
# if we have any normal jobs lets log
self._log.debug(
"Total Jobs To Process: [{0}] Current Runs: [{1}]".format(
self._job_metadata['normal'], self.get_runs()))
else:
# we only have persistent jobs to process
self.log_message_once_a_second(
"Total Jobs To Process: [{0}] Current Runs: [{1}]".format(
len(job_queue), self.get_runs()), 0)
# now lets loop through the job schedule
for job in job_queue:
# start class up
command = self._importer.load(job['module'], job['command'])
# ensure we got back the correct type
if not command:
self._log.error(
"Failed To Load Command [{0}] of [{1}]".format(
job['command'], job['module']),
hipchat=True)
del command
continue
if not isinstance(command, GreaseDaemonCommand):
self._log.error(
"Instance created was not of type GreaseDaemonCommand",
hipchat=True)
del command
continue
if not job['persistent']:
# This is an on-demand job
# we just need to execute it
self.mark_job_in_progress(job['id'])
self._log.debug(
"Preparing to execute on-demand job [{0}]".format(
job['id']), True)
command.attempt_execution(job['id'], job['additional'])
else:
# This is a persistent job
if self.has_job_run(job['id']):
# Job Already Executed
continue
else:
if job['tick'] is self.get_current_run_second():
self._log.debug(
"Preparing to execute persistent job [{0}]".
format(job['id']), True)
command.attempt_execution(job['id'],
job['additional'])
self.add_job_to_completed_queue(job['id'])
else:
# continue because we are not in the tick required
continue
# Report Telemetry
self._ioc.run_daemon_telemetry(command)
if command.get_exe_state()['result']:
# job success
if job['persistent']:
self._log.debug(
"Persistent Job Successful [{0}]".format(
job['id']))
else:
self._log.debug(
"On-Demand Job Successful [{0}]".format(job['id']))
self.mark_job_complete(job['id'])
else:
# job failed
if job['persistent']:
self._log.debug("Persistent Job Failed [{0}]".format(
job['id']))
else:
self._log.debug("On-Demand Job Failed [{0}]".format(
job['id']))
self.mark_job_failure(job['id'], job['failures'])
command.__del__()
del command
return True
def process_queue_threaded(self):
# type: () -> bool
self.thread_check()
job_queue = self.get_assigned_jobs()
if len(job_queue) is 0:
# have we moved forward since the last second
self.log_message_once_a_second(
"Total Jobs To Process: [0] Current Runs: [{0}]".format(
self.get_runs()), -1)
else:
if len(self._ContextMgr) >= int(
self._config.get('GREASE_THREAD_MAX', '15')):
self.log_message_once_a_second(
"Thread Maximum Reached::Current Runs: [{0}]".format(
self.get_runs()), -10)
return True
# We have some jobs to process
if self._job_metadata['normal'] is 0:
# we only have persistent jobs to process
self.log_message_once_a_second(
"Total Jobs To Process: [{0}] Current Runs: [{1}]".format(
len(job_queue), self.get_runs()), 0)
else:
self._log.debug(
"Total Jobs To Process: [0] Current Runs: [{0}]".format(
self.get_runs()), -1)
# now lets loop through the job schedule
for job in job_queue:
# start class up
command = self._importer.load(job['module'], job['command'])
# ensure we got back the correct type
if not command:
self._log.error(
"Failed To Load Command [{0}] of [{1}]".format(
job['command'], job['module']),
hipchat=True)
del command
continue
if not isinstance(command, GreaseDaemonCommand):
self._log.error(
"Instance created was not of type GreaseDaemonCommand",
hipchat=True)
del command
continue
if not job['persistent']:
# This is an on-demand job
# we just need to execute it
self._log.debug(
"Passing on-demand job [{0}] to thread manager".format(
job['id']), True)
self.thread_execute(command, job['id'], job['additional'],
False, job['failures'])
else:
# This is a persistent job
if self.has_job_run(job['id']):
# Job Already Executed
command.__del__()
del command
continue
else:
if job['tick'] is self.get_current_run_second():
self._log.debug(
"Passing persistent job [{0}] to thread manager"
.format(job['id']), True)
self.thread_execute(command, job['id'],
job['additional'], True)
self.add_job_to_completed_queue(job['id'])
else:
# continue because we are not in the tick required
command.__del__()
del command
continue
self.thread_check()
return True
def thread_execute(self, command, cid, additional, persistent, failures=0):
# type: (GreaseDaemonCommand, int, dict, bool, int) -> None
# first ensure the command ID isn't already running
process_running = False
for item in self._ContextMgr:
if item[2] == cid:
process_running = True
break
if process_running:
# if it is return out
self._log.debug(
"Bailing on job [{0}], already executing".format(cid), True)
return None
# start process
proc = threading.Thread(
target=command.attempt_execution,
args=(cid, additional),
name="GREASE EXECUTION THREAD::CID [{0}]".format(cid))
# set for background
proc.daemon = True
if persistent:
self._log.debug(
"Beginning persistent execution of job [{0}] on thread".format(
cid), True)
else:
self.mark_job_in_progress(cid)
self._log.debug(
"Beginning on-demand execution of job [{0}] on thread".format(
cid), True)
# start
proc.start()
# add command to pool
self._ContextMgr.append([command, proc, cid, persistent, failures])
return None
def thread_check(self):
final = []
# Check for tread completion else add back to list
for command in self._ContextMgr:
if command[1].isAlive():
final.append(command)
else:
self._log.info("Job completed [{0}]".format(command[2]), True)
self.record_telemetry(command[0], command[2], command[4],
command[3])
self._ContextMgr = final
return
def record_telemetry(self, command, cid, failures, persistent):
# type: (GreaseDaemonCommand, int, int, bool) -> None
# Report Telemetry
command.set_exe_state('command_id', cid)
self._ioc.run_daemon_telemetry(command)
if command.get_exe_state()['result']:
# job success
if persistent:
self._log.debug("Persistent Job Successful [{0}]".format(cid))
else:
self._log.debug("On-Demand Job Successful [{0}]".format(cid))
self.mark_job_complete(cid)
else:
# job failed
if persistent:
self._log.debug("Persistent Job Failed [{0}]".format(cid))
else:
self._log.debug("On-Demand Job Failed [{0}]".format(cid))
self.mark_job_failure(cid, failures)
command.__del__()
del command
pass
def mark_job_in_progress(self, job_id):
"""
sets job as in progress
:param job_id: int
:return: bool
"""
stmt = update(JobQueue).where(JobQueue.id == job_id).values(
in_progress=True, completed=False)
self._alchemyConnection.get_session().execute(stmt)
self._alchemyConnection.get_session().commit()
return True
def mark_job_complete(self, job_id):
"""
Complete a successful job
:param job_id: int
:return: bool
"""
stmt = update(JobQueue).where(JobQueue.id == job_id).values(
in_progress=False, completed=True, complete_time=datetime.now())
self._alchemyConnection.get_session().execute(stmt)
self._alchemyConnection.get_session().commit()
return True
def mark_job_failure(self, job_id, current_failures):
"""
Fail a job
:param job_id: int
:param current_failures: int
:return: bool
"""
stmt = update(JobQueue)\
.where(JobQueue.id == job_id)\
.values(in_progress=False, completed=False, complete_time=None, failures=current_failures + 1)
self._alchemyConnection.get_session().execute(stmt)
self._alchemyConnection.get_session().commit()
return True
def get_assigned_jobs(self):
"""
gets current job assignment
:return: list
"""
# type: () -> list
# reset job queue metadata
self._job_metadata['normal'] = 0
self._job_metadata['persistent'] = 0
# create final result
final = []
# first find normal jobs
result = self._alchemyConnection\
.get_session()\
.query(JobQueue, JobConfig)\
.filter(JobQueue.host_name == self._config.node_db_id())\
.filter(JobQueue.job_id == JobConfig.id) \
.filter(or_(and_(JobQueue.in_progress == False, JobQueue.completed == False), JobQueue.in_progress == True)) \
.filter(JobQueue.failures < 6)\
.all()
if not result:
# No Jobs Found
self._job_metadata['normal'] = 0
else:
# Walk the job list
for job in result:
self._job_metadata['normal'] += 1
final.append({
'id': job.JobQueue.id,
'module': job.JobConfig.command_module,
'command': job.JobConfig.command_name,
'request_time': datetime.utcnow(),
'additional': job.JobQueue.additional,
'tick': job.JobConfig.tick,
'persistent': False,
'failures': job.JobQueue.failures
})
# Now search for persistent jobs
result = self._alchemyConnection\
.get_session()\
.query(PersistentJobs, JobConfig)\
.filter(PersistentJobs.server_id == self._config.node_db_id())\
.filter(PersistentJobs.command == JobConfig.id)\
.filter(PersistentJobs.enabled == True)\
.all()
if not result:
# No Jobs Found
self._job_metadata['persistent'] = 0
else:
# Walk the job list
for job in result:
self._job_metadata['persistent'] += 1
final.append({
'id': job.PersistentJobs.id,
'module': job.JobConfig.command_module,
'command': job.JobConfig.command_name,
'request_time': datetime.utcnow(),
'additional': job.PersistentJobs.additional,
'tick': job.JobConfig.tick,
'persistent': True,
'failures': 0
})
return final
# Class Property getter/setters/methods
# run counter
def get_runs(self):
"""
returns int of amount of loops
:return: int
"""
# type: () -> int
return int(self._runs)
def inc_runs(self):
"""
increments run count
:return: None
"""
# type: () -> bool
self._runs += 1
return True
def reset_runs(self):
"""
resets the run counter to 0
:return: bool
"""
# type: () -> bool
self._runs = 0
return True
# Job Completed Queue
def add_job_to_completed_queue(self, job_ib):
"""
Adds Job to queue so we don't run the job again
:param job_ib: int
:return: bool
"""
# type: int -> bool
if int(job_ib) not in self._job_completed_queue:
self._log.debug("Job Executed This Second [{0}]".format(job_ib),
True)
self._job_completed_queue.append(int(job_ib))
return True
else:
return False
def has_job_run(self, job_id):
"""
Determines if the job ID has run during the current cycle
:param job_id: int
:return: bool
"""
# type: int -> bool
if int(job_id) in self._job_completed_queue:
return True
else:
return False
def reset_completed_job_queue(self):
"""
clears job run queue
:return: bool
"""
# type: () -> bool
self._log.debug("Completed Per-Second Queue Cleared", True)
self._job_completed_queue = []
# throttle tick
def get_throttle_tick(self):
"""
returns how many runs in this second
:return: int
"""
# type: () -> int
return int(self._throttle_tick)
def inc_throttle_tick(self):
"""
increases throttle tick by 1
:return: bool
"""
# type: () -> bool
self._throttle_tick += 1
return True
def reset_throttle_tick(self):
"""
resets throttle tick to 0
:return: bool
"""
# type: () -> bool
self._throttle_tick = 0
return True
# Process Controller
def set_process(self, process):
"""
sets the process handler
:param process: Daemon
:return: None
"""
self._process = process
return None
def get_process(self):
"""
Returns _process property
:return: Daemon/None
"""
return self._process
# time operators
@staticmethod
def get_current_real_second():
"""
Gets current second
:return: int
"""
# type: () -> int
return datetime.now().second
def get_current_run_second(self):
"""
Gets the current observed second
:return: int
"""
# type: () -> int
return int(self._current_run_second)
def set_current_run_second(self, sec):
"""
Sets current observed second
:param sec: int
:return: None
"""
# type: int -> None
self._current_run_second = int(sec)
def have_we_moved_forward_in_time(self):
"""
Answers the question "have we moved forward in time?"
so we reset our counters and return true else false
:return: bool
"""
# type: () -> bool
if self.get_current_run_second() == self.get_current_real_second():
return False
else:
self._log.debug("Time has moved forward! Restoring Context", True)
self.set_current_run_second(self.get_current_real_second())
self.reset_completed_job_queue()
self.reset_throttle_tick()
return True
class ScanOnConfig(GreaseDaemonCommand):
def __init__(self):
super(ScanOnConfig, self).__init__()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
self._scanner_config = DetectorConfiguration.ConfigurationLoader()
self._context = '{}'
self._source_data = {}
self._importer = Importer(self._ioc.message())
self._duplification_filter = SourceDeDuplify(self._ioc.message())
def __del__(self):
super(ScanOnConfig, self).__del__()
self._sql.get_session().close()
def get_source_data(self):
# type: () -> dict
return self._source_data
def set_source_data(self, data):
# type: (dict) -> None
self._source_data = data
def execute(self, context='{}'):
# engage scanning
self.scan()
# clear up this
del self._duplification_filter
return True
def scan(self):
# engage scanners scotty
for scanner in self._scanner_config.get_scanners():
self._ioc.message().debug(
"STARTING SCANNER [{0}]".format(str(scanner)), True)
# Ensure if we are focused only to process our source
if os.getenv('GREASE_SOURCE_FOCUS'):
# we have one, ensure its on the list of configured scanners
if str(os.getenv('GREASE_SOURCE_FOCUS')) != str(scanner):
# It does not match, continue
self._ioc.message().info(
"Scanner skipped because not focus: [" + str(scanner) +
"] searching for [" +
str(os.getenv('GREASE_SOURCE_FOCUS')) + "]")
continue
# lets loop through our scanners/detectors to execute the parsing
# try to load the scanner we want
parser = self._importer.load(os.getenv('GREASE_SOURCES_PKG', ''),
scanner, True)
# type check that bad boy to ensure sanity
if isinstance(parser, BaseSource):
# if we got back a valid source lets parse that sucker
self._ioc.message().debug(
"PARSING SOURCE [{0}]".format(str(scanner)), True)
parser.parse_source(
self._scanner_config.get_scanner_config(scanner))
# lets get the results of the parse
# here we run our de-duplication logic
self._ioc.message().debug(
"PASSING RESULT TO DEDUPLICATION ENGINE [{0}]".format(
str(scanner)), True)
source = self._duplification_filter.create_unique_source(
scanner, parser.duplicate_check_fields(),
list(parser.get_records()))
self._ioc.message().debug(
"ATTEMPTING DETECTION SCHEDULING [{0}]".format(
str(scanner)), True)
if self._schedule_detection(source, scanner):
self._ioc.message().info(
"Detector job scheduled from scanner: [" +
str(scanner) + "]")
else:
self._ioc.message().error(
"Failed to schedule source detection for [" +
str(scanner) + "]",
hipchat=True)
del parser
else:
# else something went haywire pls feel free to fix your config
self._ioc.message().error(
"Invalid Scanner In Configurations: [" + str(scanner) +
"]",
hipchat=True)
def _schedule_detection(self, sources, scanner):
# type: (dict, str) -> bool
# first lets get applicable servers to run detectors
# lets only get the least assigned server so we can round robin
result = self._sql.get_session()\
.query(JobServers)\
.filter(JobServers.detector == True)\
.filter(JobServers.active == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if not result:
self._ioc.message().error(
"Failed to find detection server! dropping scan!",
hipchat=True)
return False
else:
server = result.id
# Now lets insert the sources for the determined server to work
source = SourceData(source_data=sources,
source_server=self._config.node_db_id(),
detection_server=server,
scanner=scanner,
created_time=datetime.utcnow())
self._sql.get_session().add(source)
self._sql.get_session().commit()
# finally lets ensure we account for the fact our server is going to do
# that job and increment the assignment counter
stmt = update(JobServers).where(JobServers.id == server).values(
jobs_assigned=result.jobs_assigned + 1)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"SOURCE SCHEDULED FOR DETECTION [{0}] TO SERVER [{1}]".format(
str(scanner), str(server)), True)
return True
class LaunchCtl(GreaseDaemonCommand):
_config = Configuration()
_sql = SQLAlchemyConnection(_config)
def __init__(self):
super(LaunchCtl, self).__init__()
self.purpose = "Register machine with Job Control Database"
def __del__(self):
super(LaunchCtl, self).__del__()
self._sql.get_session().close()
def execute(self, context='{}'):
if len(sys.argv) >= 4:
action = str(sys.argv[3])
else:
action = ''
if action == 'register':
return bool(self._action_register())
elif action == 'kill-server':
return bool(self._action_cull_server())
elif action == 'revive-server':
return bool(self._action_restore_server())
elif action == 'list-pjobs':
return bool(self._action_list_persistent_jobs())
elif action == 'list-jobs':
return bool(self._action_list_job_schedule())
elif action == 'assign-task':
return bool(self._action_assign_task())
elif action == 'remove-task':
return bool(self._action_remove_task())
elif action == 'enable-detection':
return bool(self._action_enable_detection())
elif action == 'enable-scheduling':
return bool(self._action_enable_scheduling())
elif action == 'disable-detection':
return bool(self._action_disable_detection())
elif action == 'disable-scheduling':
return bool(self._action_disable_scheduling())
elif action == 'create-job':
return bool(self._action_create_job())
elif action == 'load-db':
return bool(self._action_load_db())
else:
print("ERR: Invalid Command Expected: ")
print("\tregister")
print("\tkill-server")
print("\trevive-server")
print("\tlist-pjobs")
print("\tlist-jobs")
print("\tassign-task")
print("\tremove-task")
print("\tenable-detection")
print("\tenable-scheduling")
print("\tdisable-detection")
print("\tdisable-scheduling")
print("\tcreate-job")
print("\tload-db")
return True
def _action_register(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
self._ioc.message().warning("Machine Already Registered With Grease Job Control")
print("Machine Already Registered With Grease Job Control")
return True
else:
# we need to register
# first lets generate a new UUID
uid = uuid.uuid4()
# lets see if we have been provided an execution env
if len(sys.argv) >= 5:
exe_env = str(sys.argv[4])
else:
exe_env = 'general'
# next lets register with the job control database
server = JobServers(
host_name=str(uid),
execution_environment=exe_env,
active=True,
activation_time=datetime.utcnow()
)
self._sql.get_session().add(server)
self._sql.get_session().commit()
file(self._config.identity_file, 'w').write(str(uid))
return True
def _action_cull_server(self):
# type: () -> bool
if len(sys.argv) >= 5:
server = str(sys.argv[4])
else:
if os.path.isfile(self._config.identity_file):
server = self._config.identity
else:
print("Server has no registration record locally")
return True
# get the server ID
result = self._sql.get_session().query(JobServers)\
.filter(JobServers.host_name == server)\
.first()
if result:
server_id = result.id
instance = Section31()
instance._declare_doctor(server_id)
instance._cull_server(server_id)
return True
else:
print("Job Server Not In Registry")
return True
def _action_restore_server(self):
# type: () -> bool
if len(sys.argv) >= 5:
server = str(sys.argv[4])
else:
if os.path.isfile(self._config.identity_file):
server = self._config.identity
else:
print("Server has no registration record locally")
return True
# get the server ID
result = self._sql.get_session().query(JobServers)\
.filter(JobServers.host_name == server)\
.first()
if result:
server_id = result.id
else:
print("Job Server Not In Registry")
return True
# clear the doctor from the server health table
stmt = update(ServerHealth)\
.where(ServerHealth.server == server_id)\
.values(doctor=None)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
# next reactivate it
stmt = update(JobServers)\
.where(JobServers.id == server_id)\
.values(active=True, activation_time=datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
return True
def _action_list_persistent_jobs(self):
# type: () -> bool
result = self._sql.get_session().query(PersistentJobs, JobConfig)\
.filter(PersistentJobs.command == JobConfig.id)\
.filter(PersistentJobs.enabled == True)\
.filter(PersistentJobs.server_id == self._config.node_db_id())\
.all()
if not result:
print("No Scheduled Jobs on this node")
else:
for job in result:
print(
"\tPackage: [{0}] Job: [{1}] Tick: [{2}] Additional: [{3}]".format(
job.JobConfig.command_module,
job.JobConfig.command_name,
job.JobConfig.tick,
job.PersistentJob.additional
)
)
return True
def _action_list_job_schedule(self):
# type: () -> bool
result = self._sql.get_session().query(JobQueue)\
.filter(JobQueue.completed == False)\
.filter(JobQueue.in_progress == False)\
.filter(JobQueue.host_name == self._config.node_db_id())\
.all()
if not result:
print("No jobs scheduled on this node")
return True
for job in result:
print("Jobs in Queue:")
print("\t Module: [{0}] Command: [{1}] Additional: [{2}]".format(
job.JobConfig.command_module,
job.JobConfig.command_name,
job.JobQueue.additional
))
return True
def _action_assign_task(self):
# type: () -> bool
if len(sys.argv) >= 5:
new_task = str(sys.argv[4])
else:
print("Please provide a command to schedule to node")
return True
result = self._sql.get_session().query(JobConfig)\
.filter(JobConfig.command_name == new_task)\
.first()
if not result:
print("Command not found! Available Commands:")
result = self._sql.get_session().query(JobConfig).all()
if not result:
print("NO JOBS CONFIGURED IN DB")
else:
for job in result:
print("{0}".format(job.command_name))
return True
else:
pJob = PersistentJobs(
server_id=self._config.node_db_id(),
command=result.id,
additional={},
enabled=True
)
self._sql.get_session().add(pJob)
self._sql.get_session().commit()
print("TASK ASSIGNED")
return True
def _action_remove_task(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
server = self._config.node_db_id()
else:
print("Server has no registration record locally")
return True
if len(sys.argv) >= 5:
new_task = str(sys.argv[4])
result = self._sql.get_session().query(JobConfig).filter(JobConfig.command_name == new_task).first()
if not result:
print("Failed to find job in configuration tables")
return True
else:
stmt = update(PersistentJobs)\
.where(and_(PersistentJobs.server_id == server, PersistentJobs.command == result.id))\
.values(enabled=False)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
print("TASK UNASSIGNED")
return True
def _action_enable_detection(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
server = self._config.identity
stmt = update(JobServers).where(JobServers.host_name == server).values(detector=True)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
print("DETECTION ENABLED")
return True
else:
print("ERR: SERVER UNREGISTERED")
return False
def _action_enable_scheduling(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
server = self._config.identity
stmt = update(JobServers).where(JobServers.host_name == server).values(scheduler=True)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
print("SCHEDULING ENABLED")
return True
else:
print("ERR: SERVER UNREGISTERED")
return False
def _action_disable_detection(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
server = self._config.identity
stmt = update(JobServers).where(JobServers.host_name == server).values(detector=False)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
print("DETECTION DISABLED")
return True
else:
print("ERR: SERVER UNREGISTERED")
return False
def _action_disable_scheduling(self):
# type: () -> bool
if os.path.isfile(self._config.identity_file):
server = self._config.identity
stmt = update(JobServers).where(JobServers.host_name == server).values(scheduler=False)
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
print("SCHEDULING DISABLED")
return True
else:
print("ERR: SERVER UNREGISTERED")
return False
def _action_create_job(self):
# type: () -> bool
if len(sys.argv) >= 6:
new_task_module = str(sys.argv[4])
new_task_class = str(sys.argv[5])
# ensure a job doesn't exist
result = self._sql.get_session().query(JobConfig)\
.filter(JobConfig.command_module == new_task_module)\
.filter(JobConfig.command_name == new_task_class)\
.all()
if result:
print("Job already exists")
return True
NewJob = JobConfig(
command_module=new_task_module,
command_name=new_task_class
)
self._sql.get_session().add(NewJob)
self._sql.get_session().commit()
print("Job Created")
return True
else:
print("ERR: INVALID SUBCOMMAND::MUST PASS MODULE AND CLASS NAME")
return False
def _action_load_db(self):
print("LOADING DB")
RDBMSTypes.__main__()
if not os.path.isfile(self._config.identity_file):
self._action_register()
return True
def __init__(self):
super(Scheduler, self).__init__()
self._scanner_config = DetectorConfiguration.ConfigurationLoader()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
class Scheduler(GreaseDaemonCommand):
def __init__(self):
super(Scheduler, self).__init__()
self._scanner_config = DetectorConfiguration.ConfigurationLoader()
self._config = Configuration()
self._sql = SQLAlchemyConnection(self._config)
def __del__(self):
super(Scheduler, self).__del__()
self._sql.get_session().close()
def execute(self, context='{}'):
# Lets go get the jobs needing to be scheduled by this server
result = self._sql.get_session().query(SourceData, JobServers)\
.filter(SourceData.scheduling_server == JobServers.id)\
.filter(JobServers.id == self._config.node_db_id())\
.filter(SourceData.detection_start_time != None)\
.filter(SourceData.detection_end_time != None)\
.filter(SourceData.detection_complete == True)\
.filter(SourceData.scheduling_start_time == None)\
.filter(SourceData.scheduling_end_time == None)\
.filter(SourceData.scheduling_complete == False)\
.limit(15)\
.all()
if not result:
self._ioc.message().debug("No Sources to schedule", True)
return True
else:
for schedule in result:
# lets own this
self._take_ownership(schedule.SourceData.id)
self._schedule_via_sources(schedule.SourceData.source_data)
# lets finish out
self._complete(schedule.SourceData.id)
return True
def _schedule_via_sources(self, sources):
# type: (dict) -> None
for index in sources:
# check to see if we already assigned this source
if 'grease_internal_assigned' in sources[index]:
if sources[index]['grease_internal_assigned']:
# we already successfully did the thing
self._ioc.message().debug(
"SCHEDULING ALREADY PROCEEDED ON INDEX [{0}]".format(
str(index)), True)
continue
else:
# it failed previously
self._ioc.message().warning(
"Record Failed To Be Assigned::Loop Over")
continue
else:
# we have not attempted to schedule this record yet
if len(sources[index]['rule_processing']) > 0:
# rules got processed on this source
for rule_name, rule_results in sources[index][
'rule_processing'].iteritems():
self._ioc.message().debug(
"PROCESSING SOURCE [{0}] FOR RULE [{1}]".format(
str(index), str(rule_name)), True)
if rule_results['status']:
self._ioc.message().debug(
"SOURCE [{0}] RULE [{1}] PASSED DETECTION".
format(str(index), str(rule_name)), True)
# rule was successful time to schedule
# lets go load the rule config
rule_config = self._scanner_config.get_config(
rule_name)
self._ioc.message().debug(
"READING CONFIG FOR RULE [{0}]".format(
str(rule_name)), True)
# setup the execution environment variable
if 'exe_env' in rule_config:
if len(rule_config['exe_env']) > 0:
# if we have a valid string then just set it
exe_env = rule_config['exe_env']
else:
# else they left it blank, default to general
exe_env = 'general'
else:
# they didn't provide one so default to general
exe_env = 'general'
# next lets get the ID
if 'incident_number' in sources[index]:
# Set the incident Number
i_num = sources[index]['incident_number']
elif 'number' in sources[index]:
# Set the events number
i_num = sources[index]['number']
else:
# default to md5 hash of values list to ensure unique ID
i_num = hashlib.sha256(
json.dumps(
sources[index].values())).hexdigest()
# Now lets set the job additional parameters
additional = dict()
if 'parameters' in sources[index][
'rule_processing'][rule_name]:
if isinstance(
sources[index]['rule_processing']
[rule_name]['parameters'], dict):
additional = sources[index][
'rule_processing'][rule_name][
'parameters']
else:
self._ioc.message().warning(
"Parameters were not dictionary for rule: ["
+ str(rule_name) + "]")
# Now lets setup the ticket info
additional['ticket'] = i_num
self._ioc.message().debug(
"PREPARING TO SCHEDULE JOB [{0}] FOR EXECUTION::"
"EXE_ENV: [{1}] ADDITIONAL: [{2}] ticket: [{3}]"
.format(str(rule_config['job']), str(exe_env),
str(additional), str(i_num)), True)
if self._assign(
rule_config['job'], exe_env,
str(
self._config.get(
'SCHEDULE_PKG',
'localhost_generic')), str(i_num),
additional):
# we successfully assigned the ticket
self._ioc.message().debug(
"JOB EXECUTION SCHEDULING SUCCESSFUL [{0}] FOR RULE [{1}]"
.format(str(index), str(rule_name)), True)
sources[index][
'grease_internal_assigned'] = True
continue
else:
# we failed to assign the ticket
self._ioc.message().warning(
"JOB EXECUTION SCHEDULING FAILED [{0}] FOR RULE [{1}]"
.format(str(index), str(rule_name)), True)
sources[index][
'grease_internal_assigned'] = False
continue
else:
# rule failed fine then
self._ioc.message().debug(
"RULE FAILED FOR SOURCE [{0}] RULE [{1}]".
format(str(index), str(rule_name)), True)
continue
else:
# No rules were processed on this source
self._ioc.message().debug(
"RULE PROCESSING WAS EMPTY FOR SOURCE [{0}]".format(
str(index)), True)
continue
def _assign(self, job, exec_env, package, ticket, additional=dict):
# type: (str, str, str, str, dict) -> bool
# check to ensure this ticket isn't already on the schedule
self._ioc.message().debug(
"Job Scheduling Starting::Job [{0}] Exec_Env [{1}] Package [{2}] Ticket [{3}] Additional [{4}]"
.format(str(job), str(exec_env), str(package), str(ticket),
str(additional)), True)
if len(ticket) > 0:
self._ioc.message().debug(
"Validating ticket ID not already in Job Queue", True)
result = self._sql.get_session().query(JobQueue)\
.filter(JobQueue.ticket == ticket)\
.filter(or_(and_(JobQueue.in_progress == False, JobQueue.completed == False), JobQueue.in_progress == True))\
.all()
if result:
self._ioc.message().warning(
"Ticket Already in Job Queue for Execution Ticket")
return False
# lets only get the least assigned server so we can round robin
self._ioc.message().debug("Searching for Execution Server", True)
result = self._sql.get_session()\
.query(JobServers)\
.filter(JobServers.execution_environment == exec_env)\
.filter(JobServers.active == True)\
.order_by(JobServers.jobs_assigned)\
.first()
if not result:
self._ioc.message().error(
"No Execution Environments Found For Job: [" + job + "]",
hipchat=True)
return False
server_info = result.id
server_job_count = int(result.jobs_assigned)
self._ioc.message().debug(
"Job Server Selected [{0}] current assignment total [{1}]".format(
server_info, server_job_count), True)
self._ioc.message().debug("Searching for Job Configuration", True)
result = self._sql.get_session().query(JobConfig)\
.filter(JobConfig.command_module == package)\
.filter(JobConfig.command_name == job)\
.first()
if not result:
self._ioc.message().error(
"No Jobs Configured For Requested Job: [" + job +
"] for package: [" + package + "]",
hipchat=True)
return False
job_id = result.id
# Proceed to schedule
self._ioc.message().debug(
"Creating new job in queue for job [{0}] on node [{1}]".format(
job_id, server_info), True)
JobToQueue = JobQueue(host_name=server_info,
job_id=job_id,
ticket=ticket,
additional=additional)
self._sql.get_session().add(JobToQueue)
self._sql.get_session().commit()
# that job and increment the assignment counter
self._ioc.message().debug(
"incrementing jobs assigned on server [{0}]".format(server_info),
True)
self._sql.get_session().query(JobServers).filter_by(
id=server_info).update({'jobs_assigned': server_job_count + 1})
self._sql.get_session().commit()
self._ioc.message().debug(
"JOB [{0}] SCHEDULED FOR SERVER [{1}]".format(
str(job_id), str(server_info)), True)
return True
def _take_ownership(self, source_file_id):
# type: (int) -> None
stmt = update(SourceData)\
.where(SourceData.id == source_file_id)\
.values(scheduling_start_time=datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"TAKING OWNERSHIP OF SOURCE [{0}]".format(str(source_file_id), ),
True)
def _complete(self, source_file_id):
# type: (int) -> None
stmt = update(SourceData)\
.where(SourceData.id == source_file_id)\
.values(scheduling_complete=True, scheduling_end_time=datetime.utcnow())
self._sql.get_session().execute(stmt)
self._sql.get_session().commit()
self._ioc.message().debug(
"COMPLETING SOURCE [{0}]".format(str(source_file_id), ), True)
class UnixDaemon(GreaseDaemonCommon):
def __init__(self, router):
super(UnixDaemon, self).__init__(router)
self._config = Configuration()
# self._pidfile = '/tmp/grease/grease.pid'
self._pidfile = self._config.grease_dir + self._config.fs_Separator + 'grease.pid'
self._pid = os.getpid()
def daemonize(self):
"""Standard Double Fork"""
# First Fork
try:
pid = os.fork()
if pid > 0:
# Exit Parent Now
sys.exit(0)
except OSError as e:
self._router._log.critical(
"Fork Failure [1]: {0} ({1}) Host: [{2}]".format(
e.errno, e.strerror, self._config.node_identity()))
self._router.bad_exit(
"Fork Failure: {0} ({1})".format(e.errno, e.strerror), 2)
# Decouple from the parent process env data
os.chdir("/")
os.setsid()
os.umask(0)
# Second Fork
try:
pid = os.fork()
if pid > 0:
# Exit Parent Now
sys.exit(0)
except OSError as e:
self._router._log.critical(
"Fork Failure [1]: {0} ({1}) Host: [{2}]".format(
e.errno, e.strerror, self._config.node_identity()))
self._router.bad_exit(
"Fork Failure: {0} ({1})".format(e.errno, e.strerror), 2)
# Finally lets write our pid file
# Register Our Daemon
atexit.register(self.del_pid)
pid = str(os.getpid())
file(self._pidfile, 'w+').write(str(pid))
self._pid = pid
def del_pid(self):
os.remove(str(self._pidfile))
def restart(self):
self.stop()
self.start()
def stop(self):
# Attempt to get lock on PID
try:
pidfile = file(self._pidfile, 'r')
pid = int(pidfile.read().strip())
pidfile.close()
except IOError:
pid = None
# If there was no File / Pid Found
if not pid:
self._router._log.error("Daemon Not Running Cannot Stop")
self._router.bad_exit(
"Daemon Is Not Running Currently, Failed To Stop", 3)
# try to kill the running PID
try:
while 1:
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError as err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self._pidfile):
os.remove(self._pidfile)
else:
self._router._log.exception(err)
self._router.bad_exit(err, 4)
def start(self):
# First check if daemon is already running
try:
pidfile = file(self._pidfile, 'r')
pid = int(pidfile.read().strip())
pidfile.close()
except IOError:
pid = None
# if already in Daemon then ignore else start
if pid:
self._router._log.info("Daemon already running")
print("Daemon Already In Service")
sys.exit(0)
else:
self.daemonize()
self.run()