def deserialize(d):
task = hxtool_scheduler_task(
d['profile_id'],
d['name'],
task_id=d['task_id'],
parent_id=d.get('parent_id', None),
wait_for_parent=d.get('wait_for_parent', True),
start_time=HXAPI.dt_from_str(d['start_time']),
end_time=HXAPI.dt_from_str(d['end_time'])
if d['end_time'] else None,
next_run=HXAPI.dt_from_str(d['next_run'])
if d['next_run'] else None,
enabled=d['enabled'],
immutable=d['immutable'],
stop_on_fail=d['stop_on_fail'],
defer_interval=d['defer_interval'])
task.last_run = HXAPI.dt_from_str(
d['last_run']) if d['last_run'] else None
task.parent_complete = d.get('parent_complete', False)
task.last_run_state = d.get('last_run_state', None)
task.state = d.get('state')
schedule = d.get('schedule', None)
if schedule is dict:
task.set_schedule(**schedule)
task._calculate_next_run()
for s in d['steps']:
# I hate this
step_module = eval(s['module'])
task.add_step(step_module, s['function'], s['args'], s['kwargs'])
return task
def initialize_task_api_sessions(self):
# Loop through background credentials and start the API sessions
profiles = hxtool_global.hxtool_db.profileList()
for profile in profiles:
task_api_credential = hxtool_global.hxtool_db.backgroundProcessorCredentialGet(
profile['profile_id'])
if task_api_credential:
try:
salt = HXAPI.b64(task_api_credential['salt'], True)
iv = HXAPI.b64(task_api_credential['iv'], True)
key = crypt_pbkdf2_hmacsha256(salt, TASK_API_KEY)
decrypted_background_password = crypt_aes(
key,
iv,
task_api_credential['hx_api_encrypted_password'],
decrypt=True)
self._add_task_api_task(
profile['profile_id'], profile['hx_host'],
profile['hx_port'],
task_api_credential['hx_api_username'],
decrypted_background_password)
decrypted_background_password = None
except UnicodeDecodeError:
logger.error(
"Please reset the background credential for {} ({}).".
format(profile['hx_host'], profile['profile_id']))
else:
logger.info("No background credential for {} ({}).".format(
profile['hx_host'], profile['profile_id']))
def oiocCreate(self, iocname, ioc, username):
return self._db_openioc.insert_one({'ioc_id' : str(secure_uuid4()),
'iocname': str(iocname),
'username' : str(username),
'ioc' : str(ioc),
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def initialize_task_api_sessions(self):
# Loop through background credentials and start the API sessions
profiles = hxtool_global.hxtool_db.profileList()
for profile in profiles:
task_api_credential = hxtool_global.hxtool_db.backgroundProcessorCredentialGet(profile['profile_id'])
if task_api_credential:
decrypted_background_password = keyring.get_password("hxtool_{}".format(profile['profile_id']), task_api_credential['hx_api_username'])
# TODO: eventually remove this code once most people are using keyring
if not decrypted_background_password:
logger.info("Background credential for {} is not using keyring, moving it.".format(profile['profile_id']))
try:
salt = HXAPI.b64(task_api_credential['salt'], True)
iv = HXAPI.b64(task_api_credential['iv'], True)
key = crypt_pbkdf2_hmacsha256(salt, TASK_API_KEY)
decrypted_background_password = crypt_aes(key, iv, task_api_credential['hx_api_encrypted_password'], decrypt = True)
keyring.set_password("hxtool_{}".format(profile['profile_id']), task_api_credential['hx_api_username'], decrypted_background_password)
hxtool_db.backgroundProcessorCredentialRemove(profile['profile_id'])
hxtool_db.backgroundProcessorCredentialCreate(profile['profile_id'], task_api_credential['hx_api_username'])
except (UnicodeDecodeError, ValueError):
logger.error("Please reset the background credential for {} ({}).".format(profile['hx_host'], profile['profile_id']))
if decrypted_background_password:
self._add_task_api_task(profile['profile_id'], profile['hx_host'], profile['hx_port'], task_api_credential['hx_api_username'], decrypted_background_password)
decrypted_background_password = None
else:
logger.info("No background credential for {} ({}).".format(profile['hx_host'], profile['profile_id']))
def scriptCreate(self, scriptname, script, username):
return self._db_scripts.insert_one({'script_id' : str(secure_uuid4()),
'scriptname': str(scriptname),
'username' : str(username),
'script' : str(script),
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def serialize(self, include_module_data=True):
r = {
'profile_id': self.profile_id,
'profile_name': self.profile_name,
'task_id': self.task_id,
'name': self.name,
'schedule': self.schedule,
'start_time': HXAPI.dt_to_str(self.start_time),
'end_time':
HXAPI.dt_to_str(self.end_time) if self.end_time else None,
'last_run':
HXAPI.dt_to_str(self.last_run) if self.last_run else None,
'next_run':
HXAPI.dt_to_str(self.next_run) if self.next_run else None,
'enabled': self.enabled,
'immutable': self.immutable,
'stop_on_fail': self.stop_on_fail,
'parent_id': self.parent_id,
'parent_complete': self.parent_complete,
'wait_for_parent': self.wait_for_parent,
'defer_interval': self.defer_interval,
'state': self.state,
'last_run_state': self.last_run_state,
}
if include_module_data:
r['stored_result'] = self.stored_result
r['steps'] = [{
'module': m.__module__,
'function': f,
'args': a,
'kwargs': ka
} for m, f, a, ka in self.steps]
return r
def parse_schedule(request_params):
start_time = None
schedule = None
schedule_type = request_params.get('schedule', None)
if schedule_type:
if schedule_type == "run_at":
start_time = HXAPI.dt_from_str(request_params['run_at_value'])
elif schedule_type == "run_interval":
schedule = {}
interval_value = int(request_params['interval_value'])
interval_unit = request_params['interval_unit']
if interval_unit == "second":
schedule['seconds'] = interval_value
elif interval_unit == "minute":
schedule['minutes'] = interval_value
elif interval_unit == "hour":
schedule['hours'] = interval_value
elif interval_unit == "day":
schedule['days'] = interval_value
elif interval_unit == "week":
schedule['weeks'] = interval_value
elif interval_unit == "month":
schedule['months'] = interval_value
if request_params['interval_start'] == "interval_start_at":
start_time = HXAPI.dt_from_str(
request_params['interval_start_value'])
return (start_time, schedule)
def rotate_task_key(self, hxtool_db, use_legacy_key=False):
if use_legacy_key:
k = TASK_API_KEY
else:
k = self._read_task_api_key()
new_key = crypt_generate_random(32)
# Loop through background credentials and start the API sessions
profiles = hxtool_db.profileList()
for profile in profiles:
task_api_credential = hxtool_db.backgroundProcessorCredentialGet(
profile['profile_id'])
if task_api_credential:
salt = HXAPI.b64(task_api_credential['salt'], True)
iv = HXAPI.b64(task_api_credential['iv'], True)
key = crypt_pbkdf2_hmacsha256(salt, k)
decrypted_background_password = crypt_aes(
key,
iv,
task_api_credential['hx_api_encrypted_password'],
decrypt=True)
hxtool_db.backgroundProcessorCredentialRemove(
profile['profile_id'])
iv, salt, encrypted_password = self._encrypt_task_credential(
new_key, decrypted_background_password)
hxtool_db.backgroundProcessorCredentialCreate(
profile['profile_id'],
task_api_credential['hx_api_username'], HXAPI.b64(iv),
HXAPI.b64(salt), encrypted_password)
decrypted_background_password = None
self._write_task_api_key(new_key)
def taskProfileAdd(self, name, actor, params):
return self._db_taskprofiles.insert_one({'taskprofile_id' : str(secure_uuid4()),
'name': str(name),
'actor' : str(actor),
'params' : params,
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def hostGroupAdd(self, profile_id, name, actor, agent_ids = []):
return self._db_hostgroups.insert_one({'profile_id' : profile_id,
'hostgroup_id' : str(secure_uuid4()),
'name': str(name),
'actor' : str(actor),
'agent_ids' : agent_ids,
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def scriptCreate(self, scriptname, script, username):
with self._lock:
return self._db.table('scripts').insert({'script_id' : str(secure_uuid4()),
'scriptname': str(scriptname),
'username' : str(username),
'script' : str(script),
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def oiocCreate(self, iocname, ioc, username):
with self._lock:
return self._db.table('openioc').insert({'ioc_id' : str(secure_uuid4()),
'iocname': str(iocname),
'username' : str(username),
'ioc' : str(ioc),
'create_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow()),
'update_timestamp' : HXAPI.dt_to_str(datetime.datetime.utcnow())})
def add_task_api_session(self, profile_id, hx_host, hx_port, username,
password):
iv = crypt_generate_random(16)
salt = crypt_generate_random(32)
key = crypt_pbkdf2_hmacsha256(salt, TASK_API_KEY)
encrypted_password = crypt_aes(key, iv, password)
hxtool_global.hxtool_db.backgroundProcessorCredentialCreate(
profile_id, username, HXAPI.b64(iv), HXAPI.b64(salt),
encrypted_password)
encrypted_password = None
self._add_task_api_task(profile_id, hx_host, hx_port, username,
password)
password = None
def add_task_api_session(self, profile_id, hx_host, hx_port, username,
password):
iv = crypt_generate_random(16)
salt = crypt_generate_random(32)
k = self._read_task_api_key()
iv, salt, encrypted_password = self._encrypt_task_credential(
k, password)
hxtool_global.hxtool_db.backgroundProcessorCredentialCreate(
profile_id, username, HXAPI.b64(iv), HXAPI.b64(salt),
encrypted_password)
encrypted_password = None
self._add_task_api_task(profile_id, hx_host, hx_port, username,
password)
password = None
def crypt_aes(key, iv, data, decrypt = False, base64_coding = True):
cipher = AES.new(key, AES.MODE_OFB, iv)
if decrypt:
if base64_coding:
data = HXAPI.b64(data, True)
data = cipher.decrypt(data)
data = unpad(data, 16).decode('utf-8')
return data
else:
data = data.encode('utf-8')
data = pad(data, 16)
data = cipher.encrypt(data)
if base64_coding:
data = HXAPI.b64(data)
return data
def fileListingCreate(self,
profile_id,
username,
bulk_download_eid,
path,
regex,
depth,
display_name,
api_mode=False):
ts = HXAPI.dt_to_str(datetime.datetime.utcnow())
r = self._db_file_listing.insert_one({
'profile_id':
profile_id,
'display_name':
display_name,
'bulk_download_eid':
str(bulk_download_eid),
'username':
username,
'stopped':
False,
'files': [],
'cfg': {
'path': path,
'regex': regex,
'depth': depth,
'api_mode': api_mode
},
'create_timestamp':
ts,
'update_timestamp':
ts
})
return r.inserted_id
def stackJobStop(self, stack_job_eid):
return self._db_stacking.update_one({"_id": ObjectId(stack_job_eid)}, {
"$set": {
"stopped": True,
"update_timestamp": HXAPI.dt_to_str(datetime.datetime.utcnow())
}
})
def multiFileCreate(self,
username,
profile_id,
display_name=None,
file_listing_id=None,
api_mode=False):
r = None
with self._lock:
ts = HXAPI.dt_to_str(datetime.datetime.utcnow())
try:
return self._db.table('multi_file').insert({
'display_name':
display_name or "Unnamed File Request",
'username':
username,
'profile_id':
profile_id,
'files': [],
'stopped':
False,
'api_mode':
api_mode,
'create_timestamp':
ts,
'update_timestamp':
ts,
'file_listing_id':
file_listing_id
})
except:
#TODO: Not sure if the value returns that we'd ever see an exception
if r:
self._db.table('multi_file').remove(eids=[r])
raise
return None
def bulkDownloadCreate(self,
profile_id,
hostset_name=None,
hostset_id=None,
task_profile=None):
r = None
with self._lock:
try:
ts = HXAPI.dt_to_str(datetime.datetime.utcnow())
r = self._db.table('bulk_download').insert({
'profile_id':
profile_id,
'hostset_id':
int(hostset_id),
'hostset_name':
hostset_name,
'hosts': {},
'task_profile':
task_profile,
'stopped':
False,
'complete':
False,
'create_timestamp':
ts,
'update_timestamp':
ts
})
except:
self._db.table('bulk_download').remove(eids=[r])
raise
return r
def sessionUpdate(self, session_id, session_data):
return self._db_session.replace_one(
{"session_id": session_id}, {
'session_id': session_id,
'session_data': dict(session_data),
'update_timestamp': HXAPI.dt_to_str(datetime.datetime.utcnow())
})
def stackJobCreate(self, profile_id, bulk_download_eid, stack_type):
r = None
with self._lock:
ts = HXAPI.dt_to_str(datetime.datetime.utcnow())
try:
r = self._db.table('stacking').insert({
'profile_id':
profile_id,
'bulk_download_eid':
int(bulk_download_eid),
'stopped':
False,
'stack_type':
stack_type,
'hosts': [],
'results': [],
'last_index':
None,
'last_groupby': [],
'create_timestamp':
ts,
'update_timestamp':
ts
})
except:
self._db.table('stacking').remove(eids=[r])
raise
return r
def transform(element): for i in element[list_name]: if i[query_key] == query_value: i[k] = v break if update_timestamp and 'update_timestamp' in element: element['update_timestamp'] = HXAPI.dt_to_str(datetime.datetime.utcnow())
def transform(element): if type(value) is list: element[list_name].extend(value) else: element[list_name].append(value) if update_timestamp and 'update_timestamp' in element: element['update_timestamp'] = HXAPI.dt_to_str(datetime.datetime.utcnow())
def ruleGet(self, rule_id):
with self._lock:
r = self._db.table('rules').get((tinydb.Query()['id'] == rule_id))
if r:
return HXAPI.b64(r['rule'], decode = True, decode_string = True)
else:
return False
def sessionCreate(self, session_id):
return self._db_session.insert_one({
'session_id':
session_id,
'session_data': {},
'update_timestamp':
HXAPI.dt_to_str(datetime.datetime.utcnow())
})
def sessionUpdate(self, session_id, session_data):
with self._lock:
return self._db.table('session').update(
{
'session_data': session_data,
'update_timestamp': HXAPI.dt_to_str(
datetime.datetime.utcnow())
}, (tinydb.Query()['session_id'] == session_id))
def multiFileStop(self, multi_file_id):
with self._lock:
return self._db.table('multi_file').update(
{
'stopped': True,
'update_timestamp': HXAPI.dt_to_str(
datetime.datetime.utcnow())
},
eids=[int(multi_file_id)])
def fileListingStop(self, file_listing_id):
with self._lock:
return self._db.table('file_listing').update(
{
'stopped': True,
'update_timestamp': HXAPI.dt_to_str(
datetime.datetime.utcnow())
},
eids=[int(file_listing_id)])
def stackJobStop(self, stack_job_eid):
with self._lock:
return self._db.table('stacking').update(
{
'stopped': True,
'update_timestamp': HXAPI.dt_to_str(
datetime.datetime.utcnow())
},
eids=[int(stack_job_eid)])
def sessionCreate(self, session_id):
with self._lock:
return self._db.table('session').insert({
'session_id':
session_id,
'session_data': {},
'update_timestamp':
HXAPI.dt_to_str(datetime.datetime.utcnow())
})
