def hash_pastie(self):
if self.pastie_content:
try:
self.md5 = hashlib.md5(self.pastie_content.encode('utf-8')).hexdigest()
logger.debug('Pastie {site} {id} has md5: "{md5}"'.format(site=self.site.name, id=self.id, md5=self.md5))
except Exception as e:
logger.error('Pastie {site} {id} md5 problem: {e}'.format(site=self.site.name, id=self.id, e=e))
def run(self):
while not self.kill_received:
sleep_time = random.randint(self.update_min, self.update_max)
try:
# grabs site from queue
logger.info(
'Downloading list of new pastes from {name}. '
'Will check again in {time} seconds'.format(
name=self.name, time=sleep_time))
# get the list of last pasties, but reverse it
# so we first have the old entries and then the new ones
last_pasties = self.get_last_pasties()
if last_pasties:
for pastie in reversed(last_pasties):
queues[self.name].put(pastie) # add pastie to queue
logger.info("Found {amount} new pasties for site {site}. There are now {qsize} pasties to be downloaded.".format(amount=len(last_pasties),
site=self.name,
qsize=queues[self.name].qsize()))
# catch unknown errors
except Exception as e:
msg = 'Thread for {name} crashed unexpectectly, '\
'recovering...: {e}'.format(name=self.name, e=e)
logger.error(msg)
logger.debug(traceback.format_exc())
time.sleep(sleep_time)
def run(self):
self.db_conn = sqlite3.connect(self.filename)
# create the db if it doesn't exist
self.c = self.db_conn.cursor()
try:
# LATER maybe create a table per site. Lookups will be faster as less text-searching is needed
self.c.execute('''
CREATE TABLE IF NOT EXISTS pasties (
site TEXT,
id TEXT,
md5 TEXT,
url TEXT,
local_path TEXT,
timestamp DATE,
matches TEXT
)''')
self.db_conn.commit()
except sqlite3.DatabaseError as e:
logger.error('Problem with the SQLite database {0}: {1}'.format(self.filename, e))
return None
# loop over the queue
while not self.kill_received:
try:
# grabs pastie from queue
pastie = self.queue.get()
# add the pastie to the DB
self.add_or_update(pastie)
# signals to queue job is done
self.queue.task_done()
# catch unknown errors
except Exception as e:
logger.error("Thread for SQLite crashed unexpectectly, recovering...: {e}".format(e=e))
logger.debug(traceback.format_exc())
def _get_data(key: int, file_type: str, url: str) -> dict:
try:
return_object = None
file = '{0}/{1}_{2}.json'.format(config['data']['directory'], key,
file_type)
abs_path = os.path.abspath(file)
if os.path.isfile(abs_path):
logger.debug('Returning {0} {1} data from file {2}'.format(
key, file_type, abs_path))
with open(abs_path) as f:
return_object = json.load(f)
f.close()
return return_object
result = requests.get(url)
if result.status_code == requests.codes.ok:
return_object = result.json()
with open(abs_path, 'w') as f:
logger.debug('Saving {0} {1} data to file {2}'.format(
key, file_type, abs_path))
json.dump(return_object, f)
f.close()
return return_object
raise ValueError('API call returned status code {0}'.format(
result.status_code))
except Exception as ex:
logger.exception(ex)
def __init__(self, name, download_url, archive_url, archive_regex):
threading.Thread.__init__(self)
self.kill_received = False
self.name = name
self.download_url = download_url
self.archive_url = archive_url
self.archive_regex = archive_regex
try:
self.ip_addr = yamlconfig['network']['ip']
true_socket = socket.socket
socket.socket = make_bound_socket(self.ip_addr)
except:
logger.debug("Using default IP address")
self.save_dir = yamlconfig['archive']['dir'] + os.sep + name
self.archive_dir = yamlconfig['archive']['dir-all'] + os.sep + name
if yamlconfig['archive']['save'] and not os.path.exists(self.save_dir):
os.makedirs(self.save_dir)
if yamlconfig['archive']['save-all'] and not os.path.exists(self.archive_dir):
os.makedirs(self.archive_dir)
self.archive_compress = yamlconfig['archive']['compress']
self.update_max = 30 # TODO set by config file
self.update_min = 10 # TODO set by config file
self.pastie_classname = None
self.seen_pasties = deque('', 1000) # max number of pasties ids in memory
def run(self):
while not self.kill_received:
mtime = os.stat(self.filename).st_mtime
if mtime != self.last_mtime:
logger.debug('Proxy configuration file changed. Reloading proxy list.')
proxies_lock.acquire()
load_proxies_from_file(self.filename)
self.last_mtime = mtime
proxies_lock.release()
def load_proxies_from_file(filename):
global proxies_list
try:
f = open(filename)
except Exception as e:
logger.error('Configuration problem: proxyfile "{file}" not found or not readable: {e}'.format(file=filename, e=e))
for line in f:
line = line.strip()
if line: # LATER verify if the proxy line has the correct structure
proxies_list.add(line)
logger.debug('Found {count} proxies in file "{file}"'.format(file=filename, count=len(proxies_list)))
def load_user_agents_from_file(filename):
global user_agents_list
try:
f = open(filename)
except Exception as e:
logger.error('Configuration problem: user-agent-file "{file}" not found or not readable: {e}'.format(file=filename, e=e))
for line in f:
line = line.strip()
if line:
user_agents_list.append(line)
logger.debug('Found {count} UserAgents in file "{file}"'.format(file=filename, count=len(user_agents_list)))
def add(self, pastie):
try:
data = {'site': pastie.site.name,
'id': pastie.id,
'md5': pastie.md5,
'url': pastie.url,
'local_path': pastie.site.archive_dir + os.sep + pastie.site.pastie_id_to_filename(pastie.id),
'timestamp': datetime.now(),
'matches': pastie.matches_to_text()
}
self.c.execute('INSERT INTO pasties VALUES (:site, :id, :md5, :url, :local_path, :timestamp, :matches)', data)
self.db_conn.commit()
except sqlite3.DatabaseError as e:
logger.error('Cannot add pastie {site} {id} in the SQLite database: {error}'.format(site=pastie.site.name, id=pastie.id, error=e))
logger.debug('Added pastie {site} {id} in the SQLite database.'.format(site=pastie.site.name, id=pastie.id))
def get(file: str = None) -> dict:
global _secrets
if _secrets == None:
if file == None:
file = 'secrets.yaml'
if os.path.isfile(file):
logger = modules.logger.get()
abs_path = os.path.abspath(file)
logger.debug('reading secrets from {0}'.format(abs_path))
with open(abs_path, 'r') as f:
_secrets = yaml.safe_load(f.read())
f.close()
else:
raise ValueError('{0} is not a file'.format(file))
return _secrets
def update(self, pastie):
try:
data = {'site': pastie.site.name,
'id': pastie.id,
'md5': pastie.md5,
'url': pastie.url,
'local_path': pastie.site.archive_dir + os.sep + pastie.site.pastie_id_to_filename(pastie.id),
'timestamp': datetime.now(),
'matches': pastie.matches_to_text()
}
self.c.execute('''UPDATE pasties SET md5 = :md5,
url = :url,
local_path = :local_path,
timestamp = :timestamp,
matches = :matches
WHERE site = :site AND id = :id''', data)
self.db_conn.commit()
except sqlite3.DatabaseError as e:
logger.error('Cannot add pastie {site} {id} in the SQLite database: {error}'.format(site=pastie.site.name, id=pastie.id, error=e))
logger.debug('Updated pastie {site} {id} in the SQLite database.'.format(site=pastie.site.name, id=pastie.id))
def run(self):
while not self.kill_received:
try:
# grabs pastie from queue
pastie = self.queue.get()
pastie_content = pastie.fetch_and_process_pastie()
logger.debug("Queue {name} size: {size}".format(
size=self.queue.qsize(), name=self.name))
if pastie_content:
logger.debug(
"Saved new pastie from {0} "
"with id {1}".format(self.name, pastie.id))
else:
# pastie already downloaded OR error ?
pass
# signals to queue job is done
self.queue.task_done()
# catch unknown errors
except Exception as e:
msg = "ThreadPasties for {name} crashed unexpectectly, "\
"recovering...: {e}".format(name=self.name, e=e)
logger.error(msg)
logger.debug(traceback.format_exc())
def Open(self, file_object, wal_file_object=None):
"""Opens a SQLite database file.
Since pysqlite cannot read directly from a file-like object a temporary
copy of the file is made. After creating a copy the database file this
function sets up a connection with the database and determines the names
of the tables.
Args:
file_object (dfvfs.FileIO): file-like object.
wal_file_object (Optional[dfvfs.FileIO]): file-like object for the
Write-Ahead Log (WAL) file.
Raises:
IOError: if the file-like object cannot be read.
OSError: if the file-like object cannot be read.
sqlite3.DatabaseError: if the database cannot be parsed.
ValueError: if the file-like object is missing.
"""
if not file_object:
raise ValueError('Missing file object.')
# TODO: Current design copies the entire file into a buffer
# that is parsed by each SQLite parser. This is not very efficient,
# especially when many SQLite parsers are ran against a relatively
# large SQLite database. This temporary file that is created should
# be usable by all SQLite parsers so the file should only be read
# once in memory and then deleted when all SQLite parsers have completed.
# TODO: Change this into a proper implementation using APSW
# and virtual filesystems when that will be available.
# Info: http://apidoc.apsw.googlecode.com/hg/vfs.html#vfs and
# http://apidoc.apsw.googlecode.com/hg/example.html#example-vfs
# Until then, just copy the file into a tempfile and parse it.
temporary_file = tempfile.NamedTemporaryFile(
delete=False, dir=self._temporary_directory)
try:
self._CopyFileObjectToTemporaryFile(file_object, temporary_file)
self._temp_db_file_path = temporary_file.name
except IOError:
os.remove(temporary_file.name)
raise
finally:
temporary_file.close()
if wal_file_object:
# Create WAL file using same filename so it is available for
# sqlite3.connect()
temporary_filename = '{0:s}-wal'.format(self._temp_db_file_path)
temporary_file = open(temporary_filename, 'wb')
try:
self._CopyFileObjectToTemporaryFile(wal_file_object,
temporary_file)
self._temp_wal_file_path = temporary_filename
except IOError:
os.remove(temporary_filename)
raise
finally:
temporary_file.close()
self._database = sqlite3.connect(self._temp_db_file_path)
try:
self._database.row_factory = sqlite3.Row
cursor = self._database.cursor()
sql_results = cursor.execute(self.SCHEMA_QUERY)
self.schema = {
table_name: ' '.join(query.split())
for table_name, query in sql_results
}
for table_name in self.schema.keys():
self.columns_per_table.setdefault(table_name, [])
pragma_results = cursor.execute(
'PRAGMA table_info({0:s})'.format(table_name))
for pragma_result in pragma_results:
self.columns_per_table[table_name].append(
pragma_result['name'])
except sqlite3.DatabaseError as exception:
self._database.close()
self._database = None
os.remove(self._temp_db_file_path)
self._temp_db_file_path = ''
if self._temp_wal_file_path:
os.remove(self._temp_wal_file_path)
self._temp_wal_file_path = ''
logger.debug(
'Unable to parse SQLite database: {0:s} with error: {1!s}'.
format(self._filename, exception))
raise
self._is_open = True
def main():
signal.signal(signal.SIGINT, signal_handle)
signal.signal(signal.SIGTERM, signal_handle)
parser = argparse.ArgumentParser()
parser.add_argument("name")
parser.add_argument("--verbose", action="store_true")
parser.add_argument("--mqtt-host")
parser.add_argument("--mqtt-port")
parser.add_argument("--mqtt-user")
parser.add_argument("--mqtt-pass")
parser.add_argument("--mqtt-topic-req")
parser.add_argument("--mqtt-topic-res")
args = parser.parse_args()
name = args.name
load_dotenv()
log_level = os.getenv("LOG_LEVEL", "info").lower()
setLevel(args.verbose or log_level == 'debug')
mqtt_host = args.mqtt_host or os.getenv("MQTT_HOST")
mqtt_port = args.mqtt_port or os.getenv("MQTT_PORT")
mqtt_user = args.mqtt_user or os.getenv("MQTT_USER")
mqtt_pass = args.mqtt_pass or os.getenv("MQTT_PASS")
mqtt_topic_req = args.mqtt_topic_req or os.getenv('MQTT_TOPIC_REQ')
mqtt_topic_res = args.mqtt_topic_res or os.getenv('MQTT_TOPIC_RES')
topic_req = f"{mqtt_topic_req}/{name}"
topic_res = f"{mqtt_topic_res}/{name}"
logger.debug("Starting MQTT")
nextConnectionAt = datetime.now()
connected = False
HOME = os.getenv("HOME")
pattern = re.compile(r'^Modify: (.*)\n')
while True:
now = datetime.now()
if not connected and now > nextConnectionAt:
try:
@subscribe(
topic_req, {
"host": mqtt_host,
"port": int(mqtt_port),
"user": mqtt_user,
"pass": mqtt_pass
})
def message_handle(payload, emit):
try:
if 'id' not in payload:
raise Exception("request id is not present")
if 'command' not in payload:
raise Exception("command is not present")
command = payload['command']
if command == 'status':
settings = read_config()
logger.info("settings: [%s]", settings)
found = glob.glob(
f"{HOME}/.pm2/pids/hackrf-control-*")
status = 'stopped'
uptime = None
if found:
status = 'online'
with open(found[0]) as fd:
pid = fd.read()
out = subprocess.check_output(
f"stat /proc/{pid} | grep Modify",
shell=True,
encoding="utf-8")
res = pattern.findall(out)
uptime = res[0] if res else None
emit(
topic_res, {
'id': payload['id'],
'settings': settings,
'process': {
'status': status,
'uptime': uptime
}
})
elif command == 'logs':
lines = payload.get('lines', 10)
out = subprocess.check_output(
f"tail {HOME}/.pm2/logs/hackrf-control-error.log -n {lines}",
shell=True,
encoding="utf-8")
data = []
for x in out.split('\n'):
created_at = x[0:23]
pos = x.find(" ", 24)
level = x[23:pos]
pos = x.find(" ", pos + 1)
content = x[pos:]
data.append({
'created_at': created_at,
'level': level,
'content': content
})
emit(topic_res, {
'id': payload['id'],
'data': data
})
elif command == 'config':
if 'settings' not in payload:
raise Exception("settings is not present")
settings = payload['settings']
settings['_waveform'] = 'waveform' in settings
logger.info("settings: [%s]", settings)
save_config(settings)
emit(topic_res, {'id': payload['id']})
else:
emit(topic_res, {'id': payload['id']})
except Exception as ex:
logger.warning("%s", payload)
logger.error(ex)
emit(topic_res, {'id': payload['id'], 'error': ex})
logger.info("mqtt connected")
connected = True
except Exception as ex:
logger.error(ex)
connected = False
nextConnectionAt = now + timedelta(seconds=10)
logger.debug("Reconnecting mqtt at 10 seconds")
time.sleep(0.1)
def Connect(self, configuration, source_path_spec, knowledge_base):
"""Connector to connect to ESE database modules.
Args:
configuration: configuration values.
source_path_spec (dfvfs.PathSpec): path specification of the source file.
knowledge_base (KnowledgeBase): knowledge base.
"""
if source_path_spec.parent.type_indicator != dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION:
par_id = configuration.partition_list['p1']
else:
par_id = configuration.partition_list[getattr(
source_path_spec.parent, 'location', None)[1:]]
if par_id == None:
return False
print('[MODULE]: ESE Database Analyzer Start! - partition ID(%s)' %
par_id)
# Load Schema
if not self.LoadSchemaFromYaml(
'../modules/schema/esedb/lv1_os_win_esedb.yaml'):
logger.error('cannot load schema from yaml: {0:s}'.format(
self.TABLE_NAME))
return False
# Search artifact paths
paths = self._schema['Paths']
separator = self._schema['Path_Separator']
environment_variables = knowledge_base.GetEnvironmentVariables()
find_specs = self.BuildFindSpecs(paths, separator,
environment_variables)
if len(find_specs) < 0:
return False
esedb_file = pyesedb.file()
for spec in find_specs:
file_object = self.LoadTargetFileToMemory(
source_path_spec=source_path_spec,
configuration=configuration,
file_spec=spec)
try:
esedb_file.open_file_object(file_object)
except IOError as exception:
logger.debug(
'[{0:s}] unable to open file with error: {0!s}'.format(
self.NAME, exception))
return
try:
esedb_parsers = esedb_parser.ESEDBParser.GetESEDBParserObjects(
)
table_names = frozenset(
esedb_parser.ESEDBParser.GetTableNames(esedb_file))
for parser in esedb_parsers.values():
if not parser.required_tables.issubset(table_names):
continue
try:
parser.Process(database=esedb_file)
info = tuple([
par_id, configuration.case_id,
configuration.evidence_id
])
# internet explorer
if 'Containers' in parser.required_tables:
if len(parser.GetHistoryRecords) > 0:
table_name = 'lv1_os_win_esedb_ie_history'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetHistorySchema)
if not ret:
logger.error(
'cannot create database table name: {0:s}'
.format(table_name))
return False
for record in parser.GetHistoryRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record)
query = self.InsertQueryBuilder(
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetHistorySchema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0:s}'.
format(exception))
if len(parser.GetContentRecords) > 0:
table_name = 'lv1_os_win_esedb_ie_content'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetContentSchema)
if not ret:
logger.error(
'cannot create database table name: {0:s}'
.format(table_name))
return False
for record in parser.GetContentRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record)
query = self.InsertQueryBuilder(
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetContentSchema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0:s}'.
format(exception))
if len(parser.GetCookiesRecords) > 0:
table_name = 'lv1_os_win_esedb_ie_cookies'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetCookiesSchema)
if not ret:
logger.error(
'cannot create database table name: {0:s}'
.format(table_name))
return False
for record in parser.GetCookiesRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record)
query = self.InsertQueryBuilder(
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetCookiesSchema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0:s}'.
format(exception))
if len(parser.GetDownloadRecords) > 0:
table_name = 'lv1_os_win_esedb_ie_download'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetDownloadSchema)
if not ret:
logger.error(
'cannot create database table name: {0:s}'
.format(table_name))
return False
for record in parser.GetDownloadRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record)
query = self.InsertQueryBuilder(
table_name=table_name,
schema=tuple(
['par_id', 'case_id', 'evd_id']) +
parser.GetDownloadSchema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0:s}'.
format(exception))
except errors.UnableToParseFile as exception:
logger.debug(
'[{0:s}] unable to parse file with error: {1!s}'.
format(self.NAME, exception))
finally:
esedb_file.close()
file_object.close()
def message_handle(payload, emit):
try:
if 'id' not in payload:
raise Exception("request id is not present")
if 'command' not in payload:
raise Exception("command is not present")
command = payload['command']
if command == 'config':
logger.debug(
"Config Temperature %s", {
'id': payload['id'],
'status': 'Config max temperature',
'data': {
'temp': payload['temp']
}
})
emit(
topic_res, {
'id': payload['id'],
'status': 'Config max temperature',
'data': {
'temp': payload['temp']
}
})
try:
newTemp = str(payload['temp'])
data = {"temp_max": newTemp}
save_config(data)
#updateTemperature('.local/config/hackrf-sensors.json', 0, '{"temp_max":' + newTemp + "}")
except Exception as ex:
logger.warning("%s", payload)
logger.error(ex)
emit(topic_res, {
'id': payload['id'],
'error': ex
})
elif command == 'status':
#INTEGRAR PROCESO QUE OBTIENE T° DE LA MAQUINA
# Y PASARLO COMO VARIABLE A EMIT(TOPIC_RES)
logger.debug("Getting sensors data")
emit(
topic_res, {
'id': payload['id'],
'status': 'Data sensors found',
'data': {
'temp': payload['temp']
}
})
else:
emit(topic_res, {'id': payload['id']})
except Exception as ex:
logger.warning("%s", payload)
logger.error(ex)
emit(topic_res, {'id': payload['id'], 'error': ex})
def create_host_directories_and_tar(self):
"""Main packaging function
Works in 3 parts:
1. Validate app data and configurations
2. Create tmp directories for each host with loaded apps and manifest
3. Package (tar) up host tmp directories for distribution
"""
Helpers.delete_path(self.tmp_folder)
Helpers.create_path(self.tars_folder, True)
self.repo_manager.set_commit_id()
master_commit_log = self.repo_manager.get_commit_log()
errors_found = False
changes_found = False
for host in self.appetite_hosts: # pylint: disable=too-many-nested-blocks
# Per host build apps folder and tar up based on class
hostname = host.hostname
apps = host.get_apps(self.args.refname)
tarname = host.tarname
apps = sorted(apps, key=lambda app: app.commit_id)
tmp_hostname_dir = os.path.join(self.hosts_folder, hostname)
tmp_hostname_meta = os.path.join(tmp_hostname_dir, Consts.META_DIR)
apps_meta = []
if len(apps) < 1:
Logger.warn("Host with no apps", hostname=hostname)
continue
# Parse the remote meta file from the host
# This file might not exist
remote_meta_file = host.local_meta_file
remote_metas_loaded = False
if os.path.exists(remote_meta_file):
try:
with open(remote_meta_file) as remote_data_file:
remote_metas_master = json.load(remote_data_file)
remote_metas_content = remote_metas_master['content'] \
if 'content' in remote_metas_master else remote_metas_master
remote_metas = [
AppetiteHost.create_app_from_object(
self.repo_manager, self.deployment_manager,
meta_data)
for meta_data in remote_metas_content
]
remote_metas_loaded = True
except Exception as exception:
Logger.error("Problems loading meta file",
error=exception.message,
path=remote_meta_file)
elif not self.args.dryrun:
Logger.warn("Local version of remote meta not found",
file=remote_meta_file)
ordered_unique_apps = sorted(list(set(apps)),
key=lambda single_app:
(single_app.name, single_app.
commit_id, single_app.method_name))
for iapp in ordered_unique_apps:
app_occurrences = apps.count(iapp)
if app_occurrences > 1:
Logger.warn("Dup app found",
host=host.hostname,
app_info=iapp.app_key,
occurences=app_occurrences)
# Validate app data and configurations
# Go through the apps and checks to see if there are any errors
# This is where the remote meta is compared to the newly generated
# lists of apps from the manifest
for app in apps:
raw_app_path = os.path.join(self.apps_folder, app.name)
# Check the commit Id for problems
if app.commit_id:
self.repo_manager.set_commit_id(app.commit_id)
else: # pylint: disable=else-if-used
if self.args.strict_commitids:
Logger.error("Application with missing commit Id",
hostname=hostname,
app=app.name)
errors_found = True
continue
else:
app._commit_id = master_commit_log['app_commit_id'] # pylint: disable=protected-access
self.repo_manager.set_commit_id(app.commit_id)
# Checks if app listed in the manifest
# exists with the correct commit id
if Helpers.check_path(raw_app_path):
meta_to_append = None
app.refresh_version_info(self.args.refname,
Consts.META_APP_UNCHANGED)
remote_meta = None
# Check to see what has changed
if remote_metas_loaded:
# Searches remote meta to see if application already exists
remote_meta = next((rmeta for rmeta in remote_metas
if app.check_names(rmeta)), None)
if remote_meta:
# If app does exist on system, have the commit ids changed
if remote_meta.commit_id != app.commit_id:
meta_to_append = app.set_status_changed()
else:
# meta has not changed so use existing meta
meta_to_append = app.clone
meta_to_append.update_app_version(app)
# to track if an app is removed from the remote meta
remote_metas.remove(remote_meta)
if not meta_to_append:
# There is no remote meta so all files should be added
meta_to_append = app.set_status_added()
if remote_meta and meta_to_append:
meta_outcome = Helpers.debug_app_versions(
meta_to_append, remote_meta, meta_to_append.status)
Logger.debug("Check meta logic", outcome=meta_outcome)
if meta_to_append.has_changed:
Logger.info("App change", logic=meta_outcome)
apps_meta.append(meta_to_append)
else:
Logger.error("Missing application",
hostname=hostname,
app=app.name,
path=raw_app_path)
continue
if remote_metas_loaded and len(remote_metas) > 0:
# Any apps left in the remote meta do not exist in the current
# manifest and should be deleted
delete_list = []
for deleted_app in remote_metas:
if deleted_app.method_info:
deleted_app.set_status_deleted()
# Added logic check to catch method changes
added_app_found = next((
app for app in apps_meta
if app.status == Consts.META_APP_ADDED and app.name
== deleted_app.name and app.method_info['path'] ==
deleted_app.method_info['path']), None)
if added_app_found:
added_app_found.set_status_changed()
else:
delete_list.append(deleted_app)
else:
Logger.error(
"Problems with method info for deleted app.",
hostname=hostname,
app=deleted_app.name)
apps_meta += delete_list
# Only do something if there has been a change
if len([app for app in apps_meta if not app.is_unchanged]) < 1:
continue
# No point continuing if there is no connection to the host
if not self.check_host_connection(host):
continue
# Clean command lines for auth params
# This data is ingested so creds should be removed
# apps_meta = [updated_app.clone for updated_app in apps_meta]
if not self.args.disable_logging:
for updated_app in apps_meta:
Logger.log_event(updated_app.to_dict)
# Applications that actually needs to be updated
tar_apps = sorted([
updated_app for updated_app in apps_meta if updated_app.updated
],
key=lambda tar_app: tar_app.app)
use_templating = self.template_values and self.args.templating
# Checking will allow templating otherwise will skip steps
Helpers.create_path(
os.path.join(tmp_hostname_meta, Consts.HOST_LOGS_FOLDER_NAME),
True)
if len(tar_apps) > 0:
# All error checks have been done above, build out
# the hosts directory and tar up
for updated_app in tar_apps:
app_path = os.path.join(tmp_hostname_dir,
updated_app.method_info['path'])
Helpers.create_path(app_path, True)
raw_app_path = os.path.join(self.apps_folder,
updated_app.name)
self.repo_manager.set_commit_id(updated_app.commit_id)
if updated_app.update_method_is_copy:
app_dest = os.path.join(app_path,
updated_app.app_clean)
else:
app_dest = app_path
copy_tree(raw_app_path, app_dest)
lookups_inclusion_location = os.path.join(
app_dest, self.deployment_manager.inclusion_filename)
ignore_dir = os.path.join(app_dest, Consts.TMP_IGNORE_DIR)
# Ignore files/folders set in the global configurations
if self.args.install_ignore:
content_ignored_results = Helpers.move_regexed_files(
self.args.install_ignore.split(';'), app_dest,
ignore_dir)
files_included = content_ignored_results['files_moved']
if len(files_included) > 0:
Logger.error(
"Globally these files should not exist in the App. "
"The files have been removed from the install.",
files=files_included,
hostname=hostname,
app=updated_app.name)
# Users should not have the capability to include files from the
# global ignore.
Helpers.delete_path(ignore_dir)
# Defined folders/files are to move out of application.
# This is defined in the deploymentmethods.conf
# If an app is installed for the first time, all files should be included
if 'install_ignore' in updated_app.method_info and not updated_app.is_added:
Helpers.move_regexed_files(
updated_app.method_info['install_ignore'],
app_dest, ignore_dir)
# If there is a inclusion file, include files back into app.
# This is defined on a per app basis
if os.path.isfile(lookups_inclusion_location):
with open(lookups_inclusion_location, "r") as f:
lines = [l.strip() for l in f.readlines()]
lookup_inclusion_results = Helpers.move_regexed_files(
lines, ignore_dir, app_dest)
if lookup_inclusion_results['errors_found']:
Logger.warn(
"Lookup inclusion error found",
paths=lookup_inclusion_results[
'path_errors'],
hostname=hostname,
app=updated_app.name,
todo="Remove file/path from inclusion..")
# Problem with host inclusion,
# move to next host
continue
updated_app.method_info['inclusions'] = \
lookup_inclusion_results['files_moved']
# Update objects with inclusions
updated_app.copy_value_to_method_info(
'inclusions', apps_meta)
os.remove(lookups_inclusion_location)
Helpers.delete_path(ignore_dir)
if use_templating and not updated_app.method_info[
'skip_templating']:
# Can template based on vars from templated
# values, hosts vars and app vars
Helpers.template_directory(app_dest, [
self.template_values, host.to_dict,
updated_app.to_dict
])
# Should only change access and create version file if a whole app is copied
if updated_app.update_method_is_copy:
for host_path, host_dir, host_files in os.walk(
app_dest): # pylint: disable=unused-variable
for host_file in host_files:
# Splunk apps can have active binaries in multiple languages
# This is a catch all to make sure apps have all the required
# permissions.
chmod = 0755
os.chmod(os.path.join(host_path, host_file),
chmod)
if not updated_app.method_info['no_appetite_changes']:
with open(
os.path.join(
app_dest,
Helpers.get_app_version_filename()),
"w") as f:
f.write(updated_app.to_json)
AppVersioning.create_app_version(
app_dest,
updated_app.commit_log['app_abbrev_commit_id'])
apps_distro = Helpers.content_wrapper(apps_meta,
Consts.META_CURRENT,
hostname, self.track)
# Meta file used as source of truth on instance
master_meta = self.create_meta_files(tmp_hostname_meta, '',
apps_distro)
# check be used to update and test manifest changes locally
if self.args.dryrun:
Helpers.create_path(host.local_meta_file)
shutil.copy(master_meta, host.local_meta_file)
# Always want clean logs ingested
selected_apps = Helpers.select_and_update_apps(
apps_meta, Consts.META_CURRENT, False)
self.create_meta_log(tmp_hostname_meta, '', selected_apps,
Helpers.get_utc())
host.updates = Helpers.content_process(apps_meta,
Consts.META_UPDATED,
hostname, self.track, True)
# Create the meta change file
self.create_meta_files(tmp_hostname_meta, '_update',
Helpers.content_convert(host.updates))
# Clean updates file for logging
selected_apps = Helpers.select_and_update_apps(
apps_meta, Consts.META_UPDATED, True)
self.create_meta_log(tmp_hostname_meta, '_update', selected_apps,
Helpers.get_utc())
Logger.info("Changes found",
updates=Helpers.content_wrapper(
apps_meta, Consts.META_UPDATED, hostname,
self.track, True))
# Package (tar) up host tmp directories for distribution
tar = tarfile.open(
os.path.join(self.tars_folder, "%s.tar.gz" % tarname), "w:gz")
tar.add(tmp_hostname_dir, arcname=os.path.basename(self.base_name))
tar.close()
changes_found = True
if errors_found:
sys.exit(1)
self.repo_manager.set_commit_id()
return changes_found
def download_url(url, data=None, cookie=None, loop_client=0, loop_server=0):
# Client errors (40x): if more than 5 recursions, give up on URL (used for the 404 case)
if loop_client >= retries_client:
return None, None
# Server errors (50x): if more than 100 recursions, give up on URL
if loop_server >= retries_server:
return None, None
try:
opener = None
# Random Proxy if set in config
random_proxy = get_random_proxy()
if random_proxy:
proxyh = urllib2.ProxyHandler({'http': random_proxy})
opener = urllib2.build_opener(proxyh, NoRedirectHandler())
# We need to create an opener if it didn't exist yet
if not opener:
opener = urllib2.build_opener(NoRedirectHandler())
# Random User-Agent if set in config
user_agent = get_random_user_agent()
opener.addheaders = [('Accept-Charset', 'utf-8')]
if user_agent:
opener.addheaders.append(('User-Agent', user_agent))
if cookie:
opener.addheaders.append(('Cookie', cookie))
logger.debug(
'Downloading url: {url} with proxy: {proxy} and user-agent: {ua}'.format(
url=url, proxy=random_proxy, ua=user_agent))
if data:
response = opener.open(url, data)
else:
response = opener.open(url)
htmlPage = unicode(response.read(), errors='replace')
return htmlPage, response.headers
except urllib2.HTTPError as e:
failed_proxy(random_proxy)
logger.warning("!!Proxy error on {0}.".format(url))
if 404 == e.code:
htmlPage = e.read()
logger.warning("404 from proxy received for {url}. Waiting 1 minute".format(url=url))
time.sleep(60)
loop_client += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_client, total=retries_client, url=url))
return download_url(url, loop_client=loop_client)
if 500 == e.code:
htmlPage = e.read()
logger.warning("500 from proxy received for {url}. Waiting 1 minute".format(url=url))
time.sleep(60)
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
if 504 == e.code:
htmlPage = e.read()
logger.warning("504 from proxy received for {url}. Waiting 1 minute".format(url=url))
time.sleep(60)
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
if 502 == e.code:
htmlPage = e.read()
logger.warning("502 from proxy received for {url}. Waiting 1 minute".format(url=url))
time.sleep(60)
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
if 403 == e.code:
htmlPage = e.read()
if 'Please slow down' in htmlPage or 'has temporarily blocked your computer' in htmlPage or 'blocked' in htmlPage:
logger.warning("Slow down message received for {url}. Waiting 1 minute".format(url=url))
time.sleep(60)
return download_url(url)
logger.warning("ERROR: HTTP Error ##### {e} ######################## {url}".format(e=e, url=url))
return None, None
except urllib2.URLError as e:
logger.debug("ERROR: URL Error ##### {e} ######################## ".format(e=e, url=url))
if random_proxy: # remove proxy from the list if needed
failed_proxy(random_proxy)
logger.warning("Failed to download the page because of proxy error {0} trying again.".format(url))
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
if 'timed out' in e.reason:
logger.warning("Timed out or slow down for {url}. Waiting 1 minute".format(url=url))
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
time.sleep(60)
return download_url(url, loop_server=loop_server)
return None, None
except socket.timeout:
logger.debug("ERROR: timeout ############################# " + url)
if random_proxy: # remove proxy from the list if needed
failed_proxy(random_proxy)
logger.warning("Failed to download the page because of socket error {0} trying again.".format(url))
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
return None, None
except Exception as e:
failed_proxy(random_proxy)
logger.warning("Failed to download the page because of other HTTPlib error proxy error {0} trying again.".format(url))
loop_server += 1
logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url))
return download_url(url, loop_server=loop_server)
def Connect(self, par_id, configuration, source_path_spec, knowledge_base):
"""Connector to connect to ESE database modules.
Args:
configuration: configuration values.
source_path_spec (dfvfs.PathSpec): path specification of the source file.
knowledge_base (KnowledgeBase): knowledge base.
"""
self._configuration = configuration
self._time_zone = knowledge_base.time_zone
# Load Schema
yaml_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'schema' + os.sep + 'esedb' \
+ os.sep + 'lv1_os_win_esedb.yaml'
if not self.LoadSchemaFromYaml(yaml_path):
logger.error('cannot load schema from yaml: {0:s}'.format(
self.TABLE_NAME))
return False
# Search artifact paths
paths = self._schema['Paths']
separator = self._schema['Path_Separator']
environment_variables = knowledge_base.GetEnvironmentVariables()
find_specs = self.BuildFindSpecs(paths, separator,
environment_variables)
if len(find_specs) < 0:
return False
esedb_file = pyesedb.file()
for spec in find_specs:
try:
file_object = self.LoadTargetFileToMemory(
source_path_spec=source_path_spec,
configuration=configuration,
file_spec=spec)
esedb_file.open_file_object(file_object)
except IOError as exception:
logger.debug(
'[{0:s}] unable to open file with error: {0!s}'.format(
self.NAME, exception))
return
path_spec_generator = self._path_spec_extractor.ExtractPathSpecs(
[source_path_spec],
find_specs=[spec],
recurse_file_system=False,
resolver_context=configuration.resolver_context)
for path_spec in path_spec_generator:
edb_path = path_spec.location
try:
esedb_parsers = esedb_parser.ESEDBParser.GetESEDBParserObjects(
)
table_names = frozenset(
esedb_parser.ESEDBParser.GetTableNames(esedb_file))
for parser in esedb_parsers.values():
if not parser.required_tables.issubset(table_names):
continue
try:
parser.Process(database=esedb_file)
info = tuple([
par_id, configuration.case_id,
configuration.evidence_id
])
# Internet Explorer
if 'Containers' in parser.required_tables:
# Internet Explorer History
if len(parser.GetHistoryRecords) > 0:
history_schema = ['par_id', 'case_id', 'evd_id'] + list(parser.GetHistorySchema) \
+ ['source']
table_name = 'lv1_os_win_esedb_ie_history'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=history_schema)
if not ret:
logger.error(
'cannot create database table name: {0:s}'
.format(table_name))
return False
for record in parser.GetHistoryRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record) + tuple(
[edb_path])
query = self.InsertQueryBuilder(
table_name=table_name,
schema=history_schema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0!s}'.
format(exception))
# Internet Explorer Content
if len(parser.GetContentRecords) > 0:
content_schema = ['par_id', 'case_id', 'evd_id'] + list(parser.GetContentSchema) \
+ ['source']
table_name = 'lv1_os_win_esedb_ie_content'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=content_schema)
if not ret:
logger.error(
'cannot create database table name: {0!s}'
.format(table_name))
return False
for record in parser.GetContentRecords:
tmp_record = list(record)
if isinstance(tmp_record[17], bytes):
tmp_record[17] = "Unknown"
else:
tmp_record[17] = tmp_record[
17].replace('"', '""').replace(
'\'', '\'\'')
result = info + tuple(tmp_record) + tuple(
[edb_path])
query = self.InsertQueryBuilder(
table_name=table_name,
schema=content_schema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0!s}'.
format(exception))
# Internet Explorer Cookies
if len(parser.GetCookiesRecords) > 0:
cookie_schema = ['par_id', 'case_id', 'evd_id'] + list(parser.GetCookiesSchema) \
+ ['source']
table_name = 'lv1_os_win_esedb_ie_cookies'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=cookie_schema)
if not ret:
logger.error(
'cannot create database table name: {0!s}'
.format(table_name))
return False
for record in parser.GetCookiesRecords:
tmp_record = list(record)
result = info + tuple(tmp_record) + tuple(
[edb_path])
query = self.InsertQueryBuilder(
table_name=table_name,
schema=cookie_schema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0!s}'.
format(exception))
if len(parser.GetDownloadRecords) > 0:
download_schema = ['par_id', 'case_id', 'evd_id'] + list(parser.GetDownloadSchema) \
+ ['source']
table_name = 'lv1_os_win_esedb_ie_download'
if not configuration.cursor.check_table_exist(
table_name):
ret = self.CreateTableWithSchema(
configuration.cursor,
table_name=table_name,
schema=download_schema)
if not ret:
logger.error(
'cannot create database table name: {0!s}'
.format(table_name))
return False
for record in parser.GetDownloadRecords:
tmp_record = list(record)
tmp_record[17] = tmp_record[17].replace(
'"', '""')
tmp_record[17] = tmp_record[17].replace(
'\'', '\'\'')
result = info + tuple(tmp_record) + tuple(
[edb_path])
query = self.InsertQueryBuilder(
table_name=table_name,
schema=download_schema,
data=result)
try:
configuration.cursor.execute_query(
query)
except Exception as exception:
logger.error(
'database execution failed: {0!s}'.
format(exception))
except errors.UnableToParseFile as exception:
logger.debug(
'[{0:s}] unable to parse file with error: {1!s}'.
format(self.NAME, exception))
finally:
esedb_file.close()
file_object.close()
def main():
read_config()
signal.signal(signal.SIGINT, signal_handle)
signal.signal(signal.SIGTERM, signal_handle)
parser = argparse.ArgumentParser()
parser.add_argument("name")
parser.add_argument("--verbose", action="store_true")
parser.add_argument("--mqtt-host")
parser.add_argument("--mqtt-port")
parser.add_argument("--mqtt-user")
parser.add_argument("--mqtt-pass")
parser.add_argument("--mqtt-topic-req")
parser.add_argument("--mqtt-topic-res")
args = parser.parse_args()
name = args.name
load_dotenv()
log_level = os.getenv("LOG_LEVEL", "info").lower()
setLevel(args.verbose or log_level == 'debug')
mqtt_host = args.mqtt_host or os.getenv("MQTT_HOST")
mqtt_port = args.mqtt_port or os.getenv("MQTT_PORT")
mqtt_user = args.mqtt_user or os.getenv("MQTT_USER")
mqtt_pass = args.mqtt_pass or os.getenv("MQTT_PASS")
mqtt_topic_req = args.mqtt_topic_req or os.getenv('MQTT_TOPIC_REQ')
mqtt_topic_res = args.mqtt_topic_res or os.getenv('MQTT_TOPIC_RES')
topic_req = f"{mqtt_topic_req}/{name}"
topic_res = f"{mqtt_topic_res}/{name}"
logger.debug("Starting MQTT")
nextConnectionAt = datetime.now()
connected = False
HOME = os.getenv("HOME")
pattern = re.compile(r'^Modify: (.*)\n')
while True:
now = datetime.now()
if not connected and now > nextConnectionAt:
try:
@subscribe(
topic_req, {
"host": mqtt_host,
"port": int(mqtt_port),
"user": mqtt_user,
"pass": mqtt_pass
})
def message_handle(payload, emit):
try:
if 'id' not in payload:
raise Exception("request id is not present")
if 'command' not in payload:
raise Exception("command is not present")
command = payload['command']
if command == 'config':
logger.debug(
"Config Temperature %s", {
'id': payload['id'],
'status': 'Config max temperature',
'data': {
'temp': payload['temp']
}
})
emit(
topic_res, {
'id': payload['id'],
'status': 'Config max temperature',
'data': {
'temp': payload['temp']
}
})
try:
newTemp = str(payload['temp'])
data = {"temp_max": newTemp}
save_config(data)
#updateTemperature('.local/config/hackrf-sensors.json', 0, '{"temp_max":' + newTemp + "}")
except Exception as ex:
logger.warning("%s", payload)
logger.error(ex)
emit(topic_res, {
'id': payload['id'],
'error': ex
})
elif command == 'status':
#INTEGRAR PROCESO QUE OBTIENE T° DE LA MAQUINA
# Y PASARLO COMO VARIABLE A EMIT(TOPIC_RES)
logger.debug("Getting sensors data")
emit(
topic_res, {
'id': payload['id'],
'status': 'Data sensors found',
'data': {
'temp': payload['temp']
}
})
else:
emit(topic_res, {'id': payload['id']})
except Exception as ex:
logger.warning("%s", payload)
logger.error(ex)
emit(topic_res, {'id': payload['id'], 'error': ex})
logger.info("mqtt connected")
connected = True
except Exception as ex:
logger.error(ex)
connected = False
nextConnectionAt = now + timedelta(seconds=10)
logger.debug("Reconnecting mqtt at 10 seconds")
time.sleep(0.1)
