def install_dtb(dtb_image):
try:
set_alt_kernel(True)
install_dtb_image(dtb_image)
set_alt_kernel(False)
except StandardError as err:
panic("", err)
def factory():
try:
set_alt_rootfs(True)
mount_recovery()
open("/mnt/var/ftk/recovery", 'a').close()
umount_recovery()
reboot()
except StandardError as err:
panic("", err)
def __init__(self, dbCursor, name):
self.c = dbCursor
self.name = ''.join(chr for chr in name if chr.isalnum())
self.overviewName = self.name + "_overview"
self.openName = self.name + "_open"
# Create the table and views if they don't exist yet.
if not self.tableExists(self.name):
if not utils.question("There is no work named '{}'. Create it?".format(name), "no"):
utils.panic("Invalid work name.")
self.createIfNotExists()
def log(db, args):
if utils.checkNumberOfArgs(args, 0):
visualize.closedTickets(db, -1)
elif utils.checkNumberOfArgs(args, 1, True):
if utils.checkIsInt(args[0]):
visualize.closedTickets(db, int(args[0]))
elif utils.checkIsStamp(args[0]):
visualize.log(db, args[0])
else:
utils.panic("Argument '{}' is neither an integer nor a month "
"(format: MM/YYYY).".format(args[0]))
def log(db, args):
if utils.checkNumberOfArgs(args, 0):
visualize.closedTickets(db, -1)
elif utils.checkNumberOfArgs(args, 1, True):
if utils.checkIsInt(args[0]):
visualize.closedTickets(db, int(args[0]))
elif utils.checkIsStamp(args[0]):
visualize.log(db, args[0])
else:
utils.panic("Argument '{}' is neither an integer nor a month "
"(format: MM/YYYY).".format(args[0]))
def parse_command_line():
#parsing command line
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-s", "--source-trace", metavar="name",
dest="src_trace_name",
help="source: MPI trace with SMPSs related events (has to be old format)")
parser.add_option("-d", "--destination-trace", metavar="name",
dest="dest_trace_name",
help="result: MPI/SMPSs trace (also old format)")
parser.add_option("-b", "--beginning-phase", metavar="INT",
dest="begin_phase",
help="From which phase to cut the trace")
parser.add_option("-e", "--ending-phase", metavar="INT",
dest="end_phase",
help="Until which phase to cut the trace")
(options, args) = parser.parse_args()
# check the avalibility of the files
if ((not options.src_trace_name) or (not options.dest_trace_name)):
utils.panic("incorrect usage (source or dest trace not specified), check: %prog --help")
# convert the phase numbers into integers
try:
if options.begin_phase == None:
begin_phase = -1
else:
begin_phase = int(options.begin_phase)
if options.end_phase == None:
end_phase = -1
else:
end_phase = int(options.end_phase)
except ValueError:
utils.panic("Failed to convert begin or end phase into integer")
#check sanity of the phase
if (options.begin_phase and options.end_phase):
if (int(options.begin_phase) >= int(options.end_phase)):
utils.panic('ERROR in phase selection: end_phase <= begin_phase, so the cut is empty')
if (options.begin_phase):
if (int(options.begin_phase) < 0):
utils.panic('ERROR in phase selection: start_phase cannot be negative number')
if (options.end_phase):
if (int(options.end_phase) < 0):
utils.panic('ERROR in phase selection: end_phase cannot be negative number')
#return phase in integer types, with -1 meaning that the phase is not specified
options.begin_phase = begin_phase
options.end_phase = end_phase
return options
def __init__(self, database_config=None):
if not database_config:
database_config = get_config()['database']
try:
self.engine = create_engine(
'postgresql://{pguser}:@{pghost}:{pgport}/{pgdatabase}'.format(
**database_config))
except OperationalError as exc:
panic("fingerprint-securedrop Postgres support relies on use of a "
"PGPASSFILE. Make sure this file exists in your homedir with "
"0600 permissions:\n{}.".format(exc))
def authenticate_to_tor_controlport(self):
self.logger.info("Authenticating to the tor controlport...")
try:
self.controller = Controller.from_port(port=self.control_port)
except stem.SocketError as exc:
panic("Unable to connect to tor on port {self.control_port}: "
"{exc}".format(**locals()))
try:
self.controller.authenticate()
except stem.connection.MissingPassword:
panic("Unable to authenticate to tor controlport. Please add "
"`CookieAuth 1` to your tor configuration file.")
def authenticate_to_tor_controlport(self):
self.logger.info("Authenticating to the tor controlport...")
try:
self.controller = Controller.from_port(port=self.control_port)
except stem.SocketError as exc:
panic("Unable to connect to tor on port {self.control_port}: "
"{exc}".format(**locals()))
try:
self.controller.authenticate()
except stem.connection.MissingPassword:
panic("Unable to authenticate to tor controlport. Please add "
"`CookieAuth 1` to your tor configuration file.")
def __init__(self, database_config=None):
if not database_config:
database_config = get_config()['database']
try:
self.engine = create_engine(
'postgresql://{pguser}:@{pghost}:{pgport}/{pgdatabase}'.format(
**database_config))
except OperationalError as exc:
panic(
"fingerprint-securedrop Postgres support relies on use of a "
"PGPASSFILE. Make sure this file exists in your homedir with "
"0600 permissions:\n{}.".format(exc))
def __init__(self, database_config=None):
if not database_config:
config = get_config()
database_config = dict(config.items("test_database"))
try:
self.engine = create_engine(
'postgresql://{pguser}:@{pghost}:{pgport}/{pgdatabase}'.format(
**database_config))
except OperationalError as exc:
panic("fingerprint-securedrop Postgres support relies on use of a "
"PGPASSFILE. Make sure this file and the env var pointing "
"to it exist and set 0600 permissions & user ownership."
"\n{}.".format(exc))
def delete(db, args):
utils.checkNumberOfArgs(args, 1, True)
utils.checkIsInt(args[0], True)
id = args[0]
print()
if visualize.ticket(db, id) == False:
utils.panic("There is no ticket #{}.".format(id))
print()
if utils.question("Really delete above ticket #{}?".format(id), "no"):
db.deleteEntry(id)
print("Deleted ticket #{}.".format(id))
else:
print("Aborted.")
def delete(db, args):
utils.checkNumberOfArgs(args, 1, True)
utils.checkIsInt(args[0], True)
id = args[0]
print()
if visualize.ticket(db, id) == False:
utils.panic("There is no ticket #{}.".format(id))
print()
if utils.question("Really delete above ticket #{}?".format(id), "no"):
db.deleteEntry(id)
print("Deleted ticket #{}.".format(id))
else:
print("Aborted.")
def add(db, args):
if len(args) < 3:
utils.panic("Command requires at least 3 argument(s) "
"({} given).".format(len(args)))
utils.checkIsDate(args[0])
utils.checkIsTime(args[1])
utils.checkIsTime(args[2])
notes = " ".join(args[3:])
visualize.data(args[0], args[1], args[2], notes)
print()
if utils.question("Create the above work record?"):
db.createEntry(args[0], args[1], args[2], notes)
print("Done:")
print()
visualize.lastInserted(db)
print()
else:
print("Aborted.")
def add(db, args):
if len(args) < 3:
utils.panic("Command requires at least 3 argument(s) "
"({} given).".format(len(args)))
utils.checkIsDate(args[0])
utils.checkIsTime(args[1])
utils.checkIsTime(args[2])
notes = " ".join(args[3:])
visualize.data(args[0], args[1], args[2], notes)
print()
if utils.question("Create the above work record?"):
db.createEntry(args[0], args[1], args[2], notes)
print("Done:")
print()
visualize.lastInserted(db)
print()
else:
print("Aborted.")
def main():
""" Main function for the factory toolkit
"""
opt = Options()
try:
opt.parse()
except StandardError as err:
panic("", err)
if opt.args.install != None:
install_rootfs_image(opt.args.install)
elif opt.args.kernel_install != None:
install_kernel(opt.args.kernel_install)
elif opt.args.dtb_install != None:
install_dtb(opt.args.dtb_install)
elif opt.args.factory:
factory()
elif opt.args.recovery:
recovery()
elif opt.args.normal:
normal()
def collect_onion_trace(self, url, hsid=None, extra_fn=None, trace_dir=None,
iteration=0):
"""Crawl an onion service and collect a complete cell sequence for the
activity at the time. Also, record additional information about the
circuits with stem. Optionally, pass a function to execute additional
actions after the page has loaded."""
# Todo: create collect_trace method that works for regular sites as
# well
assert ".onion" in url, ("This method is only suitable for crawling "
"onion services.")
self.logger.info("{url}: closing existing circuits before starting "
"crawl.".format(**locals()))
for circuit in self.controller.get_circuits():
self.controller.close_circuit(circuit.id)
sleep(self.wait_after_closing_circuits)
if not trace_dir:
trace_dir = self.make_ts_dir()
trace_name = urllib.parse.quote(url, safe="") + "-" + str(iteration)
trace_path = join(trace_dir, trace_name)
start_idx = self.get_cell_log_pos()
try:
self.crawl_url(url)
rend_circ_ids = self.get_rend_circ_ids(url)
if extra_fn:
self.execute_extra_fn(url, trace_path, start_idx)
except CrawlerLoggedError:
return "failed"
except CrawlerNoRendCircError:
self.save_debug_log(url, trace_path, start_idx)
return "failed"
except:
self.logger.exception("{url}: unusual exception "
"encountered:".format(**locals()))
# Also log active circuit info
self.controller.get_circuits()
exc_type, exc_value, exc_traceback = exc_info()
if exc_type in _sketchy_exceptions:
self.save_debug_log(url, trace_path, start_idx)
if self.restart_on_sketchy_exception:
self.restart_tb()
return "failed"
self.logger.info("{url}: saving full trace...".format(**locals()))
end_idx = self.get_cell_log_pos()
full_trace = self.get_full_trace(start_idx, end_idx)
# Save the trace to the database or write to file
if self.db_handler:
try:
new_example = {'hsid': hsid,
'crawlid': self.crawlid,
't_scrape': get_timestamp("db")}
except NameError:
panic("If using the database, and calling collect_onion_trace "
"directly, you must specify the hsid of the site.")
exampleid = self.db_handler.add_example(new_example)
self.db_handler.add_trace(str(full_trace), exampleid)
else:
with open(trace_path+"-full", "wb") as fh:
fh.write(full_trace)
return "succeeded"
def install_rootfs_image(image_name):
try:
install_recovery(image_name)
except StandardError as err:
panic("", err)
def start(db, args):
utils.checkNumberOfArgs(args, 0, True)
if db.startTicket() == False:
utils.panic("There already is an open work ticket.")
visualize.openTickets(db)
def normal():
try:
set_alt_rootfs(False)
reboot()
except StandardError as err:
panic("", err)
def recovery():
try:
set_alt_rootfs(True)
reboot()
except StandardError as err:
panic("", err)
def start(db, args):
utils.checkNumberOfArgs(args, 0, True)
if db.startTicket() == False:
utils.panic("There already is an open work ticket.")
visualize.openTickets(db)
def stop(db, args):
notes = " ".join(args)
if db.stopTicket(notes) == False:
utils.panic("There is no open work ticket.")
else:
visualize.closedTickets(db, 1)
def stop(db, args):
notes = " ".join(args)
if db.stopTicket(notes) == False:
utils.panic("There is no open work ticket.")
else:
visualize.closedTickets(db, 1)
def collect_onion_trace(self,
url,
hsid=None,
extra_fn=None,
trace_dir=None,
iteration=0):
"""Crawl an onion service and collect a complete cell sequence for the
activity at the time. Also, record additional information about the
circuits with stem. Optionally, pass a function to execute additional
actions after the page has loaded."""
# Todo: create collect_trace method that works for regular sites as
# well
assert ".onion" in url, ("This method is only suitable for crawling "
"onion services.")
self.logger.info("{url}: closing existing circuits before starting "
"crawl.".format(**locals()))
for circuit in self.controller.get_circuits():
self.controller.close_circuit(circuit.id)
sleep(self.wait_after_closing_circuits)
if not trace_dir:
trace_dir = self.make_ts_dir()
trace_name = urllib.parse.quote(url, safe="") + "-" + str(iteration)
trace_path = join(trace_dir, trace_name)
start_idx = self.get_cell_log_pos()
try:
self.crawl_url(url)
rend_circ_ids = self.get_rend_circ_ids(url)
if extra_fn:
self.execute_extra_fn(url, trace_path, start_idx)
except CrawlerLoggedError:
return "failed"
except CrawlerNoRendCircError:
self.save_debug_log(url, trace_path, start_idx)
return "failed"
except:
self.logger.exception("{url}: unusual exception "
"encountered:".format(**locals()))
# Also log active circuit info
self.controller.get_circuits()
exc_type, exc_value, exc_traceback = exc_info()
if exc_type in _sketchy_exceptions:
self.save_debug_log(url, trace_path, start_idx)
if self.restart_on_sketchy_exception:
self.restart_tb()
return "failed"
self.logger.info("{url}: saving full trace...".format(**locals()))
end_idx = self.get_cell_log_pos()
full_trace = self.get_full_trace(start_idx, end_idx)
# Save the trace to the database or write to file
if self.db_handler:
try:
new_example = {
'hsid': hsid,
'crawlid': self.crawlid,
't_scrape': get_timestamp("db")
}
except NameError:
panic("If using the database, and calling collect_onion_trace "
"directly, you must specify the hsid of the site.")
exampleid = self.db_handler.add_example(new_example)
self.db_handler.add_trace(str(full_trace), exampleid)
else:
with open(trace_path + "-full", "wb") as fh:
fh.write(full_trace)
return "succeeded"
def set_tasks_entered_so_far(self, tasks):
if (self._tasks_entered_so_far != tasks - 1):
utils.panic(
'failed because tasks_entered_so_far is %d and tasks is %d ' % (self._tasks_entered_so_far, tasks))
self._tasks_entered_so_far = tasks
