def deal_db(self):
"""
Process the data when the collection task is completed
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def save_db(self):
"""
Save module results into the database
"""
logger.log('DEBUG', f'Saving results to database')
lock.acquire()
db = Database()
db.create_table(self.domain)
db.save_db(self.domain, self.results, self.source)
db.close()
lock.release()
def export(target,
type='target',
db=None,
alive=False,
limit=None,
path=None,
format='csv',
show=False):
"""
OneForAll export from database module
Example:
python3 dbexport.py --target name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --target name --show False
python3 dbexport.py --target table_name --tb True --show False
Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
--path Result directory (default directory is ./results)
:param str target: Table to be exported
:param str type: Type of target
:param str db: Database path to be exported (default ./results/result.sqlite3)
:param bool alive: Only export the results of alive subdomains (default False)
:param str limit: Export limit (default None)
:param str format: Result format (default csv)
:param str path: Result directory (default None)
:param bool show: Displays the exported data in terminal (default False)
"""
if type == 'target':
database = Database(db)
domains = utils.get_domains(target)
datas = []
if domains:
for domain in domains:
table_name = domain_to_table(domain)
rows = database.export_data(table_name, alive, limit)
if rows is None:
continue
data = export_data(format, path, rows, show, table_name,
target)
datas.extend(data)
database.close()
if len(domains) > 1:
utils.export_all(alive, format, path, datas)
elif type == 'table':
database = Database(db)
rows = database.export_data(target, alive, limit)
data = export_data(format, path, rows, show, target, target)
database.close()
return data
def save_db(name, data, module):
"""
Save request results to database
:param str name: table name
:param list data: data to be saved
:param str module: module name
"""
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, module)
db.close()
def export(self, table):
"""
Export data from the database and do some follow-up processing
:param table: table name
:return: export data
:rtype: list
"""
db = Database()
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def mark(self):
"""
Mark the new discovered subdomain
:return: marked data
:rtype: list
"""
db = Database()
old_data = list()
now_data = db.get_data(self.domain).as_dict()
# Database pre-processing when it is not the first time to collect this subdomain
if db.exist_table(self.new_table):
# If there is the last collection result table, delete it first
db.drop_table(self.old_table)
# Rename the new table to the old table
db.rename_table(self.new_table, self.old_table)
old_data = db.get_data(self.old_table).as_dict()
db.close()
marked_data = utils.mark_subdomain(old_data, now_data)
return marked_data