def run(self):
print(banner)
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f'[*] Starting OneForAll @ {dt}\n')
logger.log('INFOR', f'开始运行OneForAll')
self.domains = utils.get_domains(self.target)
if self.domains:
for self.domain in self.domains:
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
brute = AIOBrute(self.domain)
brute.run()
table_name = self.domain.replace('.', '_')
db_conn = database.connect_db()
self.datas = database.get_data(db_conn, table_name).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
loop.run_until_complete(asyncio.sleep(0.25)) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.close()
database.clear_table(db_conn, table_name)
database.save_db(db_conn, table_name, self.datas)
# 数据库导出
if not self.path:
self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
db_conn.close()
else:
logger.log('FATAL', f'获取域名失败')
logger.log('INFOR', f'结束运行OneForAll')
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.verify is None:
self.verify = config.enable_verify_subdomain
rename_table = self.domain + '_last'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain + '_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
# 不验证子域的情况
if not self.verify:
# 数据库导出
self.valid = None
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
return
# 开始验证子域工作
self.datas = db.get_data(self.domain).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.datas)
self.datas = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.datas, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.datas, self.port)
self.datas = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
db.clear_table(self.domain)
db.save_db(self.domain, self.datas)
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
db.close()
def main(self):
"""
OneForAll main process
:return: subdomain results
:rtype: list
"""
self.old_table = self.domain + '_old_result'
self.new_table = self.domain + '_now_result'
self.origin_table = self.domain + '_origin_result'
self.resolve_table = self.domain + '_resolve_result'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# Due to there will be a large number of dns resolution requests,
# may cause other network tasks to be error
brute = Brute(self.domain, word=True, export=False)
brute.check_env = False
brute.quite = True
brute.run()
# Database processing
self.deal_db()
# Mark the new discovered subdomain
self.data = self.mark()
# Export results without resolve
if not self.dns:
return self.export(self.domain)
# Resolve subdomains
self.data = resolve.run_resolve(self.domain, self.data)
# Save resolve results
resolve.save_data(self.resolve_table, self.data)
# Export results without HTTP request
if not self.req:
return self.export(self.resolve_table)
# HTTP request
self.data = request.run_request(self.domain, self.data, self.port)
# Save HTTP request result
request.save_data(self.domain, self.data)
# Add the final result list to the total data list
self.datas.extend(self.data)
# Export
self.export(self.domain)
# Scan subdomain takeover
if self.takeover:
subdomains = utils.get_subdomains(self.data)
takeover = Takeover(subdomains)
takeover.run()
return self.data
def main(self):
"""
OneForAll实际运行主流程
:return: 子域结果
:rtype: list
"""
self.old_table = self.domain + '_old_result'
self.new_table = self.domain + '_now_result'
self.origin_table = self.domain + '_origin_result'
self.resolve_table = self.domain + '_resolve_result'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = Brute(self.domain, word=True, export=False)
brute.check_env = False
brute.run()
# 有关数据库处理
self.deal_db()
# 标记新发现子域
self.data = self.mark()
# 不解析子域直接导出结果
if not self.dns:
return self.export(self.domain)
# 解析子域
self.data = resolve.run_resolve(self.domain, self.data)
# 保存解析结果
resolve.save_data(self.resolve_table, self.data)
# 不请求子域直接导出结果
if not self.req:
return self.export(self.resolve_table)
# 请求子域
self.data = request.run_request(self.domain, self.data, self.port)
# 保存请求结果
request.save_data(self.domain, self.data)
# 将最终结果列表添加到总的数据列表中
self.datas.extend(self.data)
# 数据库导出
self.export(self.domain)
# 子域接管检查
if self.takeover:
subdomains = utils.get_subdomains(self.data)
takeover = Takeover(subdomains)
takeover.run()
return self.data
def main(self):
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain+'_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
self.datas = db.get_data(self.domain).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.datas)
self.datas = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.datas, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.datas, self.port)
self.datas = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
loop.close()
db.clear_table(self.domain)
db.save_db(self.domain, self.datas)
# 数据库导出
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = config.result_save_path.joinpath(name)
dbexport.export(self.domain, db.conn, self.valid, self.path,
self.format, self.output)
rename_table = self.domain + '_last'
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
def picture(self, t_name):
picture = list()
tweet = Tweet()
status_media = {}
value = {}
collect = Collect()
api = collect.api()
status_media = tweet.get_timeline(api, t_name)
for media in status_media:
try:
if (hasattr(status_media[media], "extended_entities")):
value = status_media[media].extended_entities
for key in value["media"]:
picture.append(key["media_url"])
except:
pass
return picture
def username(self, t_name):
username = list()
tweet = Tweet()
status_media = {}
value = {}
collect = Collect()
api = collect.api()
status_media = tweet.get_timeline(api, t_name)
for media in status_media:
try:
if (hasattr(status_media[media], "extended_entities")):
value = status_media[media].extended_entities
for key in value["media"]:
username.append(status_media[media].user._json["name"])
except:
pass
return username
def handleDiscovery(self, dev, isNewDev, isNewData):
if (dev.addr in self.macAddress and isNewData and dev.getValueText(22) != None):
datas = dev.getValueText(22)
bat = int(datas[20:22],16)
hum = int(datas[28:32],16)/100
temp = int(datas[24:28],16)/100
#testData(bat, self.dataBat)
#testData(hum, self.dataHum)
#testData(temp, self.dataTemp)
caption = CaptionDAO.getByMacAddress(dev.addr)
print("----------------------------")
print(caption.name)
print("Batterie : ", bat)
print("Humidite : ", hum)
print("Temperature : ", temp)
print("----------------------------")
collectHum = Collect(0, self.dataHum, caption, datetime.datetime.now(),hum)
collectBat = Collect(0, self.dataBat, caption, datetime.datetime.now(),bat)
collectTemp = Collect(0, self.dataTemp, caption, datetime.datetime.now(),temp)
CollectDAO.create(collectHum)
CollectDAO.create(collectBat)
CollectDAO.create(collectTemp)
from usidpatch import usidpatch
from nextday import N3R
from models import Connection
from xignite import Fundamentals
conn, c = Connection()
import sqlite3
z = Prep()
z.get_df(3000)
master_df = pd.DataFrame()
for i in z.tuple_list:
a, b, c = i[0], i[1], i[2]
# print(a,b,c)
try:
y = Collect(a, b, c, thresh=.075, n=22)
if len(y.df_sub) > 0:
print('Ticker: ', a, '\n Updating: ', len(y.df_sub), ' records')
master_df = master_df.append(y.df_sub)
else:
pass
y.update_d1Table()
except:
y = Collect_Error(a)
print('Deleted: ', a)
usid = usidpatch(master_df)
usid.df = usid.df.reset_index()
usid.df = usid.df.set_index('USID')
usid.df.to_sql('stockDT',
conn,
if_exists='append',
def add_product(self, product):
return Collect.create({'collection_id': self.id, 'product_id': product.id})
def remove_product(self, product):
collect = Collect.find_first(collection_id=self.id, product_id=product.id)
if collect:
collect.destroy()
def task_extract_time(self):
"""
定时任务
:return:
"""
send_count = 0
send_report = 0
while True:
self.logger.info("=============================浮屠长生 开始第[%s]定时任务==================================" % send_count)
"""获取发送时间段"""
send_min_max_time = BusiBaseConfService().search_key_return_value((busi_config.SEND_MIN_MAX_TIME), "9|20")
"""数据分割"""
send_time = send_min_max_time.split("|")
"""获取休眠时间"""
batch_sleep_time = BusiBaseConfService().search_key_return_value((busi_config.BUSI_BATCH_SLEEP_TIME), "5|8")
sleep_time = batch_sleep_time.split("|")
"""获取允许重发的ip"""
allow_resend_server_ip = BusiBaseConfService().search_key_return_value((busi_config.ALLOW_RESEND_SERVER_IP))
"""休眠时长"""
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
"""获取当前时间"""
hour = int(time.strftime('%H', time.localtime(time.time())))
if hour > int(send_time[1]):
"""
收取邮件
"""
Collect().instance_server_ip()
self.logger.info("当前时间未在发送时间段内容，进行休眠，休眠时长[%s]分钟" % time_out)
time.sleep(60 * time_out)
elif hour < int(send_time[0]):
"""判断是否在发送时间段内，如果不存在的话跳过 0~9"""
"""若标志位为0"""
if send_report == 0 and allow_resend_server_ip == self.public_ip and self.get_task_list():
send_report = 1
"""更新账号"""
SendEmailAccountService().update_all_status_valid()
send_count = 0
self.logger.info("当前时间未在发送时间段内容，进行休眠，休眠时长[%s]分钟" % time_out)
time.sleep(60 * time_out)
elif not self.search_task_code_by_server_ip():
self.logger.info("【提取数据】不存在发送中的任务，流程结束")
time.sleep(60 * time_out)
elif send_count != 0 and not self.get_task_list():
"""如果不是当天第一次发送，且缓存记录为空"""
"""发送次数"""
send_count += 1
send_report = 0
"""
收取邮件
"""
Collect().instance_server_ip()
"""
数据二次营销 大于5次就不再营销
"""
if allow_resend_server_ip == self.public_ip:
self.resend_instance()
self.instance()
"""休眠"""
self.logger.info("当前第[%s]次发送，提取完成，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60 * time_out)
elif self.get_task_list():
"""如果缓存记录不为空，则休眠"""
send_report = 0
self.logger.info("当前第[%s]次发送，缓存信息不为空，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60 * time_out)
else:
"""如果缓存记录为空，进行消费生产"""
send_count += 1
send_report = 0
"""生产"""
self.instance()
self.logger.info("当前第[%s]次发送，生成缓存信息，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60*time_out)
self.logger.info("=============================浮屠长生 结束本次定时任务==================================")
def add_product(self, product):
return Collect.create({'collection_id': self.id, 'product_id': product.id})
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.verify is None:
self.verify = config.enable_verify_subdomain
old_table = self.domain + '_last'
new_table = self.domain + '_now'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain+'_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 不验证子域的情况
if not self.verify:
# 数据库导出
self.valid = None
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
return
# 开始验证子域工作
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.data)
self.data = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.data, self.port)
self.data = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
db.clear_table(self.domain)
db.save_db(self.domain, self.data)
# 数据库导出
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
takeover = Takeover(subdomains)
takeover.run()
def collect():
user_socket = request.environ.get("wsgi.websocket")
if user_socket:
while True:
message = json.loads(user_socket.receive())
if message.get("msg") == "exit":
break
try:
# 开始
msg = {
"msg": "开始上传",
"percent": 0,
}
user_socket.send(json.dumps(msg))
msg_re = message.get("msg").split("\n")
cu = message.get("cu")
merchants = message.get("merchants")
print(msg_re, cu)
brands = CsBrand.query.all()
print(brands[0].brand_name)
co = Collect(user_socket, cu, msg_re, merchants, brands)
co.execute()
# 结束
msg = {
"msg": "上传完成",
"percent": 100,
}
user_socket.send(json.dumps(msg))
# 更新页数
if 1 <= int(cu) <= 7:
new_pages = []
for i, items in enumerate(session['pages'].split()):
if i == int(cu):
new_pages.append(str(int(msg_re[0]) + 1))
else:
new_pages.append(items)
session['pages'] = ' '.join(new_pages)
#并更新文件
filename = "config/" + session['username'] + ".txt"
with open(filename, 'w') as f:
f.write(session['pages'])
#更新merchantID
new_merchants_list = []
for i, items in enumerate(session['merchants_list'].split()):
if i == int(cu):
new_merchants_list.append(merchants)
else:
new_merchants_list.append(items)
session['merchants_list'] = ' '.join(new_merchants_list)
# 并更新文件
filename = "config/" + session['username'] + "-mise.txt"
with open(filename, 'w') as f:
f.write(session['merchants_list'])
except:
continue
from flask import Flask, jsonify, request, render_template
from answer import Answer
from collect import Collect
import re
regex = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
r'localhost|' #localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$',
re.IGNORECASE)
answers = Answer()
collecting = Collect()
app = Flask(__name__)
@app.route('/', methods=['POST'])
def home():
if (request.method == 'POST'):
data = request.get_json()
reply = answers.getReply(data['q'])
return jsonify({'answer': reply})
@app.route("/page")
def page():
return render_template("page.html")
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.dns is None:
self.dns = config.enable_dns_resolve
if self.req is None:
self.req = config.enable_http_request
old_table = self.domain + '_last_result'
new_table = self.domain + '_now_result'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
original_table = self.domain + '_original_result'
db.copy_table(self.domain, original_table)
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 不解析子域直接导出结果
if not self.dns:
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
# 获取事件循环
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析子域
task = resolve.bulk_resolve(self.data)
self.data = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_resolve_result'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
# 不请求子域直接导出结果
if not self.req:
# 数据库导出
dbexport.export(resolve_table,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
# 请求子域
task = request.bulk_request(self.data, self.port)
self.data = loop.run_until_complete(task)
self.datas.extend(self.data)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_valid(self.data)
logger.log('INFOR', f'经验证{self.domain}有效子域{count}个')
# 保存请求结果
db.clear_table(self.domain)
db.save_db(self.domain, self.data, 'request')
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
takeover = Takeover(subdomains)
takeover.run()
def remove_product(self, product):
collect = Collect.find_first(collection_id=self.id, product_id=product.id)
if collect:
collect.destroy()
