def batch_update_async(batch_items: dict):
# todo break this batch into posts and account updates
# if we're lagging by a large margin, don't bother updating accounts
lag = time_delta(find_latest_item(mongo, 'Posts', 'created'))
if lag > 1000:
return
if use_multi_threading:
with log_exceptions():
thread_multi(
fn=update_account,
fn_args=[mongo, None],
dep_args=batch_items['accounts_light'],
fn_kwargs=dict(load_extras=False),
max_workers=num_threads,
)
thread_multi(
fn=update_account_ops_quick,
fn_args=[mongo, None],
dep_args=batch_items['accounts_light'],
fn_kwargs=None,
max_workers=num_threads,
)
else:
for account_name in batch_items['accounts_light']:
with log_exceptions():
update_account(mongo, account_name, load_extras=False)
update_account_ops_quick(mongo, account_name)
if use_multi_threading:
with log_exceptions():
thread_multi(
fn=update_account,
fn_args=[mongo, None],
dep_args=batch_items['accounts'],
fn_kwargs=dict(load_extras=True),
max_workers=num_threads,
)
thread_multi(
fn=update_account_ops_quick,
fn_args=[mongo, None],
dep_args=batch_items['accounts'],
fn_kwargs=None,
max_workers=num_threads,
)
else:
for account_name in batch_items['accounts']:
with log_exceptions():
update_account(mongo, account_name, load_extras=True)
update_account_ops_quick(mongo, account_name)
def cve_day(today_time=''):
"""
每日新增cve
"""
so = SQLite("../data/nvd.db")
today = time_delta(delta=0, format="%Y-%m-%d")
sql = "select CVE_Items_cve_CVE_data_meta_ID,CVE_Items_publishedDate,CVE_Items_cve_description_description_data_value from nvd_cve where CVE_Items_publishedDate like '%{time}%'".format(
time=today)
r = so.query(sql)
od = OrderedDict()
for i in r:
od[i[0]] = (i[0], i[1], i[2])
return od
def CVE_monitor():
"""
存量离线解析，增量在线更新
"""
json_list = []
# 取配置文件中控制存量解析的参数,得到待解析入库文件列表
conf = configparser.ConfigParser()
conf.read('../conf/info.conf')
stock = conf.get('CVE_Feeds', 'stock_control')
if stock == 'True':
zips = glob.glob('../data/json/nvdcve-1.1-*.json.zip')
for z in zips:
with zipfile.ZipFile(z) as zf:
print("[UNZIP]:%s" % z)
zf.extractall(path=path('../data/json'))
jsons_stock = glob.glob('../data/json/nvdcve-1.1-*.json')
jsons_stock = [
i for i in jsons_stock if 'modified' or 'recent' not in i
]
json_list.extend(jsons_stock)
conf.set('CVE_Feeds', 'stock_control', str(False))
with open('../conf/info.conf', 'w') as configfile:
conf.write(configfile)
else:
print('[!] WARNING:stock is False，dont parse stock data！')
# 下载最新的modified数据
cve_data_feeds = path('../data/nvd', time_delta() + 'nvd_data_feeds.html')
if not os.path.exists(cve_data_feeds):
r = requests.get(url)
with codecs.open(cve_data_feeds, 'wb') as f:
f.write(r.content)
if os.path.exists(cve_data_feeds):
with codecs.open(cve_data_feeds, 'rb') as f:
modified_zip, modified_link = CVE_modified(f)
json_download(modified_zip, modified_link)
json_list.extend([path('../data/json', 'nvdcve-1.1-modified.json')])
# 统一解析存量和增量CVE数据
for j in json_list:
print(j)
ret = json2sql(j, dir_name='../data/nvd.db')
return ret
def mitre_expdb_exp():
"""
从cve.mitre.org中提取CVE exp label，弥补nvd.nist中Resource中Exploit标记的不足
更新策略：全量更新，返回全部数据
:return cve_exp:
:type:dict
:value:{'cve-id':'edb-id'}
"""
hfile = path('../data/nvd', time_delta() + 'source-EXPLOIT-DB.html')
if not os.path.exists(hfile):
r = requests.get(
'https://cve.mitre.org/data/refs/refmap/source-EXPLOIT-DB.html')
html = r.content
with codecs.open(hfile, 'wb') as f:
f.write(html)
cve_exp = dict()
if os.path.exists(hfile):
with codecs.open(hfile, 'rb') as f:
soup = BeautifulSoup(f, 'html.parser')
for tr in soup.find_all('tr'):
exp_db = ''
cve = ''
for td in tr.find_all('td'):
t = str(td)
if re.search(r'EXPLOIT-DB:(\d+)', t):
r = re.search(r'EXPLOIT-DB:(\d+)', t)
exp_db = r.group(1)
elif re.search(r'(CVE-[\d]+-[\d]+)', t):
r = re.findall(r'(CVE-[\d]+-[\d]+)', t)
cve = []
for c in r:
cve.append(c)
else:
continue
if exp_db and cve:
if isinstance(cve, list):
for c in cve:
cve_exp[c] = exp_db
else:
cve_exp[cve] = exp_db
return 'cve.mitre.org', cve_exp
def batch_update_async(batch_items: dict):
# todo break this batch into posts and account updates
lag = time_delta(find_latest_item(mongo, 'Posts', 'created'))
if lag > 1000:
return
for identifier in batch_items['comments']:
with log_exceptions():
upsert_comment_chain(mongo, identifier, recursive=True)
# if we're lagging by a large margin, don't bother updating accounts
if lag > 500:
return
for account_name in batch_items['accounts_light']:
with log_exceptions():
update_account(mongo, account_name, load_extras=False)
update_account_ops_quick(mongo, account_name)
for account_name in batch_items['accounts']:
with log_exceptions():
update_account(mongo, account_name,
load_extras=True) # should be True
update_account_ops_quick(mongo, account_name)
"""
def solution(A):
A.sort()
odd = None
for a in A:
if not odd:
odd = a
elif odd == a:
odd = None
else:
if odd:
break
return odd
if __name__ == '__main__':
from utils import time_delta
X = [9, 3, 9, 3, 9, 7, 9]
X = [1, 1, 1]
print(solution(X))
time_delta(solution, X)