def export2lmdb(self, lmdb_host, lmdb_dbname):
""" 匯出IMDB資料庫的資料到LMDB
:param lmdb_host:
:param lmdb_dbname:
:return:
"""
logger.info('export to lmdb')
since = 0
i = 0
while True:
movies = self.DbOperator.get_movies_to_export_lmdb(since, limit=self.DbOperator.LIMIT)
if movies:
for movie in movies:
movieid = movie[0]
imdbid = 'tt%07d' % int(movie[1])
i += 1
try:
imdbmovie = self.IMDbObj.get_movie(imdbid, movieid)
imdbmovie.save2db(lmdb_host, lmdb_dbname)
logger.info(
'%d, %s, %s, %s, %s' % (i, movieid, imdbid, imdbmovie['url'], imdbmovie['posterurl'])
)
except Exception as e:
logger.error('save db error: %s \r\n %s' % (imdbid, str(e)))
since += self.DbOperator.LIMIT
else:
break
def parser_subject(self):
try:
#certsearch?include_expired=true&include_subdomains=true&domain=jd.com
url = '{0}/certsearch?domain={1}&include_expired=true&include_subdomains=true'.format(
self.website, self.domain)
if self.req(url):
result = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-4]
total_page = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-1]
current_page = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-2]
self.token = ast.literal_eval(result)
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
if domains != None:
self.dns_names.extend(domains)
'''
while current_page < total_page:#重复请求，页面未变，该如何修改页面呢？
url = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p={0}".format(self.token)
if self.req(url):
print "xxxxx"
current_page = \
(self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
print current_page
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
self.dns_names.extend(domains)
else:
break
'''
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return
def req(self, url):
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
}
try:
resp = requests.get(url,
headers=headers,
timeout=self.timeout,
proxies=self.proxy)
if resp.status_code == 200:
if hasattr(resp, "text"):
return resp.text
else:
return resp.conten
else:
return None
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return None
def main():
logger.info(MSG_STARTING)
try:
config = get_config()
logger.info(config['app']['name'])
logger.info(MSG_GET_PAYLOAD)
payload = get_payload()
if config['debug']['postulant']:
logger.info(payload)
logger.info(MSG_GET_FIELDS_PAYLOAD)
fields_payload = get_fieds_payload()
logger.info(MSG_VALIDATE_PAYLOAD)
validate_payload(fields_payload, payload)
code = payload['code']
validate_code_response_200(code)
except QACliException as error:
logger.error(str(error))
logger.info(MSG_ENDING)
def post_info(self, data, api, res_str):
try:
response = requests.post(
api,
data=json.dumps(data),
headers={'auth-token': self.auth_header_val})
# 1. 字典序列化；2. 带请求头 content-type: application/json
if res_str == 'Client_Info': data = {'Client_Info': '....'}
print_info = '[{date_time}]POST[{res_str}]:{data} to server'.format(
date_time=datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'),
res_str=res_str,
data=data)
print(print_info)
logger.info(print_info)
# 获得返回结果
rep = json.loads(response.text)
return rep
except requests.ConnectionError as e:
msg = traceback.format_exc()
rep = {'code': 3, 'msg': msg}
logger.error(msg)
print(rep)
return rep
except ValueError as e:
msg = traceback.format_exc()
rep = {'code': 3, 'msg': msg}
logger.error(msg)
print(rep)
return rep
def check(self):
logger.info("Doing Zone Transfer Check ...")
try:
self.get_ns_server()
self.get_ns_server_nslookup()
except Exception as e:
logger.error("Error in {0}: {1}".format(__file__.split('/')[-1],e))
return False
if len(self.nsservers) == 0:
logger.info("None NS Server found for {0}.".format(self.domain))
return False
else:
for _ in self.nsservers:
has_zone_transfer = self.axfr_check(self.domain, _)
if has_zone_transfer != 0 and len(self.results) != 0:
logger.info("Zone Transfer Detected for {0}".format(self.domain))
#__file__ current file,the file contains current code.
fp = open(os.path.join(os.path.dirname(os.path.dirname(__file__)), "output", "{0}_zone_transfer.txt".format(self.domain)), "wb")
fp.writelines(self.results)
fp.close()
for item in self.results:
print item
return True
if has_zone_transfer == 0 or len(self.results) == 0:
logger.info("Zone Transfer False")
return False
def run(self):
try:
timestemp = time.time()
url = "{0}?0.{1}&callback=&k={2}&page=1&order=default&sort=desc&action=moreson&_={3}&verify={4}".format(
self.url, timestemp, self.domain, timestemp, self.verify)
#response = req.get(url,proxies=self.proxy).content
# no proxy needed for this class
response = req.get(url).content
result = json.loads(response)
if result.get('status') == '1':
for item in result.get('data'):
if is_domain(item.get('domain')):
self.domain_name.append(item.get('domain'))
elif result.get('status') == 3:
logger.warning("chaxun.la api block our ip...")
logger.info("input you verify_code")
# print('get verify_code():', self.verify)
# self.verify_code()
# self.run()
self.domain_name = list(set(self.domain_name))
except Exception as e:
logger.error("Error in {0}: {1}".format(__file__.split('/')[-1], e))
finally:
logger.info("{0} found {1} domains".format(self.engine_name, len(self.domain_name)))
return self.domain_name,self.smiliar_domain_name,self.email
def push(cls, msg):
""" Send push notifications
https://about.pushed.co/docs/api#api-method-push
"""
logger.info('Pushing message via Pushed service...')
response = requests.post(cls.API_URL + 'push',
data={
'app_key': app_conf.pushed_app_key,
'app_secret': app_conf.pushed_app_secret,
'target_type': 'app',
'content': msg,
})
response_json = json.loads(response.text)
if 'error' in response_json:
error_msg = response_json['error']['message']
logger.error(f'Message not sent. Error: {error_msg}')
return
response_type = response_json['response']['type']
if response_type == 'shipment_successfully_sent':
logger.info('Message successfully delivered to Pushed service')
else:
logger.error(f'Wrong response type from Pushed: {response_type}')
def evaluate_config(self, cfg, **kwargs):
x_train, x_test, y_train, y_test = cfg['x_train'], cfg['x_test'], cfg[
'y_train'], cfg['y_test']
params = cfg['params']
self.set_params(params)
logger.info('Testing:' + str(params))
#history = [x for x in x_train.values], [y for y in y_train.values]
#predictions = []
errors = []
scores = []
# Test the model on the train set
model_fit = self.fit(x_train, y=y_train)
train_predictions = model_fit.predict(x_train)
predictions = model_fit.predict(x_test)
if predictions is None or len(predictions) <= 0:
logger.error('No predictions made for config {}'.format(
str(params)))
else:
return {
'y': y_test,
'y_pred': np.array(predictions),
'y_train': y_train,
'y_train_pred': np.array(train_predictions),
'model': str(self),
'params': self.get_params(),
'errors': errors,
'scores': scores
}
def evaluate_config(self, cfg, **kwargs):
x_train, x_test, y_train, y_test = cfg['x_train'], cfg['x_test'], cfg[
'y_train'], cfg['y_test']
params = cfg['params']
self.set_params(params)
logger.info('Testing:' + str(params))
history = [x for x in x_train.values], [y for y in y_train.values]
predictions = []
errors = []
scores = []
# Test the model on the train set
model_fit = self.fit(x_train, y=y_train)
train_predictions = model_fit.predict(x_train)
predictions = model_fit.predict(x_test)
if False:
for i in range(len(x_test)):
# logger.info('grid_search {} {} {}/{}'.format(self.name, str(order), i+1, len(x_test)))
yhat = 0
try:
_left = len(x_test) - i
model_fit = self.fit(history[0],
y=history[1],
params=params)
scores.append(model_fit.score(history[0], y=history[1]))
# Save test set prediction
forecast = model_fit.predict(
x_test[i:]
) # ToDO: Save model, use new model only if score increases
yhat = float(
forecast[0]) # Forecast next element of the test set
if np.isnan(yhat):
yhat = 0
except Exception as e:
errors.append('Error at step {} for config {}: {}'.format(
i, str(params), str(e)))
pass
finally:
predictions.append(yhat) # add forecasted y to predictions
history[0].append(
x_test.iloc[i].values
) # Add an element from test set to history
history[1].append(
y_test.iloc[i]
) # Add an element from test set to history
if predictions is None or len(predictions) <= 0:
logger.error('No predictions made for config {}'.format(
str(params)))
else:
return {
'y': y_test,
'y_pred': np.array(predictions),
'y_train': y_train,
'y_train_pred': np.array(train_predictions),
'model': str(self),
'params': params,
'errors': errors,
'scores': scores
}
def fit(self, x, **kwargs):
params = kwargs.get('params')
try:
if params.get('beta'):
# Use Augmented Dickey-Fuller test: try to reject the null hypothesis that the series
# has a unit root, i.e. it's stationary. Statsmodels implementation has issues with window
# size lower than 6, so adapt it for now.
# Using the p-value, check if it's possible to reject the null hypothesis at 5 % significance
# level and than choose Simple Exponential Smoothing or Holt's Exponential Smoothing (additional
# trend factor).
r = adfuller(x) if x.shape[0] > 6 else adfuller(x, maxlag=4)
pvalue = r[1]
if pvalue < 0.05:
self.model = ExponentialSmoothing(x, trend=None, seasonal=None) \
.fit(smoothing_level=params.get('alpha'))
else:
self.model = ExponentialSmoothing(x, trend='additive', seasonal='additive',
seasonal_periods=params.get('seasonal_periods')) \
.fit(smoothing_level=params.get('alpha'), smoothing_slope=self.params.get('beta'))
else:
self.model = ExponentialSmoothing(x, trend=None, seasonal=None) \
.fit(smoothing_level=params.get('alpha'))
self.params = params
return self.model
except (ValueError, np.linalg.linalg.LinAlgError):
logger.error(
'Exponential Smoothing convergence error (a:{},b:{})'.format(
params.get('alpha'), params.get('beta')))
return None
def get_related_domains(self):
if self.hash_codes > 0:
main_of_domain = tldextract.extract(self.domain).domain
for hash in self.hash_codes:
try:
url = self.detail_url.format(hash)
resp = req.get(url,
timeout=self.timeout,
proxies=self.proxy,
verify=False)
if resp:
formated_string = (self.result[6:-1]).replace("\n", "")
tmplist = ast.literal_eval(
formated_string) #把list格式的string转换成list
main_of_cn_domain = tmplist[0][1][1].split(",")[-1]
if "CN\u003d" in main_of_cn_domain: #有可能响应的内容为空，判断一下
main_of_cn_domain = main_of_cn_domain.replace(
"CN\u003d", "")
main_of_cn_domain = tldextract.extract(
main_of_cn_domain).domain
if main_of_domain in main_of_cn_domain: #判断cn中的域名是否和要查询的域名相似
self.related_domain_name.extend(
tmplist[0][1][-1])
else:
continue
except Exception, e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return
def parser_subject(self):
try:
url = self.search_api.format(self.domain)
if self.req(url):
formated_string = (self.result[6:-1]).replace(
"\n", "").replace("[", "").replace("]", "")
self.token = formated_string.split(",")[-4]
#print result
#total_page = (self.result[6:-1]).replace("\n","").replace("[","").replace("]","").split(",")[-1]
#current_page = (self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
#it seems that no need to switch pages, the main purpose of this API is to get the related domains. it's enough using items in one page
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
if domains != None:
self.domain_name.extend(domains)
for item in formated_string.split(","): #
if len(
item
) >= 40 and " " not in item: #sSWg6vIw46sI1eNhDlilAaanXC9htQlVuMuHJWqyNr8=
item = item.strip("\"")
#item = unicode(item).decode()
item = item.replace("\\u003d", "=")
self.hash_codes.append(item)
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return
def sign(self):
jobs_failed = []
for job_class in self.job_list:
job = job_class(self)
# 默认使用移动设备User-agent,否则使用PC版User-Agent
# if job.is_mobile:
# job.session.headers.update({
# 'User-Agent': self.user.ua
# })
# else:
# job.session.headers.update({
# 'User-Agent': self.user.ua_pc})
try:
job.run()
except Exception as e:
logger.error('# 任务运行出错: ' + repr(e))
traceback.print_exc()
if not job.job_success:
jobs_failed.append(job.job_name)
print('=================================')
print('= 任务数: {}; 失败数: {}'.format(len(self.job_list), len(jobs_failed)))
if jobs_failed:
print('= 失败的任务: {}'.format(jobs_failed))
else:
print('= 全部成功 ~')
print('=================================')
return len(jobs_failed) == 0
def do_search(self):
try:
url = "http://{0}/search?num={1}&start={2}&hl=en&meta=&q={3}".format(
self.server, self.quantity, self.counter, self.word)
except Exception, e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
def import_listfile(self):
""" 將IMDB的資料庫文字檔匯進DB
如果匯入時發生問題，則利用上次的備份檔進行還原
:return:
"""
logger.info('import listfile to db')
try:
self.make_csvdir()
imdbpy2sql_path = os.path.normpath('%s/updateimdb/bin/' %
DIR_CRONTAB)
cmd = 'python %s/imdbpy2sql.py -d %s -u %s -c %s -i table' \
% (imdbpy2sql_path, self.Downloader.get_download_dir_path(), self.DbOperator.URI, self.get_csvdir())
subprocess.check_call(cmd, shell=True)
except Exception:
logger.error(
'error occurred during import listfile to db, try to restore the older db'
)
self.DbBackup.restoredb(
self.DbOperator.DB,
'%s/%s.bak' % (self.DbBackup.BACKUPDIR, self.DbOperator.DB))
logger.info('restore success!')
raise
logger.info('import listfile to db is success!')
def new_request(wrapUrl, timeout=None):
# create dict to save request data
kwargs = dict(wrapUrl.kwargs)
method = kwargs['method']
# check if exist timeout, else use default value
if timeout is not None:
kwargs['timeout'] = timeout
else:
kwargs['timeout'] = DEFAULT_TIMEOUT
kwargs = define_request(wrapUrl._url, **kwargs)
try:
session = requests.Session()
request = session.request(method, wrapUrl._url, **kwargs)
except Exception as e:
logger.debug(e)
finally:
if request is not None:
if request.status_code != 200:
logger.error("%s (%s)" % (request.reason, request.status_code))
else:
return request
def do_search(self):
try:
url = "https://{0}/customsearch/v1?key={1}&highRange={2}&lowRange={3}&cx={4}&start={5}&q={6}".format(
self.server, self.api_key, self.highRange, self.lowRange,
self.cse_id, self.counter, self.word)
except Exception, e:
logger.error(e)
def do_search(self):
try:
url = "http://{0}/search?text={1}&numdoc=50&lr={2}".format(
self.server, self.word,
self.counter) # %40=@ 搜索内容如：@meizu.com;在关键词前加@有何效果呢？，测试未发现不同
except Exception, e:
logger.error(e)
def do_search(self):
try:
api = shodan.Shodan(self.apikey)
self.results = api.search(self.word)
self.totalresults += str(self.results)
except shodan.APIError, e:
logger.error(e)
def do_search(self):
try:
url = "http://api.cognitive.microsoft.com/bing/v7.0/search?q={0}&mkt=en-us".format(
self.word, self.counter) # 这里的pn参数是条目数
except Exception, e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
def search(self):
temp_domains = []
try:
main_of_domain = tldextract.extract(self.domain).domain
c = censys.certificates.CensysCertificates(
api_id=self.api_id, api_secret=self.api_secret)
# iterate over certificates that match a search
fields = ["parsed.subject_dn",
"parsed.fingerprint_sha256"] #parsed.issuer_dn
for cert in c.search("{0}".format(self.domain), fields=fields):
#print cert["parsed.subject_dn"]
cn_domain = cert["parsed.subject_dn"].split(",")[-1].split(
"=")[-1] #cn一定是在最后吗
main_of_cn_domain = tldextract.extract(cn_domain).domain
if main_of_domain in main_of_cn_domain:
detail = c.view(
cert["parsed.fingerprint_sha256"]
) #print c.view("a762bf68f167f6fbdf2ab00fdefeb8b96f91335ad6b483b482dfd42c179be076")
#print detail
#print detail["parsed"]["names"]
temp_domains.extend(detail["parsed"]["names"])
temp_domains = list(set(temp_domains))
except Exception, e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
def build_wordlist(wordlist_file):
# 读入字典文件
fd = open(wordlist_file, "rb")
logger.info("start wordlist build...")
global raw_words
raw_words = fd.readlines()
logger.info("This dictionary contains %s rows" % len(raw_words))
print "This dictionary contains %s rows..." % len(raw_words)
if len(raw_words) == 0:
logger.error("This dictionary is empty...")
fd.close()
found_resume = False
words = Queue.Queue()
for word in raw_words:
word = word.rstrip()
# 这功能暂时没开
if conf['resume'] is not None:
if found_resume:
words.put(word)
else:
if word == conf['resume']:
found_resume = True
print "Resuming wordlist from : %s" % conf['resume']
else:
words.put(word)
logger.info("wordlist build is complete...")
return words
def do_search(self):
try:
url = "http://{0}/web?q={1}&pu=100&page={2}".format(
self.server, self.word,
self.counter) # %40=@ 搜索内容如：@meizu.com;在关键词前加@有何效果呢？，测试未发现不同
except Exception, e:
logger.error(e)
def req(self, req_method, url, params=None):
params = params or {}
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Referer': 'https://dnsdumpster.com'
}
try:
if req_method == 'GET':
resp = req.get(url,
headers=headers,
timeout=self.timeout,
proxies=self.proxy)
else:
resp = req.post(url,
data=params,
headers=headers,
timeout=self.timeout,
proxies=self.proxy)
if hasattr(resp, "text"):
return resp.text
else:
return resp.content
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return None
def check(self):
logger.info("Doing Zone Transfer Check ...")
try:
self.get_ns_server()
self.get_ns_server_nslookup()
except Exception as e:
logger.error(e)
if len(self.nsservers) == 0:
logger.info("None NS Server found for {0}.".format(self.domain))
else:
for _ in self.nsservers:
has_zone_transfer = self.axfr_check(self.domain, _)
if has_zone_transfer != 0 and len(self.results) != 0:
logger.info("Zone Transfer Detected for {0}".format(
self.domain))
fp = open(
"..\\output\\{0}_zone_transfer.txt".format(self.domain),
"wb")
fp.writelines(self.results)
fp.close()
for item in self.results:
print item
if has_zone_transfer == 0 or len(self.results) == 0:
logger.info("Zone Transfer False")
def update_movie_imdbid(self):
""" 更新IMDB資料庫電影的IMDBID
如果更新某部電影的IMDBID，出現錯誤連續達到10次，則不繼續嘗試更新IMDBID（可能會是網路等問題）
:return:
"""
logger.info('update imdb_id field')
count = 0
max_try = 10
while True:
movies = self.DbOperator.get_null_imdbid_movies()
if len(movies) == 0:
break
for movie in movies:
try_times = 0
count += 1
try:
logger.info('%s: %s' % (count, self.get_imdbid_result(movie[0])))
except Exception:
try_times += 1
time.sleep(3)
if try_times == max_try:
logger.error(traceback.format_exc())
return
logger.info('import db to table is success!')
def export2lmdb(self, lmdb_host, lmdb_dbname):
""" 匯出IMDB資料庫的資料到LMDB
:param lmdb_host:
:param lmdb_dbname:
:return:
"""
logger.info('export to lmdb')
since = 0
i = 0
while True:
movies = self.DbOperator.get_movies_to_export_lmdb(
since, limit=self.DbOperator.LIMIT)
if movies:
for movie in movies:
movieid = movie[0]
imdbid = 'tt%07d' % int(movie[1])
i += 1
try:
imdbmovie = self.IMDbObj.get_movie(imdbid, movieid)
imdbmovie.save2db(lmdb_host, lmdb_dbname)
logger.info('%d, %s, %s, %s, %s' %
(i, movieid, imdbid, imdbmovie['url'],
imdbmovie['posterurl']))
except Exception as e:
logger.error('save db error: %s \r\n %s' %
(imdbid, str(e)))
since += self.DbOperator.LIMIT
else:
break
def update_movie_imdbid(self):
""" 更新IMDB資料庫電影的IMDBID
如果更新某部電影的IMDBID，出現錯誤連續達到10次，則不繼續嘗試更新IMDBID（可能會是網路等問題）
:return:
"""
logger.info('update imdb_id field')
count = 0
max_try = 10
while True:
movies = self.DbOperator.get_null_imdbid_movies()
if len(movies) == 0:
break
for movie in movies:
try_times = 0
count += 1
try:
logger.info('%s: %s' %
(count, self.get_imdbid_result(movie[0])))
except Exception:
try_times += 1
time.sleep(3)
if try_times == max_try:
logger.error(traceback.format_exc())
return
logger.info('import db to table is success!')
def ph_request(url=None, header={}, value=None):
if url is None:
logger.error("URL is not found...")
exit(0)
else:
logger.info("Target url is {}".format(url))
if len(header) == 0:
logger.warning("Header is empty...")
header = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36'
}
try:
driver = webdriver.PhantomJS(executable_path=set_phantomjs_path())
except WebDriverException:
logger.error("phantomjs path error...")
exit(0)
try:
driver.get(url)
time.sleep(3)
finally:
return driver.page_source
def do_search(self):
try:
url = "https://{0}/?q={1}".format(self.server, self.word)
#must use https
except Exception, e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
def utask_process(self):
utask_res = {"utask_id": self.utask_id, "status_code": 5,
"run_time": "", "message": "",}
start_time = datetime.datetime.now()
try:
img_file = self.get_img() # 获取img文件
res = subprocess.Popen(
'sudo {update_cmd} dera update-fw {node} -f {img_file} -y 2>&1'.
format(
update_cmd=self.cmd_file,
node=self.node,
img_file=img_file
),
shell=True,
stdout=subprocess.PIPE
)
res.wait()
res_stdout = res.stdout.read().decode('utf-8')
end_time = datetime.datetime.now()
run_time = end_time - start_time
utask_res["run_time"] = str(run_time)
if 'Success' in res_stdout:
utask_res["status_code"] = 2
else:
utask_res["status_code"] = 3
utask_res["message"] = res_stdout
except Exception as e:
end_time = datetime.datetime.now()
run_time = end_time - start_time
utask_res["run_time"] = str(run_time)
utask_res["status_code"] = 3
msg = traceback.format_exc()
utask_res["message"] = msg
logger.error(msg)
json.dump(utask_res, open(self.task_res_path, 'w'))
def destroy_nodes(self):
L.error("DESTROYING ALL NODES FOR MANAGER %s"%self.name)
for node in self.nodes:
L.warning("KILLING NODE: %s"%node)
try:
call("docker-machine kill %s && sleep 10"%node)
call("docker-machine rm %s"%node)
except: pass
def PostData(self, payload):
data = payload
try:
r = self.s.post(self.url, data=data, headers=self.headers)
except:
logger.error("Time out...")
exit(0)
return r.text.encode('utf-8')
def GetLen(self, payload):
data = quote(payload)
try:
r = self.s.get(self.url + "?" + data, headers=self.headers)
except:
logger.error("Time out...")
exit(0)
lens = len(r.text.encode('utf-8'))
return lens
def PostBuildData(self, payload, llen):
data = payload
try:
r = self.s.post(self.url, data=data, headers=self.headers)
except:
logger.error("Time out...")
exit(0)
lens = len(r.text.encode('utf-8'))
# print r.text.encode('utf-8')
if lens == llen:
return True
else:
return False
def PostTimeData(self, payload, dtime):
data = payload
ptime = time.time()
try:
r = self.s.post(self.url, data=data, headers=self.headers)
except:
logger.error("Time out...")
exit(0)
rr = r.text.encode('utf-8')
ntime = time.time()
if ntime - ptime > dtime:
return True
else:
return False
def dir_bruter(word_queue, target_url, stime, extensions=None, pbar=None):
while not word_queue.empty():
pbar.update(1)
attempt = word_queue.get()
attempt_list = []
# 检查是否有文件扩展名，如果没有就是我们要暴力破解的路径
# if "." not in attempt:
# attempt_list.append("%s/" % attempt)
# else:
attempt_list.append("%s" % attempt)
# 如果我们想暴力扩展
if extensions:
for extension in extensions:
if extension == ".swp":
attempt_list.append("/.%s%s" % (attempt.strip('/'), extension))
else:
attempt_list.append("%s%s" % (attempt, extension))
# 迭代我们想要尝试的文件列表
for brute in attempt_list:
url = "%s%s" % (target_url, urllib.quote(brute))
# print url
try:
headers = {}
headers["User-Agent"] = conf['ua']
r = urllib2.Request(url, headers=headers)
# pbar.update(1)
try:
response = urllib2.urlopen(r, timeout=2)
except:
logger.error("Time out...")
continue # 有可能卡死
# 请求完成后睡眠
time.sleep(stime)
if response.code != 404:
logger.info("Get !!!!" + url)
tqdm.write("[%d] => %s" % (response.code, url))
except urllib2.URLError, e:
if hasattr(e, 'code') and e.code != 404:
tqdm.write("!!! %d => %s" % (e.code, url))
def normal_injection(select=None, source=None, conditions=None, limit=0, dealpayload=None, data=None, isStrings=False, isCount=False, sqlirequest=None):
# 生成随机字符串padding
padding = random_string()
payload = dealpayload.construct_normal_payload(select=select, source=source, conditions=conditions, limit=limit, padding=padding)
if sqlirequest == "GET":
result = data.GetData(payload)
elif sqlirequest == "POST":
result = data.PostData(payload)
else:
logger.error("sqlirequest error...")
exit(0)
if isCount:
return int(UnpackFunction(result, padding))
elif isStrings:
return UnpackFunction(result, padding)
else:
logger.error("Something error...")
def main():
# main
# activate API
if api.lunch():
logger.info("API started")
else:
logger.error("API failed start")
GPIO.setmode(GPIO.BOARD)
pir = 26
GPIO.setup(pir, GPIO.IN)
logger.info("Application started")
while True:
while active.getActive():
if GPIO.input(pir):
time.sleep(1)
if GPIO.input(pir):
active.setDetection("true")
sender.emailAlert()
active.setDetection("false")
time.sleep(0.2)
def import_listfile(self):
""" 將IMDB的資料庫文字檔匯進DB
如果匯入時發生問題，則利用上次的備份檔進行還原
:return:
"""
logger.info('import listfile to db')
try:
self.make_csvdir()
imdbpy2sql_path = os.path.normpath('%s/updateimdb/bin/' % DIR_CRONTAB)
cmd = 'python %s/imdbpy2sql.py -d %s -u %s -c %s -i table' \
% (imdbpy2sql_path, self.Downloader.get_download_dir_path(), self.DbOperator.URI, self.get_csvdir())
subprocess.check_call(cmd, shell=True)
except Exception:
logger.error('error occurred during import listfile to db, try to restore the older db')
self.DbBackup.restoredb(self.DbOperator.DB, '%s/%s.bak' % (self.DbBackup.BACKUPDIR, self.DbOperator.DB))
logger.info('restore success!')
raise
logger.info('import listfile to db is success!')
def time_injection(select=None, source=None, conditions=None, limit=0, dealpayload=None, data=None, times=0, isNumber=False, isStrings=False, isCount=False, sqlirequest=None):
"""
使用二分法注入逻辑注入数据
"""
chartbl = []
if isNumber:
chartbl = NumberInjectionList
elif isStrings:
chartbl = CharInjectionList
elif isCount:
chartbl = CountInjectionList
else:
logger.error("injection target error...")
exit(0)
while 1 != len(chartbl):
position = (len(chartbl) >> 1)
posValue = chartbl[position]
payload = dealpayload.construct_time_payload(select=select, source=source, conditions=conditions, limit=limit, compare=posValue)
# logger.debug("testing payload:" + payload)
if sqlirequest == "GET":
result = data.GetTimeData(payload, times)
elif sqlirequest == "POST":
result = data.PostTimeData(payload, times)
else:
logger.error("sqlirequest error...")
exit(0)
if result:
if type(chartbl) != xrange:
chartbl = chartbl[position:]
else:
# xrange() - extended virtual charset used for memory/space optimization
chartbl = xrange(chartbl[position], chartbl[-1] + 1)
else:
if type(chartbl) != xrange:
chartbl = chartbl[:position]
else:
chartbl = xrange(chartbl[0], chartbl[position])
# 判断结果
if len(chartbl) == 1:
# logger.debug("injection success,the chartbl[0]+1 is %d", chartbl[0]+1)
if isCount & chartbl[0] == 100:
logger.error("Count or Length >100...")
return 100
return chartbl[0] + 1
def error_out(context, name, message, e=None):
if context.obj['json']: return json.dumps({name: message})
L.error('%s:\t%s'%(name, message))
if isinstance(e, Exception): raise e
def run_content(self):
if len(self.columns_name) == 0:
SqliColumns.get_columns(self)
# 循环解包，进入注入
for database_name in self.columns_name:
for table_name in self.columns_name[database_name]:
# 获取数据的条数，如果小于设置的self.content_count，那需要设置条数等于self.content_count
content_counts = self.get_content_count(database_name, table_name)
if content_counts == 0:
logger.warning('Database %s Table %s is empty...' % (database_name, table_name))
continue
elif content_counts != self.content_count:
logger.debug('Database %s Table %s content amount change to %d' % (database_name, table_name, content_counts))
self.content_count = content_counts
else:
pass
# 声明一个表储存数据
content = PrettyTable(list(self.columns_name[database_name][table_name]))
content.padding_width = 1
content.align = "r"
# 每个表都要注入指定条数那么多次
for limits in xrange(self.content_count):
# 声明一个队列，储存返回的值
result = Queue.Queue()
# 声明线程队列、结果队列和最终插入table的数据队列
threads = []
results = []
contents = []
# 开始多线程的注入
logger.debug("Start multithreading Sqli...")
for column_name in self.columns_name[database_name][table_name]:
# 开始一个线程注入一个字段
try:
t = threading.Thread(target=self.get_content, name='thread for %s' % column_name,
args=(result, database_name, table_name, column_name, limits))
t.start()
except:
logger.error('Thread error...')
threads.append(t)
# 等待所有线程结束
for t in threads:
t.join()
# 注入处理返回数据，插入content中的一条
while not result.empty():
results.append(result.get())
# 处理返回的数据
for i in list(self.columns_name[database_name][table_name]):
for item in results:
if item[0] == i:
contents.append(item[1])
else:
continue
# 插入数据
content_str = ','.join(contents)
logger.info("Sqli success content is %s" % content_str)
content.add_row(contents)
# 输出表
logger.debug("Database %s Table %s sqli success..." % (database_name, table_name))
print "[*] Database %s Table %s content:" % (database_name, table_name)
print content
def oparser():
# 处理参数
parser = OptionParser()
parser.version = conf['version']
parser.add_option("--version", "-v", dest="showVersion", action="store_true",
help="show program's version and exit")
# 必选参数
target = OptionGroup(parser, "Target", "At least one of these options has to be provided to define the target(s)")
target.add_option("-u", dest="url", help="Target URL")
target.add_option("--test", dest="test", action="store_true", help="auto test")
# 可选参数
opt = OptionGroup(parser, "Options", "Optional parameters")
opt.add_option("-t", "--thread", dest="thread", type="int", help="thread number(default 30)")
opt.add_option("-e", "--ext", dest="extensions", action="store_true",
help="Whether to use file extension(default false)")
opt.add_option("-f", "--filename", dest="filename", help="Scanner dictionary (default ./php.txt)")
opt.add_option("-s", "--sleeptime", dest="stime", type="int", help="Each request of sleep time (default 1)")
opt.add_option("-l", dest="loglevel", type="int", help="log level(1-5) "
"1, CRITICAL; "
"2, ERROR(default); "
"3, WARN; "
"4, INFO; "
"5, DEBUG;")
parser.add_option_group(target)
parser.add_option_group(opt)
(args, _) = parser.parse_args(sys.argv)
if args.showVersion:
print parser.version
print "-- By LoRexxar"
exit(0)
if not (args.url or args.test):
errMsg = "missing a mandatory option (-u) or (--test), "
errMsg += "use -h for basic or --help for advanced help"
parser.error(errMsg)
conf['url'] = (args.url.rstrip('/').rstrip('"').rstrip('\'') if args.url else "http://testphp.vulnweb.com/")
# 处理下url用来log的名字
name = re.findall("[\w\.-]+", conf['url'])
try:
conf['name'] = (name[1] if len(name) == 2 else name[0])
except IndexError:
errMsg = "url input error!"
logger.error("url matching fail!")
parser.error(errMsg)
conf['thread'] = (args.thread if args.thread else 30)
if conf['thread'] < 1 or conf['thread'] > 50:
# 线程数为0-50
errMsg = "thread value error (1-50)"
parser.error(errMsg)
# 设置睡眠时间
conf['stime'] = (args.stime if args.stime else 1)
if conf['stime'] < 0 or conf['stime'] > 10:
# 睡眠时间为0-10
errMsg = "time value error (0-10)"
parser.error(errMsg)
conf['loglevel'] = (args.loglevel if args.loglevel else 2)
if conf['loglevel'] < 1 or conf['loglevel'] > 5:
# loglevel: 1-5
errMsg = "loglevel value error(input 1-5)"
parser.error(errMsg)
if conf['loglevel'] == 1:
conf['loglevel'] = logging.CRITICAL
elif conf['loglevel'] == 2:
conf['loglevel'] = logging.ERROR
elif conf['loglevel'] == 3:
conf['loglevel'] = logging.WARN
elif conf['loglevel'] == 4:
conf['loglevel'] = logging.INFO
elif conf['loglevel'] == 5:
conf['loglevel'] = logging.DEBUG
else:
conf['loglevel'] = logging.ERROR
# 开启log
log(conf['name'], conf['loglevel'])
# 设置扫描器字典
wordlist_file = (args.filename if args.filename else "./dic/php.txt")
word_queue = build_wordlist(wordlist_file)
extensions = ['.bak', '.orig', '.inc', '.swp', '~']
# 进度条
pbar = tqdm(total=word_queue.qsize(), leave=False)
# 开始扫描
if args.extensions:
tqdm.write('start scanning with extensions...')
for i in range(conf['thread']):
t = threading.Thread(target=dir_bruter, args=(word_queue, conf['url'], conf['stime'], extensions, pbar))
t.start()
else:
tqdm('start scanning...')
for i in range(conf['thread']):
t = threading.Thread(target=dir_bruter, args=(word_queue, conf['url'], conf['stime'], None, pbar))
t.start()