def __init__(self, url, max_depth):
self.seed_url = url
self.result_urls = []
self.queue = Queue()
self.max_depth = int(max_depth)
self.cur_depth = 1
self.db = DBHelper()
self.logger = get_logger(__name__)
def main(directories: []):
# directories = [
# # "/media/georgen/LOCAL DISK/George_Ndirangu/Music/",
# # "/media/georgen/LOCAL DISK/George_Ndirangu/Videos/Music/",
# # "/media/georgen/LOCAL DISK/George_Ndirangu/Learning/py/accoustid-music-cleaner/test-res/tinker"
# ]
list_of_paths = []
completed = []
failed = []
for directory in directories:
list_of_paths += dloader.load_file_paths(directory)
processing_queue = Queue()
Queue.populate_queue(list_of_paths, processing_queue)
for item in processing_queue.get_items():
item.update_status(QueueItemStatus.PROCESSING)
result: Optional[MusicFile] = accoustid.search(item.music_file)
if isinstance(result, MusicFile):
item.update_status(QueueItemStatus.SUCCESS)
on_success(completed, item.music_file)
processing_queue.remove(item)
elif isinstance(result, acoustid.WebServiceError):
print(
f"WARNING: Network error encountered. Will try to handle this another day.For now moving on \n {result}"
)
else:
# fallback to the next searching method
print("WARNING: fallback to the next searching method ")
print(
"WARNING: Musicbrainz search has been disabled for now..until it achives better perforance"
)
# result = musicbrainz_search.search(item.music_file)
#
# if isinstance(result, MusicFile):
# item.update_status(QueueItemStatus.SUCCESS)
# on_success(completed, item.music_file)
# processing_queue.remove(item)
# elif isinstance(result, musicbrainzngs.WebServiceError):
# logging.warning(
# f"Network error encountered. Will try to handle this another day.For now moving on \n {result}")
# else:
item.update_status(QueueItemStatus.FAILED)
on_fail(failed, item.music_file)
# processing_queue.remove(item)
print(
f"WARNING: Both accoustid and mb failed for file {item.music_file.file_path}."
)
print("completed...........")
print(completed)
print("failed...........")
print(failed)
def level_visit_one_queue(root):
queue = Queue()
queue.enqueue(root)
while queue:
queue_head = queue.dequeue()
print(queue_head.value)
if queue_head.left:
queue.enqueue(queue_head.left)
if queue_head.right:
queue.enqueue(queue_head.right)
def __init__(self, command_prefix: str, intents: discord.Intents,
**kwargs):
super().__init__(command_prefix=command_prefix,
intents=intents,
**kwargs)
self.logger = set_logger()
self.verification_queue = dict()
self.event_queue = Queue()
self.obj_cache = Cache()
self.running = True
self.default_invite = \
"https://discord.com/api/oauth2/authorize?client_id=767842408758771742&permissions=51200&scope=bot"
self.reddit = self.create_reddit_connection()
self.load_data()
def __init__(self, drafter: Drafter, draft: Draft, draft_seat: DraftSeat):
super().__init__()
self._drafter = drafter
self._draft = draft
self._draft_seat = draft_seat
self._pool = Cube()
self._messages: t.List[t.Mapping[str, t.Any]] = []
self._pick_counter = 0
self._booster_queue = Queue()
self._pick_queue = Queue()
self._out_queue = Queue()
self._current_booster: t.Optional[DraftBooster] = None
self._terminating = threading.Event()
self._booster_pusher = threading.Thread(target=self._draft_loop)
self._connect_lock = threading.Lock()
self._consumer: t.Optional[WebsocketConsumer] = None
def push_third_party_vulnerabilities(self, vulnerabilities_id):
"""
Pushed to a third-party vulnerability management platform
:param vulnerabilities_id:
:return:
"""
try:
status = Config('third_party_vulnerabilities', 'status').value
if int(status):
q = Queue(self.project_name, self.third_party_vulnerabilities_name, self.third_party_vulnerabilities_type, self.file_path, self.line_number, self.code_content, vulnerabilities_id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
def __init__(self, **kwargs):
if not hasattr(Init_App, '_has_init'):
if 'env' in kwargs.keys():
env = kwargs['env']
else:
raise Exception('缺少环境配置参数env')
# 加载配置项
self.app.config.from_object(configs[env])
# 开启csrf防护验证
# CSRFProtect(self.app)
# 设置日志
self.set_log(self.app.config.get('LOGGER_LEVAL'))
# 数据库链接实例
self.db = db.init_app(self.app)
# redis 数据库实例
self.redis_conn = redis.StrictRedis(
host=self.app.config.get('REDIS_HOST'),
port=self.app.config.get('REDIS_PORT'),
db=self.app.config.get('REDIS_SELECT'),
password=self.app.config.get('REDIS_PWD'),
charset=self.app.config.get('REDIS_CHARSET'))
global redis_conn
redis_conn = self.redis_conn
self.app.task_queue = Queue('task', redis_conn)
Session(self.app)
# 将自定义的路由转换器加入列表
self.app.url_map.converters['re'] = RegxConverter
# 将蓝图中的路由注册到app中
from apps.api_1_0 import api, html_blueprint
self.app.register_blueprint(api)
self.app.register_blueprint(html_blueprint)
# 初始化完成,修改类属性has_init为True
Init_App._has_init = True
async def create(self, players: dict, maps: dict, team_names: dict):
""" Creates match.
https://github.com/ModuleLIFT/API/blob/master/docs/modules.md#createself-players-dict-maps-dict-team_names-dict
"""
queue = Queue(
current_league=self.current_league,
current_match=self,
players=players,
maps=maps,
team_names=team_names,
)
validation = await queue.validate()
if validation.error:
return validation
# Working out player selection type.
if queue.player_type.random:
assign_random = queue.captain.random()
if assign_random.error:
return assign_random
elif queue.player_type.elo:
assign_elo = await queue.captain.elo()
if assign_elo.error:
return assign_elo
else:
assign_given = queue.captain.given()
if assign_given:
return assign_given
# Working out map selection type.
if queue.map_type.given:
queue.map.given()
elif queue.map_type.random:
queue.map.random()
else:
queue.map.veto()
# Creating the match.
return await queue.create()
def queue():
from utils.queue import Queue
"""
Pushed to a third-party vulnerability management platform
Start the queue first
celery -A daemon worker --loglevel=info
:return:
"""
# Configure the project ID and the vulnerability ID
project_id = request.json.get('project_id')
rule_id = request.json.get('rule_id')
if project_id is None or rule_id is None:
return jsonify(code=1002,
result='Project ID and Rule ID can\'t empty!')
# Project Info
project_info = CobraProjects.query.filter_by(id=project_id).first()
# Unpunched vulnerability and rule information
result_all = db.session().query(CobraRules, CobraResults).join(
CobraResults, CobraResults.rule_id == CobraRules.id).filter(
CobraResults.project_id == project_id, CobraResults.status == 0,
CobraResults.rule_id == rule_id).all()
if len(result_all) == 0:
return jsonify(code=1001,
result="There are no unpacked vulnerabilities")
# Dealing with vulnerabilities
for index, (rule, result) in enumerate(result_all):
try:
# Query the vulnerability type information
vul_info = CobraVuls.query.filter(
CobraVuls.id == rule.vul_id).first()
# Pushed to a third-party vulnerability management platform
q = Queue(project_info.name, vul_info.name, vul_info.third_v_id,
result.file, result.line, result.code, result.id)
q.push()
except:
print(traceback.print_exc())
return jsonify(
code=1001,
result=
"Successfully pushed {0} vulnerabilities to a third-party vulnerability management platform"
.format(len(result_all)))
def __init__(self,
bot: Client,
config: SubRedis,
cog: Cog,
voice: VoiceChannel,
*,
log: TextChannel = None,
run_forever: bool = False,
**session_config):
"""
Args:
voice_channel :class:`discord.VoiceChannel`: The voice channel the session should start playing in.
Kwargs:
run_forever :class:`bool`: Determines whether the session should run forever
log_channel :class:`discord.TextChannel`: Specifies a channel to log playback history.
"""
self.bot = bot
self.config = config
self.cog = cog
self.voice_channel = voice
self.log_channel = log
self.session_config = session_config
self.queue_config = self.session_config.get('queue')
self.skip_requests = list()
self.repeat_requests = list()
self.voice = None
self.current_track = None
if run_forever:
self.queue = Radio(config, self.queue_config)
else:
self.queue = Queue(config, self.queue_config)
self.volume = self.session_config.get('default_volume') or float(
self.config.hget("config:defaults", "volume"))
self.is_playing = True
self.play_next_song = Event()
def queue():
from utils.queue import Queue
"""
推送到第三方漏洞管理平台
先启动队列
celery -A daemon worker --loglevel=info
:return:
"""
# 配置项目ID和漏洞ID
project_id = request.json.get('project_id')
rule_id = request.json.get('rule_id')
if project_id is None or rule_id is None:
return jsonify(code=1002, result='项目ID和规则ID不能为空')
# 项目信息
project_info = CobraProjects.query.filter_by(id=project_id).first()
# 未推送的漏洞和规则信息
result_all = db.session().query(CobraRules, CobraResults).join(
CobraResults, CobraResults.rule_id == CobraRules.id).filter(
CobraResults.project_id == project_id, CobraResults.status == 0,
CobraResults.rule_id == rule_id).all()
if len(result_all) == 0:
return jsonify(code=1001, result="没有未推送的漏洞")
# 处理漏洞
for index, (rule, result) in enumerate(result_all):
try:
# 取出漏洞类型信息
vul_info = CobraVuls.query.filter(
CobraVuls.id == rule.vul_id).first()
# 推动到第三方漏洞管理平台
q = Queue(project_info.name, vul_info.name, vul_info.third_v_id,
result.file, result.line, result.code, result.id)
q.push()
except:
print(traceback.print_exc())
return jsonify(code=1001,
result="成功推送{0}个漏洞到第三方漏洞管理平台".format(len(result_all)))
def get_queue():
task_id = request.json.get('task_id')
res_queue = Queue(task_id, redis_conn)
counter = 0
while True:
try:
res = res_queue.get_nowait()
except res_queue.Empty:
if counter >= current_app.config.get('TASK_TIMEOUT'):
return jsonify({'code': RET.SERVERERR, 'message': '响应超时'})
counter += 1
time.sleep(1)
else:
if res == 0:
return jsonify({'code': RET.NODATA, 'message': '没有数据了,别再请求了'})
else:
return jsonify({
'code': RET.OK,
'res': res,
'message': '获取成功,还有结果请继续请求'
})
def _build_jump_nodes(self):
""" Build a list of jump nodes for the tree.
Designate the macro leaves (the leaves of the macro tree) as jump nodes.
Sort the list in linear time.
"""
self._jump_nodes = []
Q = Queue()
Q.enqueue(self._tree.root())
while not Q.is_empty():
p = Q.dequeue()
if self._tree.height(p) > self._block_size:
for ch in self._tree.children(p):
if self._tree.height(ch) <= self._block_size:
if p not in self._jump_nodes:
self._jump_nodes.append(p)
else:
Q.enqueue(ch)
# Bucket sort should be used for sorting!
self._jump_nodes.sort(key=lambda p: self._tree.depth(p), reverse=True)
# coding:utf-8
__author__ = 'admin'
# --------------------------------
# Created by admin on 2016/8/25.
# ---------------------------------
from utils.queue import PriorityQueue as Queue
from threading import Thread
import urllib2
import time
import datetime
from utils.encrypt import md5
import urlparse
from pybloom import BloomFilter, ScalableBloomFilter
import traceback
_queue = Queue()
_size = 0
_handle = None
def spider_init(pool_size, handle=lambda url, content: None):
'''初始化爬虫
pool_size 线程池的大小
handle 爬虫结果回调函数, 函数必须包含两个参数, 第一个参数是url,第二个参数是爬取到的结果
'''
print datetime.datetime.now(), "[Spider]:init...."
global _size, _queue, _url_max_num, _proxy_list, _handle
if _size == 0 and _handle == None:
_size = pool_size
_handle = handle
def setEnv(self):
self.declareGame("Egyptian Rat Screw")
self.declareRules(ERSRules().getAllRules())
self.createDeck(Deck(Deck.getDefaultDeck()['ranks'], Deck.getDefaultDeck()['suits']))
self.initializeMachine(ERSMachine(self))
self.slapQueue = Queue()
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info('Scan rule id: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
# logging.info(' '.join(param))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
if rule.regex_location == '':
# Find (special file)
file_path = line.strip().replace(
self.directory, '')
logging.debug('File: {0}'.format(file_path))
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path).first()
if exist_result is not None:
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[rule.vul_id]
['name'], vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, exist_result.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.warning("Exists Result")
else:
vul = CobraResults(self.task_id,
self.project_id, rule.id,
file_path, 0, '', 0)
db.session.add(vul)
try:
# push queue
q = Queue(
self.project_name, vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
else:
# Grep
line_split = line.split(':', 1)
file_path = line_split[0].strip()
if len(line_split) < 2:
logging.info("Line len < 2 {0}".format(line))
continue
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
if file_path in white_list or ".min.js" in file_path:
logging.info("In white list or min.js")
else:
only_match = rule.regex_location[:
1] == '(' and rule.regex_location[
-1] == ')'
"""
annotation (注释过滤)
# // /* *
Exclude:
- (rule_location) - 当定位规则左右两边为括号时不过滤注释行,比如硬编码密码
"""
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", code_content)
if match_result.group(
0) is not None and match_result.group(
0
) is not "" and only_match is not True:
logging.info("In Annotation")
else:
param_value = None
# parse file function structure
if only_match:
found_vul = True
else:
if file_path[
-3:] == 'php' and rule.regex_repair.strip(
) != '':
try:
parse_instance = parse.Parse(
rule.regex_location,
file_path, line_number,
code_content)
if parse_instance.is_controllable_param(
):
if parse_instance.is_repair(
rule.regex_repair,
rule.block_repair):
logging.info(
"Static: repaired")
continue
else:
if parse_instance.param_value is not None:
param_value = parse_instance.param_value
found_vul = True
else:
logging.info(
"Static: uncontrollable param"
)
continue
except:
print(traceback.print_exc())
found_vul = False
else:
found_vul = True
file_path = file_path.replace(
self.directory, '')
if found_vul:
logging.info('In Insert')
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path,
line=line_number).first()
if exist_result is not None:
logging.info("Exists Result")
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]
['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'],
file_path, line_number,
code_content,
exist_result.id)
q.push()
except Exception as e:
print(
traceback.print_exc())
logging.critical(e.message)
else:
code_content = code_content.encode(
'unicode_escape')
if len(code_content) > 512:
code_content = code_content[:
500] + '...'
code_content = '# Trigger\r' + code_content
if param_value is not None:
code_content = '# Param\r' + param_value + '\r//\r// ------ Continue... ------\r//\r' + code_content
logging.debug(
'File: {0}:{1} {2}'.format(
file_path, line_number,
code_content))
vul = CobraResults(
self.task_id, self.project_id,
rule.id, file_path,
line_number, code_content, 0)
db.session.add(vul)
db.session.commit()
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'], file_path,
line_number, code_content,
vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.info(
'Insert Results Success')
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def _run_uploader(input_queue, saver_list, saver_config):
uploader = ImageUploader(input_queue, saver_list, saver_config)
uploader.run()
def start_uploader(input_queue):
assert isinstance(input_queue, Queue), "Start Saver error: queue is invalid"
saver_list = SAVER_LIST
saver_config = SAVER_CONFIG
process_num = SAVER_CONFIG["process"]
for _ in range(process_num):
p = Process(target=_run_uploader, args=(input_queue, saver_list, saver_config))
p.start()
if __name__ == "__main__":
task_queue = Queue(size=1000)
picture_queue = Queue(size=1000)
upload_queue = Queue(size=1000)
tasks = [TaskInfo("phrasepicture"), TaskInfo("wordpicture"),
TaskInfo("iconpicture"), TaskInfo("ninepicture")]
for task in tasks:
task.set_num(100000)
task_queue.put(task)
start_builder(task_queue, picture_queue)
start_transformer(picture_queue, upload_queue)
start_uploader(upload_queue)
# "/media/georgen/LOCAL DISK/George_Ndirangu/Learning/py/music-cleaner/test-res/tinker"
]
for i, arg in enumerate(sys.argv):
if i == 0:
continue
directories.append(arg)
list_of_paths = []
completed = []
failed = []
for directory in directories:
list_of_paths += dloader.load_file_paths(directory)
processing_queue = Queue()
Queue.populate_queue(list_of_paths, processing_queue)
for item in processing_queue.get_items():
item.update_status(QueueItemStatus.PROCESSING)
result: Optional[MusicFile] = accoustid.search(item.music_file)
if isinstance(result, MusicFile):
item.update_status(QueueItemStatus.SUCCESS)
on_success(completed, item.music_file)
processing_queue.remove(item)
elif isinstance(result, acoustid.WebServiceError):
logging.warning(
f"Network error encountered. Will try to handle this another day.For now moving on \n {result}"
)
else:
