def add_file_handler(level, fmt, filename, mode, backup_count, limit, when):
"""Add a file handler to the global logger."""
kwargs = {}
# If the filename is not set, use the default filename
if filename is None:
logs_directory = config.Config('cobra', 'logs_directory').value
logs_directory = os.path.join(config.Config().project_directory,
logs_directory)
if os.path.isdir(logs_directory) is not True:
os.mkdir(logs_directory)
filename = logs_directory + os.sep + time.strftime("%Y-%m-%d") + '.log'
kwargs['filename'] = filename
# Choose the file_handler based on the passed arguments
if backup_count == 0: # Use FileHandler
cls = logging.FileHandler
kwargs['mode'] = mode
elif when is None: # Use RotatingFileHandler
cls = logging.handlers.RotatingFileHandler
kwargs['maxBytes'] = limit
kwargs['backupCount'] = backup_count
kwargs['mode'] = mode
else: # Use TimedRotatingFileHandler
cls = logging.handlers.TimedRotatingFileHandler
kwargs['when'] = when
kwargs['interval'] = limit
kwargs['backupCount'] = backup_count
return add_handler(cls, level, fmt, False, **kwargs)
def app():
with rw_fs.Storage(restart_player=False):
hardware.platfrom.fix_file_permissions(config.Config().storage_path())
xmain.start()
wallpaper.Wallpaper().load()
if config.Config().enable_clockd():
threads.run_in_thread(clockd.Clockd().run)
# threads.run_in_thread(networkd.Networkd().run)
# threads.run_in_thread(hdmihotplug.HdmiHotplug(onchange_callback=resolution_changed).run)
player = playercontroller.PlayerController()
player.start_playlist()
proto = protocoldispatcher.ProtocolDispatcher()
threads.run_in_thread(webui.start, ['0.0.0.0', 8080])
while True:
track = player.current_track_name()
pos = player.current_track_posiotion()
proto.send('now_playing', track=track, percent_position=pos)
watchdog.Watchdog().feed()
time.sleep(20)
def __init__(self):
self.status = config.Config('third_party_vulnerabilities', 'status').value
self.api = config.Config('third_party_vulnerabilities', 'api').value
self.key = config.Config('third_party_vulnerabilities', 'key').value
self.vulnerabilities = []
self.vuln_id = []
def pull_code(self, branch='master'):
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target,
branch=branch,
username=username,
password=password)
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err), gg
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!', gg
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed', gg
return 1001, 'Success', gg
def __init__(self, dialog, username, password):
self._debug = False
self.username = username
self.password = password
self.user_list = []
self.loop = asyncio.get_event_loop()
self.user_colors = config.Config("qt_user_colors.json")
self.settings = config.Config("settings.json")
self.ip = self.settings.get("server_ip")
self.port = self.settings.get("server_port")
renderer = Renderer()
lexer = InlineLexer(renderer)
lexer.enable_underscore()
self.markdown = mistune.Markdown(renderer, inline=lexer)
Ui_MainWindow.__init__(self)
self.setupUi(dialog)
dialog.setWindowIcon(QtGui.QIcon("./utils/ui/files/icon.png"))
self.MessageField.setFocus()
self.MessageField.returnPressed.connect(self.read_input)
self.connect_task = self.loop.create_task(self.connect())
def __init__(self):
domain = '{0}:{1}'.format(
config.Config('cobra', 'host').value,
config.Config('cobra', 'port').value)
self.api = 'http://' + domain + '/api/{0}'
self.headers = {"Content-Type": "application/json"}
self.key = common.md5('CobraAuthKey')
self.branch = 'master'
def __import_custom_script(self):
try:
import imp
self.__custom_script = imp.load_source("custom_script", config.Config().custom_script)
if self.__custom_script.log:
self.__custom_script.log = log
except:
log.exception("cannot import custom script '{s}'".format(s=config.Config().custom_script))
raise
def setup_logger(verbose_log=False):
logfile = config.Config().logfile()
if hardware.platfrom.__name__ == 'raspberry':
files.mkdir(os.path.dirname(logfile))
root_logger = logging.getLogger()
rotating_handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=2097152, backupCount=5)
root_logger.setLevel(logging.DEBUG if (verbose_log or config.Config().verbose_logging()) else logging.INFO)
child_logger = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter("[%(asctime)s] %(name)s[%(threadName)s/%(thread)d] |%(levelname)s| %(message)s", "%Y-%m-%d %H:%M:%S")
child_logger.setFormatter(formatter)
rotating_handler.setFormatter(formatter)
root_logger.addHandler(rotating_handler)
root_logger.addHandler(child_logger)
logging.info("started")
def __init__(self):
self.console_input_thread = None
self.process = None # the process for the server
self.server_status = ServerStatus.STOPPED
self.flag_interrupt = False # ctrl-c flag
self.flag_server_startup = False # set to True after server startup. used to start the rcon server
self.starting_server = False # to prevent multiple start_server() call
# will be assigned in reload_config()
self.parser = None
self.encoding_method = None
self.decoding_method = None
self.rcon_manager = None
self.logger = logger.Logger(self, constant.NAME_SHORT)
self.logger.set_file(constant.LOGGING_FILE)
self.language_manager = LanguageManager(self, constant.LANGUAGE_FOLDER)
self.config = config.Config(self, constant.CONFIG_FILE)
self.rcon_manager = RconManager(self)
self.load_config()
self.reactors = self.load_reactor(constant.REACTOR_FOLDER)
self.server_interface = ServerInterface(self)
self.command_manager = CommandManager(self)
self.plugin_manager = PluginManager(self, constant.PLUGIN_FOLDER)
self.load_plugins()
self.permission_manager = PermissionManager(self,
constant.PERMISSION_FILE)
def bootstrap(configfile, verbose_log=False):
hardware.platfrom.initialize()
config.Config().set_configfile(configfile)
setup_logger(verbose_log)
logging.info("using config file: '{c}'".format(c=configfile))
logging.info("working directory: '{w}'".format(w=os.getcwd()))
signal.signal(signal.SIGTERM, signal_handler)
def download(name):
try:
log.info("downloading artefact '{f}'".format(f=name))
return static_file(name, config.Config().artefacts_path)
except:
log.exception("error uploading file '{f}'".format(f=name))
raise
def __init__(self):
# Config and database setup
configData = config.Config()
self.heuristics = configData.heuristics()
self.weights = configData.weights()
self.db = database.DB()
# Columns
col_list = [
'category', 'start_time', 'id', 'price', 'title', 'description',
'longitude', 'latitude'
]
#SQL_future_col_names = "SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name='events_and_listings'"
#cols = self.db.get(SQL_future_col_names,[])
#cols = list(k[0] for k in cols)
# Future events
SQL_future = "SELECT category, starttime, eventid, price, title, description, longitude, latitude FROM events_and_listings WHERE pastevent = FALSE;"
self.future_events = pd.DataFrame(self.db.get(SQL_future, []),
columns=col_list)
# Past events
#SQL_past= "SELECT * FROM events_and_listings WHERE (pastEvent = TRUE) AND (id IN (%s));" % placeholders
SQL_past = "SELECT category, starttime, eventid, price, title, description, longitude, latitude FROM events_and_listings WHERE pastevent = TRUE;"
self.past_events = pd.DataFrame(self.db.get(SQL_past, []),
columns=col_list)
def status_task():
"""
查询扫描任务状态
:return:
"""
scan_id = request.json.get('scan_id')
key = request.json.get('key')
auth = CobraAuth.query.filter_by(key=key).first()
if auth is None:
return jsonify(code=4002, result=u'Key verify failed')
c = CobraTaskInfo.query.filter_by(id=scan_id).first()
if not c:
return jsonify(status=4004)
status = {0: 'init', 1: 'scanning', 2: 'done', 3: 'error'}
status_text = status[c.status]
domain = config.Config('cobra', 'domain').value
# project_id
project_info = CobraProjects.query.filter_by(repository=c.target).first()
if project_info:
report = 'http://' + domain + '/report/' + str(project_info.id)
else:
report = 'http://' + domain
result = {
'status': status_text,
'text': 'Success',
'report': report,
'allow_deploy': True
}
return jsonify(status=1001, result=result)
def test_parse4java(self):
"""
测试解析规则(Java)
:return:
"""
import os
from engine.parse import Parse
regex_location = r'new\sURL\((.*)\)'
regex_repair = r'Security.filter\({{PARAM}}\)'
file_path = os.path.join(config.Config().project_directory,
'tests/parse/test_functions.java')
tests = [{
'line': 33,
'code': "URL obj = new URL(url);",
'result': False,
}, {
'line': 66,
'code': "URL obj = new URL(url);",
'result': False,
'repair': True
}]
for test in tests:
parse = Parse(regex_location, file_path, test['line'],
test['code'])
self.assertEqual(test['result'], parse.is_controllable_param())
if 'repair' in test:
self.assertEqual(test['repair'],
parse.is_repair(regex_repair, 0))
def run():
configs = []
reccts = np.random.randint(18000000, 20000000, size=30)
for recct in reccts:
configs.append(config.Config(
recct=int(recct), opct=0, fcount=0, fav=500, fdist='u', frange=200))
folder = 'test8'
status = misc.Status(folder, len(configs))
while not status.finished:
conf = configs[status.current]
conf.load()
# init
print("\n{}: recct={}".format(
status.current, conf.recct))
stopwatch = misc.Stopwatch()
# flush, run, dump
scripts.flush()
stopwatch.lap(' - - flush: ')
scripts.redis_all()
stopwatch.lap(' - - workload: ')
scripts.dump()
stopwatch.lap(' - - dump: ')
# lz4, zeros
scripts.compress_lz4()
stopwatch.lap(' - - lz4: ')
scripts.zero_count()
stopwatch.lap(' - - zeros: ')
# write into final res folder
res_lz4_out(folder)
res_zeros_out(folder)
status.inc()
stopwatch.total(' - total: ')
def __init__(self,
repo_address,
branch='master',
username=None,
password=None):
# get upload directory
self.upload_directory = os.path.join(
config.Config('upload', 'directory').value, 'versions')
if os.path.isdir(self.upload_directory) is False:
os.makedirs(self.upload_directory)
self.repo_address = repo_address
self.repo_username = username
self.repo_password = password
self.repo_branch = branch
repo_user = self.repo_address.split('/')[-2]
repo_name = self.repo_address.split('/')[-1].replace('.git', '')
self.repo_author = repo_user
self.repo_name = repo_name
if '.git' not in self.repo_address:
self.repo_address += '.git'
else:
repo_name = repo_name.split('.')[0]
self.repo_directory = os.path.join(
os.path.join(self.upload_directory, repo_user), repo_name)
logging.info('Git class init.')
def run():
configs = []
for recct in [1000 * (i+1) for i in range(3000)]:
configs.append(config.Config(recct=recct, opct=0, fav=107))
folder = 'test7'
status = misc.Status(folder, len(configs))
while not status.finished:
conf = configs[status.current]
conf.load()
# init
print("\n{}: recct={}".format(
status.current, conf.recct))
stopwatch = misc.Stopwatch()
# flush, run, dump
scripts.flush()
stopwatch.lap(' - - flush: ')
scripts.redis_all()
stopwatch.lap(' - - workload: ')
scripts.dump()
stopwatch.lap(' - - dump: ')
# zstd, lz4, zeros
scripts.compress_zstd()
stopwatch.lap(' - - zstd: ')
scripts.compress_lz4()
stopwatch.lap(' - - lz4: ')
scripts.zero_count()
stopwatch.lap(' - - zeros: ')
# write into final res folder
res_zstd_out(folder)
res_lz4_out(folder)
res_zeros_out(folder)
status.inc()
stopwatch.total(' - total: ')
def __init__(self, old_process=None):
self.console_input_thread = None
self.info_reactor_thread = None
self.info_queue = queue.Queue(maxsize=constant.MAX_INFO_QUEUE_SIZE)
self.process = old_process # the process for the server
self.server_status = ServerStatus.STOPPED
self.flag_interrupt = False # ctrl-c flag
self.flag_rcon_startup = False # set to True after server startup. used to start the rcon server
self.flag_exit = False # MCDR exit flag
self.starting_server_lock = Lock() # to prevent multiple start_server() call
# will be assigned in reload_config()
self.encoding_method = None
self.decoding_method = None
self.logger = logger.Logger(self, constant.NAME_SHORT)
self.logger.set_file(constant.LOGGING_FILE)
self.server_logger = logger.ServerLogger('Server')
self.language_manager = LanguageManager(self, constant.LANGUAGE_FOLDER)
self.config = config.Config(self, constant.CONFIG_FILE)
self.rcon_manager = RconManager(self)
self.parser_manager = ParserManager(self)
self.load_config()
self.reactors = self.load_reactor(constant.REACTOR_FOLDER)
self.server_interface = ServerInterface(self)
self.command_manager = CommandManager(self)
self.plugin_manager = PluginManager(self, constant.PLUGIN_FOLDER)
self.load_plugins()
self.permission_manager = PermissionManager(self, constant.PERMISSION_FILE)
self.update_helper = UpdateHelper(self)
self.update_helper.check_update_start()
def __init__(self, dialog):
self.settings = config.Config("settings.json",
default={
"blocked_users": [],
"server_ip": "",
"server_port": "",
"should_render_markdown": True
})
self.window = dialog
Ui_LoginWindow.__init__(self)
self.setupUi(dialog)
dialog.setWindowIcon(QtGui.QIcon("./utils/ui/files/icon.png"))
server_ip = self.settings.get("server_ip")
server_port = self.settings.get("server_port")
do_markdown = self.settings.get("should_render_markdown")
if server_ip:
self.AddressField.setText(server_ip)
if server_port:
self.PortField.setText(server_port)
if isinstance(do_markdown, bool):
self.MarkdownCheck.setChecked(do_markdown)
else:
do_markdown = True
self.MarkdownCheck.setChecked(True)
self.LoginButton.clicked.connect(self.validate_input)
def bootstrap(configfile, console_app):
conf = config.Config(configfile)
loglevels = {
"debug": DEBUG,
"info": INFO,
"error": ERROR,
"warning": WARNING
}
basicConfig(filename=conf.logfile,
format="[%(asctime)s] %(name)s |%(levelname)s| %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=loglevels[conf.loglevel])
if console_app:
root_logger = getLogger()
child_logger = StreamHandler(stdout)
child_logger.setLevel(DEBUG)
formatter = Formatter(
"[%(asctime)s] %(name)s |%(levelname)s| %(message)s",
"%Y-%m-%d %H:%M:%S")
child_logger.setFormatter(formatter)
root_logger.addHandler(child_logger)
info("started as console application")
else:
info("started as daemon")
info("using config file: '{c}'".format(c=configfile))
debug("working directory: '{w}', pid: '{pid}'".format(w=getcwd(),
pid=getpid()))
debug("config:\n" + str(conf.get_raw_cfg()))
signal(SIGTERM, signal_handler)
def compress(self):
dc = decompress.Decompress(self.target)
ret, result_d = dc.decompress()
if ret is False:
return 1002, result_d
else:
directory = result_d
logging.info("Scan directory: {0}".format(directory))
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
p = CobraProjects.query.filter_by(repository=directory).first()
# detection framework for project
framework, language = detection.Detection(directory).framework()
if framework != '' or language != '':
project_framework = '{0} ({1})'.format(framework, language)
else:
project_framework = ''
if not p:
# insert into project table.
repo_name = directory.split('/')[-1]
project = CobraProjects(directory, '', repo_name, 'Upload',
project_framework, '', '', 1, current_time)
db.session.add(project)
db.session.commit()
project_id = project.id
else:
project_id = p.id
# update project's framework
p.framework = project_framework
db.session.add(p)
task = CobraTaskInfo(directory, '', 3, '', '', 0, 0, 0, 1, 0, 0,
current_time, current_time)
db.session.add(task)
db.session.commit()
cobra_path = os.path.join(config.Config().project_directory,
'cobra.py')
if os.path.isfile(cobra_path) is not True:
return 1004, 'Cobra Not Found'
# 扫描漏洞
subprocess.Popen([
'python', cobra_path, "scan", "-p",
str(project_id), "-i",
str(task.id), "-t", directory
])
# 统计代码行数
subprocess.Popen([
'python', cobra_path, "statistic", "-i",
str(task.id), "-t", directory
])
# 检测漏洞修复状况
subprocess.Popen(
['python', cobra_path, "repair", "-p",
str(project_id)])
result = dict()
result['scan_id'] = task.id
result['project_id'] = project_id
result['msg'] = u'success'
return 1001, result
def main(argv):
settings = config.Config("./path/to/directory/filename.csv", {
"host": "localhost",
"port": 9200
}, "twitter", "tweets", "tweet_id")
upload_data = read_file(
settings.file_url,
settings.index_name,
settings.type_name,
settings.id_field,
)
response = bulk_create(
upload_data,
settings.es_host,
settings.index_name,
)
if (response):
print('finished bulk upload...')
return 0
else:
print('there was an error in your bulk_create method call')
return 0
def __init__(self, **kwargs):
Thread.__init__(self)
self.setDaemon(True)
self.__params = kwargs
self.__params["repository_path"] = config.Config().repo_path
self.__params["artefacts_path"] = config.Config().artefacts_path
self.__start_time = 0
self.__end_time = 0
self.__log = ""
self.__last_commit_info = {}
self.__build_executor = None
self.__terminate_flag = Event()
self.__terminated = False
self.__error = True
self.__import_custom_script()
self.__error = False
def compress(self):
dc = decompress.Decompress(self.target)
ret, result_d = dc.decompress()
if ret is False:
return 1002, result_d
else:
directory = result_d
log.info("Scan directory: {0}".format(directory))
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
task = CobraTaskInfo(self.target, '', 3, '', '', 0, 0, 0, 1, 0, 0,
current_time, current_time)
db.session.add(task)
db.session.commit()
cobra_path = os.path.join(config.Config().project_directory,
'cobra.py')
if os.path.isfile(cobra_path) is not True:
return 1004, 'Cobra Not Found'
# Start Scanning
subprocess.Popen([
'python', cobra_path, "scan", "-p",
str(0), "-i",
str(task.id), "-t", directory
])
# Statistic Code
subprocess.Popen([
'python', cobra_path, "statistic", "-i",
str(task.id), "-t", directory
])
result = {}
result['scan_id'] = task.id
result['project_id'] = 0
result['msg'] = u'success'
return 1001, result
def __init__(self, username, password):
self._debug = False
self.username = username
self.password = password
self.user_list = []
self.loop = asyncio.get_event_loop()
self.user_colors = config.Config("user_colors.json")
self.settings = config.Config("settings.json")
self.ip = self.settings.get("server_ip")
self.port = self.settings.get("server_port")
renderer = Renderer()
lexer = InlineLexer(renderer)
lexer.enable_underscore()
self.clean = lambda: os.system("cls" if os.name == "nt" else "clear")
self.markdown = mistune.Markdown(renderer, inline=lexer)
def __init__(self, filename):
"""
:param filename: a file name without path.
"""
self.upload_directory = os.path.join(config.Config('upload', 'directory').value, 'uploads')
self.filename = filename
self.filepath = os.path.join(self.upload_directory, filename)
self.dir_name = os.path.splitext(self.filename)[0]
def download_delete(name):
try:
filepath = path.join(config.Config().artefacts_path, name)
log.info("deleting artefact '{f}'".format(f=filepath))
remove(filepath)
except:
log.exception("error removing file '{f}'".format(f=filepath))
raise
def decorated_function(*args, **kwargs):
token = request.args.get('token')
is_auth_token = token is not None and token == config.Config(
'cobra', 'secret_key').value
if not session.get('is_login'):
if not is_auth_token:
return redirect(ADMIN_URL + '/index')
return f(*args, **kwargs)
class TestConfig(unittest.TestCase):
conf = config.Config("../app.conf")
def test_get_string(self):
self.assertEqual(self.conf.get_string("DB", "host"), '"localhost"')
def test_get_int_ary(self):
self.assertEqual(self.conf.get_int_ary("REDIS", "test"), [1, 2, 3])
def __init__(self, biases):
self.db = database.DB()
self.config = config.Config()
self.commands = [["getAnonymousRecommendations", [int]],
["getRecommendations", [str, str, str]]]
self.__loadRecommenders()
self.__cache = {}
self.biases = biases
