Ejemplo n.º 1
0
def add_file_handler(level, fmt, filename, mode, backup_count, limit, when):
    """Add a file handler to the global logger."""
    kwargs = {}

    # If the filename is not set, use the default filename
    if filename is None:
        logs_directory = config.Config('cobra', 'logs_directory').value
        logs_directory = os.path.join(config.Config().project_directory,
                                      logs_directory)
        if os.path.isdir(logs_directory) is not True:
            os.mkdir(logs_directory)
        filename = logs_directory + os.sep + time.strftime("%Y-%m-%d") + '.log'

    kwargs['filename'] = filename

    # Choose the file_handler based on the passed arguments
    if backup_count == 0:  # Use FileHandler
        cls = logging.FileHandler
        kwargs['mode'] = mode
    elif when is None:  # Use RotatingFileHandler
        cls = logging.handlers.RotatingFileHandler
        kwargs['maxBytes'] = limit
        kwargs['backupCount'] = backup_count
        kwargs['mode'] = mode
    else:  # Use TimedRotatingFileHandler
        cls = logging.handlers.TimedRotatingFileHandler
        kwargs['when'] = when
        kwargs['interval'] = limit
        kwargs['backupCount'] = backup_count

    return add_handler(cls, level, fmt, False, **kwargs)
Ejemplo n.º 2
0
def app():
    with rw_fs.Storage(restart_player=False):
        hardware.platfrom.fix_file_permissions(config.Config().storage_path())

    xmain.start()
    wallpaper.Wallpaper().load()

    if config.Config().enable_clockd():
        threads.run_in_thread(clockd.Clockd().run)

    # threads.run_in_thread(networkd.Networkd().run)

    # threads.run_in_thread(hdmihotplug.HdmiHotplug(onchange_callback=resolution_changed).run)

    player = playercontroller.PlayerController()
    player.start_playlist()

    proto = protocoldispatcher.ProtocolDispatcher()

    threads.run_in_thread(webui.start, ['0.0.0.0', 8080])
    while True:
        track = player.current_track_name()
        pos = player.current_track_posiotion()
        proto.send('now_playing', track=track, percent_position=pos)
        watchdog.Watchdog().feed()
        time.sleep(20)
Ejemplo n.º 3
0
    def __init__(self):
        self.status = config.Config('third_party_vulnerabilities', 'status').value
        self.api = config.Config('third_party_vulnerabilities', 'api').value
        self.key = config.Config('third_party_vulnerabilities', 'key').value

        self.vulnerabilities = []
        self.vuln_id = []
Ejemplo n.º 4
0
    def pull_code(self, branch='master'):
        logging.info('Gitlab project')
        # Git
        if 'gitlab' in self.target:
            username = config.Config('git', 'username').value
            password = config.Config('git', 'password').value
        else:
            username = None
            password = None
        gg = git.Git(self.target,
                     branch=branch,
                     username=username,
                     password=password)

        # Git Clone Error
        try:
            clone_ret, clone_err = gg.clone()
            if clone_ret is False:
                return 4001, 'Clone Failed ({0})'.format(clone_err), gg
        except NotExistError:
            # update project status
            p = CobraProjects.query.filter_by(repository=self.target).first()
            if p is not None:
                if p.status == CobraProjects.get_status('on'):
                    p.status = CobraProjects.get_status('off')
                    db.session.add(p)
                    db.session.commit()
            return 4001, 'Repository Does not exist!', gg
        except AuthError:
            logging.critical('Git Authentication Failed')
            return 4001, 'Repository Authentication Failed', gg
        return 1001, 'Success', gg
Ejemplo n.º 5
0
    def __init__(self, dialog, username, password):
        self._debug = False
        self.username = username
        self.password = password
        self.user_list = []
        self.loop = asyncio.get_event_loop()
        self.user_colors = config.Config("qt_user_colors.json")
        self.settings = config.Config("settings.json")

        self.ip = self.settings.get("server_ip")
        self.port = self.settings.get("server_port")

        renderer = Renderer()
        lexer = InlineLexer(renderer)
        lexer.enable_underscore()
        self.markdown = mistune.Markdown(renderer, inline=lexer)

        Ui_MainWindow.__init__(self)
        self.setupUi(dialog)
        dialog.setWindowIcon(QtGui.QIcon("./utils/ui/files/icon.png"))

        self.MessageField.setFocus()

        self.MessageField.returnPressed.connect(self.read_input)

        self.connect_task = self.loop.create_task(self.connect())
Ejemplo n.º 6
0
 def __init__(self):
     domain = '{0}:{1}'.format(
         config.Config('cobra', 'host').value,
         config.Config('cobra', 'port').value)
     self.api = 'http://' + domain + '/api/{0}'
     self.headers = {"Content-Type": "application/json"}
     self.key = common.md5('CobraAuthKey')
     self.branch = 'master'
Ejemplo n.º 7
0
 def __import_custom_script(self):
     try:
         import imp
         self.__custom_script = imp.load_source("custom_script", config.Config().custom_script)
         if self.__custom_script.log:
             self.__custom_script.log = log
     except:
         log.exception("cannot import custom script '{s}'".format(s=config.Config().custom_script))
         raise
Ejemplo n.º 8
0
def setup_logger(verbose_log=False):
    logfile = config.Config().logfile()
    if hardware.platfrom.__name__ == 'raspberry':
        files.mkdir(os.path.dirname(logfile))

    root_logger = logging.getLogger()
    rotating_handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=2097152, backupCount=5)
    root_logger.setLevel(logging.DEBUG if (verbose_log or config.Config().verbose_logging()) else logging.INFO)
    child_logger = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter("[%(asctime)s] %(name)s[%(threadName)s/%(thread)d] |%(levelname)s| %(message)s", "%Y-%m-%d %H:%M:%S")
    child_logger.setFormatter(formatter)
    rotating_handler.setFormatter(formatter)
    root_logger.addHandler(rotating_handler)
    root_logger.addHandler(child_logger)
    logging.info("started")
Ejemplo n.º 9
0
    def __init__(self):
        self.console_input_thread = None
        self.process = None  # the process for the server
        self.server_status = ServerStatus.STOPPED
        self.flag_interrupt = False  # ctrl-c flag
        self.flag_server_startup = False  # set to True after server startup. used to start the rcon server
        self.starting_server = False  # to prevent multiple start_server() call

        # will be assigned in reload_config()
        self.parser = None
        self.encoding_method = None
        self.decoding_method = None
        self.rcon_manager = None

        self.logger = logger.Logger(self, constant.NAME_SHORT)
        self.logger.set_file(constant.LOGGING_FILE)
        self.language_manager = LanguageManager(self, constant.LANGUAGE_FOLDER)
        self.config = config.Config(self, constant.CONFIG_FILE)
        self.rcon_manager = RconManager(self)
        self.load_config()
        self.reactors = self.load_reactor(constant.REACTOR_FOLDER)
        self.server_interface = ServerInterface(self)
        self.command_manager = CommandManager(self)
        self.plugin_manager = PluginManager(self, constant.PLUGIN_FOLDER)
        self.load_plugins()
        self.permission_manager = PermissionManager(self,
                                                    constant.PERMISSION_FILE)
Ejemplo n.º 10
0
def bootstrap(configfile, verbose_log=False):
    hardware.platfrom.initialize()
    config.Config().set_configfile(configfile)
    setup_logger(verbose_log)
    logging.info("using config file: '{c}'".format(c=configfile))
    logging.info("working directory: '{w}'".format(w=os.getcwd()))
    signal.signal(signal.SIGTERM, signal_handler)
Ejemplo n.º 11
0
def download(name):
    try:
        log.info("downloading artefact '{f}'".format(f=name))
        return static_file(name, config.Config().artefacts_path)
    except:
        log.exception("error uploading file '{f}'".format(f=name))
        raise
Ejemplo n.º 12
0
    def __init__(self):

        # Config and database setup
        configData = config.Config()
        self.heuristics = configData.heuristics()
        self.weights = configData.weights()
        self.db = database.DB()

        # Columns
        col_list = [
            'category', 'start_time', 'id', 'price', 'title', 'description',
            'longitude', 'latitude'
        ]
        #SQL_future_col_names = "SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name='events_and_listings'"
        #cols = self.db.get(SQL_future_col_names,[])
        #cols = list(k[0] for k in cols)

        # Future events
        SQL_future = "SELECT category, starttime, eventid, price, title, description, longitude, latitude FROM events_and_listings WHERE pastevent = FALSE;"
        self.future_events = pd.DataFrame(self.db.get(SQL_future, []),
                                          columns=col_list)

        # Past events
        #SQL_past= "SELECT * FROM events_and_listings WHERE (pastEvent = TRUE) AND (id IN (%s));" % placeholders
        SQL_past = "SELECT category, starttime, eventid, price, title, description, longitude, latitude FROM events_and_listings WHERE pastevent = TRUE;"
        self.past_events = pd.DataFrame(self.db.get(SQL_past, []),
                                        columns=col_list)
Ejemplo n.º 13
0
Archivo: api.py Proyecto: xxoxx/cobra
def status_task():
    """
    查询扫描任务状态
    :return:
    """
    scan_id = request.json.get('scan_id')
    key = request.json.get('key')
    auth = CobraAuth.query.filter_by(key=key).first()
    if auth is None:
        return jsonify(code=4002, result=u'Key verify failed')
    c = CobraTaskInfo.query.filter_by(id=scan_id).first()
    if not c:
        return jsonify(status=4004)
    status = {0: 'init', 1: 'scanning', 2: 'done', 3: 'error'}
    status_text = status[c.status]
    domain = config.Config('cobra', 'domain').value
    # project_id
    project_info = CobraProjects.query.filter_by(repository=c.target).first()
    if project_info:
        report = 'http://' + domain + '/report/' + str(project_info.id)
    else:
        report = 'http://' + domain
    result = {
        'status': status_text,
        'text': 'Success',
        'report': report,
        'allow_deploy': True
    }
    return jsonify(status=1001, result=result)
Ejemplo n.º 14
0
 def test_parse4java(self):
     """
     测试解析规则(Java)
     :return:
     """
     import os
     from engine.parse import Parse
     regex_location = r'new\sURL\((.*)\)'
     regex_repair = r'Security.filter\({{PARAM}}\)'
     file_path = os.path.join(config.Config().project_directory,
                              'tests/parse/test_functions.java')
     tests = [{
         'line': 33,
         'code': "URL obj = new URL(url);",
         'result': False,
     }, {
         'line': 66,
         'code': "URL obj = new URL(url);",
         'result': False,
         'repair': True
     }]
     for test in tests:
         parse = Parse(regex_location, file_path, test['line'],
                       test['code'])
         self.assertEqual(test['result'], parse.is_controllable_param())
         if 'repair' in test:
             self.assertEqual(test['repair'],
                              parse.is_repair(regex_repair, 0))
Ejemplo n.º 15
0
def run():
    configs = []
    reccts = np.random.randint(18000000, 20000000, size=30)
    for recct in reccts:
        configs.append(config.Config(
            recct=int(recct), opct=0, fcount=0, fav=500, fdist='u', frange=200))

    folder = 'test8'
    status = misc.Status(folder, len(configs))

    while not status.finished:
        conf = configs[status.current]
        conf.load()
        # init
        print("\n{}:  recct={}".format(
            status.current, conf.recct))
        stopwatch = misc.Stopwatch()
        # flush, run, dump
        scripts.flush()
        stopwatch.lap(' - - flush: ')
        scripts.redis_all()
        stopwatch.lap(' - - workload: ')
        scripts.dump()
        stopwatch.lap(' - - dump: ')
        # lz4, zeros
        scripts.compress_lz4()
        stopwatch.lap(' - - lz4: ')
        scripts.zero_count()
        stopwatch.lap(' - - zeros: ')
        # write into final res folder
        res_lz4_out(folder)
        res_zeros_out(folder)
        status.inc()
        stopwatch.total(' - total: ')
Ejemplo n.º 16
0
Archivo: git.py Proyecto: bongwa/cobra
    def __init__(self,
                 repo_address,
                 branch='master',
                 username=None,
                 password=None):

        # get upload directory
        self.upload_directory = os.path.join(
            config.Config('upload', 'directory').value, 'versions')
        if os.path.isdir(self.upload_directory) is False:
            os.makedirs(self.upload_directory)

        self.repo_address = repo_address
        self.repo_username = username
        self.repo_password = password
        self.repo_branch = branch
        repo_user = self.repo_address.split('/')[-2]
        repo_name = self.repo_address.split('/')[-1].replace('.git', '')
        self.repo_author = repo_user
        self.repo_name = repo_name
        if '.git' not in self.repo_address:
            self.repo_address += '.git'
        else:
            repo_name = repo_name.split('.')[0]

        self.repo_directory = os.path.join(
            os.path.join(self.upload_directory, repo_user), repo_name)

        logging.info('Git class init.')
Ejemplo n.º 17
0
def run():
    configs = []
    for recct in [1000 * (i+1) for i in range(3000)]:
        configs.append(config.Config(recct=recct, opct=0, fav=107))

    folder = 'test7'
    status = misc.Status(folder, len(configs))

    while not status.finished:
        conf = configs[status.current]
        conf.load()
        # init
        print("\n{}:  recct={}".format(
            status.current, conf.recct))
        stopwatch = misc.Stopwatch()
        # flush, run, dump
        scripts.flush()
        stopwatch.lap(' - - flush: ')
        scripts.redis_all()
        stopwatch.lap(' - - workload: ')
        scripts.dump()
        stopwatch.lap(' - - dump: ')
        # zstd, lz4, zeros
        scripts.compress_zstd()
        stopwatch.lap(' - - zstd: ')
        scripts.compress_lz4()
        stopwatch.lap(' - - lz4: ')
        scripts.zero_count()
        stopwatch.lap(' - - zeros: ')
        # write into final res folder
        res_zstd_out(folder)
        res_lz4_out(folder)
        res_zeros_out(folder)
        status.inc()
        stopwatch.total(' - total: ')
Ejemplo n.º 18
0
	def __init__(self, old_process=None):
		self.console_input_thread = None
		self.info_reactor_thread = None
		self.info_queue = queue.Queue(maxsize=constant.MAX_INFO_QUEUE_SIZE)
		self.process = old_process  # the process for the server
		self.server_status = ServerStatus.STOPPED
		self.flag_interrupt = False  # ctrl-c flag
		self.flag_rcon_startup = False  # set to True after server startup. used to start the rcon server
		self.flag_exit = False  # MCDR exit flag
		self.starting_server_lock = Lock()  # to prevent multiple start_server() call

		# will be assigned in reload_config()
		self.encoding_method = None
		self.decoding_method = None

		self.logger = logger.Logger(self, constant.NAME_SHORT)
		self.logger.set_file(constant.LOGGING_FILE)
		self.server_logger = logger.ServerLogger('Server')
		self.language_manager = LanguageManager(self, constant.LANGUAGE_FOLDER)
		self.config = config.Config(self, constant.CONFIG_FILE)
		self.rcon_manager = RconManager(self)
		self.parser_manager = ParserManager(self)
		self.load_config()
		self.reactors = self.load_reactor(constant.REACTOR_FOLDER)
		self.server_interface = ServerInterface(self)
		self.command_manager = CommandManager(self)
		self.plugin_manager = PluginManager(self, constant.PLUGIN_FOLDER)
		self.load_plugins()
		self.permission_manager = PermissionManager(self, constant.PERMISSION_FILE)
		self.update_helper = UpdateHelper(self)
		self.update_helper.check_update_start()
Ejemplo n.º 19
0
    def __init__(self, dialog):
        self.settings = config.Config("settings.json",
                                      default={
                                          "blocked_users": [],
                                          "server_ip": "",
                                          "server_port": "",
                                          "should_render_markdown": True
                                      })
        self.window = dialog

        Ui_LoginWindow.__init__(self)
        self.setupUi(dialog)
        dialog.setWindowIcon(QtGui.QIcon("./utils/ui/files/icon.png"))

        server_ip = self.settings.get("server_ip")
        server_port = self.settings.get("server_port")
        do_markdown = self.settings.get("should_render_markdown")

        if server_ip:
            self.AddressField.setText(server_ip)

        if server_port:
            self.PortField.setText(server_port)

        if isinstance(do_markdown, bool):
            self.MarkdownCheck.setChecked(do_markdown)
        else:
            do_markdown = True
            self.MarkdownCheck.setChecked(True)

        self.LoginButton.clicked.connect(self.validate_input)
Ejemplo n.º 20
0
def bootstrap(configfile, console_app):
    conf = config.Config(configfile)

    loglevels = {
        "debug": DEBUG,
        "info": INFO,
        "error": ERROR,
        "warning": WARNING
    }

    basicConfig(filename=conf.logfile,
                format="[%(asctime)s] %(name)s |%(levelname)s| %(message)s",
                datefmt="%Y-%m-%d %H:%M:%S",
                level=loglevels[conf.loglevel])

    if console_app:
        root_logger = getLogger()
        child_logger = StreamHandler(stdout)
        child_logger.setLevel(DEBUG)
        formatter = Formatter(
            "[%(asctime)s] %(name)s |%(levelname)s| %(message)s",
            "%Y-%m-%d %H:%M:%S")
        child_logger.setFormatter(formatter)
        root_logger.addHandler(child_logger)
        info("started as console application")
    else:
        info("started as daemon")

    info("using config file: '{c}'".format(c=configfile))
    debug("working directory: '{w}', pid: '{pid}'".format(w=getcwd(),
                                                          pid=getpid()))
    debug("config:\n" + str(conf.get_raw_cfg()))
    signal(SIGTERM, signal_handler)
Ejemplo n.º 21
0
Archivo: scan.py Proyecto: z0x010/cobra
    def compress(self):
        dc = decompress.Decompress(self.target)
        ret, result_d = dc.decompress()
        if ret is False:
            return 1002, result_d
        else:
            directory = result_d
        logging.info("Scan directory: {0}".format(directory))
        current_time = time.strftime('%Y-%m-%d %X', time.localtime())

        p = CobraProjects.query.filter_by(repository=directory).first()

        # detection framework for project
        framework, language = detection.Detection(directory).framework()
        if framework != '' or language != '':
            project_framework = '{0} ({1})'.format(framework, language)
        else:
            project_framework = ''
        if not p:
            # insert into project table.
            repo_name = directory.split('/')[-1]
            project = CobraProjects(directory, '', repo_name, 'Upload',
                                    project_framework, '', '', 1, current_time)
            db.session.add(project)
            db.session.commit()
            project_id = project.id
        else:
            project_id = p.id
            # update project's framework
            p.framework = project_framework
            db.session.add(p)

        task = CobraTaskInfo(directory, '', 3, '', '', 0, 0, 0, 1, 0, 0,
                             current_time, current_time)
        db.session.add(task)
        db.session.commit()
        cobra_path = os.path.join(config.Config().project_directory,
                                  'cobra.py')
        if os.path.isfile(cobra_path) is not True:
            return 1004, 'Cobra Not Found'
        # 扫描漏洞
        subprocess.Popen([
            'python', cobra_path, "scan", "-p",
            str(project_id), "-i",
            str(task.id), "-t", directory
        ])
        # 统计代码行数
        subprocess.Popen([
            'python', cobra_path, "statistic", "-i",
            str(task.id), "-t", directory
        ])
        # 检测漏洞修复状况
        subprocess.Popen(
            ['python', cobra_path, "repair", "-p",
             str(project_id)])
        result = dict()
        result['scan_id'] = task.id
        result['project_id'] = project_id
        result['msg'] = u'success'
        return 1001, result
Ejemplo n.º 22
0
def main(argv):
    settings = config.Config("./path/to/directory/filename.csv", {
        "host": "localhost",
        "port": 9200
    }, "twitter", "tweets", "tweet_id")

    upload_data = read_file(
        settings.file_url,
        settings.index_name,
        settings.type_name,
        settings.id_field,
    )

    response = bulk_create(
        upload_data,
        settings.es_host,
        settings.index_name,
    )

    if (response):
        print('finished bulk upload...')
        return 0
    else:
        print('there was an error in your bulk_create method call')
        return 0
Ejemplo n.º 23
0
 def __init__(self, **kwargs):
     Thread.__init__(self)
     self.setDaemon(True)
     self.__params = kwargs
     self.__params["repository_path"] = config.Config().repo_path
     self.__params["artefacts_path"] = config.Config().artefacts_path
     self.__start_time = 0
     self.__end_time = 0
     self.__log = ""
     self.__last_commit_info = {}
     self.__build_executor = None
     self.__terminate_flag = Event()
     self.__terminated = False
     self.__error = True
     self.__import_custom_script()
     self.__error = False
Ejemplo n.º 24
0
 def compress(self):
     dc = decompress.Decompress(self.target)
     ret, result_d = dc.decompress()
     if ret is False:
         return 1002, result_d
     else:
         directory = result_d
     log.info("Scan directory: {0}".format(directory))
     current_time = time.strftime('%Y-%m-%d %X', time.localtime())
     task = CobraTaskInfo(self.target, '', 3, '', '', 0, 0, 0, 1, 0, 0,
                          current_time, current_time)
     db.session.add(task)
     db.session.commit()
     cobra_path = os.path.join(config.Config().project_directory,
                               'cobra.py')
     if os.path.isfile(cobra_path) is not True:
         return 1004, 'Cobra Not Found'
     # Start Scanning
     subprocess.Popen([
         'python', cobra_path, "scan", "-p",
         str(0), "-i",
         str(task.id), "-t", directory
     ])
     # Statistic Code
     subprocess.Popen([
         'python', cobra_path, "statistic", "-i",
         str(task.id), "-t", directory
     ])
     result = {}
     result['scan_id'] = task.id
     result['project_id'] = 0
     result['msg'] = u'success'
     return 1001, result
Ejemplo n.º 25
0
    def __init__(self, username, password):
        self._debug = False
        self.username = username
        self.password = password
        self.user_list = []
        self.loop = asyncio.get_event_loop()
        self.user_colors = config.Config("user_colors.json")
        self.settings = config.Config("settings.json")

        self.ip = self.settings.get("server_ip")
        self.port = self.settings.get("server_port")
        renderer = Renderer()
        lexer = InlineLexer(renderer)
        lexer.enable_underscore()
        self.clean = lambda: os.system("cls" if os.name == "nt" else "clear")

        self.markdown = mistune.Markdown(renderer, inline=lexer)
Ejemplo n.º 26
0
 def __init__(self, filename):
     """
     :param filename: a file name without path.
     """
     self.upload_directory = os.path.join(config.Config('upload', 'directory').value, 'uploads')
     self.filename = filename
     self.filepath = os.path.join(self.upload_directory, filename)
     self.dir_name = os.path.splitext(self.filename)[0]
Ejemplo n.º 27
0
def download_delete(name):
    try:
        filepath = path.join(config.Config().artefacts_path, name)
        log.info("deleting artefact '{f}'".format(f=filepath))
        remove(filepath)
    except:
        log.exception("error removing file '{f}'".format(f=filepath))
        raise
Ejemplo n.º 28
0
 def decorated_function(*args, **kwargs):
     token = request.args.get('token')
     is_auth_token = token is not None and token == config.Config(
         'cobra', 'secret_key').value
     if not session.get('is_login'):
         if not is_auth_token:
             return redirect(ADMIN_URL + '/index')
     return f(*args, **kwargs)
Ejemplo n.º 29
0
class TestConfig(unittest.TestCase):
    conf = config.Config("../app.conf")

    def test_get_string(self):
        self.assertEqual(self.conf.get_string("DB", "host"), '"localhost"')

    def test_get_int_ary(self):
        self.assertEqual(self.conf.get_int_ary("REDIS", "test"), [1, 2, 3])
Ejemplo n.º 30
0
 def __init__(self, biases):
     self.db = database.DB()
     self.config = config.Config()
     self.commands = [["getAnonymousRecommendations", [int]],
                      ["getRecommendations", [str, str, str]]]
     self.__loadRecommenders()
     self.__cache = {}
     self.biases = biases