Example #1
0
    def create_containers(self, pool=None, pool_num=10, num_containers=100, with_io=False):
        """To create number of containers parallelly on pool.
        Args:
            pool(str): pool handle.
            pool_num (int): pool number to create containers.
            num_containers (int): number of containers to create.
            with_io (bool): enable container test with execute_io.
        """

        self.log.info("==(2.%d)create_containers start.", pool_num)
        thread_manager = ThreadManager(self.create_container_and_test, self.timeout - 30)

        for cont_num in range(num_containers):
            thread_manager.add(
                pool=pool, pool_num=pool_num, cont_num=cont_num, with_io=with_io)

        # Launch the create_container_and_test threads
        self.log.info("==Launching %d create_container_and_test threads", thread_manager.qty)
        failed_thread_count = thread_manager.check_run()
        self.log.info(
            "==(2.%d) after thread_manager_run, %d containers created.", pool_num, num_containers)
        if failed_thread_count > 0:
            msg = "#(2.{}) FAILED create_container_and_test Threads".format(failed_thread_count)
            self.d_log.error(msg)
            self.fail(msg)
Example #2
0
 def __init__(self):
     self.service_manager = ServiceManager()
     ThreadManager.instance().add_thread(
         'ServicesVerifier',
         self.service_manager.verify_services_status,
         sleep_time=2)
     ThreadManager.instance().start_thread('ServicesVerifier')
Example #3
0
    def __init__(self):
        """Program entry point"""
        op = argparse.ArgumentParser()
        op.add_argument("-c", "--config", dest="file", default="/etc/mysql-statsd.conf", help="Configuration file")
        op.add_argument("-d", "--debug", dest="debug", help="Debug mode", default=False, action="store_true")

        # TODO switch the default to True, and make it fork by default in init script.
        op.add_argument("-f", "--foreground", dest="foreground", help="Dont fork main program", default=False, action="store_true")

        opt = op.parse_args()
        self.get_config(opt.file)

        logfile = self.config.get('daemon').get('logfile', '/tmp/daemon.log')
        if not opt.foreground:
            self.daemonize(stdin='/dev/null', stdout=logfile, stderr=logfile)

        # Set up queue
        self.queue = Queue.Queue()

        # split off config for each thread
        mysql_config = dict(mysql=self.config['mysql'])
        mysql_config['metrics'] = self.config['metrics']

        statsd_config = self.config['statsd']

        # Spawn MySQL polling thread
        mysql_thread = ThreadMySQL(queue=self.queue, **mysql_config)
        # t1 = ThreadMySQL(config=self.config, queue=self.queue)

        # Spawn Statsd flushing thread
        statsd_thread = ThreadStatsd(queue=self.queue, **statsd_config)

        # Get thread manager
        tm = ThreadManager(threads=[mysql_thread, statsd_thread])
        tm.run()
Example #4
0
 def __init__(self):
     """"""
     ThreadManager.__init__(self)
     self.active_downloads = {}
     self.queue_downloads = OrderedDict()
     self.complete_downloads = {}
     self.stopped_downloads = {}
     self.global_slots = Slots()
Example #5
0
 def __init__(self):
     """"""
     ThreadManager.__init__(self)
     self.active_downloads = {}
     self.queue_downloads = OrderedDict()
     self.complete_downloads = {}
     self.stopped_downloads = {}
     self.global_slots = Slots()
Example #6
0
    def __init__(self, username='', password='', get_dialogs_interval=60):

        self.delay_on_reply = config.get('vkbot_timing.delay_on_reply', 'i')
        self.chars_per_second = config.get('vkbot_timing.chars_per_second',
                                           'i')
        self.same_user_interval = config.get('vkbot_timing.same_user_interval',
                                             'i')
        self.same_conf_interval = config.get('vkbot_timing.same_conf_interval',
                                             'i')
        self.forget_interval = config.get('vkbot_timing.forget_interval', 'i')
        self.delay_on_first_reply = config.get(
            'vkbot_timing.delay_on_first_reply', 'i')
        self.stats_dialog_count = config.get('stats.dialog_count', 'i')
        self.no_leave_conf = config.get('vkbot.no_leave_conf', 'b')

        self.api = vkapi.VkApi(username,
                               password,
                               ignored_errors=ignored_errors,
                               timeout=config.get(
                                   'vkbot_timing.default_timeout', 'i'),
                               token_file=accounts.getFile('token.txt'),
                               log_file=accounts.getFile('inf.log')
                               if args.args['logging'] else '',
                               captcha_handler=createCaptchaHandler())
        stats.update('logging', bool(self.api.log_file))
        # hi java
        self.users = UserCache(
            self.api, self.fields + ',' +
            FriendController.requiredFields(_getFriendControllerParams()),
            config.get('cache.user_invalidate_interval', 'i'))
        self.confs = ConfCache(
            self.api, config.get('cache.conf_invalidate_interval', 'i'))
        self.vars = json.load(open('data/defaultvars.json', encoding='utf-8'))
        self.vars['default_bf'] = self.vars['bf']['id']
        self.initSelf(True)
        self.guid = int(time.time() * 5)
        self.last_viewed_comment = stats.get('last_comment', 0)
        self.good_conf = {}
        self.tm = ThreadManager()
        self.last_message = MessageCache()
        if os.path.isfile(accounts.getFile('msgdump.json')):
            try:
                data = json.load(open(accounts.getFile('msgdump.json')))
                self.last_message.load(data['cache'])
                self.api.longpoll = data['longpoll']
            except json.JSONDecodeError:
                logging.warning('Failed to load messages')
            os.remove(accounts.getFile('msgdump.json'))
        else:
            logging.info('Message dump does not exist')
        self.bad_conf_title = lambda s: False
        self.admin = None
        self.banned_list = []
        self.message_lock = threading.Lock()
        self.banned = set()
        self.receiver = MessageReceiver(self.api, get_dialogs_interval)
        self.receiver.longpoll_callback = self.longpollCallback
    def __evaluate_derived_cost(self):

        thread_manager = ThreadManager(1)
        for theta_index in range(len(self.__theta)):
            thread_manager.attach(derived_cost,
                                  (theta_index, self.__y, self.__x,
                                   self.__theta, self.__predict))

        return thread_manager.execute_all()
Example #8
0
 def stop_service(self, service_name):
     """Stops a service on the thread manager"""
     thread_manager = ThreadManager.instance()
     if thread_manager.is_thread_alive(service_name):
         if service_name == 'AudioRetriever':
             self.audio_retriever.stop_audio()
         ThreadManager.instance().stop_thread(service_name)
         print('\nService stopped successfully\n')
     else:
         print('\nCannot stop service that is not running\n')
 def __init__(self, runtimeDomain, fileManager, remoteObjectManager,
              basepath='.', **kwargs):
     HandlerDomain.__init__(self, **kwargs)
     self.runtimeDomain = runtimeDomain
     self.fileManager = fileManager
     self.remoteObjectManager = remoteObjectManager
     self.locationSerializer = serialize.LocationSerializer(
         fileManager, basepath)
     self.moduleSourcePathUpdater = ModuleSourcePathUpdater(
         self.debugger.GetSelectedTarget(), fileManager, basepath)
     self.thread_manager = ThreadManager(self.socket, self.locationSerializer, self.remoteObjectManager)
Example #10
0
 def main_menu(self):
     """Choice for main menu"""
     looping = True
     while looping:
         MenuMessages.instance().print_main_menu()
         option = self.parse_option(input())
         if option == 1:
             self.services_menu()
         elif option == 2:
             looping = False
             ThreadManager.instance().stop_all()
         else:
             print('Invalid option')
Example #11
0
    def __init__(self, source, resolution):
        ''' Sets the video source and creates the thread manager.

        Args:
            source (str): The file path of the video stream.
            resolution (tuple[int]): The camera resolution. 

        '''

        self.source = source
        self.resolution = resolution
        self.frame = None  # current frame is stored here by background thread

        self.thread_manager = ThreadManager(self)
Example #12
0
 def start_light_manager(self, show_message=True):
     """Starts the light maanager service"""
     thread_manager = ThreadManager.instance()
     thread_manager.add_thread('LightManager', self.light_manager.rgb_set)
     thread_manager.start_thread('LightManager')
     print('\nStarted light manager successfully\n')
     self.services_running['LightManager'] = True
Example #13
0
 def start_big_query_synchronizer(self, show_message=True):
     thread_manager = ThreadManager.instance()
     thread_manager.add_thread('BigQuerySynchronizer',
                               self.big_query_synchronizer.synchronize,
                               sleep_time=300)
     thread_manager.start_thread('BigQuerySynchronizer')
     print('\nStarted service big query synchronization successfully\n')
     self.services_running['BigQuerySynchronizer'] = True
Example #14
0
 def start_data_processor(self, show_message=True):
     """Starts the data processor service"""
     thread_manager = ThreadManager.instance()
     thread_manager.add_thread('DataProcessor',
                               self.data_processor.convert_data,
                               sleep_time=60)
     thread_manager.start_thread('DataProcessor')
     print('\nStarted data processor successfully\n')
     self.services_running['DataProcessor'] = True
Example #15
0
def minibatch_error_evaluation_function(y,
                                        x,
                                        theta,
                                        prediction_function,
                                        percentage_to_evaluate=0.3):

    setsize = math.floor(len(y) * percentage_to_evaluate)

    thread_manager = ThreadManager(1)
    for i in range(setsize):
        thread_manager.attach(evaluate_error,
                              (y[i], x[i], theta, prediction_function))

    errors = thread_manager.execute_all()

    error = sum(errors) / setsize

    return error
class ThreadedEngine(object):
    def __init__(self, source, engine):
        self.source = source
        self.engine = engine
        self.thread_manager = ThreadManager(self)
        self.pred = None

    def __next__(self):
        return self.get_prediction()

    def _thread(self):
        predictions = self.inference_gen()
        for prediction in predictions:
            self.pred = prediction
            self.thread_manager.set()

    def inference_gen(self):
        while True:
            self.frame = next(self.source)
            yield self.engine.invoke(self.frame)

    def get_prediction(self):
        self.thread_manager.wait()
        return self.pred, self.frame

    def get_max_length(self):
        return self.engine.get_max_length()

    def label(self, pred):
        return self.engine.label(pred)

    def start(self):
        self.thread_manager.start()
Example #17
0
 def start_audio_retriever(self, show_message=True):
     """Starts the audio retrieving service"""
     thread_manager = ThreadManager.instance()
     if not self.audio_retriever.is_recorder_device_ready():
         if show_message:
             print(
                 '\nCannot start audio retrieving, verify audio devices\n')
         return
     thread_manager.add_thread('AudioRetriever',
                               self.audio_retriever.retrieve_audio)
     thread_manager.start_thread('AudioRetriever')
     print('\nStarted audio retrieving successfully\n')
     self.services_running['AudioRetriever'] = True
Example #18
0
    def __init__(self):
        """Program entry point"""
        op = argparse.ArgumentParser()

        op.add_argument("-c", "--config", dest="file", default="/etc/mysql-statsd.conf", help="Configuration file")
        op.add_argument("-d", "--debug", dest="debug", help="Debug mode", default=False, action="store_true")

        self.opt = op.parse_args()
        opt = self.opt

        # Set up queue
        self.queue = Queue.Queue()

        # Spawn MySQL polling thread
        t1 = ThreadMySQL(config=self.config, queue=self.queue)

        # Spawn Statsd flushing thread
        t2 = ThreadStatsd(config=self.config, queue=self.queue)

        # Get thread manager
        tm = ThreadManager(threads=[t1, t2])

        tm.run()
Example #19
0
    def create_pools(self, num_pools=10, num_containers=100, with_io=False):
        """To create number of pools and containers parallelly.
        Args:
            num_pools (int): number of pools to create.
            num_containers (int): number of containers to create.
            with_io (bool): enable container test with execute_io.
        """

        # Setup the thread manager
        thread_manager = ThreadManager(self.create_containers, self.timeout - 30)

        for pool_number in range(num_pools):
            pool = self.get_pool()
            thread_manager.add(
                pool=pool, pool_num=pool_number, num_containers=num_containers, with_io=with_io)
            self.log.info("=(1.%d) pool created, %d.", pool_number, pool)

        # Launch the create_containers threads
        self.log.info("=Launching %d create_containers threads", thread_manager.qty)
        failed_thread_count = thread_manager.check_run()
        if failed_thread_count > 0:
            msg = "#(1.{}) FAILED create_containers Threads".format(failed_thread_count)
            self.d_log.error(msg)
            self.fail(msg)
Example #20
0
 def start_drive_synchronizer(self, show_message=True):
     thread_manager = ThreadManager.instance()
     self.drive_synchronizer.authenticate()
     if not self.drive_synchronizer._is_authenticated:
         if show_message:
             print(
                 '\nNot authenticated, cannot start drive synchronization\n'
             )
         return
     thread_manager.add_thread('DriveSynchronizer',
                               self.drive_synchronizer.synchronize,
                               sleep_time=300)
     thread_manager.start_thread('DriveSynchronizer')
     print('\nStarted service drive synchronization successfully\n')
     self.services_running['DriveSynchronizer'] = True
Example #21
0
 def __init__(self, debugger, chrome_channel, ipc_channel, is_attach, basepath='.'):
     '''
     chrome_channel: channel to send client chrome notification messages.
     ipc_channel: channel to send output/atom notification messages.
     debugger: lldb SBDebugger object.
     '''
     self._debugger = debugger
     self._chrome_channel = chrome_channel
     self._ipc_channel = ipc_channel
     self._is_attach = is_attach
     self._file_manager = FileManager(chrome_channel)
     self._remote_object_manager = RemoteObjectManager()
     self._location_serializer = serialize.LocationSerializer(
         self._file_manager, basepath)
     self._thread_manager = ThreadManager(self)
Example #22
0
 def start_service(self, service_name):
     """Starts a service on the thread manager"""
     thread_manager = ThreadManager.instance()
     if thread_manager.is_thread_alive(service_name):
         print('Service is already running')
         return
     if service_name == 'AudioRetriever':
         self.start_audio_retriever()
     if service_name == 'DataProcessor':
         self.start_data_processor()
     if service_name == 'LightManager':
         self.start_light_manager()
     if service_name == 'DriveSynchronizer':
         self.start_drive_synchronizer()
     if service.name == 'BigQuerySynchronizer':
         self.start_big_query_synchronizer()
Example #23
0
 def __init__(self, debugger, chrome_channel, ipc_channel, is_attach, basepath='.'):
     '''
     chrome_channel: channel to send client chrome notification messages.
     ipc_channel: channel to send output/atom notification messages.
     debugger: lldb SBDebugger object.
     '''
     self._debugger = debugger
     self._chrome_channel = chrome_channel
     self._ipc_channel = ipc_channel
     self._is_attach = is_attach
     self._file_manager = FileManager(chrome_channel)
     self._remote_object_manager = RemoteObjectManager()
     basepath = self._resolve_basepath_heuristic(basepath)
     log_debug('basepath: %s' % basepath)
     self._fixup_lldb_cwd_if_needed(basepath)
     self._location_serializer = serialize.LocationSerializer(
         self._file_manager, basepath)
     self._thread_manager = ThreadManager(self)
     self._debugger_settings = self._setDefaultDebuggerSettings()
Example #24
0
 def __init__(self, username='', password=''):
     self.api = vkapi.VkApi(username, password, ignored_errors=ignored_errors, timeout=config.get('vkbot.default_timeout', 'i'), token_file=accounts.getFile('token.txt'),
                            log_file=accounts.getFile('inf.log') if args.args['logging'] else '', captcha_handler=captcha.CaptchaHandler())
     self.api.initLongpoll()
     self.users = UserCache(self.api, 'sex,crop_photo,blacklisted,blacklisted_by_me,' + check_friend.fields)
     self.confs = ConfCache(self.api)
     self.vars = json.load(open('data/defaultvars.json', encoding='utf-8'))
     self.initSelf(True)
     self.guid = int(time.time() * 5)
     self.last_viewed_comment = stats.get('last_comment', 0)
     self.good_conf = {}
     self.tm = ThreadManager()
     self.last_message = MessageCache()
     self.last_message_id = 0
     self.whitelist = None
     self.whitelist_includeread = True
     self.bad_conf_title = lambda s: False
     self.admin = None
     self.banned_list = []
     self.longpoll_queue = queue.Queue()
     self.message_lock = threading.Lock()
Example #25
0
 def verify_services_status(self):
     """
     Verify the status of the services running in a thread, if one has been stopped
     by an exception it will start it again automatically
     """
     thread_manager = ThreadManager.instance()
     try:
         for key, value in self.services_running.items():
             if value and not thread_manager.stopped_manually(key):
                 if key == 'AudioRetriever':
                     self.start_audio_retriever(show_message=False)
                     logging.info(
                         'Restarting automatically service audio retrieving'
                     )
                 if key == 'DataProcessor':
                     self.start_data_processor(show_message=False)
                     logging.info(
                         'Restarting automatically service data processor')
                 if key == 'LightManager':
                     self.start_light_manager(show_message=False)
                     logging.info(
                         'Restarting automatically service light manager')
                 if key == 'DriveSynchronizer':
                     self.start_drive_synchronizer(show_message=False)
                     logging.info(
                         'Restarting automatically service drive synchronizer'
                     )
                 if key == 'BigQuerySynchronizer':
                     self.start_big_query_synchronizer(show_message=False)
                     logging.info(
                         'Restarting automatically service big query synchronizer'
                     )
     except Exception as error:
         logging.info(
             'Tried to restart a service but a fatal error ocurred ' +
             str(error))
 def __init__(self, source, engine):
     self.source = source
     self.engine = engine
     self.thread_manager = ThreadManager(self)
     self.pred = None
Example #27
0
    def test_metadata_server_restart(self):
        """JIRA ID: DAOS-1512.

        Test Description:
            This test will verify 2000 IOR small size container after server
            restart. Test will write IOR in 5 different threads for faster
            execution time. Each thread will create 400 (8bytes) containers to
            the same pool. Restart the servers, read IOR container file written
            previously and validate data integrity by using IOR option
            "-R -G 1".

        Use Cases:
            ?

        :avocado: tags=all,full_regression
        :avocado: tags=hw,large
        :avocado: tags=server,metadata,metadata_ior,nvme
        """
        self.create_pool()
        files_per_thread = 400
        total_ior_threads = 5

        processes = self.params.get("slots", "/run/ior/clientslots/*")

        list_of_uuid_lists = [[
            str(uuid.uuid4()) for _ in range(files_per_thread)
        ] for _ in range(total_ior_threads)]

        # Setup the thread manager
        thread_manager = ThreadManager(run_ior_loop, self.timeout - 30)

        # Launch threads to run IOR to write data, restart the agents and
        # servers, and then run IOR to read the data
        for operation in ("write", "read"):
            # Create the IOR threads
            for index in range(total_ior_threads):
                # Define the arguments for the run_ior_loop method
                ior_cmd = IorCommand()
                ior_cmd.get_params(self)
                ior_cmd.set_daos_params(self.server_group, self.pool)
                ior_cmd.flags.value = self.params.get(
                    "F", "/run/ior/ior{}flags/".format(operation))

                # Define the job manager for the IOR command
                self.ior_managers.append(Orterun(ior_cmd))
                env = ior_cmd.get_default_env(str(self.ior_managers[-1]))
                self.ior_managers[-1].assign_hosts(self.hostlist_clients,
                                                   self.workdir, None)
                self.ior_managers[-1].assign_processes(processes)
                self.ior_managers[-1].assign_environment(env)
                self.ior_managers[-1].verbose = False

                # Add a thread for these IOR arguments
                thread_manager.add(manager=self.ior_managers[-1],
                                   uuids=list_of_uuid_lists[index],
                                   tmpdir_base=self.test_dir)
                self.log.info("Created %s thread %s with container uuids %s",
                              operation, index, list_of_uuid_lists[index])

            # Launch the IOR threads
            self.log.info("Launching %d IOR %s threads", thread_manager.qty,
                          operation)
            failed_thread_count = thread_manager.check_run()
            if failed_thread_count > 0:
                msg = "{} FAILED IOR {} Thread(s)".format(
                    failed_thread_count, operation)
                self.d_log.error(msg)
                self.fail(msg)

            # Restart the agents and servers after the write / before the read
            if operation == "write":
                # Stop the agents
                errors = self.stop_agents()
                self.assertEqual(
                    len(errors), 0,
                    "Error stopping agents:\n  {}".format("\n  ".join(errors)))

                # Restart the servers w/o formatting the storage
                errors = self.restart_servers()
                self.assertEqual(
                    len(errors), 0, "Error stopping servers:\n  {}".format(
                        "\n  ".join(errors)))

                # Start the agents
                self.start_agent_managers()

        self.log.info("Test passed")
Example #28
0
 def __init__(self):
     """"""
     DownloadCore.__init__(self)  #inicializar download_core.py
     ThreadManager.__init__(self)  #inicializar thread_manager.py
     self.global_slots = Slots()  #slots.py
Example #29
0
 def __init__(self):
     """"""
     DownloadCore.__init__(self) #inicializar download_core.py
     ThreadManager.__init__(self) #inicializar thread_manager.py
     self.global_slots = Slots() #slots.py
Example #30
0
 def obtain_services_status(self):
     """Obtains and prints the status of services"""
     print('\n', ThreadManager.instance().status(), '\n')
Example #31
0
class VkBot:
    fields = 'sex,crop_photo,blacklisted,blacklisted_by_me'

    def __init__(self, username='', password='', get_dialogs_interval=60):

        self.delay_on_reply = config.get('vkbot_timing.delay_on_reply', 'i')
        self.chars_per_second = config.get('vkbot_timing.chars_per_second', 'i')
        self.same_user_interval = config.get('vkbot_timing.same_user_interval', 'i')
        self.same_conf_interval = config.get('vkbot_timing.same_conf_interval', 'i')
        self.forget_interval = config.get('vkbot_timing.forget_interval', 'i')
        self.delay_on_first_reply = config.get('vkbot_timing.delay_on_first_reply', 'i')
        self.stats_dialog_count = config.get('stats.dialog_count', 'i')
        self.no_leave_conf = config.get('vkbot.no_leave_conf', 'b')

        self.api = vkapi.VkApi(username, password, ignored_errors=ignored_errors, timeout=config.get('vkbot_timing.default_timeout', 'i'),
                               token_file=accounts.getFile('token.txt'),
                               log_file=accounts.getFile('inf.log') if args.args['logging'] else '', captcha_handler=createCaptchaHandler())
        stats.update('logging', bool(self.api.log_file))
        # hi java
        self.users = UserCache(self.api, self.fields + ',' + FriendController.requiredFields(_getFriendControllerParams()),
                               config.get('cache.user_invalidate_interval', 'i'))
        self.confs = ConfCache(self.api, config.get('cache.conf_invalidate_interval', 'i'))
        self.vars = json.load(open('data/defaultvars.json', encoding='utf-8'))
        self.vars['default_bf'] = self.vars['bf']['id']
        self.initSelf(True)
        self.guid = int(time.time() * 5)
        self.last_viewed_comment = stats.get('last_comment', 0)
        self.good_conf = {}
        self.tm = ThreadManager()
        self.last_message = MessageCache()
        if os.path.isfile(accounts.getFile('msgdump.json')):
            try:
                data = json.load(open(accounts.getFile('msgdump.json')))
                self.last_message.load(data['cache'])
                self.api.longpoll = data['longpoll']
            except json.JSONDecodeError:
                logging.warning('Failed to load messages')
            os.remove(accounts.getFile('msgdump.json'))
        else:
            logging.info('Message dump does not exist')
        self.bad_conf_title = lambda s: False
        self.admin = None
        self.banned_list = []
        self.message_lock = threading.Lock()
        self.banned = set()
        self.receiver = MessageReceiver(self.api, get_dialogs_interval)
        self.receiver.longpoll_callback = self.longpollCallback

    @property
    def whitelist(self):
        return self.receiver.whitelist

    @whitelist.setter
    def whitelist(self, new):
        self.receiver.whitelist = new

    def initSelf(self, sync=False):

        def do():
            try:
                res = self.api.users.get(fields='contacts,relation,bdate')[0]
            except IndexError:
                self.api.login()
                do()
                return
            self.self_id = res['id']
            self.vars['phone'] = res.get('mobile_phone') or self.vars['phone']
            self.vars['name'] = (res['first_name'], res['last_name'])
            self.vars['bf'] = res.get('relation_partner') or self.vars['bf']
            try:
                bdate = res['bdate'].split('.')
                today = datetime.date.today()
                self.vars['age'] = today.year - int(bdate[2]) - ((today.month, today.day) < (int(bdate[1]), int(bdate[0])))
            except LookupError:
                pass
            if not sync:
                logging.info('My phone: ' + self.vars['phone'])

        if sync:
            do()
        else:
            threading.Thread(target=do).start()

    def loadUsers(self, arr, key, clean=False):
        users = []
        confs = []
        for i in arr:
            try:
                pid = key(i)
                if pid <= 0:
                    continue
                if pid > CONF_START:
                    confs.append(pid - CONF_START)
                else:
                    users.append(pid)
            except Exception:
                pass
        self.users.load(users, clean)
        self.confs.load(confs, clean)

    def replyOne(self, message, gen_reply):
        if self.whitelist and getSender(message) not in self.whitelist:
            if getSender(message) > CONF_START:
                return
            if self.users[message['user_id']]['first_name'] + ' ' + self.users[message['user_id']]['last_name'] not in self.whitelist:
                return
        if message['user_id'] == self.self_id:  # chat with myself
            return
        if 'chat_id' in message and not self.checkConf(message['chat_id']):
            return
        try:
            if self.tm.isBusy(getSender(message)) and not self.tm.get(getSender(message)).attr['unimportant']:
                return
        except Exception:
            return
        if message['id'] < self.last_message.bySender(getSender(message)).get('id', 0):
            return

        try:
            ans = gen_reply(message)
        except Exception as e:
            ans = None
            logging.exception('local {}: {}'.format(e.__class__.__name__, str(e)))
            time.sleep(1)
        if ans:
            self.replyMessage(message, ans[0], ans[1])

    def replyAll(self, gen_reply):
        self.tm.gc()
        self.banned_list = []
        messages = self.receiver.getMessages()
        self.loadUsers(messages, lambda x: x['user_id'])
        self.loadUsers(messages, lambda x: x['chat_id'] + CONF_START)
        for cur in messages:
            self.replyOne(cur, gen_reply)
        if self.receiver.used_get_dialogs:
            stats.update('banned_messages', ' '.join(map(str, sorted(self.banned_list))))

    # noinspection PyUnusedLocal
    def longpollCallback(self, mid, flags, sender, ts, random_id, text, opt):
        if opt == {'source_mid': str(self.self_id), 'source_act': 'chat_kick_user', 'from': str(self.self_id)}:
            self.good_conf[sender] = False
            return True
        if opt.get('source_mid') == str(self.self_id) and opt.get('source_act') == 'chat_invite_user' and sender in self.good_conf:
            del self.good_conf[sender]
            return True

        if opt.get('source_act') == 'chat_title_update':
            del self.confs[sender - CONF_START]
            logging.info('Conf {} renamed into "{}"'.format(sender - CONF_START, opt['source_text']))
            if not self.no_leave_conf and self.bad_conf_title(opt['source_text']):
                self.leaveConf(sender - CONF_START)
                log.write('conf', 'conf ' + str(sender - CONF_START) + ' (name: {})'.format(opt['source_text']))
                return True
        if opt.get('source_act') == 'chat_invite_user' and opt['source_mid'] == str(self.self_id) and opt['from'] != str(self.self_id):
            self.logSender('%sender% added me to conf "{}" ({})'.format(self.confs[sender - CONF_START]['title'], sender - CONF_START),
                           {'user_id': int(opt['from'])})
            if not self.no_leave_conf and int(opt['from']) not in self.banned:
                self.deleteFriend(int(opt['from']))
        if flags & 2:  # out
            if not opt.get('source_act'):
                self.tm.terminate(sender)
            return True
        try:
            if 'from' in opt and int(opt['from']) != self.tm.get(sender).attr['user_id'] and not opt.get('source_act'):
                self.tm.get(sender).attr['reply'] = True
        except Exception:
            pass

    def sendMessage(self, to, msg, forward=None):
        if not self.good_conf.get(to, 1):
            return
        with self.message_lock:
            self.guid += 1
            time.sleep(1)
            if forward:
                return self.api.messages.send(peer_id=to, message=msg, random_id=self.guid, forward_messages=forward)
            else:
                return self.api.messages.send(peer_id=to, message=msg, random_id=self.guid)

    def replyMessage(self, message, answer, skip_mark_as_read=False):
        sender = getSender(message)
        sender_msg = self.last_message.bySender(sender)
        if 'id' in message and message['id'] <= sender_msg.get('id', 0):
            return

        if not answer:
            if self.tm.isBusy(sender):
                return
            if not sender_msg or time.time() - sender_msg['time'] > self.forget_interval:
                tl = Timeline().sleep(self.delay_on_first_reply).do(lambda: self.api.messages.markAsRead(peer_id=sender))
                tl.attr['unimportant'] = True
                self.tm.run(sender, tl, tl.terminate)
            elif answer is None:  # ignored
                self.api.messages.markAsRead.delayed(peer_id=sender, _once=True)
            else:
                tl = Timeline().sleep((self.delay_on_reply - 1) * random.random() + 1).do(lambda: self.api.messages.markAsRead(peer_id=sender))
                tl.attr['unimportant'] = True
                self.tm.run(sender, tl, tl.terminate)
            if answer is not None:
                self.last_message.byUser(message['user_id'])['text'] = message['body']
            self.last_message.updateTime(sender)
            if sender > CONF_START and 'action' not in message:
                sender_msg.setdefault('ignored', {})[message['user_id']] = time.time()
            return

        typing_time = 0
        if not answer.startswith('&#'):
            typing_time = len(answer) / self.chars_per_second

        resend = False
        # answer is not empty
        if sender_msg.get('reply', '').upper() == answer.upper() and sender_msg['user_id'] == message['user_id']:
            logging.info('Resending')
            typing_time = 0
            resend = True

        def _send(attr):
            if not set(sender_msg.get('ignored', [])) <= {message['user_id']}:
                ctime = time.time()
                for uid, ts in sender_msg['ignored'].items():
                    if uid != message['user_id'] and ctime - ts < self.same_conf_interval * 3:
                        attr['reply'] = True
            try:
                if resend:
                    res = self.sendMessage(sender, '', sender_msg['id'])
                elif attr.get('reply'):
                    res = self.sendMessage(sender, answer, message['id'])
                else:
                    res = self.sendMessage(sender, answer)
                if res is None:
                    del self.users[sender]
                    self.logSender('Failed to send a message to %sender%', message)
                    return
                msg = self.last_message.add(sender, message, res, answer)
                if resend:
                    msg['resent'] = True
            except Exception as e:
                logging.exception('thread {}: {}'.format(e.__class__.__name__, str(e)))

        cur_delay = (self.delay_on_reply - 1) * random.random() + 1
        send_time = cur_delay + typing_time
        user_delay = 0
        if sender_msg and sender != self.admin:
            user_delay = sender_msg['time'] - time.time() + (self.same_user_interval if sender < CONF_START else self.same_conf_interval)
            # can be negative
        tl = Timeline(max(send_time, user_delay))
        if 'chat_id' in message:
            tl.attr['user_id'] = message['user_id']
        if not sender_msg or time.time() - sender_msg['time'] > self.forget_interval:
            if not skip_mark_as_read:
                tl.sleep(self.delay_on_first_reply)
                tl.do(lambda: self.api.messages.markAsRead(peer_id=sender))
        else:
            tl.sleepUntil(send_time, (self.delay_on_reply - 1) * random.random() + 1)
            if not skip_mark_as_read:
                tl.do(lambda: self.api.messages.markAsRead(peer_id=sender))

        tl.sleep(cur_delay)
        if message.get('_onsend_actions'):
            for i in message['_onsend_actions']:
                tl.do(i)
                tl.sleep(cur_delay)
        if typing_time:
            tl.doEveryFor(vkapi.utils.TYPING_INTERVAL, lambda: self.api.messages.setActivity(type='typing', user_id=sender), typing_time)
        tl.do(_send, True)
        self.tm.run(sender, tl, tl.terminate)

    def checkConf(self, cid):
        if self.no_leave_conf:
            return True
        if cid + CONF_START in self.good_conf:
            return self.good_conf[cid + CONF_START]
        messages = self.api.messages.getHistory(chat_id=cid)['items']
        for i in messages:
            if i.get('action') == 'chat_create' and i['user_id'] not in self.banned:
                self.leaveConf(cid)
                self.deleteFriend(i['user_id'])
                log.write('conf', self.loggableName(i.get('user_id')) + ' ' + str(cid))
                return False
        title = self.confs[cid]['title']
        if self.bad_conf_title(title):
            self.leaveConf(cid)
            log.write('conf', 'conf ' + str(cid) + ' (name: {})'.format(title))
            return False
        self.good_conf[cid + CONF_START] = True
        return True

    def leaveConf(self, cid):
        if not self.confs[cid]:
            return False
        logging.info('Leaving conf {} ("{}")'.format(cid, self.confs[cid]['title']))
        self.good_conf[cid + CONF_START] = False
        return self.api.messages.removeChatUser(chat_id=cid, user_id=self.self_id)

    def addFriends(self, gen_reply, is_good):
        data = self.api.friends.getRequests(extended=1)
        to_rep = []
        self.loadUsers(data['items'], lambda x: x['user_id'], True)
        for i in data['items']:
            if self.users[i['user_id']].get('blacklisted'):
                self.api.friends.delete.delayed(user_id=i['user_id'])
                continue
            res = is_good(i['user_id'], True)
            if res is None:
                self.api.friends.add.delayed(user_id=i['user_id'])
                self.logSender('Adding %sender%', i)
                if 'message' in i:
                    ans = gen_reply(i)
                    to_rep.append((i, ans))
            else:
                self.api.friends.delete.delayed(user_id=i['user_id'])
                self.logSender('Not adding %sender% ({})'.format(res), i)
        for i in to_rep:
            self.replyMessage(i[0], i[1][0], i[1][1])
        self.api.sync()

    def unfollow(self):
        result = []
        requests = self.api.friends.getRequests(out=1)['items']
        suggested = self.api.friends.getRequests(suggested=1)['items']
        for i in requests:
            if i not in self.banned:
                result.append(i)
        for i in suggested:
            self.api.friends.delete.delayed(user_id=i)
        self.deleteFriend(result)
        return result

    def deleteFriend(self, uid):
        if type(uid) == int:
            self.api.friends.delete(user_id=uid)
        else:
            for i in uid:
                self.api.friends.delete.delayed(user_id=i)
            self.api.sync()

    def setOnline(self):
        self.api.account.setOnline()

    def getUserId(self, domain, is_conf=False):
        domain = str(domain).lower().rstrip().rstrip('}').rstrip()
        conf = re.search('sel=c(\\d+)', domain) or re.search('^c(\\d+)$', domain) or re.search('chat=(\\d+)', domain) or re.search('peer=2(\\d{9})',
                                                                                                                                   domain)
        if conf is not None:
            return int(conf.group(1)) + CONF_START
        if is_conf:
            if domain.isdigit():
                return int(domain) + CONF_START
            else:
                return None
        if '=' in domain:
            domain = domain.split('=')[-1]
        if '/' in domain:
            domain = domain.split('/')[-1]
        data = self.api.users.get(user_ids=domain)
        if not data:
            return None
        return data[0]['id']

    def deleteComment(self, rep):
        if rep['type'] == 'wall':
            self.api.wall.delete(owner_id=self.self_id, post_id=rep['feedback']['id'])
        elif rep['type'].endswith('photo'):
            self.api.photos.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])
        elif rep['type'].endswith('video'):
            self.api.video.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])
        else:
            self.api.wall.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])

    def filterComments(self, test):
        data = self.api.notifications.get(start_time=self.last_viewed_comment + 1, count=100)['items']
        to_del = set()
        to_bl = set()
        self.loadUsers(data, lambda x: x['feedback']['from_id'], True)
        for rep in data:
            if rep['date'] != 'i':
                self.last_viewed_comment = max(self.last_viewed_comment, int(rep['date']))

            def _check(s):
                if 'photo' in s:
                    return s['photo']['owner_id'] == self.self_id
                if 'video' in s:
                    return s['video']['owner_id'] == self.self_id
                if 'post' in s:
                    return s['post']['to_id'] == self.self_id

            if rep['type'].startswith('comment_') or (rep['type'].startswith('reply_comment') and _check(rep['parent'])) or rep['type'] == 'wall':
                txt = html.escape(rep['feedback']['text'])
                res = 'good'
                frid = int(rep['feedback']['from_id'])
                if self.users[frid]['blacklisted']:
                    res = 'blacklisted'
                    log.write('comments', self.loggableName(frid) + ' (blacklisted): ' + txt)
                    self.deleteComment(rep)
                    to_bl.add(frid)
                elif test(txt):
                    res = 'bad'
                    log.write('comments', self.loggableName(frid) + ': ' + txt)
                    self.deleteComment(rep)
                    to_del.add(frid)
                elif 'attachments' in rep['feedback'] and any(i.get('type') in ['video', 'link'] for i in rep['feedback']['attachments']):
                    res = 'attachment'
                    log.write('comments', self.loggableName(frid) + ' (attachment)')
                    self.deleteComment(rep)
                self.logSender('Comment {} (by %sender%) - {}'.format(txt, res), {'user_id': frid})
        stats.update('last_comment', self.last_viewed_comment)
        for i in to_bl:
            self.blacklist(i)
        return to_del

    def likeAva(self, uid):
        del self.users[uid]
        if 'crop_photo' not in self.users[uid]:
            return
        photo = self.users[uid]['crop_photo']['photo']
        self.api.likes.add(type='photo', owner_id=photo['owner_id'], item_id=photo['id'])
        self.logSender('Liked %sender%', {'user_id': uid})

    def setRelation(self, uid, set_by=None):
        if uid:
            log.write('relation', self.loggableName(uid))
        else:
            log.write('relation', self.loggableName(set_by) + ' (removed)')
            uid = self.vars['default_bf']
        self.api.account.saveProfileInfo(relation_partner_id=uid)
        self.vars['bf'] = self.users[uid]
        self.logSender('Set relationship with %sender%', {'user_id': uid})

    def waitAllThreads(self, loop_thread, reply):
        lp = self.api.longpoll.copy()
        self.receiver.terminate_monitor = True
        loop_thread.join(60)
        while not self.receiver.longpoll_queue.empty():
            self.replyAll(reply)
        for t in self.tm.all():
            t.join(60)
        with open(accounts.getFile('msgdump.json'), 'w') as f:
            json.dump({'cache': self.last_message.dump(), 'longpoll': lp}, f)

    # {name} - first_name last_name
    # {id} - id
    def printableName(self, pid, user_fmt, conf_fmt='Conf "{name}" ({id})'):
        if pid > CONF_START:
            return conf_fmt.format(id=(pid - CONF_START), name=self.confs[pid - CONF_START]['title'])
        else:
            return user_fmt.format(id=pid, name=self.users[pid]['first_name'] + ' ' + self.users[pid]['last_name'])

    def logSender(self, text, message):
        text_msg = text.replace('%sender%', self.printableSender(message, False))
        html_msg = html.escape(text).replace('%sender%', self.printableSender(message, True))
        logging.info(text_msg, extra={'db': html_msg})

    def printableSender(self, message, need_html):
        if message.get('chat_id', 0) > 0:
            if need_html:
                res = self.printableName(message['user_id'], user_fmt='Conf "%c" (%i), <a href="https://vk.com/id{id}" target="_blank">{name}</a>')
                return res.replace('%i', str(message['chat_id'])).replace('%c', html.escape(self.confs[message['chat_id']]['title']))
            else:
                res = self.printableName(message['user_id'], user_fmt='Conf "%c" (%i), {name}')
                return res.replace('%i', str(message['chat_id'])).replace('%c', html.escape(self.confs[message['chat_id']]['title']))
        else:
            if need_html:
                return self.printableName(message['user_id'], user_fmt='<a href="https://vk.com/id{id}" target="_blank">{name}</a>')
            else:
                return self.printableName(message['user_id'], user_fmt='{name}')

    def loggableName(self, uid):
        return self.printableName(uid, '{id} ({name})')

    def blacklist(self, uid):
        self.api.account.banUser(user_id=uid)

    def blacklistedCount(self):
        return self.api.account.getBanned(count=0)['count']

    def lastDialogs(self):

        def cb(req, resp):
            d.append((req['peer_id'], resp['count']))

        dialogs = self.api.messages.getDialogs(count=self.stats_dialog_count, preview_length=1)
        d = []
        confs = {}
        try:
            items = list(dialogs['items'])
            for dialog in items:
                if getSender(dialog['message']) in self.banned:
                    continue
                self.api.messages.getHistory.delayed(peer_id=getSender(dialog['message']), count=0).callback(cb)
                if 'title' in dialog['message']:
                    confs[getSender(dialog['message'])] = dialog['message']['title']
            self.api.sync()
        except TypeError:
            logging.warning('Unable to fetch dialogs')
            return (None, None, None)
        return (dialogs['count'], d, confs)

    def acceptGroupInvites(self):
        for i in self.api.groups.getInvites()['items']:
            logging.info('Joining group "{}"'.format(i['name']))
            self.api.groups.join(group_id=i['id'])
            log.write('groups', '{}: <a target="_blank" href="https://vk.com/club{}">{}</a>{}'.format(
                self.loggableName(i['invited_by']), i['id'], i['name'], ['', ' (closed)', ' (private)'][i['is_closed']]))

    def clearCache(self):
        self.users.clear()
        self.confs.clear()
Example #32
0
    def send_stat(self, item):
        (k, v, t) = item
        try:
            if t[1] == 'd':
                v = self.calculate_delta(k, v)
                t = t[0]
        except:
            pass
        sender = self.get_sender(t)
        sender(k, float(v))

    def run(self):
        while self.run:
            try:
                # Timeout after 1 second so we can respond to quit events
                item = self.queue.get(True, 1)
                self.send_stat(item)
            except Queue.Empty:
                continue


if __name__ == '__main__':
    # Run standalone to test this module, it will generate garbage
    from thread_manager import ThreadManager
    q = Queue.Queue()

    threads = [ThreadGenerateGarbage(q), ThreadStatsd(q)]
    tm = ThreadManager(threads=threads)
    tm.run()
Example #33
0
class DebuggerDomain(HandlerDomain):
    def __init__(self,
                 runtimeDomain,
                 fileManager,
                 remoteObjectManager,
                 basepath='.',
                 **kwargs):
        HandlerDomain.__init__(self, **kwargs)
        self.runtimeDomain = runtimeDomain
        self.fileManager = fileManager
        self.remoteObjectManager = remoteObjectManager
        self.locationSerializer = serialize.LocationSerializer(
            fileManager, basepath)
        self.moduleSourcePathUpdater = ModuleSourcePathUpdater(
            self.debugger.GetSelectedTarget(), fileManager, basepath)
        self.thread_manager = ThreadManager(self.socket,
                                            self.locationSerializer,
                                            self.remoteObjectManager)

    @property
    def name(self):
        return 'Debugger'

    @handler()
    def canSetScriptSource(self, params):
        # Return False, becuase we don't support
        # changing source at runtime.
        return {"result": False}

    @handler()
    def continueToLocation(self, params):
        # TODO(williamsc) - This is probably setting a one off breakpoint and continuing.
        raise UndefinedHandlerError('continueToLocation not implemented')

    @handler()
    def disable(self, params):
        # Not exactly the same as disable. Detach() might be closer to
        # what Chrome Dev Tools is trying to do.
        self.debugger.GetSelectedTarget().DisableAllBreakpoints()
        return {}

    @handler()
    def enable(self, params):
        process = self.debugger.GetSelectedTarget().process
        self.event_thread = LLDBListenerThread(
            server=self.socket,
            location_serializer=self.locationSerializer,
            remote_object_manager=self.remoteObjectManager,
            module_source_path_updater=self.moduleSourcePathUpdater,
            thread_manager=self.thread_manager,
            process=process)
        self.moduleSourcePathUpdater.modules_updated()
        self.event_thread.start()
        return {}

    @handler()
    def evaluateOnCallFrame(self, params):
        frameId = params['callFrameId']

        thread, frame = frameId.split('.')
        # TODO: These return booleans to indicate success. Throw something if False.
        self.debugger.GetSelectedTarget().process.SetSelectedThreadByIndexID(
            int(thread))
        self.debugger.GetSelectedTarget().process.GetSelectedThread(
        ).SetSelectedFrame(int(frame))

        return self.runtimeDomain.evaluate(params)

    @handler()
    def getScriptSource(self, params):
        filelike = self.fileManager.get_by_script_id(params['scriptId'])
        if filelike:
            return {'scriptSource': filelike.script_source}
        else:
            return {'scriptSource': '<Failed to fetch source.>'}

    @handler()
    def pause(self, params):
        self.debugger.GetSelectedTarget().process.Stop()
        return {}

    @handler()
    def removeBreakpoint(self, params):
        self.debugger.GetSelectedTarget().BreakpointDelete(
            int(params['breakpointId']))
        return {}

    @handler()
    def resume(self, params):
        self.debugger.GetSelectedTarget().process.Continue()
        return {}

    @handler()
    def selectThread(self, params):
        threadId = params['threadId']
        self.debugger.GetSelectedTarget().process.SetSelectedThreadByID(
            threadId)
        return {}

    @handler()
    def getThreadStack(self, params):
        threadId = params['threadId']
        thread = self.debugger.GetSelectedTarget().process.GetThreadByID(
            threadId)
        params = {"callFrames": []}
        if not thread == None:
            params["callFrames"] = self.thread_manager.get_thread_stack(thread)
        return params

    @handler()
    def searchInContent(self, params):
        raise UndefinedHandlerError('searchInContent not implemented')

    @handler()
    def setBreakpoint(self, params):
        filelike = self.fileManager.get_by_script_id(
            params['location']['scriptId'])
        if not filelike or not isinstance(filelike, file_manager.File):
            # Only support setting breakpoints in real files.
            return {}
        return self._set_breakpoint_by_filespec(
            filelike.server_obj,
            int(params['location']['lineNumber']) + 1)

    @handler()
    def setBreakpointByUrl(self, params):
        filelike = self.fileManager.get_by_client_url(params['url'])
        if not filelike or not isinstance(filelike, file_manager.File):
            raise RuntimeError('Cannot find file for breakpoint.')
        return self._set_breakpoint_by_filespec(filelike.server_obj,
                                                int(params['lineNumber']) + 1)

    @handler()
    def setBreakpointsActive(self, params):
        if params['active']:
            self.debugger.GetSelectedTarget().EnableAllBreakpoints()
        else:
            self.debugger.GetSelectedTarget().DisableAllBreakpoints()
        return {}

    @handler()
    def setPauseOnExceptions(self, params):
        # TODO(williamsc) - Support add support for pausing on exceptions
        raise UndefinedHandlerError('setPauseOnExceptions not implemented')

    @handler()
    def setScriptSource(self, params):
        raise UndefinedHandlerError('setScriptSource not supported for LLDB')

    @handler()
    def stepInto(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(
        ).StepInto()
        return {}

    @handler()
    def stepOut(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(
        ).StepOut()
        return {}

    @handler()
    def stepOver(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(
        ).StepOver()
        return {}

    def _set_breakpoint_by_filespec(self, filespec, line):
        breakpoint = self.debugger.GetSelectedTarget(
        ).BreakpointCreateByLocation(filespec, line)
        return {
            'breakpointId':
            str(breakpoint.id),
            'locations':
            self.locationSerializer.get_breakpoint_locations(breakpoint),
        }
Example #34
0
        elif t is 'r':
            return self.client.update_stats
        elif t is 'c':
            return self.client.incr
        elif t is 't':
            return self.client.timing

    def send_stat(self, item):
        (k, v, t) = item
        sender = self.get_sender(t)
        sender(k, float(v))

    def run(self):
        while self.run:
            try:
                # Timeout after 1 second so we can respond to quit events
                item = self.queue.get(True, 1)
                self.send_stat(item)
            except Queue.Empty:
                continue


if __name__ == '__main__':
    # Run standalone to test this module, it will generate garbage
    from thread_manager import ThreadManager
    q = Queue.Queue()

    threads = [ThreadGenerateGarbage(q), ThreadStatsd(q)]
    tm = ThreadManager(threads=threads)
    tm.run()
Example #35
0
class VkBot:
    delay_on_reply = config.get('vkbot.delay_on_reply', 'i')
    chars_per_second = config.get('vkbot.chars_per_second', 'i')
    same_user_interval = config.get('vkbot.same_user_interval', 'i')
    same_conf_interval = config.get('vkbot.same_conf_interval', 'i')
    typing_interval = 5
    forget_interval = config.get('vkbot.forget_interval', 'i')
    delay_on_first_reply = config.get('vkbot.delay_on_first_reply', 'i')
    stats_dialog_count = config.get('stats.dialog_count', 'i')

    def __init__(self, username='', password=''):
        self.api = vkapi.VkApi(username, password, ignored_errors=ignored_errors, timeout=config.get('vkbot.default_timeout', 'i'), token_file=accounts.getFile('token.txt'),
                               log_file=accounts.getFile('inf.log') if args.args['logging'] else '', captcha_handler=captcha.CaptchaHandler())
        self.api.initLongpoll()
        self.users = UserCache(self.api, 'sex,crop_photo,blacklisted,blacklisted_by_me,' + check_friend.fields)
        self.confs = ConfCache(self.api)
        self.vars = json.load(open('data/defaultvars.json', encoding='utf-8'))
        self.initSelf(True)
        self.guid = int(time.time() * 5)
        self.last_viewed_comment = stats.get('last_comment', 0)
        self.good_conf = {}
        self.tm = ThreadManager()
        self.last_message = MessageCache()
        self.last_message_id = 0
        self.whitelist = None
        self.whitelist_includeread = True
        self.bad_conf_title = lambda s: False
        self.admin = None
        self.banned_list = []
        self.longpoll_queue = queue.Queue()
        self.message_lock = threading.Lock()

    def initSelf(self, sync=False):
        self.users.clear()

        def do():
            res = self.api.users.get(fields='contacts,relation,bdate')[0]
            self.self_id = res['id']
            self.vars['phone'] = res.get('mobile_phone') or self.vars['phone']
            self.vars['name'] = (res['first_name'], res['last_name'])
            self.vars['bf'] = res.get('relation_partner') or self.vars['bf']
            try:
                bdate = res['bdate'].split('.')
                today = datetime.date.today()
                self.vars['age'] = today.year - int(bdate[2]) - ((today.month, today.day) < (int(bdate[1]), int(bdate[0])))
            except LookupError:
                pass
            if not sync:
                logging.info('My phone: ' + self.vars['phone'])

        if sync:
            do()
        else:
            threading.Thread(target=do).start()

    @staticmethod
    def getSender(message):
        if 'chat_id' in message:
            return CONF_START + message['chat_id']
        return message['user_id']

    def loadUsers(self, arr, key, clean=False, confs=False):
        users = []
        for i in arr:
            try:
                users.append(key(i))
            except Exception:
                pass
        (self.confs if confs else self.users).load(users, clean)

    def replyOne(self, message, gen_reply, method=None):
        if self.whitelist and self.getSender(message) not in self.whitelist:
            if self.getSender(message) > CONF_START or self.users[message['user_id']]['first_name'] + ' ' + self.users[message['user_id']]['last_name'] not in self.whitelist:
                return
        if 'chat_id' in message and not self.checkConf(message['chat_id']):
            return
        try:
            if self.tm.isBusy(self.getSender(message)) and not self.tm.get(self.getSender(message)).attr['unimportant']:
                return
        except Exception:
            return

        if method is not None:
            message['_method'] = method
        try:
            ans = gen_reply(message)
        except Exception as e:
            ans = None
            logging.exception('local {}: {}'.format(e.__class__.__name__, str(e)))
            time.sleep(1)
        if ans:
            self.replyMessage(message, ans[0], ans[1])

    def replyAll(self, gen_reply, include_read=False):
        self.tm.gc()
        self.banned_list = []
        if include_read:
            self.users.gc()
            if self.whitelist:
                messages = self.api.messages.getDialogs(unread=(0 if self.whitelist_includeread else 1), count=20)
                self.whitelist_includeread = False
            else:
                messages = self.api.messages.getDialogs(unread=1, count=200)
            try:
                messages = messages['items'][::-1]
            except TypeError:
                logging.warning('Unable to fetch messages')
                return
            self.loadUsers(messages, lambda x: x['message']['user_id'])
            self.loadUsers(messages, lambda x: x['message']['chat_id'], confs=True)
            for msg in sorted(messages, key=lambda m: m['message']['id']):
                cur = msg['message']
                if cur['out']:
                    continue
                if self.last_message_id and cur['id'] > self.last_message_id:
                    continue
                self.replyOne(cur, gen_reply, 'getDialogs')
            stats.update('banned_messages', ' '.join(map(str, sorted(self.banned_list))))

        else:
            messages = self.longpollMessages()
            self.loadUsers(messages, lambda x: x['user_id'])
            self.loadUsers(messages, lambda x: x['chat_id'], confs=True)
            for cur in sorted(messages, key=lambda m: m['id']):
                self.last_message_id = max(self.last_message_id, cur['id'])
                self.replyOne(cur, gen_reply)

    def longpollMessages(self):
        res = []
        while not self.longpoll_queue.empty():
            res.append(self.longpoll_queue.get())
        return res

    def getLongpoll(self):
        arr = self.api.getLongpoll()
        need_extra = []
        result = []
        for i in arr:
            if i[0] == 4:  # new message
                mid = i[1]
                sender = i[3]
                ts = i[4]
                text = i[6]
                opt = i[7]
                flags = i[2]
                if opt == {'source_mid': str(self.self_id), 'source_act': 'chat_kick_user', 'from': str(self.self_id)}:
                    self.good_conf[sender] = False
                    continue
                if opt.get('source_act') == 'chat_title_update':
                    del self.confs[sender - CONF_START]
                    logging.info('Conf {} renamed into "{}"'.format(sender - CONF_START, opt['source_text']))
                    if not config.get('vkbot.no_leave_conf') and self.bad_conf_title(opt['source_text']):
                        self.leaveConf(sender - CONF_START)
                        log.write('conf', 'conf ' + str(sender - CONF_START) + ' (name: {})'.format(opt['source_text']))
                        continue
                if opt.get('source_act') == 'chat_invite_user' and opt['source_mid'] == str(self.self_id) and opt['from'] != str(self.self_id):
                    self.logSender('%sender% added me to conf "{}"'.format(self.confs[sender - CONF_START]['title']), {'user_id': int(opt['from'])})
                    if not config.get('vkbot.no_leave_conf') and int(opt['from']) not in self.banned:
                        self.deleteFriend(int(opt['from']))
                if flags & 2:  # out
                    if not opt.get('source_act'):
                        self.tm.terminate(sender)
                    continue
                for number in range(1, 11):
                    if opt.get('attach{}_type'.format(number)) == 'photo':
                        del opt['attach{}_type'.format(number)]
                        del opt['attach{}'.format(number)]
                        text += ' ..'
                    if opt.get('attach{}_type'.format(number)) == 'doc':
                        if opt.get('attach{}_kind'.format(number)) == 'graffiti':
                            del opt['attach{}_type'.format(number)]
                            del opt['attach{}'.format(number)]
                            del opt['attach{}_kind'.format(number)]
                            text += ' ..'
                        if opt.get('attach{}_kind'.format(number)) == 'audiomsg':
                            del opt['attach{}_type'.format(number)]
                            del opt['attach{}'.format(number)]
                            del opt['attach{}_kind'.format(number)]
                            text += ' [Voice]'
                msg = {'id': mid, 'date': ts, 'body': text, 'out': 0, '_method': ''}
                if opt.get('source_act'):
                    msg['body'] = None
                if opt.get('attach1_type') == 'sticker':
                    msg['attachments'] = [{'type': 'sticker'}]

                if 'from' in opt:
                    msg['chat_id'] = sender - CONF_START
                    msg['user_id'] = int(opt['from'])
                else:
                    msg['user_id'] = sender
                try:
                    if 'chat_id' in msg and msg['user_id'] != self.tm.get(sender).attr['user_id'] and msg['body'] is not None:
                        self.tm.get(sender).attr['reply'] = True
                except Exception:
                    pass
                if not (set(opt) <= {'from', 'emoji'} or opt.get('attach1_type') == 'sticker') and not opt.get('source_act'):
                    need_extra.append(str(mid))
                    continue
                result.append(msg)

        if need_extra:
            need_extra = ','.join(need_extra)
            for i in self.api.messages.getById(message_ids=need_extra)['items']:
                i['_method'] = 'getById'
                result.append(i)
        return result

    def monitorLongpoll(self):
        def _monitor():
            while True:
                for i in self.getLongpoll():
                    self.longpoll_queue.put(i)
        self.longpoll_thread = threading.Thread(target=_monitor, daemon=True)
        self.longpoll_thread.start()

    def sendMessage(self, to, msg, forward=None):
        if not self.good_conf.get(to, 1):
            return
        with self.message_lock:
            self.guid += 1
            time.sleep(1)
            if forward:
                return self.api.messages.send(peer_id=to, message=msg, random_id=self.guid, forward_messages=forward)
            else:
                return self.api.messages.send(peer_id=to, message=msg, random_id=self.guid)

    def replyMessage(self, message, answer, skip_mark_as_read=False):
        sender = self.getSender(message)
        sender_msg = self.last_message.bySender(sender)
        if 'id' in message and message['id'] <= sender_msg.get('id', 0):
            return

        if not answer:
            if self.tm.isBusy(sender):
                return
            if not sender_msg or time.time() - sender_msg['time'] > self.forget_interval:
                tl = Timeline().sleep(self.delay_on_first_reply).do(lambda: self.api.messages.markAsRead(peer_id=sender))
                tl.attr['unimportant'] = True
                self.tm.run(sender, tl, tl.terminate)
            elif answer is None:  # ignored
                self.api.messages.markAsRead.delayed(peer_id=sender, _once=True)
            else:
                tl = Timeline().sleep((self.delay_on_reply - 1) * random.random() + 1).do(lambda: self.api.messages.markAsRead(peer_id=sender))
                tl.attr['unimportant'] = True
                self.tm.run(sender, tl, tl.terminate)
            self.last_message.byUser(message['user_id'])['text'] = message['body']
            self.last_message.updateTime(sender)
            if sender > CONF_START and message['body'] is not None:
                sender_msg.setdefault('ignored', {})[message['user_id']] = time.time()
            return

        typing_time = 0
        if not answer.startswith('&#'):
            typing_time = len(answer) / self.chars_per_second

        resend = False
        # answer is not empty
        if sender_msg.get('reply', '').upper() == answer.upper() and sender_msg['user_id'] == message['user_id']:
            logging.info('Resending')
            typing_time = 0
            resend = True

        def _send(attr):
            if not set(sender_msg.get('ignored', [])) <= {message['user_id']}:
                ctime = time.time()
                for uid, ts in sender_msg['ignored'].items():
                    if uid != message['user_id'] and ctime - ts < self.same_conf_interval * 3:
                        attr['reply'] = True
            try:
                if resend:
                    res = self.sendMessage(sender, '', sender_msg['id'])
                elif attr.get('reply'):
                    res = self.sendMessage(sender, answer, message['id'])
                else:
                    res = self.sendMessage(sender, answer)
                if res is None:
                    del self.users[sender]
                    self.logSender('Failed to send a message to %sender%', message)
                    return
                self.last_message.add(sender, message, res, answer)
            except Exception as e:
                logging.exception('thread {}: {}'.format(e.__class__.__name__, str(e)))

        cur_delay = (self.delay_on_reply - 1) * random.random() + 1
        send_time = cur_delay + typing_time
        user_delay = 0
        if sender_msg and sender != self.admin:
            user_delay = sender_msg['time'] - time.time() + (self.same_user_interval if sender < 2000000000 else self.same_conf_interval)  # can be negative
        tl = Timeline(max(send_time, user_delay))
        if 'chat_id' in message:
            tl.attr['user_id'] = message['user_id']
        if not sender_msg or time.time() - sender_msg['time'] > self.forget_interval:
            if not skip_mark_as_read:
                tl.sleep(self.delay_on_first_reply)
                tl.do(lambda: self.api.messages.markAsRead(peer_id=sender))
        else:
            tl.sleepUntil(send_time, (self.delay_on_reply - 1) * random.random() + 1)
            if not skip_mark_as_read:
                tl.do(lambda: self.api.messages.markAsRead(peer_id=sender))

        tl.sleep(cur_delay)
        if message.get('_onsend_actions'):
            for i in message['_onsend_actions']:
                tl.do(i)
                tl.sleep(cur_delay)
        if typing_time:
            tl.doEveryFor(self.typing_interval, lambda: self.api.messages.setActivity(type='typing', user_id=sender), typing_time)
        tl.do(_send, True)
        self.tm.run(sender, tl, tl.terminate)

    def checkConf(self, cid):
        if config.get('vkbot.no_leave_conf'):
            return True
        if cid + CONF_START in self.good_conf:
            return self.good_conf[cid + CONF_START]
        messages = self.api.messages.getHistory(chat_id=cid)['items']
        for i in messages:
            if i.get('action') == 'chat_create' and i['user_id'] not in self.banned:
                self.leaveConf(cid)
                self.deleteFriend(i['user_id'])
                log.write('conf', self.loggableName(i.get('user_id')) + ' ' + str(cid))
                return False
        title = self.confs[cid]['title']
        if self.bad_conf_title(title):
            self.leaveConf(cid)
            log.write('conf', 'conf ' + str(cid) + ' (name: {})'.format(title))
            return False
        self.good_conf[cid + CONF_START] = True
        return True

    def leaveConf(self, cid):
        logging.info('Leaving conf {} ("{}")'.format(cid, self.confs[cid]['title']))
        self.good_conf[cid + CONF_START] = False
        return self.api.messages.removeChatUser(chat_id=cid, user_id=self.self_id)

    def addFriends(self, gen_reply, is_good):
        data = self.api.friends.getRequests(extended=1)
        to_rep = []
        self.loadUsers(data['items'], lambda x: x['user_id'], True)
        for i in data['items']:
            if self.users[i['user_id']].get('blacklisted'):
                self.api.friends.delete.delayed(user_id=i['user_id'])
                continue
            res = is_good(i['user_id'], True)
            if res is None:
                self.api.friends.add.delayed(user_id=i['user_id'])
                self.logSender('Adding %sender%', i)
                if 'message' in i:
                    ans = gen_reply(i)
                    to_rep.append((i, ans))
            else:
                self.api.friends.delete.delayed(user_id=i['user_id'])
                self.logSender('Not adding %sender% ({})'.format(res), i)
        for i in to_rep:
            self.replyMessage(i[0], i[1][0], i[1][1])
        self.api.sync()

    def unfollow(self):
        result = []
        requests = self.api.friends.getRequests(out=1)['items']
        suggested = self.api.friends.getRequests(suggested=1)['items']
        for i in requests:
            if i not in self.banned:
                result.append(i)
        for i in suggested:
            self.api.friends.delete.delayed(user_id=i)
        self.deleteFriend(result)
        return result

    def deleteFriend(self, uid):
        if type(uid) == int:
            self.api.friends.delete(user_id=uid)
        else:
            for i in uid:
                self.api.friends.delete.delayed(user_id=i)
            self.api.sync()

    def setOnline(self):
        self.api.account.setOnline()

    def getUserId(self, domain):
        domain = str(domain).lower().rstrip().rstrip('}').rstrip()
        conf = re.search('sel=c(\\d+)', domain) or re.search('^c(\\d+)$', domain) or re.search('chat=(\\d+)', domain) or re.search('peer=2(\\d{9})', domain)
        if conf is not None:
            return int(conf.group(1)) + CONF_START
        if '=' in domain:
            domain = domain.split('=')[-1]
        if '/' in domain:
            domain = domain.split('/')[-1]
        data = self.api.users.get(user_ids=domain)
        if not data:
            return None
        return data[0]['id']

    def deleteComment(self, rep):
        if rep['type'].endswith('photo'):
            self.api.photos.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])
        elif rep['type'].endswith('video'):
            self.api.video.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])
        else:
            self.api.wall.deleteComment(owner_id=self.self_id, comment_id=rep['feedback']['id'])

    def filterComments(self, test):
        data = self.api.notifications.get(start_time=self.last_viewed_comment + 1, count=100)['items']
        to_del = set()
        to_bl = set()
        self.loadUsers(data, lambda x: x['feedback']['from_id'], True)
        for rep in data:
            if rep['date'] != 'i':
                self.last_viewed_comment = max(self.last_viewed_comment, int(rep['date']))
                stats.update('last_comment', self.last_viewed_comment)

            def _check(s):
                if 'photo' in s:
                    return s['photo']['owner_id'] == self.self_id
                if 'video' in s:
                    return s['video']['owner_id'] == self.self_id
                if 'post' in s:
                    return s['post']['to_id'] == self.self_id

            if rep['type'].startswith('comment_') or rep['type'].startswith('reply_comment') and _check(rep['parent']):
                txt = html.escape(rep['feedback']['text'])
                res = 'good'
                if self.users[rep['feedback']['from_id']]['blacklisted']:
                    res = 'blacklisted'
                    log.write('comments', self.loggableName(rep['feedback']['from_id']) + ' (blacklisted): ' + txt)
                    self.deleteComment(rep)
                    to_bl.add(rep['feedback']['from_id'])
                elif test(txt):
                    res = 'bad'
                    log.write('comments', self.loggableName(rep['feedback']['from_id']) + ': ' + txt)
                    self.deleteComment(rep)
                    to_del.add(rep['feedback']['from_id'])
                elif 'attachments' in rep['feedback'] and any(i.get('type') in ['video', 'link'] for i in rep['feedback']['attachments']):
                    res = 'attachment'
                    log.write('comments', self.loggableName(rep['feedback']['from_id']) + ' (attachment)')
                    self.deleteComment(rep)
                self.logSender('Comment {} (by %sender%) - {}'.format(txt, res), {'user_id': rep['feedback']['from_id']})
        for i in to_bl:
            self.blacklist(i)
        return to_del

    def likeAva(self, uid):
        del self.users[uid]
        if 'crop_photo' not in self.users[uid]:
            return
        photo = self.users[uid]['crop_photo']['photo']
        self.api.likes.add(type='photo', owner_id=photo['owner_id'], item_id=photo['id'])
        self.logSender('Liked %sender%', {'user_id': uid})

    def setRelation(self, uid):
        self.api.account.saveProfileInfo(relation_partner_id=uid)
        self.vars['bf'] = self.users[uid]
        log.write('relation', self.loggableName(uid))
        self.logSender('Set relationship with %sender%', {'user_id': uid})

    def waitAllThreads(self):
        for t in self.tm.all():
            t.join(60)

    # {name} - first_name last_name
    # {id} - id
    def printableName(self, pid, user_fmt, conf_fmt='Conf "{name}"'):
        if pid > CONF_START:
            return conf_fmt.format(id=(pid - CONF_START), name=self.confs[pid - CONF_START]['title'])
        else:
            return user_fmt.format(id=pid, name=self.users[pid]['first_name'] + ' ' + self.users[pid]['last_name'])

    def logSender(self, text, message):
        text_msg = text.replace('%sender%', self.printableSender(message, False))
        html_msg = text.replace('%sender%', self.printableSender(message, True))
        logging.info(text_msg, extra={'db': html_msg})

    def printableSender(self, message, need_html):
        if message.get('chat_id', 0) > 0:
            if need_html:
                return self.printableName(message['user_id'], user_fmt='Conf "%c" (%i), <a href="https://vk.com/id{id}" target="_blank">{name}</a>').replace('%i', str(
                    message['chat_id'])).replace('%c', html.escape(self.confs[message['chat_id']]['title']))
            else:
                return self.printableName(message['user_id'], user_fmt='Conf "%c" (%i), {name}').replace('%i', str(message['chat_id'])).replace('%c', html.escape(
                    self.confs[message['chat_id']]['title']))
        else:
            if need_html:
                return self.printableName(message['user_id'], user_fmt='<a href="https://vk.com/id{id}" target="_blank">{name}</a>')
            else:
                return self.printableName(message['user_id'], user_fmt='{name}')

    def loggableName(self, uid):
        return self.printableName(uid, '{id} ({name})')

    def blacklist(self, uid):
        self.api.account.banUser(user_id=uid)

    def blacklistedCount(self):
        return self.api.account.getBanned(count=0)['count']

    def lastDialogs(self):

        def cb(req, resp):
            d.append((req['peer_id'], resp['count']))

        dialogs = self.api.messages.getDialogs(count=self.stats_dialog_count, preview_length=1)
        d = []
        confs = {}
        try:
            items = list(dialogs['items'])
            for i in items:
                self.api.messages.getHistory.delayed(peer_id=self.getSender(i['message']), count=0).callback(cb)
                if 'title' in i['message']:
                    confs[self.getSender(i['message'])] = i['message']['title']
            self.api.sync()
        except TypeError:
            logging.warning('Unable to fetch dialogs')
            return (None, None, None)
        return (dialogs['count'], d, confs)

    def acceptGroupInvites(self):
        for i in self.api.groups.getInvites()['items']:
            logging.info('Joining group "{}"'.format(i['name']))
            self.api.groups.join(group_id=i['id'])
            log.write('groups', '{}: <a target="_blank" href="https://vk.com/club{}">{}</a>{}'.format(self.loggableName(i['invited_by']), i['id'], i['name'],
                                                                                                      ['', ' (closed)', ' (private)'][i['is_closed']]))
    def test_ior_intercept_verify_data(self):
        """Jira ID: DAOS-3502.

        Test Description:
            Purpose of this test is to run ior through dfuse with
            interception library on 5 clients and without interception
            library on 1 client for at least 30 minutes and verify the
            data integrity using ior's Read Verify and Write Verify
            options.

        Use case:
            Run ior with read, write, fpp, read verify
            write verify for 30 minutes
            Run ior with read, write, read verify
            write verify for 30 minutes

        :avocado: tags=all,full_regression
        :avocado: tags=hw,large
        :avocado: tags=daosio,dfuse,il,ior_intercept
        :avocado: tags=ior_intercept_verify_data
        """
        self.add_pool()
        self.add_container(self.pool)

        # Start dfuse for POSIX api. This is specific to interception library test requirements.
        self.start_dfuse(self.hostlist_clients, self.pool, self.container)

        # Setup the thread manager
        thread_manager = ThreadManager(run_ior, self.timeout - 30)
        index_clients_intercept_file = [
            (0, self.hostlist_clients[0:-1],
             os.path.join(self.prefix, 'lib64', 'libioil.so'),
             os.path.join(self.dfuse.mount_dir.value, "testfile_0_intercept")),
            (1, self.hostlist_clients[-1:], None,
             os.path.join(self.dfuse.mount_dir.value, "testfile_1")),
        ]
        self.job_manager = []
        for index, clients, intercept, test_file in index_clients_intercept_file:
            # Add a job manager for each ior command. Use a timeout for the ior command that leaves
            # enough time to report the summary of all the threads
            job_manager = get_job_manager(self, "Mpirun", None, False, "mpich",
                                          self.get_remaining_time() - 30)

            # Define the parameters that will be used to run an ior command in this thread
            thread_manager.add(
                test=self,
                manager=job_manager,
                log=self.client_log,
                hosts=clients,
                path=self.workdir,
                slots=None,
                group=self.server_group,
                pool=self.pool,
                container=self.container,
                processes=(self.processes // len(self.hostlist_clients)) *
                len(clients),
                intercept=intercept,
                ior_params={"test_file": test_file})
            self.log.info("Created thread %s for %s with intercept: %s", index,
                          clients, str(intercept))

        # Launch the IOR threads
        self.log.info("Launching %d IOR threads", thread_manager.qty)
        results = thread_manager.run()

        # Stop dfuse
        self.stop_dfuse()

        # Check the ior thread results
        failed_thread_count = thread_manager.check(results)
        if failed_thread_count > 0:
            msg = "{} FAILED IOR Thread(s)".format(failed_thread_count)
            self.d_log.error(msg)
            self.fail(msg)

        for index, clients, intercept, _ in index_clients_intercept_file:
            with_intercept = "without" if intercept is None else "with"
            IorCommand.log_metrics(
                self.log, "{} clients {} interception library".format(
                    len(clients), with_intercept),
                IorCommand.get_ior_metrics(results[index].result))
Example #37
0
class DebuggerDomain(HandlerDomain):
    '''Implement Chrome debugger domain protocol and
    convert into lldb python API.
    '''
    def __init__(self, runtimeDomain, fileManager, remoteObjectManager,
                 basepath='.', **kwargs):
        HandlerDomain.__init__(self, **kwargs)
        self.runtimeDomain = runtimeDomain
        self.fileManager = fileManager
        self.remoteObjectManager = remoteObjectManager
        self.locationSerializer = serialize.LocationSerializer(
            fileManager, basepath)
        self.moduleSourcePathUpdater = ModuleSourcePathUpdater(
            self.debugger.GetSelectedTarget(), fileManager, basepath)
        self.thread_manager = ThreadManager(self.socket, self.locationSerializer, self.remoteObjectManager)

    @property
    def name(self):
        return 'Debugger'

    @handler()
    def canSetScriptSource(self, params):
        # Return False, becuase we don't support
        # changing source at runtime.
        return {"result": False}

    @handler()
    def continueToLocation(self, params):
        # TODO(williamsc) - This is probably setting a one off breakpoint and continuing.
        raise UndefinedHandlerError('continueToLocation not implemented')

    @handler()
    def disable(self, params):
        # Not exactly the same as disable. Detach() might be closer to
        # what Chrome Dev Tools is trying to do.
        self.debugger.GetSelectedTarget().DisableAllBreakpoints()
        return {}

    @handler()
    def enable(self, params):
        process = self.debugger.GetSelectedTarget().process
        self.event_thread = LLDBListenerThread(
            server=self.socket,
            location_serializer=self.locationSerializer,
            remote_object_manager=self.remoteObjectManager,
            module_source_path_updater=self.moduleSourcePathUpdater,
            thread_manager = self.thread_manager,
            process=process)
        self.moduleSourcePathUpdater.modules_updated()
        self.event_thread.start()
        return {}

    @handler()
    def evaluateOnCallFrame(self, params):
        frameId = params['callFrameId']

        thread, frame = frameId.split('.')
        # TODO: These return booleans to indicate success. Throw something if False.
        self.debugger.GetSelectedTarget().process.SetSelectedThreadByIndexID(int(thread))
        self.debugger.GetSelectedTarget().process.GetSelectedThread().SetSelectedFrame(int(frame))

        return self.runtimeDomain.evaluate(params)

    @handler()
    def getScriptSource(self, params):
        filelike = self.fileManager.get_by_script_id(params['scriptId'])
        if filelike:
            return {'scriptSource': filelike.script_source}
        else:
            return {'scriptSource': '<Failed to fetch source.>'}

    @handler()
    def pause(self, params):
        self.debugger.GetSelectedTarget().process.Stop()
        return {}

    @handler()
    def removeBreakpoint(self, params):
        self.debugger.GetSelectedTarget().BreakpointDelete(int(params['breakpointId']))
        return {}

    @handler()
    def resume(self, params):
        self.debugger.GetSelectedTarget().process.Continue()
        return {}

    @handler()
    def selectThread(self, params):
        threadId = params['threadId']
        self.debugger.GetSelectedTarget().process.SetSelectedThreadByID(threadId)
        return {}

    @handler()
    def getThreadStack(self, params):
        threadId = params['threadId']
        thread = self.debugger.GetSelectedTarget().process.GetThreadByID(threadId)
        params = { "callFrames": [] }
        if not thread == None:
            params["callFrames"] = self.thread_manager.get_thread_stack(thread)
        return params

    @handler()
    def searchInContent(self, params):
        raise UndefinedHandlerError('searchInContent not implemented')

    @handler()
    def setBreakpoint(self, params):
        filelike = self.fileManager.get_by_script_id(params['location']['scriptId'])
        if not filelike or not isinstance(filelike, file_manager.File):
            # Only support setting breakpoints in real files.
            return {}
        return self._set_breakpoint_by_filespec(
            filelike.server_obj,
            int(params['location']['lineNumber']) + 1)

    @handler()
    def setBreakpointByUrl(self, params):
        filelike = self.fileManager.get_by_client_url(params['url'])
        if not filelike or not isinstance(filelike, file_manager.File):
            raise RuntimeError('Cannot find file for breakpoint.')
        return self._set_breakpoint_by_filespec(
            filelike.server_obj,
            int(params['lineNumber']) + 1)

    @handler()
    def setBreakpointsActive(self, params):
        if params['active']:
            self.debugger.GetSelectedTarget().EnableAllBreakpoints()
        else:
            self.debugger.GetSelectedTarget().DisableAllBreakpoints()
        return {}

    @handler()
    def setPauseOnExceptions(self, params):
        # TODO(williamsc) - Support add support for pausing on exceptions
        raise UndefinedHandlerError('setPauseOnExceptions not implemented')

    @handler()
    def setScriptSource(self, params):
        raise UndefinedHandlerError('setScriptSource not supported for LLDB')

    @handler()
    def stepInto(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread().StepInto()
        return {}

    @handler()
    def stepOut(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread().StepOut()
        return {}

    @handler()
    def stepOver(self, params):
        self.debugger.GetSelectedTarget().GetProcess().GetSelectedThread().StepOver()
        return {}

    def _set_breakpoint_by_filespec(self, filespec, line):
        breakpoint = self.debugger.GetSelectedTarget().BreakpointCreateByLocation(filespec, line)
        return {
            'breakpointId': str(breakpoint.id),
            'locations':
                self.locationSerializer.get_breakpoint_locations(breakpoint),
        }
    def __init__(self):
        """Program entry point"""
        op = argparse.ArgumentParser()
        op.add_argument("-c", "--config", dest="cfile",
                default="/etc/mysql-statsd.conf",
                help="Configuration file"
        )
        op.add_argument("-d", "--debug", dest="debug",
                help="Prints statsd metrics next to sending them",
                default=False, action="store_true"
        )
        op.add_argument("--dry-run", dest="dry_run",
                default=False,
                action="store_true",
                help="Print the output that would be sent to statsd without actually sending data somewhere"
        )

        # TODO switch the default to True, and make it fork by default in init script.
        op.add_argument("-f", "--foreground", dest="foreground", help="Dont fork main program", default=False, action="store_true")

        opt = op.parse_args()
        self.get_config(opt.cfile)

        if not self.config:
            sys.exit(op.print_help())

        try:
            logfile = self.config.get('daemon').get('logfile', '/tmp/daemon.log')
        except AttributeError:
            logfile = sys.stdout
            pass

        if not opt.foreground:
            self.daemonize(stdin='/dev/null', stdout=logfile, stderr=logfile)

        # Set up queue
        self.queue = Queue.Queue()

        # split off config for each thread
        mysql_config = dict(mysql=self.config['mysql'])
        mysql_config['metrics'] = self.config['metrics']

        statsd_config = self.config['statsd']

        # Spawn MySQL polling thread
        mysql_thread = ThreadMySQL(queue=self.queue, **mysql_config)
        # t1 = ThreadMySQL(config=self.config, queue=self.queue)

        # Spawn Statsd flushing thread
        statsd_thread = ThreadStatsd(queue=self.queue, **statsd_config)

        if opt.dry_run:
            statsd_thread = ThreadFakeStatsd(queue=self.queue, **statsd_config)

        if opt.debug:
            """ All debug settings go here """
            statsd_thread.debug = True

        # Get thread manager
        tm = ThreadManager(threads=[mysql_thread, statsd_thread])

        try:
            tm.run()
        except:
            # Protects somewhat from needing to kill -9 if there is an exception
            # within the thread manager by asking for a quit an joining.
            try:
                tm.stop_threads()
            except:
                pass

            raise