def create_consumer_producer_pair(endpoint, writer, admin, child_account=None): """ Create a pair of Producer-Consumer objects for each endpoint and return a list containing the asyncio tasks for running those objects. @param endpoint Log type to create producer/consumer pair for @param writer Object for writing logs to a server @param admin Object from which to get the correct API endpoints @param child_account If present, this is being used by MSP and pass appropriate account id @return list of asyncio tasks for running the Producer and Consumer objects """ # The format a log should have before being consumed and sent log_format = Config.get_log_format() log_queue = asyncio.Queue() producer = consumer = None # Create the right pair of Producer-Consumer objects based on endpoint if endpoint == Config.AUTH: if Config.account_is_msp(): producer = AuthlogProducer(admin.json_api_call, log_queue, child_account_id=child_account, url_path="/admin/v2/logs/authentication") else: producer = AuthlogProducer(admin.get_authentication_log, log_queue) consumer = AuthlogConsumer(log_format, log_queue, writer, child_account) elif endpoint == Config.TELEPHONY: if Config.account_is_msp(): producer = TelephonyProducer(admin.json_api_call, log_queue, child_account_id=child_account, url_path='/admin/v1/logs/telephony') else: producer = TelephonyProducer(admin.get_telephony_log, log_queue) consumer = TelephonyConsumer(log_format, log_queue, writer, child_account) elif endpoint == Config.ADMIN: if Config.account_is_msp(): producer = AdminactionProducer(admin.json_api_call, log_queue, child_account_id=child_account, url_path='/admin/v1/logs/administrator') else: producer = AdminactionProducer(admin.get_administrator_log, log_queue) consumer = AdminactionConsumer(log_format, log_queue, writer, child_account) else: Program.log(f"{endpoint} is not a recognized endpoint", logging.WARNING) del log_queue return [] tasks = [asyncio.ensure_future(producer.produce()), asyncio.ensure_future(consumer.consume())] return tasks
def create_tasks(server_to_writer): """ Create a pair of Producer-Consumer objects for each endpoint enabled within the account defined in config, or retrieve child accounts and do the same if the account is MSP. Return a list containing the asyncio tasks for running those objects. @param writer Dictionary mapping server ids to writer objects @return list of asyncio tasks for running the Producer and Consumer objects """ tasks = [] # Object with functions needed to utilize log API calls admin = create_admin(Config.get_account_ikey(), Config.get_account_skey(), Config.get_account_hostname(), is_msp=Config.account_is_msp(), proxy_server=Config.get_proxy_server(), proxy_port=Config.get_proxy_port()) # This is where functionality would be added to check if an account is MSP # (Config.account_is_msp), and then retrieve child accounts (ignoring those # in a blocklist) if the account is indeed MSP # TODO: Implement blocklist if Config.account_is_msp(): child_account = admin.get_child_accounts() child_accounts_id = [ account['account_id'] for account in child_account ] for account in child_accounts_id: # TODO: This can be made into a separate function for mapping in Config.get_account_endpoint_server_mappings(): # Get the writer to be used for this set of endpoints writer = server_to_writer[mapping.get('server')] for endpoint in mapping.get('endpoints'): new_tasks = create_consumer_producer_pair( endpoint, writer, admin, account) tasks.extend(new_tasks) else: for mapping in Config.get_account_endpoint_server_mappings(): # Get the writer to be used for this set of endpoints writer = server_to_writer[mapping.get('server')] for endpoint in mapping.get('endpoints'): new_tasks = create_consumer_producer_pair( endpoint, writer, admin) tasks.extend(new_tasks) return tasks
def test_account_is_msp(self): config = {'account': {'is_msp': False}} Config.set_config(config) is_msp = Config.account_is_msp() self.assertEqual(is_msp, False)
async def call_log_api(self): """ Make a call to a log-specific API and return the API result. The default implementation given here will not suffice for every type of log API and so should be overriden by a child clas when necessary. @return the result of the API call """ if Config.account_is_msp(): # Make an API call to retrieve authlog logs for MSP accounts parameters = {"mintime": six.ensure_str(str(self.log_offset)), "account_id": six.ensure_str(self.account_id)} api_result = await run_in_executor( functools.partial( self.api_call, method="GET", path=self.url_path, params=parameters ) ) else: api_result = await run_in_executor( functools.partial( self.api_call, mintime=self.log_offset ) ) return api_result
def main(): """ Kicks off DuoLogSync by setting important variables, creating and running a Producer-Consumer pair for each log-type defined in a config file passed to the program. """ arg_parser = argparse.ArgumentParser(prog='duologsync', description="Path to config file") arg_parser.add_argument('ConfigPath', metavar='config-path', type=str, help='Config to start application') args = arg_parser.parse_args() # Handle shutting down the program via Ctrl-C signal.signal(signal.SIGINT, sigint_handler) # Create a config Dictionary from a YAML file located at args.ConfigPath config = Config.create_config(args.ConfigPath) Config.set_config(config) # Do extra checks for Trust Monitor support is_dtm_in_config = check_for_specific_endpoint('trustmonitor', config) log_format = Config.get_log_format() is_msp = Config.account_is_msp() if (is_dtm_in_config and log_format != 'JSON'): Program.log(f"DuoLogSync: Trust Monitor endpoint only supports JSON", logging.WARNING) return if (is_dtm_in_config and is_msp): Program.log( f"DuoLogSync: Trust Monitor endpoint only supports non-msp", logging.WARNING) return Program.setup_logging(Config.get_log_filepath()) # Dict of writers (server id: writer) to be used for consumer tasks server_to_writer = Writer.create_writers(Config.get_servers()) # List of Producer/Consumer objects as asyncio tasks to be run tasks = create_tasks(server_to_writer) # Run the Producers and Consumers asyncio.get_event_loop().run_until_complete(asyncio.gather(*tasks)) asyncio.get_event_loop().close() if Program.is_logging_set(): print(f"DuoLogSync: shutdown successfully. Check " f"{Config.get_log_filepath()} for program logs")
async def call_log_api(self): """ Make a call to the authentication log endpoint and return the result of that API call @return the result of a call to the authentication log API endpoint """ if Config.account_is_msp(): # In case of recovering from checkpoint, self.mintime is None since its not init # anywhere. When duo_client is directly used, client will initialize self.mintime to # time.time() - 86400 (1 day in past). We will have to do similar thing when directly # calling logs endpoint for MSP accounts. Mintime is never used when next offset is # present if not self.mintime: self.mintime = (int(time.time()) - 86400) * 1000 # Make an API call to retrieve authlog logs for MSP accounts parameters = normalize_params({ "mintime": six.ensure_str(str(self.mintime)), "maxtime": six.ensure_str(str(int(time.time()) * 1000)), "limit": six.ensure_str('1000'), "account_id": six.ensure_str(self.account_id), "sort": six.ensure_str('ts:asc') }) if self.log_offset is not None: parameters["next_offset"] = self.log_offset authlog_api_result = await run_in_executor( functools.partial(self.api_call, method="GET", path=self.url_path, params=parameters)) else: # Make an API call to retrieve authlog logs authlog_api_result = await run_in_executor( functools.partial(self.api_call, api_version=2, mintime=self.mintime, next_offset=self.log_offset, sort='ts:asc', limit='1000')) return authlog_api_result