def create_consumer_producer_pair(endpoint, writer, admin, child_account=None):
"""
Create a pair of Producer-Consumer objects for each endpoint and return a
list containing the asyncio tasks for running those objects.
@param endpoint Log type to create producer/consumer pair for
@param writer Object for writing logs to a server
@param admin Object from which to get the correct API endpoints
@param child_account If present, this is being used by MSP and pass appropriate account id
@return list of asyncio tasks for running the Producer and Consumer objects
"""
# The format a log should have before being consumed and sent
log_format = Config.get_log_format()
log_queue = asyncio.Queue()
producer = consumer = None
# Create the right pair of Producer-Consumer objects based on endpoint
if endpoint == Config.AUTH:
if Config.account_is_msp():
producer = AuthlogProducer(admin.json_api_call, log_queue,
child_account_id=child_account,
url_path="/admin/v2/logs/authentication")
else:
producer = AuthlogProducer(admin.get_authentication_log, log_queue)
consumer = AuthlogConsumer(log_format, log_queue, writer, child_account)
elif endpoint == Config.TELEPHONY:
if Config.account_is_msp():
producer = TelephonyProducer(admin.json_api_call, log_queue,
child_account_id=child_account,
url_path='/admin/v1/logs/telephony')
else:
producer = TelephonyProducer(admin.get_telephony_log, log_queue)
consumer = TelephonyConsumer(log_format, log_queue, writer, child_account)
elif endpoint == Config.ADMIN:
if Config.account_is_msp():
producer = AdminactionProducer(admin.json_api_call, log_queue,
child_account_id=child_account,
url_path='/admin/v1/logs/administrator')
else:
producer = AdminactionProducer(admin.get_administrator_log, log_queue)
consumer = AdminactionConsumer(log_format, log_queue, writer, child_account)
else:
Program.log(f"{endpoint} is not a recognized endpoint", logging.WARNING)
del log_queue
return []
tasks = [asyncio.ensure_future(producer.produce()),
asyncio.ensure_future(consumer.consume())]
return tasks
def create_tasks(server_to_writer):
"""
Create a pair of Producer-Consumer objects for each endpoint enabled within
the account defined in config, or retrieve child accounts and do the same
if the account is MSP. Return a list containing the asyncio tasks for
running those objects.
@param writer Dictionary mapping server ids to writer objects
@return list of asyncio tasks for running the Producer and Consumer objects
"""
tasks = []
# Object with functions needed to utilize log API calls
admin = create_admin(Config.get_account_ikey(),
Config.get_account_skey(),
Config.get_account_hostname(),
is_msp=Config.account_is_msp(),
proxy_server=Config.get_proxy_server(),
proxy_port=Config.get_proxy_port())
# This is where functionality would be added to check if an account is MSP
# (Config.account_is_msp), and then retrieve child accounts (ignoring those
# in a blocklist) if the account is indeed MSP
# TODO: Implement blocklist
if Config.account_is_msp():
child_account = admin.get_child_accounts()
child_accounts_id = [
account['account_id'] for account in child_account
]
for account in child_accounts_id:
# TODO: This can be made into a separate function
for mapping in Config.get_account_endpoint_server_mappings():
# Get the writer to be used for this set of endpoints
writer = server_to_writer[mapping.get('server')]
for endpoint in mapping.get('endpoints'):
new_tasks = create_consumer_producer_pair(
endpoint, writer, admin, account)
tasks.extend(new_tasks)
else:
for mapping in Config.get_account_endpoint_server_mappings():
# Get the writer to be used for this set of endpoints
writer = server_to_writer[mapping.get('server')]
for endpoint in mapping.get('endpoints'):
new_tasks = create_consumer_producer_pair(
endpoint, writer, admin)
tasks.extend(new_tasks)
return tasks
def test_account_is_msp(self):
config = {'account': {'is_msp': False}}
Config.set_config(config)
is_msp = Config.account_is_msp()
self.assertEqual(is_msp, False)
async def call_log_api(self):
"""
Make a call to a log-specific API and return the API result. The default
implementation given here will not suffice for every type of log API and
so should be overriden by a child clas when necessary.
@return the result of the API call
"""
if Config.account_is_msp():
# Make an API call to retrieve authlog logs for MSP accounts
parameters = {"mintime": six.ensure_str(str(self.log_offset)),
"account_id": six.ensure_str(self.account_id)}
api_result = await run_in_executor(
functools.partial(
self.api_call,
method="GET",
path=self.url_path,
params=parameters
)
)
else:
api_result = await run_in_executor(
functools.partial(
self.api_call,
mintime=self.log_offset
)
)
return api_result
def main():
"""
Kicks off DuoLogSync by setting important variables, creating and running
a Producer-Consumer pair for each log-type defined in a config file passed
to the program.
"""
arg_parser = argparse.ArgumentParser(prog='duologsync',
description="Path to config file")
arg_parser.add_argument('ConfigPath',
metavar='config-path',
type=str,
help='Config to start application')
args = arg_parser.parse_args()
# Handle shutting down the program via Ctrl-C
signal.signal(signal.SIGINT, sigint_handler)
# Create a config Dictionary from a YAML file located at args.ConfigPath
config = Config.create_config(args.ConfigPath)
Config.set_config(config)
# Do extra checks for Trust Monitor support
is_dtm_in_config = check_for_specific_endpoint('trustmonitor', config)
log_format = Config.get_log_format()
is_msp = Config.account_is_msp()
if (is_dtm_in_config and log_format != 'JSON'):
Program.log(f"DuoLogSync: Trust Monitor endpoint only supports JSON",
logging.WARNING)
return
if (is_dtm_in_config and is_msp):
Program.log(
f"DuoLogSync: Trust Monitor endpoint only supports non-msp",
logging.WARNING)
return
Program.setup_logging(Config.get_log_filepath())
# Dict of writers (server id: writer) to be used for consumer tasks
server_to_writer = Writer.create_writers(Config.get_servers())
# List of Producer/Consumer objects as asyncio tasks to be run
tasks = create_tasks(server_to_writer)
# Run the Producers and Consumers
asyncio.get_event_loop().run_until_complete(asyncio.gather(*tasks))
asyncio.get_event_loop().close()
if Program.is_logging_set():
print(f"DuoLogSync: shutdown successfully. Check "
f"{Config.get_log_filepath()} for program logs")
async def call_log_api(self):
"""
Make a call to the authentication log endpoint and return the result of
that API call
@return the result of a call to the authentication log API endpoint
"""
if Config.account_is_msp():
# In case of recovering from checkpoint, self.mintime is None since its not init
# anywhere. When duo_client is directly used, client will initialize self.mintime to
# time.time() - 86400 (1 day in past). We will have to do similar thing when directly
# calling logs endpoint for MSP accounts. Mintime is never used when next offset is
# present
if not self.mintime:
self.mintime = (int(time.time()) - 86400) * 1000
# Make an API call to retrieve authlog logs for MSP accounts
parameters = normalize_params({
"mintime":
six.ensure_str(str(self.mintime)),
"maxtime":
six.ensure_str(str(int(time.time()) * 1000)),
"limit":
six.ensure_str('1000'),
"account_id":
six.ensure_str(self.account_id),
"sort":
six.ensure_str('ts:asc')
})
if self.log_offset is not None:
parameters["next_offset"] = self.log_offset
authlog_api_result = await run_in_executor(
functools.partial(self.api_call,
method="GET",
path=self.url_path,
params=parameters))
else:
# Make an API call to retrieve authlog logs
authlog_api_result = await run_in_executor(
functools.partial(self.api_call,
api_version=2,
mintime=self.mintime,
next_offset=self.log_offset,
sort='ts:asc',
limit='1000'))
return authlog_api_result
