def main(): config = load_config() metrics.init(config, 'iris-process-retention', stats_reset) retention_settings = config.get('retention', {}) if not retention_settings.get('enabled'): logger.info('Retention not enabled, bailing') return engine = create_engine(config['db']['conn']['str'] % config['db']['conn']['kwargs'], **config['db']['kwargs']) max_days = int(retention_settings['max_days']) if max_days < 1: logger.error('Max days needs to at least be 1') return cooldown_time = int(retention_settings['cooldown_time']) batch_size = int(retention_settings['batch_size']) run_interval = int(retention_settings['run_interval']) archive_path = retention_settings['archive_path'] spawn(metrics.emit_forever) while True: logger.info('Starting retention loop (kill messages+incidents older than %d days)', max_days) try: process_retention(engine, max_days=max_days, cooldown_time=cooldown_time, batch_size=batch_size, archive_path=archive_path) except Exception: logger.exception('Hit problem while running retention') logger.info('Waiting %d seconds until next iteration..', run_interval) sleep(run_interval)
def init_sender(config): db.init(config) cache.init(config) metrics.init(config, 'iris-sender', default_sender_metrics) api_cache.cache_priorities() api_cache.cache_applications() api_cache.cache_modes() global should_mock_gwatch_renewer, send_message if config['sender'].get('debug'): logger.info('DEBUG MODE') should_mock_gwatch_renewer = True should_skip_send = True else: should_skip_send = False should_mock_gwatch_renewer = should_mock_gwatch_renewer or config.get( 'skipgmailwatch', False) should_skip_send = should_skip_send or config.get('skipsend', False) if should_skip_send: config['vendors'] = [{ 'type': 'iris_dummy', 'name': 'iris dummy vendor' }] global quota quota = ApplicationQuota(db, cache.targets_for_role, config['sender'].get('sender_app'))
def main(): config = load_config() metrics.init(config, 'iris-sync-targets', stats_reset) default_nap_time = 3600 try: nap_time = int(config.get('sync_script_nap_time', default_nap_time)) except ValueError: nap_time = default_nap_time engine = create_engine( config['db']['conn']['str'] % config['db']['conn']['kwargs'], **config['db']['kwargs']) # Initialize these to zero at the start of the app, and don't reset them at every # metrics interval metrics.set('users_found', 0) metrics.set('teams_found', 0) metrics_task = spawn(metrics.emit_forever) while True: if not bool(metrics_task): logger.error('metrics task failed, %s', metrics_task.exception) metrics_task = spawn(metrics.emit_forever) sync(config, engine) logger.info('Sleeping for %d seconds' % nap_time) sleep(nap_time)
def main(): global ldap_timeout config = load_config() metrics.init(config, 'iris-sync-targets', stats_reset) default_ldap_timeout = 20 default_nap_time = 3600 ldap_timeout = int( config.get('sync_script_ldap_timeout', default_ldap_timeout)) try: nap_time = int(config.get('sync_script_nap_time', default_nap_time)) except ValueError: nap_time = default_nap_time engine = create_engine( config['db']['conn']['str'] % config['db']['conn']['kwargs'], **config['db']['kwargs']) # Optionally, maintain an internal list of mailing lists from ldap that can also be # used as targets. ldap_lists = config.get('ldap_lists') # Initialize these to zero at the start of the app, and don't reset them at every # metrics interval metrics.set('users_found', 0) metrics.set('teams_found', 0) metrics.set('ldap_lists_found', 0) metrics.set('ldap_memberships_found', 0) metrics_task = spawn(metrics.emit_forever) while True: if not bool(metrics_task): logger.error('metrics task failed, %s', metrics_task.exception) metrics_task = spawn(metrics.emit_forever) sync_from_oncall(config, engine) # Do ldap mailing list sync *after* we do the normal sync, to ensure we have the users # which will be in ldap already populated. if ldap_lists: if 'ldap_cert_path' in ldap_lists: ldap_cert_path = ldap_lists['ldap_cert_path'] if not os.access(ldap_cert_path, os.R_OK): logger.error("Failed to read ldap_cert_path certificate") raise IOError else: ldap_lists['cert_path'] = ldap_cert_path list_run_start = time.time() sync_ldap_lists(ldap_lists, engine) logger.info('Ldap mailing list sync took %.2f seconds', time.time() - list_run_start) logger.info('Sleeping for %d seconds' % nap_time) sleep(nap_time)
def main(): boot_time = time.time() config = load_config() metrics.init(config, 'iris-owa-sync', default_metrics) owaconfig = config.get('owa') if not owaconfig: logger.critical('Missing OWA configs') sys.exit(1) api_host = owaconfig.get('api_host', 'http://localhost:16649') iris_client = IrisClient(api_host, 0, owaconfig['iris_app'], owaconfig['iris_app_key']) proxies = owaconfig.get('proxies') # only way to configure a proxy is to monkey-patch (http adapter) a monkey-patch (baseprotocol) :/ if proxies: UseProxyHttpAdapter._my_proxies = proxies exchangelib.protocol.BaseProtocol.HTTP_ADAPTER_CLS = UseProxyHttpAdapter creds = exchangelib.Credentials(**owaconfig['credentials']) try: nap_time = int(owaconfig.get('sleep_interval', 60)) except ValueError: nap_time = 60 while True: start_time = time.time() message_count = 0 try: config = exchangelib.Configuration(credentials=creds, **owaconfig['config']) account = exchangelib.Account(config=config, access_type=exchangelib.DELEGATE, **owaconfig['account']) except (exchangelib.errors.EWSError, requests.exceptions.RequestException): logger.exception('Failed authenticating to OWA365') metrics.incr('owa_api_failure_count') else: logger.info('Receiving mail on behalf of %s', owaconfig['account'].get('primary_smtp_address')) message_count = poll(account, iris_client) now = time.time() run_time = now - start_time logger.info( 'Last run took %2.f seconds and processed %s messages. Waiting %s seconds until next poll..', run_time, message_count, nap_time) metrics.set('uptime', now - boot_time) metrics.emit() sleep(nap_time)
def main(): config = load_config() metrics.init(config, 'iris-application-stats', stats_reset) app_stats_settings = config.get('app-stats', {}) run_interval = int(app_stats_settings['run_interval']) spawn(metrics.emit_forever) db.init(config) while True: logger.info('Starting app stats calculation loop') stats_task() logger.info('Waiting %d seconds until next iteration..', run_interval) sleep(run_interval)
def init_sender(config): gevent.signal(signal.SIGINT, sender_shutdown) gevent.signal(signal.SIGTERM, sender_shutdown) gevent.signal(signal.SIGQUIT, sender_shutdown) api_host = config['sender'].get('api_host', 'http://localhost:16649') db.init(config) cache.init(api_host, config) metrics.init(config, 'iris-sender', default_sender_metrics) api_cache.cache_priorities() api_cache.cache_applications() api_cache.cache_modes() global should_mock_gwatch_renewer, send_message if config['sender'].get('debug'): logger.info('DEBUG MODE') should_mock_gwatch_renewer = True should_skip_send = True else: should_skip_send = False should_mock_gwatch_renewer = should_mock_gwatch_renewer or config.get( 'skipgmailwatch', False) should_skip_send = should_skip_send or config.get('skipsend', False) if should_skip_send: config['vendors'] = [{ 'type': 'iris_dummy', 'name': 'iris dummy vendor' }] global quota quota = ApplicationQuota(db, cache.targets_for_role, config['sender'].get('sender_app')) global coordinator zk_hosts = config['sender'].get('zookeeper_cluster', False) if zk_hosts: logger.info('Initializing coordinator with ZK: %s', zk_hosts) coordinator = Coordinator(zk_hosts=zk_hosts, hostname=socket.gethostname(), port=config['sender'].get('port', 2321), join_cluster=True) else: logger.info( 'ZK cluster info not specified. Using master status from config') coordinator = NonClusterCoordinator( is_master=config['sender'].get('is_master', True), slaves=config['sender'].get('slaves', []))
def main(): config = load_config() metrics.init(config, 'iris-owa-sync', default_metrics) owaconfig = config.get('owa') if not owaconfig: logger.critical('Missing OWA configs') sys.exit(1) api_host = owaconfig.get('api_host', 'http://localhost:16649') iris_client = IrisClient(api_host, 0, owaconfig['iris_app'], owaconfig['iris_app_key']) spawn(metrics.emit_forever) creds = Credentials(**owaconfig['credentials']) account = Account(primary_smtp_address=owaconfig['smtp_address'], credentials=creds, autodiscover=True, access_type=DELEGATE) logger.info('Receiving mail on behalf of %s', owaconfig['smtp_address']) try: nap_time = int(owaconfig.get('sleep_interval', 60)) except ValueError: nap_time = 60 while True: start_time = time.time() message_count = poll(account, iris_client) run_time = time.time() - start_time logger.info( 'Last run took %2.f seconds and processed %s messages. Waiting %s seconds until next poll..', run_time, message_count, nap_time) sleep(nap_time)
def main(): global ldap_timeout global ldap_pagination_size global update_sleep config = load_config() metrics.init(config, 'iris-sync-targets', stats_reset) default_ldap_timeout = 60 default_ldap_pagination_size = 400 default_update_sleep = 0 default_ldap_nap_time = 3600 default_oncall_nap_time = 60 ldap_timeout = int(config.get('sync_script_ldap_timeout', default_ldap_timeout)) ldap_pagination_size = int(config.get('sync_script_ldap_pagination_size', default_ldap_pagination_size)) update_sleep = float(config.get('target_update_pause', default_update_sleep)) try: ldap_nap_time = int(config.get('sync_script_ldap_nap_time', default_ldap_nap_time)) oncall_nap_time = int(config.get('sync_script_oncall_nap_time', default_oncall_nap_time)) except ValueError: ldap_nap_time = default_ldap_nap_time oncall_nap_time = default_oncall_nap_time # check if we are using special connection settings for this script if config.get('db_target_sync'): engine = create_engine(config['db_target_sync']['conn']['str'] % config['db_target_sync']['conn']['kwargs'], **config['db_target_sync']['kwargs']) else: engine = create_engine(config['db']['conn']['str'] % config['db']['conn']['kwargs'], **config['db']['kwargs']) # Optionally, maintain an internal list of mailing lists from ldap that can also be # used as targets. ldap_lists = config.get('ldap_lists') # Initialize these to zero at the start of the app, and don't reset them at every # metrics interval metrics.set('users_found', 0) metrics.set('teams_found', 0) metrics.set('ldap_lists_found', 0) metrics.set('ldap_memberships_found', 0) metrics_task = spawn(metrics.emit_forever) oncall_task = spawn(oncall_sync_loop, config, engine, oncall_nap_time) if ldap_lists: if 'ldap_cert_path' in ldap_lists: ldap_cert_path = ldap_lists['ldap_cert_path'] if not os.access(ldap_cert_path, os.R_OK): logger.error("Failed to read ldap_cert_path certificate") raise IOError else: ldap_lists['cert_path'] = ldap_cert_path ldap_task = spawn(ldap_sync_loop, ldap_lists, engine, ldap_nap_time) while True: if not bool(metrics_task): metrics.incr('failed_tasks') logger.error('metrics task failed, %s', metrics_task.exception) spawn(metrics.emit_forever) if not bool(oncall_task): metrics.incr('failed_tasks') logger.error('oncall task failed, %s', oncall_task.exception) metrics_task = spawn(oncall_sync_loop, config, engine, oncall_nap_time) if ldap_lists: if not bool(ldap_task): metrics.incr('failed_tasks') logger.error('ldap task failed, %s', ldap_task.exception) ldap_task = spawn(ldap_sync_loop, ldap_lists, engine, ldap_nap_time) sleep(10)