def __init__(self, cpu_id, total_cpus, poll_interval=1): self.keep_running = True self.host = platform.node() self.cpu_id = cpu_id self.total_cpus = total_cpus self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval
def __init__(self, process_identifier, cpu_id, poll_interval=10): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone)
def __init__(self, cpu_id, total_cpus, poll_interval=10): self.keep_running = True self.host = platform.node() self.cpu_id = cpu_id self.total_cpus = total_cpus self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.sync_hosts_for_shards = {} for database in config['DATABASE_HOSTS']: for shard in database['SHARDS']: # If no sync hosts are explicitly configured for the shard, # then try to steal from it. That way if you turn up a new # shard without properly allocating sync hosts to it, accounts # on it will still be started. self.sync_hosts_for_shards[shard['ID']] = shard.get( 'SYNC_HOSTS') or [self.host]
def __init__(self, process_identifier, cpu_id, poll_interval=10): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone)
def __init__(self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES) self.last_unloaded_account = time.time() # Fill the queue with initial values. null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)] for i in range(NUM_CPU_SAMPLES): self.rolling_cpu_counts.append(null_cpu_values)
def __init__( self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL, exit_after_min=None, exit_after_max=None, ): self.keep_running = True self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, "SYNC_MONITOR_CLS") } for p_name, _ in iteritems(providers): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info("starting mail sync process", supported_providers=list(module_registry)) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} # Randomize the poll_interval so we maintain at least a little fairness # when using a timeout while blocking on the redis queues. min_poll_interval = 5 self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval) self.semaphore = BoundedSemaphore(1) self.zone = config.get("ZONE") # Note that we don't partition by zone for the private queues. # There's not really a reason to since there's one queue per machine # anyways. Also, if you really want to send an Account to a mailsync # machine in another zone you can do so. self.private_queue = EventQueue( SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)) self.queue_group = EventQueueGroup([ shared_sync_event_queue_for_zone(self.zone), self.private_queue, ]) self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True) self._pending_avgs_provider = None self.last_unloaded_account = time.time() if exit_after_min and exit_after_max: exit_after = random.randint(exit_after_min * 60, exit_after_max * 60) self.log.info("exit after", seconds=exit_after) gevent.spawn_later(exit_after, self.stop)
def __init__(self, process_identifier, cpu_id, poll_interval=10): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) # We call cpu_percent in a non-blocking way. Because of the way # this function works, it'll always return 0.0 the first time # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent # for more details. psutil.cpu_percent(percpu=True)
def __init__(self, process_identifier, cpu_id, poll_interval=10): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) # We call cpu_percent in a non-blocking way. Because of the way # this function works, it'll always return 0.0 the first time # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent # for more details. psutil.cpu_percent(percpu=True)
def __init__(self): self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} self.log = get_logger() # { account_id: MailSyncMonitor() } self.monitors = dict() self.contact_sync_monitors = dict() # Restart existing active syncs. # (Later we will want to partition these across different machines!) with session_scope() as db_session: # XXX: I think we can do some sqlalchemy magic to make it so we # can query on the attribute sync_enabled. for account_id, in db_session.query(Account.id)\ .filter(~Account.sync_host.is_(None)): self.start_sync(account_id)
def __init__(self, poll_interval=1): self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} self.log = get_logger() self.monitors = {} self.contact_sync_monitors = {} self.poll_interval = poll_interval with session_scope() as db_session: # Restart existing active syncs. # (Later we'll want to partition these across different machines) for account_id, in db_session.query(Account.id).filter( ~Account.sync_host.is_(None)): self.start_sync(account_id) # In a separate greenlet, check for new accounts that are registered. gevent.spawn(self._new_account_listener)
def __init__(self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} # Randomize the poll_interval so we maintain at least a little fairness # when using a timeout while blocking on the redis queues. min_poll_interval = 5 self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval) self.semaphore = BoundedSemaphore(1) self.zone = config.get('ZONE') # Note that we don't partition by zone for the private queues. # There's not really a reason to since there's one queue per machine # anyways. Also, if you really want to send an Account to a mailsync # machine in another zone you can do so. self.private_queue = EventQueue(SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)) self.queue_group = EventQueueGroup([ shared_sync_event_queue_for_zone(self.zone), self.private_queue, ]) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self._pending_avgs_provider = None self.last_unloaded_account = time.time()
def __init__(self, cpu_id, total_cpus, poll_interval=1): self.cpu_id = cpu_id self.total_cpus = total_cpus self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval
def __init__(self, process_identifier, cpu_id, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES) # Fill the queue with initial values. Because of the way # cpu_percent works, it'll always return 0.0 the first time # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent # for more details. null_cpu_values = psutil.cpu_percent(percpu=True) for i in range(NUM_CPU_SAMPLES): self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES) self.last_unloaded_account = time.time() # Fill the queue with initial values. null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)] for i in range(NUM_CPU_SAMPLES): self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self, process_identifier, cpu_id, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.cpu_id = cpu_id self.process_identifier = process_identifier self.monitor_cls_for = {mod.PROVIDER: getattr( mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS')} for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval self.semaphore = BoundedSemaphore(1) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self.zone = config.get('ZONE') self.queue_client = QueueClient(self.zone) self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES) # Fill the queue with initial values. Because of the way # cpu_percent works, it'll always return 0.0 the first time # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent # for more details. null_cpu_values = psutil.cpu_percent(percpu=True) for i in range(NUM_CPU_SAMPLES): self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self, cpu_id, total_cpus, poll_interval=1): self.keep_running = True self.host = platform.node() self.cpu_id = cpu_id self.total_cpus = total_cpus self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, "SYNC_MONITOR_CLS") } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(cpu_id=cpu_id) self.log.info("starting mail sync process", supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} self.poll_interval = poll_interval