Exemplo n.º 1
0
    def __init__(self, process_identifier, cpu_id, poll_interval=10):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)

        # We call cpu_percent in a non-blocking way. Because of the way
        # this function works, it'll always return 0.0 the first time
        # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
        # for more details.
        psutil.cpu_percent(percpu=True)
Exemplo n.º 2
0
    def __init__(self, process_identifier, cpu_id, poll_interval=10):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)

        # We call cpu_percent in a non-blocking way. Because of the way
        # this function works, it'll always return 0.0 the first time
        # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
        # for more details.
        psutil.cpu_percent(percpu=True)
Exemplo n.º 3
0
    def __init__(self, cpu_id, total_cpus, poll_interval=1):
        self.keep_running = True
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.total_cpus = total_cpus
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
Exemplo n.º 4
0
    def __init__(self, process_identifier, cpu_id, poll_interval=10):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
Exemplo n.º 5
0
    def __init__(self, cpu_id, total_cpus, poll_interval=10):
        self.keep_running = True
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.total_cpus = total_cpus
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.sync_hosts_for_shards = {}
        for database in config['DATABASE_HOSTS']:
            for shard in database['SHARDS']:
                # If no sync hosts are explicitly configured for the shard,
                # then try to steal from it. That way if you turn up a new
                # shard without properly allocating sync hosts to it, accounts
                # on it will still be started.
                self.sync_hosts_for_shards[shard['ID']] = shard.get(
                    'SYNC_HOSTS') or [self.host]
Exemplo n.º 6
0
    def __init__(self, process_identifier, process_number,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
        self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
        self.last_unloaded_account = time.time()

        # Fill the queue with initial values.
        null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)]
        for i in range(NUM_CPU_SAMPLES):
            self.rolling_cpu_counts.append(null_cpu_values)
Exemplo n.º 7
0
    def __init__(
        self,
        process_identifier,
        process_number,
        poll_interval=SYNC_POLL_INTERVAL,
        exit_after_min=None,
        exit_after_max=None,
    ):
        self.keep_running = True
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, "SYNC_MONITOR_CLS")
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info(
            "starting mail sync process", supported_providers=module_registry.keys()
        )

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int(
            (random.random() * (poll_interval - min_poll_interval)) + min_poll_interval
        )
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get("ZONE")

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(
            SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)
        )
        self.queue_group = EventQueueGroup(
            [shared_sync_event_queue_for_zone(self.zone), self.private_queue,]
        )

        self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()

        if exit_after_min and exit_after_max:
            exit_after = random.randint(exit_after_min * 60, exit_after_max * 60)
            self.log.info("exit after", seconds=exit_after)
            gevent.spawn_later(exit_after, self.stop)
Exemplo n.º 8
0
    def __init__(self, process_identifier, cpu_id, poll_interval=10):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
Exemplo n.º 9
0
    def __init__(self, cpu_id, total_cpus, poll_interval=10):
        self.keep_running = True
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.total_cpus = total_cpus
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.sync_hosts_for_shards = {}
        for database in config['DATABASE_HOSTS']:
            for shard in database['SHARDS']:
                # If no sync hosts are explicitly configured for the shard,
                # then try to steal from it. That way if you turn up a new
                # shard without properly allocating sync hosts to it, accounts
                # on it will still be started.
                self.sync_hosts_for_shards[shard['ID']] = shard.get(
                    'SYNC_HOSTS') or [self.host]
Exemplo n.º 10
0
def provider_from_address(email_address, dns_resolver=_dns_resolver):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()
    mx_domains = get_mx_domains(domain, dns_resolver)
    ns_records = []
    try:
        ns_records = dns_resolver().query(domain, 'NS')
    except NoNameservers:
        log.error('NoNameservers', domain=domain)
    except NXDOMAIN:
        log.error('No such domain', domain=domain)
    except Timeout:
        log.error('Time out during resolution', domain=domain)
    except NoAnswer:
        log.error('No answer from provider', domain=domain)

    for (name, info) in providers.iteritems():
        provider_domains = info.get('domains', [])

        # If domain is in the list of known domains for a provider,
        # return the provider.
        for d in provider_domains:
            if domain.endswith(d):
                return name

    for (name, info) in providers.iteritems():
        provider_mx = info.get('mx_servers', [])

        # If a retrieved mx_domain is in the list of stored MX domains for a
        # provider, return the provider.
        if mx_match(mx_domains, provider_mx):
            return name

    for (name, info) in providers.iteritems():
        provider_ns = info.get('ns_servers', [])

        # If a retrieved name server is in the list of stored name servers for
        # a provider, return the provider.
        for rdata in ns_records:
            if str(rdata).lower() in provider_ns:
                return name

    return 'unknown'
Exemplo n.º 11
0
def provider_from_address(email_address, dns_resolver=dns_resolver):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()
    mx_domains = get_mx_domains(domain, dns_resolver)
    ns_records = []
    try:
        ns_records = dns_resolver.query(domain, 'NS')
    except NoNameservers:
        log.error('NoNameservers', domain=domain)
    except NXDOMAIN:
        log.error('No such domain', domain=domain)
    except Timeout:
        log.error('Time out during resolution', domain=domain)
    except NoAnswer:
        log.error('No answer from provider', domain=domain)

    for (name, info) in providers.iteritems():
        provider_domains = info.get('domains', [])

        # If domain is in the list of known domains for a provider,
        # return the provider.
        for d in provider_domains:
            if domain.endswith(d):
                return name

    for (name, info) in providers.iteritems():
        provider_mx = info.get('mx_servers', [])

        # If a retrieved mx_domain is in the list of stored MX domains for a
        # provider, return the provider.
        if mx_match(mx_domains, provider_mx):
            return name

    for (name, info) in providers.iteritems():
        provider_ns = info.get('ns_servers', [])

        # If a retrieved name server is in the list of stored name servers for
        # a provider, return the provider.
        for rdata in ns_records:
            if str(rdata).lower() in provider_ns:
                return name

    return 'unknown'
Exemplo n.º 12
0
def provider_from_address(email_address):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()

    mx_records = []
    try:
        mx_records = dns_resolver.query(domain, 'MX')
    except NoNameservers:
        log.error("NoMXservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    ns_records = []
    try:
        ns_records = dns_resolver.query(domain, 'NS')
    except NoNameservers:
        log.error("NoNameservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    for (p_name, p) in providers.iteritems():
        mx_servers = p.get('mx_servers', [])
        ns_servers = p.get('ns_servers', [])
        domains = p.get('domains', [])
        if domain in domains:
            return p_name

        valid = True
        for rdata in mx_records:
            if str(rdata.exchange).lower() not in mx_servers:
                valid = False
                break

        if valid:
            return p_name

        for rdata in ns_records:
            if str(rdata).lower() not in ns_servers:
                valid = False
                break

        if valid:
            return p_name

    return 'unknown'
Exemplo n.º 13
0
    def __init__(self, process_identifier, process_number,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get('ZONE')

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()
Exemplo n.º 14
0
    def __init__(self, cpu_id, total_cpus, poll_interval=1):
        self.cpu_id = cpu_id
        self.total_cpus = total_cpus
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
Exemplo n.º 15
0
def register_backends(base_name, base_path):
    """
    Dynamically loads all packages contained within thread
    backends module, including those by other module install paths
    """
    modules = load_modules(base_name, base_path)

    mod_for = {}
    for module in modules:
        if hasattr(module, 'PROVIDER'):
            provider_name = module.PROVIDER
            if provider_name == 'generic':
                for p_name, p in providers.iteritems():
                    p_type = p.get('type', None)
                    if p_type == 'generic' and p_name not in mod_for:
                        mod_for[p_name] = module
            else:
                mod_for[provider_name] = module

    return mod_for
Exemplo n.º 16
0
    def __init__(self,
                 process_identifier,
                 cpu_id,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
        self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)

        # Fill the queue with initial values. Because of the way
        # cpu_percent works, it'll always return 0.0 the first time
        # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
        # for more details.
        null_cpu_values = psutil.cpu_percent(percpu=True)
        for i in range(NUM_CPU_SAMPLES):
            self.rolling_cpu_counts.append(null_cpu_values)
Exemplo n.º 17
0
    def __init__(self,
                 process_identifier,
                 process_number,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
        self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
        self.last_unloaded_account = time.time()

        # Fill the queue with initial values.
        null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)]
        for i in range(NUM_CPU_SAMPLES):
            self.rolling_cpu_counts.append(null_cpu_values)
Exemplo n.º 18
0
    def __init__(self, process_identifier, cpu_id,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
        self.semaphore = BoundedSemaphore(1)

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self.zone = config.get('ZONE')
        self.queue_client = QueueClient(self.zone)
        self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)

        # Fill the queue with initial values. Because of the way
        # cpu_percent works, it'll always return 0.0 the first time
        # we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
        # for more details.
        null_cpu_values = psutil.cpu_percent(percpu=True)
        for i in range(NUM_CPU_SAMPLES):
            self.rolling_cpu_counts.append(null_cpu_values)
Exemplo n.º 19
0
    def __init__(self, cpu_id, total_cpus, poll_interval=1):
        self.keep_running = True
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.total_cpus = total_cpus
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, "SYNC_MONITOR_CLS")
        }

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(cpu_id=cpu_id)
        self.log.info("starting mail sync process", supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        self.poll_interval = poll_interval
Exemplo n.º 20
0
def provider_from_address(email_address):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()
    mx_records = []
    try:
        mx_records = dns_resolver.query(domain, 'MX')
    except NoNameservers:
        log.error("NoMXservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    ns_records = []
    try:
        ns_records = dns_resolver.query(domain, 'NS')
    except NoNameservers:
        log.error("NoNameservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    for (p_name, p) in providers.iteritems():
        mx_servers = p.get('mx_servers', [])
        ns_servers = p.get('ns_servers', [])
        domains = p.get('domains', [])
        if domain in domains:
            return p_name

        valid = len(mx_records)
        for rdata in mx_records:
            mx_domain = str(rdata.exchange).lower()

            # Depending on how the MX server is configured, domain may
            # refer to a relative name or to an absolute one.
            # FIXME @karim: maybe resolve the server instead.
            if mx_domain[-1] == '.':
                mx_domain = mx_domain[:-1]

            if mx_domain not in mx_servers:
                valid = False
                break

        if valid:
            return p_name

        valid = len(ns_records)
        for rdata in ns_records:
            if str(rdata).lower() not in ns_servers:
                valid = False
                break

        if valid:
            return p_name

    return 'unknown'
Exemplo n.º 21
0
Arquivo: url.py Projeto: dhootha/inbox
def provider_from_address(email_address):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()
    mx_records = []
    try:
        mx_records = dns_resolver.query(domain, 'MX')
    except NoNameservers:
        log.error("NoMXservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    ns_records = []
    try:
        ns_records = dns_resolver.query(domain, 'NS')
    except NoNameservers:
        log.error("NoNameservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    for (p_name, p) in providers.iteritems():
        mx_servers = p.get('mx_servers', [])
        ns_servers = p.get('ns_servers', [])
        domains = p.get('domains', [])
        if domain in domains:
            return p_name

        valid = len(mx_records)
        for rdata in mx_records:
            mx_domain = str(rdata.exchange).lower()

            # Depending on how the MX server is configured, domain may
            # refer to a relative name or to an absolute one.
            # FIXME @karim: maybe resolve the server instead.
            if mx_domain[-1] == '.':
                mx_domain = mx_domain[:-1]

            # match the given domain against any of the mx_server regular
            # expressions we have stored for the given domain. If none of them
            # match, then we cannot confirm this as the given provider
            match_filter = lambda x: re.match(x + '$', mx_domain)
            if len(filter(match_filter, mx_servers)) == 0:
                valid = False
                break

        if valid:
            return p_name

        valid = len(ns_records)
        for rdata in ns_records:
            if str(rdata).lower() not in ns_servers:
                valid = False
                break

        if valid:
            return p_name

    return 'unknown'
Exemplo n.º 22
0
def provider_from_address(email_address):
    if not EMAIL_REGEX.match(email_address):
        raise InvalidEmailAddressError('Invalid email address')

    domain = email_address.split('@')[1].lower()
    mx_domains = []
    try:
        mx_records = dns_resolver.query(domain, 'MX')
        mx_domains = [str(rdata.exchange).lower() for rdata in mx_records]
    except NoNameservers:
        log.error("NoMXservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)
        mx_domains = _fallback_get_mx_domains(domain)

    ns_records = []
    try:
        ns_records = dns_resolver.query(domain, 'NS')
    except NoNameservers:
        log.error("NoNameservers error", domain=domain)
    except NXDOMAIN:
        log.error("No such domain", domain=domain)
    except Timeout:
        log.error("Timed out while resolving", domain=domain)
    except NoAnswer:
        log.error("Provider didn't answer", domain=domain)

    for (p_name, p) in providers.iteritems():
        mx_servers = p.get('mx_servers', [])
        ns_servers = p.get('ns_servers', [])
        domains = p.get('domains', [])
        if domain in domains:
            return p_name

        valid = len(mx_domains)
        for mx_domain in mx_domains:
            # Depending on how the MX server is configured, domain may
            # refer to a relative name or to an absolute one.
            # FIXME @karim: maybe resolve the server instead.
            if mx_domain[-1] == '.':
                mx_domain = mx_domain[:-1]

            # match the given domain against any of the mx_server regular
            # expressions we have stored for the given domain. If none of them
            # match, then we cannot confirm this as the given provider
            match_filter = lambda x: re.match(x + '$', mx_domain)
            if len(filter(match_filter, mx_servers)) == 0:
                valid = False
                break

        if valid:
            return p_name

        valid = len(ns_records)
        for rdata in ns_records:
            if str(rdata).lower() not in ns_servers:
                valid = False
                break

        if valid:
            return p_name

    return 'unknown'