def __init__(self, process_identifier, cpu_id, poll_interval=10):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
# We call cpu_percent in a non-blocking way. Because of the way
# this function works, it'll always return 0.0 the first time
# we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
# for more details.
psutil.cpu_percent(percpu=True)
def __init__(self, process_identifier, cpu_id, poll_interval=10):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
# We call cpu_percent in a non-blocking way. Because of the way
# this function works, it'll always return 0.0 the first time
# we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
# for more details.
psutil.cpu_percent(percpu=True)
def __init__(self, cpu_id, total_cpus, poll_interval=1):
self.keep_running = True
self.host = platform.node()
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
def __init__(self, process_identifier, cpu_id, poll_interval=10):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
def __init__(self, cpu_id, total_cpus, poll_interval=10):
self.keep_running = True
self.host = platform.node()
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.sync_hosts_for_shards = {}
for database in config['DATABASE_HOSTS']:
for shard in database['SHARDS']:
# If no sync hosts are explicitly configured for the shard,
# then try to steal from it. That way if you turn up a new
# shard without properly allocating sync hosts to it, accounts
# on it will still be started.
self.sync_hosts_for_shards[shard['ID']] = shard.get(
'SYNC_HOSTS') or [self.host]
def __init__(self, process_identifier, process_number,
poll_interval=SYNC_POLL_INTERVAL):
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
self.last_unloaded_account = time.time()
# Fill the queue with initial values.
null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)]
for i in range(NUM_CPU_SAMPLES):
self.rolling_cpu_counts.append(null_cpu_values)
def __init__(
self,
process_identifier,
process_number,
poll_interval=SYNC_POLL_INTERVAL,
exit_after_min=None,
exit_after_max=None,
):
self.keep_running = True
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, "SYNC_MONITOR_CLS")
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info(
"starting mail sync process", supported_providers=module_registry.keys()
)
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
# Randomize the poll_interval so we maintain at least a little fairness
# when using a timeout while blocking on the redis queues.
min_poll_interval = 5
self.poll_interval = int(
(random.random() * (poll_interval - min_poll_interval)) + min_poll_interval
)
self.semaphore = BoundedSemaphore(1)
self.zone = config.get("ZONE")
# Note that we don't partition by zone for the private queues.
# There's not really a reason to since there's one queue per machine
# anyways. Also, if you really want to send an Account to a mailsync
# machine in another zone you can do so.
self.private_queue = EventQueue(
SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)
)
self.queue_group = EventQueueGroup(
[shared_sync_event_queue_for_zone(self.zone), self.private_queue,]
)
self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
self._pending_avgs_provider = None
self.last_unloaded_account = time.time()
if exit_after_min and exit_after_max:
exit_after = random.randint(exit_after_min * 60, exit_after_max * 60)
self.log.info("exit after", seconds=exit_after)
gevent.spawn_later(exit_after, self.stop)
def __init__(self, process_identifier, cpu_id, poll_interval=10):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
def __init__(self, cpu_id, total_cpus, poll_interval=10):
self.keep_running = True
self.host = platform.node()
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.sync_hosts_for_shards = {}
for database in config['DATABASE_HOSTS']:
for shard in database['SHARDS']:
# If no sync hosts are explicitly configured for the shard,
# then try to steal from it. That way if you turn up a new
# shard without properly allocating sync hosts to it, accounts
# on it will still be started.
self.sync_hosts_for_shards[shard['ID']] = shard.get(
'SYNC_HOSTS') or [self.host]
def provider_from_address(email_address, dns_resolver=_dns_resolver):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_domains = get_mx_domains(domain, dns_resolver)
ns_records = []
try:
ns_records = dns_resolver().query(domain, 'NS')
except NoNameservers:
log.error('NoNameservers', domain=domain)
except NXDOMAIN:
log.error('No such domain', domain=domain)
except Timeout:
log.error('Time out during resolution', domain=domain)
except NoAnswer:
log.error('No answer from provider', domain=domain)
for (name, info) in providers.iteritems():
provider_domains = info.get('domains', [])
# If domain is in the list of known domains for a provider,
# return the provider.
for d in provider_domains:
if domain.endswith(d):
return name
for (name, info) in providers.iteritems():
provider_mx = info.get('mx_servers', [])
# If a retrieved mx_domain is in the list of stored MX domains for a
# provider, return the provider.
if mx_match(mx_domains, provider_mx):
return name
for (name, info) in providers.iteritems():
provider_ns = info.get('ns_servers', [])
# If a retrieved name server is in the list of stored name servers for
# a provider, return the provider.
for rdata in ns_records:
if str(rdata).lower() in provider_ns:
return name
return 'unknown'
def provider_from_address(email_address, dns_resolver=dns_resolver):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_domains = get_mx_domains(domain, dns_resolver)
ns_records = []
try:
ns_records = dns_resolver.query(domain, 'NS')
except NoNameservers:
log.error('NoNameservers', domain=domain)
except NXDOMAIN:
log.error('No such domain', domain=domain)
except Timeout:
log.error('Time out during resolution', domain=domain)
except NoAnswer:
log.error('No answer from provider', domain=domain)
for (name, info) in providers.iteritems():
provider_domains = info.get('domains', [])
# If domain is in the list of known domains for a provider,
# return the provider.
for d in provider_domains:
if domain.endswith(d):
return name
for (name, info) in providers.iteritems():
provider_mx = info.get('mx_servers', [])
# If a retrieved mx_domain is in the list of stored MX domains for a
# provider, return the provider.
if mx_match(mx_domains, provider_mx):
return name
for (name, info) in providers.iteritems():
provider_ns = info.get('ns_servers', [])
# If a retrieved name server is in the list of stored name servers for
# a provider, return the provider.
for rdata in ns_records:
if str(rdata).lower() in provider_ns:
return name
return 'unknown'
def provider_from_address(email_address):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_records = []
try:
mx_records = dns_resolver.query(domain, 'MX')
except NoNameservers:
log.error("NoMXservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
ns_records = []
try:
ns_records = dns_resolver.query(domain, 'NS')
except NoNameservers:
log.error("NoNameservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
for (p_name, p) in providers.iteritems():
mx_servers = p.get('mx_servers', [])
ns_servers = p.get('ns_servers', [])
domains = p.get('domains', [])
if domain in domains:
return p_name
valid = True
for rdata in mx_records:
if str(rdata.exchange).lower() not in mx_servers:
valid = False
break
if valid:
return p_name
for rdata in ns_records:
if str(rdata).lower() not in ns_servers:
valid = False
break
if valid:
return p_name
return 'unknown'
def __init__(self, process_identifier, process_number,
poll_interval=SYNC_POLL_INTERVAL):
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
# Randomize the poll_interval so we maintain at least a little fairness
# when using a timeout while blocking on the redis queues.
min_poll_interval = 5
self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval)
self.semaphore = BoundedSemaphore(1)
self.zone = config.get('ZONE')
# Note that we don't partition by zone for the private queues.
# There's not really a reason to since there's one queue per machine
# anyways. Also, if you really want to send an Account to a mailsync
# machine in another zone you can do so.
self.private_queue = EventQueue(SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
self.queue_group = EventQueueGroup([
shared_sync_event_queue_for_zone(self.zone),
self.private_queue,
])
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self._pending_avgs_provider = None
self.last_unloaded_account = time.time()
def __init__(self, cpu_id, total_cpus, poll_interval=1):
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
def register_backends(base_name, base_path):
"""
Dynamically loads all packages contained within thread
backends module, including those by other module install paths
"""
modules = load_modules(base_name, base_path)
mod_for = {}
for module in modules:
if hasattr(module, 'PROVIDER'):
provider_name = module.PROVIDER
if provider_name == 'generic':
for p_name, p in providers.iteritems():
p_type = p.get('type', None)
if p_type == 'generic' and p_name not in mod_for:
mod_for[p_name] = module
else:
mod_for[provider_name] = module
return mod_for
def __init__(self,
process_identifier,
cpu_id,
poll_interval=SYNC_POLL_INTERVAL):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
# Fill the queue with initial values. Because of the way
# cpu_percent works, it'll always return 0.0 the first time
# we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
# for more details.
null_cpu_values = psutil.cpu_percent(percpu=True)
for i in range(NUM_CPU_SAMPLES):
self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self,
process_identifier,
process_number,
poll_interval=SYNC_POLL_INTERVAL):
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
self.last_unloaded_account = time.time()
# Fill the queue with initial values.
null_cpu_values = [0.0 for cpu in psutil.cpu_percent(percpu=True)]
for i in range(NUM_CPU_SAMPLES):
self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self, process_identifier, cpu_id,
poll_interval=SYNC_POLL_INTERVAL):
self.host = platform.node()
self.cpu_id = cpu_id
self.process_identifier = process_identifier
self.monitor_cls_for = {mod.PROVIDER: getattr(
mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
if hasattr(mod, 'SYNC_MONITOR_CLS')}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info('starting mail sync process',
supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
self.semaphore = BoundedSemaphore(1)
self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
self.zone = config.get('ZONE')
self.queue_client = QueueClient(self.zone)
self.rolling_cpu_counts = collections.deque(maxlen=NUM_CPU_SAMPLES)
# Fill the queue with initial values. Because of the way
# cpu_percent works, it'll always return 0.0 the first time
# we call it. See: https://pythonhosted.org/psutil/#psutil.cpu_percent
# for more details.
null_cpu_values = psutil.cpu_percent(percpu=True)
for i in range(NUM_CPU_SAMPLES):
self.rolling_cpu_counts.append(null_cpu_values)
def __init__(self, cpu_id, total_cpus, poll_interval=1):
self.keep_running = True
self.host = platform.node()
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, "SYNC_MONITOR_CLS")
}
for p_name, p in providers.iteritems():
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(cpu_id=cpu_id)
self.log.info("starting mail sync process", supported_providers=module_registry.keys())
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
self.poll_interval = poll_interval
def provider_from_address(email_address):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_records = []
try:
mx_records = dns_resolver.query(domain, 'MX')
except NoNameservers:
log.error("NoMXservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
ns_records = []
try:
ns_records = dns_resolver.query(domain, 'NS')
except NoNameservers:
log.error("NoNameservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
for (p_name, p) in providers.iteritems():
mx_servers = p.get('mx_servers', [])
ns_servers = p.get('ns_servers', [])
domains = p.get('domains', [])
if domain in domains:
return p_name
valid = len(mx_records)
for rdata in mx_records:
mx_domain = str(rdata.exchange).lower()
# Depending on how the MX server is configured, domain may
# refer to a relative name or to an absolute one.
# FIXME @karim: maybe resolve the server instead.
if mx_domain[-1] == '.':
mx_domain = mx_domain[:-1]
if mx_domain not in mx_servers:
valid = False
break
if valid:
return p_name
valid = len(ns_records)
for rdata in ns_records:
if str(rdata).lower() not in ns_servers:
valid = False
break
if valid:
return p_name
return 'unknown'
def provider_from_address(email_address):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_records = []
try:
mx_records = dns_resolver.query(domain, 'MX')
except NoNameservers:
log.error("NoMXservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
ns_records = []
try:
ns_records = dns_resolver.query(domain, 'NS')
except NoNameservers:
log.error("NoNameservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
for (p_name, p) in providers.iteritems():
mx_servers = p.get('mx_servers', [])
ns_servers = p.get('ns_servers', [])
domains = p.get('domains', [])
if domain in domains:
return p_name
valid = len(mx_records)
for rdata in mx_records:
mx_domain = str(rdata.exchange).lower()
# Depending on how the MX server is configured, domain may
# refer to a relative name or to an absolute one.
# FIXME @karim: maybe resolve the server instead.
if mx_domain[-1] == '.':
mx_domain = mx_domain[:-1]
# match the given domain against any of the mx_server regular
# expressions we have stored for the given domain. If none of them
# match, then we cannot confirm this as the given provider
match_filter = lambda x: re.match(x + '$', mx_domain)
if len(filter(match_filter, mx_servers)) == 0:
valid = False
break
if valid:
return p_name
valid = len(ns_records)
for rdata in ns_records:
if str(rdata).lower() not in ns_servers:
valid = False
break
if valid:
return p_name
return 'unknown'
def provider_from_address(email_address):
if not EMAIL_REGEX.match(email_address):
raise InvalidEmailAddressError('Invalid email address')
domain = email_address.split('@')[1].lower()
mx_domains = []
try:
mx_records = dns_resolver.query(domain, 'MX')
mx_domains = [str(rdata.exchange).lower() for rdata in mx_records]
except NoNameservers:
log.error("NoMXservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
mx_domains = _fallback_get_mx_domains(domain)
ns_records = []
try:
ns_records = dns_resolver.query(domain, 'NS')
except NoNameservers:
log.error("NoNameservers error", domain=domain)
except NXDOMAIN:
log.error("No such domain", domain=domain)
except Timeout:
log.error("Timed out while resolving", domain=domain)
except NoAnswer:
log.error("Provider didn't answer", domain=domain)
for (p_name, p) in providers.iteritems():
mx_servers = p.get('mx_servers', [])
ns_servers = p.get('ns_servers', [])
domains = p.get('domains', [])
if domain in domains:
return p_name
valid = len(mx_domains)
for mx_domain in mx_domains:
# Depending on how the MX server is configured, domain may
# refer to a relative name or to an absolute one.
# FIXME @karim: maybe resolve the server instead.
if mx_domain[-1] == '.':
mx_domain = mx_domain[:-1]
# match the given domain against any of the mx_server regular
# expressions we have stored for the given domain. If none of them
# match, then we cannot confirm this as the given provider
match_filter = lambda x: re.match(x + '$', mx_domain)
if len(filter(match_filter, mx_servers)) == 0:
valid = False
break
if valid:
return p_name
valid = len(ns_records)
for rdata in ns_records:
if str(rdata).lower() not in ns_servers:
valid = False
break
if valid:
return p_name
return 'unknown'