def test_data99(mock_subproc_popen):
wa = WhoAre()
mock_subproc_popen.return_value = whois_from_txt(
'whoare/zone_parsers/ar/sample_data99.txt')
wa.load('data99.com.ar')
assert mock_subproc_popen.called
assert wa.domain.base_name == 'data99'
assert wa.domain.zone == 'com.ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(2010, 4, 12, 0, 0, 0),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2020, 3, 24, 8, 26, 1,
899786),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2021, 4, 12, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'gomez pedro'
assert wa.registrant.legal_uid == '20xxxxxxxx8'
assert wa.registrant.created == tz.localize(datetime(2013, 8, 20, 0, 0, 0),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2020, 5, 4, 19, 34, 57, 928489),
is_dst=True)
assert wa.dnss[0].name == 'ns2.cluster311.com'
assert wa.dnss[1].name == 'ns1.cluster311.com'
def test_fernet(mock_subproc_popen):
wa = WhoAre()
mock_subproc_popen.return_value = whois_from_txt(
'whoare/zone_parsers/ar/sample_fernet.txt')
wa.load('fernet.com.ar')
assert mock_subproc_popen.called
expected_dict = {
"domain": {
"base_name": 'fernet',
"zone": 'com.ar',
"is_free": False,
"registered":
wa.domain.registered.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"changed": wa.domain.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"expire": wa.domain.expire.strftime('%Y-%m-%d %H:%M:%S.%f %z')
},
"registrant": {
"name": wa.registrant.name,
"legal_uid": wa.registrant.legal_uid,
"created":
wa.registrant.created.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"changed":
wa.registrant.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z')
},
"dnss": ['ns2.sedoparking.com', 'ns1.sedoparking.com']
}
assert wa.as_dict() == expected_dict
def test_fernet(mock_subproc_popen):
wa = WhoAre()
mock_subproc_popen.return_value = whois_from_txt(
'whoare/zone_parsers/ar/sample_fernet.txt')
wa.load('fernet.com.ar')
assert mock_subproc_popen.called
assert wa.domain.base_name == 'fernet'
assert wa.domain.zone == 'com.ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(
2020, 5, 7, 10, 44, 4, 210977),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2020, 5, 7, 14, 34, 40,
828423),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2021, 5, 7, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'perez juan'
assert wa.registrant.legal_uid == '20xxxxxxxx9'
assert wa.registrant.created == tz.localize(datetime(2013, 8, 19, 0, 0, 0),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2020, 10, 11, 17, 37, 54, 831645),
is_dst=True)
assert wa.dnss[0].name == 'ns2.sedoparking.com'
assert wa.dnss[1].name == 'ns1.sedoparking.com'
def test_nic(mock_subproc_popen):
wa = WhoAre()
mock_subproc_popen.return_value = whois_from_txt(
'whoare/zone_parsers/ar/sample_nic.txt')
wa.load('nic.ar')
assert mock_subproc_popen.called
assert wa.domain.base_name == 'nic'
assert wa.domain.zone == 'ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(1998, 5, 29, 0, 0, 0),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2019, 10, 7, 16, 23, 59,
503535),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2020, 5, 1, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'nic.ar'
assert wa.registrant.legal_uid == '99999999994'
assert wa.registrant.created == tz.localize(datetime(
2016, 9, 26, 12, 4, 48, 869673),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2016, 9, 26, 17, 39, 38, 373610),
is_dst=True)
assert wa.dnss[0].name == 'ns5.rdns.ar'
assert wa.dnss[1].name == 'ns6.rdns.ar'
assert wa.dnss[2].name == 'ns1.rdns.ar'
assert wa.dnss[3].name == 'ns2.rdns.ar'
assert wa.dnss[4].name == 'ns3.rdns.ar'
def add_from_whois(cls, domain, just_new=False, mock_from_txt_file=None):
""" create or update a domain from WhoIs info
Returns None if error or the domain object
"""
logger.info(f'Adding from WhoIs {domain}')
wa = WhoAre()
domain_name, zone = wa.detect_zone(domain)
zona, _ = Zona.objects.get_or_create(nombre=zone)
if just_new:
dominio = cls.get_from_full_domain(domain)
if dominio is not None and dominio.estado == STATUS_NO_DISPONIBLE:
# Already is in the database and will be updated by priority later
return True, 'Already exists', None
try:
wa.load(domain, mock_from_txt_file=mock_from_txt_file)
except TooManyQueriesError:
return False, 'Too many queries', None
except Exception as e:
return False, str(e), None
# bad domain, don't needed
if just_new and wa.domain.is_free:
return True, 'Is a free domain', None
dominio, dominio_created = Dominio.objects.get_or_create(nombre=wa.domain.base_name, zona=zona)
# create a domain after being sure we don't have any whoare errors
logger.info(f' - Dominio {dominio} Created: {dominio_created}')
# return the changes list
return True, None, dominio.update_from_wa_object(wa, just_created=dominio_created)
def test_free_01(mock_subproc_popen):
wa = WhoAre()
mock_subproc_popen.return_value = whois_from_txt(
'whoare/zone_parsers/ar/sample_free.txt')
wa.load('domainfree.com.ar')
assert mock_subproc_popen.called
assert wa.domain.is_free
def get_from_full_domain(cls, full_domain):
wa = WhoAre()
domain_name, zone = wa.detect_zone(full_domain)
zona, _ = Zona.objects.get_or_create(nombre=zone)
dominios = Dominio.objects.filter(nombre=domain_name, zona=zona)
if dominios.count() > 0:
return dominios[0]
return None
def update_from_whoare(self, request):
data = request.data # require to be parsed
logger.info(f'update_from_whoare: {data}')
real_data_str = data['domain']
logger.info(f'real data: {real_data_str}')
# final_data = ast.literal_eval(real_data_str)
final_data = json.loads(real_data_str)
if final_data.get('whoare_version', None) is None:
return JsonResponse(
{
'ok': False,
'error': 'Missing WhoAre version'
}, status=400)
if final_data['whoare_version'] < '0.1.40':
return JsonResponse(
{
'ok': False,
'error': 'Unexpected WhoAre version'
},
status=400)
wa = WhoAre()
wa.from_dict(final_data)
# if exists at pre-domains, remove it: we are done with this domain
pres = PreDominio.objects.filter(dominio=wa.domain.full_name())
if pres.count() > 0:
pre = pres[0]
pre.delete()
# skipp not-real domains when it come from pre-domains
if wa.domain.is_free:
# Si ya existe entonces es un update, si no entonces no lo necesitamos
dominio = Dominio.get_from_full_domain(wa.domain.full_name())
if dominio is None:
return JsonResponse(
{
'ok': False,
'error': 'We expect a REGISTERED domain'
},
status=400)
zona, _ = Zona.objects.get_or_create(nombre=wa.domain.zone)
dominio, dominio_created = Dominio.objects.get_or_create(
nombre=wa.domain.base_name, zona=zona)
cambios = dominio.update_from_wa_object(wa,
just_created=dominio_created)
res = {'ok': True, 'created': dominio_created, 'cambios': cambios}
return JsonResponse(res)
def save(self, **kwargs):
# si ya existe como dominio, omitir
wa = WhoAre()
domain_name, zone = wa.detect_zone(self.dominio)
try:
zona = Zona.objects.get(nombre=zone)
except:
logger.error(f'Bad zone {zone}')
self.id = 0
return
dominios = Dominio.objects.filter(nombre=domain_name, zona=zona)
if dominios.count() > 0:
# TODO, este injerto no parece bueno
self.id = 0
# TODO si trate de agregarlo como predominio y existe pero esta disponible
# es posible que se haya vuelto a registrar. Debería aqui darle mas prioridad
# a su actualizacion
else:
return super().save(**kwargs)
def load_one(self, domain, torify):
""" analyze and push one domain """
logger.info(f'Domain {domain} tor:{torify}')
self.total_analizados += 1
if domain in self.processed:
logger.error(f'Duplicated domain {domain}. Skipping')
return
self.processed.append(domain)
# preserve only last N elements
if len(self.processed) > 100:
self.processed.pop(0)
wa = WhoAre()
try:
# default directo al 43 si es de argentina
# TODO permitir configuracion de host por dominios
if domain.endswith('.ar'):
wa.load(domain,
torify=torify,
method='43',
host='whois.nic.ar')
logger.info('REG [43]: {}'.format(wa.registrant))
else:
wa.load(domain, torify=torify)
logger.info('REG: {}'.format(wa.registrant))
except Exception as e:
logger.error(f'Serve Whois ERROR {e}')
self.errores += 1
else:
self.post_one(wa)
def get_from_predomain(self):
nuevos = PreDominio.objects.filter(priority__gt=0)
nuevos = nuevos.order_by('-priority', 'dominio')[:100]
random_item = random.choice(nuevos)
if random_item.priority == 0:
# se acabaron
return self.get_from_domain()
random_item.priority = 0
random_item.save()
# si ya existe en dominios, omitir
wa = WhoAre()
domain_name, zone = wa.detect_zone(random_item.dominio)
zona = Zona.objects.get(nombre=zone)
dominios = Dominio.objects.filter(nombre=domain_name, zona=zona)
if dominios.count() > 0:
return self.get_from_domain()
self.serializer_class = FlatPreDominioSerializer
res = PreDominio.objects.filter(pk=random_item.id)
return res
def test_without_dns():
wa = WhoAre()
wa.load('sindns.com.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_whitout_dns.txt')
assert len(wa.dnss) == 0
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_idna():
wa = WhoAre()
wa.load('cañaconruda.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_idna.txt')
assert wa.domain.base_name == 'cañaconruda'
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_free_01():
wa = WhoAre()
wa.load('domainfree.com.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_free.txt')
assert wa.domain.is_free
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_fernet():
wa = WhoAre()
wa.load('fernet.com.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_fernet.txt')
assert wa.domain.base_name == 'fernet'
assert wa.domain.zone == 'com.ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(
2020, 5, 7, 10, 44, 4, 210977),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2020, 5, 7, 14, 34, 40,
828423),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2021, 5, 7, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'perez juan'
assert wa.registrant.legal_uid == '20xxxxxxxx9'
assert wa.registrant.created == tz.localize(datetime(2013, 8, 19, 0, 0, 0),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2020, 10, 11, 17, 37, 54, 831645),
is_dst=True)
assert wa.dnss[0].name == 'ns2.sedoparking.com'
assert wa.dnss[1].name == 'ns1.sedoparking.com'
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_data99():
wa = WhoAre()
wa.load('data99.com.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_data99.txt')
assert wa.domain.base_name == 'data99'
assert wa.domain.zone == 'com.ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(2010, 4, 12, 0, 0, 0),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2020, 3, 24, 8, 26, 1,
899786),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2021, 4, 12, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'gomez pedro'
assert wa.registrant.legal_uid == '20xxxxxxxx8'
assert wa.registrant.created == tz.localize(datetime(2013, 8, 20, 0, 0, 0),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2020, 5, 4, 19, 34, 57, 928489),
is_dst=True)
assert wa.dnss[0].name == 'ns2.cluster311.com'
assert wa.dnss[1].name == 'ns1.cluster311.com'
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_nic():
wa = WhoAre()
wa.load('nic.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_nic.txt')
assert wa.domain.base_name == 'nic'
assert wa.domain.zone == 'ar'
assert not wa.domain.is_free
assert wa.domain.registered == tz.localize(datetime(1998, 5, 29, 0, 0, 0),
is_dst=True)
assert wa.domain.changed == tz.localize(datetime(2019, 10, 7, 16, 23, 59,
503535),
is_dst=True)
assert wa.domain.expire == tz.localize(datetime(2020, 5, 1, 0, 0, 0),
is_dst=True)
assert wa.registrant.name == 'nic.ar'
assert wa.registrant.legal_uid == '99999999994'
assert wa.registrant.created == tz.localize(datetime(
2016, 9, 26, 12, 4, 48, 869673),
is_dst=True)
assert wa.registrant.changed == tz.localize(datetime(
2016, 9, 26, 17, 39, 38, 373610),
is_dst=True)
assert wa.dnss[0].name == 'ns5.rdns.ar'
assert wa.dnss[1].name == 'ns6.rdns.ar'
assert wa.dnss[2].name == 'ns1.rdns.ar'
assert wa.dnss[3].name == 'ns2.rdns.ar'
assert wa.dnss[4].name == 'ns3.rdns.ar'
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
def test_fernet2():
wa = WhoAre()
wa.load('fernet.com.ar',
mock_from_txt_file='whoare/zone_parsers/ar/sample_fernet.txt')
expected_dict = {
"domain": {
"base_name": 'fernet',
"zone": 'com.ar',
"is_free": False,
"registered":
wa.domain.registered.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"changed": wa.domain.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"expire": wa.domain.expire.strftime('%Y-%m-%d %H:%M:%S.%f %z')
},
"registrant": {
"name": wa.registrant.name,
"legal_uid": wa.registrant.legal_uid,
"created":
wa.registrant.created.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"changed":
wa.registrant.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z')
},
"dnss": ['ns2.sedoparking.com', 'ns1.sedoparking.com']
}
assert wa.as_dict() == expected_dict
expected_plain_dict = {
"domain_base_name":
'fernet',
"domain_zone":
'com.ar',
"domain_is_free":
False,
"domain_registered":
wa.domain.registered.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"domain_changed":
wa.domain.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"domain_expire":
wa.domain.expire.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"registrant_name":
wa.registrant.name,
"registrant_legal_uid":
wa.registrant.legal_uid,
"registrant_created":
wa.registrant.created.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"registrant_changed":
wa.registrant.changed.strftime('%Y-%m-%d %H:%M:%S.%f %z'),
"dns1":
'ns2.sedoparking.com',
"dns2":
'ns1.sedoparking.com'
}
assert wa.as_plain_dict() == expected_plain_dict
wa2 = WhoAre()
wa2.from_dict(data=wa.as_dict())
assert wa == wa2
wa3 = WhoAre()
wa3.from_plain_dict(data=wa.as_plain_dict())
assert wa == wa3
