def _create_address(self, instance, type, address, postal_code, city, country, primary=False):
"""
Create an Address for given instance
Arguments:
instance (instance): instance to add Address to.
type (str): kind of address
address (str): street and number
postal_code (str): postal code
city (str): city
country (str): country
"""
if address and city:
street, number, complement = parse_address(address)
address_kwargs = dict()
address_kwargs["type"] = type
address_kwargs["street"] = street
# Set streetnumber.
if _isint(number) and int(number) < 32766:
address_kwargs["street_number"] = int(number)
address_kwargs["complement"] = complement
# Set postal code.
if len(postal_code) > 10:
postal_code = postal_code[:10]
address_kwargs["postal_code"] = postal_code
address_kwargs["city"] = city
# Check if country is present.
if country:
country = country.upper().strip()
country = self.country_map.get(country, country)
# In case of unkown county leave empty.
if country and len(country) > 2 or country not in self.country_codes:
country = ""
# Set country.
address_kwargs["country"] = country
# Set tenant.
address_kwargs["tenant_id"] = self.tenant_pk
# Check if adress does not already exists to avoid duplicates.
if not instance.addresses.filter(street=address_kwargs["street"], city=address_kwargs["city"]).exists():
try:
address = Address.objects.create(**address_kwargs)
except DataError:
logger.warning("could not create address: %s" % address_kwargs)
else:
instance.addresses.add(address)
else:
address_kwargs.pop("tenant_id")
instance.addresses.filter(street=address_kwargs["street"], city=address_kwargs["city"]).update(
**address_kwargs
)
def _create_address(self, instance, type, address, postal_code, city, country, primary=False):
"""
Create an Address for given instance
Arguments:
instance (instance): instance to add Address to.
type (str): kind of address
address (str): street and number
postal_code (str): postal code
city (str): city
country (str): country
"""
if address and city:
street, number, complement = parse_address(address)
address_kwargs = dict()
address_kwargs['type'] = type
address_kwargs['street'] = street
if _isint(number) and int(number) < 32766:
address_kwargs['street_number'] = int(number)
address_kwargs['complement'] = complement
if postal_code and len(postal_code) > 10:
postal_code = postal_code[:10]
else:
postal_code = ''
address_kwargs['postal_code'] = postal_code
address_kwargs['city'] = city
if country:
country = country.upper().strip()
country = self.country_map.get(country, country)
if country and len(country) > 2 or country not in self.country_codes:
country = ''
address_kwargs['country'] = country
address_kwargs['tenant_id'] = self.tenant_pk
if not instance.addresses.filter(**address_kwargs).exists():
try:
address = Address.objects.create(**address_kwargs)
except DataError:
task_logger.warning('could not create address: %s' % address_kwargs)
else:
instance.addresses.add(address)
def _create_address(self,
instance,
type,
address,
postal_code,
city,
country,
primary=False):
"""
Create an Address for given instance
Arguments:
instance (instance): instance to add Address to.
type (str): kind of address
address (str): street and number
postal_code (str): postal code
city (str): city
country (str): country
"""
if address and city:
street, number, complement = parse_address(address)
address_kwargs = dict()
address_kwargs['type'] = type
address_kwargs['street'] = street
# Set streetnumber.
if is_int(number) and int(number) < 32766:
address_kwargs['street_number'] = int(number)
address_kwargs['complement'] = complement
# Set postal code.
if len(postal_code) > 10:
postal_code = postal_code[:10]
address_kwargs['postal_code'] = postal_code
address_kwargs['city'] = city
# Check if country is present.
if country:
country = country.upper().strip()
country = self.country_map.get(country, country)
# In case of unkown county leave empty.
if country and len(
country) > 2 or country not in self.country_codes:
country = ''
# Set country.
address_kwargs['country'] = country
# Set tenant.
address_kwargs['tenant_id'] = self.tenant_pk
# Check if adress does not already exists to avoid duplicates.
if not instance.addresses.filter(
street=address_kwargs['street'],
city=address_kwargs['city']).exists():
try:
address = Address.objects.create(**address_kwargs)
except DataError:
logger.warning('could not create address: %s' %
address_kwargs)
else:
instance.addresses.add(address)
else:
address_kwargs.pop('tenant_id')
instance.addresses.filter(
street=address_kwargs['street'],
city=address_kwargs['city']).update(**address_kwargs)
def set_view_output(self):
"""
Create a generic json format for account information based on the json from Dataprovider.
"""
phone_number_limit = 5
email_limit = 5
address_limit = 3
# Expected api output is json
self.api_output = json.loads(self.api_output)
# Return 404 when the api returned an error
if self.api_output.get('error'):
raise Http404()
# Return error message when nothing was found
if self.api_output.get('total') == 0:
raise Exception(
_('I\'m so sorry, I couldn\'t find any data for this website.')
)
# Filter useful data
result = self.api_output['data'][0]
# Get company name
company = result.get('company')
# Get website description
description = result.get('description')
# Get the keywords and convert to list
tags = result.get('keywords')
if tags:
tags = result.get('keywords').strip().rstrip(',').split(',')
# Get email addresses and convert to a list if needed
emails = result.get('emailaddresses', []) or []
if not isinstance(emails, list):
emails = [emails]
# Determine primary email since Dataprovider doesn't provide it
primary_email = None
if emails:
primary_email = self._get_primary_email(emails)
# Set primary email to the first in the list
emails.index(primary_email)
emails.remove(primary_email)
emails.insert(0, primary_email)
# Limit number of emails
emails = emails[:email_limit]
phone_numbers = []
# Get primary phone number and convert to a nicer representation
phone_number = result.get('phonenumber')
if phone_number:
phone_number = parse_phone_number(phone_number)
phone_numbers.append(phone_number)
# Get phone numbers and convert to list if needed
raw_phone_numbers = result.get('phonenumbers', []) or []
if not isinstance(raw_phone_numbers, list):
raw_phone_numbers = [raw_phone_numbers]
# Convert all phone numbers to a nicer representation
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Limit number of phonenumbers
phone_numbers = phone_numbers[:phone_number_limit]
# Get what kind of company it is (e.g. LLC)
legalentity = result.get('legalentity')
# Get the VAT (Value Added Tax) identifaction number
taxnumber = result.get('taxnumber')
# Get bank account number
bankaccountnumber = result.get('bankaccountnumber')
# Get the CoC (Chamber of Commerce) number
cocnumber = result.get('cocnumber')
# Get the IBAN (Internation Bank Account Number)
iban = result.get('iban')
# Get the BIC (Bank Identifier Code)
bic = result.get('bic')
# Try to parse the address
address = result.get('address')
if address:
street, street_number, complement = parse_address(address)
else:
street, street_number, complement = None, None, None
# Make the full address
addresses = []
if address or result.get('city') or result.get(
'zipcode') or result.get('country'):
addresses = [{
'street': street,
'street_number': street_number,
'complement': complement,
'city': result.get('city'),
'country': result.get('country'),
'postal_code': result.get('zipcode'),
}]
addresses = addresses[:address_limit]
# Build dict with account information
self.view_output = {
'name': company,
'description': description,
'tags': tags,
'email_addresses': emails,
'primary_email': primary_email,
'phone_numbers': phone_numbers,
'phone_number': phone_number,
'addresses': addresses,
'legalentity': legalentity,
'taxnumber': taxnumber,
'bankaccountnumber': bankaccountnumber,
'cocnumber': cocnumber,
'iban': iban,
'bic': bic,
}
return json.dumps(self.view_output)
def set_view_output(self):
"""
Create a generic json format for account information based on the json from Dataprovider.
"""
# Expected api output is json
self.api_output = anyjson.deserialize(self.api_output)
# Return 404 when the api returned an error
if self.api_output.get('error'):
raise Http404()
# Return error message when nothing was found
if self.api_output.get('total') == 0:
raise Exception(_('I\'m so sorry, I couldn\'t find any data for this website.'))
# Filter useful data
result = self.api_output['data'][0]
# Get company name
company = result.get('company')
# Get website description
description = result.get('description')
# Get the keywords and convert to list
tags = result.get('keywords')
if tags:
tags = result.get('keywords').strip().rstrip(',').split(',')
# Get email addresses and convert to a list if needed
emails = result.get('emailaddresses', []) or []
if not isinstance(emails, list):
emails = [emails]
# Determine primary email since Dataprovider doesn't provide it
primary_email = None
if emails:
primary_email = self.get_primary_email(emails)
# Get phone numbers and convert to list if needed
raw_phone_numbers = result.get('phonenumbers', []) or []
if not isinstance(raw_phone_numbers, list):
raw_phone_numbers = [raw_phone_numbers]
# Convert all phone numbers to a nicer representation
phone_numbers = []
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Get primary phone number and convert to a nicer representation
phone_number = result.get('phonenumber')
if phone_number:
phone_number = parse_phone_number(result.get('phonenumber'))
# Get what kind of company it is (e.g. LLC)
legalentity = result.get('legalentity')
# Get the VAT (Value Added Tax) identifaction number
taxnumber = result.get('taxnumber')
# Get bank account number
bankaccountnumber = result.get('bankaccountnumber')
# Get the CoC (Chamber of Commerce) number
cocnumber = result.get('cocnumber')
# Get the IBAN (Internation Bank Account Number)
iban = result.get('iban')
# Get the BIC (Bank Identifier Code)
bic = result.get('bic')
# Try to parse the address
address = result.get('address')
if address:
street, street_number, complement = parse_address(address)
else:
street, street_number, complement = None, None, None
# Make the full address
addresses = []
if address or result.get('city') or result.get('zipcode') or result.get('country'):
addresses = [{
'street': street,
'street_number': street_number,
'complement': complement,
'city': result.get('city'),
'country': result.get('country'),
'postal_code': result.get('zipcode'),
}]
# Build dict with account information
self.view_output = {
'name': company,
'description': description,
'tags': tags,
'emails': emails,
'primary_email': primary_email,
'phone_numbers': phone_numbers,
'phone_number': phone_number,
'addresses': addresses,
'legalentity': legalentity,
'taxnumber': taxnumber,
'bankaccountnumber': bankaccountnumber,
'cocnumber': cocnumber,
'iban': iban,
'bic': bic,
}
return anyjson.serialize(self.view_output)
def _get_account_information(self, json):
"""
Create a dict with account information based on the json response from Dataprovider.
"""
phone_number_limit = 5
email_limit = 5
address_limit = 3
# Return 404 when the api returned an error.
if json.get('error'):
raise Http404(json.get('error'))
if json.get('total') == 0:
# No results from Dataprovider so return an empty list.
return []
# Filter useful data.
try:
# Lookup API returns data as a dict.
result = json['data'][0]
except (KeyError, IndexError):
# Enrich API returns a single hit.
result = json['data']
if not result:
# No results from Dataprovider so return an empty list.
return []
# Get the keywords and convert to list.
tags = result.get('keywords')
if tags:
tags = [tag.strip() for tag in tags.split(',')]
tags = filter(None, tags) # Remove empty tags.
# Get email addresses as a list.
emails = list(result.get('emailaddresses', []))
# Determine primary email since Dataprovider doesn't provide it.
primary_email = None
if emails:
primary_email = self._get_primary_email(emails)
# Set primary email to the first in the list.
emails.index(primary_email)
emails.remove(primary_email)
emails.insert(0, primary_email)
phone_numbers = []
# Get primary phone number and convert to a nicer representation.
phone_number = result.get('phonenumber')
if phone_number:
phone_number = parse_phone_number(phone_number)
phone_numbers.append(phone_number)
# Get phone numbers as a list.
raw_phone_numbers = list(result.get('phonenumbers', []))
# Convert all phone numbers to a nicer representation.
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Try to parse the address.
address = result.get('address')
address_line = ''
if address:
# Construct address_line, instead of assigning address to address_line directly,
# because parse_address() also santizes the result.
street, street_number, complement = parse_address(address)
if street:
address_line = street
if street_number:
address_line += ' ' + street_number
if complement:
address_line += complement
# Make the full address.
addresses = []
if address or result.get('city') or result.get('zipcode') or result.get('country'):
# There are some exceptions on the country codes, so make sure results of Dataprovider conform to the
# ISO 3166-1 alpha-2 standard we use.
mapping = {
'UK': 'GB',
'EL': 'GR',
}
country = mapping.get(result.get('country'), result.get('country')) if result.get('country') else ''
addresses = [{
'address': address_line,
'city': result.get('city', ''),
'country': country,
'postal_code': result.get('zipcode', ''),
}]
# Get social media profiles.
social_profiles = result.get('socialprofiles')
# Group profiles by platform.
# Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest.
social_media = {}
if social_profiles:
for profile in social_profiles:
if profile.startswith('twitter.com/'):
if 'twitter' not in social_media:
social_media['twitter'] = []
social_media['twitter'].append(profile)
elif profile.startswith('www.linkedin.com/in/'):
if 'linkedin' not in social_media:
social_media['linkedin'] = []
social_media['linkedin'].append(profile)
primary_twitter = ''
if 'twitter' in social_media:
primary_twitter = self._get_primary_profile(social_media['twitter'], result.get('company'))
primary_linkedin = ''
if 'linkedin' in social_media:
primary_linkedin = self._get_primary_profile(social_media['linkedin'], result.get('company'))
description = result.get('description')
website = result.get('hostname')
# Build dict with account information.
account_information = {
'name': result.get('company'),
'description': description,
'tags': tags,
'email_addresses': emails[:email_limit],
'primary_email': primary_email,
'phone_numbers': phone_numbers[:phone_number_limit],
'phone_number': phone_number,
'addresses': addresses[:address_limit],
'legalentity': result.get('legalentity'),
'taxnumber': result.get('taxnumber'),
'bankaccountnumber': result.get('bankaccountnumber'),
'cocnumber': result.get('cocnumber'),
'iban': result.get('iban'),
'bic': result.get('bic'),
'social_media_profiles': social_media,
'twitter': primary_twitter,
'linkedin': primary_linkedin,
'primary_website': website,
}
return account_information
def _get_account_information(self, api_output):
"""
Create a dict with account information based on the json response from Dataprovider.
"""
phone_number_limit = 5
email_limit = 5
address_limit = 3
# Expected API output is json.
api_output_json = json.loads(api_output)
# Return 404 when the api returned an error.
if api_output_json.get('error'):
raise Http404(api_output_json.get('error'))
# Return error message when nothing was found.
if api_output_json.get('total') == 0:
raise APIException(
_('I\'m so sorry, I couldn\'t find any data for this website.')
)
# Filter useful data.
result = api_output_json['data'][0]
# Get the keywords and convert to list.
tags = result.get('keywords')
if tags:
tags = result.get('keywords').strip().rstrip(',').split(',')
# Get email addresses as a list.
emails = list(result.get('emailaddresses', []))
# Determine primary email since Dataprovider doesn't provide it.
primary_email = None
if emails:
primary_email = self._get_primary_email(emails)
# Set primary email to the first in the list.
emails.index(primary_email)
emails.remove(primary_email)
emails.insert(0, primary_email)
phone_numbers = []
# Get primary phone number and convert to a nicer representation.
phone_number = result.get('phonenumber')
if phone_number:
phone_number = parse_phone_number(phone_number)
phone_numbers.append(phone_number)
# Get phone numbers as a list.
raw_phone_numbers = list(result.get('phonenumbers', []))
# Convert all phone numbers to a nicer representation.
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Try to parse the address.
address = result.get('address')
address_line = ''
if address:
# Construct address_line, instead of assigning address to address_line directly,
# because parse_address() also santizes the result.
street, street_number, complement = parse_address(address)
if street:
address_line = street
if street_number:
address_line += ' ' + street_number
if complement:
address_line += complement
# Make the full address.
addresses = []
if address or result.get('city') or result.get(
'zipcode') or result.get('country'):
addresses = [{
'address': address_line,
'city': result.get('city'),
'country': result.get('country'),
'postal_code': result.get('zipcode'),
}]
# Get social media profiles.
social_profiles = result.get('socialprofiles')
# Group profiles by platform.
# Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest.
social_media = {}
for profile in social_profiles:
if profile.startswith('twitter.com/'):
if 'twitter' not in social_media:
social_media['twitter'] = []
social_media['twitter'].append(profile)
elif profile.startswith('www.linkedin.com/in/'):
if 'linkedin' not in social_media:
social_media['linkedin'] = []
social_media['linkedin'].append(profile)
primary_twitter = ''
if 'twitter' in social_media:
primary_twitter = self._get_primary_profile(
social_media['twitter'], result.get('company'))
primary_linkedin = ''
if 'linkedin' in social_media:
primary_linkedin = self._get_primary_profile(
social_media['linkedin'], result.get('company'))
# Build dict with account information.
account_information = {
'name': result.get('company'),
'description': result.get('description'),
'tags': tags,
'email_addresses': emails[:email_limit],
'primary_email': primary_email,
'phone_numbers': phone_numbers[:phone_number_limit],
'phone_number': phone_number,
'addresses': addresses[:address_limit],
'legalentity': result.get('legalentity'),
'taxnumber': result.get('taxnumber'),
'bankaccountnumber': result.get('bankaccountnumber'),
'cocnumber': result.get('cocnumber'),
'iban': result.get('iban'),
'bic': result.get('bic'),
'social_media_profiles': social_media,
'twitter': primary_twitter,
'linkedin': primary_linkedin,
}
return account_information
def set_view_output(self):
"""
Create a generic json format for account information based on the json from Dataprovider.
"""
phone_number_limit = 5
email_limit = 5
address_limit = 3
# Expected api output is json
self.api_output = json.loads(self.api_output)
# Return 404 when the api returned an error
if self.api_output.get("error"):
raise Http404()
# Return error message when nothing was found
if self.api_output.get("total") == 0:
raise Exception(_("I'm so sorry, I couldn't find any data for this website."))
# Filter useful data
result = self.api_output["data"][0]
# Get company name
company = result.get("company")
# Get website description
description = result.get("description")
# Get the keywords and convert to list
tags = result.get("keywords")
if tags:
tags = result.get("keywords").strip().rstrip(",").split(",")
# Get email addresses and convert to a list if needed
emails = result.get("emailaddresses", []) or []
if not isinstance(emails, list):
emails = [emails]
# Determine primary email since Dataprovider doesn't provide it
primary_email = None
if emails:
primary_email = self._get_primary_email(emails)
# Set primary email to the first in the list
emails.index(primary_email)
emails.remove(primary_email)
emails.insert(0, primary_email)
# Limit number of emails
emails = emails[:email_limit]
phone_numbers = []
# Get primary phone number and convert to a nicer representation
phone_number = result.get("phonenumber")
if phone_number:
phone_number = parse_phone_number(phone_number)
phone_numbers.append(phone_number)
# Get phone numbers and convert to list if needed
raw_phone_numbers = result.get("phonenumbers", []) or []
if not isinstance(raw_phone_numbers, list):
raw_phone_numbers = [raw_phone_numbers]
# Convert all phone numbers to a nicer representation
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Limit number of phonenumbers
phone_numbers = phone_numbers[:phone_number_limit]
# Get what kind of company it is (e.g. LLC)
legalentity = result.get("legalentity")
# Get the VAT (Value Added Tax) identifaction number
taxnumber = result.get("taxnumber")
# Get bank account number
bankaccountnumber = result.get("bankaccountnumber")
# Get the CoC (Chamber of Commerce) number
cocnumber = result.get("cocnumber")
# Get the IBAN (Internation Bank Account Number)
iban = result.get("iban")
# Get the BIC (Bank Identifier Code)
bic = result.get("bic")
# Try to parse the address
address = result.get("address")
if address:
street, street_number, complement = parse_address(address)
else:
street, street_number, complement = None, None, None
# Make the full address
addresses = []
if address or result.get("city") or result.get("zipcode") or result.get("country"):
addresses = [
{
"street": street,
"street_number": street_number,
"complement": complement,
"city": result.get("city"),
"country": result.get("country"),
"postal_code": result.get("zipcode"),
}
]
addresses = addresses[:address_limit]
# Build dict with account information
self.view_output = {
"name": company,
"description": description,
"tags": tags,
"email_addresses": emails,
"primary_email": primary_email,
"phone_numbers": phone_numbers,
"phone_number": phone_number,
"addresses": addresses,
"legalentity": legalentity,
"taxnumber": taxnumber,
"bankaccountnumber": bankaccountnumber,
"cocnumber": cocnumber,
"iban": iban,
"bic": bic,
}
return json.dumps(self.view_output)
def _get_account_information(self, api_output):
"""
Create a dict with account information based on the json response from Dataprovider.
"""
phone_number_limit = 5
email_limit = 5
address_limit = 3
# Expected API output is json.
api_output_json = json.loads(api_output)
# Return 404 when the api returned an error.
if api_output_json.get('error'):
raise Http404(api_output_json.get('error'))
# Return error message when nothing was found.
if api_output_json.get('total') == 0:
raise APIException(_('I\'m so sorry, I couldn\'t find any data for this website.'))
# Filter useful data.
result = api_output_json['data'][0]
# Get the keywords and convert to list.
tags = result.get('keywords')
if tags:
tags = result.get('keywords').strip().rstrip(',').split(',')
# Get email addresses as a list.
emails = list(result.get('emailaddresses', []))
# Determine primary email since Dataprovider doesn't provide it.
primary_email = None
if emails:
primary_email = self._get_primary_email(emails)
# Set primary email to the first in the list.
emails.index(primary_email)
emails.remove(primary_email)
emails.insert(0, primary_email)
phone_numbers = []
# Get primary phone number and convert to a nicer representation.
phone_number = result.get('phonenumber')
if phone_number:
phone_number = parse_phone_number(phone_number)
phone_numbers.append(phone_number)
# Get phone numbers as a list.
raw_phone_numbers = list(result.get('phonenumbers', []))
# Convert all phone numbers to a nicer representation.
for raw_phone_number in raw_phone_numbers:
phone_numbers.append(parse_phone_number(raw_phone_number))
# Try to parse the address.
address = result.get('address')
address_line = ''
if address:
# Construct address_line, instead of assigning address to address_line directly,
# because parse_address() also santizes the result.
street, street_number, complement = parse_address(address)
if street:
address_line = street
if street_number:
address_line += ' ' + street_number
if complement:
address_line += complement
# Make the full address.
addresses = []
if address or result.get('city') or result.get('zipcode') or result.get('country'):
addresses = [{
'address': address_line,
'city': result.get('city'),
'country': result.get('country'),
'postal_code': result.get('zipcode'),
}]
# Get social media profiles.
social_profiles = result.get('socialprofiles')
# Group profiles by platform.
# Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest.
social_media = {}
for profile in social_profiles:
if profile.startswith('twitter.com/'):
if 'twitter' not in social_media:
social_media['twitter'] = []
social_media['twitter'].append(profile)
elif profile.startswith('www.linkedin.com/in/'):
if 'linkedin' not in social_media:
social_media['linkedin'] = []
social_media['linkedin'].append(profile)
primary_twitter = ''
if 'twitter' in social_media:
primary_twitter = self._get_primary_profile(social_media['twitter'], result.get('company'))
primary_linkedin = ''
if 'linkedin' in social_media:
primary_linkedin = self._get_primary_profile(social_media['linkedin'], result.get('company'))
# Build dict with account information.
account_information = {
'name': result.get('company'),
'description': result.get('description'),
'tags': tags,
'email_addresses': emails[:email_limit],
'primary_email': primary_email,
'phone_numbers': phone_numbers[:phone_number_limit],
'phone_number': phone_number,
'addresses': addresses[:address_limit],
'legalentity': result.get('legalentity'),
'taxnumber': result.get('taxnumber'),
'bankaccountnumber': result.get('bankaccountnumber'),
'cocnumber': result.get('cocnumber'),
'iban': result.get('iban'),
'bic': result.get('bic'),
'social_media_profiles': social_media,
'twitter': primary_twitter,
'linkedin': primary_linkedin,
}
return account_information
