def _create_address(self, instance, type, address, postal_code, city, country, primary=False): """ Create an Address for given instance Arguments: instance (instance): instance to add Address to. type (str): kind of address address (str): street and number postal_code (str): postal code city (str): city country (str): country """ if address and city: street, number, complement = parse_address(address) address_kwargs = dict() address_kwargs["type"] = type address_kwargs["street"] = street # Set streetnumber. if _isint(number) and int(number) < 32766: address_kwargs["street_number"] = int(number) address_kwargs["complement"] = complement # Set postal code. if len(postal_code) > 10: postal_code = postal_code[:10] address_kwargs["postal_code"] = postal_code address_kwargs["city"] = city # Check if country is present. if country: country = country.upper().strip() country = self.country_map.get(country, country) # In case of unkown county leave empty. if country and len(country) > 2 or country not in self.country_codes: country = "" # Set country. address_kwargs["country"] = country # Set tenant. address_kwargs["tenant_id"] = self.tenant_pk # Check if adress does not already exists to avoid duplicates. if not instance.addresses.filter(street=address_kwargs["street"], city=address_kwargs["city"]).exists(): try: address = Address.objects.create(**address_kwargs) except DataError: logger.warning("could not create address: %s" % address_kwargs) else: instance.addresses.add(address) else: address_kwargs.pop("tenant_id") instance.addresses.filter(street=address_kwargs["street"], city=address_kwargs["city"]).update( **address_kwargs )
def _create_address(self, instance, type, address, postal_code, city, country, primary=False): """ Create an Address for given instance Arguments: instance (instance): instance to add Address to. type (str): kind of address address (str): street and number postal_code (str): postal code city (str): city country (str): country """ if address and city: street, number, complement = parse_address(address) address_kwargs = dict() address_kwargs['type'] = type address_kwargs['street'] = street if _isint(number) and int(number) < 32766: address_kwargs['street_number'] = int(number) address_kwargs['complement'] = complement if postal_code and len(postal_code) > 10: postal_code = postal_code[:10] else: postal_code = '' address_kwargs['postal_code'] = postal_code address_kwargs['city'] = city if country: country = country.upper().strip() country = self.country_map.get(country, country) if country and len(country) > 2 or country not in self.country_codes: country = '' address_kwargs['country'] = country address_kwargs['tenant_id'] = self.tenant_pk if not instance.addresses.filter(**address_kwargs).exists(): try: address = Address.objects.create(**address_kwargs) except DataError: task_logger.warning('could not create address: %s' % address_kwargs) else: instance.addresses.add(address)
def _create_address(self, instance, type, address, postal_code, city, country, primary=False): """ Create an Address for given instance Arguments: instance (instance): instance to add Address to. type (str): kind of address address (str): street and number postal_code (str): postal code city (str): city country (str): country """ if address and city: street, number, complement = parse_address(address) address_kwargs = dict() address_kwargs['type'] = type address_kwargs['street'] = street # Set streetnumber. if is_int(number) and int(number) < 32766: address_kwargs['street_number'] = int(number) address_kwargs['complement'] = complement # Set postal code. if len(postal_code) > 10: postal_code = postal_code[:10] address_kwargs['postal_code'] = postal_code address_kwargs['city'] = city # Check if country is present. if country: country = country.upper().strip() country = self.country_map.get(country, country) # In case of unkown county leave empty. if country and len( country) > 2 or country not in self.country_codes: country = '' # Set country. address_kwargs['country'] = country # Set tenant. address_kwargs['tenant_id'] = self.tenant_pk # Check if adress does not already exists to avoid duplicates. if not instance.addresses.filter( street=address_kwargs['street'], city=address_kwargs['city']).exists(): try: address = Address.objects.create(**address_kwargs) except DataError: logger.warning('could not create address: %s' % address_kwargs) else: instance.addresses.add(address) else: address_kwargs.pop('tenant_id') instance.addresses.filter( street=address_kwargs['street'], city=address_kwargs['city']).update(**address_kwargs)
def set_view_output(self): """ Create a generic json format for account information based on the json from Dataprovider. """ phone_number_limit = 5 email_limit = 5 address_limit = 3 # Expected api output is json self.api_output = json.loads(self.api_output) # Return 404 when the api returned an error if self.api_output.get('error'): raise Http404() # Return error message when nothing was found if self.api_output.get('total') == 0: raise Exception( _('I\'m so sorry, I couldn\'t find any data for this website.') ) # Filter useful data result = self.api_output['data'][0] # Get company name company = result.get('company') # Get website description description = result.get('description') # Get the keywords and convert to list tags = result.get('keywords') if tags: tags = result.get('keywords').strip().rstrip(',').split(',') # Get email addresses and convert to a list if needed emails = result.get('emailaddresses', []) or [] if not isinstance(emails, list): emails = [emails] # Determine primary email since Dataprovider doesn't provide it primary_email = None if emails: primary_email = self._get_primary_email(emails) # Set primary email to the first in the list emails.index(primary_email) emails.remove(primary_email) emails.insert(0, primary_email) # Limit number of emails emails = emails[:email_limit] phone_numbers = [] # Get primary phone number and convert to a nicer representation phone_number = result.get('phonenumber') if phone_number: phone_number = parse_phone_number(phone_number) phone_numbers.append(phone_number) # Get phone numbers and convert to list if needed raw_phone_numbers = result.get('phonenumbers', []) or [] if not isinstance(raw_phone_numbers, list): raw_phone_numbers = [raw_phone_numbers] # Convert all phone numbers to a nicer representation for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Limit number of phonenumbers phone_numbers = phone_numbers[:phone_number_limit] # Get what kind of company it is (e.g. LLC) legalentity = result.get('legalentity') # Get the VAT (Value Added Tax) identifaction number taxnumber = result.get('taxnumber') # Get bank account number bankaccountnumber = result.get('bankaccountnumber') # Get the CoC (Chamber of Commerce) number cocnumber = result.get('cocnumber') # Get the IBAN (Internation Bank Account Number) iban = result.get('iban') # Get the BIC (Bank Identifier Code) bic = result.get('bic') # Try to parse the address address = result.get('address') if address: street, street_number, complement = parse_address(address) else: street, street_number, complement = None, None, None # Make the full address addresses = [] if address or result.get('city') or result.get( 'zipcode') or result.get('country'): addresses = [{ 'street': street, 'street_number': street_number, 'complement': complement, 'city': result.get('city'), 'country': result.get('country'), 'postal_code': result.get('zipcode'), }] addresses = addresses[:address_limit] # Build dict with account information self.view_output = { 'name': company, 'description': description, 'tags': tags, 'email_addresses': emails, 'primary_email': primary_email, 'phone_numbers': phone_numbers, 'phone_number': phone_number, 'addresses': addresses, 'legalentity': legalentity, 'taxnumber': taxnumber, 'bankaccountnumber': bankaccountnumber, 'cocnumber': cocnumber, 'iban': iban, 'bic': bic, } return json.dumps(self.view_output)
def set_view_output(self): """ Create a generic json format for account information based on the json from Dataprovider. """ # Expected api output is json self.api_output = anyjson.deserialize(self.api_output) # Return 404 when the api returned an error if self.api_output.get('error'): raise Http404() # Return error message when nothing was found if self.api_output.get('total') == 0: raise Exception(_('I\'m so sorry, I couldn\'t find any data for this website.')) # Filter useful data result = self.api_output['data'][0] # Get company name company = result.get('company') # Get website description description = result.get('description') # Get the keywords and convert to list tags = result.get('keywords') if tags: tags = result.get('keywords').strip().rstrip(',').split(',') # Get email addresses and convert to a list if needed emails = result.get('emailaddresses', []) or [] if not isinstance(emails, list): emails = [emails] # Determine primary email since Dataprovider doesn't provide it primary_email = None if emails: primary_email = self.get_primary_email(emails) # Get phone numbers and convert to list if needed raw_phone_numbers = result.get('phonenumbers', []) or [] if not isinstance(raw_phone_numbers, list): raw_phone_numbers = [raw_phone_numbers] # Convert all phone numbers to a nicer representation phone_numbers = [] for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Get primary phone number and convert to a nicer representation phone_number = result.get('phonenumber') if phone_number: phone_number = parse_phone_number(result.get('phonenumber')) # Get what kind of company it is (e.g. LLC) legalentity = result.get('legalentity') # Get the VAT (Value Added Tax) identifaction number taxnumber = result.get('taxnumber') # Get bank account number bankaccountnumber = result.get('bankaccountnumber') # Get the CoC (Chamber of Commerce) number cocnumber = result.get('cocnumber') # Get the IBAN (Internation Bank Account Number) iban = result.get('iban') # Get the BIC (Bank Identifier Code) bic = result.get('bic') # Try to parse the address address = result.get('address') if address: street, street_number, complement = parse_address(address) else: street, street_number, complement = None, None, None # Make the full address addresses = [] if address or result.get('city') or result.get('zipcode') or result.get('country'): addresses = [{ 'street': street, 'street_number': street_number, 'complement': complement, 'city': result.get('city'), 'country': result.get('country'), 'postal_code': result.get('zipcode'), }] # Build dict with account information self.view_output = { 'name': company, 'description': description, 'tags': tags, 'emails': emails, 'primary_email': primary_email, 'phone_numbers': phone_numbers, 'phone_number': phone_number, 'addresses': addresses, 'legalentity': legalentity, 'taxnumber': taxnumber, 'bankaccountnumber': bankaccountnumber, 'cocnumber': cocnumber, 'iban': iban, 'bic': bic, } return anyjson.serialize(self.view_output)
def _get_account_information(self, json): """ Create a dict with account information based on the json response from Dataprovider. """ phone_number_limit = 5 email_limit = 5 address_limit = 3 # Return 404 when the api returned an error. if json.get('error'): raise Http404(json.get('error')) if json.get('total') == 0: # No results from Dataprovider so return an empty list. return [] # Filter useful data. try: # Lookup API returns data as a dict. result = json['data'][0] except (KeyError, IndexError): # Enrich API returns a single hit. result = json['data'] if not result: # No results from Dataprovider so return an empty list. return [] # Get the keywords and convert to list. tags = result.get('keywords') if tags: tags = [tag.strip() for tag in tags.split(',')] tags = filter(None, tags) # Remove empty tags. # Get email addresses as a list. emails = list(result.get('emailaddresses', [])) # Determine primary email since Dataprovider doesn't provide it. primary_email = None if emails: primary_email = self._get_primary_email(emails) # Set primary email to the first in the list. emails.index(primary_email) emails.remove(primary_email) emails.insert(0, primary_email) phone_numbers = [] # Get primary phone number and convert to a nicer representation. phone_number = result.get('phonenumber') if phone_number: phone_number = parse_phone_number(phone_number) phone_numbers.append(phone_number) # Get phone numbers as a list. raw_phone_numbers = list(result.get('phonenumbers', [])) # Convert all phone numbers to a nicer representation. for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Try to parse the address. address = result.get('address') address_line = '' if address: # Construct address_line, instead of assigning address to address_line directly, # because parse_address() also santizes the result. street, street_number, complement = parse_address(address) if street: address_line = street if street_number: address_line += ' ' + street_number if complement: address_line += complement # Make the full address. addresses = [] if address or result.get('city') or result.get('zipcode') or result.get('country'): # There are some exceptions on the country codes, so make sure results of Dataprovider conform to the # ISO 3166-1 alpha-2 standard we use. mapping = { 'UK': 'GB', 'EL': 'GR', } country = mapping.get(result.get('country'), result.get('country')) if result.get('country') else '' addresses = [{ 'address': address_line, 'city': result.get('city', ''), 'country': country, 'postal_code': result.get('zipcode', ''), }] # Get social media profiles. social_profiles = result.get('socialprofiles') # Group profiles by platform. # Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest. social_media = {} if social_profiles: for profile in social_profiles: if profile.startswith('twitter.com/'): if 'twitter' not in social_media: social_media['twitter'] = [] social_media['twitter'].append(profile) elif profile.startswith('www.linkedin.com/in/'): if 'linkedin' not in social_media: social_media['linkedin'] = [] social_media['linkedin'].append(profile) primary_twitter = '' if 'twitter' in social_media: primary_twitter = self._get_primary_profile(social_media['twitter'], result.get('company')) primary_linkedin = '' if 'linkedin' in social_media: primary_linkedin = self._get_primary_profile(social_media['linkedin'], result.get('company')) description = result.get('description') website = result.get('hostname') # Build dict with account information. account_information = { 'name': result.get('company'), 'description': description, 'tags': tags, 'email_addresses': emails[:email_limit], 'primary_email': primary_email, 'phone_numbers': phone_numbers[:phone_number_limit], 'phone_number': phone_number, 'addresses': addresses[:address_limit], 'legalentity': result.get('legalentity'), 'taxnumber': result.get('taxnumber'), 'bankaccountnumber': result.get('bankaccountnumber'), 'cocnumber': result.get('cocnumber'), 'iban': result.get('iban'), 'bic': result.get('bic'), 'social_media_profiles': social_media, 'twitter': primary_twitter, 'linkedin': primary_linkedin, 'primary_website': website, } return account_information
def _get_account_information(self, api_output): """ Create a dict with account information based on the json response from Dataprovider. """ phone_number_limit = 5 email_limit = 5 address_limit = 3 # Expected API output is json. api_output_json = json.loads(api_output) # Return 404 when the api returned an error. if api_output_json.get('error'): raise Http404(api_output_json.get('error')) # Return error message when nothing was found. if api_output_json.get('total') == 0: raise APIException( _('I\'m so sorry, I couldn\'t find any data for this website.') ) # Filter useful data. result = api_output_json['data'][0] # Get the keywords and convert to list. tags = result.get('keywords') if tags: tags = result.get('keywords').strip().rstrip(',').split(',') # Get email addresses as a list. emails = list(result.get('emailaddresses', [])) # Determine primary email since Dataprovider doesn't provide it. primary_email = None if emails: primary_email = self._get_primary_email(emails) # Set primary email to the first in the list. emails.index(primary_email) emails.remove(primary_email) emails.insert(0, primary_email) phone_numbers = [] # Get primary phone number and convert to a nicer representation. phone_number = result.get('phonenumber') if phone_number: phone_number = parse_phone_number(phone_number) phone_numbers.append(phone_number) # Get phone numbers as a list. raw_phone_numbers = list(result.get('phonenumbers', [])) # Convert all phone numbers to a nicer representation. for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Try to parse the address. address = result.get('address') address_line = '' if address: # Construct address_line, instead of assigning address to address_line directly, # because parse_address() also santizes the result. street, street_number, complement = parse_address(address) if street: address_line = street if street_number: address_line += ' ' + street_number if complement: address_line += complement # Make the full address. addresses = [] if address or result.get('city') or result.get( 'zipcode') or result.get('country'): addresses = [{ 'address': address_line, 'city': result.get('city'), 'country': result.get('country'), 'postal_code': result.get('zipcode'), }] # Get social media profiles. social_profiles = result.get('socialprofiles') # Group profiles by platform. # Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest. social_media = {} for profile in social_profiles: if profile.startswith('twitter.com/'): if 'twitter' not in social_media: social_media['twitter'] = [] social_media['twitter'].append(profile) elif profile.startswith('www.linkedin.com/in/'): if 'linkedin' not in social_media: social_media['linkedin'] = [] social_media['linkedin'].append(profile) primary_twitter = '' if 'twitter' in social_media: primary_twitter = self._get_primary_profile( social_media['twitter'], result.get('company')) primary_linkedin = '' if 'linkedin' in social_media: primary_linkedin = self._get_primary_profile( social_media['linkedin'], result.get('company')) # Build dict with account information. account_information = { 'name': result.get('company'), 'description': result.get('description'), 'tags': tags, 'email_addresses': emails[:email_limit], 'primary_email': primary_email, 'phone_numbers': phone_numbers[:phone_number_limit], 'phone_number': phone_number, 'addresses': addresses[:address_limit], 'legalentity': result.get('legalentity'), 'taxnumber': result.get('taxnumber'), 'bankaccountnumber': result.get('bankaccountnumber'), 'cocnumber': result.get('cocnumber'), 'iban': result.get('iban'), 'bic': result.get('bic'), 'social_media_profiles': social_media, 'twitter': primary_twitter, 'linkedin': primary_linkedin, } return account_information
def set_view_output(self): """ Create a generic json format for account information based on the json from Dataprovider. """ phone_number_limit = 5 email_limit = 5 address_limit = 3 # Expected api output is json self.api_output = json.loads(self.api_output) # Return 404 when the api returned an error if self.api_output.get("error"): raise Http404() # Return error message when nothing was found if self.api_output.get("total") == 0: raise Exception(_("I'm so sorry, I couldn't find any data for this website.")) # Filter useful data result = self.api_output["data"][0] # Get company name company = result.get("company") # Get website description description = result.get("description") # Get the keywords and convert to list tags = result.get("keywords") if tags: tags = result.get("keywords").strip().rstrip(",").split(",") # Get email addresses and convert to a list if needed emails = result.get("emailaddresses", []) or [] if not isinstance(emails, list): emails = [emails] # Determine primary email since Dataprovider doesn't provide it primary_email = None if emails: primary_email = self._get_primary_email(emails) # Set primary email to the first in the list emails.index(primary_email) emails.remove(primary_email) emails.insert(0, primary_email) # Limit number of emails emails = emails[:email_limit] phone_numbers = [] # Get primary phone number and convert to a nicer representation phone_number = result.get("phonenumber") if phone_number: phone_number = parse_phone_number(phone_number) phone_numbers.append(phone_number) # Get phone numbers and convert to list if needed raw_phone_numbers = result.get("phonenumbers", []) or [] if not isinstance(raw_phone_numbers, list): raw_phone_numbers = [raw_phone_numbers] # Convert all phone numbers to a nicer representation for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Limit number of phonenumbers phone_numbers = phone_numbers[:phone_number_limit] # Get what kind of company it is (e.g. LLC) legalentity = result.get("legalentity") # Get the VAT (Value Added Tax) identifaction number taxnumber = result.get("taxnumber") # Get bank account number bankaccountnumber = result.get("bankaccountnumber") # Get the CoC (Chamber of Commerce) number cocnumber = result.get("cocnumber") # Get the IBAN (Internation Bank Account Number) iban = result.get("iban") # Get the BIC (Bank Identifier Code) bic = result.get("bic") # Try to parse the address address = result.get("address") if address: street, street_number, complement = parse_address(address) else: street, street_number, complement = None, None, None # Make the full address addresses = [] if address or result.get("city") or result.get("zipcode") or result.get("country"): addresses = [ { "street": street, "street_number": street_number, "complement": complement, "city": result.get("city"), "country": result.get("country"), "postal_code": result.get("zipcode"), } ] addresses = addresses[:address_limit] # Build dict with account information self.view_output = { "name": company, "description": description, "tags": tags, "email_addresses": emails, "primary_email": primary_email, "phone_numbers": phone_numbers, "phone_number": phone_number, "addresses": addresses, "legalentity": legalentity, "taxnumber": taxnumber, "bankaccountnumber": bankaccountnumber, "cocnumber": cocnumber, "iban": iban, "bic": bic, } return json.dumps(self.view_output)
def _get_account_information(self, api_output): """ Create a dict with account information based on the json response from Dataprovider. """ phone_number_limit = 5 email_limit = 5 address_limit = 3 # Expected API output is json. api_output_json = json.loads(api_output) # Return 404 when the api returned an error. if api_output_json.get('error'): raise Http404(api_output_json.get('error')) # Return error message when nothing was found. if api_output_json.get('total') == 0: raise APIException(_('I\'m so sorry, I couldn\'t find any data for this website.')) # Filter useful data. result = api_output_json['data'][0] # Get the keywords and convert to list. tags = result.get('keywords') if tags: tags = result.get('keywords').strip().rstrip(',').split(',') # Get email addresses as a list. emails = list(result.get('emailaddresses', [])) # Determine primary email since Dataprovider doesn't provide it. primary_email = None if emails: primary_email = self._get_primary_email(emails) # Set primary email to the first in the list. emails.index(primary_email) emails.remove(primary_email) emails.insert(0, primary_email) phone_numbers = [] # Get primary phone number and convert to a nicer representation. phone_number = result.get('phonenumber') if phone_number: phone_number = parse_phone_number(phone_number) phone_numbers.append(phone_number) # Get phone numbers as a list. raw_phone_numbers = list(result.get('phonenumbers', [])) # Convert all phone numbers to a nicer representation. for raw_phone_number in raw_phone_numbers: phone_numbers.append(parse_phone_number(raw_phone_number)) # Try to parse the address. address = result.get('address') address_line = '' if address: # Construct address_line, instead of assigning address to address_line directly, # because parse_address() also santizes the result. street, street_number, complement = parse_address(address) if street: address_line = street if street_number: address_line += ' ' + street_number if complement: address_line += complement # Make the full address. addresses = [] if address or result.get('city') or result.get('zipcode') or result.get('country'): addresses = [{ 'address': address_line, 'city': result.get('city'), 'country': result.get('country'), 'postal_code': result.get('zipcode'), }] # Get social media profiles. social_profiles = result.get('socialprofiles') # Group profiles by platform. # Disregards the other platforms provided by Dataprovider: Facebook, Google Plus and Pinterest. social_media = {} for profile in social_profiles: if profile.startswith('twitter.com/'): if 'twitter' not in social_media: social_media['twitter'] = [] social_media['twitter'].append(profile) elif profile.startswith('www.linkedin.com/in/'): if 'linkedin' not in social_media: social_media['linkedin'] = [] social_media['linkedin'].append(profile) primary_twitter = '' if 'twitter' in social_media: primary_twitter = self._get_primary_profile(social_media['twitter'], result.get('company')) primary_linkedin = '' if 'linkedin' in social_media: primary_linkedin = self._get_primary_profile(social_media['linkedin'], result.get('company')) # Build dict with account information. account_information = { 'name': result.get('company'), 'description': result.get('description'), 'tags': tags, 'email_addresses': emails[:email_limit], 'primary_email': primary_email, 'phone_numbers': phone_numbers[:phone_number_limit], 'phone_number': phone_number, 'addresses': addresses[:address_limit], 'legalentity': result.get('legalentity'), 'taxnumber': result.get('taxnumber'), 'bankaccountnumber': result.get('bankaccountnumber'), 'cocnumber': result.get('cocnumber'), 'iban': result.get('iban'), 'bic': result.get('bic'), 'social_media_profiles': social_media, 'twitter': primary_twitter, 'linkedin': primary_linkedin, } return account_information