def us_person(residence, nationality):
"""
Function to check if person is US person (taxes)
Arguments
-- residence - is SMS.1 from customer
-- nationality - is PHONE.1 from customer
Return: true/false
"""
if ut.isnull(residence) and ut.isnull(nationality):
return False
return (residence == 'US' or nationality == 'US')
def national_register_number(val):
if ut.isnull(val):
global missed_tax_id_counter
missed_tax_id_counter +=1
return None
return str(fake.random_number(digits=11, fix_len=True))
def street_converter(val):
"""
Function converts free-hand address into street name, house and flat no
Arguments -- val: single cell
Return: street name, house no, flat no
Sample addresses to test
# 91 WATERLOO ROAD
# RUE NEUVE 82
# RUE FELIX DELHASSE 14 /TM
# GRNAD-ROUTE 22A/9
# AV DE 14 JUILLET 41
# AVENUE DES CITRONNIERS 57 BTE 3
# RES. LE 205 RUE GRANDE 205/1.3
# VIA SETTI CARRARO 15/F
# VIA NICOLA SCARANO 3 INT3
"""
if ut.isnull(val):
return None, None, None
house = None
apartment = None
# remove more than two spaces
val = re.sub("[ ]{2,}", " ", val).strip()
# BTE - flat number
val = re.sub(" BTE[^a-zA-Z]", "/", val)
val = re.sub(" BTE ", "/", val)
if len(re.findall(r"[0-9]{1,}BTE[^a-zA-Z]", val)) > 0:
repl_str = re.findall(r"[0-9]{1,}BTE", val)[-1]
val = val.replace(repl_str, repl_str.replace('BTE', "/"))
# remove spaces before and after house-apartment devider
val = re.sub(" /", "/", val)
val = re.sub("/ ", "/", val)
# split the address
street = val
addr = val.split(' ')
# just one no case
if len(addr) == 1:
return (fp.street_name(val), house, apartment) if IS_TEST_ENVIRONMENT else (val, house, apartment)
# most common cases - last contain a number
if re.search(r"[0-9]{1,}", addr[-1]) and len(re.search(r"[0-9]{1,}", addr[-1]).group()) > 0:
street = ' '.join(addr[:-1])
house = addr[-1]
splitted = addr[-1].split('/')
if len(splitted) > 1:
house = splitted[0]
apartment = splitted[1]
return (fp.street_name(val), fp.street_number(val), apartment) if IS_TEST_ENVIRONMENT else (val, house, apartment)
def pep(val):
"""
Convert pep value from origin Service "N", "Y", None to Boolean
Arguments -- val: single cell
Return: converted value #Tomasz Mierzwiński decission False as default
"""
if ut.isnull(val):
return False
return True if val == 'Y' else False
def to_lowercase(val):
"""
simply converts val to lower case - change object to str
Arguments -- val: single cell
Return: converted to str lowercase cell
"""
if ut.isnull(val):
return None
val = str(val)
return val.lower()
def float_converter(val):
"""
converts value from cell to float
Arguments -- val: single cell
Return: float value of cell or none
"""
if ut.isnull(val):
return None
try:
return np.float(val.replace(' ', '').replace(',', '.'))
except:
return None
def date_converter(val):
"""
converts date from R11 files in format YYYYMMDD to YYYY-MM-DD
Arguments -- val: single cell
Return: iso date as expected in API
"""
if ut.isnull(val):
return None
# simpler approach
# return f'{val[0:4]}-{val[4:6]}-{val[-2:]}'
try:
# this approach required date to be valied
return datetime.strptime(val, '%Y%m%d').date().isoformat()
except:
# TODO??? to be decided case with invalid date
return None
def valid_email(val):
"""
simple email validation - to consider using python validate_email - existence is possible
Arguments -- val: single cell
Return: 0 - not valied, 1 valied
"""
if ut.isnull(val):
return 0
#from validate_email import validate_email
# return validate_email(val,verify=True)
#for more than one we pick-up first
if len(val.split('|')) > 0 and len(val.split('|')[0]) > 0:
val = val.split('|')[0]
return 0 if not validators.email(val) else 1
def validate_birthday(val):
"""
For a belgium validate birthday and Registry number
In T24 R11 TAX.ID is unique and mandatory if the RESIDENCE is BE. A check is be made on the national register number in Belgium which is composed of 11 digits:
the first 6 positions form the date of birth in the opposite direction. For a person born on July 30, 1985, his first 6 numbers are 850730;
The following 3 positions constitute the daily birth counter. This figure is even for women and odd for men; • the last 2 positions constitute the check digit. This check digit is a sequence of 2 digits.
This number is the complement of 97 of the modulo 97 of the number formed:
- by the first 9 digits of the national number for persons born before 1 January 2000;
- by the number 2 followed by the first 9 digits of the national number for persons born after 31 December 1999.
Checks only Belgium customers
Arguments -- val: three piped cells - Country code | tax.id | birth_day
Return:
- True/False -
"""
if ut.isnull(val):
return False
val = str(val)
val = val.split('|')
if len(val) != 3:
return False
if val[0] != 'BE':
return True
if len(val) <3 :
return False
tax_id = val[1]
tax_id = re.sub(r"\D", "", tax_id)
if len(tax_id) != 11:
return False
if val[2][0] == '2':
tax_id = '2'+tax_id
check_num, val_num = int(tax_id[-2:]), int(tax_id[:-2])
belgium_modulo_number = 97
return (belgium_modulo_number - val_num % belgium_modulo_number) == check_num
def phone_segmenter(val):
"""
Function converts phone number into "semantic" valied, prefix, phone number, if possible
Arguments -- val: single cell
Return:
- True/False - if sematic is correct (corret digits number)
- prefix - eg 48
- phone number without prefix
- True/False - phone possible - for the country
"""
if ut.isnull(val):
return False, DEFAULT_PHONE_PREFIX, DEFAULT_PHONE, False
if str(val).find(DEFAULT_PHONE) > -1:
return False, DEFAULT_PHONE_PREFIX, DEFAULT_PHONE, False
try:
num = phn.parse('+'+val, None)
return True, str(num.country_code), str(num.national_number), phn.is_possible_number(num)
except:
return False, DEFAULT_PHONE_PREFIX, DEFAULT_PHONE, False
def document_id(val):
return None if ut.isnull(val) else str(fake.random_number(digits=8, fix_len=True))
def last_name(val):
return None if ut.isnull(val) else fake.last_name()
def first_name(val):
"""Faker with logic related to fake name for female if dict val is FEMALE"""
if ut.isnull(val):
return None
return fake.first_name_female() if val == 'FEMALE' else fake.first_name_male()
def postalcode(val):
return None if ut.isnull(val) else fake.postcode()
def apartment(val):
return None if ut.isnull(val) else fake.building_number()
def street_name(val):
return None if ut.isnull(val) else fake.street_name()
def street_number(val):
return None if ut.isnull(val) else fake.building_number()
def future_date(val):
return None if ut.isnull(val) else fake.future_date().isoformat()
def city(val):
return None if ut.isnull(val) else fake.city()
def past_date(val):
return None if ut.isnull(val) else fake.past_date().isoformat()
def date_of_birth(val):
return None if ut.isnull(val) else fake.date_of_birth().isoformat()
def company(val):
return None if ut.isnull(val) else fake.company()