def __init__(self, *args):
self.gender_attr = generator.GenerateFuncAttribute(
attribute_name='gender',
function=attrgenfunct.gender,
parameters=[str('Female')])
self.name_prefix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-prefix',
function = attrgenfunct.generate_name_prefix_f)
self.gname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'given-name',
freq_file_name = os.path.abspath('lookup_files/firstname_female.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.mname_attr =\
generator.GenerateFreqAttribute(attribute_name = 'middle-name',
freq_file_name = os.path.abspath('lookup_files/firstname_female.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.name_suffix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-suffix',
function = attrgenfunct.generate_name_suffix)
# Calculating age off of frequency distribution of age. Currently referencing female file
# Male csv file also exists once we can get the age generated based on gender
self.new_age_attr = \
generator.GenerateFreqAlt(attribute_name = 'age-new',
freq_file_name = os.path.abspath('lookup_files/age_gender_ratio_female.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
def __init__(self):
self.gender_attr = generator.GenerateFuncAttribute(
attribute_name='gender',
function=attrgenfunct.gender,
parameters=[str('Male')])
self.gname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'given-name',
freq_file_name = os.path.abspath('lookup_files/firstname_male.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.mname_attr =\
generator.GenerateFreqAttribute(attribute_name = 'middle-name',
freq_file_name = os.path.abspath('lookup_files/firstname_male.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.name_suffix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-suffix',
function = attrgenfunct.generate_name_suffix)
self.sname_prev_attr = \
generator.GenerateFuncAttribute(attribute_name = 'previous-surname',
function = attrgenfunct.generate_surname_m)
self.name_prefix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-prefix',
function = attrgenfunct.generate_name_prefix_m)
self.sname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = os.path.abspath('lookup_files/lastname.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.new_age_attr = \
generator.GenerateFreqAlt(attribute_name = 'age-new',
freq_file_name = os.path.abspath('lookup_files/age_gender_ratio_male.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
sname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = 'lookup_files/surname-freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
postcode_attr = \
generator.GenerateFreqAttribute(attribute_name = 'postcode',
freq_file_name = 'lookup_files/postcode_act_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
#Cell
phone_num_cell_attr = \
generator.GenerateFuncAttribute(attribute_name = 'cell-number',
function = attrgenfunct.generate_phone_number_american)
#Work
phone_num_work_attr = \
generator.GenerateFuncAttribute(attribute_name = 'work-number',
function = attrgenfunct.generate_phone_number_australia)
#Home
phone_num_home_attr = \
generator.GenerateFuncAttribute(attribute_name = 'home-number',
function = attrgenfunct.generate_phone_number_american)
credit_card_attr = \
generator.GenerateFuncAttribute(attribute_name = 'credit-card-number',
function = attrgenfunct.generate_credit_card_number)
social_security_attr = \
unicode_encoding = unicode_encoding_used)
sname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = 'lookup-files/surname-freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
postcode_attr = \
generator.GenerateFreqAttribute(attribute_name = 'postcode',
freq_file_name = 'lookup-files/postcode_act_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
phone_num_attr = \
generator.GenerateFuncAttribute(attribute_name = 'telephone-number',
function = attrgenfunct.generate_phone_number_australia)
age_uniform_attr = \
generator.GenerateFuncAttribute(attribute_name = 'age-uniform',
function = attrgenfunct.generate_uniform_age,
parameters = [0,120])
#thiago
birth_date_uniform_attr = \
generator.GenerateFuncAttribute(attribute_name = 'birth',
function = attrgenfunct.generate_normal_date,
parameters = [1,12,1904,2000])
death_date_uniform_attr = \
generator.GenerateFuncAttribute(attribute_name = 'death',
function = attrgenfunct.generate_normal_date,
def testDataGeneration(self, test_case):
"""Test the overall generation of a data set according to the parameters
given by checking if the generated data sets follows the parameter
specification given.
"""
rec_id_attr_name = test_case[0]
num_org_rec = test_case[1]
num_dup_rec = test_case[2]
max_duplicate_per_record = test_case[3]
num_duplicates_distribution = test_case[4]
max_modification_per_attr = test_case[5]
num_modification_per_record = test_case[6]
test_res_list = ['', 'Test case parameters:']
test_res_list.append(' rec_id_attr_name = %s' % (rec_id_attr_name))
test_res_list.append(' num_org_rec = %s' % (num_org_rec))
test_res_list.append(' num_dup_rec = %s' % (num_dup_rec))
test_res_list.append(' max_duplicate_per_record = %s' % \
(max_duplicate_per_record))
test_res_list.append(' num_duplicates_distribution = %s' % \
(num_duplicates_distribution))
test_res_list.append(' max_modification_per_attr = %s' % \
(max_modification_per_attr))
test_res_list.append(' num_modification_per_record = %s' % \
(num_modification_per_record))
test_res_list.append('')
# Define the attributes to be generated (based on methods from - - - - -
# the generator.py module)
# Individual attributes
#
given_name_attr = \
generator.GenerateFreqAttribute(attribute_name = 'given-name',
freq_file_name = '../lookup-files/givenname_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
surnname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = '../lookup-files/surname-freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
postcode_attr = \
generator.GenerateFreqAttribute(attribute_name = 'postcode',
freq_file_name = '../lookup-files/postcode_act_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
oz_phone_num_attr = \
generator.GenerateFuncAttribute(attribute_name = 'oz-phone-number',
function = attrgenfunct.generate_phone_number_australia)
credit_card_attr = \
generator.GenerateFuncAttribute(attribute_name = 'credit-card-number',
function = attrgenfunct.generate_credit_card_number)
age_uniform_attr = \
generator.GenerateFuncAttribute(attribute_name = 'age-uniform',
function = attrgenfunct.generate_uniform_age,
parameters = [0,120])
age_death_normal_attr = \
generator.GenerateFuncAttribute(attribute_name = 'age-death-normal',
function = attrgenfunct.generate_normal_age,
parameters = [80,20,0,120])
income_normal_attr = \
generator.GenerateFuncAttribute(attribute_name = 'income-normal',
function = attrgenfunct.generate_normal_value,
parameters = [75000, 20000, 0, 1000000, 'float2'])
rating_normal_attr = \
generator.GenerateFuncAttribute(attribute_name = 'rating-normal',
function = attrgenfunct.generate_normal_value,
parameters = [2.5, 1.0, 0.0, 5.0, 'int'])
# Compund (dependent) attributes
#
gender_city_comp_attr = \
generator.GenerateCateCateCompoundAttribute(\
categorical1_attribute_name = 'gender',
categorical2_attribute_name = 'city',
lookup_file_name = '../lookup-files/gender-city.csv',
has_header_line = True,
unicode_encoding = unicode_encoding_used)
gender_income_comp_attr = \
generator.GenerateCateContCompoundAttribute(\
categorical_attribute_name = 'alt-gender',
continuous_attribute_name = 'income',
continuous_value_type = 'float1',
lookup_file_name = '../lookup-files/gender-income.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
gender_city_salary_comp_attr = \
generator.GenerateCateCateContCompoundAttribute(\
categorical1_attribute_name = 'alt-gender-2',
categorical2_attribute_name = 'town',
continuous_attribute_name = 'salary',
continuous_value_type = 'float4',
lookup_file_name = \
'../lookup-files/gender-city-income.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
age_blood_pressure_comp_attr = \
generator.GenerateContContCompoundAttribute(\
continuous1_attribute_name = 'medical-age',
continuous2_attribute_name = 'blood-pressure',
continuous1_funct_name = 'uniform',
continuous1_funct_param = [10,110],
continuous2_function = \
contdepfunct.blood_pressure_depending_on_age,
continuous1_value_type = 'int',
continuous2_value_type = 'float3')
age_salary_comp_attr = \
generator.GenerateContContCompoundAttribute(\
continuous1_attribute_name = 'medical-age-2',
continuous2_attribute_name = 'medical-salary',
continuous1_funct_name = 'normal',
continuous1_funct_param = [45,20,25,130],
continuous2_function = \
contdepfunct.salary_depending_on_age,
continuous1_value_type = 'int',
continuous2_value_type = 'float1')
# Define how attribute values are to be modified (corrupted) - - - - - -
# (based on methods from the corruptor.py module)
#
average_edit_corruptor = \
corruptor.CorruptValueEdit(\
position_function = corruptor.position_mod_normal,
char_set_funct = basefunctions.char_set_ascii,
insert_prob = 0.25,
delete_prob = 0.25,
substitute_prob = 0.25,
transpose_prob = 0.25)
sub_tra_edit_corruptor = \
corruptor.CorruptValueEdit(\
position_function = corruptor.position_mod_uniform,
char_set_funct = basefunctions.char_set_ascii,
insert_prob = 0.0,
delete_prob = 0.0,
substitute_prob = 0.5,
transpose_prob = 0.5)
ins_del_edit_corruptor = \
corruptor.CorruptValueEdit(\
position_function = corruptor.position_mod_normal,
char_set_funct = basefunctions.char_set_ascii,
insert_prob = 0.5,
delete_prob = 0.5,
substitute_prob = 0.0,
transpose_prob = 0.0)
surname_misspell_corruptor = \
corruptor.CorruptCategoricalValue(\
lookup_file_name = '../lookup-files/surname-misspell.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
ocr_corruptor = corruptor.CorruptValueOCR(\
position_function = corruptor.position_mod_uniform,
lookup_file_name = '../lookup-files/ocr-variations.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
keyboard_corruptor = corruptor.CorruptValueKeyboard(\
position_function = corruptor.position_mod_normal,
row_prob = 0.5,
col_prob = 0.5)
phonetic_corruptor = corruptor.CorruptValuePhonetic(\
position_function = corruptor.position_mod_uniform,
lookup_file_name = \
'../lookup-files/phonetic-variations.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
missing_val_empty_corruptor = corruptor.CorruptMissingValue()
missing_val_miss_corruptor = corruptor.CorruptMissingValue(\
missing_value='miss')
missing_val_unkown_corruptor = corruptor.CorruptMissingValue(\
missing_value='unknown')
# Define the attributes to be generated for this data set, and the data
# set itself
#
attr_name_list = ['given-name', 'surname', 'city', 'postcode',
'oz-phone-number', 'credit-card-number', 'age-uniform',
'age-death-normal', 'income-normal', 'rating-normal',
'gender', 'alt-gender', 'alt-gender-2', 'town',
'income', 'salary', 'medical-age', 'blood-pressure',
'medical-age-2', 'medical-salary']
attr_data_list = [given_name_attr, surnname_attr, postcode_attr,
oz_phone_num_attr, credit_card_attr, age_uniform_attr,
age_death_normal_attr, income_normal_attr,
rating_normal_attr, gender_city_comp_attr,
gender_income_comp_attr, gender_city_salary_comp_attr,
age_blood_pressure_comp_attr, age_salary_comp_attr]
# Initialise the main data generator
#
test_data_generator = generator.GenerateDataSet(\
output_file_name = 'no-file-name',
write_header_line = True,
rec_id_attr_name = rec_id_attr_name,
number_of_records = num_org_rec,
attribute_name_list = attr_name_list,
attribute_data_list = attr_data_list,
unicode_encoding = unicode_encoding_used)
# Define distribution of how likely an attribute will be selected for
# modification (sum of probabilities must be 1.0)
#
attr_mod_prob_dictionary = {'given-name':0.1, 'surname':0.1,
'city':0.1, 'postcode':0.1,
'oz-phone-number':0.1,
'age-death-normal':0.1,
'income-normal':0.1,'gender':0.1, 'town':0.1,
'income':0.1}
# For each attribute, a distribution of which corruptors to apply needs
# to be given, with the sum ofprobabilities to be 1.0 for each attribute
#
attr_mod_data_dictionary = \
{'given-name':[(0.25, average_edit_corruptor),
(0.25, ocr_corruptor),
(0.25, phonetic_corruptor),
(0.25, missing_val_miss_corruptor)],
'surname':[(0.5, surname_misspell_corruptor),
(0.5, average_edit_corruptor)],
'city':[(0.5, keyboard_corruptor),
(0.5, missing_val_empty_corruptor)],
'postcode':[(0.3, missing_val_unkown_corruptor),
(0.7, sub_tra_edit_corruptor)],
'oz-phone-number':[(0.2, missing_val_empty_corruptor),
(0.4, sub_tra_edit_corruptor),
(0.4, keyboard_corruptor)],
'age-death-normal':[(1.0, missing_val_unkown_corruptor)],
'income-normal':[(0.3, keyboard_corruptor),
(0.3, ocr_corruptor),
(0.4, missing_val_empty_corruptor)],
'gender':[(0.5, sub_tra_edit_corruptor),
(0.5, ocr_corruptor)],
'town':[(0.2, average_edit_corruptor),
(0.3, ocr_corruptor),
(0.2, keyboard_corruptor),
(0.3, phonetic_corruptor)],
'income':[(1.0, missing_val_miss_corruptor)]}
# Initialise the main data corruptor
#
test_data_corruptor = corruptor.CorruptDataSet(\
number_of_org_records = num_org_rec,
number_of_mod_records = num_dup_rec,
attribute_name_list = attr_name_list,
max_num_dup_per_rec = max_duplicate_per_record,
num_dup_dist = num_duplicates_distribution,
max_num_mod_per_attr = max_modification_per_attr,
num_mod_per_rec = num_modification_per_record,
attr_mod_prob_dict = attr_mod_prob_dictionary,
attr_mod_data_dict = attr_mod_data_dictionary)
passed = True # Assume the test will pass :-)
# Start the generation process
#
try:
rec_dict = test_data_generator.generate()
except Exception as exce_value: # Something bad happened
test_res_list.append(' generator.generate() raised Exception: "%s"' % \
(str(exce_value)))
return test_res_list # Abandon test
num_org_rec_gen = len(rec_dict)
if (num_org_rec_gen != num_org_rec):
passed = False
test_res_list.append(' Wrong number of original records generated:' + \
' %d, expected %d' % (num_org_rec_gen,num_org_rec))
# Corrupt (modify) the original records into duplicate records
#
try:
rec_dict = test_data_corruptor.corrupt_records(rec_dict)
except Exception as exce_value: # Something bad happened
test_res_list.append(' corruptor.corrupt_records() raised ' + \
'Exception: "%s"' % (str(exce_value)))
return test_res_list # Abandon test
num_dup_rec_gen = len(rec_dict)-num_org_rec_gen
if (num_dup_rec_gen != num_dup_rec):
passed = False
test_res_list.append(' Wrong number of duplicate records generated:' + \
' %d, expected %d' % (num_dup_rec_gen,num_dup_rec))
num_dup_counts = {} # Count how many records have a certain number of
# duplicates
# Do tests on all generated records
#
for (rec_id,rec_list) in rec_dict.iteritems():
if (len(rec_list) != len(attr_name_list)):
passed = False
test_res_list.append(' Record with identifier "%s" contains wrong' % \
(rec_id) + ' number of attributes: ' + \
' %d, expected %d' % (len(rec_list),
len(attr_name_list)))
if ('org' in rec_id): # An original record
# Check the number of duplicates for this record is what is expected
#
num_dups = 0
rec_num = rec_id.split('-')[1]
for d in range(max_duplicate_per_record*2):
tmp_rec_id = 'rec-%s-dup-%d' % (rec_num,d)
if tmp_rec_id in rec_dict:
num_dups += 1
if (num_dups > max_duplicate_per_record):
passed = False
test_res_list.append(' Too many duplicate records for original' + \
' record "%s": %d' % (rec_id), num_dups)
d_count = num_dup_counts.get(num_dups, 0) + 1
num_dup_counts[num_dups] = d_count
# Check no duplicate number is outside expected range
#
for d in range(max_duplicate_per_record,max_duplicate_per_record*2):
tmp_rec_id = 'rec-%s-dup-%d' % (rec_num,d)
if (tmp_rec_id in rec_dict):
passed = False
test_res_list.append(' Illegal duplicate number: %s' % \
(tmp_rec_id)+' (larger than max. number ' + \
'of duplicates per record %sd' % \
(max_duplicate_per_record))
# Check values in certain attributes only contain letters
#
for i in [0,1,2,10,11,12,13]:
test_val = rec_list[i].replace(' ','')
test_val = test_val.replace('-','')
test_val = test_val.replace("'",'')
if (test_val.isalpha() == False):
passed = False
test_res_list.append(' Value in attribute "%s" is not only ' % \
(attr_name_list[i]) + 'letters:')
test_res_list.append(' Org: %s' % (str(rec_list)))
# Check values in certain attributes only contain digits
#
for i in [3,4,5,6,7,8,9,14,15,16,17,18,19]:
test_val = rec_list[i].replace(' ','')
test_val = test_val.replace('.','')
if (test_val.isdigit() == False):
passed = False
test_res_list.append(' Value in attribute "%s" is not only ' % \
(attr_name_list[i]) + 'digits:')
test_res_list.append(' Org: %s' % (str(rec_list)))
# Check age values are in range
#
for i in [6,7,16]:
test_val = int(rec_list[i].strip())
if ((test_val < 0) or (test_val > 130)):
passed = False
test_res_list.append(' Age value in attribute "%s" is out of' % \
(attr_name_list[i]) + ' range:')
test_res_list.append(' Org: %s' % (str(rec_list)))
# Check length of postcode, telephone and credit card numbers
#
if (len(rec_list[3]) != 4):
passed = False
test_res_list.append(' Postcode has not 4 digits:')
test_res_list.append(' Org: %s' % (str(rec_list)))
if ((len(rec_list[4]) != 12) or (rec_list[4][0] != '0')):
passed = False
test_res_list.append(' Australian phone number has wrong format:')
test_res_list.append(' Org: %s' % (str(rec_list)))
# Check 'rating' is between 0 and 5
#
test_val = int(rec_list[9].strip())
if ((test_val < 0) or (test_val > 5)):
passed = False
test_res_list.append(' "rating-normal" value is out of range:')
test_res_list.append(' Org: %s' % (str(rec_list)))
# Check gender values
#
test_val = rec_list[10]
if (test_val not in ['male','female']):
passed = False
test_res_list.append(' "gender" value is out of range:')
test_res_list.append(' Org: %s' % (str(rec_list)))
test_val = rec_list[11]
if (test_val not in ['m','f','na']):
passed = False
test_res_list.append(' "alt-gender" value is out of range:')
test_res_list.append(' Org: %s' % (str(rec_list)))
test_val = rec_list[12]
if (test_val not in ['male','female']):
passed = False
test_res_list.append(' "alt-gender-2" value is out of range:')
test_res_list.append(' Org: %s' % (str(rec_list)))
if ('dup' in rec_id): # A duplicate record
# Get the corresponding original record
#
org_rec_id = 'rec-%s-org' % (rec_id.split('-')[1])
org_rec_list = rec_dict[org_rec_id]
# Check the duplicate number
#
dup_num = int(rec_id.split('-')[-1])
if ((dup_num < 0) or (dup_num > max_duplicate_per_record-1)):
passed = False
test_res_list.append(' Duplicate record with identifier "%s" ' % \
(rec_id) + ' has an illegal duplicate number:' \
+ ' %d' % (dup_num))
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
# Check that a duplicate record contains the expected - - - - - - - - -
# number of modifications
num_diff_val = 0 # Count how many values are different
for i in range(len(rec_list)): # Check all attribute values
if (rec_list[i] != org_rec_list[i]):
num_diff_val += 1
if (num_diff_val == 0): # No differences between org and dup record
passed = False
test_res_list.append(' Duplicate record with identifier "%s" ' % \
(rec_id) + 'is the same as it original record')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
if (num_diff_val < num_modification_per_record):
passed = False
test_res_list.append(' Duplicate record with identifier "%s" ' % \
(rec_id) + 'contains less modifications ' + \
'than expected (%d instead of %d)' % \
(num_diff_val, num_modification_per_record))
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
# Check that certain attributes have not been modified
#
for i in [5,6,9,11,12,15,16,17,18,19]:
if (rec_list[i] != org_rec_list[i]):
passed = False
test_res_list.append(' Duplicate record with identifier "%s" ' % \
(rec_id) + 'contains modified attribute ' + \
'values that should not be modified')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
# Check the content of certain attribute values, and how they
# differ between original and duplicate records
#
# Due to the possibility thatmultiple modifications are applied on the
# same attribute these tests are limited
test_org_val = org_rec_list[2] # City
test_dup_val = rec_list[2]
if (test_dup_val != ''):
if (len(test_org_val) != len(test_dup_val)):
passed = False
test_res_list.append(' "city" values have different length:')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
test_org_val = org_rec_list[4] # Australian phone number
test_dup_val = rec_list[4]
if (test_dup_val != ''):
if (len(test_org_val) != len(test_dup_val)):
passed = False
test_res_list.append(' "oz-phone-number" values have different' + \
' length:')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
test_org_val = org_rec_list[7] # Age-death-normal
test_dup_val = rec_list[7]
if (test_dup_val != 'unknown'):
if (test_org_val != test_dup_val):
passed = False
test_res_list.append(' Wrong value for "age-death-normal":')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
test_org_val = org_rec_list[14] # Income
test_dup_val = rec_list[14]
if (test_dup_val != 'miss'):
if (test_org_val != test_dup_val):
passed = False
test_res_list.append(' Wrong value for "income":')
test_res_list.append(' Org: %s' % (str(org_rec_list)))
test_res_list.append(' Dup: %s' % (str(rec_list)))
test_res_list.append(' Distribution of duplicates: ("%s" expected)' % \
num_duplicates_distribution)
dup_keys = num_dup_counts.keys()
dup_keys.sort()
for d in dup_keys:
test_res_list.append(' %d: %d records' % (d, num_dup_counts[d]))
test_res_list.append('')
if (passed == True):
test_res_list.append(' All tests passed')
test_res_list.append('')
return test_res_list
# Check if the given the unicode encoding selected is valid.
#
basefunctions.check_unicode_encoding_exists(unicode_encoding_used)
# -----------------------------------------------------------------------------
# Define the attributes to be generated (using methods from the generator.py
# module).
#
surname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = 'lookup-files/surname-freq-japanese.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
credit_card_attr = \
generator.GenerateFuncAttribute(attribute_name = 'credit-card-number',
function = attrgenfunct.generate_credit_card_number)
age_normal_attr = \
generator.GenerateFuncAttribute(attribute_name = 'age',
function = attrgenfunct.generate_normal_age,
parameters = [45,30,0,130])
gender_city_comp_attr = \
generator.GenerateCateCateCompoundAttribute(\
categorical1_attribute_name = 'gender',
categorical2_attribute_name = 'city',
lookup_file_name = 'lookup-files/gender-city-japanese.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
# -----------------------------------------------------------------------------
unicode_encoding = unicode_encoding_used)
sname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = 'lookup-files/surname-freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
postcode_attr = \
generator.GenerateFreqAttribute(attribute_name = 'postcode',
freq_file_name = 'lookup-files/postcode_act_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
phone_num_attr = \
generator.GenerateFuncAttribute(attribute_name = 'telephone-number',
function = attrgenfunct.generate_phone_number_australia)
credit_card_attr = \
generator.GenerateFuncAttribute(attribute_name = 'credit-card-number',
function = attrgenfunct.generate_credit_card_number)
age_uniform_attr = \
generator.GenerateFuncAttribute(attribute_name = 'age-uniform',
function = attrgenfunct.generate_uniform_age,
parameters = [0,120])
income_normal_attr = \
generator.GenerateFuncAttribute(attribute_name = 'income-normal',
function = attrgenfunct.generate_normal_value,
parameters = [50000,20000, 0, 1000000, 'float2'])
gender_marital_status_comp_attr = \
generator.GenerateCateCateCompoundAttribute(\
categorical1_attribute_name = 'Gender',
categorical2_attribute_name = 'Marital Status',
lookup_file_name = 'lookup-files/gender-maritalstatus.csv',
has_header_line = True,
unicode_encoding = 'ascii')
race_attr = \
generator.GenerateFreqAttribute(attribute_name = 'Race',
freq_file_name = 'lookup-files/race-freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
date_of_birth_attr = \
generator.GenerateFuncAttribute(attribute_name = 'DOB',
function = attrgenfunct.generate_date_of_birth)
social_security_number_attr = \
generator.GenerateFuncAttribute(attribute_name = 'SSN',
function = attrgenfunct.generate_social_security_number)
sex_income_comp_attr = \
generator.GenerateCateContCompoundAttribute(\
categorical_attribute_name = 'Sex',
continuous_attribute_name = 'Income',
continuous_value_type = 'float1',
lookup_file_name = 'lookup-files/gender-income.csv',
has_header_line = False,
unicode_encoding = 'ascii')
credit_card_attr = \
# Set the number of modification that are to be applied to a record.
#
num_modification_per_record = 5
# Check if the given the unicode encoding selected is valid.
#
basefunctions.check_unicode_encoding_exists(unicode_encoding_used)
# -----------------------------------------------------------------------------
# Define the attributes to be generated (using methods from the generator.py
# module).
#
name_prefix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-prefix',
function = attrgenfunct.generate_name_prefix_f)
gname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'given-name',
freq_file_name = 'lookup_files/givenname_f_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
mname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'middle-name',
freq_file_name = 'lookup_files/givenname_f_freq.csv',
has_header_line = False,
unicode_encoding = unicode_encoding_used)
name_suffix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-suffix',
function = attrgenfunct.generate_name_suffix)
def __init__(self, *args):
super(AttrMeta, self).__init__(*args)
self.race_hispanic = \
generator.GenerateFreqAlt(attribute_name = 'race-hispanic',
freq_file_name = os.path.abspath('lookup_files/race_w_hispanic_ascii.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.sname_attr = \
generator.GenerateFreqAttribute(attribute_name = 'surname',
freq_file_name = os.path.abspath('lookup_files/lastname.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.name_suffix_attr = \
generator.GenerateFuncAttribute(attribute_name = 'name-suffix',
function = attrgenfunct.generate_name_suffix)
self.sname_prev_attr = \
generator.GenerateFreqAttribute(attribute_name = 'previous-surname',
freq_file_name = os.path.abspath('lookup_files/lastname.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.nickname_attr = \
generator.GenerateFuncAttribute(attribute_name = 'nickname',
function = attrgenfunct.generate_nickname)
self.postcode_attr = \
generator.GenerateFreqAttribute(attribute_name = 'postcode',
freq_file_name = os.path.abspath('lookup_files/postcode_ascii.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
#Cell
self.phone_num_cell_attr = \
generator.GenerateFuncAttribute(attribute_name = 'cell-number',
function = attrgenfunct.generate_phone_number_american)
#Work
self.phone_num_work_attr = \
generator.GenerateFuncAttribute(attribute_name = 'work-number',
function = attrgenfunct.generate_phone_number_american)
#Home
self.phone_num_home_attr = \
generator.GenerateFuncAttribute(attribute_name = 'home-number',
function = attrgenfunct.generate_phone_number_american)
self.credit_card_attr = \
generator.GenerateFuncAttribute(attribute_name = 'credit-card-number',
function = attrgenfunct.generate_credit_card_number)
self.social_security_attr = \
generator.GenerateFuncAttribute(attribute_name = 'social-security-number',
function = attrgenfunct.generate_social_security_number)
#passport
self.passport_attr = \
generator.GenerateFuncAttribute(attribute_name = 'passport-number',
function = attrgenfunct.generate_passport_num)
# Mother maiden name
self.mother = \
generator.GenerateFreqAttribute(attribute_name = 'mother-maiden-name',
freq_file_name = os.path.abspath('lookup_files/lastname.csv'),
has_header_line = False,
unicode_encoding = unicode_encoding_used)
self.address_attr = \
generator.GenerateFuncAttribute(attribute_name = 'street-address',
function = attrgenfunct.generate_address)
self.city_attr = \
generator.GenerateFuncAttribute(attribute_name = 'city',
function = attrgenfunct.generate_city)
self.state_attr = \
generator.GenerateFuncAttribute(attribute_name = 'state',
function = attrgenfunct.generate_state)
self.primary_ID_attr = \
generator.GenerateFuncAttribute(attribute_name = 'primary_key',
function = attrgenfunct.generate_primary)
def output_alt(self, *args, **kwargs):
'selective attribute output'
required = [
self.primary_ID_attr, self.gname_attr, self.mname_attr,
self.sname_attr, self.name_suffix_attr, self.name_prefix_attr,
self.sname_prev_attr, self.nickname_attr, self.new_age_attr,
self.gender_attr, self.address_attr, self.city_attr,
self.state_attr, self.postcode_attr
]
select_tup = self.AttrCheck(
self.primary_ID_attr, self.gname_attr, self.mname_attr,
self.sname_attr, self.name_suffix_attr, self.name_prefix_attr,
self.sname_prev_attr, self.nickname_attr, self.new_age_attr,
self.gender_attr, self.address_attr, self.city_attr,
self.state_attr, self.postcode_attr, self.phone_num_cell_attr,
self.phone_num_work_attr, self.phone_num_home_attr,
self.credit_card_attr, self.social_security_attr,
self.passport_attr, self.mother)
#tick will be the true or false tuple
#j will be the select_tup
#tick = self.AttrCheck(*args)
#select=(getattr(select_tup,x) for x in select_tup._fields if getattr(tick,x)==True)
select = (getattr(select_tup, y) for y in args)
out = OrderedDict((attr.attribute_name, attr.create_attribute_value())
for attr in select)
def attr_out_set(container, attr):
container[attr.attribute_name] = attr.create_attribute_value()
self.email_attr = generator.GenerateFuncAttribute(
attribute_name='email',
function=attrgenfunct.generate_email_address,
parameters=[str(out['given-name']),
str(out['surname'])])
self.DOB_attr = generator.GenerateFuncAttribute(
attribute_name='DOB',
function=attrgenfunct.generate_DOB,
parameters=[int(out['age-new'])])
r_h = self.race_hispanic.random_pick().split('..')
self.race = r_h[1]
self.hispanic = r_h[0]
self.race_attr = generator.GenerateFuncAttribute(
attribute_name='race',
function=attrgenfunct.race,
parameters=[str(self.race)])
self.hispanic_attr = generator.GenerateFuncAttribute(
attribute_name='hispanic',
function=attrgenfunct.hispanic,
parameters=[str(self.hispanic)])
self.marriage_attr = generator.GenerateFuncAttribute(
attribute_name='marital-status',
function=attrgenfunct.marriage,
parameters=[int(out['age-new'])])
dependent = [self.email_attr, self.DOB_attr, self.race_attr,\
self.hispanic_attr, self.marriage_attr]
for value in dependent:
attr_out_set(out, value)
return out
def output(self):
'create synthetic output'
#removed all compound attribute
#single attr need create_attribute_values(), singular
#compound attr need create_attribute_values(), plural!
#must update compound context to USA
primary = [
self.primary_ID_attr, self.gname_attr, self.mname_attr,
self.sname_attr, self.name_suffix_attr, self.name_prefix_attr,
self.sname_prev_attr, self.nickname_attr, self.new_age_attr,
self.gender_attr, self.address_attr, self.city_attr,
self.state_attr, self.postcode_attr, self.phone_num_cell_attr,
self.phone_num_work_attr, self.phone_num_home_attr,
self.credit_card_attr, self.social_security_attr,
self.passport_attr, self.mother
]
out = [attr.create_attribute_value() for attr in primary]
labels = [attr.attribute_name for attr in primary]
self.email_attr = generator.GenerateFuncAttribute(
attribute_name='email',
function=attrgenfunct.generate_email_address,
parameters=[str(out[1]), str(out[3])])
out.append(self.email_attr.create_attribute_value())
labels.append(self.email_attr.attribute_name)
self.DOB_attr = generator.GenerateFuncAttribute(
attribute_name='DOB',
function=attrgenfunct.generate_DOB,
parameters=[int(out[8])])
out.append(self.DOB_attr.create_attribute_value())
labels.append(self.DOB_attr.attribute_name)
r_h = self.race_hispanic.random_pick().split('..')
self.race = r_h[1]
self.hispanic = r_h[0]
self.race_attr = generator.GenerateFuncAttribute(
attribute_name='race',
function=attrgenfunct.race,
parameters=[str(self.race)])
out.append(self.race_attr.create_attribute_value())
labels.append(self.race_attr.attribute_name)
self.hispanic_attr = generator.GenerateFuncAttribute(
attribute_name='hispanic',
function=attrgenfunct.hispanic,
parameters=[str(self.hispanic)])
out.append(self.hispanic_attr.create_attribute_value())
labels.append(self.hispanic_attr.attribute_name)
self.marriage_attr = generator.GenerateFuncAttribute(
attribute_name='marital-status',
function=attrgenfunct.marriage,
parameters=[int(out[8])])
out.append(self.marriage_attr.create_attribute_value())
labels.append(self.marriage_attr.attribute_name)
outputwork2 = OrderedDict(zip(labels, out))
return outputwork2
