not_letters_or_digits = unicode(string.punctuation)
translate_table = dict((ord(char), translate_to)
for char in not_letters_or_digits)
return to_translate.translate(translate_table)
def strip_punc(s):
s_out = s.translate(string.maketrans("",""), string.punctuation)
return s_out
#Function names below are not exact
N = len(names)
t0 = time.time()
clean_names = [psCleanup.rem_diacritics(n) for n in names]
clean_names = [psCleanup.rem_trail_spaces(n) for n in clean_names]
clean_names = [psCleanup.stdize_case(n) for n in clean_names]
clean_names = [translate_non_alphanumerics(n) for n in clean_names]
clean_names = psCleanup.master_clean_dicts(clean_names, all_dicts)
clean_names = [n.strip() for n in clean_names]
t1 = time.time()
### Works out to ~ 0.05s / entry
clean_time = t1 - t0
print clean_time / N
## Then pre-cluster by the leading 3 characters of the name
t0 = time.time()
leading_ngram_dict = psDisambig.build_leading_ngram_dict(clean_names, leading_n=3)
t1 = time.time()
leading_ngram_time = t1 - t0
print leading_ngram_time / N
return to_translate.translate(translate_table)
def strip_punc(s):
s_out = s.translate(string.maketrans("", ""), string.punctuation)
return s_out
#Function names below are not exact
N = len(names)
t0 = time.time()
clean_names = [psCleanup.rem_diacritics(n) for n in names]
clean_names = [psCleanup.rem_trail_spaces(n) for n in clean_names]
clean_names = [psCleanup.stdize_case(n) for n in clean_names]
clean_names = [translate_non_alphanumerics(n) for n in clean_names]
clean_names = psCleanup.master_clean_dicts(clean_names, all_dicts)
clean_names = [re.sub(' ', '', n) for n in clean_names]
t1 = time.time()
## Define some blocking functions
def block_by_2_ngrams(name_string, ngram_length=2):
block_dict = {}
for name in name_string:
these_ngrams = set([
''.join(name[j] for j in range(i, i + ngram_length))
for i in range((len(name) - ngram_length))
])
ngram_combos = get_combinatorial_pairs(list(these_ngrams))
for ngram in ngram_combos:
FROM tls206_person INNER JOIN tls207_pers_appln ON tls206_person.person_id = tls207_pers_appln.person_id
INNER JOIN tls201_appln ON tls201_appln.appln_id = tls207_pers_appln.appln_id
INNER JOIN tls209_appln_ipc ON tls209_appln_ipc.appln_id = tls207_pers_appln.appln_id
WHERE tls201_appln.appln_id = tls207_pers_appln.appln_id
AND YEAR(tls201_appln.appln_filing_date) = """+ year +"""
GROUP BY tls207_pers_appln.appln_id ORDER BY NULL LIMIT 10
"""
date = time.strftime('%c', time.localtime())
print 'Processing ' + year + '. Started: '+ time.strftime('%c', time.localtime())
time_start = time.time()
query_output = myquery(dataextract)
final_output = tuple_clean(query_output)
time_end = time.time()
elapsed_time = time_end - time_start
total_elapsed_time += elapsed_time
print 'Time elapsed for ' + year + ': ' + str(numpy.round(elapsed_time, 0))
print 'Overall elapsed time: ' + str(numpy.round(elapsed_time, 0))
names = [q[2] for q in query_output]
name_time_start = time.time()
test = psCleanup.master_clean_dicts(names, psCleanup.cleanup_dicts)
name_time_end = time.time()
print name_time_end - name_time_start
def clean_list_wrapper(name_list, dict_list=all_dicts):
out = [psCleanup.rem_diacritics(n) for n in name_list]
out = [psCleanup.stdize_case(n) for n in out]
out = psCleanup.master_clean_dicts(out, all_dicts)
out = [n.strip() for n in out]
return(out)
def clean_list_wrapper(name_list, dict_list=all_dicts):
out = [psCleanup.rem_diacritics(n) for n in name_list]
out = [psCleanup.stdize_case(n) for n in out]
out = psCleanup.master_clean_dicts(out, all_dicts)
out = [n.strip() for n in out]
return (out)
