if text:

punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.:;]+')

result = []

for word in punct_re.split(text.lower()):

word = normalize('NFKD', word).encode('ascii', 'ignore')

if word:

result.append(word)

return unicode(delim.join(result))

else:

column = unidecode(column)

column = re.sub(' +', ' ', column)

column = re.sub('\n', ' ', column)

column = column.strip().strip('"').strip("'").lower().strip()

inp = StringIO(contents)

reader = UnicodeCSVReader(inp)

header = reader.next()

header = [slugify(h) for h in header]

outp = StringIO()

writer = UnicodeCSVWriter(outp)

writer.writerow(header)

writer.writerows([[preProcess(unicode(i)) for i in r] for r in reader])

outp.seek(0)

conn = sqlite3.connect(':memory:')

t = Table.from_csv(outp,

name='raw_table',

blanks_as_nulls=False,

infer_types=False)

sql_table = make_table(t)

create_st = make_create_table_statement(sql_table)

parts = create_st.split('raw_table (')

create_st = '{0} raw_table ( record_id INTEGER PRIMARY KEY,{1}'.format(*parts)

insert = sql_table.insert()

curs = conn.cursor()

curs.execute(create_st)

rows = [dict(zip(header, row)) for row in t.to_rows()]

for row in rows:

curs.execute(str(insert), row)

dump = StringIO()

for line in conn.iterdump():

dump.write(unidecode(line))

dump.seek(0)

conn = sqlite3.connect(':memory:')

curs = conn.cursor()

curs.executescript(dump)

curs.execute('select * from raw_table limit 1')

header = [i[0] for i in curs.description]

sql = 'SELECT {0} from raw_table'.format(','.join([h for h in header if h != 'record_id']))

if sample:

sql = '''

SELECT r.*

FROM raw_table AS r

JOIN (

SELECT MAX(record_id) AS record_id

FROM raw_table

GROUP BY {0}

) AS s

ON r.record_id = s.record_id

'''.format(','.join(fields))

curs.execute(sql)

dd = {}

for index, row in enumerate(curs.fetchall()):

dd[index] = dict(zip(header, row))

conn = sqlite3.connect(':memory:')

curs = conn.cursor()

curs.executescript(dump)

curs.execute('select distinct {0} from raw_table'.format(field))

dist = [i[0] for i in curs.fetchall() if i[0]]

"""

Prepare deduplicated file for writing to various formats with

duplicates clustered.

"""

cluster_membership = {}

cluster_id = None

for cluster_id, cluster in enumerate(clustered_dupes):

for record_id, score in zip(*cluster):

cluster_membership[record_id] = cluster_id

unique_record_id = cluster_id + 1

clustered_rows = []

with open('{0}-converted.csv'.format(fpath), 'rb') as f:

reader = UnicodeCSVReader(f)

heading_row = reader.next()

heading_row.insert(0, 'Group ID')

rows = []

for row_id, row in enumerate(reader):

if row_id in cluster_membership:

cluster_id = cluster_membership[row_id]

else:

cluster_id = unique_record_id

unique_record_id += 1

row.insert(0, cluster_id)

rows.append(row)

rows = sorted(rows, key=itemgetter(0))

rows.insert(0, heading_row)

for row in rows:

d = OrderedDict()

for k,v in zip(heading_row, row):

d[k] = v

clustered_rows.append(d)

cluster_membership = {}

for (cluster_id, cluster) in enumerate(clustered_dupes):

for record_id, score in zip(*cluster):

cluster_membership[record_id] = cluster_id

unique_rows = []

with open('{0}-converted.csv'.format(fpath), 'rb') as f:

reader = UnicodeCSVReader(f)

rows = [reader.next()]

seen_clusters = set()

for row_id, row in enumerate(reader):

if row_id in cluster_membership:

cluster_id = cluster_membership[row_id]

if cluster_id not in seen_clusters:

rows.append(row)

seen_clusters.add(cluster_id)

else:

rows.append(row)

for row in rows:

d = OrderedDict()

for k,v in zip(rows[0], row):

d[k] = v

unique_rows.append(d)

u_path = '%s-deduped_unique.csv' % file_path

d_path = '%s-deduped.csv' % file_path

unique = open(u_path, 'wb')

writer = UnicodeCSVDictWriter(unique, unique_rows[0].keys())

writer.writeheader()

writer.writerows(unique_rows)

unique.close()

clusters = open(d_path, 'wb')

writer = UnicodeCSVDictWriter(clusters, clustered_rows[0].keys())

writer.writeheader()

writer.writerows(clustered_rows)

clusters.close()

header = rows[0].keys()

for r, row in enumerate(rows):

for c, key in enumerate(header):

value = row[key]

u_path = '%s-deduped_unique.xls' % file_path

d_path = '%s-deduped.xls' % file_path

clustered_book = xlwt.Workbook(encoding='utf-8')

clustered_sheet = clustered_book.add_sheet('Clustered Results')

for r,c,value in iterExcel(clustered_rows):

clustered_sheet.write(r,c,label=value)

clustered_book.save(d_path)

unique_book = xlwt.Workbook(encoding='utf-8')

unique_sheet = unique_book.add_sheet('Unique Results')

for r,c,value in iterExcel(unique_rows):

unique_sheet.write(r,c,label=value)

unique_book.save(u_path)

u_path = '%s-deduped_unique.xlsx' % file_path

d_path = '%s-deduped.xlsx' % file_path

d_book = Workbook()

d_ws = d_book.active

d_ws.title = 'Clustered Results'

for r,c,value in iterExcel(clustered_rows):

col = get_column_letter(c + 1)

d_ws.cell('%s%s' % (col, r + 1)).value = value

d_book.save(filename=d_path)

u_book = Workbook()

u_ws = u_book.active

u_ws.title = 'Unique Results'

for r,c,value in iterExcel(unique_rows):

col = get_column_letter(c + 1)

u_ws.cell('%s%s' % (col, r + 1)).value = value

u_book.save(filename=u_path)

training_data,

sample,

dump,

file_path,

file_type):

d = dedupe.Dedupe(field_defs, sample)

td = StringIO(json.dumps(training_data, default=_to_json))

d.readTraining(td)

d.train()

settings_path = '{0}-settings.dedupe'.format(file_path)

training_path = '{0}-training.json'.format(file_path)

with open(settings_path, 'wb') as f:

d.writeSettings(f)

with open(training_path, 'wb') as f:

d.writeTraining(f)

dd = makeDataDict(dump)

threshold = d.threshold(dd,recall_weight=2)

clusters = d.match(dd, threshold)

files = writeResults(clusters, dump, file_type, file_path)

files['training'] = os.path.relpath(training_path, __file__)

files['settings'] = os.path.relpath(settings_path, __file__)

d = dedupe.StaticDedupe(open(settings_path, 'rb'))

dd = makeDataDict(dump)

threshold = d.threshold(dd, recall_weight=float(recall_weight))

clusters = d.match(dd, threshold)

files = writeResults(clusters, dump, file_type, file_path)

cluster_count, clustered_rows = prepareResults(clusters, file_path)

unique_rows = prepareUniqueResults(clusters, file_path)

if file_type == 'csv':

deduped, deduped_unique = writeCSV(unique_rows, clustered_rows, file_path)

if file_type == 'xls':

deduped, deduped_unique = writeXLS(unique_rows, clustered_rows, file_path)

if file_type == 'xlsx':

deduped, deduped_unique = writeXLSX(unique_rows, clustered_rows, file_path)

conn = sqlite3.connect(':memory:')

curs = conn.cursor()

curs.executescript(dump)

curs.execute('select count(*) from raw_table')

line_count = [i[0] for i in curs.fetchall()]

files = {

'deduped': os.path.relpath(deduped, __file__),

'deduped_unique': os.path.relpath(deduped_unique, __file__),

'cluster_count': cluster_count,

'line_count': line_count,

}