#!/usr/bin/env python

# -*- coding: utf-8 -*-

import urllib2 # For parsing urls

import re # Stand back! I know regular expressions

from bs4 import BeautifulSoup # An html travelling tool

import Levenshtein # For fuzzy matching strings, for the lulz

import unicodedata # Because accented characters are a pain in the ass

import codecs # Ditto

import rtfunicode # Double ditto

import pdb # A debugger

opener = urllib2.build_opener()

opener.addheaders = [('User-agent', 'Mozilla/5.0')]

# Input file is from here

inFilePath = "inputs/october2.txt"

# outpuf file goes here

outFilePath = "outputs/" + inFilePath.lstrip('inputs').rpartition('.')[0] + "_HYPERLINKED.rtf"

# I have yet to try this with anything but montreal

CITY = "montreal"

output = open(outFilePath, "w")

textFile = open(inFilePath, "U")

fileData = textFile.readlines()

textFile.close()

businesses = []

sites = ['facebook', 'yelp', 'yellowpages', 'urbanspoon', 'twitter']

# For translating addresses

roadTrans = {}

roadTrans['abbr'] = {'boul': 'boulevard', 'blvd': 'boulevard', 'bd': 'boulevard', 'ch': 'chemin',

'av': 'avenue', 'o': 'ouest', 'e': 'est', 'n': 'nord', 's': 'sud'}

# key = french, value = english

roadTrans['type'] = {'rue': 'street', 'avenue': 'avenue', 'boulevard': 'boulevard', 'chemin': 'road'}

roadTrans['direction'] = {'ouest': 'west', 'nord': 'north', 'sud': 'south', 'est': 'east'}

# Read in the input data

for line in fileData:

name = unicode(line.rstrip(' \n').strip(), 'utf-8').replace('&', 'and')

if name:

if name[-1] == ':':

continue

else:

businesses.append({'searchName': name})

# When comparing two strings for smiliarity, remove these words to prevent false negatives

def removeSkipWords(words):

return words

# Properly format and change french addresses to english

def translateAddress(stringAddress):

return address['number'] + ' ' + address['street']

# See if the found business name matches the searched name

def findMatchScore(searchName, foundName) :

return max(bigR, bigR2)

# If google finds a yellowpages link for the business, attempt to get a phone/address from the page

def searchYellowpages(business):

business['website'] = str(a['href']).partition("www.")[2].rstrip(' \n')

# search yelp

def searchYelp(business):

business['address'] = translateAddress(address.getText().strip(' \n').rstrip(' \n'))

# Returns links matching the site name from a list of links

def getLink(siteName, links):

return linkString

#only if twitter isn't found, because twitter's search is teh suxxor

def searchTwitter(business):

business['twitter'] = "twitter.com" + link

# Get site links, attempt phone number and address

def searchGoogle(business):

return soup

# Gobbledeegook for the rtf header

output.write("{\\rtf1\\ansi\\ansicpg1252\\deff0\\deflang1033{" +

"\\fonttbl{\\f0\\fnil\\fcharset0 Helvetica;}}\\n{\\colortbl ;" +

"\\red0\\green0\\blue255;}\n{\\*\\generator Msftedit 5.41.21.2509;}" +

"\\viewkind4\\uc1\\pard\\sa200\\sl276\\slmult1\\lang9\\f0\\fs22\n\n")

# For every business on the input file, search and output results

for b in businesses:

print "Search: " + b['searchName']

b['foundName'] = b['searchName']

b['address'] = ''

b['phone'] = ''

b['website'] = ''

for s in sites:

b[s] = ''

# google is definitely on to me, I should be careful

gsoup = searchGoogle(b)

if b['foundName'] != b['searchName']:

print "It's probably called: " + b['foundName']

searchYelp(b)

# If we still haven't found some info, but have a yellowpages link, try it

if not (b['website'] and b['address'] and b['phone']) and (not not b['yellowpages']):

searchYellowpages(b)

# Because twitters are hard to find

if not b['twitter']:

searchTwitter(b)

# If no website is listed, use the results from the google search to try and find one

if not b['website']:

# Sites I don't want showing up as the business website, kinda hacky

ignorewords = ['restomontreal', 'googleusercontent',

'webcache', 'google', 'facebook', 'yelp', 'yellowpages',

'urbanspoon', 'twitter', 'foursquare', 'zagat', 'blogspot',

'tripadvisor', 'pagesjaunes', 'montrealplus', 'canpages',

'blackbookmag', 'adbeux', 'about', 'citeeze', 'nightlife',

'restaurant', 'canplaces', 'yahoo', 'profilecanada', 'mtlblog'

'nearyou', 'foodpages']

div = gsoup.find('div', id='search')

if div:

h3 = div.find_all('h3', class_='r')

a = []

for header in h3:

link = header.find('a')

if link:

a.append(link)

for l in a:

link = l['href']

if link:

ignore = False

words = link.partition('//')[2].partition('/')[0].split('.')

if not words[0]:

continue

for w in words:

if w in ignorewords:

ignore = True

if not ignore:

ws = link.partition('&')[0].partition('=')[2]

if ws[:4] == "http":

wsfilt = ws.lstrip("https://")

wsfilt = wsfilt.rstrip('/')

words = wsfilt.split('/')

if len(words) == 1: #ignore ones that are super lengthy and often wrong

b['website'] = ws

break

# This is just for reporting to the console

print b['foundName']

matchScore = findMatchScore(b['searchName'], b['foundName'])

print matchScore

if b['website']:

print "website: " + b['website']

if b['phone']:

# Reformat the phone number to ###.###.####

b['phone'] = b['phone'].replace('(', '').replace(')', '').replace(' ', '.').replace('-', '.')

print "phone: " + b['phone']

if b['address']:

print "address: " + b['address']

if b['facebook']:

print "facebook: " + b['facebook']

if b['twitter']:

print "twitter: " + b['twitter']

print '\n'

## write to file

# If the located business name is significantly different, report it

if matchScore < 0.8:

output.write("Search: " + b['searchName'].encode('rtfunicode') + '\line\n')

output.write(b['foundName'].encode('rtfunicode') + ' \line\n')

printAttributes = ['address', 'phone', 'website', 'facebook', 'twitter']

for att in printAttributes:

if b[att]:

if att == 'website':

name = urllib2.unquote(b[att])

name = unicode(urllib2.unquote(name), 'utf-8')

output.write("{\\field{\\*\\fldinst{HYPERLINK " + b[att].encode('rtfunicode') + "}}{\\fldrslt{\ul\cf1" + name.encode('rtfunicode') + "}}}" + ' \line\n');

continue

elif att == 'facebook' or att == 'twitter':

output.write("{\\field{\\*\\fldinst{HYPERLINK " + b[att].encode('rtfunicode') + "}}{\\fldrslt{\ul\cf1" + att.capitalize() + "}}}" + ' \line\n');

name = urllib2.unquote(b[att])

name = unicode(urllib2.unquote(name), 'utf-8')

output.write(name.encode('rtfunicode') + ' \line\n')

else:

output.write(b[att].encode('rtfunicode') + ' \line\n')

output.write('\line\n\n')

# Close out rtf file so that it is readable

output.write('}')

output.close()