def add_location(req, database): data = {} errors = [] status = '' if req.args is not None: data, arg_errors = parseargs(req.args) errors.extend(arg_errors) if not errors: try: latitude, longitude = geocoder.geocode(data['loc'], delay=16) except ProtocolError, exc: return data, status, [str(exc)] if latitude != None and longitude != None: host = req.get_remote_host(apache.REMOTE_NOLOOKUP) values = { 'lat' : latitude , 'long' : longitude , 'url' : host , } records = database.get_by_dict(values) if not records: values.update( { 'loc' : data['loc'] , 'hood' : data['hood'], 'source': 2 , } ) database.insert_location(values) status = '"%s" (%f, %f) will be added to "%s"\n' % \ (data['loc'], latitude, longitude, data['hood']) else: errors.append('"%s" (%f, %f) in "%s" has already been submitted.' % (data['loc'], latitude, longitude, data['hood'])) else: errors.append('Could not find "%s"'%data['loc']) else: #errors occurred during argument parsing, do nothing here pass
def _scrape_posting(database, hood, url): loc_pat = re.compile(r'href="?http://maps\.google\.com/\?q=loc%3A\+(.+)\+San\+Francisco\+CA\+US"') sock = urllib.urlopen(url) htmlSource = sock.read() sock.close() if test: print "html is", len(htmlSource), "bytes" loc_matches = loc_pat.findall(htmlSource) loc = None if loc_matches: loc = urllib.unquote_plus(loc_matches[0]) if test: print loc citystate = config['citystate'] latitude, longitude = None, None if loc: try: latitude, longitude = geocoder.geocode(loc, citystate=citystate) except ProtocolError, exc: log("error geocoding %r: %r" %(url, exc)) return
##modify it under the terms of the GNU General Public License ##as published by the Free Software Foundation, version 2. ##This program is distributed in the hope that it will be useful, ##but WITHOUT ANY WARRANTY; without even the implied warranty of ##MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ##GNU General Public License for more details. from hood import db, geocoder from config import config if __name__ == '__main__': database = db.DB(config) geocoder.init(config) fixable_rows = database.get_fixable() print len(fixable_rows), 'rows to fix' fixed = 0 for r in fixable_rows: id, created, updated, url, hood, hood_id, source, loc, latitude, longitude = r print 'trying to fix', hood, loc latitude, longitude = geocoder.geocode(loc) if latitude is not None and longitude is not None: print 'found', latitude, longitude fixed += 1 database.update_location(id, {'lat':latitude, 'long':longitude}) database.close_db() print 'fixed', fixed, 'rows'