% athlete_id try: response = opener.open(url) except Exception, e: print '%s - %s' % (e, athlete_id) return {'error': 'fail5'} if response.getcode() != 200: raise Exception('Athlete info: %s - %s' % \ (response.getcode(), response.msg)) soup = BeautifulSoup(response.read()) time.sleep(TIME_BT_REQUESTS) # find the data we're interested in run_info = list(soup.childGenerator())[4] # first get the 10k record try: records = list(run_info.childGenerator())[7] records = list(records.childGenerator())[3] except: return {'error': 'fail1'} ten_k = None for item in records.childGenerator(): if '10k' in str(item): record = list(item.childGenerator())[3] record = record.text # time is in the format 1:02:23 or 45:54