def func(event): if offline: # Refresh site list event = event._replace( site_info=site_list[event.site_info['name']]) data = scrape(event.html, event.site_info) return event._replace(html="", **data)
def func(event): parse_date = event.raw.get('date', '') if isinstance(parse_date, Iterable): return event._replace(date=parse_date) # Combine dates if we scraped an 'date_end' if 'date_end' in event.raw and event.raw['date_end']: parse_date = "{} to {}".format(event.raw['date'], event.raw['date_end']) new_date = date_parser(unicode_to_whitespace(parse_date)) # Convert to dict and insert if we retrieved a date if new_date is not None: new_date_dict = map(lambda l: l._asdict(), new_date) return event._replace(date=new_date_dict) else: return event
def func(event, post=post): if enough_info(event): for event_date in event.date: date = to_datetime_with_today(event_date).isoformat() name = unicode_to_whitespace(event.name) event = event._replace(date=date, name=name) post(event_to_str(event), event.id)
def func(event): address = fix_address_location(event.raw.get('address', ''), event.site_info) if not address: return event if offline: location = address_cached_get(address, lambda _: offline_location, commit=False) else: location = address_cached_get(address, sleep_and_google) location = location.get('geometry', {}).get('location', {}) lat, lng = location.get('lat', None), location.get('lng', None) if lat and lng: return event._replace(address=address, location={ 'lat': lat, 'lon': lng }) else: return event._replace(address=address)
def func(event): new_scores = event.scores.update( {'uniqueness': get_uniqueness(event.keywords, event.name)}) return event._replace(scores=new_scores)
def func(event): if event.location is not None and event.date is not None and len( event.date): return event._replace(id=date_geohash_template( date=date_hash(event), geo=geohash(event)))
def fix_name(event): if event.name.strip() is '': return event._replace(name=untitled_event_name) return event
def func(event): return event._replace(keywords=get_keywords(event.description))
def func(event): category = event.raw.get('type', '').strip().lower() category = category_fixer(category) if category == "": category = categorize_event(event.description).lower() return event._replace(type=category)
def func(event): description = shorten_longer(strip(event.raw.get('description', ''))) name = strip(event.raw.get('name', '')).title() return event._replace(description=strip(description), name=strip(name).title())