def _process_events(service_user, page):
sln_settings = get_solution_settings(service_user)
if not sln_settings:
logging.error(
"check_for_events_in_fr_deuillabarre failed: sln_settings found for %s",
service_user)
return
if SolutionModule.AGENDA not in sln_settings.modules:
logging.error(
"check_for_events_in_fr_deuillabarre failed: module found for %s",
service_user)
return
url = u"http://www.mairie-deuillabarre.fr/agenda?page=%s" % page
response = urlfetch.fetch(url, deadline=60)
if response.status_code != 200:
logging.error(
"Could not check for events in fr_deuillabarre page %s.\n%s",
page,
response.content,
_suppress=False)
return
tree = html.fromstring(response.content.decode("utf8"))
events = tree.xpath(
'//form[@id="icagenda-list"]//div[@class="event ic-event ic-clearfix"]'
)
to_put = []
for event in events:
h2_title = event.getchildren()[1].getchildren()[0].getchildren(
)[0].getchildren()[0]
div_place = event.getchildren()[1].getchildren()[2]
title = _filter_chars(h2_title.getchildren()[0].text)
url = h2_title.getchildren()[0].xpath("@href")[0]
place = _filter_chars(
div_place.text_content() if div_place is not None else u"")
place = place.replace("Lieu: ", "")
if not (url.startswith("http://") or url.startswith("https://")):
url = u"http://www.mairie-deuillabarre.fr%s" % url
url = unicode(url) if not isinstance(url, unicode) else url
t = _get_event_details(url)
if not t:
continue
start_epochs, end_hour_epochs, description = t
event_parent_key = parent_key(service_user, sln_settings.solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =", Event.SOURCE_SCRAPER).filter("external_id =",
url).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_SCRAPER,
external_id=url)
event.calendar_id = sln_settings.default_calendar
event.external_link = url
event.title = title
event.description = description
event.place = place
event.first_start_date = start_epochs[0]
event.last_start_date = start_epochs[-1]
event.start_dates = start_epochs
event.end_dates = end_hour_epochs
to_put.append(event)
if to_put:
db.put(to_put)
if events:
deferred.defer(_process_events, service_user, page + 1)
else:
sln_settings.put_identity_pending = True
sln_settings.put()
def _populate_uit_events(sln_settings, uitdatabank_secret, uitdatabank_key,
external_id, uitdatabank_actors, changed_since):
logging.debug("process event with id: %s", external_id)
detail_success, detail_result = _get_uitdatabank_events_detail(
uitdatabank_secret, uitdatabank_key, external_id)
if not detail_success:
logging.warn("Failed to get detail for cdbid: %s\n%s" %
(external_id, detail_result))
return None
if uitdatabank_secret and changed_since:
if detail_result["lastupdated"] < (changed_since * 100):
return None
if DEBUG:
logging.warn("detail result: %s", detail_result)
event_parent_key = parent_key(sln_settings.service_user,
sln_settings.solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =",
Event.SOURCE_UITDATABANK_BE).filter("external_id =",
external_id).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_UITDATABANK_BE,
external_id=external_id)
event.calendar_id = sln_settings.default_calendar
events = [event]
uitdatabank_created_by = detail_result.get("createdby", None)
logging.debug("uitdatabank_created_by: %s", uitdatabank_created_by)
uitdatabank_lastupdated_by = detail_result.get("lastupdatedby", None)
logging.debug("uitdatabank_lastupdated_by: %s", uitdatabank_lastupdated_by)
if uitdatabank_created_by or uitdatabank_lastupdated_by:
if uitdatabank_created_by and uitdatabank_created_by not in uitdatabank_actors:
uitdatabank_actors[uitdatabank_created_by] = []
if uitdatabank_lastupdated_by and uitdatabank_lastupdated_by not in uitdatabank_actors:
uitdatabank_actors[uitdatabank_lastupdated_by] = []
origanizer_settings = []
if uitdatabank_created_by:
origanizer_settings.extend(
uitdatabank_actors[uitdatabank_created_by])
if uitdatabank_lastupdated_by and uitdatabank_created_by != uitdatabank_lastupdated_by:
origanizer_settings.extend(
uitdatabank_actors[uitdatabank_lastupdated_by])
logging.debug("len(origanizer_settings): %s", len(origanizer_settings))
for organizer_sln_settings in origanizer_settings:
organizer_event_parent_key = parent_key(
organizer_sln_settings.service_user,
organizer_sln_settings.solution)
organizer_event = Event.all().ancestor(
organizer_event_parent_key).filter(
"source =", Event.SOURCE_UITDATABANK_BE).filter(
"external_id =", external_id).get()
if not organizer_event:
organizer_event = Event(parent=organizer_event_parent_key,
source=Event.SOURCE_UITDATABANK_BE,
external_id=external_id)
organizer_event.calendar_id = organizer_sln_settings.default_calendar
events.append(organizer_event)
r_event_detail = detail_result["eventdetails"]["eventdetail"]
if not r_event_detail:
logging.warn('Missing eventdetail')
return None
if isinstance(r_event_detail, list):
for x in r_event_detail:
if x['lang'] == sln_settings.main_language:
r_event_detail = x
break
else:
r_event_detail = r_event_detail[0]
event_title = r_event_detail["title"]
event_description = r_event_detail.get(
"shortdescription", r_event_detail.get("longdescription", u""))
if "physical" in detail_result["location"]["address"]:
location = detail_result["location"]["address"]["physical"]
if location.get("street", None):
if uitdatabank_secret:
event_place = "%s %s, %s %s" % (
location["street"]["value"], location.get("housenr", ""),
location["zipcode"], location["city"]["value"])
else:
event_place = "%s %s, %s %s" % (
location["street"], location.get(
"housenr", ""), location["zipcode"], location["city"])
else:
if uitdatabank_secret:
event_place = "%s %s" % (location["zipcode"],
location["city"]["value"])
else:
event_place = "%s %s" % (location["zipcode"], location["city"])
else:
event_place = detail_result["location"]["address"]["virtual"]["title"]
r_organizer = detail_result.get('organiser')
for k in ('actor', 'actordetails', 'actordetail', 'title'):
if not r_organizer:
break
r_organizer = r_organizer.get(k)
event_organizer = r_organizer
r_timestamps = detail_result["calendar"].get("timestamps")
if not r_timestamps:
logging.debug(
"skipping event because we could not determine starttime for: \n%s",
pprint.pformat(detail_result))
return None
event_start_dates, event_end_dates = get_event_start_and_end_dates(
r_timestamps, v2=uitdatabank_secret)
if not event_start_dates:
logging.info(
"Skipping event because it had no starttime (list) for:\n%s",
pprint.pformat(detail_result))
return None
for event in events:
event.title = event_title
event.description = event_description
event.place = event_place
event.organize = event_organizer
event.last_start_date = max(event_start_dates)
event.start_dates = event_start_dates
event.end_dates = event_end_dates
event.first_start_date = event.get_first_event_date()
return events
def put_google_events(service_user, calendar_str_key, calendar_id, solution,
google_calendar_finished, google_events, language):
to_put = []
no_title_text = common_translate(language, SOLUTION_COMMON, '(No title)')
for google_event in google_events:
try:
google_event_id = google_event["id"]
event_parent_key = parent_key(service_user, solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =",
Event.SOURCE_GOOGLE_CALENDAR).filter("external_id =",
google_event_id).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_GOOGLE_CALENDAR,
external_id=google_event_id)
if google_event.get("status", "confirmed") == "cancelled":
if not event.is_saved():
continue
event.deleted = True
elif not event.deleted:
event.deleted = False
event.title = google_event.get('summary', no_title_text)
event.place = google_event.get("location",
u"").replace(u"\n", u" ")
event.organizer = google_event.get("organizer",
{}).get("displayName", u"")
event.description = google_event.get('description', u"")
event.calendar_id = calendar_id
event.external_link = google_event["htmlLink"]
event.start_dates = list()
event.end_dates = list()
if google_event["start"].get("dateTime", None):
start_date_with_tz = dateutil.parser.parse(
google_event["start"]["dateTime"])
end_date_with_tz = dateutil.parser.parse(
google_event["end"]["dateTime"])
day_difference = abs(
(end_date_with_tz - start_date_with_tz).days)
if day_difference == 0:
start_epoch = get_epoch_from_datetime(
datetime(start_date_with_tz.year,
start_date_with_tz.month,
start_date_with_tz.day,
start_date_with_tz.hour,
start_date_with_tz.minute))
event.start_dates.append(start_epoch)
end_epoch = get_epoch_from_datetime(
datetime(end_date_with_tz.year, end_date_with_tz.month,
end_date_with_tz.day, end_date_with_tz.hour,
end_date_with_tz.minute))
end_date = datetime.fromtimestamp(end_epoch)
event.end_dates.append(
int(
timedelta(
hours=end_date.hour,
minutes=end_date.minute,
seconds=end_date.second).total_seconds()))
# TODO: multi day event
# else:
# start_date = datetime.strptime(google_event["start"]["date"], '%Y-%m-%d')
# end_date = datetime.strptime(google_event["end"]["date"], '%Y-%m-%d')
# day_difference = abs((end_date - start_date).days)
if event.start_dates:
event.first_start_date = event.start_dates[0]
event.last_start_date = event.start_dates[-1]
to_put.append(event)
else:
logging.info("Skipping event because it had no start_dates")
logging.debug(google_event)
except:
logging.warn('Failed to put Google Event: %s', google_event)
raise
if to_put:
db.put(to_put)
if google_calendar_finished:
def trans():
scgs = SolutionCalendarGoogleSync.get_by_key_name(
service_user.email())
if not scgs:
return True # update was run from saving a calendar
if calendar_str_key in scgs.google_calendar_keys:
scgs.google_calendar_keys.remove(calendar_str_key)
if not scgs.google_calendar_keys:
scgs.delete()
return True
scgs.put()
return False
if db.run_in_transaction(trans):
from solutions.common.bizz.provisioning import populate_identity_and_publish
sln_settings = get_solution_settings(service_user)
sln_main_branding = get_solution_main_branding(service_user)
deferred.defer(populate_identity_and_publish, sln_settings,
sln_main_branding.branding_key)