def trans():
event = Event.get(event_key)
if not event: # event can be None if the CLEANUP cron_job is being run
return
new_first_start_date = event.get_first_event_date()
if event.first_start_date != new_first_start_date:
event.first_start_date = new_first_start_date
event.put()
def _process_event_reminder(reminder_key):
reminder = EventReminder.get(reminder_key)
service_user = reminder.service_user
settings = get_solution_settings(service_user)
event = Event.get_by_id(reminder.event_id,
parent_key(service_user, settings.solution))
if event and reminder.event_start_epoch in event.start_dates:
now_ = now()
if (now_ +
reminder.remind_before) > (reminder.event_start_epoch +
timezone_offset(settings.timezone)):
if event.place:
place = "\n@ " + event.place + "\n"
else:
place = ""
dt = datetime.datetime.fromtimestamp(reminder.event_start_epoch)
language = settings.main_language or DEFAULT_LANGUAGE
when = "%s, %s" % (format_date(dt, format='full', locale=language),
format_time(dt, format='short',
locale=language))
reminderMessage = "Reminder:\n\nTitle:\n%s\n\nWhen:\n%s\n%s\nDescription:\n%s" % (
event.title, when, place, event.description)
main_branding = get_solution_main_branding(service_user)
human_user = get_human_user_from_app_user(reminder.human_user)
members = list()
members.append(human_user.email())
users.set_user(reminder.service_user)
try:
messaging.send(parent_key=None,
parent_message_key=None,
message=reminderMessage,
answers=[],
flags=Message.FLAG_ALLOW_DISMISS,
members=members,
branding=main_branding.branding_key,
tag=None,
service_identity=reminder.service_identity)
finally:
users.clear_user()
reminder.status = EventReminder.STATUS_REMINDED
reminder.put()
else:
reminder.status = EventReminder.STATUS_REMINDED
reminder.put()
def _populate_uit_events(sln_settings, uitdatabank_secret, uitdatabank_key,
external_id, uitdatabank_actors, changed_since):
logging.debug("process event with id: %s", external_id)
detail_success, detail_result = _get_uitdatabank_events_detail(
uitdatabank_secret, uitdatabank_key, external_id)
if not detail_success:
logging.warn("Failed to get detail for cdbid: %s\n%s" %
(external_id, detail_result))
return None
if uitdatabank_secret and changed_since:
if detail_result["lastupdated"] < (changed_since * 100):
return None
if DEBUG:
logging.warn("detail result: %s", detail_result)
event_parent_key = parent_key(sln_settings.service_user,
sln_settings.solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =",
Event.SOURCE_UITDATABANK_BE).filter("external_id =",
external_id).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_UITDATABANK_BE,
external_id=external_id)
event.calendar_id = sln_settings.default_calendar
events = [event]
uitdatabank_created_by = detail_result.get("createdby", None)
logging.debug("uitdatabank_created_by: %s", uitdatabank_created_by)
uitdatabank_lastupdated_by = detail_result.get("lastupdatedby", None)
logging.debug("uitdatabank_lastupdated_by: %s", uitdatabank_lastupdated_by)
if uitdatabank_created_by or uitdatabank_lastupdated_by:
if uitdatabank_created_by and uitdatabank_created_by not in uitdatabank_actors:
uitdatabank_actors[uitdatabank_created_by] = []
if uitdatabank_lastupdated_by and uitdatabank_lastupdated_by not in uitdatabank_actors:
uitdatabank_actors[uitdatabank_lastupdated_by] = []
origanizer_settings = []
if uitdatabank_created_by:
origanizer_settings.extend(
uitdatabank_actors[uitdatabank_created_by])
if uitdatabank_lastupdated_by and uitdatabank_created_by != uitdatabank_lastupdated_by:
origanizer_settings.extend(
uitdatabank_actors[uitdatabank_lastupdated_by])
logging.debug("len(origanizer_settings): %s", len(origanizer_settings))
for organizer_sln_settings in origanizer_settings:
organizer_event_parent_key = parent_key(
organizer_sln_settings.service_user,
organizer_sln_settings.solution)
organizer_event = Event.all().ancestor(
organizer_event_parent_key).filter(
"source =", Event.SOURCE_UITDATABANK_BE).filter(
"external_id =", external_id).get()
if not organizer_event:
organizer_event = Event(parent=organizer_event_parent_key,
source=Event.SOURCE_UITDATABANK_BE,
external_id=external_id)
organizer_event.calendar_id = organizer_sln_settings.default_calendar
events.append(organizer_event)
r_event_detail = detail_result["eventdetails"]["eventdetail"]
if not r_event_detail:
logging.warn('Missing eventdetail')
return None
if isinstance(r_event_detail, list):
for x in r_event_detail:
if x['lang'] == sln_settings.main_language:
r_event_detail = x
break
else:
r_event_detail = r_event_detail[0]
event_title = r_event_detail["title"]
event_description = r_event_detail.get(
"shortdescription", r_event_detail.get("longdescription", u""))
if "physical" in detail_result["location"]["address"]:
location = detail_result["location"]["address"]["physical"]
if location.get("street", None):
if uitdatabank_secret:
event_place = "%s %s, %s %s" % (
location["street"]["value"], location.get("housenr", ""),
location["zipcode"], location["city"]["value"])
else:
event_place = "%s %s, %s %s" % (
location["street"], location.get(
"housenr", ""), location["zipcode"], location["city"])
else:
if uitdatabank_secret:
event_place = "%s %s" % (location["zipcode"],
location["city"]["value"])
else:
event_place = "%s %s" % (location["zipcode"], location["city"])
else:
event_place = detail_result["location"]["address"]["virtual"]["title"]
r_organizer = detail_result.get('organiser')
for k in ('actor', 'actordetails', 'actordetail', 'title'):
if not r_organizer:
break
r_organizer = r_organizer.get(k)
event_organizer = r_organizer
r_timestamps = detail_result["calendar"].get("timestamps")
if not r_timestamps:
logging.debug(
"skipping event because we could not determine starttime for: \n%s",
pprint.pformat(detail_result))
return None
event_start_dates, event_end_dates = get_event_start_and_end_dates(
r_timestamps, v2=uitdatabank_secret)
if not event_start_dates:
logging.info(
"Skipping event because it had no starttime (list) for:\n%s",
pprint.pformat(detail_result))
return None
for event in events:
event.title = event_title
event.description = event_description
event.place = event_place
event.organize = event_organizer
event.last_start_date = max(event_start_dates)
event.start_dates = event_start_dates
event.end_dates = event_end_dates
event.first_start_date = event.get_first_event_date()
return events
def _process_events(service_user, page):
sln_settings = get_solution_settings(service_user)
if not sln_settings:
logging.error(
"check_for_events_in_fr_deuillabarre failed: sln_settings found for %s",
service_user)
return
if SolutionModule.AGENDA not in sln_settings.modules:
logging.error(
"check_for_events_in_fr_deuillabarre failed: module found for %s",
service_user)
return
url = u"http://www.mairie-deuillabarre.fr/agenda?page=%s" % page
response = urlfetch.fetch(url, deadline=60)
if response.status_code != 200:
logging.error(
"Could not check for events in fr_deuillabarre page %s.\n%s",
page,
response.content,
_suppress=False)
return
tree = html.fromstring(response.content.decode("utf8"))
events = tree.xpath(
'//form[@id="icagenda-list"]//div[@class="event ic-event ic-clearfix"]'
)
to_put = []
for event in events:
h2_title = event.getchildren()[1].getchildren()[0].getchildren(
)[0].getchildren()[0]
div_place = event.getchildren()[1].getchildren()[2]
title = _filter_chars(h2_title.getchildren()[0].text)
url = h2_title.getchildren()[0].xpath("@href")[0]
place = _filter_chars(
div_place.text_content() if div_place is not None else u"")
place = place.replace("Lieu: ", "")
if not (url.startswith("http://") or url.startswith("https://")):
url = u"http://www.mairie-deuillabarre.fr%s" % url
url = unicode(url) if not isinstance(url, unicode) else url
t = _get_event_details(url)
if not t:
continue
start_epochs, end_hour_epochs, description = t
event_parent_key = parent_key(service_user, sln_settings.solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =", Event.SOURCE_SCRAPER).filter("external_id =",
url).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_SCRAPER,
external_id=url)
event.calendar_id = sln_settings.default_calendar
event.external_link = url
event.title = title
event.description = description
event.place = place
event.first_start_date = start_epochs[0]
event.last_start_date = start_epochs[-1]
event.start_dates = start_epochs
event.end_dates = end_hour_epochs
to_put.append(event)
if to_put:
db.put(to_put)
if events:
deferred.defer(_process_events, service_user, page + 1)
else:
sln_settings.put_identity_pending = True
sln_settings.put()
def update_statistic():
# Completely rebuilds statistics on run.
current_date = now()
broadcast_count_dict = {}
for news_item in get_news_of_last_month():
service_email = get_service_user_from_service_identity_user(
news_item.sender).email()
if service_email not in broadcast_count_dict:
broadcast_count_dict[service_email] = 1
else:
broadcast_count_dict[service_email] += 1
future_event_count_dict = {}
for future_event_key in Event.get_future_event_keys(current_date):
service_email = future_event_key.parent().name()
if service_email not in future_event_count_dict:
future_event_count_dict[service_email] = 1
else:
future_event_count_dict[service_email] += 1
static_content_count_dict = {}
for static_content_key in SolutionStaticContent.get_all_keys():
service_email = static_content_key.parent().name()
if service_email not in static_content_count_dict:
static_content_count_dict[service_email] = 1
else:
static_content_count_dict[service_email] += 1
unanswered_question_count_dict = {}
# find the oldest, unanswered question per customer and add it to the statistics
for unanswered_question in SolutionInboxMessage.get_all_unanswered_questions(
5):
service_email = unanswered_question.service_user.email()
if unanswered_question.question_asked_timestamp != 0:
if not service_email in unanswered_question_count_dict:
unanswered_question_count_dict[
service_email] = unanswered_question.question_asked_timestamp
elif unanswered_question.question_asked_timestamp < unanswered_question_count_dict[
service_email]:
unanswered_question_count_dict[
service_email] = unanswered_question.question_asked_timestamp
# dict with as keys the app id from the city, value the statistics of this city.
statistics = {}
for customer in Customer.all():
if len(customer.app_ids) != 0:
service_email = customer.service_email
if customer.app_id not in statistics:
stats = AssociationStatistic(key_name=customer.app_id)
stats.generated_on = current_date
statistics[customer.app_id] = stats
else:
stats = statistics[customer.app_id]
if not service_email:
logging.error(
u'Association customer %s(%d) has no service_email!',
customer.name, customer.id)
continue
stats.customer_emails.append(service_email)
if service_email in broadcast_count_dict:
stats.broadcasts_last_month.append(
broadcast_count_dict[customer.service_email])
else:
stats.broadcasts_last_month.append(0)
if service_email in future_event_count_dict:
stats.future_events_count.append(
future_event_count_dict[service_email])
else:
stats.future_events_count.append(0)
if service_email in static_content_count_dict:
stats.static_content_count.append(
static_content_count_dict[service_email])
else:
stats.static_content_count.append(0)
if service_email in unanswered_question_count_dict:
stats.last_unanswered_questions_timestamps.append(
unanswered_question_count_dict[service_email])
else:
stats.last_unanswered_questions_timestamps.append(0)
for chunk in chunks(statistics.values(), 200):
db.put(chunk)
def new_event_received(service_user, message_flow_run_id, member, steps,
end_id, end_message_flow_id, parent_message_key, tag,
result_key, flush_id, flush_message_flow_id,
service_identity, user_details):
from solutions.common.bizz.messaging import _get_step_with_id
logging.info("_flow_member_result_new_event: \n %s" % steps)
calendar = _get_step_with_id(steps, 'message_calendar')
title = _get_step_with_id(steps, 'message_title')
if not title:
logging.error(
"Did not find step with id 'title'. Can not process message_flow_member_result with tag %s"
% tag)
return None
date_ = _get_step_with_id(steps, 'message_date')
if not date_:
logging.error(
"Did not find step with id 'date'. Can not process message_flow_member_result with tag %s"
% tag)
return None
start_time = _get_step_with_id(steps, 'message_start_time')
if not start_time:
logging.error(
"Did not find step with id 'start_time'. Can not process message_flow_member_result with tag %s"
% tag)
return None
end_time = _get_step_with_id(steps, 'message_end_time')
if not end_time:
logging.error(
"Did not find step with id 'end_time'. Can not process message_flow_member_result with tag %s"
% tag)
return None
description = _get_step_with_id(steps, 'message_description')
if not description:
logging.error(
"Did not find step with id 'description'. Can not process message_flow_member_result with tag %s"
% tag)
return None
place = _get_step_with_id(steps, 'message_place')
if not place:
logging.error(
"Did not find step with id 'place'. Can not process message_flow_member_result with tag %s"
% tag)
return None
photo = _get_step_with_id(steps, 'message_photo')
if not photo:
logging.error(
"Did not find step with id 'photo'. Can not process message_flow_member_result with tag %s"
% tag)
return None
sln_settings = get_solution_settings(service_user)
app_user = user_details[0].toAppUser()
calendars_ids_admin = get_solution_calendar_ids_for_user(
service_user, sln_settings.solution, app_user)
if len(calendars_ids_admin) == 0:
logging.warn("User %s isn't a calendar admin anymore" %
app_user.email())
return None
calendar_id = None
if calendar and calendar.form_result.result:
calendar_id = long(calendar.form_result.result.value)
else:
if sln_settings.default_calendar in calendars_ids_admin:
calendar_id = sln_settings.default_calendar
else:
calendar_id = calendars_ids_admin[0]
sc = SolutionCalendar.get_by_id(
calendar_id, parent_key(service_user, sln_settings.solution))
if not sc:
logging.warn("Calendar %s does not exists anymore" % calendar_id)
return None
event = Event(parent=parent_key(service_user, sln_settings.solution))
event.picture_version = 0
event.title = title.form_result.result.value
event.place = place.form_result.result.value
event.description = description.form_result.result.value
start_date = get_epoch_from_datetime(
datetime.fromtimestamp(date_.form_result.result.value).date()
) + start_time.form_result.result.value
event.last_start_date = start_date
event.start_dates = [start_date]
event.end_dates = [end_time.form_result.result.value]
event.first_start_date = event.get_first_event_date()
event.calendar_id = calendar_id
if photo.form_result:
result = urlfetch.fetch(photo.form_result.result.value, {},
"GET", {},
False,
False,
deadline=10 * 60)
if result.status_code != 200:
logging.error(
"Failed to download photo upload for new event whith link: %s"
% photo.form_result.result)
picture = None
else:
img_str = result.content
img_b64 = base64.b64encode(img_str)
previous_len_img = len(img_b64)
while len(img_b64) > 512 * 1024:
img = images.Image(img_str)
img.im_feeling_lucky()
img_str = img.execute_transforms(
output_encoding=images.JPEG) # Reduces quality to 85%
img_b64 = base64.b64encode(img_str)
if previous_len_img <= len(img_b64):
break
previous_len_img = len(img_b64)
picture = "data:image/jpg;base64,%s" % img_b64
else:
picture = None
event.picture = picture
event.put()
send_message(service_user, u"solutions.common.calendar.update")
deferred.defer(common_provision, service_user)
return None
def trans():
if new_event.id:
event = get_event_by_id(service_user, sln_settings.solution,
new_event.id)
if new_event.new_picture:
event.picture_version += 1
else:
event = Event(
parent=parent_key(service_user, sln_settings.solution))
event.picture_version = 0
event.title = new_event.title
event.place = new_event.place
event.organizer = new_event.organizer
event.description = new_event.description
start_dates = []
for start_date in new_event.start_dates:
start_dates.append(start_date.toEpoch())
start_dates_sorted, end_dates_sorted = zip(
*sorted(zip(start_dates, new_event.end_dates)))
startDates = list(start_dates_sorted)
new_event.end_dates = list(end_dates_sorted)
event.last_start_date = max(startDates)
event.start_dates = start_dates
event.end_dates = new_event.end_dates
event.first_start_date = event.get_first_event_date()
event.url = new_event.external_link
event.picture = picture
event.calendar_id = new_event.calendar_id
sln_settings.updates_pending = True
put_and_invalidate_cache(event, sln_settings)
return sln_settings
def _put_default_event(service_user, translate, solution):
event = Event(parent=parent_key(service_user, solution))
now = time.mktime(date.timetuple(date.today()))
event.title = u"Title"
event.place = u"Place"
event.description = u"Description"
event.last_start_date = long(now + 7 * 24 * 60 * 60 + 20 * 60 * 60)
event.start_dates = [event.last_start_date]
event.end_dates = [long(4 * 60 * 60)]
event.first_start_date = event.get_first_event_date()
event.put()
event2 = Event(parent=parent_key(service_user, solution))
event2.title = u"Title2"
event2.place = u"Place2"
event2.description = u"Description2"
event2.last_start_date = long(now + 14 * 24 * 60 * 60 + 20 * 60 * 60)
event2.start_dates = [event2.last_start_date]
event2.end_dates = [long(4 * 60 * 60)]
event2.first_start_date = event2.get_first_event_date()
event2.put()
return [event, event2]
def put_google_events(service_user, calendar_str_key, calendar_id, solution,
google_calendar_finished, google_events, language):
to_put = []
no_title_text = common_translate(language, SOLUTION_COMMON, '(No title)')
for google_event in google_events:
try:
google_event_id = google_event["id"]
event_parent_key = parent_key(service_user, solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =",
Event.SOURCE_GOOGLE_CALENDAR).filter("external_id =",
google_event_id).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_GOOGLE_CALENDAR,
external_id=google_event_id)
if google_event.get("status", "confirmed") == "cancelled":
if not event.is_saved():
continue
event.deleted = True
elif not event.deleted:
event.deleted = False
event.title = google_event.get('summary', no_title_text)
event.place = google_event.get("location",
u"").replace(u"\n", u" ")
event.organizer = google_event.get("organizer",
{}).get("displayName", u"")
event.description = google_event.get('description', u"")
event.calendar_id = calendar_id
event.external_link = google_event["htmlLink"]
event.start_dates = list()
event.end_dates = list()
if google_event["start"].get("dateTime", None):
start_date_with_tz = dateutil.parser.parse(
google_event["start"]["dateTime"])
end_date_with_tz = dateutil.parser.parse(
google_event["end"]["dateTime"])
day_difference = abs(
(end_date_with_tz - start_date_with_tz).days)
if day_difference == 0:
start_epoch = get_epoch_from_datetime(
datetime(start_date_with_tz.year,
start_date_with_tz.month,
start_date_with_tz.day,
start_date_with_tz.hour,
start_date_with_tz.minute))
event.start_dates.append(start_epoch)
end_epoch = get_epoch_from_datetime(
datetime(end_date_with_tz.year, end_date_with_tz.month,
end_date_with_tz.day, end_date_with_tz.hour,
end_date_with_tz.minute))
end_date = datetime.fromtimestamp(end_epoch)
event.end_dates.append(
int(
timedelta(
hours=end_date.hour,
minutes=end_date.minute,
seconds=end_date.second).total_seconds()))
# TODO: multi day event
# else:
# start_date = datetime.strptime(google_event["start"]["date"], '%Y-%m-%d')
# end_date = datetime.strptime(google_event["end"]["date"], '%Y-%m-%d')
# day_difference = abs((end_date - start_date).days)
if event.start_dates:
event.first_start_date = event.start_dates[0]
event.last_start_date = event.start_dates[-1]
to_put.append(event)
else:
logging.info("Skipping event because it had no start_dates")
logging.debug(google_event)
except:
logging.warn('Failed to put Google Event: %s', google_event)
raise
if to_put:
db.put(to_put)
if google_calendar_finished:
def trans():
scgs = SolutionCalendarGoogleSync.get_by_key_name(
service_user.email())
if not scgs:
return True # update was run from saving a calendar
if calendar_str_key in scgs.google_calendar_keys:
scgs.google_calendar_keys.remove(calendar_str_key)
if not scgs.google_calendar_keys:
scgs.delete()
return True
scgs.put()
return False
if db.run_in_transaction(trans):
from solutions.common.bizz.provisioning import populate_identity_and_publish
sln_settings = get_solution_settings(service_user)
sln_main_branding = get_solution_main_branding(service_user)
deferred.defer(populate_identity_and_publish, sln_settings,
sln_main_branding.branding_key)
def get_event_by_id(service_user, solution, id_):
return Event.get_by_id(id_, parent_key(service_user, solution))
def get_public_event_list(service_user, solution):
qry = Event.gql(
"WHERE ANCESTOR IS :ancestor AND deleted=False AND source = :source")
qry.bind(ancestor=parent_key(service_user, solution),
source=Event.SOURCE_CMS)
return generator(qry.run())
def get_event_list(service_user, solution):
qry = Event.gql("WHERE ANCESTOR IS :ancestor AND deleted=False")
qry.bind(ancestor=parent_key_unsafe(service_user, solution))
return generator(qry.run())
def trans():
event = Event.get(event_key)
event.first_start_date = event.get_first_event_date()
event.put()