def save_event(self, info): info = info.copy() args = dict(data_source=info['data_source'], origin_id=info['origin_id']) obj_id = "%s:%s" % (info['data_source'].id, info['origin_id']) try: obj = Event.objects.get(**args) obj._created = False assert obj.id == obj_id except Event.DoesNotExist: obj = Event(**args) obj._created = True obj.id = obj_id obj._changed = False obj._changed_fields = [] location_id = None if 'location' in info: location = info['location'] if 'id' in location: location_id = location['id'] if 'extra_info' in location: info['location_extra_info'] = location['extra_info'] assert info['start_time'] if 'has_start_time' not in info: info['has_start_time'] = True if not info['has_start_time']: # Event start time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['start_time'].tzinfo == pytz.utc: info['start_time'] = info['start_time'].astimezone(LOCAL_TZ) info['start_time'] = info['start_time'].replace(hour=0, minute=0, second=0) # If no end timestamp supplied, we treat the event as ending at midnight. if 'end_time' not in info or not info['end_time']: info['end_time'] = info['start_time'] info['has_end_time'] = False if 'has_end_time' not in info: info['has_end_time'] = True # If end date is supplied but no time, the event ends at midnight of the following day. if not info['has_end_time']: # Event end time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['end_time'].tzinfo == pytz.utc: info['end_time'] = info['start_time'].astimezone(LOCAL_TZ) info['end_time'] = info['end_time'].replace(hour=0, minute=0, second=0) info['end_time'] += datetime.timedelta(days=1) skip_fields = [ 'id', 'location', 'publisher', 'offers', 'keywords', 'images' ] self._update_fields(obj, info, skip_fields) self._set_field(obj, 'location_id', location_id) self._set_field(obj, 'publisher_id', info['publisher'].id) self._set_field(obj, 'deleted', False) if obj._created: # We have to save new objects here to be able to add related fields. # Changed objects will be saved only *after* related fields have been changed. try: obj.save() except ValidationError as error: logger.error('Event {} could not be saved: {}'.format( obj, error)) raise # many-to-many fields if 'images' in info: self.set_images(obj, info['images']) keywords = info.get('keywords', []) new_keywords = set([kw.id for kw in keywords]) old_keywords = set(obj.keywords.values_list('id', flat=True)) if new_keywords != old_keywords: if obj.is_user_edited(): # this prevents overwriting manually added keywords if not new_keywords <= old_keywords: obj.keywords.add(*new_keywords) obj._changed = True else: obj.keywords.set(new_keywords) obj._changed = True obj._changed_fields.append('keywords') audience = info.get('audience', []) new_audience = set([kw.id for kw in audience]) old_audience = set(obj.audience.values_list('id', flat=True)) if new_audience != old_audience: if obj.is_user_edited(): # this prevents overwriting manually added audience if not new_audience <= old_audience: obj.audience.add(*new_audience) obj._changed = True else: obj.audience.set(new_audience) obj._changed = True obj._changed_fields.append('audience') in_language = info.get('in_language', []) new_languages = set([lang.id for lang in in_language]) old_languages = set(obj.in_language.values_list('id', flat=True)) if new_languages != old_languages: if obj.is_user_edited(): # this prevents overwriting manually added languages if not new_languages <= old_languages: obj.in_language.add(*new_languages) obj._changed = True else: obj.in_language.set(in_language) obj._changed = True obj._changed_fields.append('in_language') # one-to-many fields with foreign key pointing to event offers = [] for offer in info.get('offers', []): offer_obj = Offer(event=obj) self._update_fields(offer_obj, offer, skip_fields=['id']) offers.append(offer_obj) val = operator.methodcaller('simple_value') if set(map(val, offers)) != set(map(val, obj.offers.all())): # this prevents overwriting manually added offers. do not update offers if we have added ones if not obj.is_user_edited() or len(set(map( val, offers))) >= obj.offers.count(): obj.offers.all().delete() for o in offers: o.save() obj._changed = True obj._changed_fields.append('offers') links = [] if 'external_links' in info: for lang in info['external_links'].keys(): for l in info['external_links'][lang]: l['language'] = lang links += info['external_links'][lang] # TODO: use simple_value logic like for offers above? def obj_make_link_id(obj): return '%s:%s:%s' % (obj.language_id, obj.name, obj.link) def info_make_link_id(info): return '%s:%s:%s' % (info['language'], info.get('name', ''), info['link']) new_links = set([info_make_link_id(link) for link in links]) old_links = set( [obj_make_link_id(link) for link in obj.external_links.all()]) if old_links != new_links: # this prevents overwriting manually added links. do not update links if we have added ones if not obj.is_user_edited() or len(new_links) >= len(old_links): obj.external_links.all().delete() for link in links: link_obj = EventLink(event=obj, language_id=link['language'], link=link['link']) if len(link['link']) > 200: continue if 'name' in link: link_obj.name = link['name'] link_obj.save() obj._changed = True obj._changed_fields.append('links') if 'extension_course' in settings.INSTALLED_APPS: extension_data = info.get('extension_course') if extension_data is not None: from extension_course.models import Course try: course = obj.extension_course course._changed = False for field in EXTENSION_COURSE_FIELDS: self._set_field(course, field, extension_data.get(field)) course_changed = course._changed if course_changed: course.save() except Course.DoesNotExist: Course.objects.create( event=obj, **{ field: extension_data.get(field) for field in EXTENSION_COURSE_FIELDS }) course_changed = True if course_changed: obj._changed = True obj._changed_fields.append('extension_course') # If event start time changed, it was rescheduled. if 'start_time' in obj._changed_fields: self._set_field(obj, 'event_status', Event.Status.RESCHEDULED) if obj._changed or obj._created: # Finally, we must save the whole object, even when only related fields changed. # Also, we want to log all that happened. try: obj.save() except ValidationError as error: print('Event ' + str(obj) + ' could not be saved: ' + str(error)) raise if obj._created: verb = "created" else: verb = "changed (fields: %s)" % ', '.join(obj._changed_fields) logger.debug("{} {}".format(obj, verb)) return obj
def save_event(self, info): info = info.copy() args = dict(data_source=info['data_source'], origin_id=info['origin_id']) obj_id = "%s:%s" % (info['data_source'].id, info['origin_id']) try: obj = Event.objects.get(**args) obj._created = False assert obj.id == obj_id except Event.DoesNotExist: obj = Event(**args) obj._created = True obj.id = obj_id obj._changed = False location_id = None if 'location' in info: location = info['location'] if 'id' in location: location_id = location['id'] if 'extra_info' in location: info['location_extra_info'] = location['extra_info'] assert info['start_time'] if 'has_start_time' not in info: info['has_start_time'] = True if not info['has_start_time']: # Event start time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['start_time'].tzinfo == pytz.utc: info['start_time'] = info['start_time'].astimezone(LOCAL_TZ) info['start_time'] = info['start_time'].replace(hour=0, minute=0, second=0) # If no end timestamp supplied, we treat the event as ending at midnight. if 'end_time' not in info or not info['end_time']: info['end_time'] = info['start_time'] info['has_end_time'] = False if 'has_end_time' not in info: info['has_end_time'] = True # If end date is supplied but no time, the event ends at midnight of the following day. if not info['has_end_time']: # Event end time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['end_time'].tzinfo == pytz.utc: info['end_time'] = info['start_time'].astimezone(LOCAL_TZ) info['end_time'] = info['end_time'].replace(hour=0, minute=0, second=0) info['end_time'] += datetime.timedelta(days=1) skip_fields = ['id', 'location', 'publisher', 'offers', 'keywords', 'images'] self._update_fields(obj, info, skip_fields) self._set_field(obj, 'location_id', location_id) self._set_field(obj, 'publisher_id', info['publisher'].id) self._set_field(obj, 'deleted', False) if obj._created or obj._changed: try: obj.save() except ValidationError as error: logger.error('Event {} could not be saved: {}'.format(obj, error)) raise # many-to-many fields if 'images' in info: self.set_images(obj, info['images']) keywords = info.get('keywords', []) new_keywords = set([kw.id for kw in keywords]) old_keywords = set(obj.keywords.values_list('id', flat=True)) if new_keywords != old_keywords: if obj.is_user_edited(): # this prevents overwriting manually added keywords if not new_keywords <= old_keywords: obj.keywords.add(*new_keywords) obj._changed = True else: obj.keywords.set(new_keywords) obj._changed = True audience = info.get('audience', []) new_audience = set([kw.id for kw in audience]) old_audience = set(obj.audience.values_list('id', flat=True)) if new_audience != old_audience: if obj.is_user_edited(): # this prevents overwriting manually added audience if not new_audience <= old_audience: obj.audience.add(*new_audience) obj._changed = True else: obj.audience.set(new_audience) obj._changed = True in_language = info.get('in_language', []) new_languages = set([lang.id for lang in in_language]) old_languages = set(obj.in_language.values_list('id', flat=True)) if new_languages != old_languages: if obj.is_user_edited(): # this prevents overwriting manually added languages if not new_languages <= old_languages: obj.in_language.add(*new_languages) obj._changed = True else: obj.in_language.set(in_language) obj._changed = True # one-to-many fields with foreign key pointing to event offers = [] for offer in info.get('offers', []): offer_obj = Offer(event=obj) self._update_fields(offer_obj, offer, skip_fields=['id']) offers.append(offer_obj) val = operator.methodcaller('simple_value') if set(map(val, offers)) != set(map(val, obj.offers.all())): # this prevents overwriting manually added offers. do not update offers if we have added ones if not obj.is_user_edited() or len(set(map(val, offers))) >= obj.offers.count(): obj.offers.all().delete() for o in offers: o.save() obj._changed = True links = [] if 'external_links' in info: for lang in info['external_links'].keys(): for l in info['external_links'][lang]: l['language'] = lang links += info['external_links'][lang] # TODO: use simple_value logic like for offers above? def obj_make_link_id(obj): return '%s:%s:%s' % (obj.language_id, obj.name, obj.link) def info_make_link_id(info): return '%s:%s:%s' % (info['language'], info.get('name', ''), info['link']) new_links = set([info_make_link_id(link) for link in links]) old_links = set([obj_make_link_id(link) for link in obj.external_links.all()]) if old_links != new_links: # this prevents overwriting manually added links. do not update links if we have added ones if not obj.is_user_edited() or len(new_links) >= len(old_links): obj.external_links.all().delete() for link in links: link_obj = EventLink(event=obj, language_id=link['language'], link=link['link']) if len(link['link']) > 200: continue if 'name' in link: link_obj.name = link['name'] link_obj.save() obj._changed = True if 'extension_course' in settings.INSTALLED_APPS: extension_data = info.get('extension_course') if extension_data is not None: from extension_course.models import Course try: course = obj.extension_course course._changed = False for field in EXTENSION_COURSE_FIELDS: self._set_field(course, field, extension_data.get(field)) course_changed = course._changed if course_changed: course.save() except Course.DoesNotExist: Course.objects.create( event=obj, **{field: extension_data.get(field) for field in EXTENSION_COURSE_FIELDS} ) course_changed = True if course_changed: obj._changed = True if obj._changed or obj._created: # save again after adding related fields to update last_modified_time! try: obj.save() except ValidationError as error: print('Event ' + str(obj) + ' could not be saved: ' + str(error)) raise if obj._created: verb = "created" else: verb = "changed" logger.debug("{} {}".format(obj, verb)) return obj
def save_event(self, info): info = info.copy() args = dict(data_source=info['data_source'], origin_id=info['origin_id']) obj_id = "%s:%s" % (info['data_source'].id, info['origin_id']) try: obj = Event.objects.get(**args) obj._created = False assert obj.id == obj_id except Event.DoesNotExist: obj = Event(**args) obj._created = True obj.id = obj_id obj._changed = False location_id = None if 'location' in info: location = info['location'] if 'id' in location: location_id = location['id'] if 'extra_info' in location: info['location_extra_info'] = location['extra_info'] assert info['start_time'] if 'has_start_time' not in info: info['has_start_time'] = True if not info['has_start_time']: info['start_time'] = info['start_time'].replace(hour=0, minute=0, second=0) # If no end timestamp supplied, we treat the event as ending at midnight. if 'end_time' not in info or not info['end_time']: info['end_time'] = info['start_time'] info['has_end_time'] = False if 'has_end_time' not in info: info['has_end_time'] = True # If end date is supplied but no time, the event ends at midnight of the following day. if not info['has_end_time']: info['end_time'] = info['end_time'].replace(hour=0, minute=0, second=0) info['end_time'] += datetime.timedelta(days=1) skip_fields = ['id', 'location', 'publisher', 'offers', 'keywords', 'image', 'image_license'] self._update_fields(obj, info, skip_fields) self._set_field(obj, 'location_id', location_id) self._set_field(obj, 'publisher_id', info['publisher'].id) image_url = info.get('image', '').strip() image_object = self.get_or_create_image(image_url) if 'image_license' in info: license_id = info['image_license'] if image_object.license_id != license_id: try: license_object = License.objects.get(id=license_id) except License.DoesNotExist: print('Invalid license id "%s" image %s event %s' % (license_id, image_url, obj)) return image_object.license = license_object image_object.save(update_fields=('license',)) self.set_image(obj, image_object) self._set_field(obj, 'deleted', False) if obj._created or obj._changed: try: obj.save() except ValidationError as error: print('Event ' + str(obj) + ' could not be saved: ' + str(error)) raise # many-to-many fields keywords = info.get('keywords', []) new_keywords = set([kw.id for kw in keywords]) old_keywords = set(obj.keywords.values_list('id', flat=True)) if new_keywords != old_keywords: if obj.is_user_edited(): # this prevents overwriting manually added keywords if not new_keywords <= old_keywords: obj.keywords.add(*new_keywords) obj._changed = True else: obj.keywords = new_keywords obj._changed = True audience = info.get('audience', []) new_audience = set([kw.id for kw in audience]) old_audience = set(obj.audience.values_list('id', flat=True)) if new_audience != old_audience: if obj.is_user_edited(): # this prevents overwriting manually added audience if not new_audience <= old_audience: obj.audience.add(*new_audience) obj._changed = True else: obj.audience = new_audience obj._changed = True # one-to-many fields with foreign key pointing to event offers = [] for offer in info.get('offers', []): offer_obj = Offer(event=obj) self._update_fields(offer_obj, offer, skip_fields=['id']) offers.append(offer_obj) val = operator.methodcaller('simple_value') if set(map(val, offers)) != set(map(val, obj.offers.all())): # this prevents overwriting manually added offers. do not update offers if we have added ones if not obj.is_user_edited() or len(set(map(val, offers))) >= obj.offers.count(): obj.offers.all().delete() for o in offers: o.save() obj._changed = True links = [] if 'external_links' in info: for lang in info['external_links'].keys(): for l in info['external_links'][lang]: l['language'] = lang links += info['external_links'][lang] # TODO: use simple_value logic like for offers above? def obj_make_link_id(obj): return '%s:%s:%s' % (obj.language_id, obj.name, obj.link) def info_make_link_id(info): return '%s:%s:%s' % (info['language'], info.get('name', ''), info['link']) new_links = set([info_make_link_id(link) for link in links]) old_links = set([obj_make_link_id(link) for link in obj.external_links.all()]) if old_links != new_links: # this prevents overwriting manually added links. do not update links if we have added ones if not obj.is_user_edited() or len(new_links) >= len(old_links): obj.external_links.all().delete() for link in links: link_obj = EventLink(event=obj, language_id=link['language'], link=link['link']) if len(link['link']) > 200: continue if 'name' in link: link_obj.name = link['name'] link_obj.save() obj._changed = True if obj._changed or obj._created: if obj._created: verb = "created" else: verb = "changed" print("%s %s" % (obj, verb)) return obj
def save_event(self, info): info = info.copy() args = dict(data_source=info['data_source'], origin_id=info['origin_id']) obj_id = "%s:%s" % (info['data_source'].id, info['origin_id']) try: obj = Event.objects.get(**args) obj._created = False assert obj.id == obj_id except Event.DoesNotExist: obj = Event(**args) obj._created = True obj.id = obj_id obj._changed = False obj._changed_fields = [] location_id = None if 'location' in info: location = info['location'] if 'id' in location: location_id = location['id'] if 'extra_info' in location: info['location_extra_info'] = location['extra_info'] assert info['start_time'] if 'has_start_time' not in info: info['has_start_time'] = True if not info['has_start_time']: # Event start time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['start_time'].tzinfo == pytz.utc: info['start_time'] = info['start_time'].astimezone(LOCAL_TZ) info['start_time'] = info['start_time'].replace(hour=0, minute=0, second=0) # If no end timestamp supplied, we treat the event as ending at midnight. if 'end_time' not in info or not info['end_time']: info['end_time'] = info['start_time'] info['has_end_time'] = False if 'has_end_time' not in info: info['has_end_time'] = True # If end date is supplied but no time, the event ends at midnight of the following day. if not info['has_end_time']: # Event end time is not exactly defined. # Use midnight in event timezone, or, if given in utc, local timezone if info['end_time'].tzinfo == pytz.utc: info['end_time'] = info['start_time'].astimezone(LOCAL_TZ) info['end_time'] = info['end_time'].replace(hour=0, minute=0, second=0) info['end_time'] += datetime.timedelta(days=1) skip_fields = [ 'id', 'location', 'publisher', 'offers', 'keywords', 'images' ] self._update_fields(obj, info, skip_fields) self._set_field(obj, 'location_id', location_id) self._set_field(obj, 'publisher_id', info['publisher'].id) self._set_field(obj, 'deleted', False) if obj._created: # We have to save new objects here to be able to add related fields. # Changed objects will be saved only *after* related fields have been changed. try: obj.save() except ValidationError as error: logger.error('Event {} could not be saved: {}'.format( obj, error)) raise # many-to-many fields # if images change and event has been user edited, do not reinstate old image!!! if not obj.is_user_edited() and 'images' in info: self.set_images(obj, info['images']) keywords = info.get('keywords', []) new_keywords = set([kw.id for kw in keywords]) old_keywords = set(obj.keywords.values_list('id', flat=True)) if new_keywords != old_keywords: if obj.is_user_edited(): # this prevents overwriting manually added keywords if not new_keywords <= old_keywords: obj.keywords.add(*new_keywords) obj._changed = True elif self._has_espoo_keywords(old_keywords): # This prevents overwriting place keywords added by the add_espoo_places management command. # Note that this a somewhat ugly hack since it tries to infer whether the event keywords have been # modified by, e.g., the add_espoo_places management command by checking if the event has any Espoo # keywords. That is, if the event hasn't been modified by a user but it has Espoo keywords, the Espoo # keywords have probably been added by the add_espoo_places management command. In that case, we don't # want to overwrite the Espoo keywords added by the add_espoo_places management command. Instead, we # want to merge the Espoo keywords added by the add_espoo_places management command with the new # keywords. This way, if any of the non-Espoo keywords have changed, then those changes will be # reflected in the event's keywords. # # Note that this doesn't necessarily work correctly if the importer itself has added any Espoo keywords # for the event. The check for Espoo keywords naively relies on checking whether a keyword is prefixed # with ':espoo' or not. Thus, it can't distinguish whether a Espoo keyword has been added by the # add_espoo_places management command or by the importer itself. So, if the importer itself has added # any Espoo keywords previously but they have been deleted in data source after that, then the event # will still have those keywords since the following code assumes that they've been added by the # add_espoo_places command and thus keeps them. espoo_keywords = self._get_espoo_keywords_from_set( old_keywords) obj.keywords.set(new_keywords.union(espoo_keywords)) obj._changed = True else: obj.keywords.set(new_keywords) obj._changed = True obj._changed_fields.append('keywords') audience = info.get('audience', []) new_audience = set([kw.id for kw in audience]) old_audience = set(obj.audience.values_list('id', flat=True)) if new_audience != old_audience: if obj.is_user_edited(): # this prevents overwriting manually added audience if not new_audience <= old_audience: obj.audience.add(*new_audience) obj._changed = True elif self._has_espoo_keywords(old_audience): # This prevents overwriting audience keywords added by the add_espoo_audience management command. # Note that this a somewhat ugly hack since it tries to infer whether the event audiences have been # modified by, e.g., the add_espoo_audience management command by checking if the event has any Espoo # audiences. That is, if the event hasn't been modified by a user but it has Espoo audiences, the Espoo # audiences have probably been added by the add_espoo_audience management command. In that case, we # don't want to overwrite the Espoo audiences added by the add_espoo_audience management command. # Instead, we want to merge the Espoo audiences added by the add_espoo_audience management command with # the new audiences. This way, if any of the non-Espoo audiences have changed, then those changes will # be reflected in the event's audiences. # # Note that this doesn't necessarily work correctly if the importer itself has added any Espoo # audiences for the event. The check for Espoo audiences naively relies on checking whether an audience # is prefixed with ':espoo' or not. Thus, it can't distinguish whether an Espoo audience has been added # by the add_espoo_audience management command or by the importer itself. So, if the importer itself # has added any Espoo audiences previously but they have been deleted in data source after that, then # the event will still have those audiences since the following code assumes that they've been added by # the add_espoo_audience command and thus keeps them. espoo_audiences = self._get_espoo_keywords_from_set( old_audience) obj.audience.set(new_audience.union(espoo_audiences)) obj._changed = True else: obj.audience.set(new_audience) obj._changed = True obj._changed_fields.append('audience') in_language = info.get('in_language', []) new_languages = set([lang.id for lang in in_language]) old_languages = set(obj.in_language.values_list('id', flat=True)) if new_languages != old_languages: if obj.is_user_edited(): # this prevents overwriting manually added languages if not new_languages <= old_languages: obj.in_language.add(*new_languages) obj._changed = True else: obj.in_language.set(in_language) obj._changed = True obj._changed_fields.append('in_language') # one-to-many fields with foreign key pointing to event offers = [] for offer in info.get('offers', []): offer_obj = Offer(event=obj) self._update_fields(offer_obj, offer, skip_fields=['id']) offers.append(offer_obj) val = operator.methodcaller('simple_value') if set(map(val, offers)) != set(map(val, obj.offers.all())): # this prevents overwriting manually added offers. do not update offers if we have added ones if not obj.is_user_edited() or len(set(map( val, offers))) >= obj.offers.count(): obj.offers.all().delete() for o in offers: o.save() obj._changed = True obj._changed_fields.append('offers') if info['external_links']: ExternalLink = namedtuple('ExternalLink', ['language', 'name', 'url']) def list_obj_links(obj): links = set() for link in obj.external_links.all(): links.add( ExternalLink(link.language.id, link.name, link.link)) return links def list_external_links(external_links): links = set() for language in external_links.keys(): for link_name in external_links[language].keys(): links.add( ExternalLink(language, link_name, external_links[language][link_name])) return links new_links = list_external_links(info['external_links']) old_links = list_obj_links(obj) if not obj.is_user_edited() and new_links != old_links: obj.external_links.all().delete() for link in new_links: if len(link.url) > 200: logger.error( f'{obj} required external link of length {len(link.url)}, current limit 200' ) continue link_obj = EventLink(event=obj, language_id=link.language, name=link.name, link=link.url) link_obj.save() obj._changed = True obj._changed_fields.append('links') if 'extension_course' in settings.INSTALLED_APPS: extension_data = info.get('extension_course') if extension_data is not None: from extension_course.models import Course try: course = obj.extension_course course._changed = False for field in EXTENSION_COURSE_FIELDS: self._set_field(course, field, extension_data.get(field)) course_changed = course._changed if course_changed: course.save() except Course.DoesNotExist: Course.objects.create( event=obj, **{ field: extension_data.get(field) for field in EXTENSION_COURSE_FIELDS }) course_changed = True if course_changed: obj._changed = True obj._changed_fields.append('extension_course') # If event start time changed, it was rescheduled. if 'start_time' in obj._changed_fields: self._set_field(obj, 'event_status', Event.Status.RESCHEDULED) # The event may be cancelled status = info.get('event_status', None) if status: self._set_field(obj, 'event_status', status) if obj._changed or obj._created: # Finally, we must save the whole object, even when only related fields changed. # Also, we want to log all that happened. try: obj.save() except ValidationError as error: print('Event ' + str(obj) + ' could not be saved: ' + str(error)) raise if obj._created: verb = "created" else: verb = "changed (fields: %s)" % ', '.join(obj._changed_fields) logger.debug("{} {}".format(obj, verb)) return obj