def save_place(self, info):
args = dict(data_source=info['data_source'],
origin_id=info['origin_id'])
obj_id = "%s:%s" % (info['data_source'].id, info['origin_id'])
try:
obj = Place.objects.get(**args)
obj._created = False
assert obj.id == obj_id
except Place.DoesNotExist:
obj = Place(**args)
obj._created = True
obj.id = obj_id
obj._changed = False
skip_fields = ['id', 'position', 'custom_fields', 'publisher']
self._update_fields(obj, info, skip_fields)
n = info.get('latitude', 0)
e = info.get('longitude', 0)
position = None
if n and e:
p = Point(e, n, srid=4326) # GPS coordinate system
if p.within(self.bounding_box):
if self.target_srid != 4326:
p.transform(self.gps_to_target_ct)
position = p
else:
logger.warning("Invalid coordinates (%f, %f) for %s" %
(n, e, obj))
if position and obj.position:
# If the distance is less than 10cm, assume the position
# hasn't changed.
assert obj.position.srid == settings.PROJECTION_SRID
if position.distance(obj.position) < 0.10:
position = obj.position
if position != obj.position:
obj._changed = True
obj.position = position
# we may end up reinstating deleted locations whenever they are imported back and forth
if obj.deleted:
obj.deleted = False
obj._changed = True
self._set_field(obj, 'publisher_id', info['publisher'].id)
if obj._changed or obj._created:
if obj._created:
verb = "created"
else:
verb = "changed"
logger.debug("%s %s" % (obj, verb))
obj.save()
return obj
def get_or_create_place_id(self, street_address):
"""
Return the id of the event place corresponding to the street_address.
Create the event place if not found.
Espoo website does not maintain a place object with a dedicated id.
This function tries to map the address to an existing place or create
a new one if no place is found.
"""
address_data = clean_street_address(street_address)
street_address = address_data.get('street_address', None)
if not street_address:
return
espoo_loc_id = self.location_cache.get(street_address, None)
if espoo_loc_id:
return espoo_loc_id
places = Place.objects.filter(deleted=False,
street_address__icontains='%s' %
street_address).order_by('id')
place = places.first() # Choose one place arbitrarily if many.
if len(places) > 1:
self.logger.warning('Several tprek_id match the address "%s".' %
street_address)
if not place:
origin_id = self._get_next_place_id("espoo")
address_data.update({
'publisher': self.organization,
'origin_id': origin_id,
'id': 'espoo:%s' % origin_id,
'data_source': self.data_source,
})
place = Place(**address_data)
place.save()
# Cached the location to speed up
self.location_cache.update({street_address: place.id})
return place.id
def _import_unit(self, syncher, info):
n = 0
e = 0
isPositionOk = True
try:
n = info['location']['coordinates'][0] # latitude
e = info['location']['coordinates'][1] # longitude
except:
isPositionOk == False
POSITIONERROR.append(info['id'])
if isPositionOk:
obj = syncher.get(str(info['id']))
obj_id = 'tpr:%s' % str(info['id'])
if not obj:
obj = Place(data_source=self.data_source, origin_id=info['id'])
obj._changed = True
obj._created = True
obj.id = obj_id
else:
assert obj.id == obj_id
obj._created = False
obj._changed_fields = []
try:
self._save_translated_field_multilevel(obj, 'name', info, 'name')
self._save_translated_field_multilevel(obj, 'description', info, 'description')
self._save_translated_field_multilevel(obj, 'street_address', info, 'street_address')
except:
NONEVALUES.append('unit ID: ' + str(info['id']) + ' some multilanguage field is none')
pass
try:
self._save_field(obj, 'info_url', info, 'www', max_length=1000)
except:
NONEVALUES.append('unit ID: ' + str(info['id']) + ' info_url field is empty!')
pass
try:
self._save_field(obj, 'address_locality', info, 'municipality')
except:
NONEVALUES.append('unit ID: ' + str(info['id']) + ' address_locality field is empty!')
pass
try:
self._save_field(obj, 'telephone', info, 'phone')
except:
NONEVALUES.append('unit ID: ' + str(info['id']) + ' telephone field is empty!')
pass
field_map = {
'address_zip': 'postal_code',
'address_postal_full': None,
'email': 'email',
}
for src_field, obj_field in field_map.items():
if not obj_field:
continue
val = info.get(src_field, None)
if getattr(obj, obj_field) != val:
setattr(obj, obj_field, val)
obj._changed_fields.append(obj_field)
obj._changed = True
position = None
if n and e:
if os.name == 'nt':
p = Point(e, n, srid=4326) # GPS coordinate system (WGS 84)
else:
p = Point(n, e, srid=4326) # GPS coordinate system (WGS 84)
if p.within(self.bounding_box):
if self.target_srid != 4326:
p.transform(self.gps_to_target_ct)
position = p
else:
logger.warning("Invalid coordinates (%f, %f) for %s" % (n, e, obj))
try:
picture_url = info.get('picture_url', '').strip()
except:
NONEVALUES.append('unit ID: ' + str(info['id']) + ' picture_url field is empty!')
pass
if position and obj.position:
# If the distance is less than 10cm, assume the location
# hasn't changed.
assert obj.position.srid == settings.PROJECTION_SRID
if position.distance(obj.position) < 0.10:
position = obj.position
if position != obj.position:
obj._changed = True
obj._changed_fields.append('position')
obj.position = position
if obj.publisher_id != self.organization.id:
obj.publisher = self.organization
obj._changed_fields.append('publisher')
obj._changed = True
if obj.deleted:
obj.deleted = False
replace_location(from_source='matko', by=obj)
obj._changed_fields.append('deleted')
obj._changed = True
if obj._changed:
if obj._created:
verb = "created"
else:
verb = "changed (fields: %s)" % ', '.join(obj._changed_fields)
logger.info("%s %s" % (obj, verb))
obj.save()
syncher.mark(obj)
def get_or_create_place_id(self, street_address, lang, name='', url=''):
"""
Return the id of the event place corresponding to the street_address.
Create the event place with the address, name and url if not found.
Espoo website does not maintain a place object with a dedicated id.
This function tries to map the address to an existing place or create
a new one if no place is found.
:param street_address: The exact street address of the location
:type street_address: String
:param lang: The language the strings are provided in
:type lang: String
:param name: Optional name for the location
:type name: String
:param url: Optional info URL for the location
:type url: String
:rtype: id for the location
"""
address_data = clean_street_address(street_address)
street_address = address_data.get('street_address', None)
if not street_address:
return
espoo_loc_id = self.location_cache.get(street_address, None)
if espoo_loc_id:
return espoo_loc_id
address_lang = lang
if address_lang not in ADDRESS_LANGUAGES:
# pick the first preferred language (e.g. Finnish) if lang (e.g. English) has no address translations
address_lang = ADDRESS_LANGUAGES[0]
# do not use street addresses, we want metadata for espoo event locations too!
filter_params = {
'data_source__in': (self.tprek_data_source, self.data_source),
'deleted': False,
'street_address_' + address_lang + '__icontains': street_address,
}
# prefer tprek, prefer existing event locations
places = Place.objects.filter(**filter_params).order_by(
'-data_source', '-n_events')
place = places.first() # Choose one place arbitrarily if many.
if len(places) > 1:
logger.warning(
'Several tprek and/or espoo id match the address "{}".'.format(
street_address))
if not place:
origin_id = self._get_next_place_id("espoo")
# address must be saved in the right language!
address_data['street_address_' +
address_lang] = address_data.pop('street_address')
address_data.update({
'publisher': self.organization,
'origin_id': origin_id,
'id': 'espoo:%s' % origin_id,
'data_source': self.data_source,
'name_' + lang: name,
'info_url_' + lang: url
})
place = Place(**address_data)
place.save()
elif place.data_source == self.data_source:
# update metadata in the given language if the place belongs to espoo:
setattr(place, 'name_' + lang, name)
setattr(place, 'info_url_' + lang, url)
setattr(place, 'street_address_' + address_lang, street_address)
place.save(update_fields=[
'name_' + lang, 'info_url_' + lang, 'street_address_' + lang
])
# Cached the location to speed up
self.location_cache.update(
{street_address:
place.id}) # location cache need not care about address language
return place.id
def _import_address(self, syncher, address_obj):
# addresses have no static ids, just format the address cleanly
origin_id = str(address_obj).replace(' - ',
'-').replace(',', '').replace(
' ', '_').lower()
obj = syncher.get(origin_id)
obj_id = 'osoite:' + origin_id
if not obj:
obj = Place(data_source=self.data_source, origin_id=origin_id)
obj._changed = True
obj._created = True
obj.id = obj_id
else:
assert obj.id == obj_id
obj._created = False
obj._changed_fields = []
# we must construct the names and street addresses from the address object
info = {}
for lang in self.supported_languages:
info['name_' + lang] = self.get_whole_address(address_obj, lang)
info['street_address_' + lang] = self.get_street_address(
address_obj, lang)
info['municipality_' + lang] = getattr(address_obj.street.municipality, 'name_' + lang) \
or getattr(address_obj.street.municipality, 'name_fi')
self._save_translated_field(obj, 'name', info, 'name')
self._save_translated_field(obj, 'street_address', info,
'street_address')
self._save_translated_field(obj, 'address_locality', info,
'municipality')
position = address_obj.location
#print("[OSOITE.PY]: ---> POSITION: ", position)
#position = None
obj._changed = True
if position and obj.position:
# If the distance is less than 10cm, assume the location
# hasn't changed.
assert obj.position.srid == settings.PROJECTION_SRID
if position.distance(obj.position) < 0.10:
position = obj.position
#position = None
if position != obj.position:
obj._changed = True
obj._changed_fields.append('position')
obj.position = position
if obj.publisher_id != self.organization.id:
obj.publisher = self.organization
obj._changed_fields.append('publisher')
obj._changed = True
if obj.deleted:
obj.deleted = False
# address has been reinstated, hip hip hooray!
# there's no way we can find any events from other addresses that should now be in this address
# so we cannot do address replace here (the way we do with tprek units)
obj._changed_fields.append('deleted')
obj._changed = True
if obj._changed:
if obj._created:
verb = "created"
else:
verb = "changed (fields: %s)" % ', '.join(obj._changed_fields)
logger.info("%s %s" % (obj, verb))
obj.save()
syncher.mark(obj)
def _import_unit(self, syncher, info):
obj = syncher.get(str(info['id']))
obj_id = 'tprek:%s' % str(info['id'])
if not obj:
obj = Place(data_source=self.data_source, origin_id=info['id'])
obj._changed = True
obj._created = True
obj.id = obj_id
else:
assert obj.id == obj_id
obj._created = False
obj._changed_fields = []
self._save_translated_field(obj, 'name', info, 'name')
self._save_translated_field(obj, 'description', info, 'desc')
self._save_translated_field(obj, 'street_address', info, 'street_address')
self._save_translated_field(obj, 'address_locality', info, 'address_city')
self._save_translated_field(obj, 'info_url', info, 'www', max_length=1000)
self._save_field(obj, 'telephone', info, 'phone')
field_map = {
'address_zip': 'postal_code',
'address_postal_full': None,
'email': 'email',
}
for src_field, obj_field in field_map.items():
if not obj_field:
continue
val = info.get(src_field, None)
if getattr(obj, obj_field) != val:
setattr(obj, obj_field, val)
obj._changed_fields.append(obj_field)
obj._changed = True
n = info.get('latitude', 0)
e = info.get('longitude', 0)
position = None
if n and e:
p = Point(e, n, srid=4326) # GPS coordinate system
if p.within(self.bounding_box):
if self.target_srid != 4326:
p.transform(self.gps_to_target_ct)
position = p
else:
logger.warning("Invalid coordinates (%f, %f) for %s" % (n, e, obj))
picture_url = info.get('picture_url', '').strip()
if picture_url:
self.set_image(obj, {'url': picture_url})
if position and obj.position:
# If the distance is less than 10cm, assume the location
# hasn't changed.
assert obj.position.srid == settings.PROJECTION_SRID
if position.distance(obj.position) < 0.10:
position = obj.position
if position != obj.position:
obj._changed = True
obj._changed_fields.append('position')
obj.position = position
if obj.publisher_id != self.organization.id:
obj.publisher = self.organization
obj._changed_fields.append('publisher')
obj._changed = True
if obj.deleted:
obj.deleted = False
# location has been reinstated in tprek, hip hip hooray!
replace_location(from_source='matko', by=obj)
obj._changed_fields.append('deleted')
obj._changed = True
if obj._changed:
if obj._created:
verb = "created"
else:
verb = "changed (fields: %s)" % ', '.join(obj._changed_fields)
logger.info("%s %s" % (obj, verb))
obj.save()
syncher.mark(obj)
def get_or_create_place_id(self, street_address, lang, name='', url=''):
"""
Return the id of the event place corresponding to the street_address.
Create the event place with the address, name and url if not found.
Espoo website does not maintain a place object with a dedicated id.
This function tries to map the address to an existing place or create
a new one if no place is found.
:param street_address: The exact street address of the location
:type street_address: String
:param lang: The language the strings are provided in
:type lang: String
:param name: Optional name for the location
:type name: String
:param url: Optional info URL for the location
:type url: String
:rtype: id for the location
"""
address_data = clean_street_address(street_address)
street_address = address_data.get('street_address', None)
if not street_address:
return
espoo_loc_id = self.location_cache.get(street_address, None)
if espoo_loc_id:
return espoo_loc_id
filter_params = {
'deleted': False,
'street_address_' + lang + '__icontains': street_address,
}
places = Place.objects.filter(**filter_params).order_by('id')
place = places.first() # Choose one place arbitrarily if many.
if len(places) > 1:
self.logger.warning('Several tprek_id match the address "%s".' %
street_address)
if not place:
origin_id = self._get_next_place_id("espoo")
address_data.update({
'publisher': self.organization,
'origin_id': origin_id,
'id': 'espoo:%s' % origin_id,
'data_source': self.data_source,
'name_' + lang: name,
'info_url_' + lang: url
})
place = Place(**address_data)
place.save()
elif place.data_source == self.data_source:
# update metadata in the given language if the place belongs to espoo:
setattr(place, 'name_' + lang, name)
setattr(place, 'info_url_' + lang, url)
setattr(place, 'street_address_' + lang, street_address)
place.save(update_fields=[
'name_' + lang, 'info_url_' + lang, 'street_address_' + lang
])
# Cached the location to speed up
self.location_cache.update({street_address: place.id})
return place.id
def do(self):
logger.debug("cron job starting for: FB")
#get the event worker for heltech
try:
ew = EventWorker.objects.get(
page_name="HEL Tech") #assume one worker for hel tech
except ObjectDoesNotExist:
logger.error("No event workers configured, cannot check events!")
return
except MultipleObjectsReturned:
logger.error(
"Multiple workers found, do not know which one to run yet!")
return
graph = facebook.GraphAPI(access_token=ew.page_token, version="2.1")
#get the page events and the database events
fb_events = graph.get_connections(id=ew.page_id,
connection_name="events")
db_events = Event.objects.all()
for event in fb_events['data']:
try:
e = Event.objects.get(eid=event['id'])
except ObjectDoesNotExist:
e = Event()
ep = EventParser(event, ew.parse_speakers)
ep.parse()
#exists, lets update the participant count
ac = graph.get_object(id=event['id'], fields='attending_count')
e.attending_count = ac['attending_count'] or 0
# fetch the cover photo
cover = graph.get_object(id=event['id'], fields=['cover'])
try:
e.cover_uri = cover['cover']['source']
except KeyError:
e.cover_uri = ""
# fill in the data
e.eid = ep.eid
e.title = ep.title
e.start_time = ep.start_time
e.end_time = ep.end_time
e.programme = ep.programme
e.description = ep.description
e.punchline = ep.punchline
#add the place
try:
p = Place.objects.get(name=ep.place)
except ObjectDoesNotExist:
p = Place()
p.name = ep.place
p.streetaddr = ep.addr
p.save()
e.place = p
e.save() #that's all for now
# fill the speakers
if ew.parse_speakers:
edt = parser.parse(e.start_time)
now = datetime.datetime.now(edt.tzinfo)
if edt >= now or (
not Speaker.objects.filter(event__eid=e.eid).exists()):
(count, thedict) = speakers = Speaker.objects.filter(
event__eid=e.eid).delete()
logger.debug("deleted: " + str(count) +
" speakers, details: " + str(thedict))
for speaker in ep.speakers:
org = None
sobj = Speaker()
if speaker['org']:
try:
org = Organisation.objects.get(
name=speaker['org'])
except ObjectDoesNotExist:
org = Organisation()
org.name = speaker['org']
org.save()
speaker = Speaker(full_name=speaker['name'],
title=speaker['title'],
role=speaker['role'],
organisation=org,
event=e)
speaker.save()