def sync_content_tourism(sender, **kwargs):
if not settings.TOURISM_ENABLED:
return
server_settings = kwargs['server_settings']
inputkw = kwargs['input_kwargs']
languages = server_settings.languages.available.keys()
logger.debug("Languages: %s" % languages)
for language in languages:
inputkwlang = dict(language=language, **inputkw)
InputFile(DataSource.filepath, **inputkwlang).pull_if_modified()
for datasource in DataSource.tmp_objects.filter(
language=language).all():
try:
InputFile(datasource.geojson_url,
**inputkwlang).pull_if_modified()
except IOError:
logger.error("Could not retrieve datasource %s" %
datasource.geojson_url)
if not settings.TOURISM_DATASOURCE_FAULT_TOLERANT:
raise
for datasource in DataSource.tmp_objects.filter(
language=language).all():
InputFile(datasource.pictogram_url, **inputkw).pull_if_modified()
def content(self):
try:
logger.debug("Read content from %s" % self.fullpath)
with open(self.fullpath, 'r') as f:
content = f.read()
return content
except IOError:
logger.error("Could not read '%s'" % self.fullpath)
return '[]'
def content(self):
try:
logger.debug("Read content from %s" % self.fullpath)
with open(self.fullpath, 'r') as f:
content = f.read()
return content
except IOError:
logger.error("Could not read '%s'" % self.fullpath)
return '[]'
def pull(self, ifmodified=False):
"""
Pull a file served by a Geotrek server.
Set 'if-modified-since' HTTP request header to reduce bandwidth.
"""
headers = {}
if self.language:
cprint('/' + self.language, 'cyan', end='', file=self.stdout)
headers.update({'Accept-language': self.language})
if ifmodified:
try:
# If local file is empty, force retrieval
assert getsize(self.path) > settings.MIN_BYTE_SYZE
# Use datetime of previous file to set header
mtime = getmtime(self.path)
headers.update({'if-modified-since': http_date(mtime)})
except (OSError, AssertionError):
pass
cprint('/%s ...' % self.url,
'white',
attrs=['bold'],
end=' ',
file=self.stdout)
self.stdout.flush()
self.reply = self.client.get(self.url, headers=headers)
if self.reply.status_code in (304, ):
cprint("%s (Up-to-date)" % self.reply.status_code,
'green',
attrs=['bold'],
file=self.stdout)
return
elif self.reply.status_code != requests.codes.ok:
cprint("%s (Failed)" % self.reply.status_code,
'red',
attrs=['bold'],
file=self.stderr)
raise IOError("Failed to retrieve %s (code: %s)" %
(self.reply.url, self.reply.status_code))
else:
cprint("%s (Download)" % self.reply.status_code,
'yellow',
file=self.stdout)
mkdir_p(dirname(self.path_tmp))
with open(self.path_tmp, 'wb') as f:
f.write(self.content())
f.write("\n")
logger.debug(" %s\n" %
self.path.replace(settings.INPUT_DATA_ROOT, ''))
last_modified = parse_http_date_safe(
self.reply.headers.get('last-modified'))
if last_modified:
utime(self.path_tmp, (last_modified, last_modified))
def sync_content_feedback(sender, **kwargs):
if not settings.FEEDBACK_FORM_ENABLED:
return
server_settings = kwargs['server_settings']
inputkw = kwargs['input_kwargs']
languages = server_settings.languages.available.keys()
logger.debug("Languages: %s" % languages)
for language in languages:
inputkwlang = dict(language=language, **inputkw)
InputFile(FeedbackCategory.filepath, **inputkwlang).pull_if_modified()
def post(self, url, **kwargs):
url = self.absolute_url(url)
kwargs = self._set_headers_kwargs(kwargs)
data = kwargs['data']
if 'csrfmiddlewaretoken' not in data:
response = self.session.head(url)
assert response.status_code == requests.codes.ok, "Failed to obtain CSRF token"
csrftoken = response.cookies.get('csrftoken', '')
data['csrfmiddlewaretoken'] = csrftoken
kwargs['data'] = data
logger.debug("POST %s (%s)" % (url, kwargs))
return self.session.post(url, **kwargs)
def post(self, url, **kwargs):
url = self.absolute_url(url)
kwargs = self._set_headers_kwargs(kwargs)
data = kwargs['data']
if 'csrfmiddlewaretoken' not in data:
response = self.session.head(url)
assert response.status_code == requests.codes.ok, "Failed to obtain CSRF token"
csrftoken = response.cookies.get('csrftoken', '')
data['csrfmiddlewaretoken'] = csrftoken
kwargs['data'] = data
logger.debug("POST %s (%s)" % (url, kwargs))
return self.session.post(url, **kwargs)
def pull(self, ifmodified=False):
"""
Pull a file served by a Geotrek server.
Set 'if-modified-since' HTTP request header to reduce bandwidth.
"""
headers = {}
if self.language:
cprint('/' + self.language, 'cyan', end='', file=self.stdout)
headers.update({'Accept-language': self.language})
if ifmodified:
try:
# If local file is empty, force retrieval
assert getsize(self.path) > settings.MIN_BYTE_SYZE
# Use datetime of previous file to set header
mtime = getmtime(self.path)
headers.update({'if-modified-since': http_date(mtime)})
except (OSError, AssertionError):
pass
cprint('/%s ...' % self.url, 'white', attrs=['bold'], end=' ', file=self.stdout)
self.stdout.flush()
self.reply = self.client.get(self.url, headers=headers)
if self.reply.status_code in (304,):
cprint("%s (Up-to-date)" % self.reply.status_code, 'green', attrs=['bold'], file=self.stdout)
return
elif self.reply.status_code != requests.codes.ok:
cprint("%s (Failed)" % self.reply.status_code, 'red', attrs=['bold'], file=self.stderr)
raise IOError("Failed to retrieve %s (code: %s)" % (self.reply.url,
self.reply.status_code))
else:
cprint("%s (Download)" % self.reply.status_code, 'yellow', file=self.stdout)
mkdir_p(dirname(self.path_tmp))
with open(self.path_tmp, 'wb') as f:
f.write(self.content())
f.write("\n")
logger.debug(" %s\n" % self.path.replace(settings.INPUT_DATA_ROOT, ''))
last_modified = parse_http_date_safe(self.reply.headers.get('last-modified'))
if last_modified:
utime(self.path_tmp, (last_modified, last_modified))
def sync_content_tourism(sender, **kwargs):
if not settings.TOURISM_ENABLED:
return
server_settings = kwargs['server_settings']
inputkw = kwargs['input_kwargs']
languages = server_settings.languages.available.keys()
logger.debug("Languages: %s" % languages)
for language in languages:
inputkwlang = dict(language=language, **inputkw)
InputFile(DataSource.filepath, **inputkwlang).pull_if_modified()
for datasource in DataSource.tmp_objects.filter(language=language).all():
try:
InputFile(datasource.geojson_url, **inputkwlang).pull_if_modified()
except IOError:
logger.error("Could not retrieve datasource %s" % datasource.geojson_url)
if not settings.TOURISM_DATASOURCE_FAULT_TOLERANT:
raise
for datasource in DataSource.tmp_objects.filter(language=language).all():
InputFile(datasource.pictogram_url, **inputkw).pull_if_modified()
def get(self, url, **kwargs):
logger.debug("GET %s" % url)
url = self.absolute_url(url)
kwargs = self._set_headers_kwargs(kwargs)
return self.session.get(url, **kwargs)
def sync_content_trekking(sender, **kwargs):
server_settings = kwargs['server_settings']
input_kwargs = kwargs['input_kwargs']
languages = server_settings.languages.available.keys()
logger.debug("Languages: %s" % languages)
for language in languages:
inputkwlang = dict(language=language, **input_kwargs)
TrekListInputFile(**inputkwlang).pull()
for trek in models.Trek.tmp_objects.filter(language=language).all():
InputFile(trek.properties.gpx, **inputkwlang).pull_if_modified()
InputFile(trek.properties.kml, **inputkwlang).pull_if_modified()
if trek.properties.information_desk_layer:
# Only available in Geotrek 0.24
InformationDeskInputFile(trek.properties.information_desk_layer, **inputkwlang).pull_if_modified()
if settings.PRINT_ENABLED:
InputFile(trek.properties.printable, **inputkwlang).pull_if_modified()
# Fetch media only once, since they do not depend on language
for trek in models.Trek.tmp_objects.filter(language=settings.LANGUAGE_CODE).all():
InputFile(trek.properties.map_image_url, **input_kwargs).pull_if_modified()
InputFile(trek.properties.altimetric_profile, **input_kwargs).pull_if_modified()
profile_svg_url = trek.properties.altimetric_profile.replace('.json', '.svg')
InputFile(profile_svg_url, **input_kwargs).pull_if_modified()
if trek.properties.thumbnail:
InputFile(trek.properties.thumbnail, **input_kwargs).pull_if_modified()
for picture in trek.properties.pictures:
InputFile(picture.url, **input_kwargs).pull_if_modified()
for theme in trek.properties.themes:
InputFile(theme.pictogram, **input_kwargs).pull_if_modified()
for usage in trek.properties.usages:
InputFile(usage.pictogram, **input_kwargs).pull_if_modified()
for network in trek.properties.networks:
if network.pictogram:
InputFile(network.pictogram, **input_kwargs).pull_if_modified()
if trek.properties.difficulty and trek.properties.difficulty.pictogram:
InputFile(trek.properties.difficulty.pictogram, **input_kwargs).pull_if_modified()
if trek.properties.route and trek.properties.route.pictogram:
InputFile(trek.properties.route.pictogram, **input_kwargs).pull_if_modified()
for weblink in trek.properties.web_links:
if weblink.category:
InputFile(weblink.category.pictogram, **input_kwargs).pull_if_modified()
for poi in models.POIs.tmp_objects.filter(trek__pk=trek.pk,
language=settings.LANGUAGE_CODE).all():
if poi.properties.thumbnail:
InputFile(poi.properties.thumbnail, **input_kwargs).pull_if_modified()
for picture in poi.properties.pictures:
InputFile(picture.url, **input_kwargs).pull_if_modified()
InputFile(poi.properties.type.pictogram, **input_kwargs).pull_if_modified()
if 'information_desks' in trek.properties:
for desk in trek.properties.information_desks:
if desk and desk.get('photo_url', ''):
InputFile(desk.photo_url, **input_kwargs).pull_if_modified()
#
# Fetch attachments
if settings.FILELIST_ENABLED:
attachments = models.AttachmentFile.tmp_objects.filter(trek__pk=trek.properties.pk,
language=settings.LANGUAGE_CODE)
for attachment in attachments.all():
InputFile(attachment.url, **input_kwargs).pull_if_modified()
def content(self):
content = self.reply.json()
if content is None:
return super(TrekListInputFile, self).content()
features = []
for feature in content['features']:
properties = feature['properties']
pk = properties.get('pk', -1)
# Ignore treks that are not published
if not properties.get('published', False):
logger.debug('Trek %s is not published.' % pk)
continue
# Ignore treks that are not linestring
if feature['geometry']['type'].lower() != 'linestring':
msg = 'Trek %s was ignored (not linestring).' % pk
cprint(msg, 'red', attrs=['bold'], file=self.stderr)
continue
# Fill with detail properties
detailpath = models.Trek.detailpath.format(pk=pk)
detailfile = TrekInputFile(detailpath, **self.initkwargs)
detailfile.pull()
detail = json.loads(detailfile.content())
properties.update(detail)
# Remove rooturl from relative URLs
relative_props = ['altimetric_profile', 'elevation_area_url', 'gpx', 'kml',
'map_image_url', 'printable', 'poi_layer', 'information_desk_layer']
for k in relative_props:
properties[k] = properties[k].replace(self.client.rooturl, '') if properties.get(k) else properties.get(k)
# Reroot information desks photos
if 'information_desks' in properties:
properties['information_desks'] = reroot(properties['information_desks'], attr='photo_url')
else:
properties['information_desks'] = [properties['information_desk']]
# Download attachments list file
url = reroot(properties['filelist_url'])
destination = models.AttachmentFile.filepath.format(trek__pk=pk)
# Store destination as new official url (e.g. for Geotrek mobile)
properties['filelist_url'] = destination
f = AttachmentInputFile(url, store=destination, **self.initkwargs)
f.pull()
# Add POIs information in list, useful for textual search
f = POIsInputFile(models.POIs.filepath.format(trek__pk=pk), **self.initkwargs)
f.pull()
poiscontent = json.loads(f.content())
poisprops = [poi['properties'] for poi in poiscontent['features']]
properties['pois'] = [{'name': poiprop['name'],
'description': poiprop['description'],
'type': poiprop['type']['label']}
for poiprop in poisprops]
feature['properties'] = properties
features.append(feature)
content['features'] = features
return self.serialize_json(content)
def sync_content_trekking(sender, **kwargs):
server_settings = kwargs['server_settings']
input_kwargs = kwargs['input_kwargs']
languages = server_settings.languages.available.keys()
logger.debug("Languages: %s" % languages)
for language in languages:
inputkwlang = dict(language=language, **input_kwargs)
TrekListInputFile(**inputkwlang).pull()
for trek in models.Trek.tmp_objects.filter(language=language).all():
InputFile(trek.properties.gpx, **inputkwlang).pull_if_modified()
InputFile(trek.properties.kml, **inputkwlang).pull_if_modified()
if trek.properties.information_desk_layer:
# Only available in Geotrek 0.24
InformationDeskInputFile(
trek.properties.information_desk_layer,
**inputkwlang).pull_if_modified()
if settings.PRINT_ENABLED:
InputFile(trek.properties.printable,
**inputkwlang).pull_if_modified()
# Fetch media only once, since they do not depend on language
for trek in models.Trek.tmp_objects.filter(
language=settings.LANGUAGE_CODE).all():
InputFile(trek.properties.map_image_url,
**input_kwargs).pull_if_modified()
InputFile(trek.properties.altimetric_profile,
**input_kwargs).pull_if_modified()
profile_svg_url = trek.properties.altimetric_profile.replace(
'.json', '.svg')
InputFile(profile_svg_url, **input_kwargs).pull_if_modified()
if trek.properties.thumbnail:
InputFile(trek.properties.thumbnail,
**input_kwargs).pull_if_modified()
for picture in trek.properties.pictures:
InputFile(picture.url, **input_kwargs).pull_if_modified()
for theme in trek.properties.themes:
InputFile(theme.pictogram, **input_kwargs).pull_if_modified()
for usage in trek.properties.usages:
InputFile(usage.pictogram, **input_kwargs).pull_if_modified()
for network in trek.properties.networks:
if network.pictogram:
InputFile(network.pictogram, **input_kwargs).pull_if_modified()
if trek.properties.difficulty and trek.properties.difficulty.pictogram:
InputFile(trek.properties.difficulty.pictogram,
**input_kwargs).pull_if_modified()
if trek.properties.route and trek.properties.route.pictogram:
InputFile(trek.properties.route.pictogram,
**input_kwargs).pull_if_modified()
for weblink in trek.properties.web_links:
if weblink.category:
InputFile(weblink.category.pictogram,
**input_kwargs).pull_if_modified()
for poi in models.POIs.tmp_objects.filter(
trek__pk=trek.pk, language=settings.LANGUAGE_CODE).all():
if poi.properties.thumbnail:
InputFile(poi.properties.thumbnail,
**input_kwargs).pull_if_modified()
for picture in poi.properties.pictures:
InputFile(picture.url, **input_kwargs).pull_if_modified()
InputFile(poi.properties.type.pictogram,
**input_kwargs).pull_if_modified()
if 'information_desks' in trek.properties:
for desk in trek.properties.information_desks:
if desk and desk.get('photo_url', ''):
InputFile(desk.photo_url,
**input_kwargs).pull_if_modified()
#
# Fetch attachments
if settings.FILELIST_ENABLED:
attachments = models.AttachmentFile.tmp_objects.filter(
trek__pk=trek.properties.pk, language=settings.LANGUAGE_CODE)
for attachment in attachments.all():
InputFile(attachment.url, **input_kwargs).pull_if_modified()
def content(self):
content = self.reply.json()
if content is None:
return super(TrekListInputFile, self).content()
features = []
for feature in content['features']:
properties = feature['properties']
pk = properties.get('pk', -1)
# Ignore treks that are not published
if not properties.get('published', False):
logger.debug('Trek %s is not published.' % pk)
continue
# Ignore treks that are not linestring
if feature['geometry']['type'].lower() != 'linestring':
msg = 'Trek %s was ignored (not linestring).' % pk
cprint(msg, 'red', attrs=['bold'], file=self.stderr)
continue
# Fill with detail properties
detailpath = models.Trek.detailpath.format(pk=pk)
detailfile = TrekInputFile(detailpath, **self.initkwargs)
detailfile.pull()
detail = json.loads(detailfile.content())
properties.update(detail)
# Remove rooturl from relative URLs
relative_props = [
'altimetric_profile', 'elevation_area_url', 'gpx', 'kml',
'map_image_url', 'printable', 'poi_layer',
'information_desk_layer'
]
for k in relative_props:
properties[k] = properties[k].replace(
self.client.rooturl,
'') if properties.get(k) else properties.get(k)
# Reroot information desks photos
if 'information_desks' in properties:
properties['information_desks'] = reroot(
properties['information_desks'], attr='photo_url')
else:
properties['information_desks'] = [
properties['information_desk']
]
# Download attachments list file
url = reroot(properties['filelist_url'])
destination = models.AttachmentFile.filepath.format(trek__pk=pk)
# Store destination as new official url (e.g. for Geotrek mobile)
properties['filelist_url'] = destination
f = AttachmentInputFile(url, store=destination, **self.initkwargs)
f.pull()
# Add POIs information in list, useful for textual search
f = POIsInputFile(models.POIs.filepath.format(trek__pk=pk),
**self.initkwargs)
f.pull()
poiscontent = json.loads(f.content())
poisprops = [poi['properties'] for poi in poiscontent['features']]
properties['pois'] = [{
'name': poiprop['name'],
'description': poiprop['description'],
'type': poiprop['type']['label']
} for poiprop in poisprops]
feature['properties'] = properties
features.append(feature)
content['features'] = features
return self.serialize_json(content)
def get(self, url, **kwargs):
logger.debug("GET %s" % url)
url = self.absolute_url(url)
kwargs = self._set_headers_kwargs(kwargs)
return self.session.get(url, **kwargs)