def sync_content_tourism(sender, **kwargs): if not settings.TOURISM_ENABLED: return server_settings = kwargs['server_settings'] inputkw = kwargs['input_kwargs'] languages = server_settings.languages.available.keys() logger.debug("Languages: %s" % languages) for language in languages: inputkwlang = dict(language=language, **inputkw) InputFile(DataSource.filepath, **inputkwlang).pull_if_modified() for datasource in DataSource.tmp_objects.filter( language=language).all(): try: InputFile(datasource.geojson_url, **inputkwlang).pull_if_modified() except IOError: logger.error("Could not retrieve datasource %s" % datasource.geojson_url) if not settings.TOURISM_DATASOURCE_FAULT_TOLERANT: raise for datasource in DataSource.tmp_objects.filter( language=language).all(): InputFile(datasource.pictogram_url, **inputkw).pull_if_modified()
def content(self): try: logger.debug("Read content from %s" % self.fullpath) with open(self.fullpath, 'r') as f: content = f.read() return content except IOError: logger.error("Could not read '%s'" % self.fullpath) return '[]'
def content(self): try: logger.debug("Read content from %s" % self.fullpath) with open(self.fullpath, 'r') as f: content = f.read() return content except IOError: logger.error("Could not read '%s'" % self.fullpath) return '[]'
def pull(self, ifmodified=False): """ Pull a file served by a Geotrek server. Set 'if-modified-since' HTTP request header to reduce bandwidth. """ headers = {} if self.language: cprint('/' + self.language, 'cyan', end='', file=self.stdout) headers.update({'Accept-language': self.language}) if ifmodified: try: # If local file is empty, force retrieval assert getsize(self.path) > settings.MIN_BYTE_SYZE # Use datetime of previous file to set header mtime = getmtime(self.path) headers.update({'if-modified-since': http_date(mtime)}) except (OSError, AssertionError): pass cprint('/%s ...' % self.url, 'white', attrs=['bold'], end=' ', file=self.stdout) self.stdout.flush() self.reply = self.client.get(self.url, headers=headers) if self.reply.status_code in (304, ): cprint("%s (Up-to-date)" % self.reply.status_code, 'green', attrs=['bold'], file=self.stdout) return elif self.reply.status_code != requests.codes.ok: cprint("%s (Failed)" % self.reply.status_code, 'red', attrs=['bold'], file=self.stderr) raise IOError("Failed to retrieve %s (code: %s)" % (self.reply.url, self.reply.status_code)) else: cprint("%s (Download)" % self.reply.status_code, 'yellow', file=self.stdout) mkdir_p(dirname(self.path_tmp)) with open(self.path_tmp, 'wb') as f: f.write(self.content()) f.write("\n") logger.debug(" %s\n" % self.path.replace(settings.INPUT_DATA_ROOT, '')) last_modified = parse_http_date_safe( self.reply.headers.get('last-modified')) if last_modified: utime(self.path_tmp, (last_modified, last_modified))
def sync_content_feedback(sender, **kwargs): if not settings.FEEDBACK_FORM_ENABLED: return server_settings = kwargs['server_settings'] inputkw = kwargs['input_kwargs'] languages = server_settings.languages.available.keys() logger.debug("Languages: %s" % languages) for language in languages: inputkwlang = dict(language=language, **inputkw) InputFile(FeedbackCategory.filepath, **inputkwlang).pull_if_modified()
def post(self, url, **kwargs): url = self.absolute_url(url) kwargs = self._set_headers_kwargs(kwargs) data = kwargs['data'] if 'csrfmiddlewaretoken' not in data: response = self.session.head(url) assert response.status_code == requests.codes.ok, "Failed to obtain CSRF token" csrftoken = response.cookies.get('csrftoken', '') data['csrfmiddlewaretoken'] = csrftoken kwargs['data'] = data logger.debug("POST %s (%s)" % (url, kwargs)) return self.session.post(url, **kwargs)
def post(self, url, **kwargs): url = self.absolute_url(url) kwargs = self._set_headers_kwargs(kwargs) data = kwargs['data'] if 'csrfmiddlewaretoken' not in data: response = self.session.head(url) assert response.status_code == requests.codes.ok, "Failed to obtain CSRF token" csrftoken = response.cookies.get('csrftoken', '') data['csrfmiddlewaretoken'] = csrftoken kwargs['data'] = data logger.debug("POST %s (%s)" % (url, kwargs)) return self.session.post(url, **kwargs)
def pull(self, ifmodified=False): """ Pull a file served by a Geotrek server. Set 'if-modified-since' HTTP request header to reduce bandwidth. """ headers = {} if self.language: cprint('/' + self.language, 'cyan', end='', file=self.stdout) headers.update({'Accept-language': self.language}) if ifmodified: try: # If local file is empty, force retrieval assert getsize(self.path) > settings.MIN_BYTE_SYZE # Use datetime of previous file to set header mtime = getmtime(self.path) headers.update({'if-modified-since': http_date(mtime)}) except (OSError, AssertionError): pass cprint('/%s ...' % self.url, 'white', attrs=['bold'], end=' ', file=self.stdout) self.stdout.flush() self.reply = self.client.get(self.url, headers=headers) if self.reply.status_code in (304,): cprint("%s (Up-to-date)" % self.reply.status_code, 'green', attrs=['bold'], file=self.stdout) return elif self.reply.status_code != requests.codes.ok: cprint("%s (Failed)" % self.reply.status_code, 'red', attrs=['bold'], file=self.stderr) raise IOError("Failed to retrieve %s (code: %s)" % (self.reply.url, self.reply.status_code)) else: cprint("%s (Download)" % self.reply.status_code, 'yellow', file=self.stdout) mkdir_p(dirname(self.path_tmp)) with open(self.path_tmp, 'wb') as f: f.write(self.content()) f.write("\n") logger.debug(" %s\n" % self.path.replace(settings.INPUT_DATA_ROOT, '')) last_modified = parse_http_date_safe(self.reply.headers.get('last-modified')) if last_modified: utime(self.path_tmp, (last_modified, last_modified))
def sync_content_tourism(sender, **kwargs): if not settings.TOURISM_ENABLED: return server_settings = kwargs['server_settings'] inputkw = kwargs['input_kwargs'] languages = server_settings.languages.available.keys() logger.debug("Languages: %s" % languages) for language in languages: inputkwlang = dict(language=language, **inputkw) InputFile(DataSource.filepath, **inputkwlang).pull_if_modified() for datasource in DataSource.tmp_objects.filter(language=language).all(): try: InputFile(datasource.geojson_url, **inputkwlang).pull_if_modified() except IOError: logger.error("Could not retrieve datasource %s" % datasource.geojson_url) if not settings.TOURISM_DATASOURCE_FAULT_TOLERANT: raise for datasource in DataSource.tmp_objects.filter(language=language).all(): InputFile(datasource.pictogram_url, **inputkw).pull_if_modified()
def get(self, url, **kwargs): logger.debug("GET %s" % url) url = self.absolute_url(url) kwargs = self._set_headers_kwargs(kwargs) return self.session.get(url, **kwargs)
def sync_content_trekking(sender, **kwargs): server_settings = kwargs['server_settings'] input_kwargs = kwargs['input_kwargs'] languages = server_settings.languages.available.keys() logger.debug("Languages: %s" % languages) for language in languages: inputkwlang = dict(language=language, **input_kwargs) TrekListInputFile(**inputkwlang).pull() for trek in models.Trek.tmp_objects.filter(language=language).all(): InputFile(trek.properties.gpx, **inputkwlang).pull_if_modified() InputFile(trek.properties.kml, **inputkwlang).pull_if_modified() if trek.properties.information_desk_layer: # Only available in Geotrek 0.24 InformationDeskInputFile(trek.properties.information_desk_layer, **inputkwlang).pull_if_modified() if settings.PRINT_ENABLED: InputFile(trek.properties.printable, **inputkwlang).pull_if_modified() # Fetch media only once, since they do not depend on language for trek in models.Trek.tmp_objects.filter(language=settings.LANGUAGE_CODE).all(): InputFile(trek.properties.map_image_url, **input_kwargs).pull_if_modified() InputFile(trek.properties.altimetric_profile, **input_kwargs).pull_if_modified() profile_svg_url = trek.properties.altimetric_profile.replace('.json', '.svg') InputFile(profile_svg_url, **input_kwargs).pull_if_modified() if trek.properties.thumbnail: InputFile(trek.properties.thumbnail, **input_kwargs).pull_if_modified() for picture in trek.properties.pictures: InputFile(picture.url, **input_kwargs).pull_if_modified() for theme in trek.properties.themes: InputFile(theme.pictogram, **input_kwargs).pull_if_modified() for usage in trek.properties.usages: InputFile(usage.pictogram, **input_kwargs).pull_if_modified() for network in trek.properties.networks: if network.pictogram: InputFile(network.pictogram, **input_kwargs).pull_if_modified() if trek.properties.difficulty and trek.properties.difficulty.pictogram: InputFile(trek.properties.difficulty.pictogram, **input_kwargs).pull_if_modified() if trek.properties.route and trek.properties.route.pictogram: InputFile(trek.properties.route.pictogram, **input_kwargs).pull_if_modified() for weblink in trek.properties.web_links: if weblink.category: InputFile(weblink.category.pictogram, **input_kwargs).pull_if_modified() for poi in models.POIs.tmp_objects.filter(trek__pk=trek.pk, language=settings.LANGUAGE_CODE).all(): if poi.properties.thumbnail: InputFile(poi.properties.thumbnail, **input_kwargs).pull_if_modified() for picture in poi.properties.pictures: InputFile(picture.url, **input_kwargs).pull_if_modified() InputFile(poi.properties.type.pictogram, **input_kwargs).pull_if_modified() if 'information_desks' in trek.properties: for desk in trek.properties.information_desks: if desk and desk.get('photo_url', ''): InputFile(desk.photo_url, **input_kwargs).pull_if_modified() # # Fetch attachments if settings.FILELIST_ENABLED: attachments = models.AttachmentFile.tmp_objects.filter(trek__pk=trek.properties.pk, language=settings.LANGUAGE_CODE) for attachment in attachments.all(): InputFile(attachment.url, **input_kwargs).pull_if_modified()
def content(self): content = self.reply.json() if content is None: return super(TrekListInputFile, self).content() features = [] for feature in content['features']: properties = feature['properties'] pk = properties.get('pk', -1) # Ignore treks that are not published if not properties.get('published', False): logger.debug('Trek %s is not published.' % pk) continue # Ignore treks that are not linestring if feature['geometry']['type'].lower() != 'linestring': msg = 'Trek %s was ignored (not linestring).' % pk cprint(msg, 'red', attrs=['bold'], file=self.stderr) continue # Fill with detail properties detailpath = models.Trek.detailpath.format(pk=pk) detailfile = TrekInputFile(detailpath, **self.initkwargs) detailfile.pull() detail = json.loads(detailfile.content()) properties.update(detail) # Remove rooturl from relative URLs relative_props = ['altimetric_profile', 'elevation_area_url', 'gpx', 'kml', 'map_image_url', 'printable', 'poi_layer', 'information_desk_layer'] for k in relative_props: properties[k] = properties[k].replace(self.client.rooturl, '') if properties.get(k) else properties.get(k) # Reroot information desks photos if 'information_desks' in properties: properties['information_desks'] = reroot(properties['information_desks'], attr='photo_url') else: properties['information_desks'] = [properties['information_desk']] # Download attachments list file url = reroot(properties['filelist_url']) destination = models.AttachmentFile.filepath.format(trek__pk=pk) # Store destination as new official url (e.g. for Geotrek mobile) properties['filelist_url'] = destination f = AttachmentInputFile(url, store=destination, **self.initkwargs) f.pull() # Add POIs information in list, useful for textual search f = POIsInputFile(models.POIs.filepath.format(trek__pk=pk), **self.initkwargs) f.pull() poiscontent = json.loads(f.content()) poisprops = [poi['properties'] for poi in poiscontent['features']] properties['pois'] = [{'name': poiprop['name'], 'description': poiprop['description'], 'type': poiprop['type']['label']} for poiprop in poisprops] feature['properties'] = properties features.append(feature) content['features'] = features return self.serialize_json(content)
def sync_content_trekking(sender, **kwargs): server_settings = kwargs['server_settings'] input_kwargs = kwargs['input_kwargs'] languages = server_settings.languages.available.keys() logger.debug("Languages: %s" % languages) for language in languages: inputkwlang = dict(language=language, **input_kwargs) TrekListInputFile(**inputkwlang).pull() for trek in models.Trek.tmp_objects.filter(language=language).all(): InputFile(trek.properties.gpx, **inputkwlang).pull_if_modified() InputFile(trek.properties.kml, **inputkwlang).pull_if_modified() if trek.properties.information_desk_layer: # Only available in Geotrek 0.24 InformationDeskInputFile( trek.properties.information_desk_layer, **inputkwlang).pull_if_modified() if settings.PRINT_ENABLED: InputFile(trek.properties.printable, **inputkwlang).pull_if_modified() # Fetch media only once, since they do not depend on language for trek in models.Trek.tmp_objects.filter( language=settings.LANGUAGE_CODE).all(): InputFile(trek.properties.map_image_url, **input_kwargs).pull_if_modified() InputFile(trek.properties.altimetric_profile, **input_kwargs).pull_if_modified() profile_svg_url = trek.properties.altimetric_profile.replace( '.json', '.svg') InputFile(profile_svg_url, **input_kwargs).pull_if_modified() if trek.properties.thumbnail: InputFile(trek.properties.thumbnail, **input_kwargs).pull_if_modified() for picture in trek.properties.pictures: InputFile(picture.url, **input_kwargs).pull_if_modified() for theme in trek.properties.themes: InputFile(theme.pictogram, **input_kwargs).pull_if_modified() for usage in trek.properties.usages: InputFile(usage.pictogram, **input_kwargs).pull_if_modified() for network in trek.properties.networks: if network.pictogram: InputFile(network.pictogram, **input_kwargs).pull_if_modified() if trek.properties.difficulty and trek.properties.difficulty.pictogram: InputFile(trek.properties.difficulty.pictogram, **input_kwargs).pull_if_modified() if trek.properties.route and trek.properties.route.pictogram: InputFile(trek.properties.route.pictogram, **input_kwargs).pull_if_modified() for weblink in trek.properties.web_links: if weblink.category: InputFile(weblink.category.pictogram, **input_kwargs).pull_if_modified() for poi in models.POIs.tmp_objects.filter( trek__pk=trek.pk, language=settings.LANGUAGE_CODE).all(): if poi.properties.thumbnail: InputFile(poi.properties.thumbnail, **input_kwargs).pull_if_modified() for picture in poi.properties.pictures: InputFile(picture.url, **input_kwargs).pull_if_modified() InputFile(poi.properties.type.pictogram, **input_kwargs).pull_if_modified() if 'information_desks' in trek.properties: for desk in trek.properties.information_desks: if desk and desk.get('photo_url', ''): InputFile(desk.photo_url, **input_kwargs).pull_if_modified() # # Fetch attachments if settings.FILELIST_ENABLED: attachments = models.AttachmentFile.tmp_objects.filter( trek__pk=trek.properties.pk, language=settings.LANGUAGE_CODE) for attachment in attachments.all(): InputFile(attachment.url, **input_kwargs).pull_if_modified()
def content(self): content = self.reply.json() if content is None: return super(TrekListInputFile, self).content() features = [] for feature in content['features']: properties = feature['properties'] pk = properties.get('pk', -1) # Ignore treks that are not published if not properties.get('published', False): logger.debug('Trek %s is not published.' % pk) continue # Ignore treks that are not linestring if feature['geometry']['type'].lower() != 'linestring': msg = 'Trek %s was ignored (not linestring).' % pk cprint(msg, 'red', attrs=['bold'], file=self.stderr) continue # Fill with detail properties detailpath = models.Trek.detailpath.format(pk=pk) detailfile = TrekInputFile(detailpath, **self.initkwargs) detailfile.pull() detail = json.loads(detailfile.content()) properties.update(detail) # Remove rooturl from relative URLs relative_props = [ 'altimetric_profile', 'elevation_area_url', 'gpx', 'kml', 'map_image_url', 'printable', 'poi_layer', 'information_desk_layer' ] for k in relative_props: properties[k] = properties[k].replace( self.client.rooturl, '') if properties.get(k) else properties.get(k) # Reroot information desks photos if 'information_desks' in properties: properties['information_desks'] = reroot( properties['information_desks'], attr='photo_url') else: properties['information_desks'] = [ properties['information_desk'] ] # Download attachments list file url = reroot(properties['filelist_url']) destination = models.AttachmentFile.filepath.format(trek__pk=pk) # Store destination as new official url (e.g. for Geotrek mobile) properties['filelist_url'] = destination f = AttachmentInputFile(url, store=destination, **self.initkwargs) f.pull() # Add POIs information in list, useful for textual search f = POIsInputFile(models.POIs.filepath.format(trek__pk=pk), **self.initkwargs) f.pull() poiscontent = json.loads(f.content()) poisprops = [poi['properties'] for poi in poiscontent['features']] properties['pois'] = [{ 'name': poiprop['name'], 'description': poiprop['description'], 'type': poiprop['type']['label'] } for poiprop in poisprops] feature['properties'] = properties features.append(feature) content['features'] = features return self.serialize_json(content)
def get(self, url, **kwargs): logger.debug("GET %s" % url) url = self.absolute_url(url) kwargs = self._set_headers_kwargs(kwargs) return self.session.get(url, **kwargs)