def handle(self, *args, **options): if not os.path.exists(settings.GEOIP_CONFIG_ROOT): os.makedirs(settings.GEOIP_CONFIG_ROOT) print("Downloading IPv4 database... ") compressed = '%s.gz' % settings.GEOIP_CITY_PATH urlretrieve( 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz', compressed) with open(settings.GEOIP_CITY_PATH, 'wb') as _out, gzip.open(compressed, 'rb') as _in: _out.write(_in.read()) os.remove(compressed) print("Downloading IPv6 database... ") compressed = '%s.gz' % settings.GEOIP_CITY_V6_PATH urlretrieve( 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz', compressed) with open(settings.GEOIP_CITY_V6_PATH, 'wb') as _out, gzip.open(compressed, 'rb') as _in: _out.write(_in.read()) os.remove(compressed) print("Done downloading GeoIP databases.")
def ensure_local_file(self, url): filename = url.split('/')[-1] path = os.path.join(self.data_dir, filename) if not os.path.exists(path): urlretrieve(url, path) return path
def download(self, url): """ Downloads the given URL and returns the file name. """ def cleanup_url(url): tmp = url.rstrip('/') filename = tmp.split('/')[-1] if url.endswith('/'): display_url = tmp + '/' else: display_url = url return filename, display_url prefix = 'django_%s_template_' % self.app_or_project tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download') self.paths_to_remove.append(tempdir) filename, display_url = cleanup_url(url) if self.verbosity >= 2: self.stdout.write("Downloading %s\n" % display_url) try: the_path, info = urlretrieve(url, path.join(tempdir, filename)) except IOError as e: raise CommandError( "couldn't download URL %s to %s: %s" % (url, filename, e) ) used_name = the_path.split('/')[-1] # Trying to get better name from response headers content_disposition = info.get('content-disposition') if content_disposition: _, params = cgi.parse_header(content_disposition) guessed_filename = params.get('filename') or used_name else: guessed_filename = used_name # Falling back to content type guessing ext = self.splitext(guessed_filename)[1] content_type = info.get('content-type') if not ext and content_type: ext = mimetypes.guess_extension(content_type) if ext: guessed_filename += ext # Move the temporary file to a filename that has better # chances of being recognized by the archive utils if used_name != guessed_filename: guessed_path = path.join(tempdir, guessed_filename) shutil.move(the_path, guessed_path) return guessed_path # Giving up return the_path
def download(self, url): """ Downloads the given URL and returns the file name. """ def cleanup_url(url): tmp = url.rstrip('/') filename = tmp.split('/')[-1] if url.endswith('/'): display_url = tmp + '/' else: display_url = url return filename, display_url prefix = 'django_%s_template_' % self.app_or_project tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download') self.paths_to_remove.append(tempdir) filename, display_url = cleanup_url(url) if self.verbosity >= 2: self.stdout.write("Downloading %s\n" % display_url) try: the_path, info = urlretrieve(url, path.join(tempdir, filename)) except IOError as e: raise CommandError("couldn't download URL %s to %s: %s" % (url, filename, e)) used_name = the_path.split('/')[-1] # Trying to get better name from response headers content_disposition = info.get('content-disposition') if content_disposition: _, params = cgi.parse_header(content_disposition) guessed_filename = params.get('filename') or used_name else: guessed_filename = used_name # Falling back to content type guessing ext = self.splitext(guessed_filename)[1] content_type = info.get('content-type') if not ext and content_type: ext = mimetypes.guess_extension(content_type) if ext: guessed_filename += ext # Move the temporary file to a filename that has better # chances of being recognnized by the archive utils if used_name != guessed_filename: guessed_path = path.join(tempdir, guessed_filename) shutil.move(the_path, guessed_path) return guessed_path # Giving up return the_path
def handle(self, *args, **options): if not os.path.exists(settings.GEOIP_CONFIG_ROOT): os.makedirs(settings.GEOIP_CONFIG_ROOT) print("Downloading IPv4 database... ") compressed = "%s.gz" % settings.GEOIP_CITY_PATH urlretrieve("http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz", compressed) with open(settings.GEOIP_CITY_PATH, "wb") as _out, gzip.open(compressed, "rb") as _in: _out.write(_in.read()) os.remove(compressed) print("Downloading IPv6 database... ") compressed = "%s.gz" % settings.GEOIP_CITY_V6_PATH urlretrieve( "http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz", compressed ) with open(settings.GEOIP_CITY_V6_PATH, "wb") as _out, gzip.open(compressed, "rb") as _in: _out.write(_in.read()) os.remove(compressed) print("Done downloading GeoIP databases.")
def open_raster_file(self): """ Get raster source file to extract tiles from. This makes a local copy of rasterfile, unzips the raster and reprojects it into web mercator if necessary. The reprojected raster is stored for reuse such that reprojection does only happen once. The local copy of the raster is needed if files are stored on remote storages. """ reproj, created = RasterLayerReprojected.objects.get_or_create( rasterlayer=self.rasterlayer) # Check if the raster has already been reprojected has_reprojected = reproj.rasterfile.name not in (None, '') # Create workdir raster_workdir = getattr(settings, 'RASTER_WORKDIR', None) self.tmpdir = tempfile.mkdtemp(dir=raster_workdir) # Choose source for raster data, use the reprojected version if it exists. if self.rasterlayer.source_url and not has_reprojected: url = self.rasterlayer.source_url if url.lower().startswith('http') or url.startswith('file'): url_path = urlparse(self.rasterlayer.source_url).path filename = url_path.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) urlretrieve(self.rasterlayer.source_url, filepath) elif url.startswith('s3'): # Get the bucket name and file key, assuming the following url # strucure: s3://BUCKET_NAME/BUCKET_KEY bucket_name = url.split('s3://')[1].split('/')[0] bucket_key = '/'.join(url.split('s3://')[1].split('/')[1:]) # Assume the file name is the last piece of the key. filename = bucket_key.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) # Get file from s3. s3 = boto3.resource('s3', endpoint_url=self.s3_endpoint_url) bucket = s3.Bucket(bucket_name) bucket.download_file(bucket_key, filepath, ExtraArgs={'RequestPayer': 'requester'}) else: raise RasterException( 'Only http(s) and s3 urls are supported.') else: if has_reprojected: rasterfile_source = reproj.rasterfile else: rasterfile_source = self.rasterlayer.rasterfile if not rasterfile_source.name: raise RasterException( 'No data source found. Provide a rasterfile or a source url.' ) # Copy raster file source to local folder filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name)) rasterfile = open(filepath, 'wb') for chunk in rasterfile_source.chunks(): rasterfile.write(chunk) rasterfile.close() # If the raster file is compressed, decompress it, otherwise try to # open the source file directly. if os.path.splitext(filepath)[1].lower() == '.zip': # Open and extract zipfile zf = zipfile.ZipFile(filepath) zf.extractall(self.tmpdir) # Remove zipfile os.remove(filepath) # Get filelist from directory matches = [] for root, dirnames, filenames in os.walk(self.tmpdir): for filename in fnmatch.filter(filenames, '*.*'): matches.append(os.path.join(root, filename)) # Open the first raster file found in the matched files. self.dataset = None for match in matches: try: self.dataset = GDALRaster(match) break except GDALException: pass # Raise exception if no file could be opened by gdal. if not self.dataset: raise RasterException('Could not open rasterfile.') else: self.dataset = GDALRaster(filepath) # Override srid if provided if self.rasterlayer.srid: try: self.dataset = GDALRaster(self.dataset.name, write=True) except GDALException: raise RasterException( 'Could not override srid because the driver for this ' 'type of raster does not support write mode.') self.dataset.srs = self.rasterlayer.srid
def open_raster_file(self): """ Get raster source file to extract tiles from. This makes a local copy of rasterfile, unzips the raster and reprojects it into web mercator if necessary. The reprojected raster is stored for reuse such that reprojection does only happen once. The local copy of the raster is needed if files are stored on remote storages. """ reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer) # Check if the raster has already been reprojected has_reprojected = reproj.rasterfile.name not in (None, '') # Create workdir raster_workdir = getattr(settings, 'RASTER_WORKDIR', None) self.tmpdir = tempfile.mkdtemp(dir=raster_workdir) # Choose source for raster data, use the reprojected version if it exists. if self.rasterlayer.source_url and not has_reprojected: url_path = urlparse(self.rasterlayer.source_url).path filename = url_path.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) urlretrieve(self.rasterlayer.source_url, filepath) else: if has_reprojected: rasterfile_source = reproj.rasterfile else: rasterfile_source = self.rasterlayer.rasterfile if not rasterfile_source.name: raise RasterException('No data source found. Provide a rasterfile or a source url.') # Copy raster file source to local folder filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name)) rasterfile = open(filepath, 'wb') for chunk in rasterfile_source.chunks(): rasterfile.write(chunk) rasterfile.close() # If the raster file is compressed, decompress it, otherwise try to # open the source file directly. if os.path.splitext(filepath)[1].lower() == '.zip': # Open and extract zipfile zf = zipfile.ZipFile(filepath) zf.extractall(self.tmpdir) # Remove zipfile os.remove(filepath) # Get filelist from directory matches = [] for root, dirnames, filenames in os.walk(self.tmpdir): for filename in fnmatch.filter(filenames, '*.*'): matches.append(os.path.join(root, filename)) # Open the first raster file found in the matched files. self.dataset = None for match in matches: try: self.dataset = GDALRaster(match) break except GDALException: pass # Raise exception if no file could be opened by gdal. if not self.dataset: raise RasterException('Could not open rasterfile.') else: self.dataset = GDALRaster(filepath) # Override srid if provided if self.rasterlayer.srid: try: self.dataset = GDALRaster(self.dataset.name, write=True) except GDALException: raise RasterException( 'Could not override srid because the driver for this ' 'type of raster does not support write mode.' ) self.dataset.srs = self.rasterlayer.srid
def retrieve(self, location, path): request.urlretrieve(parse.urljoin(self.url, location), path) return True, path
import cgi