def handle(self, *args, **options):
if not os.path.exists(settings.GEOIP_CONFIG_ROOT):
os.makedirs(settings.GEOIP_CONFIG_ROOT)
print("Downloading IPv4 database... ")
compressed = '%s.gz' % settings.GEOIP_CITY_PATH
urlretrieve(
'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz',
compressed)
with open(settings.GEOIP_CITY_PATH,
'wb') as _out, gzip.open(compressed, 'rb') as _in:
_out.write(_in.read())
os.remove(compressed)
print("Downloading IPv6 database... ")
compressed = '%s.gz' % settings.GEOIP_CITY_V6_PATH
urlretrieve(
'http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz',
compressed)
with open(settings.GEOIP_CITY_V6_PATH,
'wb') as _out, gzip.open(compressed, 'rb') as _in:
_out.write(_in.read())
os.remove(compressed)
print("Done downloading GeoIP databases.")
def ensure_local_file(self, url):
filename = url.split('/')[-1]
path = os.path.join(self.data_dir, filename)
if not os.path.exists(path):
urlretrieve(url, path)
return path
def download(self, url):
"""
Downloads the given URL and returns the file name.
"""
def cleanup_url(url):
tmp = url.rstrip('/')
filename = tmp.split('/')[-1]
if url.endswith('/'):
display_url = tmp + '/'
else:
display_url = url
return filename, display_url
prefix = 'django_%s_template_' % self.app_or_project
tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download')
self.paths_to_remove.append(tempdir)
filename, display_url = cleanup_url(url)
if self.verbosity >= 2:
self.stdout.write("Downloading %s\n" % display_url)
try:
the_path, info = urlretrieve(url, path.join(tempdir, filename))
except IOError as e:
raise CommandError(
"couldn't download URL %s to %s: %s" % (url, filename, e)
)
used_name = the_path.split('/')[-1]
# Trying to get better name from response headers
content_disposition = info.get('content-disposition')
if content_disposition:
_, params = cgi.parse_header(content_disposition)
guessed_filename = params.get('filename') or used_name
else:
guessed_filename = used_name
# Falling back to content type guessing
ext = self.splitext(guessed_filename)[1]
content_type = info.get('content-type')
if not ext and content_type:
ext = mimetypes.guess_extension(content_type)
if ext:
guessed_filename += ext
# Move the temporary file to a filename that has better
# chances of being recognized by the archive utils
if used_name != guessed_filename:
guessed_path = path.join(tempdir, guessed_filename)
shutil.move(the_path, guessed_path)
return guessed_path
# Giving up
return the_path
def download(self, url):
"""
Downloads the given URL and returns the file name.
"""
def cleanup_url(url):
tmp = url.rstrip('/')
filename = tmp.split('/')[-1]
if url.endswith('/'):
display_url = tmp + '/'
else:
display_url = url
return filename, display_url
prefix = 'django_%s_template_' % self.app_or_project
tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download')
self.paths_to_remove.append(tempdir)
filename, display_url = cleanup_url(url)
if self.verbosity >= 2:
self.stdout.write("Downloading %s\n" % display_url)
try:
the_path, info = urlretrieve(url, path.join(tempdir, filename))
except IOError as e:
raise CommandError("couldn't download URL %s to %s: %s" %
(url, filename, e))
used_name = the_path.split('/')[-1]
# Trying to get better name from response headers
content_disposition = info.get('content-disposition')
if content_disposition:
_, params = cgi.parse_header(content_disposition)
guessed_filename = params.get('filename') or used_name
else:
guessed_filename = used_name
# Falling back to content type guessing
ext = self.splitext(guessed_filename)[1]
content_type = info.get('content-type')
if not ext and content_type:
ext = mimetypes.guess_extension(content_type)
if ext:
guessed_filename += ext
# Move the temporary file to a filename that has better
# chances of being recognnized by the archive utils
if used_name != guessed_filename:
guessed_path = path.join(tempdir, guessed_filename)
shutil.move(the_path, guessed_path)
return guessed_path
# Giving up
return the_path
def handle(self, *args, **options):
if not os.path.exists(settings.GEOIP_CONFIG_ROOT):
os.makedirs(settings.GEOIP_CONFIG_ROOT)
print("Downloading IPv4 database... ")
compressed = "%s.gz" % settings.GEOIP_CITY_PATH
urlretrieve("http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz", compressed)
with open(settings.GEOIP_CITY_PATH, "wb") as _out, gzip.open(compressed, "rb") as _in:
_out.write(_in.read())
os.remove(compressed)
print("Downloading IPv6 database... ")
compressed = "%s.gz" % settings.GEOIP_CITY_V6_PATH
urlretrieve(
"http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz", compressed
)
with open(settings.GEOIP_CITY_V6_PATH, "wb") as _out, gzip.open(compressed, "rb") as _in:
_out.write(_in.read())
os.remove(compressed)
print("Done downloading GeoIP databases.")
def open_raster_file(self):
"""
Get raster source file to extract tiles from.
This makes a local copy of rasterfile, unzips the raster and reprojects
it into web mercator if necessary. The reprojected raster is stored for
reuse such that reprojection does only happen once.
The local copy of the raster is needed if files are stored on remote
storages.
"""
reproj, created = RasterLayerReprojected.objects.get_or_create(
rasterlayer=self.rasterlayer)
# Check if the raster has already been reprojected
has_reprojected = reproj.rasterfile.name not in (None, '')
# Create workdir
raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)
# Choose source for raster data, use the reprojected version if it exists.
if self.rasterlayer.source_url and not has_reprojected:
url = self.rasterlayer.source_url
if url.lower().startswith('http') or url.startswith('file'):
url_path = urlparse(self.rasterlayer.source_url).path
filename = url_path.split('/')[-1]
filepath = os.path.join(self.tmpdir, filename)
urlretrieve(self.rasterlayer.source_url, filepath)
elif url.startswith('s3'):
# Get the bucket name and file key, assuming the following url
# strucure: s3://BUCKET_NAME/BUCKET_KEY
bucket_name = url.split('s3://')[1].split('/')[0]
bucket_key = '/'.join(url.split('s3://')[1].split('/')[1:])
# Assume the file name is the last piece of the key.
filename = bucket_key.split('/')[-1]
filepath = os.path.join(self.tmpdir, filename)
# Get file from s3.
s3 = boto3.resource('s3', endpoint_url=self.s3_endpoint_url)
bucket = s3.Bucket(bucket_name)
bucket.download_file(bucket_key,
filepath,
ExtraArgs={'RequestPayer': 'requester'})
else:
raise RasterException(
'Only http(s) and s3 urls are supported.')
else:
if has_reprojected:
rasterfile_source = reproj.rasterfile
else:
rasterfile_source = self.rasterlayer.rasterfile
if not rasterfile_source.name:
raise RasterException(
'No data source found. Provide a rasterfile or a source url.'
)
# Copy raster file source to local folder
filepath = os.path.join(self.tmpdir,
os.path.basename(rasterfile_source.name))
rasterfile = open(filepath, 'wb')
for chunk in rasterfile_source.chunks():
rasterfile.write(chunk)
rasterfile.close()
# If the raster file is compressed, decompress it, otherwise try to
# open the source file directly.
if os.path.splitext(filepath)[1].lower() == '.zip':
# Open and extract zipfile
zf = zipfile.ZipFile(filepath)
zf.extractall(self.tmpdir)
# Remove zipfile
os.remove(filepath)
# Get filelist from directory
matches = []
for root, dirnames, filenames in os.walk(self.tmpdir):
for filename in fnmatch.filter(filenames, '*.*'):
matches.append(os.path.join(root, filename))
# Open the first raster file found in the matched files.
self.dataset = None
for match in matches:
try:
self.dataset = GDALRaster(match)
break
except GDALException:
pass
# Raise exception if no file could be opened by gdal.
if not self.dataset:
raise RasterException('Could not open rasterfile.')
else:
self.dataset = GDALRaster(filepath)
# Override srid if provided
if self.rasterlayer.srid:
try:
self.dataset = GDALRaster(self.dataset.name, write=True)
except GDALException:
raise RasterException(
'Could not override srid because the driver for this '
'type of raster does not support write mode.')
self.dataset.srs = self.rasterlayer.srid
def open_raster_file(self):
"""
Get raster source file to extract tiles from.
This makes a local copy of rasterfile, unzips the raster and reprojects
it into web mercator if necessary. The reprojected raster is stored for
reuse such that reprojection does only happen once.
The local copy of the raster is needed if files are stored on remote
storages.
"""
reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer)
# Check if the raster has already been reprojected
has_reprojected = reproj.rasterfile.name not in (None, '')
# Create workdir
raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)
# Choose source for raster data, use the reprojected version if it exists.
if self.rasterlayer.source_url and not has_reprojected:
url_path = urlparse(self.rasterlayer.source_url).path
filename = url_path.split('/')[-1]
filepath = os.path.join(self.tmpdir, filename)
urlretrieve(self.rasterlayer.source_url, filepath)
else:
if has_reprojected:
rasterfile_source = reproj.rasterfile
else:
rasterfile_source = self.rasterlayer.rasterfile
if not rasterfile_source.name:
raise RasterException('No data source found. Provide a rasterfile or a source url.')
# Copy raster file source to local folder
filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name))
rasterfile = open(filepath, 'wb')
for chunk in rasterfile_source.chunks():
rasterfile.write(chunk)
rasterfile.close()
# If the raster file is compressed, decompress it, otherwise try to
# open the source file directly.
if os.path.splitext(filepath)[1].lower() == '.zip':
# Open and extract zipfile
zf = zipfile.ZipFile(filepath)
zf.extractall(self.tmpdir)
# Remove zipfile
os.remove(filepath)
# Get filelist from directory
matches = []
for root, dirnames, filenames in os.walk(self.tmpdir):
for filename in fnmatch.filter(filenames, '*.*'):
matches.append(os.path.join(root, filename))
# Open the first raster file found in the matched files.
self.dataset = None
for match in matches:
try:
self.dataset = GDALRaster(match)
break
except GDALException:
pass
# Raise exception if no file could be opened by gdal.
if not self.dataset:
raise RasterException('Could not open rasterfile.')
else:
self.dataset = GDALRaster(filepath)
# Override srid if provided
if self.rasterlayer.srid:
try:
self.dataset = GDALRaster(self.dataset.name, write=True)
except GDALException:
raise RasterException(
'Could not override srid because the driver for this '
'type of raster does not support write mode.'
)
self.dataset.srs = self.rasterlayer.srid
def retrieve(self, location, path):
request.urlretrieve(parse.urljoin(self.url, location), path)
return True, path
import cgi
