def fetch(self, url, path, filename):
"""Verify if the file is already downloaded and complete. If they don't
exists or if are not complete, use homura download function to fetch
files. Return a list with the path of the downloaded file and the size
of the remote file.
"""
print('\nDownloading file:\t{}'.format(filename))
remote_file_size = self.get_remote_file_size(url)
file_path = os.path.join(path, filename)
if os.path.exists(os.path.join(path, filename)):
size = os.path.getsize(file_path)
if size == remote_file_size:
print('File already exists')
values = {
"name": filename,
"path": file_path,
"type": filename.split("_")[-1].split(".")[0],
"size": size
}
return values
fetch(url, path)
print('stored at {}'.format(path))
values = {
"name": filename,
"path": file_path,
"type": filename.split("_")[-1].split(".")[0],
"size": remote_file_size
}
return values
def fetch(self, url, path, filename):
""" Downloads the given url.
:param url:
The url to be downloaded.
:type url:
String
:param path:
The directory path to where the image should be stored
:type path:
String
:param filename:
The filename that has to be downloaded
:type filename:
String
:returns:
Boolean
"""
self.output('Downloading: %s' % filename, normal=True, arrow=True)
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == self.get_remote_file_size(url):
self.output('%s already exists on your system' % filename, normal=True, color='green', indent=1)
return False
fetch(url, path)
self.output('stored at %s' % path, normal=True, color='green', indent=1)
return True
def download_meta(url, download_path):
dpath = download_path if download_path else mkdtemp()
dpath = os.path.join(dpath, 'LANDSAT_8_C1.csv')
# don't download if the file is downloaded in the last 6 hours
if os.path.isfile(dpath):
mtime = os.path.getmtime(dpath)
if time.time() - mtime < (6 * 60 * 60):
return open(dpath, 'r')
fetch(url, dpath)
return open(dpath, 'r')
def download_meta(url, download_path):
dpath = download_path if download_path else mkdtemp()
dpath = os.path.join(dpath, 'LANDSAT_8_C1.csv')
# don't download if the file is downloaded in the last 6 hours
if os.path.isfile(dpath):
mtime = os.path.getmtime(dpath)
if time.time() - mtime < (6 * 60 * 60):
return open(dpath, 'r')
fetch(url, dpath)
return open(dpath, 'r')
def fetch(self, url, path):
""" Downloads the given url.
:param url:
The url to be downloaded.
:type url:
String
:param path:
The directory path to where the image should be stored
:type path:
String
:param filename:
The filename that has to be downloaded
:type filename:
String
:returns:
Boolean
"""
segments = url.split('/')
filename = segments[-1]
# remove query parameters from the filename
filename = filename.split('?')[0]
self.output('Downloading: %s' % filename, normal=True, arrow=True)
# print(join(path, filename))
# raise Exception
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == self.get_remote_file_size(url):
self.output('%s already exists on your system' % filename,
normal=True,
color='green',
indent=1)
else:
fetch(url, path)
self.output('stored at %s' % path,
normal=True,
color='green',
indent=1)
return join(path, filename)
def fetch(self, url, path, filename):
"""Verify if the file is already downloaded and complete. If they don't
exists or if are not complete, use homura download function to fetch
files. Return a list with the path of the downloaded file and the size
of the remote file.
"""
print(('Downloading: %s' % filename))
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == self.get_remote_file_size(url):
print(('%s already exists on your system' % filename))
return [join(path, filename), self.get_remote_file_size(url)]
fetch(url, path)
print(('stored at %s' % path))
return [join(path, filename), self.get_remote_file_size(url)]
def fetch(self, url, path, filename):
self.output('Downloading: %s' % filename, normal=True, arrow=True)
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == self.get_remote_file_size(url):
self.output('%s already exists on your system' % filename,
normal=True,
color='green',
indent=1)
return False
fetch(url, path)
self.output('stored at %s' % path,
normal=True,
color='green',
indent=1)
return True
def fetch(self, url, path):
""" Downloads the given url.
:param url:
The url to be downloaded.
:type url:
String
:param path:
The directory path to where the image should be stored
:type path:
String
:param filename:
The filename that has to be downloaded
:type filename:
String
:returns:
Boolean
"""
segments = url.split('/')
filename = segments[-1]
# remove query parameters from the filename
filename = filename.split('?')[0]
self.output('Downloading: %s' % filename, normal=True, arrow=True)
# print(join(path, filename))
# raise Exception
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == self.get_remote_file_size(url):
self.output('%s already exists on your system' % filename, normal=True, color='green', indent=1)
else:
fetch(url, path)
self.output('stored at %s' % path, normal=True, color='green', indent=1)
return join(path, filename)
def fetch(self, url, path, filename):
"""Verify if the file is already downloaded and complete. If they don't
exists or if are not complete, use homura download function to fetch
files. Return a list with the path of the downloaded file and the size
of the remote file.
"""
logger.debug('initializing download in ', url)
remote_file_size = self.get_remote_file_size(url)
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == remote_file_size:
logger.error('%s already exists on your system' % filename)
print('%s already exists on your system' % filename)
return [join(path, filename), size]
logger.debug('Downloading: %s' % filename)
print('Downloading: %s' % filename)
fetch(url, path)
print('stored at %s' % path)
logger.debug('stored at %s' % path)
return [join(path, filename), remote_file_size]
def fetch(self, url, path, filename):
"""Verify if the file is already downloaded and complete. If they don't
exists or if are not complete, use homura download function to fetch
files. Return a list with the path of the downloaded file and the size
of the remote file.
"""
logger.debug('initializing download in ', url)
remote_file_size = self.get_remote_file_size(url)
if exists(join(path, filename)):
size = getsize(join(path, filename))
if size == remote_file_size:
logger.error('%s already exists on your system' % filename)
print('%s already exists on your system' % filename)
return [join(path, filename), size]
logger.debug('Downloading: %s' % filename)
print('Downloading: %s' % filename)
fetch(url, path)
print('stored at %s' % path)
logger.debug('stored at %s' % path)
return [join(path, filename), remote_file_size]
def download(self, scenes):
"""Download Function from Google Storage
Args:
SceneID {data_tye: List}
Retruns:
Tarred landsat data files
"""
self.google = landsat.settings.GOOGLE_STORAGE
# Check for the validity of SceneID
for scene in scenes:
# An empty list in accordance to landsat unique ID
scene_anatomy = {
'path': None,
'row': None,
'sat': None,
'scene': scene
}
if isinstance(scene, str) and len(scene) == 21:
scene_anatomy['path'] = scene[3:6]
scene_anatomy['row'] = scene[6:9]
scene_anatomy['sat'] = 'L' + scene[2:3]
else:
raise IncorrectSceneId('Bad List')
# Create a folder to download tarred landsat Scenes
path = check_create_folder(self.download_dir)
filename = scene + '.tar.bz'
print path
print scene_anatomy
# Builds the URL to a google storage file
url = url_builder([self.google,scene_anatomy['sat'], scene_anatomy['path'], scene_anatomy['row'],filename])
print url
# Downloads and save at the location defined under path
fetch(url,path)
return path
def process_landsat(idlist, download, extract, create, warp, thumbnail,dir,ctype, ltype,overwrite):
thumb_list = [['' for i in range(13)] for j in range(43 - 28 + 1)]
date_list = ['-1' for i in range(13)]
path_list = ['-1' for i in range(13)]
for id in idlist:
path = id[3:6]
row = id[6:9]
year = id[9:13]
ddd = id[13:16]
refdate = datetime.date(int(year), 1, 1)
d = refdate + datetime.timedelta(days=int(ddd))
date = d.strftime('%Y-%m-%d')
file = id + ".tar.bz"
thumb_list[int(row) - 28][int(path) - 104] = id + ".jpg"
date_list[int(path) - 104] = date
path_list[int(path) - 104] = path
url = "http://storage.googleapis.com/earthengine-public/landsat/L8/%s/%s/%s" % (
path, row, file)
fpath = join(dir, file)
unzipdir = join(dir, id)
if download:
if (not exists(fpath) or overwrite):
if not exists(dir):
print "skip! not exist:" + dir + " directory"
else:
print "download " + url
fetch(url, fpath)
# no info download
#urllib.urlretrieve(url, fpath)
else:
print "already exist:" + fpath
if extract:
if (not exists(unzipdir) or overwrite):
if not exists(fpath):
print "skip! not exist:" + fpath
else:
print "unzip " + unzipdir
tar = tarfile.open(fpath, 'r')
tar.extractall(path=unzipdir)
tar.close()
else:
print "already exist:" + unzipdir
if create:
if (not exists(join(dir, id + ctype + ".tif")) or overwrite):
if not exists(unzipdir):
print "skip! not exist:" + unzipdir
else:
print "create " + id + ctype + ".tif"
if ctype == "ndvi":
landsat_color.ndvi_process(id, join(dir, id), dir,ltype)
else:
landsat_color.process(id, join(dir, id), dir,ltype)
else:
print "already exist:" + join(dir, id + ctype + ".tif")
if warp:
if (not exists(join(dir, id + ctype + "_w.tif")) or overwrite):
if not exists(join(dir, id + ctype +".tif")):
print "skip! not exist:" + join(dir, id + ctype + ".tif")
else:
cmd = "gdalwarp -overwrite --config GDAL_CACHEMAX 2048 -wm 2048 -multi -co TFW=YES -t_srs EPSG:3857 %s %s" % (
join(dir, id + ctype+ ".tif"), join(dir, id + ctype + "_w.tif"))
print cmd
subprocess.call(cmd, shell=True)
cmd = "nearblack --config GDAL_CACHEMAX 2048 %s" % (join(dir,id + ctype +"_w.tif"))
print cmd
subprocess.call(cmd, shell=True)
else:
print "already exist:" + join(dir, id + ctype + "_w.tif")
if thumbnail:
if (not exists(join(dir, id + ctype + "_t.tif")) or overwrite):
if not exists(join(dir, id + ctype + "_w.tif")):
print "skip! not exist:" + join(dir, id + ctype + "_w.tif")
else:
cmd = "gdal_translate -outsize 5%% 5%% %s %s" % (
join(dir, id + ctype + "_w.tif"), join(dir, id + ctype + "_t.tif"))
print cmd
subprocess.call(cmd, shell=True)
else:
print "already exist:" + join(dir, id + ctype + "_t.tif")