def download_status(self):
open_dnld_statusfile = open(self.dnld_statusfile, 'a')
csv_dnld_statusfile = csv.writer(open_dnld_statusfile, delimiter=';')
self.end_dnld_datetime = datetime.today()
csv_dnld_statusfile.writerow([])
csv_dnld_statusfile.writerow([
'########### START LOG STATUS FOR: ' + self.dnld_name + ' - ' +
self.dnld_date
])
csv_dnld_statusfile.writerow([self.dnld_name, self.dnld_date])
csv_dnld_statusfile.writerow([
'started: ' + datetime_format(self.start_dnld_datetime),
'finished: ' + datetime_format(self.end_dnld_datetime)
])
for status in self.dnld_status:
csv_dnld_statusfile.writerow(status)
# report errors
csv_dnld_statusfile.writerow([])
if len(self.errors) != 0:
csv_dnld_statusfile.writerow(['Errors:'])
for error in self.errors:
csv_dnld_statusfile.writerow([error])
else:
csv_dnld_statusfile.writerow(['No errors reported'])
# close log file
del csv_dnld_statusfile
open_dnld_statusfile.close()
def run(config_run):
if config_run.p1_qc4sd not in [None, 'None']:
msg = 'Warning: The process {0} was executed before\n'.format(
config_run.process_name)
config_run.process_logfile.write(msg)
print(msg)
source_path = os.path.join(config_run.working_directory, 'p0_download')
dir_process = os.path.join(config_run.working_directory,
config_run.process_name)
if not os.path.isdir(source_path):
msg = '\nError: The directory of previous process: {0}\n' \
'not exist, please run the previous process before it.\n'.format(source_path)
config_run.process_logfile.write(msg)
print(msg)
# save in setting
config_run.p1_qc4sd = 'with errors! - ' + datetime_format(
datetime.today())
config_run.save()
return
if os.path.isdir(dir_process):
shutil.rmtree(dir_process)
if not os.path.isdir(dir_process):
os.makedirs(dir_process)
# process the time series (in the whole period) with QC4SD by tile and by mode
for root, dirs, files in os.walk(source_path):
if len(files) != 0:
files = [x for x in files if x.endswith(('.hdf', '.HDF'))]
if files:
tile = os.path.basename(root)
mode = os.path.basename(os.path.dirname(root))
files = sorted(files)
msg = 'Making the quality control process (using QC4SD) for {0} in tile {1}:'.format(
mode, tile)
config_run.process_logfile.write(msg + '\n')
print(msg)
files_with_path = [os.path.join(root, file) for file in files]
# process the mode MXD09A1/GA for the bands 3,4,5,6 and 7
if mode in ['MOD09A1', 'MYD09A1'] + ['MOD09GA', 'MYD09GA']:
qc4sd.run('default', [3, 4, 5, 6, 7], files_with_path,
dir_process)
# process the mode MXD09Q1/GQ for the bands 1 and 2
if mode in ['MOD09Q1', 'MYD09Q1'] + ['MOD09GQ', 'MYD09GQ']:
qc4sd.run('default', [1, 2], files_with_path, dir_process)
# finishing the process
msg = '\nThe process {0} completed - ({1})'.format(
config_run.process_name, datetime_format(datetime.today()))
config_run.process_logfile.write(msg + '\n')
print(msg)
# save in setting
config_run.p1_qc4sd = 'done - ' + datetime_format(datetime.today())
config_run.save()
def daemon_request(self, urls_files):
if len(urls_files) == 0 or len(self.errors) != 0:
self.dnld_logfile.write('Errors reported before download:\n')
for error in self.errors:
self.dnld_logfile.write(' ' + error + '\n')
self.dnld_logfile.flush()
if len(urls_files) == 0:
self.dnld_logfile.write(' no files to download!!, ' +
datetime_format(datetime.today()) +
'\n')
self.dnld_logfile.flush()
return False
else:
self.dnld_logfile.write('Getting list of download files: OK, ' +
datetime_format(datetime.today()) + '\n')
self.dnld_logfile.flush()
self.dnld_logfile.write('\n')
for attempt in range(self.NUM_ATTEMPT_DAEMON):
url_file_to_test = urls_files[randint(0,
len(urls_files) -
1)] # take a random file
# check if file exists or exists but is corrupt (200 and 508 respectively)
if get_http_code(url_file_to_test) in [200, 508]:
self.dnld_logfile.write('ready to download ' +
datetime_format(datetime.today()) +
'\n')
self.dnld_logfile.flush()
return True
else:
self.dnld_logfile.write(
'waiting for available files, attempt ' + str(attempt) +
', ' + datetime_format(datetime.today()) + '\n')
self.dnld_logfile.flush()
sleep(self.WAIT_TIME_DAEMON)
# impossible request files to download... exit
self.dnld_logfile.write(
'maximum of attempt for daemon, impossible request files to download, exiting '
+ datetime_format(datetime.today()) + '\n')
self.dnld_logfile.flush()
# TODO: sendmail error
exit()
def run(config_run):
if config_run.p4_stats not in [None, 'None']:
msg = 'Warning: The process {0} was executed before\n'.format(
config_run.process_name)
config_run.process_logfile.write(msg)
print(msg)
# optional argument prev_rundir only for p4_stats for computes all statistics
if config_run.prev_rundir:
msg = "p4_stats will make some statistics with previous run, located in:\n\t" + \
os.path.join(config_run.prev_rundir, os.path.basename(config_run.working_directory)) + '\n'
config_run.process_logfile.write(msg)
print(msg)
# define and check files in previous run directory
previous_p3_mosaic_dir = os.path.join(
config_run.prev_rundir,
os.path.basename(config_run.working_directory), "p3_mosaic")
if not os.path.isdir(previous_p3_mosaic_dir):
msg = '\nWARNING: The directory of previous p3_mosaic run: {0}\n' \
'not exist, please run the previous process before it.\n'\
'continue but some statistics don\'t will processed.\n'.format(previous_p3_mosaic_dir)
config_run.process_logfile.write(msg)
print(msg)
config_run.prev_rundir = None
else:
msg = '\nWARNING: Not defined \'--prev_rundir\' directory of previous run,\n' \
'continue but some statistics don\'t will processed.\n'
config_run.process_logfile.write(msg)
print(msg)
source_path = os.path.join(config_run.working_directory, 'p3_mosaic')
dir_process = os.path.join(config_run.working_directory,
config_run.process_name)
if not os.path.isdir(source_path):
msg = '\nError: The directory of previous process: {0}\n' \
'not exist, please run the previous process before it.\n'.format(source_path)
config_run.process_logfile.write(msg)
print(msg)
# save in setting
config_run.p4_stats = 'with errors! - ' + datetime_format(
datetime.today())
config_run.save()
return
if os.path.isdir(dir_process):
shutil.rmtree(dir_process)
# process file by file
def process():
for root, dirs, files in os.walk(source_path):
if len(files) != 0:
files = [x for x in files if x.endswith(('.tif', '.TIF'))]
for file in files:
##############
# Calculate the median statistical
msg = 'Calculating the median statistical for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# median directory
median_dir = os.path.join(dir_process, 'median')
if not os.path.isdir(median_dir):
os.makedirs(median_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
median_dir,
os.path.splitext(file)[0] + '_median.tif')
try:
statistics('median', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating median statistic\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Calculate the mean statistical
msg = 'Calculating the mean statistical for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# mean directory
mean_dir = os.path.join(dir_process, 'mean')
if not os.path.isdir(mean_dir):
os.makedirs(mean_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
mean_dir,
os.path.splitext(file)[0] + '_mean.tif')
try:
statistics('mean', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating mean statistic\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Calculate the standard deviation
msg = 'Calculating the standard deviation for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# standard deviation directory
std_dir = os.path.join(dir_process, 'std')
if not os.path.isdir(std_dir):
os.makedirs(std_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
std_dir,
os.path.splitext(file)[0] + '_std.tif')
try:
statistics('std', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating standard deviation\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Calculate the valid data
# this count the valid data (no nans) across the layers (time axis) in percentage (0-100%)
msg = 'Calculating the valid data for {0}: '.format(file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# valid data directory
vd_dir = os.path.join(dir_process, 'valid_data')
if not os.path.isdir(vd_dir):
os.makedirs(vd_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
vd_dir,
os.path.splitext(file)[0] + '_valid_data.tif')
try:
statistics('valid_data', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating valid data\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Calculate the signal-to-noise ratio
# this signal-to-noise ratio defined as the mean divided by the standard deviation.
msg = 'Calculating the signal-to-noise ratio for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# valid data directory
snr_dir = os.path.join(dir_process, 'snr')
if not os.path.isdir(snr_dir):
os.makedirs(snr_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
snr_dir,
os.path.splitext(file)[0] + '_snr.tif')
try:
statistics('snr', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating signal-to-noise ratio\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Computes the coefficient of variation, the ratio of the biased standard
# deviation to the mean.
msg = 'Calculating the coefficient of variation for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# coefficient of variation directory
coeff_var_dir = os.path.join(dir_process, 'coeff_var')
if not os.path.isdir(coeff_var_dir):
os.makedirs(coeff_var_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
coeff_var_dir,
os.path.splitext(file)[0] + '_coeff_var.tif')
try:
statistics('coeff_var', in_file, out_file)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating coefficient of variation\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
##############
# Calculate the Pearson's correlation coefficient between this time series (x)
# with the previous time series (y): pearson_corr = covar(x,y)/(std(x)*std(y))
if config_run.prev_rundir:
msg = 'Calculating the Pearson\'s correlation coefficient for {0}: '.format(
file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg, end='')
# Pearson's correlation coefficient directory
pearson_corr_dir = os.path.join(
dir_process, 'pearson_corr')
if not os.path.isdir(pearson_corr_dir):
os.makedirs(pearson_corr_dir)
in_file = os.path.join(root, file)
out_file = os.path.join(
pearson_corr_dir,
os.path.splitext(file)[0] + '_pearson_corr.tif')
try:
statistics('pearson_corr', in_file, out_file,
previous_p3_mosaic_dir)
msg = 'OK'
config_run.process_logfile.write(msg + '\n')
print(msg)
except Exception as error:
msg = 'FAIL\nError: While calculating Pearson\'s correlation coefficient\n' + error
config_run.process_logfile.write(msg + '\n')
print(msg)
return msg
return 0
return_code = process()
# finishing the process
msg = '\nThe process {0} completed {1}- ({2})'.format(
config_run.process_name, 'with errors! ' if return_code != 0 else '',
datetime_format(datetime.today()))
config_run.process_logfile.write(msg + '\n')
print(msg)
# save in setting
config_run.p4_stats = ('with errors! - ' if return_code != 0 else
'done - ') + datetime_format(datetime.today())
config_run.save()
def run(config_run):
if config_run.p3_mosaic not in [None, 'None']:
msg = 'Warning: The process {0} was executed before\n'.format(
config_run.process_name)
config_run.process_logfile.write(msg)
print(msg)
source_path = os.path.join(config_run.working_directory, 'p2_reproj')
dir_process = os.path.join(config_run.working_directory,
config_run.process_name)
if not os.path.isdir(source_path):
msg = '\nError: The directory of previous process: {0}\n' \
'not exist, please run the previous process before it.\n'.format(source_path)
config_run.process_logfile.write(msg)
print(msg)
# save in setting
config_run.p3_mosaic = 'with errors! - ' + datetime_format(
datetime.today())
config_run.save()
return
if os.path.isdir(dir_process):
shutil.rmtree(dir_process)
imgs_group_by_var = {}
scenes = [
'h10v07',
'h10v08',
'h10v09',
'h11v07',
'h11v08',
'h11v09',
]
# process file by file detecting the groups of bands and subproducts data
# inside directory for make the mosaic based on the name of the directory
for root, dirs, files in os.walk(source_path):
if len(files) != 0:
files = [x for x in files if x[-4::] == '.tif']
if files:
var = os.path.basename(root)
imgs_group_by_var[var] = []
imgs_by_scene = []
files_temp_list = list(files)
while files_temp_list:
# detect the repeat name for each variable for process the group
file_group = files_temp_list[0]
for group_name in scenes:
if group_name in os.path.basename(file_group):
scene_group_name = file_group.split(group_name)[1]
# selecting the file for mosaic
mosaic_input_list = []
for file in files_temp_list:
if scene_group_name in file:
mosaic_input_list.append(file)
# output name file for mosaic
scene_group_name = scene_group_name.replace(var, '')
scene_group_name = scene_group_name.replace('.tif', '')
scene_group_name = scene_group_name[
0:-1] if scene_group_name[
-1] == '_' else scene_group_name
scene_group_name = scene_group_name[
1::] if scene_group_name[0] == '_' else scene_group_name
# define file names
mosaic_name_tmp = scene_group_name + '_tmp.tif'
mosaic_name = scene_group_name + '.tif'
# mosaic path
mosaic_dest = os.path.dirname(
os.path.join(
dir_process,
os.path.join(root, file).split('/p2_reproj/')[-1]))
# list of file for make mosaic
mosaic_input_list_fullpath = [
os.path.join(root, f) for f in mosaic_input_list
]
# mosaic process
msg = 'Processing mosaic {0}: '.format(mosaic_name)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg)
return_code, msg = mosaic(mosaic_input_list_fullpath,
mosaic_dest, mosaic_name_tmp)
config_run.process_logfile.write(msg + '\n')
print(msg)
if return_code != 0: break
# clipping Colombia shape
return_code, msg = clipping_colombia(
os.path.join(mosaic_dest, mosaic_name_tmp),
os.path.join(mosaic_dest, mosaic_name))
print(msg)
if return_code != 0: break
# clean process files in list and sorted
files_temp_list = sorted(
list(set(files_temp_list) - set(mosaic_input_list)))
os.remove(os.path.join(mosaic_dest, mosaic_name_tmp))
if return_code != 0: break
# finishing the process
msg = '\nThe process {0} completed {1}- ({2})'.format(
config_run.process_name, 'with errors! ' if return_code != 0 else '',
datetime_format(datetime.today()))
config_run.process_logfile.write(msg + '\n')
print(msg)
# save in setting
config_run.p3_mosaic = ('with errors! - ' if return_code != 0 else
'done - ') + datetime_format(datetime.today())
config_run.save()
def run(config_run):
if config_run.p5_layerstack not in [None, 'None']:
msg = 'Warning: The process {0} was executed before\n'.format(
config_run.process_name)
config_run.process_logfile.write(msg)
print(msg)
source_path = os.path.join(config_run.working_directory, 'p4_stats')
dir_process = os.path.join(config_run.working_directory,
config_run.process_name)
if not os.path.isdir(source_path):
msg = '\nError: The directory of previous process: {0}\n' \
'not exist, please run the previous process before it.\n'.format(source_path)
config_run.process_logfile.write(msg)
print(msg)
# save in setting
config_run.p5_layerstack = 'with errors! - ' + datetime_format(
datetime.today())
config_run.save()
return
if os.path.isdir(dir_process):
shutil.rmtree(dir_process)
####### unir todas las bandas GeoTiff reproyectadas por cada imagen a GeoTiff multibanda
sorted_bands = [0, 1, 4, 5, 6, 2, 3]
# process file by file
for root, dirs, files in os.walk(source_path):
if len(files) != 0:
files = [x for x in files if x[-4::] == '.tif']
if files:
var = os.path.basename(root)
msg = 'Generating layer stack for variable {0}: '.format(var)
config_run.process_logfile.write(msg + '\n')
print(msg)
dest = os.path.join(dir_process, var)
if not os.path.isdir(dest):
os.makedirs(dest)
input_all_band = sorted(files)
# ordenar las bandas segun sorted_bands
input_all_band = [
x for (y, x) in sorted(zip(sorted_bands, input_all_band))
]
input_all_band = [
os.path.join(root, x) for x in input_all_band
]
# get the lower pixel size of all input band
all_pixel_sizes = [get_pixel_size(f) for f in input_all_band]
lower_pixel_size = sorted(all_pixel_sizes)[0]
msg = ' calculating the lower pixel size for all bands: ' + str(
lower_pixel_size)
config_run.process_logfile.write(msg + '\n')
config_run.process_logfile.flush()
print(msg)
# nombre del layer stack
out_file = os.path.join(dest, "LayerStack_" + var + ".tif")
# combinacion de bandas a GeoTiff multibanda usando gdal
return_code = call([
"gdal_merge.py", "-o", out_file, "-of", "GTiff",
"-separate", "-ps",
str(lower_pixel_size),
str(lower_pixel_size), "-co", "COMPRESS=LZW", "-co",
"PREDICTOR=2", "-co", "TILED=YES"
] + input_all_band)
if return_code == 0: # successfully
msg = ' was converted successfully'
config_run.process_logfile.write(msg + '\n')
print(msg)
else:
msg = '\nError: Problems in layerstack process, please check\n' \
'error message above, likely the files not were\n' \
'processed successfully.'
config_run.process_logfile.write(msg + '\n')
print(msg)
break
# finishing the process
msg = '\nThe process {0} completed {1}- ({2})'.format(
config_run.process_name, 'with errors! ' if return_code != 0 else '',
datetime_format(datetime.today()))
config_run.process_logfile.write(msg + '\n')
print(msg)
# save in setting
config_run.p5_layerstack = ('with errors! - ' if return_code != 0 else
'done - ') + datetime_format(datetime.today())
config_run.save()
def run(config_run):
if config_run.p2_reproj not in [None, 'None']:
msg = 'Warning: The process {0} was executed before\n'.format(config_run.process_name)
config_run.process_logfile.write(msg)
print(msg)
source_path = os.path.join(config_run.working_directory, 'p1_qc4sd')
dir_process = os.path.join(config_run.working_directory, config_run.process_name)
if not os.path.isdir(source_path):
msg = '\nError: The directory of previous process: {0}\n' \
'not exist, please run the previous process before it.\n'.format(source_path)
config_run.process_logfile.write(msg)
print(msg)
# save in setting
config_run.p2_reproj = 'with errors! - ' + datetime_format(datetime.today())
config_run.save()
return
if os.path.isdir(dir_process):
shutil.rmtree(dir_process)
if not os.path.isdir(dir_process):
os.makedirs(dir_process)
# process file by file
for root, dirs, files in os.walk(source_path):
if len(files) != 0:
files = [x for x in files if x.endswith(('.tif', '.TIF'))]
for file in files:
msg = 'Reprojection file {0} with Gdal: '.format(file)
config_run.process_logfile.write(msg)
config_run.process_logfile.flush()
print(msg)
##############
# first reprojection to The WGS LatLon (EPSG:4326)
in_file = os.path.join(root, file)
file_tmp = "tmp_"+file
out_file_tmp = os.path.join(dir_process, file_tmp)
# Reprojection with Gdal
return_code = call(["gdalwarp", in_file, out_file_tmp, "-r", "near", "-t_srs", "EPSG:4326"])
if return_code == 0: # successfully
msg = 'was reprojected to EPSG:4326 successfully'
config_run.process_logfile.write(msg + '\n')
print(msg)
else:
msg = '\nError: Problem while reprojecting in Gdal\n'
config_run.process_logfile.write(msg + '\n')
print(msg)
break
##############
# second reprojection to WGS 84 / UTM zone 18N (EPSG:32618)
out_file = os.path.join(dir_process, file)
mode = file.split('_')[1]
# process the mode MXD09A1: adjust pixel size
if mode in ['MOD09A1', 'MYD09A1']:
pixel_size = ["500", "500"]
# process the mode MXD09Q1: adjust pixel size
if mode in ['MOD09Q1', 'MYD09Q1']:
pixel_size = ["250", "250"]
# Reprojection with Gdal
return_code = call(["gdalwarp", out_file_tmp, out_file, "-r", "near", "-t_srs", "EPSG:32618",
"-co", "COMPRESS=LZW", "-co", "PREDICTOR=2", "-co", "TILED=YES", "-tr"] + pixel_size)
if return_code == 0: # successfully
msg = 'was reprojected to EPSG:32618 successfully'
config_run.process_logfile.write(msg + '\n')
print(msg)
else:
msg = '\nError: Problem while reprojecting in Gdal\n'
config_run.process_logfile.write(msg + '\n')
print(msg)
break
os.remove(out_file_tmp)
# finishing the process
msg = '\nThe process {0} completed {1}- ({2})'.format(config_run.process_name,
'with errors! ' if return_code != 0 else '',
datetime_format(datetime.today()))
config_run.process_logfile.write(msg + '\n')
print(msg)
# save in setting
config_run.p2_reproj = ('with errors! - ' if return_code != 0 else 'done - ') + datetime_format(datetime.today())
config_run.save()
def worker(self):
while True:
file, Q = self.Q.get()
file_download_ok = False
# Check if file exists and check it and not download if is corrected
if os.path.isfile(file.path):
# check cksum
check_status, check_msg = file.check(self)
if check_status in [0]:
self.dnld_logfile.write('file exits and it is correct: ' +
file.url + ' (' +
datetime_format(datetime.today()) +
') - ' + check_msg + '\n')
print('file exits and it is correct: ' +
os.path.basename(file.url))
self.dnld_logfile.flush()
file_download_ok = True
Q.put(('OK (' + check_msg + ')', file))
return
if check_status in [2, 3]:
self.dnld_logfile.write(
'file exits but this is corrupt: ' + file.url + ' (' +
datetime_format(datetime.today()) + ') - ' +
check_msg + '\n')
print('file exits but this is corrupt: ' +
os.path.basename(file.url))
self.dnld_logfile.flush()
os.remove(file.path)
if check_status in [4]:
self.dnld_logfile.write(
'xml file is corrupt, not check the file but continue: '
+ file.url + ' (' + datetime_format(datetime.today()) +
') - ' + check_msg + '\n')
print('xml file is corrupt, not check but continue: ' +
os.path.basename(file.url))
self.dnld_logfile.flush()
file_download_ok = True
Q.put(('NO CHECKED (' + check_msg + ')', file))
return
# download the file with some num of attempt if has error
for attempt in range(self.NUM_ATTEMPT):
self.dnld_logfile.write('download started: ' + file.url +
' (' +
datetime_format(datetime.today()) +
')\n')
print('download started: ' + os.path.basename(file.url))
self.dnld_logfile.flush()
# download with wget
wget_status = call(self.wget_cmd + [file.url],
shell=False) # TODO: Download real file
# wget_status = call(self.wget_cmd + [file.url+ '.xml'], shell=False) # TODO: Download xml file
# TODO: check time for wget process
# check wget_status
if wget_status == 0:
# check cksum
check_status, check_msg = file.check(
self) # TODO: Download real file and .check(self)
# check_status, check_msg = 0,'ok' # TODO: no check for download test xml file
if check_status in [0, 1]:
self.dnld_logfile.write(
'download finished: ' + file.url + ' (' +
datetime_format(datetime.today()) + ') - ' +
check_msg + '\n')
print('download finished: ' +
os.path.basename(file.url))
self.dnld_logfile.flush()
file_download_ok = True
Q.put(('OK (' + check_msg + ')', file))
return
elif check_status in [4]:
self.dnld_logfile.write(
'download finished: ' + file.url + ' (' +
datetime_format(datetime.today()) + ') - ' +
check_msg + '\n')
print('download finished: ' +
os.path.basename(file.url))
self.dnld_logfile.flush()
file_download_ok = True
Q.put(('NO CHECKED (' + check_msg + ')', file))
return
else:
self.dnld_logfile.write(
'error downloading, attempt ' + str(attempt) +
', ' + check_msg + ', try again: ' + file.url +
' (' + datetime_format(datetime.today()) + ')\n')
self.dnld_logfile.flush()
os.remove(file.path)
sleep(self.WAIT_TIME_ATTEMPT)
elif get_http_code(file.url) == 508:
self.dnld_logfile.write(
'the file is corrupt, file not downloaded (download continue): '
+ file.url + ' (' + datetime_format(datetime.today()) +
') - ' + '\n')
print('file is corrupt: ' + os.path.basename(file.url))
self.dnld_logfile.flush()
file_download_ok = True
Q.put(('CORRUPT (NOT DOWNLOADED)', file))
return
else:
# if wget_status != 0 is due a some error
self.dnld_logfile.write('error downloading, attempt ' +
str(attempt) + ', ' + file.url +
' (' +
datetime_format(datetime.today()) +
')\n')
self.dnld_logfile.flush()
os.remove(file.path)
sleep(self.WAIT_TIME_ATTEMPT)
if not file_download_ok:
Q.put(('ERROR', file))
def main(self, urls_files, DEST=False):
Q = Queue() # Create a second queue so the worker
self.DEST = DEST
self.start_dnld_datetime = datetime.today()
# make destination directory
if self.DEST:
if not os.path.isdir(self.DEST):
os.makedirs(self.DEST)
msg = '\n## Start download for: ' + self.dnld_name + '/' + self.dnld_date + ' - (' + datetime_format(
datetime.today()) + ')'
self.dnld_logfile.write(msg + '\n')
print(msg)
# is destination was defined
if self.DEST:
self.wget_cmd = self.WGET + ["-P", self.DEST]
else: # save in the same directory of run
self.wget_cmd = self.WGET
# start daemon request for check available files
self.daemon_request(urls_files)
# threads can send the data back again
for url in urls_files:
# create file instance
new_file = File(self, url)
# Add the URLs in `urls` to be downloaded asynchronously
self.Q.put((new_file, Q))
dnld_status = []
for i in range(len(urls_files)):
# Get the data as it arrives, raising
# any exceptions if they occur
status, file = Q.get()
dnld_status.append([file.name, file.getsize(), status])
self.dnld_status = dnld_status
# save errors produced in the download
for status in self.dnld_status:
if status[2] == 'ERROR':
self.errors.append('error downloading file ' + status[0])
DownloadManager.DNLD_ERRORS = len(self.errors)
# download
download_main.run(config_run)
########################################### process ###########################################
if args.make == 'process':
# ignore warnings
warnings.filterwarnings("ignore")
# save make in config_run
config_run.make = args.make
# set the log file for process
config_run.process_logfile = open(
os.path.join(config_run.working_directory, 'process.log'), 'a')
# init log of process
msg = '\n\n########### START LOG FOR PROCESS: ' + args.process + \
' - (' + datetime_format(datetime.today()) + ') ###########' + \
'\n#### in dir: ' + os.path.basename(config_run.working_directory) + '\n'
config_run.process_logfile.write(msg + '\n')
print(msg)
config_run.process_name = args.process
############################### Quality Control process - QC4SD ###############################
if args.process == 'p1_qc4sd':
p1_qc4sd.run(config_run)
#################################### Reprojection process #####################################
if args.process == 'p2_reproj':
p2_reproj.run(config_run)
######################################### make mosaic #########################################
def do_download():
for source in config_run.source:
global dnld_errors
# save the current source to download
config_run.current_source = source
######################################## pre download ########################################
# prepare directory to download
config_run.download_path = os.path.join(
config_run.working_directory, config_run.current_source,
'p0_download')
if not os.path.isdir(config_run.download_path):
os.makedirs(config_run.download_path)
# set the log file for download
config_run.dnld_logfile = open(
os.path.join(config_run.download_path, 'download.log'), 'a')
# init log of download
msg = '\n\n########### START LOG FOR: '+config_run.current_source.upper() + \
' - (' + datetime_format(datetime.today()) + ') ###########' + \
'\n#### target: ' + config_run.target_date.date.strftime('%Y-%m-%d') + \
'\n#### in dir: ' + os.path.basename(config_run.working_directory) + '\n'
config_run.dnld_logfile.write(msg + '\n')
print(msg)
# check if the download date is greater than the target date
today = date.today()
if config_run.target_date.date >= today:
msg = '\nThe target date for download files {0} is greater than current date {1}' \
.format(config_run.target_date.date.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d'))
config_run.dnld_logfile.write(msg + '\n')
print(msg)
exit()
########################################## download ##########################################
# print message of start download
msg = '\n#### Download target {0} (in dir: {1}) - ({2})'.format(
config_run.target_date.date.strftime('%Y-%m-%d'),
os.path.basename(config_run.working_directory),
datetime_format(datetime.today()))
config_run.dnld_logfile.write(msg + '\n')
print(msg)
# for Modis products: MOD09A1, MOD09Q1, MYD09A1, MYD09Q1
# surface Reflectance 8-Day L3: http://modis.gsfc.nasa.gov/data/dataprod/mod09.php
if config_run.modis_type == "mxd09a1_q1":
# download from terra
if config_run.current_source == 'terra':
dnld_errors_A1, status_file_A1 = modis.download(
config_run, 'MOD09A1')
dnld_errors_Q1, status_file_Q1 = modis.download(
config_run, 'MOD09Q1')
# download from aqua
if config_run.current_source == 'aqua':
dnld_errors_A1, status_file_A1 = modis.download(
config_run, 'MYD09A1')
dnld_errors_Q1, status_file_Q1 = modis.download(
config_run, 'MYD09Q1')
# for Modis products: MOD09GA, MOD09GQ, MYD09GA, MYD09GQ
# surface Reflectance Daily L2G: http://modis.gsfc.nasa.gov/data/dataprod/mod09.php
if config_run.modis_type == "mxd09ga_gq":
# download from terra
if config_run.current_source == 'terra':
dnld_errors_A1, status_file_A1 = modis.download(
config_run, 'MOD09GA')
dnld_errors_Q1, status_file_Q1 = modis.download(
config_run, 'MOD09GQ')
# download from aqua
if config_run.current_source == 'aqua':
dnld_errors_A1, status_file_A1 = modis.download(
config_run, 'MYD09GA')
dnld_errors_Q1, status_file_Q1 = modis.download(
config_run, 'MYD09GQ')
msg_error = None
## check errors from download
if dnld_errors_A1 != 0 or dnld_errors_Q1 != 0:
dnld_errors.append(config_run.target_date.date)
msg_error = 'with'
elif config_run.dnld_errors is not None:
# without error, then delete date in errors list if exist
if config_run.target_date.date.strftime(
'%Y-%m-%d') in config_run.dnld_errors:
config_run.dnld_errors.remove(
config_run.target_date.date.strftime('%Y-%m-%d'))
msg_error = 'without'
# print message of state of download
msg = '\n#### Download finished for {0} {1} errors - ({2})'.format(
config_run.target_date.date.strftime('%Y-%m-%d'), msg_error,
datetime_format(datetime.today()))
config_run.dnld_logfile.write(msg + '\n')
print(msg)
# update the target date
config_run.target_date.next()
# rename folder
update_working_directory(config_run)
# update/save config file
config_run.save()