def collect_log_files(tmp_dir, tar_dir, test=None):
"""
Collect and compress log files related to current processing.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:return:
"""
logger.info("Collect mpmd and task log-files")
tsub = pygac_commit + "-log_files"
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
source = os.path.join(ecfbase, "mpmd/generated")
cmd = ["cp", "-r", source, tdir]
logger.info("{cmd}".format(cmd=cmd))
exe_sb(cmd=cmd)
source = os.path.join(datadir, "mpmd/log")
cmd = ["cp", "-r", source, tdir]
logger.info("{cmd}".format(cmd=cmd))
exe_sb(cmd=cmd)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
def collect_sql_files(tmp_dir, tar_dir, test=None):
"""
Collect and compress SQLite databases related to current processing.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:return:
"""
logger.info("Collect sqlite files")
tsub = pygac_commit + "-sql_files"
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
copy(src=sql_gacdb_archive, dst=tdir)
copy(src=sql_pystat_output, dst=tdir)
copy(src=sql_pygac_logout, dst=tdir)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name,
format='gztar',
root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
def collect_log_files(tmp_dir, tar_dir, test=None):
"""
Collect and compress log files related to current processing.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:return:
"""
logger.info("Collect mpmd and task log-files")
tsub = pygac_commit + "-log_files"
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
source = os.path.join(ecfbase, "mpmd/generated")
cmd = ["cp", "-r", source, tdir]
logger.info("{cmd}".format(cmd=cmd))
exe_sb(cmd=cmd)
source = os.path.join(datadir, "mpmd/log")
cmd = ["cp", "-r", source, tdir]
logger.info("{cmd}".format(cmd=cmd))
exe_sb(cmd=cmd)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name,
format='gztar',
root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
def l2(args_l2):
"""
Subcommand for archiving L2 data into ECFS.
"""
# -- create date string
datestring = str(args_l2.year) + str('%02d' % args_l2.month)
# -- sensor and platform settings
if args_l2.instrument.upper() == "AVHRR":
(satstr, satnum) = split_platform_string(args_l2.satellite)
sensor = args_l2.instrument.upper()
platform = satstr.upper() + '-' + satnum
else:
sensor = args_l2.instrument.upper()
platform = args_l2.satellite.upper()
# -- list everything in input directory
alldirs = os.listdir(args_l2.inpdir)
# -- list of files to be archived
tarfile_list = list()
# *** L2 archiving ***
# -- find latest job via ID-US number
if len(alldirs) > 0:
getdirs = list()
# collect right subfolders
for ad in alldirs:
if datestring in ad \
and args_l2.instrument.upper() in ad \
and args_l2.satellite.lower() in ad \
and 'retrieval' in ad:
getdirs.append(ad)
# sort list
getdirs.sort()
# get last element from list, should be last job
lastdir = getdirs.pop()
# get ID number from the last job
idnumber = get_id(lastdir)
# archive data
(tlist, tempdir) = tar_results("L2", args_l2.inpdir,
datestring, sensor,
platform, idnumber)
tarfile_list = tlist
else:
logger.info("Check your input directory, maybe it is empty ?")
# copy tarfile into ECFS
logger.info("Copy2ECFS: tarfile_list")
copy_into_ecfs(datestring, tarfile_list, args_l2.ecfsdir)
# delete tempdir
logger.info("Delete \'{0}\'".format(tempdir))
delete_dir(tempdir)
def clear_l1(args_l1):
"""
Remove L1 satellite input data if retrieval was successful.
"""
# find platform
if args_l1.satellite.upper() == "TERRA" or \
args_l1.satellite.upper() == "MOD":
platform = "MOD"
sensor = "MODIS"
elif args_l1.satellite.upper() == "AQUA" or \
args_l1.satellite.upper() == "MYD":
platform = "MYD"
sensor = "MODIS"
elif args_l1.satellite.upper().startswith("NOAA"):
platform = args_l1.satellite.lower()
sensor = "AVHRR"
elif args_l1.satellite.upper().startswith("METOP"):
platform = args_l1.satellite.lower()
sensor = "AVHRR"
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
# delete corr. config file
# config_proc_1_getdata_avhrr_2008_01_NOAA15.file
# config_proc_1_getdata_modis_2008_01_AQUA.file
cfgbase = cfg_prefix + "1_getdata_" + sensor.lower() + '_' + \
str(args_l1.year) + '_' + str('%02d' % args_l1.month) + \
'_' + platform.upper() + cfg_suffix
cfgfile = os.path.join(args_l1.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\'".format(cfgfile))
delete_file(cfgfile)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".
format(cfgfile))
ipath = os.path.join(args_l1.inpdir, sensor, platform,
str(args_l1.year), str('%02d' % args_l1.month))
if os.path.isdir(ipath):
logger.info("Delete: \'{0}\' -> retrieval was successful!".
format(ipath))
delete_dir(ipath)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".
format(ipath))
def clear_aux(args_aux):
"""
Remove auxiliary data if complete month was successful.
"""
ipath = os.path.join(args_aux.inpdir, args_aux.auxdata)
if not os.path.isdir(ipath):
logger.info("The argument for --auxdata should be equal "
"to the name of the subfolder you want to clean up.")
sys.exit(0)
# delete corr. config file
# config_proc_1_getdata_aux_2008_01.file
# config_proc_1_getdata_era_2008_01.file
if args_aux.auxdata.lower().startswith("aux"):
auxkey = "aux"
elif args_aux.auxdata.lower().startswith("era"):
auxkey = "era"
cfgbase = cfg_prefix + "1_getdata_" + auxkey + '_' + \
str(args_aux.year) + '_' + str('%02d' % args_aux.month) + \
cfg_suffix
cfgfile = os.path.join(args_aux.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\' ".format(cfgfile))
delete_file(cfgfile)
else:
logger.info(
"Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile))
ilist = os.listdir(ipath)
for i in ilist:
if not i.isdigit():
# aux
# BRDF_dir # albedo_dir # emissivity_dir # ice_snow_dir
ispath = os.path.join(ipath, i, str(args_aux.year),
str('%02d' % args_aux.month))
else:
# ERAinterim
# 2008
ispath = os.path.join(ipath, str(args_aux.year),
str('%02d' % args_aux.month))
if os.path.isdir(ispath):
logger.info(
"Delete: \'{0}\' -> month was successful!".format(ispath))
delete_dir(ispath)
else:
logger.info(
"Nothing to delete: \'{0}\' doesn't exist!".format(ispath))
def clear_aux(args_aux):
"""
Remove auxiliary data if complete month was successful.
"""
ipath = os.path.join(args_aux.inpdir, args_aux.auxdata)
if not os.path.isdir(ipath):
logger.info("The argument for --auxdata should be equal "
"to the name of the subfolder you want to clean up.")
sys.exit(0)
# delete corr. config file
# config_proc_1_getdata_aux_2008_01.file
# config_proc_1_getdata_era_2008_01.file
if args_aux.auxdata.lower().startswith("aux"):
auxkey = "aux"
elif args_aux.auxdata.lower().startswith("era"):
auxkey = "era"
cfgbase = cfg_prefix + "1_getdata_" + auxkey + '_' + \
str(args_aux.year) + '_' + str('%02d' % args_aux.month) + \
cfg_suffix
cfgfile = os.path.join(args_aux.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\' ".format(cfgfile))
delete_file(cfgfile)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile))
ilist = os.listdir(ipath)
for i in ilist:
if not i.isdigit():
# aux
# BRDF_dir # albedo_dir # emissivity_dir # ice_snow_dir
ispath = os.path.join(ipath, i, str(args_aux.year),
str('%02d' % args_aux.month))
else:
# ERAinterim
# 2008
ispath = os.path.join(ipath, str(args_aux.year),
str('%02d' % args_aux.month))
if os.path.isdir(ispath):
logger.info("Delete: \'{0}\' -> month was successful!".
format(ispath))
delete_dir(ispath)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".
format(ispath))
def clear_l1(args_l1):
"""
Remove L1 satellite input data if retrieval was successful.
"""
# find platform
if args_l1.satellite.upper() == "TERRA" or \
args_l1.satellite.upper() == "MOD":
platform = "MOD"
sensor = "MODIS"
elif args_l1.satellite.upper() == "AQUA" or \
args_l1.satellite.upper() == "MYD":
platform = "MYD"
sensor = "MODIS"
elif args_l1.satellite.upper().startswith("NOAA"):
platform = args_l1.satellite.lower()
sensor = "AVHRR"
elif args_l1.satellite.upper().startswith("METOP"):
platform = args_l1.satellite.lower()
sensor = "AVHRR"
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
# delete corr. config file
# config_proc_1_getdata_avhrr_2008_01_NOAA15.file
# config_proc_1_getdata_modis_2008_01_AQUA.file
cfgbase = cfg_prefix + "1_getdata_" + sensor.lower() + '_' + \
str(args_l1.year) + '_' + str('%02d' % args_l1.month) + \
'_' + platform.upper() + cfg_suffix
cfgfile = os.path.join(args_l1.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\'".format(cfgfile))
delete_file(cfgfile)
else:
logger.info(
"Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile))
ipath = os.path.join(args_l1.inpdir, sensor, platform, str(args_l1.year),
str('%02d' % args_l1.month))
if os.path.isdir(ipath):
logger.info(
"Delete: \'{0}\' -> retrieval was successful!".format(ipath))
delete_dir(ipath)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".format(ipath))
def collect_invalid_orbits(tmp_dir, tar_dir, sy, ey, test=None):
"""
Collect invalid AVHRR GAC L1c orbits and compress files per satellite.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:param sy: start year
:param ey: end year
:return:
"""
logger.info("Collect invalid L1c orbits")
sats = pygac_sat_list()
for s in sats:
year = sy
while year <= ey:
pattern = "ECC_GAC_*" + s + "*_" + str(year) + "*.h5"
files = get_file_list(pattern=pattern, path=relict_dir)
if files:
logger.info("Working on {sat} and {year}".format(
sat=s, year=str(year)))
files.sort()
tsub = pygac_commit + "-invalid_l1c_orbits_" + s + "_" + str(
year)
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
for f in files:
copy(src=f, dst=tdir)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name,
format='gztar',
root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history,
ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
year += 1
else:
year += 1
continue
def collect_invalid_orbits(tmp_dir, tar_dir, sy, ey, test=None):
"""
Collect invalid AVHRR GAC L1c orbits and compress files per satellite.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:param sy: start year
:param ey: end year
:return:
"""
logger.info("Collect invalid L1c orbits")
sats = pygac_sat_list()
for s in sats:
year = sy
while year <= ey:
pattern = "ECC_GAC_*" + s + "*_" + str(year) + "*.h5"
files = get_file_list(pattern=pattern, path=relict_dir)
if files:
logger.info("Working on {sat} and {year}".format(sat=s, year=str(year)))
files.sort()
tsub = pygac_commit + "-invalid_l1c_orbits_" + s + "_" + str(year)
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
for f in files:
copy(src=f, dst=tdir)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
year += 1
else:
year += 1
continue
def collect_sql_files(tmp_dir, tar_dir, test=None):
"""
Collect and compress SQLite databases related to current processing.
:param tmp_dir: copy files for tarball creation to this directory
:param tar_dir: tarball written to this directory
:return:
"""
logger.info("Collect sqlite files")
tsub = pygac_commit + "-sql_files"
tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub)
copy(src=sql_gacdb_archive, dst=tdir)
copy(src=sql_pystat_output, dst=tdir)
copy(src=sql_pygac_logout, dst=tdir)
tar_name = os.path.join(tar_dir, tsub)
tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir)
delete_dir(tmpdir=tdir)
if not test:
copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch)
logger.info("DONE {0}".format(os.path.basename(tar_arch)))
parser.add_argument('--eyear', type=int, default='2017',
help='End Year, e.g. 2006')
parser.add_argument('--test', action="store_true",
help="Testing option, copy to ECFS disabled.")
args = parser.parse_args()
# Call function associated with the selected subcommand
logger.info("{0} start for {1}".format(sys.argv[0], args))
save_dir = make_temp_dir(temp_dir=datadir, temp_sub=pygac_commit)
if args.logs:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_logs")
collect_log_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test)
delete_dir(tmpdir=work_dir)
if args.sqls:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_sqls")
collect_sql_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test)
delete_dir(tmpdir=work_dir)
if args.invalids:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_invalids")
collect_invalid_orbits(tmp_dir=work_dir, tar_dir=save_dir,
sy=args.syear, ey=args.eyear, test=args.test)
delete_dir(tmpdir=work_dir)
logger.info("{0} finished!".format(os.path.basename(__file__)))
stderr=subprocess.PIPE)
stdout, stderr = p4.communicate()
logger.info("add2sqlite_l1c_info STDOUT:{0}".format(stdout))
logger.info("add2sqlite_l1c_info STDERR:{0}".format(stderr))
# -- end of loop over l1bfiles
# -- check if output is equal input
output1 = os.path.join(out, "*avhrr*h5")
output2 = os.path.join(out, "*sunsatangles*h5")
output3 = os.path.join(out, "*qualflags*h5")
avhrr_cnt = len(glob.glob(output1))
sunsat_cnt = len(glob.glob(output2))
flags_cnt = len(glob.glob(output3))
if avhrr_cnt != l1bcnt and sunsat_cnt != l1bcnt and flags_cnt != l1bcnt:
logger.info("PYGAC FAILED {0} time(s):".format(len(failed_l1b_orbits)))
for failed in failed_l1b_orbits:
logger.info("{0}".format(failed))
else:
# -- delete dirs: where L1b files are extracted
delete_dir(inp)
logger.info("TarFile {0} finished: {1} L1b -> {2} L1c\n".format(
tarbase, l1bcnt, avhrr_cnt))
# -- end of loop over tarfiles
logger.info("{0} finished!".format(os.path.basename(__file__)))
for root, dirs, files in os.walk(make_tar_tmp_dir):
for filename in fnmatch.filter(files, tar_pattern):
tar_files_list.append(os.path.join(root, filename))
# -- sort list
tar_files_list.sort()
# -- create ecp_tar_upload directory
if not os.path.exists(ecp_tar_upload):
os.makedirs(ecp_tar_upload)
# -- move all tarfiles to final subfolder
move_files(tar_files_list, ecp_tar_upload)
# -- delete tar tmp directory
delete_dir(make_tar_tmp_dir)
# -- delete pygac results, i.e. l1c input
delete_dir(inp)
# -- delete l1b main input tarfile in ecp_tar_download
db = AvhrrGacDatabase(dbfile=sql_gacdb_archive)
results = db.get_tarfiles(start_date=args.start_date,
end_date=args.end_date,
sats=[args.satellite],
include_blacklisted=False,
strict=False)
db.close()
for res in results:
# "NOAA7_1985_01.tar"
def l3(args_l3):
"""
Subcommand for archiving l3 data into ECFS.
"""
# -- create date string
datestring = str(args_l3.year) + str('%02d' % args_l3.month)
print "_______________________________________"
# -- sensor and platform settings for L3U and L3C
if args_l3.prodtype.upper() != "L3S" and args_l3.satellite:
if args_l3.instrument.upper() == "AVHRR" and not args_l3.satellite.upper().startswith("METOP"):
(satstr, satnum) = split_platform_string(args_l3.satellite)
sensor = args_l3.instrument.upper()
platform = satstr.upper() + '-' + satnum
else:
sensor = args_l3.instrument.upper()
platform = args_l3.satellite.upper()
elif args_l3.prodtype.upper() == "L3S":
sensor = args_l3.instrument.upper()
platform = "MERGED"
print sensor + " " + platform
else:
logger.info("You chose prodtype={0},"
"thus give me also a satellite name".
format(args_l3.prodtype))
sys.exit(0)
# -- set prodtype_name
if args_l3.prodtype.upper() == "L3U":
prodtype_name = "daily_samples"
elif args_l3.prodtype.upper() == "L3C" or args_l3.prodtype.upper() == "L3S":
prodtype_name = "monthly_means"
else:
logger.info("Wrong --prodtype input!")
sys.exit(0)
# -- list everything in input directory
alldirs = os.listdir(args_l3.inpdir)
# -- archive now: L3U or L3C (satellite AND instrument)
if args_l3.prodtype.upper() != "L3S":
# -- find latest job via ID-US number
if len(alldirs) > 0:
getdirs = list()
# collect right subfolders
for ad in alldirs:
if args_l3.local:
if datestring in ad \
and args_l3.instrument.upper() in ad \
and args_l3.satellite.upper() in ad \
and args_l3.prodtype.upper() in ad \
and "Europe" in ad \
and prodtype_name in ad:
getdirs.append(ad)
else:
if datestring in ad \
and args_l3.instrument.upper() in ad \
and args_l3.satellite.upper() in ad \
and args_l3.prodtype.upper() in ad \
and "Europe" not in ad \
and prodtype_name in ad:
getdirs.append(ad)
# sort list
getdirs.sort(key=lambda s: os.path.getmtime(os.path.join(args_l3.inpdir, s)), reverse=True)
# get first element from list, should be last job
lastdir = getdirs[0]
# get ID number from the last job
idnumber = get_id(lastdir)
if args_l3.prodtype.upper() == "L3U":
if args_l3.write_l3u_splitlist:
full_lastdir = os.path.join(args_l3.inpdir, lastdir)
for f in os.listdir(full_lastdir):
if ("CLD_PRODUCTS" in f and f.endswith(".nc")):
full_f = os.path.join(full_lastdir, f)
p = sub.Popen(['cdo', 'sinfon', full_f],stdout=sub.PIPE,stderr=sub.PIPE)
output, errors = p.communicate()
# split at spaces, remove invalid entries and next-line strings
output = filter(None, output.split(" "))
output = [s.strip('\n') for s in output]
cld_products = 'selname,' + ','.join([s for s in output if "ctt_" in s or "cth_" in s or "ctp_" in s or "cer_" in s or ("cot_" in s and not "cccot_" in s) or "cwp_" in s or "stemp_" in s or "cph_" in s or "qcflag_" in s or "time_" in s])
cld_masktype = 'selname,' + ','.join([s for s in output if "cty_" in s or "cccot_" in s or "cmask_" in s or "cph_" in s or "time_" in s or "ann_phase_" in s])
cld_angles = 'selname,' + ','.join([s for s in output if "illum_" in s or "satzen_" in s or "solzen_" in s or "relazi_" in s or "time_" in s])
rad_products = 'selname,' + ','.join([s for s in output if "toa_" in s or "boa_" in s or "cla_" in s or "cee_" in s or "time_" in s])
if args_l3.local:
sat_measurements = 'selname,' + ','.join([s for s in output if "refl_" in s or "bt_" in s or "time_" in s])
taskfile = open(args_l3.inpdir + "/" + "_".join([args_l3.instrument.upper(), args_l3.satellite.upper(), idnumber, "L3U_splitting_tasklist.txt"]), "w")
for path in getdirs:
full_path = os.path.join(args_l3.inpdir, path)
cd = 'cd ' + full_path
files = os.listdir(full_path)
for l3u_file in files:
if ("CLD_PRODUCTS" in l3u_file and l3u_file.endswith(".nc")):
mask = split_l3u(cld_masktype, l3u_file, "CLD_MASKTYPE")
ang = split_l3u(cld_angles, l3u_file, "CLD_ANGLES")
rad = split_l3u(rad_products, l3u_file, "RAD_PRODUCTS")
call = " && ".join([cd, mask, ang, rad])
if args_l3.local:
toa = split_l3u(sat_measurements, l3u_file, "SAT_MEASUREMENTS")
call = call + " && " + toa
prod = split_l3u(cld_products, l3u_file, "CLD_PRODUCTS")
call = call + " && " + prod + "\n"
taskfile.write(call)
taskfile.close()
sys.exit(0)
if args_l3.check_splitting:
for root, dirs, files in os.walk(args_l3.inpdir):
for name in files:
if idnumber in name:
taskfile_name = os.path.join(root, name)
with open(taskfile_name) as f:
taskfile = f.readlines()
taskfile = [x.strip() for x in taskfile]
for name in dirs:
matches = []
if name.endswith(idnumber):
dirfiles = os.listdir(os.path.join(root, name))
for filename in fnmatch.filter(dirfiles, '*.nc'):
matches.append(filename)
nfiles = len(matches)
if (nfiles < 4 and nfiles > 0 and not args_l3.local) or (nfiles < 5 and nfiles > 0 and args_l3.local):
print "Not enough NetCDF output files in " + name + ", so will split sequentially"
yyyymmdd = name[0:8]
for line in taskfile:
if yyyymmdd in line:
cwd = line.split()[1]
calls = line.split(" && ")[1:]
for call in calls:
process = sub.Popen(call.split(), stdout=sub.PIPE, stderr=sub.PIPE, cwd=cwd)
output, error = process.communicate()
sys.exit(0)
# make tarfile
(tlist, tempdir) = tar_results(args_l3.prodtype.upper(), args_l3.inpdir,
datestring, sensor, platform, idnumber, args_l3.local)
logger.info("Copy2ECFS: {0}".format(tlist))
copy_into_ecfs(datestring, tlist, args_l3.ecfsdir)
logger.info("Move files to {0}".format(tmp_data_storage))
create_dir(tmp_data_storage)
move_files(tlist, tmp_data_storage)
logger.info("Delete \'{0}\'".format(tempdir))
delete_dir(tempdir)
else:
logger.info("Check your input directory, maybe it is empty ?")
# -- archive now: L3S (sensor family, i.e. all AVHRR or MODIS)
elif args_l3.prodtype.upper() == "L3S":
# -- find latest job via ID-US number
if len(alldirs) > 0:
getdirs = list()
# collect right subfolders
for ad in alldirs:
if datestring in ad \
and args_l3.instrument.upper() in ad \
and args_l3.prodtype.upper() in ad \
and prodtype_name in ad:
getdirs.append(ad)
# sort list
getdirs.sort()
print "________________________________________"
print getdirs
# get last element from list, should be last job
lastdir = getdirs.pop()
# get ID number from the last job
idnumber = get_id(lastdir)
print "________________________________________"
print idnumber
print args_l3.prodtype.upper() + " " + args_l3.inpdir + " " + datestring + " " + sensor + " " + platform + " " + idnumber #+ " " + args_l3.local
print args_l3.local
# make tarfile
(tlist, tempdir) = tar_results(args_l3.prodtype.upper(), args_l3.inpdir,
datestring, sensor, platform, idnumber, args_l3.local)
logger.info("Copy2ECFS: {0}".format(tlist))
copy_into_ecfs(datestring, tlist, args_l3.ecfsdir)
logger.info("Move files to {0}".format(tmp_data_storage))
create_dir(tmp_data_storage)
move_files(tlist, tmp_data_storage)
logger.info("Delete \'{0}\'".format(tempdir))
delete_dir(tempdir)
else:
logger.info("Check your input directory, maybe it is empty ?")
else:
logger.info("Nothing was archived for l3 parameters passed:")
logger.info(" * INPDIR :{0} ".format(args_l3.inpdir))
logger.info(" * ECFSDIR :{0} ".format(args_l3.ecfsdir))
logger.info(" * PRODTYPE:{0} ".format(args_l3.prodtype))
def clear_l2(args_l2):
"""
Search for ID and then remove all directories,
which belong to this ID and satellite and sensor and date.
"""
if args_l2.satellite.upper() == "TERRA":
platform = "TERRA"
elif args_l2.satellite.upper() == "AQUA":
platform = "AQUA"
elif args_l2.satellite.upper() == "MOD":
platform = "TERRA"
elif args_l2.satellite.upper() == "MYD":
platform = "AQUA"
elif args_l2.satellite.upper().startswith("NOAA"):
platform = args_l2.satellite.lower()
elif args_l2.satellite.upper().startswith("METOP"):
platform = args_l2.satellite.lower()
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
# delete corr. config file
# config_proc_2_process_2008_06_NOAA15.file
# config_proc_2_process_2008_06_AQUA.file
if "output" in args_l2.inpdir:
cfgbase = cfg_prefix + "2_process_" + str(args_l2.year) + \
'_' + str('%02d' % args_l2.month) + \
'_' + platform.upper() + cfg_suffix
cfgfile = os.path.join(args_l2.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\' ".format(cfgfile))
delete_file(cfgfile)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".
format(cfgfile))
# date string
datestr = str(args_l2.year) + str('%02d' % args_l2.month)
# get dirs list containing all subdirs of given path
alldirs = os.listdir(args_l2.inpdir)
# if alldirs is not empty
if len(alldirs) > 0:
# get dirs list matching the arguments
getdirs = list()
for ad in alldirs:
if datestr in ad \
and args_l2.instrument.upper() in ad \
and platform in ad and 'retrieval' in ad:
getdirs.append( os.path.join(args_l2.inpdir, ad) )
# check if getdirs list is empty
if len(getdirs) == 0:
logger.info("Nothing to delete in {0} for {1} {2} {3}".
format(args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
else:
# sort list
getdirs.sort()
# get last element from list, should be last job
lastdir = getdirs[-1]
# get ID number from the last job
id_number = get_id(lastdir)
# remove all subdirs matching the id number
for gdir in getdirs:
if id_number in gdir:
logger.info("Delete: {0}".format(gdir))
delete_dir(gdir)
# pattern
pattern = datestr + '*' + args_l2.instrument.upper() + '_' + \
platform + '_retrieval_' + id_number
# remove all dirs matching pattern
logger.info("Delete: \'{0}\' -> retrieval failed or was successful!".
format(pattern))
else:
logger.info("Nothing to delete in {0} for {1} {2} {3}".
format(args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
def clear_l3(args_l3):
"""
Remove L3 results if archiving was successful.
"""
# find platform
if args_l3.prodtype.lower() != "l3s":
if args_l3.satellite:
if args_l3.satellite.upper() == "TERRA" or \
args_l3.satellite.upper() == "MOD":
platform = "TERRA"
elif args_l3.satellite.upper() == "AQUA" or \
args_l3.satellite.upper() == "MYD":
platform = "AQUA"
elif args_l3.satellite.upper().startswith("NOAA"):
platform = args_l3.satellite.upper()
elif args_l3.satellite.upper().startswith("METOP"):
platform = args_l3.satellite.upper()
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
else:
logger.info("You chose prodtype={0}, "
"so tell me which platform!".format(args_l3.prodtype))
sys.exit(0)
# find sensor
sensor = args_l3.instrument.upper()
# delete corr. config file
cfg_file_list = list()
# -- monthly config files
# config_proc_3_make_l3c_NOAA18_2008_01.file
# config_proc_3_make_l3c_AQUA_2008_01.file
if args_l3.prodtype.lower() == "l3c":
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \
'_' + platform.upper() + '_' + str(args_l3.year) + \
'_' + str('%02d' % args_l3.month) + '_' + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- monthly config files
# config_proc_3_make_l3s_AVHRR_2008_01.file
# config_proc_3_make_l3s_MODIS_2008_01.file
elif args_l3.prodtype.lower() == "l3s":
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \
'_' + sensor.upper() + '_' + str(args_l3.year) + \
'_' + str('%02d' % args_l3.month) + '_' + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- daily config files
# config_proc_3_make_l3u_NOAA18_2008_01_01.file
# config_proc_3_make_l3u_TERRA_2008_01_31.file
elif args_l3.prodtype.lower() == "l3u":
# calendar.monthrange
# Returns weekday of first day of the month and
# number of days in month, for the specified year and month.
last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1]
for iday in range(last_day_of_month):
iday += 1
if args_l3.local:
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + "_Europe" + cfg_suffix
else:
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- daily config files
# config_proc_3_make_l2b_sum_NOAA18_2008_01_01.file
# config_proc_3_make_l2b_sum_NOAA18_2008_01_31.file
elif args_l3.prodtype.lower() == "l2b_sum":
last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1]
for iday in range(last_day_of_month):
iday += 1
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# now remove all config files in the list
for cfile in cfg_file_list:
if os.path.isfile(cfile):
logger.info("Delete: \'{0}\' ".format(cfile))
delete_file(cfile)
else:
logger.info(
"Nothing to delete: \'{0}\' doesn't exist!".format(cfile))
# date string
datestr = str(args_l3.year) + str('%02d' % args_l3.month)
# get dirs list containing all subdirs of given path
alldirs = os.listdir(args_l3.inpdir)
if len(alldirs) > 0:
# get dirs list matching the arguments
getdirs = list()
for ad in alldirs:
# L3S: no platform, sensor fam. monthly averages
if args_l3.prodtype.lower() == "l3s":
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad:
getdirs.append(os.path.join(args_l3.inpdir, ad))
# L3C, L3U, L2B_SUM: sensor and platform in subdirectory_name
else:
if args_l3.local:
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad and platform in ad \
and 'Europe' in ad:
getdirs.append(os.path.join(args_l3.inpdir, ad))
else:
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad and platform in ad \
and not 'Europe' in ad:
getdirs.append(os.path.join(args_l3.inpdir, ad))
# check if getdirs list is empty
if len(getdirs) == 0:
logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".format(
args_l3.inpdir, sensor, platform, datestr,
args_l3.prodtype.upper()))
else:
# sort list
getdirs.sort()
# get last element from list, should be last job
lastdir = getdirs[-1]
# get ID number from the last job
id_num = get_id(lastdir)
# remove all subdirs matching the id number
for gdir in getdirs:
if id_num in gdir:
logger.info("Delete: {0}".format(gdir))
delete_dir(gdir)
# pattern
# L3S: sensor fam. monthly averages
if args_l3.prodtype.lower() == "l3s":
pattern = datestr + '*' + sensor + '_ORAC*' + \
'*' + args_l3.prodtype.upper() + '*' + id_num
else:
pattern = datestr + '*' + sensor + '_ORAC*' + platform + \
'*' + args_l3.prodtype.upper() + '*' + id_num
# remove all dirs matching pattern
logger.info(
"Delete: \'{0}\' -> L2toL3 was successful!".format(pattern))
else:
logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".format(
args_l3.inpdir, sensor, platform, datestr, args_l3.prodtype))
def clear_l3(args_l3):
"""
Remove L3 results if archiving was successful.
"""
# find platform
if args_l3.prodtype.lower() != "l3s":
if args_l3.satellite:
if args_l3.satellite.upper() == "TERRA" or \
args_l3.satellite.upper() == "MOD":
platform = "TERRA"
elif args_l3.satellite.upper() == "AQUA" or \
args_l3.satellite.upper() == "MYD":
platform = "AQUA"
elif args_l3.satellite.upper().startswith("NOAA"):
platform = args_l3.satellite.upper()
elif args_l3.satellite.upper().startswith("METOP"):
platform = args_l3.satellite.upper()
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
else:
logger.info("You chose prodtype={0}, "
"so tell me which platform!".
format(args_l3.prodtype))
sys.exit(0)
# find sensor
sensor = args_l3.instrument.upper()
# delete corr. config file
cfg_file_list = list()
# -- monthly config files
# config_proc_3_make_l3c_NOAA18_2008_01.file
# config_proc_3_make_l3c_AQUA_2008_01.file
if args_l3.prodtype.lower() == "l3c":
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \
'_' + platform.upper() + '_' + str(args_l3.year) + \
'_' + str('%02d' % args_l3.month) + '_' + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- monthly config files
# config_proc_3_make_l3s_AVHRR_2008_01.file
# config_proc_3_make_l3s_MODIS_2008_01.file
elif args_l3.prodtype.lower() == "l3s":
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \
'_' + sensor.upper() + '_' + str(args_l3.year) + \
'_' + str('%02d' % args_l3.month) + '_' + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- daily config files
# config_proc_3_make_l3u_NOAA18_2008_01_01.file
# config_proc_3_make_l3u_TERRA_2008_01_31.file
elif args_l3.prodtype.lower() == "l3u":
# calendar.monthrange
# Returns weekday of first day of the month and
# number of days in month, for the specified year and month.
last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1]
for iday in range(last_day_of_month):
iday += 1
if args_l3.local:
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + "_Europe" + cfg_suffix
else:
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# -- daily config files
# config_proc_3_make_l2b_sum_NOAA18_2008_01_01.file
# config_proc_3_make_l2b_sum_NOAA18_2008_01_31.file
elif args_l3.prodtype.lower() == "l2b_sum":
last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1]
for iday in range(last_day_of_month):
iday += 1
cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \
platform.upper() + '_' + \
str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \
str('%02d' % iday) + cfg_suffix
cfgfile = os.path.join(args_l3.cfgdir, cfgbase)
cfg_file_list.append(cfgfile)
# now remove all config files in the list
for cfile in cfg_file_list:
if os.path.isfile(cfile):
logger.info("Delete: \'{0}\' ".format(cfile))
delete_file(cfile)
else:
logger.info("Nothing to delete: \'{0}\' doesn't exist!".
format(cfile))
# date string
datestr = str(args_l3.year) + str('%02d' % args_l3.month)
# get dirs list containing all subdirs of given path
alldirs = os.listdir(args_l3.inpdir)
if len(alldirs) > 0:
# get dirs list matching the arguments
getdirs = list()
for ad in alldirs:
# L3S: no platform, sensor fam. monthly averages
if args_l3.prodtype.lower() == "l3s":
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad:
getdirs.append( os.path.join(args_l3.inpdir, ad) )
# L3C, L3U, L2B_SUM: sensor and platform in subdirectory_name
else:
if args_l3.local:
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad and platform in ad \
and 'Europe' in ad:
getdirs.append( os.path.join(args_l3.inpdir, ad) )
else:
if datestr in ad and sensor in ad \
and args_l3.prodtype.upper() in ad \
and 'ORAC' in ad and platform in ad \
and not 'Europe' in ad:
getdirs.append( os.path.join(args_l3.inpdir, ad) )
# check if getdirs list is empty
if len(getdirs) == 0:
logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".
format(args_l3.inpdir, sensor, platform, datestr,
args_l3.prodtype.upper()))
else:
# sort list
getdirs.sort()
# get last element from list, should be last job
lastdir = getdirs[-1]
# get ID number from the last job
id_num = get_id(lastdir)
# remove all subdirs matching the id number
for gdir in getdirs:
if id_num in gdir:
logger.info("Delete: {0}".format(gdir))
delete_dir(gdir)
# pattern
# L3S: sensor fam. monthly averages
if args_l3.prodtype.lower() == "l3s":
pattern = datestr + '*' + sensor + '_ORAC*' + \
'*' + args_l3.prodtype.upper() + '*' + id_num
else:
pattern = datestr + '*' + sensor + '_ORAC*' + platform + \
'*' + args_l3.prodtype.upper() + '*' + id_num
# remove all dirs matching pattern
logger.info("Delete: \'{0}\' -> L2toL3 was successful!".
format(pattern))
else:
logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".
format(args_l3.inpdir, sensor, platform, datestr,
args_l3.prodtype))
p4 = subprocess.Popen(c4, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p4.communicate()
logger.info("add2sqlite_l1c_info STDOUT:{0}".format(stdout))
logger.info("add2sqlite_l1c_info STDERR:{0}".format(stderr))
# -- end of loop over l1bfiles
# -- check if output is equal input
output1 = os.path.join(out, "*avhrr*h5")
output2 = os.path.join(out, "*sunsatangles*h5")
output3 = os.path.join(out, "*qualflags*h5")
avhrr_cnt = len(glob.glob(output1))
sunsat_cnt = len(glob.glob(output2))
flags_cnt = len(glob.glob(output3))
if avhrr_cnt != l1bcnt and sunsat_cnt != l1bcnt and flags_cnt != l1bcnt:
logger.info("PYGAC FAILED {0} time(s):".format(len(failed_l1b_orbits)))
for failed in failed_l1b_orbits:
logger.info("{0}".format(failed))
else:
# -- delete dirs: where L1b files are extracted
delete_dir(inp)
logger.info("TarFile {0} finished: {1} L1b -> {2} L1c\n".
format(tarbase, l1bcnt, avhrr_cnt))
# -- end of loop over tarfiles
logger.info("{0} finished!".format(os.path.basename(__file__)))
def clear_l2(args_l2):
"""
Search for ID and then remove all directories,
which belong to this ID and satellite and sensor and date.
"""
if args_l2.satellite.upper() == "TERRA":
platform = "TERRA"
elif args_l2.satellite.upper() == "AQUA":
platform = "AQUA"
elif args_l2.satellite.upper() == "MOD":
platform = "TERRA"
elif args_l2.satellite.upper() == "MYD":
platform = "AQUA"
elif args_l2.satellite.upper().startswith("NOAA"):
platform = args_l2.satellite.lower()
elif args_l2.satellite.upper().startswith("METOP"):
platform = args_l2.satellite.lower()
else:
logger.info("WRONG SATELLITE NAME!")
sys.exit(0)
# delete corr. config file
# config_proc_2_process_2008_06_NOAA15.file
# config_proc_2_process_2008_06_AQUA.file
if "output" in args_l2.inpdir:
cfgbase = cfg_prefix + "2_process_" + str(args_l2.year) + \
'_' + str('%02d' % args_l2.month) + \
'_' + platform.upper() + cfg_suffix
cfgfile = os.path.join(args_l2.cfgdir, cfgbase)
if os.path.isfile(cfgfile):
logger.info("Delete: \'{0}\' ".format(cfgfile))
delete_file(cfgfile)
else:
logger.info(
"Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile))
# date string
datestr = str(args_l2.year) + str('%02d' % args_l2.month)
# get dirs list containing all subdirs of given path
alldirs = os.listdir(args_l2.inpdir)
# if alldirs is not empty
if len(alldirs) > 0:
# get dirs list matching the arguments
getdirs = list()
for ad in alldirs:
if datestr in ad \
and args_l2.instrument.upper() in ad \
and platform in ad and 'retrieval' in ad:
getdirs.append(os.path.join(args_l2.inpdir, ad))
# check if getdirs list is empty
if len(getdirs) == 0:
logger.info("Nothing to delete in {0} for {1} {2} {3}".format(
args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
else:
# sort list
getdirs.sort()
# get last element from list, should be last job
lastdir = getdirs[-1]
# get ID number from the last job
id_number = get_id(lastdir)
# remove all subdirs matching the id number
for gdir in getdirs:
if id_number in gdir:
logger.info("Delete: {0}".format(gdir))
delete_dir(gdir)
# pattern
pattern = datestr + '*' + args_l2.instrument.upper() + '_' + \
platform + '_retrieval_' + id_number
# remove all dirs matching pattern
logger.info(
"Delete: \'{0}\' -> retrieval failed or was successful!".
format(pattern))
else:
logger.info("Nothing to delete in {0} for {1} {2} {3}".format(
args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
# -- sort list
tar_files_list.sort()
# -- create ecp_tar_upload directory
if not os.path.exists(ecp_tar_upload):
os.makedirs(ecp_tar_upload)
# -- move all tarfiles to final subfolder
move_files(tar_files_list, ecp_tar_upload)
# -- delete tar tmp directory
delete_dir(make_tar_tmp_dir)
# -- delete pygac results, i.e. l1c input
delete_dir(inp)
# -- delete l1b main input tarfile in ecp_tar_download
db = AvhrrGacDatabase(dbfile=sql_gacdb_archive)
results = db.get_tarfiles(start_date=args.start_date, end_date=args.end_date,
sats=[args.satellite], include_blacklisted=False,
strict=False)
db.close()
for res in results:
# "NOAA7_1985_01.tar"
parser.add_argument('--test',
action="store_true",
help="Testing option, copy to ECFS disabled.")
args = parser.parse_args()
# Call function associated with the selected subcommand
logger.info("{0} start for {1}".format(sys.argv[0], args))
save_dir = make_temp_dir(temp_dir=datadir, temp_sub=pygac_commit)
if args.logs:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_logs")
collect_log_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test)
delete_dir(tmpdir=work_dir)
if args.sqls:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_sqls")
collect_sql_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test)
delete_dir(tmpdir=work_dir)
if args.invalids:
work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_invalids")
collect_invalid_orbits(tmp_dir=work_dir,
tar_dir=save_dir,
sy=args.syear,
ey=args.eyear,
test=args.test)
delete_dir(tmpdir=work_dir)