def collect_log_files(tmp_dir, tar_dir, test=None): """ Collect and compress log files related to current processing. :param tmp_dir: copy files for tarball creation to this directory :param tar_dir: tarball written to this directory :return: """ logger.info("Collect mpmd and task log-files") tsub = pygac_commit + "-log_files" tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub) source = os.path.join(ecfbase, "mpmd/generated") cmd = ["cp", "-r", source, tdir] logger.info("{cmd}".format(cmd=cmd)) exe_sb(cmd=cmd) source = os.path.join(datadir, "mpmd/log") cmd = ["cp", "-r", source, tdir] logger.info("{cmd}".format(cmd=cmd)) exe_sb(cmd=cmd) tar_name = os.path.join(tar_dir, tsub) tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir) delete_dir(tmpdir=tdir) if not test: copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch) logger.info("DONE {0}".format(os.path.basename(tar_arch)))
def collect_sql_files(tmp_dir, tar_dir, test=None): """ Collect and compress SQLite databases related to current processing. :param tmp_dir: copy files for tarball creation to this directory :param tar_dir: tarball written to this directory :return: """ logger.info("Collect sqlite files") tsub = pygac_commit + "-sql_files" tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub) copy(src=sql_gacdb_archive, dst=tdir) copy(src=sql_pystat_output, dst=tdir) copy(src=sql_pygac_logout, dst=tdir) tar_name = os.path.join(tar_dir, tsub) tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir) delete_dir(tmpdir=tdir) if not test: copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch) logger.info("DONE {0}".format(os.path.basename(tar_arch)))
def l2(args_l2): """ Subcommand for archiving L2 data into ECFS. """ # -- create date string datestring = str(args_l2.year) + str('%02d' % args_l2.month) # -- sensor and platform settings if args_l2.instrument.upper() == "AVHRR": (satstr, satnum) = split_platform_string(args_l2.satellite) sensor = args_l2.instrument.upper() platform = satstr.upper() + '-' + satnum else: sensor = args_l2.instrument.upper() platform = args_l2.satellite.upper() # -- list everything in input directory alldirs = os.listdir(args_l2.inpdir) # -- list of files to be archived tarfile_list = list() # *** L2 archiving *** # -- find latest job via ID-US number if len(alldirs) > 0: getdirs = list() # collect right subfolders for ad in alldirs: if datestring in ad \ and args_l2.instrument.upper() in ad \ and args_l2.satellite.lower() in ad \ and 'retrieval' in ad: getdirs.append(ad) # sort list getdirs.sort() # get last element from list, should be last job lastdir = getdirs.pop() # get ID number from the last job idnumber = get_id(lastdir) # archive data (tlist, tempdir) = tar_results("L2", args_l2.inpdir, datestring, sensor, platform, idnumber) tarfile_list = tlist else: logger.info("Check your input directory, maybe it is empty ?") # copy tarfile into ECFS logger.info("Copy2ECFS: tarfile_list") copy_into_ecfs(datestring, tarfile_list, args_l2.ecfsdir) # delete tempdir logger.info("Delete \'{0}\'".format(tempdir)) delete_dir(tempdir)
def clear_l1(args_l1): """ Remove L1 satellite input data if retrieval was successful. """ # find platform if args_l1.satellite.upper() == "TERRA" or \ args_l1.satellite.upper() == "MOD": platform = "MOD" sensor = "MODIS" elif args_l1.satellite.upper() == "AQUA" or \ args_l1.satellite.upper() == "MYD": platform = "MYD" sensor = "MODIS" elif args_l1.satellite.upper().startswith("NOAA"): platform = args_l1.satellite.lower() sensor = "AVHRR" elif args_l1.satellite.upper().startswith("METOP"): platform = args_l1.satellite.lower() sensor = "AVHRR" else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) # delete corr. config file # config_proc_1_getdata_avhrr_2008_01_NOAA15.file # config_proc_1_getdata_modis_2008_01_AQUA.file cfgbase = cfg_prefix + "1_getdata_" + sensor.lower() + '_' + \ str(args_l1.year) + '_' + str('%02d' % args_l1.month) + \ '_' + platform.upper() + cfg_suffix cfgfile = os.path.join(args_l1.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\'".format(cfgfile)) delete_file(cfgfile) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!". format(cfgfile)) ipath = os.path.join(args_l1.inpdir, sensor, platform, str(args_l1.year), str('%02d' % args_l1.month)) if os.path.isdir(ipath): logger.info("Delete: \'{0}\' -> retrieval was successful!". format(ipath)) delete_dir(ipath) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!". format(ipath))
def clear_aux(args_aux): """ Remove auxiliary data if complete month was successful. """ ipath = os.path.join(args_aux.inpdir, args_aux.auxdata) if not os.path.isdir(ipath): logger.info("The argument for --auxdata should be equal " "to the name of the subfolder you want to clean up.") sys.exit(0) # delete corr. config file # config_proc_1_getdata_aux_2008_01.file # config_proc_1_getdata_era_2008_01.file if args_aux.auxdata.lower().startswith("aux"): auxkey = "aux" elif args_aux.auxdata.lower().startswith("era"): auxkey = "era" cfgbase = cfg_prefix + "1_getdata_" + auxkey + '_' + \ str(args_aux.year) + '_' + str('%02d' % args_aux.month) + \ cfg_suffix cfgfile = os.path.join(args_aux.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\' ".format(cfgfile)) delete_file(cfgfile) else: logger.info( "Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile)) ilist = os.listdir(ipath) for i in ilist: if not i.isdigit(): # aux # BRDF_dir # albedo_dir # emissivity_dir # ice_snow_dir ispath = os.path.join(ipath, i, str(args_aux.year), str('%02d' % args_aux.month)) else: # ERAinterim # 2008 ispath = os.path.join(ipath, str(args_aux.year), str('%02d' % args_aux.month)) if os.path.isdir(ispath): logger.info( "Delete: \'{0}\' -> month was successful!".format(ispath)) delete_dir(ispath) else: logger.info( "Nothing to delete: \'{0}\' doesn't exist!".format(ispath))
def clear_aux(args_aux): """ Remove auxiliary data if complete month was successful. """ ipath = os.path.join(args_aux.inpdir, args_aux.auxdata) if not os.path.isdir(ipath): logger.info("The argument for --auxdata should be equal " "to the name of the subfolder you want to clean up.") sys.exit(0) # delete corr. config file # config_proc_1_getdata_aux_2008_01.file # config_proc_1_getdata_era_2008_01.file if args_aux.auxdata.lower().startswith("aux"): auxkey = "aux" elif args_aux.auxdata.lower().startswith("era"): auxkey = "era" cfgbase = cfg_prefix + "1_getdata_" + auxkey + '_' + \ str(args_aux.year) + '_' + str('%02d' % args_aux.month) + \ cfg_suffix cfgfile = os.path.join(args_aux.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\' ".format(cfgfile)) delete_file(cfgfile) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile)) ilist = os.listdir(ipath) for i in ilist: if not i.isdigit(): # aux # BRDF_dir # albedo_dir # emissivity_dir # ice_snow_dir ispath = os.path.join(ipath, i, str(args_aux.year), str('%02d' % args_aux.month)) else: # ERAinterim # 2008 ispath = os.path.join(ipath, str(args_aux.year), str('%02d' % args_aux.month)) if os.path.isdir(ispath): logger.info("Delete: \'{0}\' -> month was successful!". format(ispath)) delete_dir(ispath) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!". format(ispath))
def clear_l1(args_l1): """ Remove L1 satellite input data if retrieval was successful. """ # find platform if args_l1.satellite.upper() == "TERRA" or \ args_l1.satellite.upper() == "MOD": platform = "MOD" sensor = "MODIS" elif args_l1.satellite.upper() == "AQUA" or \ args_l1.satellite.upper() == "MYD": platform = "MYD" sensor = "MODIS" elif args_l1.satellite.upper().startswith("NOAA"): platform = args_l1.satellite.lower() sensor = "AVHRR" elif args_l1.satellite.upper().startswith("METOP"): platform = args_l1.satellite.lower() sensor = "AVHRR" else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) # delete corr. config file # config_proc_1_getdata_avhrr_2008_01_NOAA15.file # config_proc_1_getdata_modis_2008_01_AQUA.file cfgbase = cfg_prefix + "1_getdata_" + sensor.lower() + '_' + \ str(args_l1.year) + '_' + str('%02d' % args_l1.month) + \ '_' + platform.upper() + cfg_suffix cfgfile = os.path.join(args_l1.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\'".format(cfgfile)) delete_file(cfgfile) else: logger.info( "Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile)) ipath = os.path.join(args_l1.inpdir, sensor, platform, str(args_l1.year), str('%02d' % args_l1.month)) if os.path.isdir(ipath): logger.info( "Delete: \'{0}\' -> retrieval was successful!".format(ipath)) delete_dir(ipath) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!".format(ipath))
def collect_invalid_orbits(tmp_dir, tar_dir, sy, ey, test=None): """ Collect invalid AVHRR GAC L1c orbits and compress files per satellite. :param tmp_dir: copy files for tarball creation to this directory :param tar_dir: tarball written to this directory :param sy: start year :param ey: end year :return: """ logger.info("Collect invalid L1c orbits") sats = pygac_sat_list() for s in sats: year = sy while year <= ey: pattern = "ECC_GAC_*" + s + "*_" + str(year) + "*.h5" files = get_file_list(pattern=pattern, path=relict_dir) if files: logger.info("Working on {sat} and {year}".format( sat=s, year=str(year))) files.sort() tsub = pygac_commit + "-invalid_l1c_orbits_" + s + "_" + str( year) tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub) for f in files: copy(src=f, dst=tdir) tar_name = os.path.join(tar_dir, tsub) tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir) delete_dir(tmpdir=tdir) if not test: copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch) logger.info("DONE {0}".format(os.path.basename(tar_arch))) year += 1 else: year += 1 continue
def collect_invalid_orbits(tmp_dir, tar_dir, sy, ey, test=None): """ Collect invalid AVHRR GAC L1c orbits and compress files per satellite. :param tmp_dir: copy files for tarball creation to this directory :param tar_dir: tarball written to this directory :param sy: start year :param ey: end year :return: """ logger.info("Collect invalid L1c orbits") sats = pygac_sat_list() for s in sats: year = sy while year <= ey: pattern = "ECC_GAC_*" + s + "*_" + str(year) + "*.h5" files = get_file_list(pattern=pattern, path=relict_dir) if files: logger.info("Working on {sat} and {year}".format(sat=s, year=str(year))) files.sort() tsub = pygac_commit + "-invalid_l1c_orbits_" + s + "_" + str(year) tdir = make_temp_dir(temp_dir=tmp_dir, temp_sub=tsub) for f in files: copy(src=f, dst=tdir) tar_name = os.path.join(tar_dir, tsub) tar_arch = make_archive(base_name=tar_name, format='gztar', root_dir=tmp_dir) delete_dir(tmpdir=tdir) if not test: copy_to_ecfs_archive(ecfs_dir=proc_history, ecfs_file=tar_arch) logger.info("DONE {0}".format(os.path.basename(tar_arch))) year += 1 else: year += 1 continue
parser.add_argument('--eyear', type=int, default='2017', help='End Year, e.g. 2006') parser.add_argument('--test', action="store_true", help="Testing option, copy to ECFS disabled.") args = parser.parse_args() # Call function associated with the selected subcommand logger.info("{0} start for {1}".format(sys.argv[0], args)) save_dir = make_temp_dir(temp_dir=datadir, temp_sub=pygac_commit) if args.logs: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_logs") collect_log_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test) delete_dir(tmpdir=work_dir) if args.sqls: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_sqls") collect_sql_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test) delete_dir(tmpdir=work_dir) if args.invalids: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_invalids") collect_invalid_orbits(tmp_dir=work_dir, tar_dir=save_dir, sy=args.syear, ey=args.eyear, test=args.test) delete_dir(tmpdir=work_dir) logger.info("{0} finished!".format(os.path.basename(__file__)))
stderr=subprocess.PIPE) stdout, stderr = p4.communicate() logger.info("add2sqlite_l1c_info STDOUT:{0}".format(stdout)) logger.info("add2sqlite_l1c_info STDERR:{0}".format(stderr)) # -- end of loop over l1bfiles # -- check if output is equal input output1 = os.path.join(out, "*avhrr*h5") output2 = os.path.join(out, "*sunsatangles*h5") output3 = os.path.join(out, "*qualflags*h5") avhrr_cnt = len(glob.glob(output1)) sunsat_cnt = len(glob.glob(output2)) flags_cnt = len(glob.glob(output3)) if avhrr_cnt != l1bcnt and sunsat_cnt != l1bcnt and flags_cnt != l1bcnt: logger.info("PYGAC FAILED {0} time(s):".format(len(failed_l1b_orbits))) for failed in failed_l1b_orbits: logger.info("{0}".format(failed)) else: # -- delete dirs: where L1b files are extracted delete_dir(inp) logger.info("TarFile {0} finished: {1} L1b -> {2} L1c\n".format( tarbase, l1bcnt, avhrr_cnt)) # -- end of loop over tarfiles logger.info("{0} finished!".format(os.path.basename(__file__)))
for root, dirs, files in os.walk(make_tar_tmp_dir): for filename in fnmatch.filter(files, tar_pattern): tar_files_list.append(os.path.join(root, filename)) # -- sort list tar_files_list.sort() # -- create ecp_tar_upload directory if not os.path.exists(ecp_tar_upload): os.makedirs(ecp_tar_upload) # -- move all tarfiles to final subfolder move_files(tar_files_list, ecp_tar_upload) # -- delete tar tmp directory delete_dir(make_tar_tmp_dir) # -- delete pygac results, i.e. l1c input delete_dir(inp) # -- delete l1b main input tarfile in ecp_tar_download db = AvhrrGacDatabase(dbfile=sql_gacdb_archive) results = db.get_tarfiles(start_date=args.start_date, end_date=args.end_date, sats=[args.satellite], include_blacklisted=False, strict=False) db.close() for res in results: # "NOAA7_1985_01.tar"
def l3(args_l3): """ Subcommand for archiving l3 data into ECFS. """ # -- create date string datestring = str(args_l3.year) + str('%02d' % args_l3.month) print "_______________________________________" # -- sensor and platform settings for L3U and L3C if args_l3.prodtype.upper() != "L3S" and args_l3.satellite: if args_l3.instrument.upper() == "AVHRR" and not args_l3.satellite.upper().startswith("METOP"): (satstr, satnum) = split_platform_string(args_l3.satellite) sensor = args_l3.instrument.upper() platform = satstr.upper() + '-' + satnum else: sensor = args_l3.instrument.upper() platform = args_l3.satellite.upper() elif args_l3.prodtype.upper() == "L3S": sensor = args_l3.instrument.upper() platform = "MERGED" print sensor + " " + platform else: logger.info("You chose prodtype={0}," "thus give me also a satellite name". format(args_l3.prodtype)) sys.exit(0) # -- set prodtype_name if args_l3.prodtype.upper() == "L3U": prodtype_name = "daily_samples" elif args_l3.prodtype.upper() == "L3C" or args_l3.prodtype.upper() == "L3S": prodtype_name = "monthly_means" else: logger.info("Wrong --prodtype input!") sys.exit(0) # -- list everything in input directory alldirs = os.listdir(args_l3.inpdir) # -- archive now: L3U or L3C (satellite AND instrument) if args_l3.prodtype.upper() != "L3S": # -- find latest job via ID-US number if len(alldirs) > 0: getdirs = list() # collect right subfolders for ad in alldirs: if args_l3.local: if datestring in ad \ and args_l3.instrument.upper() in ad \ and args_l3.satellite.upper() in ad \ and args_l3.prodtype.upper() in ad \ and "Europe" in ad \ and prodtype_name in ad: getdirs.append(ad) else: if datestring in ad \ and args_l3.instrument.upper() in ad \ and args_l3.satellite.upper() in ad \ and args_l3.prodtype.upper() in ad \ and "Europe" not in ad \ and prodtype_name in ad: getdirs.append(ad) # sort list getdirs.sort(key=lambda s: os.path.getmtime(os.path.join(args_l3.inpdir, s)), reverse=True) # get first element from list, should be last job lastdir = getdirs[0] # get ID number from the last job idnumber = get_id(lastdir) if args_l3.prodtype.upper() == "L3U": if args_l3.write_l3u_splitlist: full_lastdir = os.path.join(args_l3.inpdir, lastdir) for f in os.listdir(full_lastdir): if ("CLD_PRODUCTS" in f and f.endswith(".nc")): full_f = os.path.join(full_lastdir, f) p = sub.Popen(['cdo', 'sinfon', full_f],stdout=sub.PIPE,stderr=sub.PIPE) output, errors = p.communicate() # split at spaces, remove invalid entries and next-line strings output = filter(None, output.split(" ")) output = [s.strip('\n') for s in output] cld_products = 'selname,' + ','.join([s for s in output if "ctt_" in s or "cth_" in s or "ctp_" in s or "cer_" in s or ("cot_" in s and not "cccot_" in s) or "cwp_" in s or "stemp_" in s or "cph_" in s or "qcflag_" in s or "time_" in s]) cld_masktype = 'selname,' + ','.join([s for s in output if "cty_" in s or "cccot_" in s or "cmask_" in s or "cph_" in s or "time_" in s or "ann_phase_" in s]) cld_angles = 'selname,' + ','.join([s for s in output if "illum_" in s or "satzen_" in s or "solzen_" in s or "relazi_" in s or "time_" in s]) rad_products = 'selname,' + ','.join([s for s in output if "toa_" in s or "boa_" in s or "cla_" in s or "cee_" in s or "time_" in s]) if args_l3.local: sat_measurements = 'selname,' + ','.join([s for s in output if "refl_" in s or "bt_" in s or "time_" in s]) taskfile = open(args_l3.inpdir + "/" + "_".join([args_l3.instrument.upper(), args_l3.satellite.upper(), idnumber, "L3U_splitting_tasklist.txt"]), "w") for path in getdirs: full_path = os.path.join(args_l3.inpdir, path) cd = 'cd ' + full_path files = os.listdir(full_path) for l3u_file in files: if ("CLD_PRODUCTS" in l3u_file and l3u_file.endswith(".nc")): mask = split_l3u(cld_masktype, l3u_file, "CLD_MASKTYPE") ang = split_l3u(cld_angles, l3u_file, "CLD_ANGLES") rad = split_l3u(rad_products, l3u_file, "RAD_PRODUCTS") call = " && ".join([cd, mask, ang, rad]) if args_l3.local: toa = split_l3u(sat_measurements, l3u_file, "SAT_MEASUREMENTS") call = call + " && " + toa prod = split_l3u(cld_products, l3u_file, "CLD_PRODUCTS") call = call + " && " + prod + "\n" taskfile.write(call) taskfile.close() sys.exit(0) if args_l3.check_splitting: for root, dirs, files in os.walk(args_l3.inpdir): for name in files: if idnumber in name: taskfile_name = os.path.join(root, name) with open(taskfile_name) as f: taskfile = f.readlines() taskfile = [x.strip() for x in taskfile] for name in dirs: matches = [] if name.endswith(idnumber): dirfiles = os.listdir(os.path.join(root, name)) for filename in fnmatch.filter(dirfiles, '*.nc'): matches.append(filename) nfiles = len(matches) if (nfiles < 4 and nfiles > 0 and not args_l3.local) or (nfiles < 5 and nfiles > 0 and args_l3.local): print "Not enough NetCDF output files in " + name + ", so will split sequentially" yyyymmdd = name[0:8] for line in taskfile: if yyyymmdd in line: cwd = line.split()[1] calls = line.split(" && ")[1:] for call in calls: process = sub.Popen(call.split(), stdout=sub.PIPE, stderr=sub.PIPE, cwd=cwd) output, error = process.communicate() sys.exit(0) # make tarfile (tlist, tempdir) = tar_results(args_l3.prodtype.upper(), args_l3.inpdir, datestring, sensor, platform, idnumber, args_l3.local) logger.info("Copy2ECFS: {0}".format(tlist)) copy_into_ecfs(datestring, tlist, args_l3.ecfsdir) logger.info("Move files to {0}".format(tmp_data_storage)) create_dir(tmp_data_storage) move_files(tlist, tmp_data_storage) logger.info("Delete \'{0}\'".format(tempdir)) delete_dir(tempdir) else: logger.info("Check your input directory, maybe it is empty ?") # -- archive now: L3S (sensor family, i.e. all AVHRR or MODIS) elif args_l3.prodtype.upper() == "L3S": # -- find latest job via ID-US number if len(alldirs) > 0: getdirs = list() # collect right subfolders for ad in alldirs: if datestring in ad \ and args_l3.instrument.upper() in ad \ and args_l3.prodtype.upper() in ad \ and prodtype_name in ad: getdirs.append(ad) # sort list getdirs.sort() print "________________________________________" print getdirs # get last element from list, should be last job lastdir = getdirs.pop() # get ID number from the last job idnumber = get_id(lastdir) print "________________________________________" print idnumber print args_l3.prodtype.upper() + " " + args_l3.inpdir + " " + datestring + " " + sensor + " " + platform + " " + idnumber #+ " " + args_l3.local print args_l3.local # make tarfile (tlist, tempdir) = tar_results(args_l3.prodtype.upper(), args_l3.inpdir, datestring, sensor, platform, idnumber, args_l3.local) logger.info("Copy2ECFS: {0}".format(tlist)) copy_into_ecfs(datestring, tlist, args_l3.ecfsdir) logger.info("Move files to {0}".format(tmp_data_storage)) create_dir(tmp_data_storage) move_files(tlist, tmp_data_storage) logger.info("Delete \'{0}\'".format(tempdir)) delete_dir(tempdir) else: logger.info("Check your input directory, maybe it is empty ?") else: logger.info("Nothing was archived for l3 parameters passed:") logger.info(" * INPDIR :{0} ".format(args_l3.inpdir)) logger.info(" * ECFSDIR :{0} ".format(args_l3.ecfsdir)) logger.info(" * PRODTYPE:{0} ".format(args_l3.prodtype))
def clear_l2(args_l2): """ Search for ID and then remove all directories, which belong to this ID and satellite and sensor and date. """ if args_l2.satellite.upper() == "TERRA": platform = "TERRA" elif args_l2.satellite.upper() == "AQUA": platform = "AQUA" elif args_l2.satellite.upper() == "MOD": platform = "TERRA" elif args_l2.satellite.upper() == "MYD": platform = "AQUA" elif args_l2.satellite.upper().startswith("NOAA"): platform = args_l2.satellite.lower() elif args_l2.satellite.upper().startswith("METOP"): platform = args_l2.satellite.lower() else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) # delete corr. config file # config_proc_2_process_2008_06_NOAA15.file # config_proc_2_process_2008_06_AQUA.file if "output" in args_l2.inpdir: cfgbase = cfg_prefix + "2_process_" + str(args_l2.year) + \ '_' + str('%02d' % args_l2.month) + \ '_' + platform.upper() + cfg_suffix cfgfile = os.path.join(args_l2.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\' ".format(cfgfile)) delete_file(cfgfile) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!". format(cfgfile)) # date string datestr = str(args_l2.year) + str('%02d' % args_l2.month) # get dirs list containing all subdirs of given path alldirs = os.listdir(args_l2.inpdir) # if alldirs is not empty if len(alldirs) > 0: # get dirs list matching the arguments getdirs = list() for ad in alldirs: if datestr in ad \ and args_l2.instrument.upper() in ad \ and platform in ad and 'retrieval' in ad: getdirs.append( os.path.join(args_l2.inpdir, ad) ) # check if getdirs list is empty if len(getdirs) == 0: logger.info("Nothing to delete in {0} for {1} {2} {3}". format(args_l2.inpdir, args_l2.instrument.upper(), platform, datestr)) else: # sort list getdirs.sort() # get last element from list, should be last job lastdir = getdirs[-1] # get ID number from the last job id_number = get_id(lastdir) # remove all subdirs matching the id number for gdir in getdirs: if id_number in gdir: logger.info("Delete: {0}".format(gdir)) delete_dir(gdir) # pattern pattern = datestr + '*' + args_l2.instrument.upper() + '_' + \ platform + '_retrieval_' + id_number # remove all dirs matching pattern logger.info("Delete: \'{0}\' -> retrieval failed or was successful!". format(pattern)) else: logger.info("Nothing to delete in {0} for {1} {2} {3}". format(args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
def clear_l3(args_l3): """ Remove L3 results if archiving was successful. """ # find platform if args_l3.prodtype.lower() != "l3s": if args_l3.satellite: if args_l3.satellite.upper() == "TERRA" or \ args_l3.satellite.upper() == "MOD": platform = "TERRA" elif args_l3.satellite.upper() == "AQUA" or \ args_l3.satellite.upper() == "MYD": platform = "AQUA" elif args_l3.satellite.upper().startswith("NOAA"): platform = args_l3.satellite.upper() elif args_l3.satellite.upper().startswith("METOP"): platform = args_l3.satellite.upper() else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) else: logger.info("You chose prodtype={0}, " "so tell me which platform!".format(args_l3.prodtype)) sys.exit(0) # find sensor sensor = args_l3.instrument.upper() # delete corr. config file cfg_file_list = list() # -- monthly config files # config_proc_3_make_l3c_NOAA18_2008_01.file # config_proc_3_make_l3c_AQUA_2008_01.file if args_l3.prodtype.lower() == "l3c": cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \ '_' + platform.upper() + '_' + str(args_l3.year) + \ '_' + str('%02d' % args_l3.month) + '_' + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- monthly config files # config_proc_3_make_l3s_AVHRR_2008_01.file # config_proc_3_make_l3s_MODIS_2008_01.file elif args_l3.prodtype.lower() == "l3s": cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \ '_' + sensor.upper() + '_' + str(args_l3.year) + \ '_' + str('%02d' % args_l3.month) + '_' + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- daily config files # config_proc_3_make_l3u_NOAA18_2008_01_01.file # config_proc_3_make_l3u_TERRA_2008_01_31.file elif args_l3.prodtype.lower() == "l3u": # calendar.monthrange # Returns weekday of first day of the month and # number of days in month, for the specified year and month. last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1] for iday in range(last_day_of_month): iday += 1 if args_l3.local: cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + "_Europe" + cfg_suffix else: cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- daily config files # config_proc_3_make_l2b_sum_NOAA18_2008_01_01.file # config_proc_3_make_l2b_sum_NOAA18_2008_01_31.file elif args_l3.prodtype.lower() == "l2b_sum": last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1] for iday in range(last_day_of_month): iday += 1 cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # now remove all config files in the list for cfile in cfg_file_list: if os.path.isfile(cfile): logger.info("Delete: \'{0}\' ".format(cfile)) delete_file(cfile) else: logger.info( "Nothing to delete: \'{0}\' doesn't exist!".format(cfile)) # date string datestr = str(args_l3.year) + str('%02d' % args_l3.month) # get dirs list containing all subdirs of given path alldirs = os.listdir(args_l3.inpdir) if len(alldirs) > 0: # get dirs list matching the arguments getdirs = list() for ad in alldirs: # L3S: no platform, sensor fam. monthly averages if args_l3.prodtype.lower() == "l3s": if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad: getdirs.append(os.path.join(args_l3.inpdir, ad)) # L3C, L3U, L2B_SUM: sensor and platform in subdirectory_name else: if args_l3.local: if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad and platform in ad \ and 'Europe' in ad: getdirs.append(os.path.join(args_l3.inpdir, ad)) else: if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad and platform in ad \ and not 'Europe' in ad: getdirs.append(os.path.join(args_l3.inpdir, ad)) # check if getdirs list is empty if len(getdirs) == 0: logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".format( args_l3.inpdir, sensor, platform, datestr, args_l3.prodtype.upper())) else: # sort list getdirs.sort() # get last element from list, should be last job lastdir = getdirs[-1] # get ID number from the last job id_num = get_id(lastdir) # remove all subdirs matching the id number for gdir in getdirs: if id_num in gdir: logger.info("Delete: {0}".format(gdir)) delete_dir(gdir) # pattern # L3S: sensor fam. monthly averages if args_l3.prodtype.lower() == "l3s": pattern = datestr + '*' + sensor + '_ORAC*' + \ '*' + args_l3.prodtype.upper() + '*' + id_num else: pattern = datestr + '*' + sensor + '_ORAC*' + platform + \ '*' + args_l3.prodtype.upper() + '*' + id_num # remove all dirs matching pattern logger.info( "Delete: \'{0}\' -> L2toL3 was successful!".format(pattern)) else: logger.info("Nothing to delete in {0} for {1} {2} {3} {4}".format( args_l3.inpdir, sensor, platform, datestr, args_l3.prodtype))
def clear_l3(args_l3): """ Remove L3 results if archiving was successful. """ # find platform if args_l3.prodtype.lower() != "l3s": if args_l3.satellite: if args_l3.satellite.upper() == "TERRA" or \ args_l3.satellite.upper() == "MOD": platform = "TERRA" elif args_l3.satellite.upper() == "AQUA" or \ args_l3.satellite.upper() == "MYD": platform = "AQUA" elif args_l3.satellite.upper().startswith("NOAA"): platform = args_l3.satellite.upper() elif args_l3.satellite.upper().startswith("METOP"): platform = args_l3.satellite.upper() else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) else: logger.info("You chose prodtype={0}, " "so tell me which platform!". format(args_l3.prodtype)) sys.exit(0) # find sensor sensor = args_l3.instrument.upper() # delete corr. config file cfg_file_list = list() # -- monthly config files # config_proc_3_make_l3c_NOAA18_2008_01.file # config_proc_3_make_l3c_AQUA_2008_01.file if args_l3.prodtype.lower() == "l3c": cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \ '_' + platform.upper() + '_' + str(args_l3.year) + \ '_' + str('%02d' % args_l3.month) + '_' + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- monthly config files # config_proc_3_make_l3s_AVHRR_2008_01.file # config_proc_3_make_l3s_MODIS_2008_01.file elif args_l3.prodtype.lower() == "l3s": cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + \ '_' + sensor.upper() + '_' + str(args_l3.year) + \ '_' + str('%02d' % args_l3.month) + '_' + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- daily config files # config_proc_3_make_l3u_NOAA18_2008_01_01.file # config_proc_3_make_l3u_TERRA_2008_01_31.file elif args_l3.prodtype.lower() == "l3u": # calendar.monthrange # Returns weekday of first day of the month and # number of days in month, for the specified year and month. last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1] for iday in range(last_day_of_month): iday += 1 if args_l3.local: cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + "_Europe" + cfg_suffix else: cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # -- daily config files # config_proc_3_make_l2b_sum_NOAA18_2008_01_01.file # config_proc_3_make_l2b_sum_NOAA18_2008_01_31.file elif args_l3.prodtype.lower() == "l2b_sum": last_day_of_month = calendar.monthrange(args_l3.year, args_l3.month)[1] for iday in range(last_day_of_month): iday += 1 cfgbase = cfg_prefix + "3_make_" + args_l3.prodtype.lower() + '_' + \ platform.upper() + '_' + \ str(args_l3.year) + '_' + str('%02d' % args_l3.month) + '_' + \ str('%02d' % iday) + cfg_suffix cfgfile = os.path.join(args_l3.cfgdir, cfgbase) cfg_file_list.append(cfgfile) # now remove all config files in the list for cfile in cfg_file_list: if os.path.isfile(cfile): logger.info("Delete: \'{0}\' ".format(cfile)) delete_file(cfile) else: logger.info("Nothing to delete: \'{0}\' doesn't exist!". format(cfile)) # date string datestr = str(args_l3.year) + str('%02d' % args_l3.month) # get dirs list containing all subdirs of given path alldirs = os.listdir(args_l3.inpdir) if len(alldirs) > 0: # get dirs list matching the arguments getdirs = list() for ad in alldirs: # L3S: no platform, sensor fam. monthly averages if args_l3.prodtype.lower() == "l3s": if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad: getdirs.append( os.path.join(args_l3.inpdir, ad) ) # L3C, L3U, L2B_SUM: sensor and platform in subdirectory_name else: if args_l3.local: if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad and platform in ad \ and 'Europe' in ad: getdirs.append( os.path.join(args_l3.inpdir, ad) ) else: if datestr in ad and sensor in ad \ and args_l3.prodtype.upper() in ad \ and 'ORAC' in ad and platform in ad \ and not 'Europe' in ad: getdirs.append( os.path.join(args_l3.inpdir, ad) ) # check if getdirs list is empty if len(getdirs) == 0: logger.info("Nothing to delete in {0} for {1} {2} {3} {4}". format(args_l3.inpdir, sensor, platform, datestr, args_l3.prodtype.upper())) else: # sort list getdirs.sort() # get last element from list, should be last job lastdir = getdirs[-1] # get ID number from the last job id_num = get_id(lastdir) # remove all subdirs matching the id number for gdir in getdirs: if id_num in gdir: logger.info("Delete: {0}".format(gdir)) delete_dir(gdir) # pattern # L3S: sensor fam. monthly averages if args_l3.prodtype.lower() == "l3s": pattern = datestr + '*' + sensor + '_ORAC*' + \ '*' + args_l3.prodtype.upper() + '*' + id_num else: pattern = datestr + '*' + sensor + '_ORAC*' + platform + \ '*' + args_l3.prodtype.upper() + '*' + id_num # remove all dirs matching pattern logger.info("Delete: \'{0}\' -> L2toL3 was successful!". format(pattern)) else: logger.info("Nothing to delete in {0} for {1} {2} {3} {4}". format(args_l3.inpdir, sensor, platform, datestr, args_l3.prodtype))
p4 = subprocess.Popen(c4, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p4.communicate() logger.info("add2sqlite_l1c_info STDOUT:{0}".format(stdout)) logger.info("add2sqlite_l1c_info STDERR:{0}".format(stderr)) # -- end of loop over l1bfiles # -- check if output is equal input output1 = os.path.join(out, "*avhrr*h5") output2 = os.path.join(out, "*sunsatangles*h5") output3 = os.path.join(out, "*qualflags*h5") avhrr_cnt = len(glob.glob(output1)) sunsat_cnt = len(glob.glob(output2)) flags_cnt = len(glob.glob(output3)) if avhrr_cnt != l1bcnt and sunsat_cnt != l1bcnt and flags_cnt != l1bcnt: logger.info("PYGAC FAILED {0} time(s):".format(len(failed_l1b_orbits))) for failed in failed_l1b_orbits: logger.info("{0}".format(failed)) else: # -- delete dirs: where L1b files are extracted delete_dir(inp) logger.info("TarFile {0} finished: {1} L1b -> {2} L1c\n". format(tarbase, l1bcnt, avhrr_cnt)) # -- end of loop over tarfiles logger.info("{0} finished!".format(os.path.basename(__file__)))
def clear_l2(args_l2): """ Search for ID and then remove all directories, which belong to this ID and satellite and sensor and date. """ if args_l2.satellite.upper() == "TERRA": platform = "TERRA" elif args_l2.satellite.upper() == "AQUA": platform = "AQUA" elif args_l2.satellite.upper() == "MOD": platform = "TERRA" elif args_l2.satellite.upper() == "MYD": platform = "AQUA" elif args_l2.satellite.upper().startswith("NOAA"): platform = args_l2.satellite.lower() elif args_l2.satellite.upper().startswith("METOP"): platform = args_l2.satellite.lower() else: logger.info("WRONG SATELLITE NAME!") sys.exit(0) # delete corr. config file # config_proc_2_process_2008_06_NOAA15.file # config_proc_2_process_2008_06_AQUA.file if "output" in args_l2.inpdir: cfgbase = cfg_prefix + "2_process_" + str(args_l2.year) + \ '_' + str('%02d' % args_l2.month) + \ '_' + platform.upper() + cfg_suffix cfgfile = os.path.join(args_l2.cfgdir, cfgbase) if os.path.isfile(cfgfile): logger.info("Delete: \'{0}\' ".format(cfgfile)) delete_file(cfgfile) else: logger.info( "Nothing to delete: \'{0}\' doesn't exist!".format(cfgfile)) # date string datestr = str(args_l2.year) + str('%02d' % args_l2.month) # get dirs list containing all subdirs of given path alldirs = os.listdir(args_l2.inpdir) # if alldirs is not empty if len(alldirs) > 0: # get dirs list matching the arguments getdirs = list() for ad in alldirs: if datestr in ad \ and args_l2.instrument.upper() in ad \ and platform in ad and 'retrieval' in ad: getdirs.append(os.path.join(args_l2.inpdir, ad)) # check if getdirs list is empty if len(getdirs) == 0: logger.info("Nothing to delete in {0} for {1} {2} {3}".format( args_l2.inpdir, args_l2.instrument.upper(), platform, datestr)) else: # sort list getdirs.sort() # get last element from list, should be last job lastdir = getdirs[-1] # get ID number from the last job id_number = get_id(lastdir) # remove all subdirs matching the id number for gdir in getdirs: if id_number in gdir: logger.info("Delete: {0}".format(gdir)) delete_dir(gdir) # pattern pattern = datestr + '*' + args_l2.instrument.upper() + '_' + \ platform + '_retrieval_' + id_number # remove all dirs matching pattern logger.info( "Delete: \'{0}\' -> retrieval failed or was successful!". format(pattern)) else: logger.info("Nothing to delete in {0} for {1} {2} {3}".format( args_l2.inpdir, args_l2.instrument.upper(), platform, datestr))
# -- sort list tar_files_list.sort() # -- create ecp_tar_upload directory if not os.path.exists(ecp_tar_upload): os.makedirs(ecp_tar_upload) # -- move all tarfiles to final subfolder move_files(tar_files_list, ecp_tar_upload) # -- delete tar tmp directory delete_dir(make_tar_tmp_dir) # -- delete pygac results, i.e. l1c input delete_dir(inp) # -- delete l1b main input tarfile in ecp_tar_download db = AvhrrGacDatabase(dbfile=sql_gacdb_archive) results = db.get_tarfiles(start_date=args.start_date, end_date=args.end_date, sats=[args.satellite], include_blacklisted=False, strict=False) db.close() for res in results: # "NOAA7_1985_01.tar"
parser.add_argument('--test', action="store_true", help="Testing option, copy to ECFS disabled.") args = parser.parse_args() # Call function associated with the selected subcommand logger.info("{0} start for {1}".format(sys.argv[0], args)) save_dir = make_temp_dir(temp_dir=datadir, temp_sub=pygac_commit) if args.logs: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_logs") collect_log_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test) delete_dir(tmpdir=work_dir) if args.sqls: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_sqls") collect_sql_files(tmp_dir=work_dir, tar_dir=save_dir, test=args.test) delete_dir(tmpdir=work_dir) if args.invalids: work_dir = make_temp_dir(temp_dir=datadir, temp_sub="save_invalids") collect_invalid_orbits(tmp_dir=work_dir, tar_dir=save_dir, sy=args.syear, ey=args.eyear, test=args.test) delete_dir(tmpdir=work_dir)