def start_uploading(settings):
"""
start uploading process based on "upload_scheduler.txt"
:return: None
"""
today = datetime.date.today()
with open("upload_scheduler.txt", "r") as f:
lines = [l for l in f if l.strip()]
for i, lin in enumerate(lines):
path, time, status = lin.split()
program = os.path.basename(os.path.dirname(path))
target_date = datetime.datetime.strptime(time, "%Y-%m-%d").date()
if status == "pending" and target_date == today:
if program not in args.observing_programs:
Message.addMessage(
"skipping uploading program: {} ({})".format(
program, os.path.basename(path)),
dump="header")
continue
Message.clearMessage("program")
Message.clearMessage("session")
Message.clearMessage("download")
Message.clearMessage("log")
upload = settings[program].get("upload", "no").lower()
if upload == "ivs":
code = os.path.basename(os.path.dirname(path))
Transfer.upload(path)
emails = Helper.read_emails(settings[program],
args.fallback_email)
SendMail.writeMail_upload(code, emails)
elif upload == "no":
pass
elif upload == "gow":
code = os.path.basename(os.path.dirname(path))
Transfer.upload_GOW_ftp(path)
emails = Helper.read_emails(settings[program],
args.fallback_email)
SendMail.writeMail_upload(code, emails)
else:
emails = upload.split(",")
with open(os.path.join(path, "selected", "email.txt"),
"r") as f:
body = f.read()
SendMail.writeMail(os.path.join(path, "selected"), emails,
body)
lines[i] = lin.replace("pending", "uploaded")
with open("upload_scheduler.txt", "w") as f:
for lout in lines:
path, time, status = lout.split()
target_date = datetime.datetime.strptime(time, "%Y-%m-%d").date()
# do not list sessions older than 1 year in upload_scheduler.txt
if target_date + datetime.timedelta(days=365) > today:
f.write(lout)
def start(master,
path_scheduler,
code,
code_regex,
select_best,
emails,
delta_days,
delta_days_upload,
statistic_field,
output_path="./Schedules/",
upload=False,
pre_fun=None,
post_fun=None):
"""
start auto processing for one observing program
:param master: list of dictionaries with session specific fields read from session master
:param path_scheduler: path to VieSched++ executable
:param code: observing program code
:param code_regex: regular expression to match session name
:param select_best: function to select best schedule from statistics dataframe
:param emails: list of email addresses
:param statistic_field: fields to be stored in statistics file
:param delta_days: time offset in days from where schedule should be generated
:param delta_days_upload: time offset in days when schedule should be updated
:param output_path: prefix for output path
:param upload: flag if session needs to be uploaded
:param pre_fun: list of functions executed prior to scheduling
:param post_fun: list of functions executed after to scheduling
:return: None
"""
Message.clearMessage("program")
pattern = re.compile(code_regex)
Message.addMessage("=== {} observing program ===".format(code),
dump="program")
Message.addMessage("contact:", dump="program")
for email in emails:
Message.addMessage(" {}".format(email), dump="program")
Message.addMessage("schedule master contained {} sessions".format(
len(master)),
dump="program")
today = datetime.date.today()
sessions = []
if delta_days == "next":
for s in master:
if s["date"].date() < today:
continue
if pattern.match(s["name"]):
sessions.append(s)
break
upload = False
else:
target_day = today + datetime.timedelta(days=delta_days)
Message.addMessage("date offset: {} days".format(delta_days),
dump="program")
Message.addMessage(
"target start time: {:%B %d, %Y}".format(target_day),
dump="program")
sessions = [
s for s in master if pattern.match(s["name"])
if s["date"].date() == target_day
]
Message.addMessage("{} session(s) will be processed".format(len(sessions)),
dump="program")
# get list of templates
templates = []
template_path = os.path.join("Templates", code)
for file in os.listdir(template_path):
if file.endswith(".xml"):
templates.append(os.path.join(template_path, file))
# loop over all sessions
for session in sessions:
Message.clearMessage("session")
Message.clearMessage("log")
Message.addMessage("##### {} #####".format(session["code"].upper()))
Message.addMessage(
"{name} ({code}) start {date} duration {duration}h stations {stations}"
.format(**session))
xmls = adjust_template(output_path, session, templates, pre_fun)
xml_dir = os.path.dirname(xmls[0])
df_list = []
flag_VLBA = any(
["VLBA" in sta or "PIETOWN" in sta for sta in session["stations"]])
flag_DSS = any([sta.startswith("DSS") for sta in session["stations"]])
if flag_VLBA or flag_DSS:
post_fun.append(post_scheduling_functions._vex_in_sked_format)
if flag_VLBA:
post_fun.append(post_scheduling_functions._vlba_vex_adjustments)
# loop over all templates
for xml in xmls:
Message.addMessage(" processing file: {}".format(xml))
xml = os.path.abspath(xml)
p = subprocess.run([path_scheduler, xml],
cwd=xml_dir,
capture_output=True,
text=True)
log = p.stdout
if log:
Message.addMessage(log, dump="log")
errlog = p.stderr
if errlog:
Message.addMessage(errlog, dump="log")
p.check_returncode()
# rename statistics.csv and simulation_summary file to avoid name clashes
statistic_in = os.path.join(xml_dir, "statistics.csv")
statistic_out = "statistics_{}.csv".format(
os.path.basename(os.path.splitext(xml)[0]))
statistic_out = os.path.join(xml_dir, statistic_out)
if os.path.exists(statistic_out):
os.remove(statistic_out)
os.rename(statistic_in, statistic_out)
simulation_summary_in = os.path.join(xml_dir,
"simulation_summary.txt")
simulation_summary_out = "simulation_summary_{}.txt".format(
os.path.basename(os.path.splitext(xml)[0]))
simulation_summary_out = os.path.join(xml_dir,
simulation_summary_out)
if os.path.exists(simulation_summary_out):
os.remove(simulation_summary_out)
os.rename(simulation_summary_in, simulation_summary_out)
# read statistics.csv file
df = pd.read_csv(statistic_out, index_col=0)
df_list.append(df)
# concatenate all statistics.csv files
stats = pd.concat(df_list)
stats = stats.drop_duplicates()
stats.sort_index(inplace=True)
# find best schedule based on statistics
best_idx = select_best(stats, template_path=template_path)
Message.addMessage("best version: v{:03d}".format(best_idx))
if upload:
Message.addMessage(
"this session will be uploaded on: {:%B %d, %Y}".format(
today +
datetime.timedelta(days=delta_days - delta_days_upload)))
if delta_days - delta_days_upload < 1:
Message.addMessage("[WARNING]: upload date already passed!")
else:
Message.addMessage("this session will NOT be uploaded!")
summary_file = os.path.join(os.path.dirname(xml_dir), "summary.txt")
summary_df = Helper.addStatistics(stats, best_idx, statistic_field,
session["code"].upper(),
summary_file)
# copy best schedule to selected folder
version_pattern = "_v{:03d}".format(best_idx)
bestFiles = glob.glob(
os.path.join(xml_dir, "*{}*".format(version_pattern)))
xml_dir_selected = os.path.join(xml_dir, "selected")
if os.path.exists(xml_dir_selected):
shutil.rmtree(xml_dir_selected)
os.makedirs(xml_dir_selected)
for f in bestFiles:
fname = os.path.basename(f).replace(version_pattern, "")
destination = os.path.join(xml_dir_selected, fname)
shutil.copy(f, destination)
stats.to_csv(os.path.join(xml_dir_selected, "merged_statistics.csv"))
if upload:
Helper.update_uploadScheduler(xml_dir_selected,
delta_days - delta_days_upload,
upload)
try:
skdFile = os.path.join(xml_dir_selected,
"{}.skd".format(session["code"].lower()))
skd = skd_parser.skdParser(skdFile)
skd.parse()
Plotting.summary(summary_df, xml_dir_selected)
Plotting.polar_plots(skd, xml_dir_selected, "duration")
Plotting.polar_plots(skd, xml_dir_selected, "start_time")
Plotting.close_all()
except:
Message.addMessage("#### ERROR ####")
Message.addMessage(traceback.format_exc())
for post_f in post_fun:
post_f(path=xml_dir_selected,
ds=stats.loc[best_idx, :],
session=session,
program_code=code)
SendMail.writeMail(xml_dir_selected, emails)