def get_non_conflicting_groups(passes, delay=timedelta(seconds=0)):
"""Get the different non-conflicting solutions in a group of conflicting
passes.
"""
# Uses graphs and maximal clique finding with the Bron-Kerbosch algorithm.
order = len(passes)
if order == 1:
return [passes]
graph = Graph(order)
for i, overpass in enumerate(sorted(passes, key=lambda x: x.risetime)):
for j in range(i + 1, order):
if not overpass.overlaps(passes[j], delay):
graph.add_edge(i, j)
groups = []
for res in graph.bron_kerbosch(set(), set(graph.vertices), set()):
grp = []
for vertex in res:
grp.append(passes[vertex])
groups.append(sorted(grp))
return groups
def get_non_conflicting_groups(passes, delay=timedelta(seconds=0)):
"""Get the different non-conflicting solutions in a group of conflicting
passes.
"""
# Uses graphs and maximal clique finding with the Bron-Kerbosch algorithm.
order = len(passes)
if order == 1:
return [passes]
graph = Graph(order)
for i, overpass in enumerate(sorted(passes, key=lambda x: x.risetime)):
for j in range(i + 1, order):
if not overpass.overlaps(passes[j], delay):
graph.add_edge(i, j)
groups = []
for res in graph.bron_kerbosch(set(), set(graph.vertices), set()):
grp = []
for vertex in res:
grp.append(passes[vertex])
groups.append(sorted(grp))
return groups
def get_best_sched(overpasses, area_of_interest, delay, avoid_list=None):
"""Get the best schedule based on *area_of_interest*.
"""
avoid_list = avoid_list or []
passes = sorted(overpasses, key=lambda x: x.risetime)
grs = conflicting_passes(passes, delay)
logger.debug("conflicting %s", str(grs))
ncgrs = [get_non_conflicting_groups(gr, delay) for gr in grs]
logger.debug("non conflicting %s", str(ncgrs))
n_vertices = len(passes)
graph = Graph(n_vertices=n_vertices + 2)
def add_arc(graph, p1, p2, hook=None):
logger.debug("Adding arc between " + str(p1) +
" and " + str(p2) + "...")
if p1 in avoid_list or p2 in avoid_list:
w = 0
logger.debug("...0 because in the avoid_list!")
else:
w = combine(p1, p2, area_of_interest)
logger.debug("...with weight " + str(w))
# with open("/tmp/schedule.gv", "a") as fp_:
# fp_.write(' "' + str(p1) + '" -> "' + str(p2) +
# '" [ label = "' + str(w) + '" ];\n')
graph.add_arc(passes.index(p1) + 1,
passes.index(p2) + 1, w)
if hook is not None:
hook()
prev = set()
for ncgr in ncgrs:
for pr in prev:
foll = set(gr[0] for gr in ncgr)
for f in foll:
add_arc(graph, pr, f)
prev = set(sorted(gr, key=lambda x: x.falltime)[-1] for gr in ncgr)
for gr in ncgr:
if len(gr) > 1:
for p1, p2 in zip(gr[:-1], gr[1:]):
add_arc(graph, p1, p2)
for pr in prev:
graph.add_arc(passes.index(pr) + 1, n_vertices + 1)
for first in ncgrs[0][0]:
graph.add_arc(0, passes.index(first) + 1)
dist, path = graph.dag_longest_path(0, n_vertices + 1)
del dist
return [passes[idx - 1] for idx in path[1:-1]], (graph, passes)
def get_best_sched(overpasses, area_of_interest, scores, delay, avoid_list=None):
"""Get the best schedule based on *area_of_interest*.
"""
avoid_list = avoid_list or []
print avoid_list
raw_input()
passes = sorted(overpasses, key=lambda x: x.risetime)
grs = conflicting_passes(passes, delay)
logger.debug("conflicting %s", str(grs))
ncgrs = [get_non_conflicting_groups(gr, delay) for gr in grs]
logger.debug("non conflicting %s", str(ncgrs))
n_vertices = len(passes)
graph = Graph(n_vertices=n_vertices + 2)
def add_arc(graph, p1, p2, hook=None):
logger.debug("Adding arc between " + str(p1) +
" and " + str(p2) + "...")
if p1 in avoid_list or p2 in avoid_list:
w = 0
logger.debug("...0 because in the avoid_list!")
else:
w = combine(p1, p2, area_of_interest, scores)
logger.debug("...with weight " + str(w))
with open("/tmp/schedule.gv", "a") as fp_:
fp_.write(' "' + str(p1) + '" -> "' + str(p2) +
'" [ label = "' + str(w) + '" ];\n')
graph.add_arc(passes.index(p1) + 1,
passes.index(p2) + 1, w)
if hook is not None:
hook()
prev = set()
for ncgr in ncgrs:
for pr in prev:
foll = set(gr[0] for gr in ncgr)
for f in foll:
add_arc(graph, pr, f)
prev = set(sorted(gr, key=lambda x: x.falltime)[-1] for gr in ncgr)
for gr in ncgr:
if len(gr) > 1:
for p1, p2 in zip(gr[:-1], gr[1:]):
add_arc(graph, p1, p2)
for pr in prev:
graph.add_arc(passes.index(pr) + 1, n_vertices + 1)
for first in ncgrs[0][0]:
graph.add_arc(0, passes.index(first) + 1)
dist, path = graph.dag_longest_path(0, n_vertices + 1)
del dist
return [passes[idx - 1] for idx in path[1:-1]], (graph, passes)
def run():
"""The schedule command."""
import argparse
global logger
parser = argparse.ArgumentParser()
# general arguments
parser.add_argument("-c",
"--config",
default=None,
help="configuration file to use")
parser.add_argument("-t", "--tle", default=None, help="tle file to use")
parser.add_argument("-l",
"--log",
default=None,
help="File to log to (defaults to stdout)")
parser.add_argument("-m",
"--mail",
nargs="*",
default=None,
help="mail address(es) to send error messages to.")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="print debug messages too")
# argument group: coordinates and times
group_postim = parser.add_argument_group(
title="start-parameter",
description="(or set values in the configuration file)")
group_postim.add_argument("--lat",
type=float,
help="Latitude, degrees north")
group_postim.add_argument("--lon",
type=float,
help="Longitude, degrees east")
group_postim.add_argument("--alt", type=float, help="Altitude, km")
group_postim.add_argument("-f",
"--forward",
type=float,
help="time ahead to compute the schedule")
group_postim.add_argument("-s",
"--start-time",
type=parse_datetime,
help="start time of the schedule to compute")
group_postim.add_argument("-d",
"--delay",
default=60,
type=float,
help="delay (in seconds) needed between two " +
"consecutive passes (60 seconds by default)")
# argument group: special behaviour
group_spec = parser.add_argument_group(
title="special",
description="(additional parameter changing behaviour)")
group_spec.add_argument("-a",
"--avoid",
help="xml request file with passes to avoid")
group_spec.add_argument("--no-aqua-terra-dump",
action="store_false",
help="do not consider Aqua/Terra-dumps")
group_spec.add_argument("--multiproc",
action="store_true",
help="use multiple parallel processes")
# argument group: output-related
group_outp = parser.add_argument_group(
title="output",
description="(file pattern are taken from configuration file)")
group_outp.add_argument("-o",
"--output-dir",
default=None,
help="where to put generated files")
group_outp.add_argument(
"-u",
"--output-url",
default=None,
help="URL where to put generated schedule file(s)" +
", otherwise use output-dir")
group_outp.add_argument("-x",
"--xml",
action="store_true",
help="generate an xml request file (schedule)")
group_outp.add_argument("-r",
"--report",
action="store_true",
help="generate an xml report file (schedule)")
group_outp.add_argument("--scisys",
action="store_true",
help="generate a SCISYS schedule file")
group_outp.add_argument("-p",
"--plot",
action="store_true",
help="generate plot images")
group_outp.add_argument("-g",
"--graph",
action="store_true",
help="save graph info")
group_outp.add_argument("--meos",
action="store_true",
help="generate a MEOS schedule file")
group_outp.add_argument("--metno-xml",
action="store_true",
help="generate a METNO xml pass data file")
opts = parser.parse_args()
if opts.config:
# read_config() returns:
# [(coords, station, area, scores)], forward, start, {pattern}
# station_list, forward, start, pattern = utils.read_config(opts.config)
scheduler = utils.read_config(opts.config)
# TODO make config file compulsory
if (not opts.config) and (not (opts.lon or opts.lat or opts.alt)):
parser.error("Coordinates must be provided in the absence of "
"configuration file.")
if not (opts.xml or opts.scisys or opts.report or opts.metno_xml):
parser.error("No output specified, use '--scisys' or '-x/--xml'")
if opts.output_dir is None:
opts.output_dir = os.path.curdir
if "dir_output" not in scheduler.patterns:
pattern["dir_output"] = opts.output_dir
if opts.log:
previous = os.path.exists(opts.log)
handler = logging.handlers.RotatingFileHandler(opts.log, backupCount=7)
if previous:
handler.doRollover()
else:
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter(
"[%(levelname)s: %(asctime)s :"
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S'))
if opts.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
if opts.mail:
mhandler = logging.handlers.SMTPHandler(
"localhost", "*****@*****.**", opts.mail,
"Scheduler")
mhandler.setLevel(logging.WARNING)
logging.getLogger('').addHandler(mhandler)
logger = logging.getLogger("trollsched")
tle_file = opts.tle
if opts.start_time:
start_time = opts.start_time
else:
start_time = datetime.utcnow()
allpasses = {}
graph = {}
logger.debug("start: %s forward: %s" %
(scheduler.start, scheduler.forward))
pattern_args = {
"output_dir": opts.output_dir,
"date": start_time.strftime("%Y%m%d"),
"time": start_time.strftime("%H%M%S")
}
dir_output = build_filename("dir_output", scheduler.patterns, pattern_args)
if not os.path.exists(dir_output):
logger.debug("Create output dir " + dir_output)
os.makedirs(dir_output)
if len(scheduler.stations) > 1:
opts.comb = True
import pickle
ph = open(os.path.join(dir_output, "opts.pkl"), "wb")
pickle.dump(opts, ph)
ph.close()
else:
opts.comb = False
scheduler.opts = opts
# single- or multi-processing?
if not opts.multiproc or len(scheduler.stations) == 1:
# sequential processing all stations' single schedule.
for station in scheduler.stations:
graph[station.id], allpasses[station.id] = station.single_station(
scheduler, start_time, tle_file)
else:
# processing the stations' single schedules with multiprocessing.
process_single = {}
statlst_ordered = []
# first round through the stations, forking sub-processes to do the
# "single station calculations" in parallel.
# the pickling of passes and graphs is done inside single_station().
for station in scheduler.stations:
statlst_ordered.append(station.id)
from multiprocessing import Process
process_single[station.id] = Process(target=station.single_station,
args=(scheduler, start_time,
tle_file))
process_single[station.id].start()
# second round through the stations, collecting the sub-processes and
# their results.
for station_id in statlst_ordered:
process_single[station_id].join()
pattern_args["station"] = station_id
# load graph for station
graph[station_id] = Graph()
graph[station_id].load(
build_filename("file_graph", scheduler.patterns, pattern_args)
+ ".npz")
# load pickled passes for station
ph = open(
os.path.join(dir_output, "allpasses.%s.pkl" % station_id),
"rb")
allpasses[station_id] = pickle.load(ph)
ph.close()
if opts.comb:
combined_stations(scheduler, start_time, graph, allpasses)
def main():
import logging
import logging.handlers
import os
import pickle
from trollsched.schedule import parse_datetime
from trollsched.schedule import combined_stations, build_filename
try:
from trollsched.schedule import read_config
import argparse
logger = logging.getLogger("trollsched")
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config",
default=None,
help="configuration file to use")
parser.add_argument("-s",
"--start-time",
type=parse_datetime,
help="start time of the schedule to compute")
parser.add_argument("-o",
"--output-dir",
default=None,
help="where to put generated files")
parser.add_argument("-x",
"--xml",
action="store_true",
help="generate an xml request file (schedule)")
parser.add_argument("-r",
"--report",
action="store_true",
help="generate an xml report file (schedule)")
parser.add_argument("--scisys",
action="store_true",
help="generate a SCISYS schedule file")
parser.add_argument("-p",
"--plot",
action="store_true",
help="generate plot images")
parser.add_argument("-g",
"--graph",
action="store_true",
help="save graph info")
opts = parser.parse_args()
if opts.config is None:
parser.error("Configuration file required.")
if opts.start_time:
start_time = opts.start_time
else:
start_time = datetime.utcnow()
# [coords, station, area, scores], forward, start, pattern
station_list, forward, start, pattern = read_config(opts.config)
pattern_args = {
"output_dir": opts.output_dir,
"date": start_time.strftime("%Y%m%d"),
"time": start_time.strftime("%H%M%S")
}
dir_output = build_filename("dir_output", pattern, pattern_args)
if not os.path.exists(dir_output):
print(dir_output, "does not exist!")
sys.exit(1)
ph = open(os.path.join(dir_output, "opts.pkl"), "rb")
opts = pickle.load(ph)
ph.close()
graph = {}
allpasses = {}
for coords, station, area, scores in station_list:
pattern_args["station"] = station
graph[station] = Graph()
graph[station].load(
build_filename("file_graph", pattern, pattern_args) + ".npz")
# print "---",station,"---"
# print_matrix(graph[station].adj_matrix, ly=5)
# print_matrix(graph[station].weight_matrix, ly=5, lx=-1)
# allpasses[station] = get_passes_from_xml_file(os.path.join(opts.report, "acquisition-schedule-report." + station + ".xml"))
# print len(allpasses[station]),allpasses[station]
# for v in graph[station].neighbours(1):
# print v, " : ", allpasses[station][v].risetime, "->", graph[station].weight(1, v)
ph = open(
os.path.join(
build_filename("dir_output", pattern, pattern_args),
"allpasses.%s.pkl" % station), "rb")
allpasses[station] = pickle.load(ph)
ph.close()
from trollsched.schedule import conflicting_passes
totpas = []
for s, sp in allpasses.items():
print("len(sp)", s, len(sp))
totpas.extend(list(sp))
passes = sorted(totpas, key=lambda x: x.risetime)
cpg = conflicting_passes(passes, timedelta(seconds=600))
print("ALLPASSES", len(allpasses)) # ,allpasses
print("PASSES", len(passes)) # ,passes
print("CONFLGRPS", len(cpg)) # ,cpg
print("MAX", max([len(g) for g in cpg]))
combined_stations(opts, pattern, station_list, graph, allpasses,
start_time, start, forward)
except:
logger.exception("Something wrong happened!")
raise
def add_graphs(graphs, passes, delay=timedelta(seconds=0)):
"""Add all graphs to one combined graph. """
statlst = graphs.keys()
def count_neq_passes(pl):
"""Counts how many satellite passes in a list are really distinct (satellite/epoch)."""
if len(pl):
r = []
s = 1
for q in pl[1:]:
if pl[0] != q:
r.append(q)
s += count_neq_passes(r)
return s
else:
return 0
for s, g in graphs.items():
logger.debug("station: %s, order: %d", s, g.order)
# Graphs and allpasses are hashmaps of sets, or similar, but we need
# lists of lists, forthat they are copied.
grl = []
pl = []
for s in statlst:
grl.append(graphs[s])
pl.append(sorted(passes[s], key=lambda x: x.risetime))
# Rough estimate for the size of the combined passes' graph.
n_vertices = 1
for g in grl:
n_vertices += g.order
n_vertices *= len(statlst) * 2
newgraph = Graph(n_vertices=n_vertices)
logger.debug("newgraph order: %d", newgraph.order)
# This value signals the end, when no more passes from any antenna are available.
stopper = tuple((None, None) for s in range(len(statlst)))
# The new passes list, it'll be filled with tuples, each with of one pass per antenna.
# It's initialized with the first passes.
#
# TODO: ideally something like next line, but this doesn't work faultless
# if one or more stations have multiple "first passes":
# newpasses = [tuple((pl[s][p - 1], None) for s in range(len(statlst)) for p in grl[s].neighbours(0))]
#
# TODO: not "just the first vertix" with the line:
# parlist = [newpasses[0]]
#
newpasses = [
tuple((pl[s][grl[s].neighbours(0)[0] - 1], None)
for s in range(len(statlst)))
]
parlist = [newpasses[0]]
while len(parlist):
newparlist = []
for parnode in parlist:
if parnode == stopper:
# All antennas reached the end of passes list in this path of
# possibilities.
# stopper == ((None,None) * stations)
#
# If this happens for all elements of parlist, newparlist will
# stay empty and (at the bottom of this loop) replace parlist,
# which as an empty list will cause the surrounding while-loop
# to end.
continue
collected_newnodes = collect_nodes(0, parnode, grl, newgraph,
newpasses, pl, delay)
for newnode_list in collected_newnodes:
newnode = tuple(newnode_list)
if newnode not in newpasses:
newpasses.append(newnode)
if newnode not in newparlist:
newparlist.append(newnode)
# Collecting the weights from each stations weight-matrix ...
# (could be more compact if it weren't for the None-values)
wl = []
for s, p, n in zip(range(len(statlst)), parnode, newnode):
try:
if n[0] is None:
wl.append(0)
else:
wl.append(n[1] or grl[s].weight(
pl[s].index(p[0]) + 1, pl[s].index(n[0]) + 1))
except:
logger.error(
"Collecting weights: stat %d - parnode %s %s - newnode %s %s",
s,
parnode,
p,
newnode,
n,
exc_info=1)
raise
# Apply vertix-count to the sum of collected weights.
# vertix-count: number of vertices with reference to same
# satellite pass, it can result to 0, 1, 2.
w = sum(wl) / 2**(
(2 * len(parnode)) - count_neq_passes(parnode) -
count_neq_passes(newnode))
# TODO: if the starting point isn't "just the first vertix",
# the comparison must be changed
if parnode == newpasses[0]:
# "virtual" weight for the starting point.
newgraph.add_arc(0, newpasses.index(parnode) + 1, w)
newgraph.add_arc(
newpasses.index(parnode) + 1,
newpasses.index(newnode) + 1, w)
parlist = newparlist
logger.debug("newpasses length: %d", len(newpasses))
return statlst, newgraph, newpasses
def add_graphs(graphs, passes, delay=timedelta(seconds=0)):
"""Add all graphs to one combined graph. """
statlst = graphs.keys()
def count_neq_passes(pl):
"""Counts how many satellite passes in a list are really distinct (satellite/epoch)."""
if len(pl):
r = []
s = 1
for q in pl[1:]:
if pl[0] != q:
r.append(q)
s += count_neq_passes(r)
return s
else:
return 0
for s, g in graphs.items():
logger.debug("station: %s, order: %d", s, g.order)
# Graphs and allpasses are hashmaps of sets, or similar, but we need
# lists of lists, forthat they are copied.
grl = []
pl = []
for s in statlst:
grl.append(graphs[s])
pl.append(sorted(passes[s], key=lambda x: x.risetime))
# Rough estimate for the size of the combined passes' graph.
n_vertices = 1
for g in grl:
n_vertices += g.order
n_vertices *= len(statlst) * 2
newgraph = Graph(n_vertices=n_vertices)
logger.debug("newgraph order: %d", newgraph.order)
# This value signals the end, when no more passes from any antenna are available.
stopper = tuple((None, None) for s in range(len(statlst)))
# The new passes list, it'll be filled with tuples, each with of one pass per antenna.
# It's initialized with the first passes.
#
# TODO: ideally something like next line, but this doesn't work faultless
# if one or more stations have multiple "first passes":
# newpasses = [tuple((pl[s][p - 1], None) for s in range(len(statlst)) for p in grl[s].neighbours(0))]
#
# TODO: not "just the first vertix" with the line:
# parlist = [newpasses[0]]
#
newpasses = [tuple((pl[s][grl[s].neighbours(0)[0] - 1], None) for s in range(len(statlst)))]
parlist = [newpasses[0]]
while len(parlist):
newparlist = []
for parnode in parlist:
if parnode == stopper:
# All antennas reached the end of passes list in this path of
# possibilities.
# stopper == ((None,None) * stations)
#
# If this happens for all elements of parlist, newparlist will
# stay empty and (at the bottom of this loop) replace parlist,
# which as an empty list will cause the surrounding while-loop
# to end.
continue
collected_newnodes = collect_nodes(0, parnode, grl, newgraph, newpasses, pl, delay)
for newnode_list in collected_newnodes:
newnode = tuple(newnode_list)
if newnode not in newpasses:
newpasses.append(newnode)
if newnode not in newparlist:
newparlist.append(newnode)
# Collecting the weights from each stations weight-matrix ...
# (could be more compact if it weren't for the None-values)
wl = []
for s, p, n in zip(range(len(statlst)), parnode, newnode):
try:
if n[0] is None:
wl.append(0)
else:
wl.append(n[1] or grl[s].weight(pl[s].index(p[0]) + 1, pl[s].index(n[0]) + 1))
except:
logger.error(
"Collecting weights: stat %d - parnode %s %s - newnode %s %s", s, parnode, p, newnode, n, exc_info=1)
raise
# Apply vertix-count to the sum of collected weights.
# vertix-count: number of vertices with reference to same
# satellite pass, it can result to 0, 1, 2.
w = sum(wl) / 2 ** ((2 * len(parnode)) - count_neq_passes(parnode) - count_neq_passes(newnode))
# TODO: if the starting point isn't "just the first vertix",
# the comparison must be changed
if parnode == newpasses[0]:
# "virtual" weight for the starting point.
newgraph.add_arc(0, newpasses.index(parnode) + 1, w)
newgraph.add_arc(newpasses.index(parnode) + 1, newpasses.index(newnode) + 1, w)
parlist = newparlist
logger.debug("newpasses length: %d", len(newpasses))
return statlst, newgraph, newpasses
