class Scheduler:
"""The list of analytics"""
analytics = defaultdict(Analytic)
def __init__(self):
if len(sys.argv) < 4:
logger.error("Usage: {} IN.csv Func.py [0-1 Algorithm] [*.log]"
.format(sys.argv[0]))
sys.exit(1)
self.schedule = []
self.input_file = sys.argv[1]
self.load_function_file = sys.argv[2]
try:
self.load_module = importlib.import_module(self.load_function_file
.replace('.py', ''))
# for name in dir(self.load_module):
# print(getattr(self.load_module, name))
logger.info("Load function loaded")
except ImportError:
logger.error("Import error")
self.algorithm_type = sys.argv[3]
self.log_files = sys.argv[4:]
self.input_analytics = []
self.schedule = []
self.database = Database("data.p")
def main(self):
# Loads existing data
data = self.database.load_data()
if data is not None:
self.analytics = data
# Read new logs and merge them into each analytic
[
self.analytics[id].merge_run(run)
for id, run
in LogParser.parse_log_files(self.log_files)
]
# Assign the id of the dictionary to the analytic
for id in self.analytics.keys():
self.analytics[id].analytic_id = id
# Saves the loaded analytic data
self.database.save_data(self.analytics)
# Load CSV of jobs to get frequency
self.load_analytic_list(self.input_file)
# Call scheduler
if (self.algorithm_type == '0'):
logger.info("***Random Scheduling has been chosen")
self.scheduling_algorithm = RandomScheduling(7 * 24 * 60 * 60)
else:
logger.info("***Block Scheduling has been chosen")
self.scheduling_algorithm = BlockingAlgorithm(7 * 24 * 60 * 60)
# TODO: Map the analytics with something along the lines of:
# [self.analytics[i] for i in self.input_analytics] (filter by input)
self.schedule = self.scheduling_algorithm \
.build_schedule(self.analytics.values(),
self.load_module)
logger.info("Built schedule")
logger.debug("Schedule: {}".format(self.schedule))
# Create schedule CSV
self.schedule = self.scheduling_algorithm.schedule
self.write_schedule()
# TODO: Remove random save
print("Max: {}".format(max(self.scheduling_algorithm.current_load)))
np.save("schedule.npy", self.scheduling_algorithm.current_load)
def load_analytic_list(self, file_path):
""" Loads in the CSV of analytics.
Arguments:
file_path: the relative path of the .csv
list of analytics that will be scheduled
"""
with open(file_path) as csv_file:
reader = csv.reader(csv_file, delimiter=',')
for row in reader:
self.analytics[row[0]].run_frequency = row[2]
logger.info("Analytic ID:" + row[0] + " Frequency: " + row[1])
def write_schedule(self):
""" Writes the schedule to a CSV file
named schedule.csv.
"""
with open('schedule.csv', 'w') as csv_file:
schedule_writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
for analytic in self.schedule:
schedule_writer.writerow(list(analytic))
def save_data_test(self):
db = Database("test.txt");
Database.save_data();
