def __init__(self, toyid, arrival, duration):
self.reference_start_time = datetime.datetime(2014, 1, 1, 0, 0) # set when elf starts working on toy
self.id = toyid
self.arrival_minute = Hours.convert_to_minute(arrival)
self.duration = int(duration)
self.completed_minute = 0
self.work_window_days = 25
def evaluate(num_elves, toys_filename, results_filename):
toy_file = os.path.join(os.getcwd(), toys_filename)
myToys = read_toys(toy_file)
print ' -- All toys read. Starting to score submission. '
sub_file = os.path.join(os.getcwd(), results_filename)
hrs = Hours()
score_submission(sub_file, myToys, hrs, num_elves)
def __init__(self, toyid, arrival, duration):
self.reference_start_time = datetime.datetime(
2014, 1, 1, 0, 0) # set when elf starts working on toy
self.id = toyid
self.arrival_minute = Hours.convert_to_minute(arrival)
self.duration = int(duration)
self.completed_minute = 0
def play_elf(self, output, elf_object, toy_id, toy_duration, work_start_time=None):
"""
:param elf_id: Elf id
:param toy_id: Toy_Id
:param work_start_time: Minute since Jan, 2014
:return:
"""
hrs = Hours();
ref_time = datetime.datetime(2014, 1, 1, 0, 0);
if work_start_time == None:
work_start_time = elf_object.next_available_time;
elf_object.next_available_time, work_duration = \
self.assign_elf_to_toy(work_start_time, elf_object, toy_duration, hrs);
elf_object.update_elf(hrs, Toy(toy_id, '2014 01 01 01 01' ,toy_duration), work_start_time, work_duration); #Every toy has default arrival time(irrelevant)
tt = ref_time + datetime.timedelta(seconds=60*work_start_time)
time_string = " ".join([str(tt.year), str(tt.month), str(tt.day), str(tt.hour), str(tt.minute)])
if elf_object.id in output:
output[elf_object.id].append((toy_id, time_string, work_duration, work_start_time)); #work_start_time will be used to print in sorted order
else:
output[elf_object.id] = [(toy_id, time_string, work_duration, work_start_time)];
return tt.year;
def sort_toys():
import pandas as pd
toys = pd.read_csv("toys_rev2.csv")
toys["Start"] = toys["Arrival_time"].apply(lambda x: Hours.convert_to_minute(x))
toys["Finish"] = toys["Start"] + toys["Duration"]
toys = toys.sort("Finish")
toys.drop("Start", axis=1, inplace=True)
toys.drop("Finish", axis=1, inplace=True)
toys.to_csv("toys.csv", index=False)
def solution_firstAvailableElf(toy_file, soln_file, myelves):
""" Creates a simple solution where the next available elf is assigned a toy. Elves do not start
work outside of sanctioned hours.
:param toy_file: filename for toys file (input)
:param soln_file: filename for solution file (output)
:param myelves: list of elves in a priority queue ordered by next available time
:return:
"""
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
with open(toy_file, 'rb') as f:
toysfile = csv.reader(f)
toysfile.next() # header row
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
for row in toysfile:
current_toy = Toy(row[0], row[1], row[2])
# get next available elf
elf_available_time, current_elf = heapq.heappop(myelves)
work_start_time = elf_available_time
if current_toy.arrival_minute > elf_available_time:
work_start_time = current_toy.arrival_minute
# work_start_time cannot be before toy's arrival
if work_start_time < current_toy.arrival_minute:
print 'Work_start_time before arrival minute: {0}, {1}'.\
format(work_start_time, current_toy.arrival_minute)
exit(-1)
current_elf.next_available_time, work_duration = \
assign_elf_to_toy(work_start_time, current_elf, current_toy, hrs)
current_elf.update_elf(hrs, current_toy, work_start_time,
work_duration)
# put elf back in heap
heapq.heappush(myelves,
(current_elf.next_available_time, current_elf))
# write to file in correct format
tt = ref_time + datetime.timedelta(seconds=60 *
work_start_time)
time_string = " ".join([
str(tt.year),
str(tt.month),
str(tt.day),
str(tt.hour),
str(tt.minute)
])
wcsv.writerow([
current_toy.id, current_elf.id, time_string, work_duration
])
def solve(self, toys):
hrs = Hours()
next_toy = None
current_time = 540 # Santa's Workshop opens Jan 1, 2014 9:00 (= 540 minutes)
while True:
next_elf_time, elf = heapq.heappop(self.elves)
current_time = max(current_time, next_elf_time)
if (next_toy != None and next_toy.arrival_minute <= current_time):
self.pending_toys.insert((next_toy, next_toy.duration))
next_toy = None
if (next_toy == None):
for toy in toys:
if (toy.arrival_minute <= current_time):
self.pending_toys.insert((toy, toy.duration))
else:
next_toy = toy
break
if (len(self.pending_toys) == 0 and next_toy == None):
raise StopIteration()
if (len(self.pending_toys) == 0):
current_time = next_toy.arrival_minute
continue
remaining_time = hrs.get_remaining_sanctioned_time(current_time)
if (remaining_time == hrs.sanctioned_minutes_per_day
and elf.rating >= 4):
toy, duration = self.pending_toys.pop_le(sys.maxint)
else:
try:
toy, duration = self.pending_toys.pop_le(remaining_time)
except ValueError:
toy, duration = self.pending_toys.pop_le(sys.maxint)
work_duration = elf.asign_toy(current_time, toy, hrs)
heapq.heappush(self.elves, (elf.next_available_time, elf))
yield toy.id, elf.id, current_time, work_duration, elf.rating
def solve(self, toys):
hrs = Hours()
for toy in toys:
elf_available_time, elf = heapq.heappop(self.elves)
work_start_time = max(elf_available_time, toy.arrival_minute)
work_duration = elf.asign_toy(work_start_time, toy, hrs)
heapq.heappush(self.elves, (elf.next_available_time, elf))
yield toy.id, elf.id, work_start_time, work_duration, elf.rating
def write_jobs(self, out_file):
jobs = sorted(self.jobs, key=lambda x: x[2])
with open(out_file, 'w') as f:
f.write('ToyId,ElfId,StartTime,Duration\n')
for toy_id, elf_id, start_minute, duration in jobs:
start_time = Hours.convert_to_datetime(start_minute)
f.write('%d,%d,%d %d %d %d %d,%d\n' % (toy_id, elf_id,
start_time.year, start_time.month,
start_time.day, start_time.hour,
start_time.minute, duration))
def sort_toys():
import pandas as pd
toys = pd.read_csv("toys_rev2.csv")
toys["Start"] = toys["Arrival_time"].apply(
lambda x: Hours.convert_to_minute(x))
toys["Finish"] = toys["Start"] + toys["Duration"]
toys = toys.sort("Finish")
toys.drop("Start", axis=1, inplace=True)
toys.drop("Finish", axis=1, inplace=True)
toys.to_csv("toys.csv", index=False)
def solve(self, toys):
hrs = Hours()
next_toy = None
current_time = 540 # Santa's Workshop opens Jan 1, 2014 9:00 (= 540 minutes)
while True:
next_elf_time, elf = heapq.heappop(self.elves)
current_time = max(current_time, next_elf_time)
if (next_toy != None and next_toy.arrival_minute <= current_time):
self.pending_toys.insert((next_toy, next_toy.duration))
next_toy = None
if (next_toy == None):
for toy in toys:
if (toy.arrival_minute <= current_time):
self.pending_toys.insert((toy, toy.duration))
else:
next_toy = toy
break
if (len(self.pending_toys) == 0 and next_toy == None):
raise StopIteration()
if (len(self.pending_toys) == 0):
current_time = next_toy.arrival_minute
continue
remaining_time = hrs.get_remaining_sanctioned_time(current_time)
if (remaining_time == hrs.sanctioned_minutes_per_day and elf.rating >= 4):
toy, duration = self.pending_toys.pop_le(sys.maxint)
else:
try:
toy, duration = self.pending_toys.pop_le(remaining_time)
except ValueError:
toy, duration = self.pending_toys.pop_le(sys.maxint)
work_duration = elf.asign_toy(current_time, toy, hrs)
heapq.heappush(self.elves, (elf.next_available_time, elf))
yield toy.id, elf.id, current_time, work_duration, elf.rating
class SolutionWriter:
def __init__(self, soln_file):
self.w = open(soln_file, 'wt')
self.wcsv = csv.writer(self.w)
self.wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
self.ref_time = datetime.datetime(2014, 1, 1, 0, 0)
self.hours = Hours()
def write(self, toy, elf, work_start_time, work_duration):
time_string = self.hours.get_time_string(work_start_time)
self.wcsv.writerow([toy.id, elf.id, time_string, work_duration])
def close(self):
self.w.close()
def __init__(self, NUM_ELVES, toy_file):
#Output file
self.hrs = Hours()
self.NUM_ELVES = NUM_ELVES;
self.ref_time = datetime.datetime(2014, 1, 1, 0, 0);
self.toy_baskets = dict();
self.max_duration_log = 11.0; #Predetermined from analysis
self.elves = dict(); #elf_id -> (elf Object, list of allocated toys, optimimum toy workflow)
self.boost_productive_th = 24; #All jobs with hr duration less than this qualify as boosters
self.create_toy_baskets(toy_file)
#Every elf maintains list of toys which it has to optically work on
for i in xrange(1, NUM_ELVES+1):
elf = Elf(i);
self.elves[elf.id] = (elf, [], []);
def global_optimizer(sub_file, myToys, hrs, NUM_ELVES, output):
""" Score the submission file, performing constraint checking. Returns the time (in minutes) when
final present is complete.
:param sub_file: submission file name. Headers: ToyId, ElfId, Start_Time, Duration
:param myToys: toys dictionary
:param hrs: hours object
:return: time (in minutes) when final present is complete
"""
file_handler = open(output, "wb")
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
threshold_minute = hrs.convert_to_minute('2450 1 1 0 0')
basket_toys_above_th = []
basket_toys_below_th = []
#Unchanged
elf_availability = {}
#Maintains the earliest time when elves finish their last jobs
for i in range(1, NUM_ELVES + 1):
elf_availability[i] = [0, 0]
row_count = 0
with open(sub_file, 'rb') as f:
fcsv = csv.reader(f)
fcsv.next() # header
for row in fcsv:
row_count += 1
if row_count % 50000 == 0:
print 'Starting toy: {0}'.format(row_count)
current_toy = int(row[0])
current_elf = int(row[1])
start_minute = hrs.convert_to_minute(row[2])
duration = int(row[3])
if start_minute > threshold_minute:
basket_toys_above_th.append(current_toy)
else:
basket_toys_below_th.append((row[0], row[1], row[2], row[3]))
if elf_availability[current_elf][0] < start_minute:
elf_availability[current_elf][0] = start_minute
elf_availability[current_elf][1] = duration
myelves = []
for i in elf_availability.keys():
elf = Elf(i)
elf.rating = 0.25
elf.next_available_time = elf_availability[current_elf][0] + int(
math.ceil(elf_availability[current_elf][1] / elf.rating))
heapq.heappush(myelves, (elf.next_available_time, elf))
while len(basket_toys_above_th) > 0:
#Retrive the toy object
current_toy_duration = myToys[basket_toys_above_th[0]]
#Pick the next available elf and execute it
elf_available_time, current_elf = heapq.heappop(myelves)
work_start_time = elf_available_time
current_elf.next_available_time, work_duration = \
assign_elf_to_toy(work_start_time, current_elf, current_toy_duration, hrs)
current_elf.update_elf(hrs, Toy(basket_toys_above_th[0], '2014 01 01 01 01' ,current_toy_duration), \
work_start_time, work_duration)
# put elf back in heap
heapq.heappush(myelves, (current_elf.next_available_time, current_elf))
tt = ref_time + datetime.timedelta(seconds=60 * work_start_time)
time_string = " ".join([
str(tt.year),
str(tt.month),
str(tt.day),
str(tt.hour),
str(tt.minute)
])
#Add it to the basket below the threshold
basket_toys_below_th.append((basket_toys_above_th[0], current_elf.id,
time_string, work_duration))
del basket_toys_above_th[0]
basket_toys_below_th = sorted(
basket_toys_below_th,
key=lambda element: hrs.convert_to_minute(element[2]))
file_handler.write("ToyId,ElfId,StartTime,Duration\n")
for obj in basket_toys_below_th:
file_handler.write(
str(obj[0]) + "," + str(obj[1]) + "," + str(obj[2]) + "," +
str(obj[3]) + "\n")
file_handler.close()
def __init__(self, toy_id, arrival, duration):
self.id = int(toy_id)
self.arrival_minute = Hours.convert_to_minute(arrival)
self.duration = int(duration)
def matts_solution(bj, tj, gj, soln_file, myelves,low_thresh, high_thresh):
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
f = open(gj, 'rb')
f.readline()
print "loading toys"
available_toys = []
while True:
try:
line=f.readline()
available_toys.append(newtoy(line))
except:
break
f.close()
print "toys loaded"
tjfile = open(tj)
tjfile.readline()
bjfile = open(bj)
bjfile.readline()
print "assigning toys"
toys_ass = 0
next_print = list(range(0,1000000, 1000))
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
next_tiny = next_newtoy(tjfile)
next_big = next_newtoy(bjfile)
while len(available_toys)>0:
if toys_ass > next_print[0]:
next_print.pop(0)
print "assigned toys: ", toys_ass
for elf in myelves:
try:
next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(elf.next_available_time, elf, next_tiny, hrs)
if (unsanctioned > 0):
elf.next_available_time = hrs.tomorrow_minutes(elf.next_available_time)
except:
pass
# print 'elf ', elf.id
job_to_ass=None
if elf.rating >= high_thresh and next_big is not None:
#print 'long'
job_to_ass = next_big
next_big = next_newtoy(bjfile)
elif elf.rating>=low_thresh:
#print 'med'
if (hrs.day_minutes_remaining(elf.next_available_time)< \
(hrs.hours_per_day*60-10)):
job_to_ass = get_best_job(available_toys, elf.rating, elf.next_available_time, hrs, force=False)
else:
job_to_ass = get_best_job(available_toys, elf.rating, elf.next_available_time, hrs, force=True)
# print job_to_ass
if job_to_ass==None and next_tiny is not None:
#print 'short'
#print 'nexttiny'
#print next_tiny
if (elf.rating <= low_thresh):
job_to_ass = next_tiny
next_tiny = next_newtoy(tjfile)
else:
elf.next_available_time = hrs.tomorrow_minutes(elf.next_available_time)
#print job_to_ass
if job_to_ass is None:
job_to_ass = get_best_job(available_toys, elf.rating, elf.next_available_time, hrs, force=True)
work_start_time = elf.next_available_time
elf.next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(work_start_time, elf, job_to_ass, hrs)
elf.update_elf(hrs, job_to_ass, work_start_time, work_duration)
tt = ref_time + datetime.timedelta(seconds=60*work_start_time)
time_string = " ".join([str(tt.year), str(tt.month), str(tt.day), str(tt.hour), str(tt.minute)])
wcsv.writerow([job_to_ass.id, elf.id, time_string, work_duration])
#print job_to_ass.id, elf.id, time_string, work_duration, elf.rating
toys_ass+=1
if len(available_toys)<=0:
break
print "Medium jobs exhausted, toys assigned: ", toys_ass
if next_tiny is None:
print "Tiny jobs exhausted!"
if next_big is None:
print "Big jobs exhausted!"
while next_big is not None or next_tiny is not None:
if toys_ass > next_print[0]:
next_print.pop[0]
print "assigned toys: ", toys_ass
for elf in myelves:
#print 'elf ', elf.id
job_to_ass=None
if elf.rating >= high_thresh and next_big is not None:
#print 'long'
job_to_ass = next_big
next_big = next_newtoy(bjfile)
elif next_tiny is not None:
#print 'short'
#print 'nexttiny'
#print next_tiny
job_to_ass = next_tiny
next_tiny = next_newtoy(tjfile)
else:
#there are no tiny jobs left
job_to_ass = next_big
next_big = next_newtoy(bjfile)
#print job_to_ass
work_start_time = elf.next_available_time
elf.next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(work_start_time, elf, job_to_ass, hrs)
elf.update_elf(hrs, job_to_ass, work_start_time, work_duration)
tt = ref_time + datetime.timedelta(seconds=60*work_start_time)
time_string = " ".join([str(tt.year), str(tt.month), str(tt.day), str(tt.hour), str(tt.minute)])
wcsv.writerow([job_to_ass.id, elf.id, time_string, work_duration])
#print job_to_ass.id, elf.id, time_string, work_duration, elf.rating
toys_ass+=1
if next_big is None and next_tiny is None:
break
print "Total toys assigned: ", toys_ass
bjfile.close()
tjfile.close()
return
# Définition de quelques variables
working_date = datetime.date(2014, 1, 1)
NUM_ELVES = int(sys.argv[1])
NUM_TOYS = int(sys.argv[2])
PRODUCTIVITY_THRESHOLD = float(sys.argv[3])
MINUTES_LEFT_END_OF_DAY = int(sys.argv[4])
PRODUCTIVITY_THRESHOLD_STR = str(PRODUCTIVITY_THRESHOLD).replace(".", "_")
toy_file = os.path.join(os.getcwd(), '..', 'DATA', 'toys_rev2.csv')
soln_file = os.path.join(os.getcwd(), '..', 'DATA', 'my_solution_third_num_elves_%d_num_toys_%d_prod_%s_minutes_%d.csv' % (NUM_ELVES, NUM_TOYS, PRODUCTIVITY_THRESHOLD_STR, MINUTES_LEFT_END_OF_DAY))
# Objet hours
hrs = Hours()
# Création du pool d'elfes
myelfpool = ElfPool(NUM_ELVES)
# Création du pool de jouets
mytoypool = ToyPool()
mytoypool.add_file_content(toy_file, NUM_TOYS)
# Fichier dans lequel logger la solution
w = open(soln_file, 'wb')
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration', 'Original_Toy_Duration', 'Old_Productivity', 'New_Productivity', 'Sanctioned', 'Unsanctioned'])
c = 0
def solution_firstAvailableElf(toy_file, soln_file):
""" Creates a simple solution where the next available elf is assigned a toy. Elves do not start
work outside of sanctioned hours.
:param toy_file: filename for toys file (input)
:param soln_file: filename for solution file (output)
:param my_elves: list of elves in a priority queue ordered by next available time
:return:
"""
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
toy_list = []
my_elves = create_elves(NUM_ELVES)
#Build list of toys
with open(toy_file, 'rb') as f:
toysfile = csv.reader(f)
toysfile.next() # header row
for row in toysfile:
current_toy = Toy(row[0], row[1], row[2])
toy_list.append(current_toy)
big_toys = sorted([x for x in toy_list if x.duration > 600], key = lambda e: -e.duration)
small_toys = sorted([x for x in toy_list if x.duration <= 600], key = lambda e: e.duration)
toy_list = None
gc.collect()
sorted_toy_list = big_toys + small_toys
big_toys = small_toys = None
gc.collect()
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
while len(sorted_toy_list) > 0:
elf_available_time, current_elf = heapq.heappop(my_elves)
if current_elf.rating > 3:
current_toy = sorted_toy_list.pop()
else:
val = hrs.get_sanctioned_time_left(elf_available_time)
current_toy_idx = find_closest_idx(sorted_toy_list, current_elf.rating * (val*1.025))
current_toy = sorted_toy_list.pop(current_toy_idx)
if len(sorted_toy_list) % 1000 == 0:
print "[Elf %s @ %0.2f] => toy %s @ %s -- %0.2f%% done" % (current_elf.id, current_elf.rating, current_toy.id, current_toy.duration, len(sorted_toy_list) / 10000.0)
# get next available elf
work_start_time = elf_available_time
#!# work_start_time cannot be before toy's arrival
#!if work_start_time < current_toy.arrival_minute:
#! print 'Work_start_time before arrival minute: {0}, {1}'.\
#! format(work_start_time, current_toy.arrival_minute)
#! exit(-1)
current_elf.next_available_time, work_duration = assign_elf_to_toy(work_start_time, current_elf, current_toy, hrs)
current_elf.update_elf(hrs, current_toy, work_start_time, work_duration)
# put elf back in heap
heapq.heappush(my_elves, (current_elf.next_available_time, current_elf))
# write to file in correct format
tt = ref_time + datetime.timedelta(seconds=60*work_start_time)
time_string = " ".join([str(tt.year), str(tt.month), str(tt.day), str(tt.hour), str(tt.minute)])
wcsv.writerow([current_toy.id, current_elf.id, time_string, work_duration])
def solution(toy_file, soln_file, num_elves, TARGET):
""" Creates a simple solution where the next available elf is assigned a toy. Elves do not start
work outside of sanctioned hours.
:param toy_file: filename for toys file (input)
:param soln_file: filename for solution file (output)
:param myelves: list of elves in a priority queue ordered by next available time
:return:
"""
hrs = Hours()
toy_loader = ToyLoader(toy_file)
solution_writer = SolutionWriter(soln_file)
busy_elves_heap = BusyElvesHeap(num_elves)
elves_ready = ElvesReady()
scheduler = Scheduler(TARGET, num_elves)
start = time.time()
current_time = 497340
print("Getting initial toys")
print('time taken = {0}, current_time = {1}'.format(time.time() - start, hrs.get_time_string(current_time)))
new_toy_orders = toy_loader.get_toys_up_to_minute(current_time)
print("Loading initial toys into backlog")
print('time taken = {0}, current_time = {1}'.format(time.time() - start, hrs.get_time_string(current_time)))
toy_backlog = ToyBacklogV2(new_toy_orders)
print("Finished initializing toys into backlog")
print('time taken = {0}, current_time = {1}'.format(time.time() - start, hrs.get_time_string(current_time)))
toys_finished = 0
toys_left_at_end = []
time_of_last_toy_assigned = current_time
while not (toy_loader.done() and toy_backlog.done() and len(toys_left_at_end) == 0):
# step 1 process newly arrived toys and fresh elves
new_elves = busy_elves_heap.get_elves_for_min(current_time)
elves_ready.add_elves(new_elves)
if (current_time % 120 == 60):
print('time taken = {0}, current_time = {1}'.format(time.time() - start, hrs.get_time_string(current_time)))
print('easy_toys:{0},\t constant_toys:{1},\t variable_toys:{2},\t hardest_toys:{3},\t toys_left_at_end:{4}'.format(
toy_backlog.easy_toy_list.size, len(toy_backlog.constant_rating_list),
len(toy_backlog.variable_toy_list), len(toy_backlog.hardest_toy_list), len(toys_left_at_end)))
print('elves ready:{0}, high-perf-elves:{1}'.format(len(elves_ready.training_elf_list),
len(elves_ready.high_performance_elf_list)))
print('toys finished = {0}'.format(toys_finished))
if (len(elves_ready.training_elf_list) + len(elves_ready.high_performance_elf_list)) == 0:
current_time = hrs.next_sanctioned_minute(current_time)
continue
if (toy_loader.done() and current_time - time_of_last_toy_assigned > 2880 and len(elves_ready.training_elf_list) == num_elves and len(toys_left_at_end) == 0):
print("starting cleanup")
for toy in toy_backlog.easy_toy_list.all_toys():
toys_left_at_end.append(toy)
toy_backlog.easy_toy_list.clear()
for toy in toy_backlog.constant_rating_list:
toys_left_at_end.append(toy)
toy_backlog.constant_rating_list = []
while len(toy_backlog.variable_toy_list) > 0:
toys_left_at_end.append(toy_backlog.pop_variable_toy())
while len(toy_backlog.hardest_toy_list) > 0:
toys_left_at_end.append(toy_backlog.pop_hardest_toy())
if (toy_loader.done() and len(toys_left_at_end) > 0):
# clean up last toys
toys_finished += scheduler.clean_up(toys_left_at_end, elves_ready, busy_elves_heap, current_time, solution_writer)
else:
# step 2 pair off as many elves and toys as possible
toys_newly_finished = scheduler.schedule(toy_backlog, elves_ready, busy_elves_heap, current_time,
solution_writer, toy_loader)
toys_finished += toys_newly_finished
if (toys_newly_finished > 0):
time_of_last_toy_assigned = current_time
current_time = hrs.next_sanctioned_minute(current_time)
toy_loader.close()
solution_writer.close()
print '\nSuccess!'
print ' Score = {0}'.format(score)
# ======================================================================= #
# === MAIN === #
if __name__ == '__main__':
""" Evaluation script for Helping Santa's Helpers, the 2014 Kaggle Holiday Optimization Competition. """
import sys
if len(sys.argv) == 2:
toy_file_path = os.path.join(os.getcwd(), 'data/toys_rev2.csv')
out_file_path = sys.argv[1]
num_toys = 10000000
num_elves = 900
else:
toy_file_path = sys.argv[1]
out_file_path = sys.argv[2]
num_toys = int(sys.argv[3])
num_elves = int(sys.argv[4])
start = time.time()
myToys = read_toys(toy_file_path, num_toys)
print ' -- All toys read. Starting to score submission. '
hrs = Hours()
score_submission(out_file_path, myToys, hrs, num_toys, num_elves)
print 'total time = {0}'.format(time.time() - start)
def __init__(self, soln_file):
self.w = open(soln_file, 'wt')
self.wcsv = csv.writer(self.w)
self.wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
self.ref_time = datetime.datetime(2014, 1, 1, 0, 0)
self.hours = Hours()
def optimize(self, NUM_ELVES, boosters, big_jobs):
"""
:param elf_object: Elf Object
:param boosters: List of (Toy_Id, Duration)
:param big_jobs: List of (Toy_Id, Duration)
:return:
"""
output = dict();
file_handler = open( ("data/finalSubmission1" + ".csv"), "wb" );
hrs = Hours();
last_job_completed_year = 0;
min_desired_rating = 0.32;
total_no_of_toys = len(boosters) + len(big_jobs);
#Sort the big jobs in descending order of duration
big_jobs.sort(key=operator.itemgetter(1),reverse=True);
#Sort the boosters in ascending order of duration
boosters.sort(key=operator.itemgetter(1));
rating_change_index1 = 0;
rating_change_index2 = 0;
for i in range(0, len(big_jobs)):
if int(big_jobs[i][1]/60.0) <= 300 and int(big_jobs[i][1]/60.0) > 200:
rating_change_index1 = i;
if int(big_jobs[i][1]/60.0) <= 200:
rating_change_index2 = i;
no_completed_toys = 0;
big_job_counter = 0;
elves = Queue.Queue();
for i in xrange(1, NUM_ELVES+1):
elf = Elf(i);
elves.put(elf);
while no_completed_toys < total_no_of_toys:
if no_completed_toys % 10000 == 0:
print(" Algorithm : Boost_Play , Boosters : " + str(len(boosters)) + " Big Jobs : " + str(len(big_jobs)));
#print("**Seeking a rating : " + str(min_desired_rating));
if len(big_jobs) == 0:
break;
#Pick an elf
elf_object = elves.get();
#Iterate through the boosters and increase the productivity
no_of_toys_played = 0;
while ( round(elf_object.rating,3) + 0.002 < round(min_desired_rating,3) ):
if len(boosters) == 0 or no_of_toys_played > 20:
break;
#Generate a random no
random_toy = randint(0, len(boosters)-1);
#Play the one with no unsanctioned time
sanctioned, unsanctioned = self.breakDownWork(elf_object.next_available_time, elf_object, boosters[random_toy][1], hrs);
if unsanctioned == 0:
completion_yr = self.play_elf(output, elf_object, boosters[random_toy][0], boosters[random_toy][1]);
#Remove the toy played
del boosters[random_toy];
else:
#Play the first toy as it has the least amount of unsanctioned time
#Set the next available time to next day, 9:00 am
work_start_time = hrs.day_start + int(hrs.minutes_in_24h * math.ceil(elf_object.next_available_time / float(hrs.minutes_in_24h)));
completion_yr = self.play_elf(output, elf_object, boosters[0][0], boosters[0][1], work_start_time);
del boosters[0];
if completion_yr > last_job_completed_year:
last_job_completed_year = completion_yr;
no_completed_toys += 1;
no_of_toys_played += 1;
if len(big_jobs) > 0:
if big_job_counter >= rating_change_index1:
min_desired_rating = 0.31;
if big_job_counter >= rating_change_index2:
min_desired_rating = 0.29;
#Play the first toy in the queue on the following day
work_start_time = hrs.day_start + int(hrs.minutes_in_24h * math.ceil(elf_object.next_available_time / float(hrs.minutes_in_24h)));
completion_yr = self.play_elf(output, elf_object, big_jobs[0][0], big_jobs[0][1], work_start_time);
#completion_yr = play_elf(output, elf_object, big_jobs[0][0], big_jobs[0][1]);
if completion_yr > last_job_completed_year:
last_job_completed_year = completion_yr;
#Delete this toy
del big_jobs[0];
no_completed_toys += 1;
big_job_counter += 1;
#Put the elf back in
elves.put(elf_object);
#Play naively
myelves = []
while not elves.empty():
current_elf = elves.get();
heapq.heappush(myelves, (current_elf.next_available_time, current_elf));
if len(boosters) == 0:
remaining_jobs = big_jobs;
else:
remaining_jobs = boosters;
while len(remaining_jobs) > 0:
print(" Algorithm : Naive , Boosters : " + str(len(boosters)) + " Big Jobs : " + str(len(remaining_jobs)));
elf_available_time, current_elf = heapq.heappop(myelves);
completion_yr = self.play_elf(output, current_elf, remaining_jobs[0][0], remaining_jobs[0][1]);
if completion_yr > last_job_completed_year:
last_job_completed_year = completion_yr;
# put elf back in heap
heapq.heappush(myelves, (current_elf.next_available_time, current_elf));
no_completed_toys += 1;
print(current_elf.id);
del remaining_jobs[0];
for elf_id in output:
#Sort the output based on work_start_time
output[elf_id].sort(key=operator.itemgetter(3));
#Flush the output of this elf to file
for toy in output[elf_id]:
file_handler.write( str(toy[0]) + "," + str(elf_id) + "," \
+ str(toy[1]) + "," + str(toy[2]) + "\n");
file_handler.close();
return last_job_completed_year;
def solve(toy_file, num_toys, num_elves, out_file):
all_toys = read_toys(toy_file, num_toys)
with open(TOY_OBJ_FILE, 'w') as f:
cPickle.dump(all_toys, f, cPickle.HIGHEST_PROTOCOL)
hrs = Hours()
"""
all_toys_idx = range(0, len(all_toys))
del all_toys
elves_init = Elves(num_elves)
elves_init_file = './tmp/elves_init.obj'
with open(elves_init_file, 'w') as f:
cPickle.dump(elves_init, f, cPickle.HIGHEST_PROTOCOL)
del elves_init
elves_best = None
for i in range(0, 1):
for s in glob.glob('./tmp/*.pkl'):
os.remove(s)
elves_file = assign_elves(all_toys_idx, elves_init_file, hrs)
with open(elves_file, 'r') as f:
elves = cPickle.load(f)
if elves_best is None or elves_best.cost() > elves.cost():
elves_best = elves
random.shuffle(all_toys_idx)
"""
toy_groups = {}
for toy_idx, toy in enumerate(all_toys):
if toy.arrival_minute not in toy_groups:
toy_groups[toy.arrival_minute] = []
toy_groups[toy.arrival_minute].append(toy_idx)
del all_toys
toy_groups = [(arrival_minute, toys)
for arrival_minute, toys in toy_groups.iteritems()]
elves_best = None
for i in range(0, min(len(toy_groups), 1)):
random.shuffle(toy_groups)
elves_init = Elves(num_elves)
elves_init_file = './tmp/elves_init.obj'
with open(elves_init_file, 'w') as f:
cPickle.dump(elves_init, f, cPickle.HIGHEST_PROTOCOL)
del elves_init
elves_group_file = elves_init_file
for grp_idx, (arrival_minute, toys) in enumerate(toy_groups):
print 'Working on group %d/%d' % (grp_idx, len(toy_groups))
# find the "best" way to assign this group (toys)
elves_group_best_file = None
elves_group_best_score = None
for j in range(0, min(len(toys), 1)):
random.shuffle(toys)
elves_new_file = assign_elves(toys, elves_group_file, hrs)
with open(elves_new_file, 'r') as f:
elves_new = cPickle.load(f)
if elves_group_best_score is None or elves_group_best_score > elves_new.cost(
):
elves_group_best_score = elves_new.cost()
elves_group_best_file = elves_new_file
elves_group_file = elves_group_best_file
with open(elves_group_file, 'r') as f:
elves = cPickle.load(f)
if elves_best is None or elves_best.cost() > elves.cost():
elves_best = elves
elves_best.write_jobs(out_file)
print 'Cost:', elves_best.cost()
def matts_solution(bj, tj, gj, soln_file, myelves, low_thresh, high_thresh):
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
f = open(gj, 'rb')
f.readline()
print "loading toys"
available_toys = []
while True:
try:
line = f.readline()
available_toys.append(newtoy(line))
except:
break
f.close()
print "toys loaded"
tjfile = open(tj)
tjfile.readline()
bjfile = open(bj)
bjfile.readline()
print "assigning toys"
toys_ass = 0
next_print = list(range(0, 1000000, 1000))
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
next_tiny = next_newtoy(tjfile)
next_big = next_newtoy(bjfile)
while len(available_toys) > 0:
if toys_ass > next_print[0]:
next_print.pop(0)
print "assigned toys: ", toys_ass
for elf in myelves:
try:
next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(elf.next_available_time, elf, next_tiny, hrs)
if (unsanctioned > 0):
elf.next_available_time = hrs.tomorrow_minutes(
elf.next_available_time)
except:
pass
# print 'elf ', elf.id
job_to_ass = None
if elf.rating >= high_thresh and next_big is not None:
#print 'long'
job_to_ass = next_big
next_big = next_newtoy(bjfile)
elif elf.rating >= low_thresh:
#print 'med'
if (hrs.day_minutes_remaining(elf.next_available_time)< \
(hrs.hours_per_day*60-10)):
job_to_ass = get_best_job(available_toys,
elf.rating,
elf.next_available_time,
hrs,
force=False)
else:
job_to_ass = get_best_job(available_toys,
elf.rating,
elf.next_available_time,
hrs,
force=True)
# print job_to_ass
if job_to_ass == None and next_tiny is not None:
#print 'short'
#print 'nexttiny'
#print next_tiny
if (elf.rating <= low_thresh):
job_to_ass = next_tiny
next_tiny = next_newtoy(tjfile)
else:
elf.next_available_time = hrs.tomorrow_minutes(
elf.next_available_time)
#print job_to_ass
if job_to_ass is None:
job_to_ass = get_best_job(available_toys,
elf.rating,
elf.next_available_time,
hrs,
force=True)
work_start_time = elf.next_available_time
elf.next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(work_start_time, elf, job_to_ass, hrs)
elf.update_elf(hrs, job_to_ass, work_start_time, work_duration)
tt = ref_time + datetime.timedelta(seconds=60 *
work_start_time)
time_string = " ".join([
str(tt.year),
str(tt.month),
str(tt.day),
str(tt.hour),
str(tt.minute)
])
wcsv.writerow(
[job_to_ass.id, elf.id, time_string, work_duration])
#print job_to_ass.id, elf.id, time_string, work_duration, elf.rating
toys_ass += 1
if len(available_toys) <= 0:
break
print "Medium jobs exhausted, toys assigned: ", toys_ass
if next_tiny is None:
print "Tiny jobs exhausted!"
if next_big is None:
print "Big jobs exhausted!"
while next_big is not None or next_tiny is not None:
if toys_ass > next_print[0]:
next_print.pop[0]
print "assigned toys: ", toys_ass
for elf in myelves:
#print 'elf ', elf.id
job_to_ass = None
if elf.rating >= high_thresh and next_big is not None:
#print 'long'
job_to_ass = next_big
next_big = next_newtoy(bjfile)
elif next_tiny is not None:
#print 'short'
#print 'nexttiny'
#print next_tiny
job_to_ass = next_tiny
next_tiny = next_newtoy(tjfile)
else:
#there are no tiny jobs left
job_to_ass = next_big
next_big = next_newtoy(bjfile)
#print job_to_ass
work_start_time = elf.next_available_time
elf.next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(work_start_time, elf, job_to_ass, hrs)
elf.update_elf(hrs, job_to_ass, work_start_time, work_duration)
tt = ref_time + datetime.timedelta(seconds=60 *
work_start_time)
time_string = " ".join([
str(tt.year),
str(tt.month),
str(tt.day),
str(tt.hour),
str(tt.minute)
])
wcsv.writerow(
[job_to_ass.id, elf.id, time_string, work_duration])
#print job_to_ass.id, elf.id, time_string, work_duration, elf.rating
toys_ass += 1
if next_big is None and next_tiny is None:
break
print "Total toys assigned: ", toys_ass
bjfile.close()
tjfile.close()
return
def solution_noovertime(toy_file, soln_file, myelves):
""" Creates a simple solution where the next available elf is assigned a toy. Elves do not start
work outside of sanctioned hours.
:param toy_file: filename for toys file (input)
:param soln_file: filename for solution file (output)
:param myelves: list of elves in a priority queue ordered by next available time
:return:
"""
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
with open(toy_file, 'rb') as f:
toysfile = csv.reader(f)
toysfile.next() # header row
available_toys = OrderedDict([])
for row in toysfile:
available_toys[row[0]] = Toy(row[0], row[1], row[2])
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
toys_assigned = [
9999,
]
start_times = np.array([a.next_available_time for a in myelves])
while len(available_toys.keys()) > 0:
print 'available_toys ', len(available_toys.keys())
tot_ass = 0
to_pop = []
for toycount, currtoy in available_toys.items():
best_times2 = np.where(currtoy.arrival_minute >= start_times,
np.nan, start_times)
best_times = np.argsort(best_times2)
for elfpos in best_times:
try:
work_start_time = int(best_times2[elfpos])
except ValueError:
break
current_elf = myelves[elfpos]
# work_start_time cannot be before toy's arrival
#if work_start_time < current_toy.arrival_minute:
# print 'Work_start_time before arrival minute: {0}, {1}'.\
# format(work_start_time, current_toy.arrival_minute)
# exit(-1)
next_available_time, work_duration, unsanctioned = \
assign_elf_to_toy(work_start_time, current_elf, currtoy, hrs)
if (unsanctioned == 0 or toys_assigned[-1] == 0
or current_elf.rating > 3.5):
current_elf.next_available_time = next_available_time
current_elf.update_elf(hrs, currtoy, work_start_time,
work_duration)
start_times[elfpos] = current_elf.next_available_time
# write to file in correct format
tt = ref_time + datetime.timedelta(seconds=60 *
work_start_time)
time_string = " ".join([
str(tt.year),
str(tt.month),
str(tt.day),
str(tt.hour),
str(tt.minute)
])
wcsv.writerow([
currtoy.id, current_elf.id, time_string,
work_duration
])
tot_ass += 1
del available_toys[toycount]
break
toys_assigned.append(tot_ass)
print 'productivities'
print[a.rating for a in myelves]
print toys_assigned
return
def solution_firstAvailableElf(toy_file, soln_file, myelves):
""" Creates a simple solution where the next available elf is assigned a toy. Elves do not start
work outside of sanctioned hours.
:param toy_file: filename for toys file (input)
:param soln_file: filename for solution file (output)
:param myelves: list of elves in a priority queue ordered by next available time
:return:
"""
hrs = Hours()
ref_time = datetime.datetime(2014, 1, 1, 0, 0)
row_count = 0
toy_list = []
loop_count = 0
#Build list of toys
with open(toy_file, 'rb') as f:
toysfile = csv.reader(f)
toysfile.next() # header row
for row in toysfile:
current_toy = Toy(row[0], row[1], row[2])
toy_list.append(current_toy)
sorted_toy_list = sorted(toy_list,key = lambda e: e.duration)
toy_list = None
gc.collect()
print 'All is sorted:'
with open(soln_file, 'wb') as w:
wcsv = csv.writer(w)
wcsv.writerow(['ToyId', 'ElfId', 'StartTime', 'Duration'])
while len(sorted_toy_list) > 0:
elf_available_time, current_elf = heapq.heappop(myelves)
if current_elf.rating > 3.9:
current_toy = sorted_toy_list.pop()
print 'Elf {0} eff {1} toy {2} duration {3} BIG TOY---------------------------------'.format(current_elf.id, current_elf.rating, current_toy.id, current_toy.duration)
loop_count = loop_count + 1
else:
val = hrs.get_sanctioned_time_left(elf_available_time)
current_toy_idx = find_closest_idx(sorted_toy_list, current_elf.rating * (val*1.15))
current_toy = sorted_toy_list.pop(current_toy_idx)
if len(sorted_toy_list) % 10 == 0:
print 'Elf {0} eff {1} toy {2} duration {3} SMALL TOY {4}> {5}'.format(current_elf.id, current_elf.rating, current_toy.id, current_toy.duration, len(sorted_toy_list), len(sorted_toy_list))
# get next available elf
work_start_time = elf_available_time
#!# work_start_time cannot be before toy's arrival
#!if work_start_time < current_toy.arrival_minute:
#! print 'Work_start_time before arrival minute: {0}, {1}'.\
#! format(work_start_time, current_toy.arrival_minute)
#! exit(-1)
current_elf.next_available_time, work_duration = \
assign_elf_to_toy(work_start_time, current_elf, current_toy, hrs)
current_elf.update_elf(hrs, current_toy, work_start_time, work_duration)
# put elf back in heap
heapq.heappush(myelves, (current_elf.next_available_time, current_elf))
# write to file in correct format
tt = ref_time + datetime.timedelta(seconds=60*work_start_time)
time_string = " ".join([str(tt.year), str(tt.month), str(tt.day), str(tt.hour), str(tt.minute)])
wcsv.writerow([current_toy.id, current_elf.id, time_string, work_duration])
print 'loop_count = {0}'.format(loop_count)
def optimize(elf_object, boosters, big_jobs):
"""
:param elf_object: Elf Object
:param boosters: List of (Toy_Id, Duration)
:param big_jobs: List of (Toy_Id, Duration)
:return:
"""
output = [];
file_handler = open( ("data/submission_" + str(elf_object.id) + ".csv"), "wb" );
hrs = Hours();
last_job_completed_year = 0;
alpha = 0.02; #BumpupParameter
min_desired_rating = (0.25 + alpha); #Base Min Rate + Bumpup Parameter
total_no_of_toys = len(boosters) + len(big_jobs);
#Sort the big jobs in descending order of duration
big_jobs.sort(key=operator.itemgetter(1),reverse=True);
#Sort the boosters in ascending order of duration
boosters.sort(key=operator.itemgetter(1));
no_completed_toys = 0;
big_job_counter = 0;
ratings = generateOptimumRatings(big_jobs, boosters, hrs, alpha);
spill_over = 0.0;
while no_completed_toys < total_no_of_toys:
#print("Optimizing : Elf " + str(elf_object.id) + " Rating : " + str(elf_object.rating) + " Completed : " + str(no_completed_toys) + " Boosters : " + str(len(boosters)) + " Big Jobs : " + str(len(big_jobs)) + " Last Completed Year : " +str(last_job_completed_year));
if len(big_jobs) > 0:
if big_job_counter > 0 and len(boosters) > 0:
spill_over = (elf_object.rating - ratings[big_job_counter-1]);
#print("*Spill Over : " + str(spill_over));
#Play the first toy in the queue on the following day
work_start_time = hrs.day_start + int(hrs.minutes_in_24h * math.ceil(elf_object.next_available_time / float(hrs.minutes_in_24h)));
completion_yr = play_elf(output, elf_object, big_jobs[0][0], big_jobs[0][1], work_start_time);
#completion_yr = play_elf(output, elf_object, big_jobs[0][0], big_jobs[0][1]);
if completion_yr > last_job_completed_year:
last_job_completed_year = completion_yr;
#Delete this toy
del big_jobs[0];
#Set the desired rating for the next job
if big_job_counter < len(ratings):
min_desired_rating = ratings[big_job_counter];
if (min_desired_rating - spill_over) >= 0.25:
min_desired_rating -= spill_over;
#Rating cannot drop below
if min_desired_rating < (0.25 + alpha):
min_desired_rating = (0.25 + alpha);
ratings[big_job_counter] = min_desired_rating;
big_job_counter += 1;
no_completed_toys += 1;
#print("**Seeking a rating : " + str(min_desired_rating));
#Iterate through the boosters and increase the productivity
while ( round(elf_object.rating,3) + 0.002 < round(min_desired_rating,3) or len(big_jobs) == 0) and no_completed_toys < total_no_of_toys:
#print("***Optimizing : Elf " + str(elf_object.id) + " Rating : " + str(elf_object.rating) + " Completed : " + str(no_completed_toys) + " Boosters : " + str(len(boosters)) + " Big Jobs : " + str(len(big_jobs)))
if len(boosters) == 0:
break;
#Generate a random no
random_toy = randint(0, len(boosters)-1);
#Play the one with no unsanctioned time
sanctioned, unsanctioned = breakDownWork(elf_object.next_available_time, elf_object, boosters[random_toy][1], hrs);
if unsanctioned == 0:
completion_yr = play_elf(output, elf_object, boosters[random_toy][0], boosters[random_toy][1]);
#Remove the toy played
del boosters[random_toy];
else:
#Play the first toy as it has the least amount of unsanctioned time
#Set the next available time to next day, 9:00 am
work_start_time = hrs.day_start + int(hrs.minutes_in_24h * math.ceil(elf_object.next_available_time / float(hrs.minutes_in_24h)));
completion_yr = play_elf(output, elf_object, boosters[0][0], boosters[0][1], work_start_time);
del boosters[0];
if completion_yr > last_job_completed_year:
last_job_completed_year = completion_yr;
no_completed_toys += 1;
#Sort the output based on work_start_time
output.sort(key=operator.itemgetter(3));
#Flush the output of this elf to file
for toy in output:
file_handler.write( str(toy[0]) + "," + str(elf_object.id) + "," \
+ str(toy[1]) + "," + str(toy[2]) + "\n");
file_handler.close();
return last_job_completed_year;
