def create_sI_run_filelist(glob_file='/path/to/core50_root_dir/*/*/*',
dest_bp='/insert/your/path/sI_inc/',
dest_cum_bp='/insert/your/path/sI_cum/',
all_sess=range(11),
all_objs=range(50),
cumulative=True,
train_sess=[0, 1, 3, 4, 5, 7, 8, 10],
test_sess=[2, 6, 9],
batch_order=[x for x in range(8)]):
""" Given some parameters, it creates the batches filelist and
eventually the cumulative ones. """
# Adjusting the batch order
app = [-1] * 8
for i, batch_idx in enumerate(batch_order):
app[i] = train_sess[batch_idx]
train_sess = app
# Loading all the file lists divided by session
filelist_all_sess = load_filelist_per_sess(glob_file)
# Create training batches filelists
for i, sess in enumerate(train_sess):
# create first batch
create_filelist(dest_bp + "train_batch_" + str(i).zfill(2),
filelist_all_sess, [sess], all_objs)
# Creating test filelist
create_filelist(dest_bp + "test", filelist_all_sess, test_sess, all_objs)
# Creating the cumulative version if requested
if cumulative:
all_lines = []
for batch_id in range(len(train_sess)):
with open(
dest_bp + 'train_batch_' + str(batch_id).zfill(2) +
'_filelist.txt', 'r') as f:
all_lines += f.readlines()
with open(
dest_cum_bp + 'train_batch_' + str(batch_id).zfill(2) +
'_filelist.txt', 'w') as f:
for line in all_lines:
f.write(line)
shutil.copy(dest_bp + "test_filelist.txt", dest_cum_bp)
def create_sII_run_filelist(
glob_file='/path/to/core50_root_dir/*/*/*',
dest_bp='/insert/your/path/sII_inc/',
dest_cum_bp='/insert/your/path/sII_cum/',
all_sess=range(11),
all_objs=range(50),
change_labels=True,
cumulative=True,
train_sess=[0, 1, 3, 4, 5, 7, 8, 10],
test_sess=[2, 6, 9],
batch_order=[x for x in range(9)]):
""" Given some parameters, it creates the batches filelist and
eventually the cumulative ones. """
# Loading the filelists devided by sessions
filelist_all_sess = load_filelist_per_sess(glob_file)
objs = [[]] * 9
objs_test = []
# Here the creations of the batches (which class to choose for
# each of them) is **independent** by the external seed. This means that
# the units are static throughout the runs while only their order
# can change. This is the same as for the NI scenario where the content of
# the batches is fixed.
# first batch
objs[0] = [i * 5 for i in range(10)]
objs_test += objs[0][:]
# inc batches
for batch_id in range(1, 9):
if batch_id % 2 == 0:
objs[batch_id] = [i * 5 + int(round(batch_id / 2.0)) for i in
range(5, 10)]
else:
objs[batch_id] = [i * 5 + int(round(batch_id / 2.0)) for i in
range(5)]
# the first batch stay the same regardless of the order of inc batches
app = [[]] * 9
app[0] = objs[0][:]
for i, batch_idx in enumerate(batch_order[1:]):
app[i+1] = objs[batch_idx][:]
objs_test += objs[batch_idx][:]
objs = app
# Since the freezeweights strategy needs the encountered classes to be in
# adjacent neurons the labels are changed for this task (i.e. o1 has not
# necessary the label 0).
if change_labels:
label_map = {label: i for i, label in enumerate(objs_test)}
print "Label map:", label_map
print "obj train:", len(objs), sorted(objs)
for batch_id in range(9):
create_filelist(dest_bp + "train_batch_" + str(batch_id).zfill(2),
filelist_all_sess, train_sess, objs[batch_id],
label_map=label_map)
print "obj test:", len(objs_test), sorted(objs_test)
create_filelist(dest_bp + "test", filelist_all_sess,
test_sess, objs_test, label_map=label_map)
# create the cumulative version
if cumulative:
all_lines = []
for batch_id in range(len(batch_order)):
with open(dest_bp + 'train_batch_' +
str(batch_id).zfill(2) + '_filelist.txt',
'r') as f:
all_lines += f.readlines()
with open(dest_cum_bp + 'train_batch_' +
str(batch_id).zfill(2) + '_filelist.txt',
'w') as f:
for line in all_lines:
f.write(line)
shutil.copy(dest_bp + "test_filelist.txt", dest_cum_bp)
def create_sIII_run_filelist(
glob_file='data/core50_128x128/*/*/*',
dest_bp='/insert/your/path/sIII_inc/',
dest_cum_bp='/insert/your/path/sIII_cum/',
all_sess=range(11),
all_objs=range(50),
cumulative=True,
batch_order=[x for x in range(79)]):
""" Given some parameters, it creates the batches filelist and
eventually the cumulative ones. """
# Here the creations of the units (which obj in which sess)
# is **independent** by the external seed. This means that the units are
# static throughout the runs while only their order can change. This is
# the same as for the NI and NC scenarios where the batches are fixed.
rnd_state = np.random.get_state()
np.random.seed(0)
filelist_all_sess = load_filelist_per_sess(glob_file)
train_sess = [0, 1, 3, 4, 5, 7, 8, 10]
test_sess = [2, 6, 9]
# Selecting the five objs for batch
first_ten_objs = [i * 5 for i in range(10)]
objs_after_first_b = []
for id in all_objs:
if id not in first_ten_objs:
objs_after_first_b.append(id)
np.random.shuffle(objs_after_first_b)
objs_per_batch = np.reshape(objs_after_first_b, (8, 5))
objs_per_batch = [row for row in objs_per_batch]
# Creating units for classes after first batch
units = []
for sess in train_sess:
for objs_id in objs_per_batch:
units.append((sess, objs_id))
# Creating for the first 10 classes split in two groups
for sess in train_sess[1:]:
units.append((sess, first_ten_objs[:5]))
units.append((sess, first_ten_objs[5:]))
# Suffling units
np.random.shuffle(units)
print("Number of incremental units: ", len(units))
print("----- Unit details (sess, objs) ------")
for unit in units:
print(unit)
# Creating first batch
create_filelist(dest_bp + "train_batch_00", filelist_all_sess, [0],
first_ten_objs)
# Creating test
create_filelist(dest_bp + "test", filelist_all_sess, test_sess,
all_objs)
# Reordering incremental units based on batch order
new_units = [[]] * 78
for i, id in enumerate(batch_order[1:]):
new_units[i] = units[id-1]
units = new_units
# Creating incremental batches with units
for batch_id, unit in enumerate(units):
create_filelist(dest_bp + "train_batch_" +
str(batch_id + 1).zfill(2),
filelist_all_sess, [unit[0]], unit[1])
# Creating the cumulative version
if cumulative:
all_lines = []
for batch_id in range(len(units) + 1):
with open(dest_bp + 'train_batch_' +
str(batch_id).zfill(2) + '_filelist.txt', 'r') as f:
all_lines += f.readlines()
with open(dest_cum_bp + 'train_batch_' +
str(batch_id).zfill(2) + '_filelist.txt', 'w') as f:
for line in all_lines:
f.write(line)
shutil.copy(dest_bp + "test_filelist.txt", dest_cum_bp)
# Resetting previous rnd state
np.random.set_state(rnd_state)
def create_sII_run_filelist(glob_file='data/core50_128x128/*/*/*',
dest_bp='data/sII_inc/',
dest_cum_bp='data/sII_cum/',
cumulative=False,
train_sess=[0, 1, 3, 4, 5, 7, 8, 10],
test_sess=[2, 6, 9],
batch_order=[x for x in range(9)]):
""" Given some parameters, it creates the batches filelist and
eventually the cumulative ones. """
# Loading the filelists devided by sessions
filelist_all_sess = load_filelist_per_sess(glob_file)
objs = [[]] * 9
objs_test = []
# Here the creations of the batches (which class to choose for
# each of them) is **independent** by the external seed. This means that
# the units are static throughout the runs while only their order
# can change. This is the same as for the NI scenario where the content of
# the batches is fixed.
# first batch
# objs[0] = [i * 5 for i in range(10)]
objs[0] = [i * 5 for i in range(10)]
objs_test += objs[0][:]
# inc batches
for batch_id in range(1, 9):
if batch_id % 2 == 0:
objs[batch_id] = [
i * 5 + int(math.ceil(batch_id / 2.0)) for i in range(5, 10)
]
else:
objs[batch_id] = [
i * 5 + int(math.ceil(batch_id / 2.0)) for i in range(5)
]
# the first batch stay the same regardless of the order of inc batches
app = [[]] * 9
app[0] = objs[0][:]
for i, batch_idx in enumerate(batch_order[1:]):
app[i + 1] = objs[batch_idx][:]
objs_test += objs[batch_idx][:]
objs = app
print("obj train:", len(objs), sorted(objs))
for batch_id in range(9):
create_filelist(dest_bp + "train_batch_" + str(batch_id).zfill(2),
filelist_all_sess, train_sess, objs[batch_id])
print("obj test:", len(objs_test), sorted(objs_test))
create_filelist(dest_bp + "test", filelist_all_sess, test_sess, objs_test)
# create the cumulative version
if cumulative:
all_lines = []
for batch_id in range(len(batch_order)):
with open(
dest_bp + 'train_batch_' + str(batch_id).zfill(2) +
'_filelist.txt', 'r') as f:
all_lines += f.readlines()
with open(
dest_cum_bp + 'train_batch_' + str(batch_id).zfill(2) +
'_filelist.txt', 'w') as f:
for line in all_lines:
f.write(line)
shutil.copy(dest_bp + "test_filelist.txt", dest_cum_bp)