def getRandomSites(self, xStart, xEnd, yStart, yEnd, qty, radius=0):
import random
rangeX = (xStart, xEnd)
rangeY = (yStart, yEnd)
deltas = set()
for x in range(-radius, radius + 1):
for y in range(-radius, radius + 1):
if x * x + y * y <= radius * radius:
deltas.add((x, y))
randPoints = []
excluded = set()
i = 0
while i < qty:
x = random.randrange(*rangeX)
y = random.randrange(*rangeY)
if (x, y) in excluded: continue
randPoints.append((x, y))
i += 1
excluded.update((x + dx, y + dy) for (dx, dy) in deltas)
return randPoints
def set_energy(self):
if self.classic == "vege":
self.energy = random.randrange(10, 20, 1)
elif self.classic == "fruit":
self.energy = random.randrange(10, 30, 1)
else:
self.energy = random.randrange(30, 50, 1)
def level_3(self):
""" rising """
for i in range(10):
smallbun = RisingBun("sources/images/sausage.png",
SPRITE_SMALLBUN_SCALING)
smallbun.center_x = random.randrange(SCREEN_WIDTH)
smallbun.center_y = random.randrange(-SCREEN_HEIGHT, 0)
self.smallbun_list.append(smallbun)
def level_2(self):
""" falling """
for i in range(10):
smallbun = FallingBun("sources/images/pizza.png",
SPRITE_SMALLBUN_SCALING)
smallbun.center_x = random.randrange(SCREEN_WIDTH)
smallbun.center_y = random.randrange(SCREEN_HEIGHT,
SCREEN_HEIGHT * 2)
self.smallbun_list.append(smallbun)
def Generate(self):
for i in range(self.total):
(red_delta,green_delta,blue_delta) = map(int,numpy.random.normal(125,40,3))
red_delta = self.Correct(red_delta)
green_delta = self.Correct(green_delta)
blue_delta = self.Correct(blue_delta)
(width,height) = self.surface.get_size()
rand_x = random.randrange(width-self.resource.rect.width)
rand_y = random.randrange(height-self.resource.rect.height)
self.resources.append(Resource.Resource(pygame.Rect(rand_x,rand_y,
self.resource.rect.width,self.resource.rect.height)
,(red_delta,blue_delta,green_delta)))
def plotter(combinedDatatoPlot, width, height):
i = list(combinedDatatoPlot.columns)
i.remove("Stock")
for x in i:
mpl.rcParams.update(mpl.rcParamsDefault)
fig = plt.figure(figsize=(width / 100., height / 100.), dpi=100)
stocks = combinedDatatoPlot[['Stock', x]]
num_of_stocks = len(stocks.Stock.unique())
color = iter(plt.cm.jet(np.linspace(0, 1, num_of_stocks)))
for stock in stocks.Stock.unique():
c = next(color)
stockDFRaw = stocks[stocks.Stock == stock]
stockDF = pandas.DataFrame({stock: stockDFRaw[x]})
stockDF.reset_index(level=0, inplace=True)
plotindex = random.randrange(1, len(stockDF.index) - 1)
plt.plot(stockDF["date_time"], stockDF[stock], c=c, label=stock)
plt.ylabel(x)
plt.xlabel('Date')
plt.legend(loc="upper left", fontsize=10)
plt.annotate(stock, (mdates.date2num(
stockDF["date_time"][plotindex]), stockDF[stock][plotindex]),
xytext=(15, 15),
textcoords='offset points',
arrowprops=dict(arrowstyle='-|>', color=c),
color=c)
fig_name = str(x) + "_plot.png"
fig.savefig(fig_name, dpi=600)
plt.clf()
plt.close()
def update(self, e=None):
# Pick a new cell if there is nothing more to do with the old cell.
if self._cell is None or self._cell.degree() <= 0:
valid_permutation_found = False
while not valid_permutation_found:
k = random.randrange(3, 14)
l = numpy.random.permutation(k)
self._cell = pyradbar.Cell( l, suspend=False )
if self._cell.num_fixed_pts_lam() == 0:
valid_permutation_found = True
self.update_wrt_cell()
else:
# Take a face of the cell.
i = random.randrange(0, self._cell.degree() + 1)
self._cell.face(i)
self.update_wrt_cell()
def level_1(self):
# create small buns
for i in range(10):
smallbun = arcade.Sprite("sources/images/chicken.png",
SPRITE_SMALLBUN_SCALING)
# positions of buns
smallbun.center_x = random.randrange(SCREEN_WIDTH)
smallbun.center_y = random.randrange(SCREEN_HEIGHT - 100)
# angles of buns
smallbun.angle = random.randrange(30)
smallbun.change_angle = random.randrange(-5, 6)
# add buns to list
self.smallbun_list.append(smallbun)
def time_qselect_worst(nPerms, sizeIni, sizeFin, step, pivotF=pivotP):
"""
Funcion que mide el tiempo peor de quickselect para nPerms de sizeIni a sizeFin
con un step para diferentes funciones pivote
:param nPerms: numero de permutaciones a realizar para cada tamano
:param sizeIni: tamano inicial
:param sizeFin: tamano final
:param step: step entre cada 2 tamanos
:param pivotF: funcion pivote a emplear
:return lista de tiempos empleados en el caso peor para cada paso
"""
timelist = []
for i in range(sizeIni, sizeFin, step):
timeW = 0
for j in range(nPerms):
t = np.random.permutation(i)
p = min(t)
u = max(t)
k = random.randrange(len(t))
time1 = time.clock()
qselect(t, p, u, k, pivotF)
time2 = time.clock()
timeW = max(time2 - time1, timeW)
timelist.append(timeW)
return timelist
def time_qselect_worst(nPerms, sizeIni, sizeFin, step, pivotF=pivotP):
"""
Funcion que mide el tiempo peor de quickselect para nPerms de sizeIni a sizeFin
con un step para diferentes funciones pivote
:param nPerms: numero de permutaciones a realizar para cada tamano
:param sizeIni: tamano inicial
:param sizeFin: tamano final
:param step: step entre cada 2 tamanos
:param pivotF: funcion pivote a emplear
:return lista de tiempos empleados en el caso peor para cada paso
"""
timelist = []
for i in range(sizeIni, sizeFin, step):
timeW = 0
for j in range(nPerms):
t = np.random.permutation(i)
p = min(t)
u = max(t)
k = random.randrange(len(t))
time1 = time.clock()
qselect(t, p, u, k, pivotF)
time2 = time.clock()
timeW = max(time2 - time1, timeW)
timelist.append(timeW)
return timelist
def fill(self):
seed = random.randrange(0, self.n_items)
self.__nfill_s(int(self.width/2), int(self.height/2), seed)
self.__place(self.__missing())
self.score = self.__score()
rmatrix = self.__compact()
self.size = rmatrix.shape[0] * rmatrix.shape[1]
print(".", end="")
def initial_grouping(n, cn):
#get list of nodes
nodes = [node for node in range(0, n)]
#assign group number to each nodes
group_assign = [random.randrange(0, cn) for g_num in range(len(nodes))]
return nodes, group_assign
def next_genmo(self):
size = len(self._inds)
i_dying = random.randrange(size)
f = [x.calculate_ind_sc() / size for x in self._inds]
parent = random.choice(self._inds, 2, False, f)
father = parent[0]
mother = parent[1]
next_p = father.replicate_error().make_zygote()
next_m = mother.replicate_error().make_zygote()
self._inds[i_dying] = Diploid(next_p, next_m)
return self
def sim_random(n,latencies,traces,outfiles):
print "Simulating Random multi-path scheme..."
traces_file = natsorted(glob.glob(traces[0]+'/*.cell'))
for instance_file in traces_file:
instance = open(instance_file,'r')
instance = instance.read().split('\n')[:-1]
# Get n circuits latencies as a multipath virtual structure of the same client for this instance
mplatencies = getCircuitLatencies(latencies, n) # it is a list of list of latencies for each of m circuits. length = m
routes = [random.randrange(0, n, 1) for _ in range(len(instance))] # Random routes
new_instance = multipath.simulate(instance,mplatencies,routes) # Simulate the multipath effect for the given latencies and routes
saveInFile(instance_file,new_instance,routes,outfiles) # Save the transformed current instance according to thei respective routes
def test_api(assert_200=True, num_objects=5, desired_dimension=2,
total_pulls_per_client=4, num_experiments=1, num_clients=6):
pool = Pool(processes=num_clients)
supported_alg_ids = ['TestAlg']
alg_list = []
for idx, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({'alg_label': algorithm['alg_label'],
'proportion': 1./len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# Test POST Experiment
initExp_args_dict = {}
initExp_args_dict['app_id'] = 'Tests'
initExp_args_dict['args'] = {}
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many'
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['targets'] = {}
initExp_args_dict['args']['targets']['n'] = num_objects
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_uid = test_utils.initExp(initExp_args_dict)
exp_info += [exp_uid]
# Generate participants
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid, participant_uid, total_pulls_per_client, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def __init__(self, classic, value, scale=1):
self.classic = classic
self.value = value
# images for foodie
self.image_file_name = f"sources/images/{self.classic}{self.value}.png"
# if this foodie has been selected
self.chosen = False
# foodie's hungrylevel
self.hungry_level = random.randrange(50, 90, 1)
# call the parent
super().__init__(self.image_file_name, scale)
def setup(self):
""" Setup the game and initial the variables """
# list of musics
self.music_list = ["sources/music/funkyrobot.mp3"]
self.current_song = 0
self.play_song()
# create sprite lists
# score
self.score = 0
# timer reset
self.total_time = 20.0
# set up the player
self.player_sprite.center_x = random.randrange(SCREEN_WIDTH)
self.player_sprite.center_y = random.randrange(SCREEN_HEIGHT - 100)
self.player_list.append(self.player_sprite)
# level set up
self.level = 1
self.level_1()
def begin_rotate(self, monomer_idx = None):
"""
rotate with begin
"""
random_direction = randrange(0, Lattice.z)
#new_position = Lattice.get_coordinate(self.chain[1]._position, random_direction)
new_position = self.chain[1].neighbours_position[random_direction]
if self.box.is_occupied(new_position) or new_position in self.positions_list:
#logging.info("Reject rotate; position: (%d, %d, %d) is occupied" % (new_position))
return False
new_monomer = self.chain[0]
old_position = self.chain[0]._position
old_open_position = self.chain[0]._open
old_next_direction = self.chain[0].next_direction
old_energy = self.total_energy
self.chain[0].position = new_position
self.chain[0]._open = Lattice.get_open_coordinate(self.chain[1]._open, random_direction)
self.chain[0].next_direction = Lattice.TURN_DIRECTION[random_direction]
self.positions_list[0] = new_position
perform_move = True
new_energy = None
if self.box.athermal_state == False:
new_energy = self.box.calculate_chain_energy(self)
dE = new_energy - old_energy
if self.box.accept(dE, None):
perform_move = True
else:
perform_move = False
if perform_move:
if not new_energy:
new_energy = self.box.calculate_chain_energy(self)
self.total_energy = new_energy
else:
new_monomer.position = old_position
new_monomer._open = old_open_position
new_monomer.next_direction = old_next_direction
self.chain[0] = new_monomer
self.positions_list[0] = old_position
# restore old energy
self.total_energy = old_energy
return perform_move
def predict():
# Extracts the file from the upload and save it into
# a temporary file.
#
f = request.files['file']
filename = "%s.wav" % ((random.randrange(0, 1000000000)))
f.save(filename)
# Loads the file and convert features.
#
mfcc = convert_mfcc(filename)
input_l = []
input_r = []
s, w, h = X.shape
input_l.append(mfcc.reshape(w, h, 1))
y_pred = np.zeros(shape=(s))
###loop through the audio mfccs of the baseline audio files.
for i in range(X.shape[0]):
w, h = mfcc.shape
pairs = [np.zeros((1, w, h, 1)) for i in range(2)]
pairs[0][0, :, :, :] = input_l[0]
pairs[1][0, :, :, :] = X[i].reshape(w, h, 1)
y_pred[i] = (model.predict(pairs).ravel())[0]
# Use our Keras model to predict the output.
#
prediction = y[np.argmax(y_pred)]
result = json.dumps({
# Place the class index with the highest probability into
# the "best_class" attribute.
#
# The use of item() converts best_class (which is a numpy.int64
# data type) to a native Python int.
#
#"best_class": best_class.item(),
"person_name": prediction
# Return the full prediction from Keras.
# Convert a Numpy array to a native Python list.
#
#"full_prediction" : full_prediction.tolist()[0]
})
os.remove(filename)
return Response(result, mimetype='application/json')
def segment_rotate(self, monomer_idx = None):
"""
@Monomer|int monomer: monomer or monomer_idx on chain_list
Rotate :monomer
"""
if monomer_idx is None:
monomer_idx = randrange(0, self._total_length - 1)
if monomer_idx == 0 or monomer_idx == self._total_length - 1:
#logging.debug("Tried to rotate monomer of id: %d which is in the end of polymer chain. It doesn't make sense" % monomer_idx)
return False
if monomer_idx + 1 >= self._total_length:
return False
prev_monomer = self.chain[monomer_idx - 1]
rotate_monomer = self.chain[monomer_idx]
next_monomer = self.chain[monomer_idx + 1]
## we can chouse only position from this set
common_neighbours_positions = set(prev_monomer.neighbours_position).intersection(next_monomer.neighbours_position)
number_of_positions = len(common_neighbours_positions)
if number_of_positions == 0:
return False
new_position = list(common_neighbours_positions)[randrange(0, number_of_positions)]
if not new_position or len(new_position) != 3:
if __debug__: logging.debug("New_position %s:" % str(new_position))
return False
try:
direction = prev_monomer.neighbours_position.index(new_position)
except Exception, e:
if __debug__: logging.error("Reject segment, error in direction, (%d, %d, %d)" % (new_position))
return False
def crossover(poblacion, elitismo, probCrossover):
nuevaGeneracion = []
rango = len(poblacion)
if elitismo:
poblacion.sort(key=lambda cromosoma: cromosoma.objetivo)
for pos, cElit in enumerate(poblacion):
if pos < rango * 0.2:
nuevaGeneracion.append(copy.deepcopy(cElit))
poblacion.remove(cElit)
rango = (int)(rango * 0.8)
random.shuffle(poblacion)
for _ in range((int)(rango / 2)):
p1 = poblacion.pop(random.randrange(
0, len(poblacion))) # Padre 1: cromosoma
p2 = poblacion.pop(random.randrange(
0, len(poblacion))) # Padre 2: cromosoma
if random.uniform(0, 1) <= probCrossover:
c = 0 # Cursor
ciclos = []
x = 0
while sum([len(s) for s in ciclos]) == 24:
ciclo = []
while ciclo[0][0] == ciclo[x][1]:
permutacion = [
c, p2.index(p1[c])
] # Par con la pocicion del elemento c en p1 y la pocicion de ese elemento en p2
ciclo.append(permutacion)
c = ciclo[x][1]
x += 1
ciclos.append(ciclo)
for m, ciclo in enumerate(ciclos):
if m % 2 == 0:
cambios = [c[0] for c in ciclo]
for i in cambios:
p1[i], p2[i] = p2[i], p1[i]
nuevaGeneracion.extend([p1, p2])
return nuevaGeneracion
def create_agent(self, type, near=None):
if near:
pos = random.choice(list(self.neighbors(near)))
else:
pos = tuple(random.randrange(size) for size in self.grid_shape)
# NOTE(Pontus): This makes sure that
# the more crowded it is, the less agents will be born
if not self.is_safe_position(pos):
return None
agent = Agent(type, pos)
x, y = pos
self.lattice[x][y].append(agent)
self.agents.append(agent)
return agent
def key_pressed(clock, key_list = None):
keys = event.getKeys()
rt = clock.getTime()
if 'escape' in keys:
quitExpe()
if key_list != None:
ret_keys = [ k for k in key_list if k in keys ]
# this is for testing purposes (automatize input, randomly)
if _debug_on and len(key_list):
ret_keys = [ key_list[random.randrange(0, len(key_list))] ]
else:
ret_keys = keys
return rt, ret_keys
def calculate_probability(odds, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in range(odds):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in range(rep):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in range(odds):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
return (writelist, percentlist, writelist2, percentlist2)
except(KeyboardInterrupt, SystemExit):
try:
os.remove(str(file_count)+'.csv')
finally:
exit()
def select(self):
if self._free == 0:
if self._auto_reset:
self.reset(self._size)
else:
return -1
p = random.randrange(0, self._free) + 1
for i in xrange(0, self._size):
if self._info[i] == 0:
p -= 1
if p == 0:
self._info[i] = 1
self._free -= 1
break
return i
def reservoir_sampling(generator, limit):
"perform a reservoir sampling because for a large state space it is impossible to enumerate states in memory"
import numpy as np
import random
if limit is None:
results = np.array(list(generator))
else:
results = np.array([c for c, _ in zip(generator, range(limit))])
i = limit
step = 1
for result in generator:
i += 1
if (i % step) == 0:
if i == step * 10:
step = i
j = random.randrange(i)
if j < limit:
results[j] = result
print("done reservoir sampling")
return results
def calcMajority(layer, classes):
'''calcualte majority over values in column# start to column# end'''
num_features = layer.GetFeatureCount()
for i in range(num_features):
feature = layer.GetFeature(i)
zonal = [feature.GetField(x) for x in classes]
max_cnt = max(zonal)
labels=[a for a,b in enumerate(zonal) if b==max_cnt]
#resolve ties
if len(labels)==1:
label=labels[0]
elif len(labels)>1 and max_cnt>0:
label=labels[random.randrange(0,len(labels))] #if tied, take random
else: label=255 #no data
feature.SetField('label', label)
layer.SetFeature(feature)
source = None
print 'shapefile updated'
def level_1(self):
# set up the player
for foodie_classic in FOODIE_CLASSIC:
for foodie_value in FOODIE_VALUES:
foodie = Foodie(foodie_classic, foodie_value,
SPRITE_FOODIE_SCALING)
foodie.position = SCREEN_WIDTH / 2, FOODIE_HEIGHT / 2
self.foodie_list.append(foodie)
# shuffle the foodie
for pos1 in range(len(self.foodie_list)):
pos2 = random.randrange(len(self.foodie_list))
self.foodie_list[pos1], self.foodie_list[pos2] = self.foodie_list[
pos2], self.foodie_list[pos1]
self.foodie = self.foodie_list[0]
# create food list
self.food_list = arcade.SpriteList()
# create every food
for food_classic in FOOD_CLASSIC:
for food_value in FOOD_VALUES:
food = Food(food_classic, food_value, SPRITE_FOOD_SCALING)
food.position = FOOD_START_X + FOOD_VALUES.index(food_value) * FOOD_X_SPACING, \
FOOD_START_Y + FOOD_CLASSIC.index(food_classic) * FOOD_Y_SPACING
self.food_list.append(food)
# list of foods we are dragging with mouse
self.held_foods = []
# original position of the dragging foods if they need to go back
self.held_foods_original_location = []
# create dishes
self.dish_list: arcade.SpriteList = arcade.SpriteList()
for i in range(4):
# dish = arcade.SpriteSolidColor(DISH_WIDTH, DISH_HEIGHT, arcade.csscolor.DARK_OLIVE_GREEN)
dish = arcade.Sprite("sources/images/dish.jpeg",
SPRITE_FOOD_SCALING)
dish.position = DISH_START_X + i * DISH_X_SPACING, DISH_START_Y
self.dish_list.append(dish)
def calcMajority(layer, classes):
'''calcualte majority over values in column# start to column# end'''
num_features = layer.GetFeatureCount()
for i in range(num_features):
feature = layer.GetFeature(i)
zonal = [feature.GetField(x) for x in classes]
max_cnt = max(zonal)
labels = [a for a, b in enumerate(zonal) if b == max_cnt]
#resolve ties
if len(labels) == 1:
label = labels[0]
elif len(labels) > 1 and max_cnt > 0:
label = labels[random.randrange(
0, len(labels))] #if tied, take random
else:
label = 255 #no data
feature.SetField('label', label)
layer.SetFeature(feature)
source = None
print 'shapefile updated'
def create_stsq(size=None, typ='stream',
name=span.utils.name2num('Spik'), nchannels=16, sort_code=0,
fmt=np.float32, fs=4882.8125,
samples_per_channel=None):
if size is None:
size = randint(2 ** 5, 2 ** 6)
if samples_per_channel is None:
samples_per_channel = randint(2 ** 6, 2 ** 7)
names = ('size', 'type', 'name', 'channel', 'sort_code', 'timestamp',
'fp_loc', 'format', 'fs', 'shank')
nsamples = samples_per_channel * nchannels
index = Int64Index(np.arange(nsamples))
size = Series(size, index=index, name='size')
typ = Series(typ, index=index, name='type')
name = Series(name, index=index, name='name')
channel = Series(np.tile(np.arange(nchannels),
(samples_per_channel, 1))[:, ::-1].ravel(),
index=index, name='channel')
sort_code = Series(sort_code, index=index, name='sort_code')
fp_loc = Series(index, name='fp_loc')
fmt = Series([fmt] * index.size, index=index, name='format')
start = randrange(100e7, 128e7)
ts = start + (size[0] / fs) * np.r_[:samples_per_channel]
ts = np.tile(ts, (nchannels, 1)).T.ravel()
timestamp = Series(ts, index=index, name='timestamp')
srt = Indexer.sort('channel').reset_index(drop=True)
shank = srt.shank[channel].reset_index(drop=True)
data = (size, typ, name, channel, sort_code, timestamp, fp_loc, fmt, fs,
shank)
return DataFrame(dict(zip(names, data)))
def run_all(assert_200, num_clients):
def timeit(f):
"""
Utility used to time the duration of code execution. This script can be composed with any other script.
Usage::\n
def f(n):
return n**n
def g(n):
return n,n**n
answer0,dt = timeit(f)(3)
answer1,answer2,dt = timeit(g)(3)
"""
def timed(*args, **kw):
ts = time.time()
result = f(*args, **kw)
te = time.time()
if type(result)==tuple:
return result + ((te-ts),)
else:
return result,(te-ts)
return timed
app_id = 'PoolBasedTripletMDS'
# num_clients = 1
# assert_200 = True
# Setup the exp_uid's
client_exp_uids = []
client_exp_keys = []
client_participant_uids = []
for cl in range(num_clients):
participants = []
for i in range(10):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
client_participant_uids.append(participants)
# Args for the experiments
num_objects = 100
desired_dimension = 2
x = numpy.linspace(0,1,num_objects)
X_true = numpy.vstack([x,x]).transpose()
# total_pulls = 100000*num_clients
total_pulls = 3
# Generate the list of algorithms
alg_list = []
test_alg = {}
test_alg['alg_id'] = 'RandomSampling'
test_alg['alg_label'] = 'Test'
test_alg['test_alg_label'] = 'Test'
test_alg['params'] = {}
alg_list.append(test_alg)
random_alg = {}
random_alg['alg_id'] = 'RandomSampling'
random_alg['alg_label'] = 'Random'
random_alg['test_alg_label'] = 'Test'
random_alg['params'] = {}
alg_list.append(random_alg)
uncertainty_sampling_alg = {}
uncertainty_sampling_alg['alg_id'] = 'UncertaintySampling'
uncertainty_sampling_alg['alg_label'] = 'Uncertainty Sampling'
uncertainty_sampling_alg['test_alg_label'] = 'Test'
uncertainty_sampling_alg['params'] = {}
alg_list.append(uncertainty_sampling_alg)
params = {}
test_proportion = 0.2
params['proportions'] = []
for algorithm in alg_list:
if algorithm['alg_label'] == 'Test':
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':test_proportion } )
else:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':(1. - test_proportion)/(len(alg_list)-1.) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# args paramaters
n = num_objects
d = desired_dimension
delta = 0.01
#################################################
# Test POST Experiment with no app id
#################################################
print
print 'Test POST Experiment with no app id'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert int(response.status_code) == 400
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with empty args
#################################################
print
print 'Test POST Experiment with empty args'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert int(response.status_code) == 400
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with letter for numerical value (failure_probability)
#################################################
print
print 'Test POST Experiment with letter for numerical value (failure_probability)'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = 'Failure Prob'
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
# This is a problem! 200 is returned each time
# if assert_200: assert response.status_code == 400
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with letter for numerical value (dimension)
#################################################
print
print 'Test POST Experiment with letter for numerical value (dimension)'
# print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = 'Huge Dimension'
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code == 400
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with letter for numerical value (num objects)
#################################################
print
print 'Test POST Experiment with letter for numerical value (num objects)'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = 'Huge Number of Objects'
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code == 400
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with empty alg_list
#################################################
print
print 'Test POST Experiment with empty alg_list'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = []
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code == 200
initExp_response_dict = json.loads(response.text)
#################################################
# Test POST Experiment with no optional fields present
#################################################
print
print 'Test POST Experiment with no optional fields present'
print
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code == 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
client_exp_uids.append(exp_uid)
client_exp_keys.append(exp_key)
#################################################
# Test GET Experiment
#################################################
print
print 'Test POST Experiment with no optional fields present'
print
url = "http://"+HOSTNAME+"/api/experiment/"+client_exp_uids[cl]+"/"+client_exp_keys[cl]
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
print
# Now we will do many get queries over a random set of exp_uid's to generate data
seconds_between_API_hits = .001
t = 0
while t<total_pulls:
t+=1
# print t
time.sleep(seconds_between_API_hits)
# grab a random exp_uid
exp_uid = client_exp_uids[t%len(client_exp_uids)] #random.choice(client_exp_uids)
exp_key = client_exp_keys[t%len(client_exp_keys)] #random.choice(client_exp_uids)
participant_uids = client_participant_uids[t%len(client_exp_uids)]
participant_uid = numpy.random.choice(participants)
#######################################
# test GET getQuery #
#######################################
getQuery_args_dict = {}
getQuery_args_dict['exp_uid'] = exp_uid
getQuery_args_dict['exp_key'] = exp_key
getQuery_args_dict['args'] = {}
getQuery_args_dict['args']['participant_uid'] = participant_uid
if t == 3:
getQuery_args_dict['args']['participant_uid'] = 'fake participant_uid'
print
print 'Test GET getQuery with fake participant_uid'
print
url = 'http://'+HOSTNAME+'/api/experiment/getQuery'
response,dt = timeit(requests.post)(url, json.dumps(getQuery_args_dict),headers={'content-type':'application/json'})
print "POST getQuery response = ", response.text, response.status_code
# This should not be 200
if assert_200: assert response.status_code is 400
print "POST getQuery duration = ", dt
print
else:
url = 'http://'+HOSTNAME+'/api/experiment/getQuery'
response,dt = timeit(requests.post)(url, json.dumps(getQuery_args_dict),headers={'content-type':'application/json'})
print "POST getQuery response = ", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST getQuery duration = ", dt
print
getQuery_response_dict = json.loads(response.text)
query_uid = getQuery_response_dict['query_uid']
targets = getQuery_response_dict['target_indices']
# print targets
for target in targets:
if target['label'] == 'center':
index_center = target['index']
elif target['label'] == 'left':
index_left = target['index']
elif target['label'] == 'right':
index_right = target['index']
#############################################
# test POST processAnswer
#############################################
# generate simulated reward
direction = norm(X_true[index_left]-X_true[index_center])-norm(X_true[index_right]-X_true[index_center])
r = numpy.random.rand()
if r<.1:
direction = - direction
if direction<0.:
target_winner = index_left
else:
target_winner = index_right
processAnswer_args_dict = {}
processAnswer_args_dict["exp_uid"] = exp_uid
processAnswer_args_dict["exp_key"] = exp_key
processAnswer_args_dict["args"] = {}
processAnswer_args_dict["args"]["query_uid"] = query_uid
processAnswer_args_dict["args"]["target_winner"] = target_winner
if t == 1:
processAnswer_args_dict["args"]["query_uid"] = ''
print
print 'test POST processAnswer with empty query_uid'
print
processAnswer_args_dict["args"]["target_winner"] = target_winner
url = 'http://'+HOSTNAME+'/api/experiment/processAnswer'
print "POST processAnswer args = ", processAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(processAnswer_args_dict), headers={'content-type':'application/json'})
print "POST processAnswer response", response.text, response.status_code
if assert_200: assert response.status_code is 400
print "POST processAnswer duration = ", dt
print
processAnswer_json_response = eval(response.text)
elif t == 2:
processAnswer_args_dict["args"]["target_winner"] = ''
print
print 'test POST processAnswer with empty target_winner'
print
url = 'http://'+HOSTNAME+'/api/experiment/processAnswer'
print "POST processAnswer args = ", processAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(processAnswer_args_dict), headers={'content-type':'application/json'})
print "POST processAnswer response", response.text, response.status_code
# This should not be 200
if assert_200: assert response.status_code is 400
print "POST processAnswer duration = ", dt
print
processAnswer_json_response = eval(response.text)
else:
url = 'http://'+HOSTNAME+'/api/experiment/processAnswer'
print "POST processAnswer args = ", processAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(processAnswer_args_dict), headers={'content-type':'application/json'})
print "POST processAnswer response", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST processAnswer duration = ", dt
print
processAnswer_json_response = eval(response.text)
def test_api(assert_200=True, num_objects=4, desired_dimension=1,
total_pulls_per_client=5, num_experiments=1,
num_clients=7):
true_weights = numpy.zeros(desired_dimension)
true_weights[0] = 1.
pool = Pool(processes=num_clients)
supported_alg_ids = ['RandomSamplingLinearLeastSquares','RandomSamplingLinearLeastSquares']
alg_list = []
for idx,alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
if idx==0:
alg_item['alg_label'] = 'Test'
else:
alg_item['alg_label'] = alg_id
alg_item['test_alg_label'] = 'Test'
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({'alg_label': algorithm['alg_label'],
'proportion': 1./len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
targetset = []
for i in range(num_objects):
features = list(numpy.random.randn(desired_dimension))
targetset.append({'primary_description': str(features),
'primary_type':'text',
'alt_description':'%d' % (i),
'alt_type':'text',
'meta': {'features':features}})
# Test POST Experiment
print '\n'*2 + 'Testing POST initExp...'
initExp_args_dict = {}
initExp_args_dict['app_id'] = 'PoolBasedBinaryClassification'
initExp_args_dict['args'] = {}
initExp_args_dict['args']['failure_probability'] = 0.01
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['targets'] = {'targetset': targetset}
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_info_ = test_utils.initExp(initExp_args_dict)
exp_info += [exp_info_]
exp_uid = initExp_response_dict['exp_uid']
exp_info.append({'exp_uid':exp_uid,})
# Test GET Experiment
initExp_response_dict = test_utils.getExp(exp_uid)
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid,participant_uid,total_pulls_per_client,true_weights,assert_200))
print "participants are", participants
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
import numpy as np import time from numpy.random import * import random from multiprocessing import Process, cpu_count, current_process, Array """ 行列の積 C = A B """ #行列 #a = np.array([[3,0,0,3,3,5,2,2,4], [1,2,0,3,6,2,1,5,2],[1,4,7,2,5,3,3,6,5],[3,5,2,5,2,5,3,6,3],[4,6,2,5,2,5,7,3,4],[6,2,6,4,2,5,6,2,6],[2,5,2,5,3,5,6,4,5],[4,2,4,5,2,5,3,4,5],[3,5,3,6,4,5,3,1,2]]) #b = np.array([[3,0,0,3,3,5,2,2,4], [1,2,0,3,6,2,1,5,2],[1,4,7,2,5,3,3,6,5],[3,5,2,5,2,5,3,6,3],[4,6,2,5,2,5,7,3,4],[6,2,6,4,2,5,6,2,6],[2,5,2,5,3,5,6,4,5],[4,2,4,5,2,5,3,4,5],[3,5,3,6,4,5,3,1,2]]) r1 = random.randrange(2000) #2000までのN #N×Nの正方行列を生成 a = np.random.rand(r1, r1) #行列A b = np.random.rand(r1, r1) #行列B print(random.randrange(2000)) #何行何列の正方行列か表示 start = time.time() #時間計測スタート np.dot(a, b) #行列積の逐次計算 process_time = time.time() - start #時間計測終わり print(process_time) #時間計測出力 print(np.dot(a, b)) #行列積計算結果
def run_all(assert_200, num_clients):
def timeit(f):
"""
Utility used to time the duration of code execution. This script can be composed with any other script.
Usage::\n
def f(n):
return n**n
def g(n):
return n,n**n
answer0,dt = timeit(f)(3)
answer1,answer2,dt = timeit(g)(3)
"""
def timed(*args, **kw):
ts = time.time()
result = f(*args, **kw)
te = time.time()
if type(result)==tuple:
return result + ((te-ts),)
else:
return result,(te-ts)
return timed
app_id = 'PoolBasedTripletMDS'
# Setup the exp_uid's
client_exp_uids = []
client_exp_keys = []
client_participant_uids = []
for cl in range(num_clients):
participants = []
for i in range(10):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
client_participant_uids.append(participants)
# Args for the experiments
num_objects = 30
desired_dimension = 2
x = numpy.linspace(0,1,num_objects)
X_true = numpy.vstack([x,x]).transpose()
# total_pulls = 100000*num_clients
# total_pulls = 10*desired_dimension*num_objects*numpy.floor(numpy.log(num_objects))*num_clients*4
total_pulls = 50
# Generate the list of algorithms
alg_list = []
test_alg = {}
test_alg['alg_id'] = 'RandomSampling'
test_alg['alg_label'] = 'Test'
test_alg['test_alg_label'] = 'Test'
test_alg['params'] = {}
alg_list.append(test_alg)
random_alg = {}
random_alg['alg_id'] = 'RandomSampling'
random_alg['alg_label'] = 'Random'
random_alg['test_alg_label'] = 'Test'
random_alg['params'] = {}
alg_list.append(random_alg)
uncertainty_sampling_alg = {}
uncertainty_sampling_alg['alg_id'] = 'UncertaintySampling'
uncertainty_sampling_alg['alg_label'] = 'Uncertainty Sampling'
uncertainty_sampling_alg['test_alg_label'] = 'Test'
uncertainty_sampling_alg['params'] = {}
alg_list.append(uncertainty_sampling_alg)
crowd_kernel_alg = {}
crowd_kernel_alg['alg_id'] = 'CrowdKernel'
crowd_kernel_alg['alg_label'] = 'Crowd Kernel'
crowd_kernel_alg['test_alg_label'] = 'Test'
crowd_kernel_alg['params'] = {}
alg_list.append(crowd_kernel_alg)
params = {}
test_proportion = 0.2
params['proportions'] = []
for algorithm in alg_list:
if algorithm['alg_label'] == 'Test':
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':test_proportion } )
else:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':(1. - test_proportion)/(len(alg_list)-1.) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# args paramaters
n = num_objects
d = desired_dimension
delta = 0.01
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['alg_list'] = alg_list
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
for cl in range(num_clients):
# convert python dictionary to json dictionary
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
client_exp_uids.append(exp_uid)
client_exp_keys.append(exp_key)
#################################################
# Test GET Experiment
#################################################
url = "http://"+HOSTNAME+"/api/experiment/"+client_exp_uids[cl]+"/"+client_exp_keys[cl]
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
# Now we will do many get queries over a random set of exp_uid's to generate data
seconds_between_API_hits = .001
t = 0
while t<total_pulls:
t+=1
print t
time.sleep(seconds_between_API_hits)
# grab a random exp_uid
exp_uid = client_exp_uids[t%len(client_exp_uids)] #random.choice(client_exp_uids)
exp_key = client_exp_keys[t%len(client_exp_keys)] #random.choice(client_exp_uids)
participant_uids = client_participant_uids[t%len(client_exp_uids)]
participant_uid = numpy.random.choice(participants)
#######################################
# test GET getQuery #
#######################################
getQuery_args_dict = {}
getQuery_args_dict['exp_uid'] = exp_uid
getQuery_args_dict['exp_key'] = exp_key
getQuery_args_dict['args'] = {}
getQuery_args_dict['args']['participant_uid'] = participant_uid
url = 'http://'+HOSTNAME+'/api/experiment/getQuery'
response,dt = timeit(requests.post)(url, json.dumps(getQuery_args_dict),headers={'content-type':'application/json'})
print "POST getQuery response = ", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST getQuery duration = ", dt
print
getQuery_response_dict = json.loads(response.text)
query_uid = getQuery_response_dict['query_uid']
targets = getQuery_response_dict['target_indices']
print targets
for target in targets:
if target['label'] == 'center':
index_center = target['index']
elif target['label'] == 'left':
index_left = target['index']
elif target['label'] == 'right':
index_right = target['index']
#############################################
# test POST processAnswer
#############################################
# generate simulated reward
direction = norm(X_true[index_left]-X_true[index_center])-norm(X_true[index_right]-X_true[index_center])
r = numpy.random.rand()
if r<.1:
direction = - direction
if direction<0.:
target_winner = index_left
else:
target_winner = index_right
processAnswer_args_dict = {}
processAnswer_args_dict["exp_uid"] = exp_uid
processAnswer_args_dict["exp_key"] = exp_key
processAnswer_args_dict["args"] = {}
processAnswer_args_dict["args"]["query_uid"] = query_uid
processAnswer_args_dict["args"]["target_winner"] = target_winner
url = 'http://'+HOSTNAME+'/api/experiment/processAnswer'
print "POST processAnswer args = ", processAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(processAnswer_args_dict), headers={'content-type':'application/json'})
print "POST processAnswer response", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST processAnswer duration = ", dt
print
processAnswer_json_response = eval(response.text)
# #############################################
# # test GET logs
# #############################################
# r = numpy.random.rand()
# if r <.005:
# url = 'http://'+HOSTNAME+'/api/experiment/'+exp_uid+'/'+exp_key'/logs'
# response,dt = timeit(requests.get)(url)
# print "GET Logs response", response.text, response.status_code
# print "GET Logs duration = ", dt
# print
# #############################################
# # test GET participants
# #############################################
# r = numpy.random.rand()
# if r <.005:
# url = 'http://'+HOSTNAME+'/api/experiment/'+exp_uid+'/'+exp_key+'/participants'
# response,dt = timeit(requests.get)(url)
# print "Participants response", response.text, response.status_code
# print "Participants duration = ", dt
# print
############################################
# test POST stats
###########################################
# Create all our potential /experiments/stats requests and then loop over them
args_list = []
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'api_activity_histogram'
getStats_args_dict['params'] = {'task':'getQuery'}
args_list.append(getStats_args_dict)
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'api_processAnswer_activity_stacked_histogram'
getStats_args_dict['params'] = {}
args_list.append(getStats_args_dict)
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'compute_duration_multiline_plot'
getStats_args_dict['params'] = {'task':'getQuery'}
args_list.append(getStats_args_dict)
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'compute_duration_detailed_stacked_area_plot'
getStats_args_dict['params'] = {'task':'getQuery','alg_label':alg_list[-1]['alg_label']}
args_list.append(getStats_args_dict)
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'test_error_multiline_plot'
getStats_args_dict['params'] = {}
args_list.append(getStats_args_dict)
for cl in range(num_clients):
getStats_args_dict = {}
getStats_args_dict["exp_uid"] = client_exp_uids[cl]
getStats_args_dict["exp_key"] = client_exp_keys[cl]
for args in args_list:
getStats_args_dict["args"] = args
url = 'http://'+HOSTNAME+'/api/experiment/stats'
response = requests.post(url, json.dumps(getStats_args_dict) ,headers={'content-type':'application/json'})
getStats_json_response = eval(response.text)
print "/experiment/stats "+args['stat_id'], str(getStats_json_response), response.status_code
if assert_200: assert response.status_code is 200
print
def test_geometric_spanner_condition_Delaunay_triangulation_sphere_surface(self):
'''The geometric spanner condition (http://en.wikipedia.org/wiki/Delaunay_triangulation#Properties) indicates that the length of the shortest edge-traveled path between two nodes in a Delaunay triangulation is no longer than 2.42 times the straight-line Euclidean distance between them.'''
#create a networkx graph object of the Delaunay triangulation vertices & edges
voronoi_instance_small = voronoi_utility.Voronoi_Sphere_Surface(self.simple_sphere_coordinate_array)
Delaunay_point_array_small = voronoi_instance_small.delaunay_triangulation_spherical_surface() #should be shape (N,3,3) for N triangles and their vertices in 3D space
node_dictionary = {}
node_counter = 1
list_nodes_identified_debug = []
#assign an integer node (vertex) number to each unique coordinate on the test sphere:
for node_coordinate in self.simple_sphere_coordinate_array:
node_dictionary[node_counter] = node_coordinate
node_counter += 1
#print 'self.simple_sphere_coordinate_array:', self.simple_sphere_coordinate_array
#print 'self.simple_sphere_coordinate_array.shape:', self.simple_sphere_coordinate_array.shape
#print 'node_dictionary.values():', node_dictionary.values() #there seem to be multiple duplicates / rounding variations for the polar point at [0. 0. 2.]
def identify_node_based_on_coordinate(coordinate_array,node_dictionary):
'''Return the node number based on the coordinates in the original test sphere data set.'''
nodenum = 0
num_positives_debug = 0
for node_number, node_coordinates in node_dictionary.iteritems():
if numpy.allclose(node_coordinates,coordinate_array,atol=1e-18):
nodenum = node_number
num_positives_debug += 1
#if num_positives_debug > 1:
#print 'duplicate offender:', node_coordinates, coordinate_array
#else:
#print 'original match:', node_coordinates, coordinate_array
assert num_positives_debug == 1, "Only a single node should correspond to the input coordinates."
return nodenum
def produce_networkx_edges_from_triangle_data(triangle_array_data,node_dictionary):
'''Input should be shape (3,3) array of coordinate data for a Delaunay triangle.'''
list_networkx_edge_tuples = []
#each triangle will, of course, have 3 edges
triangle_array_row_indices_for_edge_vertices = [[0,1],[1,2],[2,0]]
for triangle_row_indices_of_edge in triangle_array_row_indices_for_edge_vertices:
first_triangle_row_index, second_triangle_row_index = triangle_row_indices_of_edge
first_vertex_coord = triangle_array_data[first_triangle_row_index]
second_vertex_coord = triangle_array_data[second_triangle_row_index]
graph_node_number_first_vertex_current_edge = identify_node_based_on_coordinate(first_vertex_coord,node_dictionary)
graph_node_number_second_vertex_current_edge = identify_node_based_on_coordinate(second_vertex_coord,node_dictionary)
list_nodes_identified_debug.extend([graph_node_number_first_vertex_current_edge,graph_node_number_second_vertex_current_edge]) #missing nodes with debug list growth here, but no missing nodes if I grow the debug list from within the identify_node_based_on_coordinate() function itself; why????
#the edge weight for networkx should be the Euclidean straight-line distance between the vertices
weight = scipy.spatial.distance.euclidean(first_vertex_coord,second_vertex_coord)
networkx_edge_tuple = (graph_node_number_first_vertex_current_edge,graph_node_number_second_vertex_current_edge,weight)
list_networkx_edge_tuples.append(networkx_edge_tuple)
return list_networkx_edge_tuples
#build networkx graph object from Delaunay triangles (simplices)
G = nx.Graph()
triangle_counter = 0
for triangle_coord_array in Delaunay_point_array_small:
current_list_networkx_edge_tuples = produce_networkx_edges_from_triangle_data(triangle_coord_array,node_dictionary)
#print 'current_list_networkx_edge_tuples:', current_list_networkx_edge_tuples
G.add_weighted_edges_from(current_list_networkx_edge_tuples) #duplicates will simply be updated
triangle_counter += 1
#print 'Triangle:', triangle_counter, 'total edges:', G.size(), 'total nodes:', G.order()
#print 'size:', G.size()
#print 'list of edges:', G.edges()
#print 'num nodes:', len(G.nodes())
#print 'dict size:', len(node_dictionary)
#print 'dict keys:', node_dictionary.keys()
#print 'nodes:', G.nodes()
#print 'edges:', G.edges()
#print 'ordered set nodes identified:', set(sorted(list_nodes_identified_debug))
self.assertEqual(len(G),self.num_triangulation_input_points) #obviously, the number of nodes in the graph should match the number of points on the sphere
#perform the geometric spanner test for a random subset of nodes:
num_tests = 0
while num_tests < 10: #do 10 random tests
first_node_number = random.randrange(1,len(G),1)
second_node_number = random.randrange(1,len(G),1)
minimum_distance_between_nodes_following_Delaunay_edges = nx.dijkstra_path_length(G,first_node_number,second_node_number)
#compare with straight line Euclidean distance:
first_node_coordinate = node_dictionary[first_node_number]
second_node_coordinate = node_dictionary[second_node_number]
Euclidean_distance = scipy.spatial.distance.euclidean(first_node_coordinate,second_node_coordinate)
self.assertLess(minimum_distance_between_nodes_following_Delaunay_edges/Euclidean_distance,2.42) #the geometric spanner condition
num_tests += 1
def run_all(assert_200):
app_id = 'CardinalBanditsPureExploration'
num_arms = 50
true_means = numpy.array(range(num_arms)) / float(num_arms)
total_pulls_per_client = 30
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 200
pool = Pool(processes=num_clients)
# input test parameters
n = num_arms
delta = 0.05
R = 2 # assumes scores in range [1,5]
# supported_alg_ids = ['SimpleUCB','AlternativeUCB']
supported_alg_ids = ['RandomSampling', 'LUCB', 'LilUCB', 'RoundRobin']
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append({
'alg_label': algorithm['alg_label'],
'proportion': 1. / len(alg_list)
})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['R'] = R
initExp_args_dict['args'][
'participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args'][
'algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args'][
'instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args'][
'debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['labels'] = {'1': 1, '2': 2, '3': 3}
initExp_args_dict['app_id'] = app_id
exp_info = []
for ell in range(num_experiments):
url = "http://" + HOSTNAME + "/api/experiment"
response = requests.post(url,
json.dumps(initExp_args_dict),
headers={'content-type': 'application/json'})
print "POST initExp response =", response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
exp_info.append({'exp_uid': exp_uid, 'exp_key': exp_key})
#################################################
# Test GET Experiment
#################################################
url = "http://" + HOSTNAME + "/api/experiment/" + exp_uid + "/" + exp_key
response = requests.get(url)
print "GET experiment response =", response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
exp_key = experiment['exp_key']
pool_args.append((exp_uid, exp_key, participant_uid,
total_pulls_per_client, true_means, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
def test_api(assert_200=True, num_arms=5, num_clients=8, delta=0.05,
total_pulls_per_client=5, num_experiments=1,
params={'num_tries': 5}):
app_id = 'DuelingBanditsPureExploration'
true_means = numpy.array(range(num_arms)[::-1])/float(num_arms)
pool = Pool(processes=num_clients)
supported_alg_ids = ['BR_LilUCB', 'BR_Random', 'ValidationSampling', 'BR_KLUCB']
alg_list = []
for i, alg_id in enumerate(supported_alg_ids):
alg_item = {}
if alg_id == 'ValidationSampling':
alg_item['params'] = params
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id+'_'+str(i)
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({'alg_label': algorithm['alg_label'], 'proportion':1./len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
print algorithm_management_settings
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {'alg_list': alg_list,
'algorithm_management_settings': algorithm_management_settings,
'context': 'Context for Dueling Bandits',
'context_type': 'text',
'debrief': 'Test debried.',
'failure_probability': 0.05,
'instructions': 'Test instructions.',
'participant_to_algorithm_management': 'one_to_many',
'targets': {'n': num_arms}}
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
exp_info = []
for ell in range(num_experiments):
exp_info += [test_utils.initExp(initExp_args_dict)[1]]
# Generate participants
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid, participant_uid, total_pulls_per_client,
true_means,assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def pull_move(self, monomer_idx = None):
"""
Pull move, N. Lesh, M. Mitzemacher, S. Whitesides
"""
## flags
valid_configuration = False
if settings.DEBUG: self.valid_chain()
#if monomer_idx is None:
monomer_idx = randrange(0, self._total_length - 1)
## two special cases at the begin/end
case = 3
old_energy = self.total_energy
is_occupied = self.box.is_occupied
is_neighbour = self.box.lattice.is_neighbour
## trials with optimization
last_affected_idx = 0
# case
# 0 - left
# 1 - right
# 3 - inside
# 0 - right
# 1 - left
#mod_pos = 0
if monomer_idx == 0: ## left
dir_1 = randrange(0, Lattice.z)
dir_2 = randrange(0, Lattice.z)
pos_1 = self.chain[0].neighbours_position[dir_1]
pos_2 = self.box.lattice.get_coordinate(pos_1, dir_2)
if is_occupied(pos_1) or is_occupied(pos_2):
return False
#if not is_neighbour(pos_1, pos_2):
# return False
m1 = self.chain[0]
m2 = self.chain[1]
open_pos_1 = self.box.lattice.get_open_coordinate(m1._open, dir_1)
open_pos_2 = self.box.lattice.get_open_coordinate(open_pos_1, dir_2)
old_position_list = self.positions_list[:]
#old_monomer_position = [ m._position for m in self.chain ]
old_monomer_position = old_position_list
old_monomer_open_position = [ m._open for m in self.chain ]
old_monomer_next_direction = [ m.next_direction for m in self.chain ]
m1.position = pos_2
m1._open = open_pos_2
m2.position = pos_1
m2._open = open_pos_1
## old energy
#self[0] = m1
m1.chain = self
self.chain[0] = m1
self.positions_list[0] = m1._position
#self[1] = m2
m2.chain = self
self.chain[1] = m2
self.positions_list[1] = m2._position
#mod_pos = 2
for idx in xrange(2, self._total_length):
i_monomer = self.chain[idx]
p_monomer = self.chain[idx - 1]
diff = np.array(i_monomer._open) - np.array(p_monomer._open)
diff = diff.dot(diff)
if diff == 2:
p_monomer.next_direction = p_monomer.neighbours_position.index(i_monomer._position)
break
#mod_pos += 1
i_monomer.position = old_monomer_position[idx - 2]
i_monomer._open = old_monomer_open_position[idx - 2]
i_monomer.next_direction = old_monomer_next_direction[idx - 2]
#self[idx] = i_monomer
i_monomer.chain = self
self.chain[idx] = i_monomer
self.positions_list[idx] = i_monomer._position
last_affected_idx = idx
self.chain[0].next_direction = self.chain[0].neighbours_position.index(self.chain[1]._position)
self.chain[1].next_direction = self.chain[1].neighbours_position.index(self.chain[2]._position)
case = 0
if settings.DEBUG: self.valid_chain()
elif monomer_idx == self._total_length - 1: ## right
dir_1 = randrange(0, Lattice.z)
dir_2 = randrange(0, Lattice.z)
pos_1 = self.chain[-1].neighbours_position[dir_1]
pos_2 = self.box.lattice.get_coordinate(pos_1, dir_2)
if is_occupied(pos_1) or is_occupied(pos_2):
return False
open_pos_1 = self.box.lattice.get_open_coordinate(self.chain[-1]._open, dir_1)
open_pos_2 = self.box.lattice.get_open_coordinate(open_pos_1, dir_2)
old_position_list = self.positions_list[:]
#old_monomer_position = [ m._position for m in self.chain ]
old_monomer_position = old_position_list
old_monomer_open_position = [ m._open for m in self.chain ]
old_monomer_next_direction = [ m.next_direction for m in self.chain ]
m1 = self.chain[-1]
m2 = self.chain[-2]
m1.position = pos_2
m1._open = open_pos_2
m2.position = pos_1
m2._open = open_pos_1
#mod_pos = 2
for idx in xrange(self._total_length - 2, -1, -1):
i_monomer = self.chain[idx]
n_monomer = self.chain[idx+1]
diff = np.array(i_monomer._open) - np.array(n_monomer._open)
diff = diff.dot(diff)
if diff == 2:
i_monomer.next_direction = i_monomer.neighbours_position.index(n_monomer._position)
break
#mod_pos += 1
i_monomer.position = old_monomer_position[idx + 2]
i_monomer._open = old_monomer_open_position[idx + 2]
i_monomer.next_direction = old_monomer_next_direction[idx + 2]
#self[idx] = i_monomer
i_monomer.chain = self
self.chain[idx] = i_monomer
self.positions_list[idx] = i_monomer._position
last_affected_idx = idx
case = 1
self.chain[-2].next_direction = self.chain[-2].neighbours_position.index(self.chain[-1]._position)
self.chain[-1].next_direction = 0
if settings.DEBUG: self.valid_chain()
if case == 3:
prev_monomer = self.chain[monomer_idx - 1]
monomer = self.chain[monomer_idx]
next_monomer = self.chain[monomer_idx + 1]
random_neighbour_prev = randrange(0, Lattice.z)
#if self.box.is_occupied(prev_monomer.neighbours_position[random_neighbour_prev]):
# return False
#L can be adjenct to i-1 or i+1, randomly choose
sub_case = randrange(0, 2) ## 0 - right, 1 - left
if sub_case == 0:
L_position = next_monomer.neighbours_position[random_neighbour_prev]
L_open_position = Lattice.get_open_coordinate(next_monomer._open, random_neighbour_prev)
Lnext_direction = self.box.lattice.get_neighbours(L_position).index(next_monomer._position)
elif sub_case == 1:
L_position = prev_monomer.neighbours_position[random_neighbour_prev]
L_open_position = Lattice.get_open_coordinate(prev_monomer._open, random_neighbour_prev)
Lprev_direction = self.box.lattice.get_neighbours(L_position).index(prev_monomer._position)
C_position = monomer.neighbours_position[random_neighbour_prev]
C_open_position = Lattice.get_open_coordinate(monomer._open, random_neighbour_prev)
diff = np.array(L_position) - np.array(C_position)
diff = diff.dot(diff)
if diff != 2:
return False
if is_occupied(L_position):
return False
if is_occupied(C_position):
if (sub_case == 0 and C_position != prev_monomer._position) or (sub_case == 1 and C_position != next_monomer._position):
return False
# old_energy
#old_energy = self.box.calculate_chain_energy(self)
# direction L->C
C_nb = self.box.lattice.get_neighbours(C_position)
CL_direction = C_nb.index(L_position)
LC_direction = self.box.lattice.TURN_DIRECTION[CL_direction]
# save old position list
old_position_list = self.positions_list[:]
#old_monomer_position = [ m._position for m in self.chain ]
old_monomer_position = old_position_list
old_monomer_open_position = [ m._open for m in self.chain ]
old_monomer_next_direction = [ m.next_direction for m in self.chain ]
if (sub_case == 0 and prev_monomer._position == C_position) or \
(sub_case == 1 and next_monomer._position == C_position):
valid_configuration = True
monomer.position = L_position
monomer._open = L_open_position
if sub_case == 0:
monomer.next_direction = Lnext_direction
elif sub_case == 1:
monomer.next_direction = LC_direction
#self[monomer_idx] = monomer
monomer.chain = self
self.chain[monomer_idx] = monomer
self.positions_list[monomer_idx] = monomer._position
if sub_case == 0:
prev_monomer.position = C_position
prev_monomer._open = C_open_position
prev_monomer.next_direction = CL_direction
#self[monomer_idx - 1] = prev_monomer
prev_monomer.chain = self
self.chain[monomer_idx - 1] = prev_monomer
self.positions_list[monomer_idx - 1] = prev_monomer._position
elif sub_case == 1:
if not valid_configuration:
next_monomer.position = C_position
next_monomer._open = C_open_position
next_monomer.next_direction = self.box.lattice.TURN_DIRECTION[random_neighbour_prev]
#self[monomer_idx + 1] = next_monomer
next_monomer.chain = self
self.chain[monomer_idx + 1] = next_monomer
self.positions_list[monomer_idx + 1] = next_monomer._position
prev_monomer.next_direction = random_neighbour_prev
#self[monomer_idx - 1] = prev_monomer
prev_monomer.chain = self
self.chain[monomer_idx - 1] = prev_monomer
self.positions_list[monomer_idx - 1] = prev_monomer._position
#mod_pos = 2
if not valid_configuration:
if sub_case == 0: ## right
for j in xrange(monomer_idx - 2, -1, -1):
j_monomer = self.chain[j]
i_monomer = self.chain[j+1]
diff = np.array(j_monomer._open) - np.array(i_monomer._open)
diff = diff.dot(diff)
if diff == 2:
j_monomer.next_direction = j_monomer.neighbours_position.index(i_monomer._position)
break
#mod_pos += 1
j_monomer.position = old_monomer_position[j+2]
j_monomer._open = old_monomer_open_position[j+2]
j_monomer.next_direction = j_monomer.neighbours_position.index(i_monomer._position)
#self[j] = j_monomer
j_monomer.chain = self
self.chain[j] = j_monomer
self.positions_list[j] = j_monomer._position
elif sub_case == 1:
for j in xrange(monomer_idx + 2, self._total_length):
j_monomer = self.chain[j]
i_monomer = self.chain[j-1]
diff = np.array(j_monomer._open) - np.array(i_monomer._open)
diff = diff.dot(diff)
if diff == 2:
i_monomer.next_direction = i_monomer.neighbours_position.index(j_monomer._position)
break
#mod_pos += 1
j_monomer.position = old_monomer_position[j-2]
j_monomer._open = old_monomer_open_position[j-2]
i_monomer.next_direction = i_monomer.neighbours_position.index(j_monomer._position)
#self[j] = j_monomer
j_monomer.chain = self
self.chain[j] = j_monomer
self.positions_list[j] = j_monomer._position
if settings.DEBUG: self.valid_chain()
## end move procedure
# old_energy, emergency case
new_energy = self.box.calculate_chain_energy(self)
dE = new_energy - old_energy
accept_move = False
if dE <= 0.0 or self.box.athermal_state:
accept_move = True
else:
accept_move = self.box.accept(dE, None)
if accept_move is True:
self.total_energy = new_energy
#self.positions_list = [ m._position for m in self.chain ]
self.box.global_energy_update([self.idx])
#f = open("mod_pos_%d" % self._total_length, "w+")
#f.write("%d\n" % mod_pos)
#f.close()
#self.valid_chain()
else: ## revert
if case == 3:
if sub_case == 0:
monomer_list = xrange(0, monomer_idx+2)
elif sub_case == 1:
monomer_list = xrange(monomer_idx-1, self._total_length)
elif case == 0:
monomer_list = xrange(0, self._total_length)
elif case == 1: ##
monomer_list = xrange(0, self._total_length)
#monomer_list = changed_monomer_idx
for idx in monomer_list:
m = self.chain[idx]
old_pos = old_monomer_position[idx]
old_dir = old_monomer_next_direction[idx]
if m._position != old_pos or m.next_direction != old_dir:
m.position = old_monomer_position[idx]
m._open = old_monomer_open_position[idx]
m.next_direction = old_monomer_next_direction[idx]
#self[idx] = m
m.chain = self
self.chain[idx] = m
self.positions_list[idx] = m._position
#self.positions_list = old_position_list[:]
self.total_energy = old_energy
#self.valid_chain()
return accept_move
def run_all(assert_200):
app_id = 'PoolBasedTripletMDS'
num_objects = 30
desired_dimension = 2
x = numpy.linspace(0,1,num_objects)
X_true = numpy.vstack([x,x]).transpose()
total_pulls_per_client = 20000
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 5
pool = Pool(processes=num_clients)
# input test parameters
n = num_objects
d = desired_dimension
delta = 0.01
supported_alg_ids = ['RandomSampling','RandomSampling','UncertaintySampling','CrowdKernel', 'STE']
alg_list = []
for idx,alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
if idx==0:
alg_item['alg_label'] = 'Test'
else:
alg_item['alg_label'] = alg_id
alg_item['test_alg_label'] = 'Test'
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['d'] = d
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['context_type'] = 'text'
initExp_args_dict['args']['context'] = 'Boom baby triplet works'
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
exp_info = []
for ell in range(num_experiments):
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
exp_info.append( {'exp_uid':exp_uid,'exp_key':exp_key} )
#################################################
# Test GET Experiment
#################################################
url = "http://"+HOSTNAME+"/api/experiment/"+exp_uid+"/"+exp_key
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
exp_key = experiment['exp_key']
pool_args.append( (exp_uid,exp_key,participant_uid,total_pulls_per_client,X_true,assert_200) )
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
def _snake(self, move_type):
"""
Snake move
"""
#direction_to_choice = range(0, Lattice.z)
if settings.DEBUG: self.valid_chain()
if move_type == 0:
monomer = self.chain[0]
elif move_type == 1:
monomer = self.chain[-1]
#random new position
random_direction = randrange(0, Lattice.z)
new_position = monomer.neighbours_position[random_direction]
open_position = Lattice.get_open_coordinate(monomer._open, random_direction)
if self.box.is_occupied(new_position) or new_position in self.positions_list:
return False
# copy
old_position_list = self.positions_list[:]
old_open_position_list = [ x._open for x in self.chain ]
old_next_direction_list = [ x.next_direction for x in self.chain ]
old_energy = self.total_energy
self.positions_list = []
"""
Update position of monomers in chain (copy)
"""
if move_type == 0: # left
self.positions_list = [new_position]
for idx in range(1, self._total_length):
self.chain[idx].position = old_position_list[idx - 1]
self.chain[idx]._open = old_open_position_list[idx - 1]
self.chain[idx].next_direction = old_next_direction_list[idx - 1]
self.positions_list += old_position_list[0: self._total_length - 1]
self.chain[0].position = new_position
self.chain[0]._open = open_position
self.chain[0].next_direction = self.box.lattice.TURN_DIRECTION[random_direction]
elif move_type == 1: # right
for idx in range(0, self._total_length - 1):
self.chain[idx].position = old_position_list[idx + 1]
self.chain[idx]._open = old_open_position_list[idx + 1]
self.chain[idx].next_direction = old_next_direction_list[idx + 1]
self.chain[-1].position = new_position
self.chain[-1]._open = open_position
self.chain[-2].next_direction = random_direction
self.positions_list = old_position_list[1:] + [new_position]
new_energy = self.box.calculate_chain_energy(self)
dE = new_energy - old_energy
if dE <= 0.0 or self.box.athermal_state:
if settings.DEBUG: self.valid_chain()
self.total_energy = new_energy
return True
elif self.box.accept(dE, None):
if settings.DEBUG: self.valid_chain()
self.total_energy = new_energy
return True
else:
for idx in xrange(self._total_length):
self.chain[idx].position = old_position_list[idx]
self.chain[idx]._open = old_open_position_list[idx]
self.chain[idx].next_direction = old_next_direction_list[idx]
self.positions_list = old_position_list[:]
self.total_energy = old_energy
if settings.DEBUG: self.valid_chain()
return False
def run_all(assert_200):
def timeit(f):
"""
Utility used to time the duration of code execution. This script can be composed with any other script.
Usage::\n
def f(n):
return n**n
def g(n):
return n,n**n
answer0,dt = timeit(f)(3)
answer1,answer2,dt = timeit(g)(3)
"""
def timed(*args, **kw):
ts = time.time()
result = f(*args, **kw)
te = time.time()
if type(result)==tuple:
return result + ((te-ts),)
else:
return result,(te-ts)
return timed
#HOSTNAME= '52.11.148.214:8001'
app_id = 'TupleBanditsPureExploration'
num_arms = 10
num_objects_to_display = 4
true_means = numpy.array(range(num_arms))/float(num_arms)
total_pulls = 10*num_arms
# total_pulls = 50
# rm = ResourceManager()
# print
# print utils.get_app_about(app_id)
# print
# # get all the relevant algs
# supported_alg_ids = utils.get_app_supported_algs(app_id)
# print
# print "supported_alg_ids : " + str(supported_alg_ids)
# print
supported_alg_ids = ['RandomSampling']
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# input test parameters
n = num_arms
k = num_objects_to_display
delta = 0.01
participants = []
for i in range(10):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
########################
#
# Test initExp
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['k'] = k
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['context_type'] = 'text'
initExp_args_dict['args']['context'] = 'Boom baby tuple works'
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
url = "http://"+HOSTNAME+"/api/experiment/"+exp_uid+"/"+exp_key
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
initExp_response_dict = json.loads(response.text)
# url = "http://"+HOSTNAME+"/widgets/temp-widget-keys"
# args_dict={ 'exp_uid':exp_uid,
# 'exp_key':exp_key,
# 'n':1, #number of widget keys
# 'tries':1000,
# 'duration':10000 }
# print "temp-widget-keys = " + str(args_dict)
# response = requests.post(url, json.dumps(args_dict),headers={'content-type':'application/json'})
# widget_key_dict = json.loads(response.text)
# widget_keys = widget_key_dict['keys']
# print "POST temp-widget-keys response = ", response.text, response.status_code
for t in range(total_pulls):
# time.sleep(.001)
print t
# test getQuery #
#################
getQuery_args_dict = {}
getQuery_args_dict['exp_uid'] = exp_uid
getQuery_args_dict['exp_key'] = exp_key
getQuery_args_dict['args'] = {}
getQuery_args_dict['args']['participant_uid'] = numpy.random.choice(participants)
url = 'http://'+HOSTNAME+'/api/experiment/getQuery'
response,dt = timeit(requests.post)(url, json.dumps(getQuery_args_dict),headers={'content-type':'application/json'})
print "POST getQuery response = ", response.text, response.status_code
print "POST getQuery duration = ", dt
print
query_dict = json.loads(response.text)
print query_dict
query_uid = query_dict['query_uid']
targets = query_dict['target_indices']
# generate simulated reward #
#############################
rewards = numpy.zeros(len(targets)).tolist()
for i, target_index in enumerate(targets):
rewards[i] = true_means[target_index['index']] + numpy.random.randn()*0.5
target_winner = targets[numpy.argmax(rewards)]['target']['target_id']
# test reportAnswer #
#####################
reportAnswer_args_dict = {}
reportAnswer_args_dict["exp_uid"] = exp_uid
reportAnswer_args_dict["exp_key"] = exp_key
reportAnswer_args_dict["args"] = {}
reportAnswer_args_dict["args"]["query_uid"] = query_uid
reportAnswer_args_dict["args"]["target_winner"] = target_winner
url = 'http://'+HOSTNAME+'/api/experiment/reportAnswer'
print "POST reportAnswer args = ", reportAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(reportAnswer_args_dict), headers={'content-type':'application/json'})
print "POST reportAnswer response", response.text, response.status_code
print "POST reportAnswer duration = ", dt
print
reportAnswer_json_response = eval(response.text)
# test getStats #
#################
# args_list = []
# getStats_args_dict = {}
# getStats_args_dict['stat_id'] = 'most_current_ranking'
# getStats_args_dict['params'] = {'alg_label':'RandomSampling'}
# args_list.append(getStats_args_dict)
# getStats_args_dict = {}
# getStats_args_dict["exp_uid"] = exp_uid
# getStats_args_dict["exp_key"] = exp_key
# for args in args_list:
# getStats_args_dict["args"] = args
# url = 'http://'+HOSTNAME+'/api/experiment/stats'
# response = requests.post(url, json.dumps(getStats_args_dict) ,headers={'content-type':'application/json'})
# getStats_json_response = eval(response.text)
# print "/experiment/stats "+args['stat_id'], str(getStats_json_response), response.status_code
# print
print "%s : All tests compeleted successfully" % (app_id)
if __name__ == '__main__':
run_all(False)
def run_all(assert_200):
app_id = "TupleBanditsPureExploration"
num_arms = 20
num_objects_to_display = 4
true_means = numpy.array(range(num_arms)) / float(num_arms)
total_pulls_per_client = 50
num_experiments = 10
# clients run in simultaneous fashion using multiprocessing library
num_clients = 100
pool = Pool(processes=num_clients)
# input test parameters
n = num_arms
k = num_objects_to_display
delta = 0.01
supported_alg_ids = ["RandomSampling"]
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item["alg_id"] = alg_id
alg_item["alg_label"] = alg_id
alg_item["params"] = {}
alg_list.append(alg_item)
params = {}
params["proportions"] = []
for algorithm in alg_list:
params["proportions"].append({"alg_label": algorithm["alg_label"], "proportion": 1.0 / len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings["mode"] = "fixed_proportions"
algorithm_management_settings["params"] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict["args"] = {}
initExp_args_dict["args"]["n"] = n
initExp_args_dict["args"]["k"] = k
initExp_args_dict["args"]["failure_probability"] = delta
initExp_args_dict["args"]["participant_to_algorithm_management"] = "one_to_many" # 'one_to_one' #optional field
initExp_args_dict["args"]["algorithm_management_settings"] = algorithm_management_settings # optional field
initExp_args_dict["args"]["alg_list"] = alg_list # optional field
initExp_args_dict["args"]["instructions"] = "You want instructions, here are your test instructions"
initExp_args_dict["args"]["debrief"] = "You want a debrief, here is your test debrief"
initExp_args_dict["args"]["context_type"] = "text"
initExp_args_dict["args"]["context"] = "Boom baby dueling works"
initExp_args_dict["app_id"] = app_id
initExp_args_dict["site_id"] = "replace this with working site id"
initExp_args_dict["site_key"] = "replace this with working site key"
exp_info = []
for ell in range(num_experiments):
url = "http://" + HOSTNAME + "/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={"content-type": "application/json"})
print "POST initExp response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict["exp_uid"]
exp_key = initExp_response_dict["exp_key"]
exp_info.append({"exp_uid": exp_uid, "exp_key": exp_key})
#################################################
# Test GET Experiment
#################################################
url = "http://" + HOSTNAME + "/api/experiment/" + exp_uid + "/" + exp_key
response = requests.get(url)
print "GET experiment response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = "%030x" % random.randrange(16 ** 30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment["exp_uid"]
exp_key = experiment["exp_key"]
pool_args.append((exp_uid, exp_key, participant_uid, total_pulls_per_client, true_means, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
print(loss_valid_G)
print(acc_valid_G)
alpha = 0.1
beta = 0.5
B = 64
model = train_fully_connected_sgd(Xs_tr, Ys_tr, d1, d2, alpha, beta, B, epochs)
floss,faccuracy = evaluate_model(Xs_te, Ys_te, model[0])
print(floss)
print(faccuracy)
#%%"""P2-3 Random search"""
for i in range(18):
print(i)
alpha = random.randrange(1,11,1)/100
print(alpha)
beta = random.randrange(5,10,1)/10
print(beta)
loss_valid_G = []
acc_valid_G = []
B = random.randrange(64,129,64)
print(B)
model = train_fully_connected_sgd(Xs_tr, Ys_tr, d1, d2, alpha, beta, B, epochs)
floss,faccuracy = evaluate_model(Xs_te, Ys_te, model[0])
temp1 = model[1].history['val_loss']
temp2 = model[1].history['val_accuracy']
loss_valid_G.append(temp1[-1])
acc_valid_G.append(temp2[-1])
print(loss_valid_G)
print(acc_valid_G)
def perform_voting_classification(skClassifiers,
trainSamplesInfo,
imgFileInfo,
classAreaMask,
classMaskPxlVal,
tmpDIR,
tmpImgBase,
outClassImg,
gdalformat='KEA',
numCores=-1):
"""
A function which will perform a number of classification creating a combined classification by a simple vote.
The classifier parameters can be differed as a list of classifiers is provided (the length of the list is equal
to the number of votes), where the training data is resampled for each classifier. The analysis can be performed
using multiple processing cores.
Where:
:param skClassifiers: a list of classifiers (from scikit-learn), the number of classifiers defined
will be equal to the number of votes.
:param trainSamplesInfo: a list of rsgislib.classification.classimgutils.SamplesInfoObj objects used to
parameters the classifer and extract training data.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are
to be used for the classification so it adheres to the training data.
:param classAreaMask: a mask image which is used to specified the areas of the scene which are to be classified.
:param classMaskPxlVal: is the pixel value within the classAreaMask image for the areas of the image
which are to be classified.
:param tmpDIR: a temporary file location which will be created and removed during processing.
:param tmpImgBase: the same name of files written to the tmpDIR
:param outClassImg: the final output image file.
:param gdalformat: the output file format for outClassImg
:param numCores: is the number of processing cores to be used for the analysis (if -1 then all cores on the machine will be used).
Example::
classVoteTemp = os.path.join(imgTmp, 'ClassVoteTemp')
imgFileInfo = [rsgislib.imageutils.ImageBandInfo(img2010dB, 'sardb', [1,2]), rsgislib.imageutils.ImageBandInfo(imgSRTM, 'srtm', [1])]
trainSamplesInfo = []
trainSamplesInfo.append(SamplesInfoObj(className='Water', classID=1, maskImg=classTrainRegionsMask, maskPxlVal=1, outSampImgFile='WaterSamples.kea', numSamps=500, samplesH5File='WaterSamples_pxlvals.h5', red=0, green=0, blue=255))
trainSamplesInfo.append(SamplesInfoObj(className='Land', classID=2, maskImg=classTrainRegionsMask, maskPxlVal=2, outSampImgFile='LandSamples.kea', numSamps=500, samplesH5File='LandSamples_pxlvals.h5', red=150, green=150, blue=150))
trainSamplesInfo.append(SamplesInfoObj(className='Mangroves', classID=3, maskImg=classTrainRegionsMask, maskPxlVal=3, outSampImgFile='MangroveSamples.kea', numSamps=500, samplesH5File='MangroveSamples_pxlvals.h5', red=0, green=153, blue=0))
skClassifiers = []
for i in range(5):
skClassifiers.append(ExtraTreesClassifier(n_estimators=50))
for i in range(5):
skClassifiers.append(ExtraTreesClassifier(n_estimators=100))
for i in range(5):
skClassifiers.append(ExtraTreesClassifier(n_estimators=50, max_depth=2))
for i in range(5):
skClassifiers.append(ExtraTreesClassifier(n_estimators=100, max_depth=2))
mangroveRegionClassImg = MangroveRegionClass.kea
classsklearn.perform_voting_classification(skClassifiers, trainSamplesInfo, imgFileInfo, classWithinMask, 1, classVoteTemp, 'ClassImgSample', mangroveRegionClassImg, gdalformat='KEA', numCores=-1)
"""
def _apply_voting_classifier(inParams):
"""
Internal function which is used by performVotingClassification
"""
skClassifier = inParams['skClassifier']
cTmpDIR = inParams['cTmpDIR']
classAreaMask = inParams['classAreaMask']
classMaskPxlVal = inParams['classMaskPxlVal']
imgFileInfo = inParams['imgFileInfo']
tmpClassImgOut = inParams['tmpClassImgOut']
gdalformat = inParams['gdalformat']
trainSamplesInfo = inParams['trainSamplesInfo']
rndSeed = inParams['rndSeed']
classTrainInfo = dict()
for trainSamples in trainSamplesInfo:
rsgislib.imageutils.performRandomPxlSampleInMaskLowPxlCount(
inputImage=trainSamples.maskImg,
outputImage=os.path.join(cTmpDIR, trainSamples.outSampImgFile),
gdalformat=gdalformat,
maskvals=[trainSamples.maskPxlVal],
numSamples=trainSamples.numSamps,
rndSeed=rndSeed)
rsgislib.imageutils.extractZoneImageBandValues2HDF(
imgFileInfo, os.path.join(cTmpDIR,
trainSamples.outSampImgFile),
os.path.join(cTmpDIR, trainSamples.samplesH5File),
trainSamples.maskPxlVal)
classTrainInfo[trainSamples.className] = ClassSimpleInfoObj(
id=trainSamples.classID,
fileH5=os.path.join(cTmpDIR, trainSamples.samplesH5File),
red=trainSamples.red,
green=trainSamples.green,
blue=trainSamples.blue)
train_sklearn_classifier(classTrainInfo, skClassifier)
apply_sklearn_classifer(classTrainInfo, skClassifier, classAreaMask,
classMaskPxlVal, imgFileInfo, tmpClassImgOut,
gdalformat)
rsgisUtils = rsgislib.RSGISPyUtils()
if type(skClassifiers) is not list:
raise rsgislib.RSGISPyException(
"A list of classifiers must be provided")
numOfVotes = len(skClassifiers)
if numCores <= 0:
numCores = rsgisUtils.numProcessCores()
tmpPresent = True
if not os.path.exists(tmpDIR):
os.makedirs(tmpDIR)
tmpPresent = False
outClassImgs = []
mCoreParams = []
dirs2DEL = []
rndGen = random.seed()
for i in range(numOfVotes):
cTmpDIR = os.path.join(tmpDIR, str(i))
if os.path.exists(cTmpDIR):
shutil.rmtree(cTmpDIR, ignore_errors=True)
os.makedirs(cTmpDIR)
dirs2DEL.append(cTmpDIR)
tmpClassImgOut = os.path.join(tmpDIR,
tmpImgBase + '_' + str(i) + '.kea')
outClassImgs.append(tmpClassImgOut)
inParams = dict()
inParams['skClassifier'] = skClassifiers[i]
inParams['cTmpDIR'] = cTmpDIR
inParams['classAreaMask'] = classAreaMask
inParams['classMaskPxlVal'] = classMaskPxlVal
inParams['imgFileInfo'] = imgFileInfo
inParams['tmpClassImgOut'] = tmpClassImgOut
inParams['gdalformat'] = 'KEA'
inParams['trainSamplesInfo'] = trainSamplesInfo
inParams['rndSeed'] = random.randrange(1000)
mCoreParams.append(inParams)
# Run processing on multiple cores.
mProccesPool = Pool(numCores)
mProccesPool.map(_apply_voting_classifier, mCoreParams)
# Combine results using MODE.
rsgislib.imagecalc.calcMultiImgBandStats(outClassImgs, outClassImg,
rsgislib.SUMTYPE_MODE, gdalformat,
rsgislib.TYPE_8UINT, 0, True)
rsgislib.rastergis.populateStats(clumps=outClassImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
# Colour output classification image.
ratDataset = gdal.Open(outClassImg, gdal.GA_Update)
red = rat.readColumn(ratDataset, 'Red')
green = rat.readColumn(ratDataset, 'Green')
blue = rat.readColumn(ratDataset, 'Blue')
ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255'))
for trainSample in trainSamplesInfo:
print("Apply Colour to class \'" + trainSample.className + "\'")
red[trainSample.classID] = trainSample.red
green[trainSample.classID] = trainSample.green
blue[trainSample.classID] = trainSample.blue
ClassName[trainSample.classID] = trainSample.className
rat.writeColumn(ratDataset, "Red", red)
rat.writeColumn(ratDataset, "Green", green)
rat.writeColumn(ratDataset, "Blue", blue)
rat.writeColumn(ratDataset, "ClassName", ClassName)
ratDataset = None
if not tmpPresent:
shutil.rmtree(tmpDIR, ignore_errors=True)
else:
for cDIR in dirs2DEL:
shutil.rmtree(cDIR, ignore_errors=True)
def run_all(assert_200):
app_id = 'StochasticDuelingBordaBanditsPureExploration'
num_arms = 25
true_means = numpy.array(range(num_arms))/float(num_arms)
total_pulls_per_client = 1000
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 200
pool = Pool(processes=num_clients)
# input test parameters
n = num_arms
delta = 0.05
supported_alg_ids = ['BR_LilUCB','BR_Random','BR_SuccElim','BeatTheMean','BR_Thompson']
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['context_type'] = 'text'
initExp_args_dict['args']['context'] = 'Boom baby dueling works'
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
exp_info = []
for ell in range(num_experiments):
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
exp_info.append( {'exp_uid':exp_uid,'exp_key':exp_key} )
#################################################
# Test GET Experiment
#################################################
url = "http://"+HOSTNAME+"/api/experiment/"+exp_uid+"/"+exp_key
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
exp_key = experiment['exp_key']
pool_args.append( (exp_uid,exp_key,participant_uid,total_pulls_per_client,true_means,assert_200) )
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
def run_all(assert_200):
app_id = "PoolBasedTripletMDS"
num_objects = 30
desired_dimension = 2
x = numpy.linspace(0, 1, num_objects)
X_true = numpy.vstack([x, x]).transpose()
total_pulls_per_client = 20000
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 5
pool = Pool(processes=num_clients)
# input test parameters
n = num_objects
d = desired_dimension
delta = 0.01
supported_alg_ids = ["RandomSampling", "RandomSampling", "UncertaintySampling", "CrowdKernel", "STE"]
alg_list = []
for idx, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item["alg_id"] = alg_id
if idx == 0:
alg_item["alg_label"] = "Test"
else:
alg_item["alg_label"] = alg_id
alg_item["test_alg_label"] = "Test"
alg_item["params"] = {}
alg_list.append(alg_item)
params = {}
params["proportions"] = []
for algorithm in alg_list:
params["proportions"].append({"alg_label": algorithm["alg_label"], "proportion": 1.0 / len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings["mode"] = "fixed_proportions"
algorithm_management_settings["params"] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict["args"] = {}
initExp_args_dict["args"]["n"] = n
initExp_args_dict["args"]["d"] = d
initExp_args_dict["args"]["failure_probability"] = delta
initExp_args_dict["args"]["participant_to_algorithm_management"] = "one_to_many" # 'one_to_one' #optional field
initExp_args_dict["args"]["algorithm_management_settings"] = algorithm_management_settings # optional field
initExp_args_dict["args"]["alg_list"] = alg_list # optional field
initExp_args_dict["args"]["instructions"] = "You want instructions, here are your test instructions"
initExp_args_dict["args"]["debrief"] = "You want a debrief, here is your test debrief"
initExp_args_dict["args"]["context_type"] = "text"
initExp_args_dict["args"]["context"] = "Boom baby triplet works"
initExp_args_dict["app_id"] = app_id
initExp_args_dict["site_id"] = "replace this with working site id"
initExp_args_dict["site_key"] = "replace this with working site key"
exp_info = []
for ell in range(num_experiments):
url = "http://" + HOSTNAME + "/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={"content-type": "application/json"})
print "POST initExp response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict["exp_uid"]
exp_key = initExp_response_dict["exp_key"]
exp_info.append({"exp_uid": exp_uid, "exp_key": exp_key})
#################################################
# Test GET Experiment
#################################################
url = "http://" + HOSTNAME + "/api/experiment/" + exp_uid + "/" + exp_key
response = requests.get(url)
print "GET experiment response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = "%030x" % random.randrange(16 ** 30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment["exp_uid"]
exp_key = experiment["exp_key"]
pool_args.append((exp_uid, exp_key, participant_uid, total_pulls_per_client, X_true, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
def test_api(assert_200=True, num_arms=5,
num_experiments=1, num_clients=10, total_pulls=5):
app_id = 'CardinalBanditsPureExploration'
true_means = numpy.array(range(num_arms)[::-1])/float(num_arms)
pool = Pool(processes=num_clients)
# input test parameters
n = num_arms
delta = 0.05
supported_alg_ids = ['LilUCB', 'RoundRobin']
labels = [{'label':'bad','reward':1.},{'label':'neutral','reward':2.},{'label':'good','reward':3.}]
alg_list = []
for i, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id+'_'+str(i)
#alg_item['params'] = {}
alg_list.append(alg_item)
params = []
#params['proportions'] = []
for algorithm in alg_list:
params.append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
print "alg mangement settings", algorithm_management_settings
# Test POST Experiment
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['targets'] = {'n':n}
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['context_type'] = 'text'
initExp_args_dict['args']['context'] = 'This is a context'
initExp_args_dict['args']['rating_scale'] = {'labels':labels}
# initExp_args_dict['args']['HAHA'] = {'labels':labels}
initExp_args_dict['app_id'] = app_id
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_uid = test_utils.initExp(initExp_args_dict)
exp_info += [exp_uid]
exp_uid = initExp_response_dict['exp_uid']
exp_info += [{'exp_uid':exp_uid,}]
# Test GET Experiment
initExp_response_dict = test_utils.getExp(exp_uid)
# Generate participants
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid,participant_uid,total_pulls,true_means,assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def test_api(assert_200=True,
num_objects=5,
desired_dimension=2,
total_pulls_per_client=4,
num_experiments=1,
num_clients=6):
x = numpy.linspace(0, 1, num_objects)
X_true = numpy.vstack([x, x]).transpose()
pool = Pool(processes=num_clients)
supported_alg_ids = [
'CrowdKernel', 'RandomSampling', 'UncertaintySampling',
'ValidationSampling', 'STE'
]
alg_list = []
for idx, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
if alg_id == 'ValidationSampling':
alg_item['alg_label'] = 'Test'
alg_item['params'] = {
'query_list': [[q1, q2, q3] for q1 in [0, 1, 2]
for q2 in [0, 1, 2] for q3 in [0, 1, 2]]
}
else:
alg_item['alg_label'] = alg_id
alg_item['test_alg_label'] = 'Test'
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({
'alg_label': algorithm['alg_label'],
'proportion': 1. / len(alg_list)
})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# Test POST Experiment
initExp_args_dict = {}
initExp_args_dict['app_id'] = 'PoolBasedTripletMDS'
initExp_args_dict['args'] = {}
initExp_args_dict['args']['d'] = desired_dimension
initExp_args_dict['args']['failure_probability'] = 0.01
initExp_args_dict['args'][
'participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args'][
'algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args'][
'instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args'][
'debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['targets'] = {}
initExp_args_dict['args']['targets']['n'] = num_objects
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_uid = test_utils.initExp(initExp_args_dict)
exp_info += [exp_uid]
# Generate participants
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid, participant_uid, total_pulls_per_client,
X_true, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def test_api(assert_200=True,
num_objects=6,
total_pulls_per_client=5,
num_experiments=1,
num_clients=2):
pool = Pool(processes=num_clients)
supported_alg_ids = ['RoundRobin']
alg_list = []
for idx, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
if idx == 0:
alg_item['alg_label'] = 'Test'
else:
alg_item['alg_label'] = alg_id
alg_item['test_alg_label'] = 'Test'
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({
'alg_label': algorithm['alg_label'],
'proportion': 1. / len(alg_list)
})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
targetset = []
for i in range(num_objects):
features = list(numpy.random.randn(6))
targetset.append({
'primary_description': str(features),
'primary_type': 'text',
'alt_description': '%d' % (i),
'alt_type': 'text',
'target_id': str(i),
'meta': {
'features': features
}
})
# Test POST Experiment
print '\n' * 2 + 'Testing POST initExp...'
initExp_args_dict = {}
initExp_args_dict['app_id'] = app_id
initExp_args_dict['args'] = {}
initExp_args_dict['args'][
'participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args'][
'algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args'][
'instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args'][
'debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['targets'] = {'targetset': targetset}
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_info_ = test_utils.initExp(
initExp_args_dict)
exp_info += [exp_info_]
exp_uid = initExp_response_dict['exp_uid']
exp_info.append({
'exp_uid': exp_uid,
})
# Test GET Experiment
initExp_response_dict = test_utils.getExp(exp_uid)
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append((exp_uid, participant_uid, total_pulls_per_client,
true_weights, assert_200))
print "participants are", participants
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def treatments_player(self):
# create the dictionary with the variables
treatments_dic = {
'original_trial_num': [],
'cluster': [],
'last_fix_condition': [],
'first_fix_value': [],
'last_fix_value': [],
'number_of_fixations': [],
'lose_value': [],
'gain_value': [],
'gain_condition': [],
'loss_condition': [],
'lose_times': [],
'gain_times': [],
}
clusters = [
0, 1
] # 0 - fast scanning (1 or 2 fixations), 1 - long fixations (3,4 or 5 fix)
last_fix_conditions = [-1, 0, 1] # -1 - shorter, 0 - equal, 1 - longer
low_losses = list(range(-13, -21, -1))
high_losses = list(range(-21, -28, -1))
losses_lists = [high_losses, low_losses]
low_gains = list(range(20, 29, 1))
high_gains = list(range(29, 39, 1))
gains_lists = [low_gains, high_gains]
last_fixes = ['gain', 'loss']
trial_time_clus0 = 1.365
trial_time_clus1 = 2.110
# what is (in %) the last fixation difference is?
last_fix_larger_by = .25
count = 1
# use for loop to create all combinations of treatments
for clus in clusters:
for last_fix_condition in last_fix_conditions:
for gains in gains_lists:
for losses in losses_lists:
for last_fix_val in last_fixes:
gain = random.choice(gains)
lose = random.choice(losses)
treatments_dic['original_trial_num'].append(count)
treatments_dic['cluster'].append(clus)
treatments_dic['last_fix_condition'].append(
last_fix_condition)
treatments_dic['lose_value'].append(lose)
treatments_dic['gain_value'].append(gain)
if gains == low_gains:
treatments_dic['gain_condition'].append(
'low_gains')
elif gains == high_gains:
treatments_dic['gain_condition'].append(
'high_gains')
if losses == high_losses:
treatments_dic['loss_condition'].append(
'high_losses')
elif losses == low_losses:
treatments_dic['loss_condition'].append(
'low_losses')
if clus == 0:
fix_num = 2
treatments_dic['number_of_fixations'].append(2)
time_per_fix = trial_time_clus0 / fix_num
elif clus == 1:
# define how many fixations will there be
fix_num = random.randint(3, 5)
treatments_dic['number_of_fixations'].append(
fix_num)
time_per_fix = trial_time_clus1 / fix_num
# compute lose and gain times
time_per_fix_larger = time_per_fix * (
1 + last_fix_larger_by)
time_per_fix_smaller = time_per_fix * (
1 - last_fix_larger_by)
lose_times = []
gain_times = []
first_value = None
# define whether gains or losses are shown last
treatments_dic['last_fix_value'].append(
last_fix_val)
# now define which value will be shown first
if (fix_num % 2 == 0) & (last_fix_val == 'gain'):
first_value = 'loss'
elif (fix_num % 2 == 0) & (last_fix_val == 'loss'):
first_value = 'gain'
elif (fix_num % 2 != 0) & (last_fix_val == 'gain'):
first_value = 'gain'
elif (fix_num % 2 != 0) & (last_fix_val == 'loss'):
first_value = 'loss'
treatments_dic['first_fix_value'].append(
first_value)
# create arrays that will include all fixation times
for time_ind in range(fix_num):
# check if it's a first fixation to mitigate the reaction delay
if time_ind == 0:
time_per_fix = time_per_fix + .165
if (first_value == 'gain') & (
(time_ind % 2) == 0):
gain_times.append(time_per_fix)
elif (first_value == 'loss') & (
(time_ind % 2) == 0):
lose_times.append(time_per_fix)
elif (first_value == 'gain') & (
(time_ind % 2) != 0):
lose_times.append(time_per_fix)
elif (first_value == 'loss') & (
(time_ind % 2) != 0):
gain_times.append(time_per_fix)
# substract the error time if this was the first fixation
if time_ind == 0:
time_per_fix = time_per_fix - .165
if time_ind == fix_num - 1:
if last_fix_condition == 1:
if last_fix_val == 'gain':
gain_times[
-1] = time_per_fix_larger
elif last_fix_val == 'loss':
lose_times[
-1] = time_per_fix_larger
elif last_fix_condition == -1:
if last_fix_val == 'gain':
gain_times[
-1] = time_per_fix_smaller
elif last_fix_val == 'loss':
lose_times[
-1] = time_per_fix_smaller
# add the fixation times to the dictionary
treatments_dic['lose_times'].append(lose_times)
treatments_dic['gain_times'].append(gain_times)
count += 1
treatments_df = pd.DataFrame(treatments_dic)
# randomize the table using random-generated number (which will be the same for all trials for a specific participant)
randomized = treatments_df.sample(
frac=1, random_state=self.in_round(1).rand_int)
# re-index the new table in order so we could present the new randomized table from start to the end
randomized['new_indexing'] = list(range(0, len(randomized)))
randomized = randomized.set_index(randomized['new_indexing'])
# Check whether the trials are practice to decide whether to show random or
# ordered rows from the randomized table
pt = self.practice_trials()
if pt == 0:
row_number = self.round_number - 1 - Constants.num_practice_rounds
elif pt == 1:
row_number = random.randrange(0, len(randomized))
# write down the data for the participant for each row so we can see it during the data analysis
self.original_trial_num = randomized.loc[row_number,
'original_trial_num']
self.cluster = randomized.loc[row_number, 'cluster']
self.lose_value = randomized.loc[row_number, 'lose_value']
self.gain_value = randomized.loc[row_number, 'gain_value']
self.last_fix_condition = randomized.loc[row_number,
'last_fix_condition']
self.first_fix_value = randomized.loc[row_number, 'first_fix_value']
self.last_fix_value = randomized.loc[row_number, 'last_fix_value']
self.number_of_fixations = randomized.loc[row_number,
'number_of_fixations']
self.row_number = row_number
self.gain_condition = randomized.loc[row_number, 'gain_condition']
self.loss_condition = randomized.loc[row_number, 'loss_condition']
# Create distinct variables for all fix times
row_gain_times = randomized.loc[row_number, 'gain_times']
row_lose_times = randomized.loc[row_number, 'lose_times']
for index in range(1, 6):
var = 'lose_time_' + str(index)
if len(row_lose_times) < index:
lose_fix_time = 0
else:
lose_fix_time = row_lose_times[index - 1]
exec("self.%s = %f" % (var, lose_fix_time))
var = 'gain_time_' + str(index)
if len(row_gain_times) < index:
gain_fix_time = 0
else:
gain_fix_time = row_gain_times[index - 1]
exec("self.%s = %f" % (var, gain_fix_time))
return randomized
def run_all(assert_200):
app_id = 'CardinalBanditsPureExploration'
num_arms = 50
true_means = numpy.array(range(num_arms))/float(num_arms)
total_pulls_per_client = 30
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 200
pool = Pool(processes=num_clients)
# input test parameters
n = num_arms
delta = 0.05
R = 2 # assumes scores in range [1,5]
# supported_alg_ids = ['SimpleUCB','AlternativeUCB']
supported_alg_ids = ['RandomSampling','LUCB','LilUCB','RoundRobin']
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['R'] = R
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['labels'] = {'1':1, '2':2, '3':3}
initExp_args_dict['app_id'] = app_id
exp_info = []
for ell in range(num_experiments):
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
exp_info.append( {'exp_uid':exp_uid,'exp_key':exp_key} )
#################################################
# Test GET Experiment
#################################################
url = "http://"+HOSTNAME+"/api/experiment/"+exp_uid+"/"+exp_key
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
exp_key = experiment['exp_key']
pool_args.append( (exp_uid,exp_key,participant_uid,total_pulls_per_client,true_means,assert_200) )
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
def test_api(assert_200=True, num_objects=5, desired_dimension=2,
total_pulls_per_client=4, num_experiments=1, num_clients=6):
x = numpy.linspace(0,1,num_objects)
X_true = numpy.vstack([x,x]).transpose()
pool = Pool(processes=num_clients)
supported_alg_ids = ['CrowdKernel', 'RandomSampling',
'UncertaintySampling', 'ValidationSampling', 'STE']
alg_list = []
for idx, alg_id in enumerate(supported_alg_ids):
alg_item = {}
alg_item['alg_id'] = alg_id
if alg_id == 'ValidationSampling':
alg_item['alg_label'] = 'Test'
alg_item['params'] = {'query_list': [
[q1, q2, q3] for q1 in [0, 1, 2]
for q2 in [0, 1, 2]
for q3 in [0, 1, 2]
]}
else:
alg_item['alg_label'] = alg_id
alg_item['test_alg_label'] = 'Test'
alg_list.append(alg_item)
params = []
for algorithm in alg_list:
params.append({'alg_label': algorithm['alg_label'],
'proportion': 1./len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# Test POST Experiment
initExp_args_dict = {}
initExp_args_dict['app_id'] = 'PoolBasedTripletMDS'
initExp_args_dict['args'] = {}
initExp_args_dict['args']['d'] = desired_dimension
initExp_args_dict['args']['failure_probability'] = 0.01
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['targets'] = {}
initExp_args_dict['args']['targets']['n'] = num_objects
exp_info = []
for ell in range(num_experiments):
initExp_response_dict, exp_uid = test_utils.initExp(initExp_args_dict)
exp_info += [exp_uid]
# Generate participants
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment['exp_uid']
pool_args.append( (exp_uid,participant_uid,total_pulls_per_client,X_true,assert_200) )
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list)
def test_geometric_spanner_condition_Delaunay_triangulation_sphere_surface(
self):
'''The geometric spanner condition (http://en.wikipedia.org/wiki/Delaunay_triangulation#Properties) indicates that the length of the shortest edge-traveled path between two nodes in a Delaunay triangulation is no longer than 2.42 times the straight-line Euclidean distance between them.'''
#create a networkx graph object of the Delaunay triangulation vertices & edges
voronoi_instance_small = voronoi_utility.Voronoi_Sphere_Surface(
self.simple_sphere_coordinate_array)
Delaunay_point_array_small = voronoi_instance_small.delaunay_triangulation_spherical_surface(
) #should be shape (N,3,3) for N triangles and their vertices in 3D space
node_dictionary = {}
node_counter = 1
list_nodes_identified_debug = []
#assign an integer node (vertex) number to each unique coordinate on the test sphere:
for node_coordinate in self.simple_sphere_coordinate_array:
node_dictionary[node_counter] = node_coordinate
node_counter += 1
#print 'self.simple_sphere_coordinate_array:', self.simple_sphere_coordinate_array
#print 'self.simple_sphere_coordinate_array.shape:', self.simple_sphere_coordinate_array.shape
#print 'node_dictionary.values():', node_dictionary.values() #there seem to be multiple duplicates / rounding variations for the polar point at [0. 0. 2.]
def identify_node_based_on_coordinate(coordinate_array,
node_dictionary):
'''Return the node number based on the coordinates in the original test sphere data set.'''
nodenum = 0
num_positives_debug = 0
for node_number, node_coordinates in node_dictionary.iteritems():
if numpy.allclose(node_coordinates,
coordinate_array,
atol=1e-18):
nodenum = node_number
num_positives_debug += 1
#if num_positives_debug > 1:
#print 'duplicate offender:', node_coordinates, coordinate_array
#else:
#print 'original match:', node_coordinates, coordinate_array
assert num_positives_debug == 1, "Only a single node should correspond to the input coordinates."
return nodenum
def produce_networkx_edges_from_triangle_data(triangle_array_data,
node_dictionary):
'''Input should be shape (3,3) array of coordinate data for a Delaunay triangle.'''
list_networkx_edge_tuples = []
#each triangle will, of course, have 3 edges
triangle_array_row_indices_for_edge_vertices = [[0, 1], [1, 2],
[2, 0]]
for triangle_row_indices_of_edge in triangle_array_row_indices_for_edge_vertices:
first_triangle_row_index, second_triangle_row_index = triangle_row_indices_of_edge
first_vertex_coord = triangle_array_data[
first_triangle_row_index]
second_vertex_coord = triangle_array_data[
second_triangle_row_index]
graph_node_number_first_vertex_current_edge = identify_node_based_on_coordinate(
first_vertex_coord, node_dictionary)
graph_node_number_second_vertex_current_edge = identify_node_based_on_coordinate(
second_vertex_coord, node_dictionary)
list_nodes_identified_debug.extend(
[
graph_node_number_first_vertex_current_edge,
graph_node_number_second_vertex_current_edge
]
) #missing nodes with debug list growth here, but no missing nodes if I grow the debug list from within the identify_node_based_on_coordinate() function itself; why????
#the edge weight for networkx should be the Euclidean straight-line distance between the vertices
weight = scipy.spatial.distance.euclidean(
first_vertex_coord, second_vertex_coord)
networkx_edge_tuple = (
graph_node_number_first_vertex_current_edge,
graph_node_number_second_vertex_current_edge, weight)
list_networkx_edge_tuples.append(networkx_edge_tuple)
return list_networkx_edge_tuples
#build networkx graph object from Delaunay triangles (simplices)
G = nx.Graph()
triangle_counter = 0
for triangle_coord_array in Delaunay_point_array_small:
current_list_networkx_edge_tuples = produce_networkx_edges_from_triangle_data(
triangle_coord_array, node_dictionary)
#print 'current_list_networkx_edge_tuples:', current_list_networkx_edge_tuples
G.add_weighted_edges_from(current_list_networkx_edge_tuples
) #duplicates will simply be updated
triangle_counter += 1
#print 'Triangle:', triangle_counter, 'total edges:', G.size(), 'total nodes:', G.order()
#print 'size:', G.size()
#print 'list of edges:', G.edges()
#print 'num nodes:', len(G.nodes())
#print 'dict size:', len(node_dictionary)
#print 'dict keys:', node_dictionary.keys()
#print 'nodes:', G.nodes()
#print 'edges:', G.edges()
#print 'ordered set nodes identified:', set(sorted(list_nodes_identified_debug))
self.assertEqual(
len(G), self.num_triangulation_input_points
) #obviously, the number of nodes in the graph should match the number of points on the sphere
#perform the geometric spanner test for a random subset of nodes:
num_tests = 0
while num_tests < 10: #do 10 random tests
first_node_number = random.randrange(1, len(G), 1)
second_node_number = random.randrange(1, len(G), 1)
minimum_distance_between_nodes_following_Delaunay_edges = nx.dijkstra_path_length(
G, first_node_number, second_node_number)
#compare with straight line Euclidean distance:
first_node_coordinate = node_dictionary[first_node_number]
second_node_coordinate = node_dictionary[second_node_number]
Euclidean_distance = scipy.spatial.distance.euclidean(
first_node_coordinate, second_node_coordinate)
self.assertLess(
minimum_distance_between_nodes_following_Delaunay_edges /
Euclidean_distance, 2.42) #the geometric spanner condition
num_tests += 1
def run_all(assert_200):
def timeit(f):
"""
Utility used to time the duration of code execution. This script can be composed with any other script.
Usage::\n
def f(n):
return n**n
def g(n):
return n,n**n
answer0,dt = timeit(f)(3)
answer1,answer2,dt = timeit(g)(3)
"""
def timed(*args, **kw):
ts = time.time()
result = f(*args, **kw)
te = time.time()
if type(result)==tuple:
return result + ((te-ts),)
else:
return result,(te-ts)
return timed
app_id = 'DuelingBanditsPureExploration'
# assert_200 = False
num_arms = 10
true_means = numpy.array(range(num_arms))/float(num_arms)
total_pulls = num_arms*10
# total_pulls = 50
# rm = ResourceManager()
# print
# print utils.get_app_about(app_id)
# print
# # get all the relevant algs
# supported_alg_ids = utils.get_app_supported_algs(app_id)
# print
# print "supported_alg_ids : " + str(supported_alg_ids)
# print
supported_alg_ids = ['BR_LilUCB','BR_Random','BR_SuccElim','BeatTheMean']
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item['alg_id'] = alg_id
alg_item['alg_label'] = alg_id
alg_item['params'] = {}
alg_list.append(alg_item)
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list) } )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
# input test parameters
n = num_arms
delta = 0.01
participants = []
for i in range(10):
participant_uid = '%030x' % random.randrange(16**30)
participants.append(participant_uid)
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict['args'] = {}
initExp_args_dict['args']['n'] = n
initExp_args_dict['args']['failure_probability'] = delta
initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' # 'one_to_one' #optional field
initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings #optional field
initExp_args_dict['args']['alg_list'] = alg_list #optional field
initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions'
initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief'
initExp_args_dict['args']['context_type'] = 'text'
initExp_args_dict['args']['context'] = 'Boom baby dueling works'
initExp_args_dict['app_id'] = app_id
initExp_args_dict['site_id'] = 'replace this with working site id'
initExp_args_dict['site_key'] = 'replace this with working site key'
url = "http://"+HOSTNAME+"/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={'content-type':'application/json'})
print "POST initExp response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict['exp_uid']
exp_key = initExp_response_dict['exp_key']
#################################################
# Test GET Experiment
#################################################
url = "http://"+HOSTNAME+"/api/experiment/"+exp_uid+"/"+exp_key
response = requests.get(url)
print "GET experiment response =",response.text, response.status_code
if assert_200: assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
# url = "http://"+HOSTNAME+"/widgets/temp-widget-keys"
# args_dict={ 'exp_uid':exp_uid,
# 'exp_key':exp_key,
# 'n':1, #number of widget keys
# 'tries':1000,
# 'duration':10000 }
# print "temp-widget-keys = " + str(args_dict)
# response = requests.post(url, json.dumps(args_dict),headers={'content-type':'application/json'})
# widget_key_dict = json.loads(response.text)
# widget_keys = widget_key_dict['keys']
# print "POST temp-widget-keys response = ", response.text, response.status_code
# url = "http://"+HOSTNAME+"/widgets/getwidget"
# args_dict={ 'name':'getQuery',
# 'exp_uid':exp_uid,
# 'app_id':app_id,
# 'widget_key':widget_keys[0],
# 'args':{}}
# print "getwidget args = " + str(args_dict)
# response = requests.post(url, json.dumps(args_dict),headers={'content-type':'application/json'})
# print "POST getwidget response = ", response.text, response.status_code
# response = requests.get(url)
for t in range(total_pulls):
# time.sleep(.001)
print t
#######################################
# test POST getQuery #
#######################################
getQuery_args_dict = {}
getQuery_args_dict['exp_uid'] = exp_uid
getQuery_args_dict['exp_key'] = exp_key
getQuery_args_dict['args'] = {}
getQuery_args_dict['args']['participant_uid'] = numpy.random.choice(participants)
url = 'http://'+HOSTNAME+'/api/experiment/getQuery'
response,dt = timeit(requests.post)(url, json.dumps(getQuery_args_dict),headers={'content-type':'application/json'})
print "POST getQuery response = ", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST getQuery duration = ", dt
print
ts = time.time()
query_dict = json.loads(response.text)
query_uid = query_dict['query_uid']
targets = query_dict['target_indices']
for target in targets:
if target['label'] == 'left':
index_left = target['index']
if target['label'] == 'right':
index_right = target['index']
if target['flag'] == 1:
index_painted = target['index']
# generate simulated reward #
#############################
reward_left = true_means[index_left] + numpy.random.randn()*0.5
reward_right = true_means[index_right] + numpy.random.randn()*0.5
if reward_left>reward_right:
index_winner = index_left
else:
index_winner = index_right
response_time = time.time() - ts
#############################################
# test POST processAnswer
#############################################
processAnswer_args_dict = {}
processAnswer_args_dict["exp_uid"] = exp_uid
processAnswer_args_dict["exp_key"] = exp_key
processAnswer_args_dict["args"] = {}
processAnswer_args_dict["args"]["query_uid"] = query_uid
processAnswer_args_dict["args"]['target_winner'] = index_winner
processAnswer_args_dict["args"]['response_time'] = response_time
url = 'http://'+HOSTNAME+'/api/experiment/processAnswer'
print "POST processAnswer args = ", processAnswer_args_dict
response,dt = timeit(requests.post)(url, json.dumps(processAnswer_args_dict), headers={'content-type':'application/json'})
print "POST processAnswer response", response.text, response.status_code
if assert_200: assert response.status_code is 200
print "POST processAnswer duration = ", dt
print
processAnswer_json_response = eval(response.text)
# # test predict #
# ################
# # get stateless app
# app = utils.get_app(app_id)
# # convert python dictionary to json dictionary
# predict_id = 'arm_ranking'
# params = {'alg_label':alg_label}
# predict_args_dict = {'predict_id':predict_id,'params':params}
# predict_args_json = json.dumps(predict_args_dict)
# print "predict_args_json = " + str(predict_args_json)
# predict_json,didSucceed,message = app.predict(exp_uid=exp_uid,args_json=predict_args_json)
# print "predict_response_json = " + str(predict_json)
# if not didSucceed:
# raise Exception(message)
# # print "iter %d : pulled arm %d, prediction = %s" % (t,index,predict_json)
# # convert json dictionary to python dictionary
# predict_dict = json.loads(predict_json)
# test getStats #
################
# get stateless app
# stat_list = rm.get_app_supported_stats(app_id)
# alg_dict_list = rm.get_algs_for_exp_uid(exp_uid)
# alg_label_list = [x['alg_label'] for x in alg_dict_list]
# args_list = []
# for stat in stat_list:
# stat_id = stat['stat_id']
# necessary_params = stat['necessary_params']
# if ('alg_label' in necessary_params) and ('task' in necessary_params):
# for task in ['getQuery','processAnswer','predict']:
# for alg_label in alg_label_list:
# getStats_dict = {}
# getStats_dict['stat_id'] = stat_id
# getStats_dict['params'] = {'alg_label':alg_label,'task':task}
# getStats_args_json = json.dumps(getStats_dict)
# args_list.append(getStats_args_json)
# elif ('alg_label' in necessary_params):
# for alg_label in alg_label_list:
# getStats_dict = {}
# getStats_dict['stat_id'] = stat_id
# getStats_dict['params'] = {'alg_label':alg_label}
# getStats_args_json = json.dumps(getStats_dict)
# args_list.append(getStats_args_json)
# elif ('task' in necessary_params):
# for task in ['getQuery','processAnswer','predict']:
# getStats_dict = {}
# getStats_dict['stat_id'] = stat_id
# getStats_dict['params'] = {'task':task}
# getStats_args_json = json.dumps(getStats_dict)
# args_list.append(getStats_args_json)
# else:
# getStats_dict = {}
# getStats_dict['stat_id'] = stat_id
# getStats_dict['params'] = {}
# getStats_args_json = json.dumps(getStats_dict)
# args_list.append(getStats_args_json)
# # get stateless app
# app = utils.get_app(app_id)
# for getStats_args_json in args_list:
# print
# print
# print "getStats_args_json = " + str(getStats_args_json)
# getStats_response_json,didSucceed,message = app.getStats(exp_uid=exp_uid,args_json=getStats_args_json)
# # print "getStats_response_json = "
# print getStats_response_json
# if not didSucceed:
# raise Exception(message)
############################################
# test POST stats
###########################################
args_list = []
getStats_args_dict = {}
getStats_args_dict['stat_id'] = 'most_current_ranking'
getStats_args_dict['params'] = {'alg_label':'BR_LilUCB'}
args_list.append(getStats_args_dict)
getStats_args_dict = {}
getStats_args_dict["exp_uid"] = exp_uid
getStats_args_dict["exp_key"] = exp_key
for args in args_list:
getStats_args_dict["args"] = args
url = 'http://'+HOSTNAME+'/api/experiment/stats'
response = requests.post(url, json.dumps(getStats_args_dict) ,headers={'content-type':'application/json'})
getStats_json_response = eval(response.text)
print "/experiment/stats "+args['stat_id'], str(getStats_json_response), response.status_code
if assert_200: assert response.status_code is 200
print
url = 'http://'+HOSTNAME+'/api/experiment/'+exp_uid+'/'+exp_key+'/participants'
response = requests.get(url)
participants_response = eval(response.text)
print 'participants_response = ' + str(participants_response)
print "%s : All tests compeleted successfully" % (app_id)
import numpy
import numpy.random
import random
from utils.prog_bar import ProgBar
from dpgmm import DPGMM
# Parameters...
trainCount = 1024
testCount = 256
# Generate a random set of Gaussians to test with...
print "Generating model..."
count = random.randrange(2, 6)
mix = numpy.random.rand(count) + 1.0
mix /= mix.sum()
mean = numpy.random.randint(-2, 3, (count, 3))
sd = 0.4 * numpy.random.rand(count) + 0.1
for i in xrange(count):
print "%i: mean = %s; sd = %f" % (i, str(mean[i, :]), sd[i])
# Draw two sets of samples from them...
print "Generating data..."
train = []
for _ in xrange(trainCount):
which = numpy.random.multinomial(1, mix).argmax()
covar = sd[which] * numpy.identity(3)
def run_all(assert_200):
num_examples = 1000
X = numpy.random.randn(num_examples, 3)
app_id = "PoolBasedBinaryClassification"
true_means = numpy.tanh(X[:, 0])
total_pulls_per_client = 100
num_experiments = 1
# clients run in simultaneous fashion using multiprocessing library
num_clients = 10
pool = Pool(processes=num_clients)
# input test parameters
delta = 0.05
example_pool = X.tolist()
test_labels = [[i, numpy.sign(X[i, 0])] for i in range(num_examples)]
supported_alg_ids = ["RandomSamplingLinearLeastSquares"]
alg_list = []
for alg_id in supported_alg_ids:
alg_item = {}
alg_item["alg_id"] = alg_id
alg_item["alg_label"] = alg_id
alg_item["params"] = {}
alg_list.append(alg_item)
params = {}
params["proportions"] = []
for algorithm in alg_list:
params["proportions"].append({"alg_label": algorithm["alg_label"], "proportion": 1.0 / len(alg_list)})
algorithm_management_settings = {}
algorithm_management_settings["mode"] = "fixed_proportions"
algorithm_management_settings["params"] = params
#################################################
# Test POST Experiment
#################################################
initExp_args_dict = {}
initExp_args_dict["args"] = {}
initExp_args_dict["args"]["example_pool"] = example_pool
initExp_args_dict["args"]["failure_probability"] = delta
initExp_args_dict["args"]["test_labels"] = test_labels
initExp_args_dict["args"]["participant_to_algorithm_management"] = "one_to_many" # 'one_to_one' #optional field
initExp_args_dict["args"]["algorithm_management_settings"] = algorithm_management_settings # optional field
initExp_args_dict["args"]["alg_list"] = alg_list # optional field
initExp_args_dict["args"]["instructions"] = "Please answer the yes or no question to the best of your ability."
initExp_args_dict["args"]["debrief"] = "Thank you for participating"
initExp_args_dict["args"]["context"] = "Is the target a positive example?"
initExp_args_dict["args"]["context_type"] = "text"
initExp_args_dict["app_id"] = app_id
initExp_args_dict["site_id"] = "replace this with working site id"
initExp_args_dict["site_key"] = "replace this with working site key"
exp_info = []
for ell in range(num_experiments):
url = "http://" + HOSTNAME + "/api/experiment"
response = requests.post(url, json.dumps(initExp_args_dict), headers={"content-type": "application/json"})
print "POST initExp response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
exp_uid = initExp_response_dict["exp_uid"]
exp_key = initExp_response_dict["exp_key"]
exp_info.append({"exp_uid": exp_uid, "exp_key": exp_key})
#################################################
# Test GET Experiment
#################################################
url = "http://" + HOSTNAME + "/api/experiment/" + exp_uid + "/" + exp_key
response = requests.get(url)
print "GET experiment response =", response.text, response.status_code
if assert_200:
assert response.status_code is 200
initExp_response_dict = json.loads(response.text)
###################################
# Generate participants
###################################
participants = []
pool_args = []
for i in range(num_clients):
participant_uid = "%030x" % random.randrange(16 ** 30)
participants.append(participant_uid)
experiment = numpy.random.choice(exp_info)
exp_uid = experiment["exp_uid"]
exp_key = experiment["exp_key"]
pool_args.append((exp_uid, exp_key, participant_uid, total_pulls_per_client, true_means, assert_200))
results = pool.map(simulate_one_client, pool_args)
for result in results:
print result
