def createReadLibrary(df, reads_file_1, reads_file_2, args):
'''loop through a dataframe containing sequences (ref and alt alleles)
and expression levels (ref and alt counts) and create two dictionaries of
simulated NGS reads (fwd and reverse read pairs)
'''
READ_COUNTER = 0
args.read_target = np.sum(df.ref_read_count) + np.sum(df.alt_read_count)
insert_distribution = [int(i) for i in list(stats.skewnorm.rvs(
a=args.skew,
size=args.read_target+10000, # add a buffer in case of rounding errors etc.
loc=args.insert_mean,
scale=args.insert_sd,
))]
random.shuffle(insert_distribution)
args.insert_distribution = RandomDict([(k,v) for k,v in enumerate(insert_distribution)])
args.insert_distribution_min = min(insert_distribution)
args.qual_scores = RandomDict([(k,v) for k,v in enumerate(readQualscores(args))])
# iterate over each gene/transcript
for row in df.itertuples():
# create ref_seq reads
for i in range(row.ref_read_count):
READ_COUNTER = createReadPair(
row.seq, reads_file_1, reads_file_2, READ_COUNTER, args)
# create alt_seq reads
for i in range(row.alt_read_count):
READ_COUNTER = createReadPair(
row.alt_seq, reads_file_1, reads_file_2, READ_COUNTER, args)
def test_random_value():
r = RandomDict()
for i in range(10000):
r[i] = i
values = set(range(10000))
for i in range(100000):
assert r.random_value() in values
def test_update(self):
t = RandomDict({'a': 1, 'b': 2})
self.assertEqual(2, len(t))
self.assertEqual(1, t['a'])
self.assertEqual(2, t['b'])
t.update({'c': 7, 'a': 8})
self.assertEqual(3, len(t))
self.assertEqual(8, t['a'])
self.assertEqual(2, t['b'])
self.assertEqual(7, t['c'])
def test_del(self):
t = RandomDict()
t['a'] = 1
t['b'] = 2
del t['a']
self.assertEqual(2, t['b'])
self.assertEqual(1, len(t))
self.assertEqual(2, t.pop('b'))
self.assertEqual(0, len(t))
def test_random_key():
import string
r = RandomDict()
for k in string.ascii_lowercase:
r[k] = 1
keyset = set(string.ascii_lowercase)
while len(r) > 0:
k = r.random_key()
assert k in keyset
del r[k]
def test_set_get(self):
t = RandomDict()
t['a'] = 1
t['b'] = 2
self.assertEqual(1, t['a'])
self.assertEqual(2, t['b'])
t['a'] = 10
self.assertEqual(10, t['a'])
self.assertEqual(2, t['b'])
self.assertEqual(2, len(t))
self.assertEqual(2, t.get('b'))
t.setdefault('c', 89)
self.assertEqual(89, t['c'])
def test_many_inserts_deletes():
r = RandomDict()
for i in range(10000):
r[i] = 1
for i in range(10000):
del r[i]
assert len(r) == 0
def _parse_initial_node_table(initial_node_table: bfcp_pb2.NodeTable)\
-> Dict[bytes, bfcp_pb2.NodeTableEntry]:
result = RandomDict()
for entry in initial_node_table.entries:
pub_key = proto_to_pubkey(entry.node.public_key)
result[pubkey_to_deterministic_string(pub_key)] = entry
return result
def init_choice_set(image_point):
choices = RandomDict()
n = len(image_point)
for i in range(n):
for j in range(i + 1, n):
image_id1, _ = image_point[i]
image_id2, _ = image_point[j]
choices[len(choices)] = (image_id1, image_id2)
return choices
def initializePatches(self):
#Instantiate Patches
#Create a list to hold the patches. We first fill these with
#zeros to hold the place for each Patch object
self.patch_dict = {
i: {
j: 0
}
for i in range(self.rows) for j in range(self.cols)
}
for i in range(self.rows):
for j in range(self.cols):
#replace zeros with actual Patch objects
good = "sugar" if i + j >= self.rows else "water"
self.patch_dict[i][j] = Patch(self, i, j, self.sugarMap[i][j],
good)
self.empty_patches = RandomDict({(i, j): self.patch_dict[i][j]
for i in range(self.rows)
for j in range(self.cols)})
class RandomCacheStorage(CacheStorage):
def __init__(self, size, time_out):
self.size = size
self.time_out = time_out
self.slot = RandomDict()
self.clock = {}
self.ver = '0.1'
def put(self, key, val):
if len(self.slot) == self.size:
k = self.slot.randdel()
del self.clock[k]
self.slot[key] = val
self.clock[key] = time.time()
def get(self, key, default=None):
if key not in self.clock:
return default
if time.time() - self.clock[key] > self.time_out:
self.invalidate(key)
return default
return self.slot[key]
def clear(self):
self.slot.clear()
self.clock.clear()
def invalidate(self, key):
del self.slot[key]
del self.clock[key]
def load(self, fp):
slot, clock, ver = pickle.load(fp)
if self.ver == ver:
self.slot = slot
self.clock = clock
def dump(self, fp):
pickle.dump((self.slot, self.clock, self.ver), fp)
def initializePatches(self):
#Instantiate Patches
#Create a dictionary to hold the patches, organize as grid.
#We first fill these with zeros as placeh holders
self.patch_dict = {
row: {
col: 0
}
for row in range(self.rows) for col in range(self.cols)
}
for row in range(self.rows):
for col in range(self.cols):
# replace zeros with actual Patch objects
good = "sugar" if row + col < self.cols else "water"
self.patch_dict[row][col] = Patch(self, row, col,
self.sugarMap[row][col],
good)
# use RandomDict - O(n) time complexity - for choosing random empty patch
self.empty_patches = RandomDict({(row, col): self.patch_dict[row][col]
for row in range(self.rows)
for col in range(self.cols)})
def __init__(self,
definitions,
use_string_pattern,
init_rand_values,
max_string_len=200):
"""Creates a Replicator for given type
:parameter definitions all object definitions that could be referenced in the type that should be replicated
:parameter use_string_pattern if valid, generated patterns instead of random character strings
:parameter init_rand_values indicates if the replicated values should be random or - as per default - be 0 or ''
:parameter max_string_len maximum length of strings, which are generated
"""
self.definitions = definitions
self.init_rand_values = init_rand_values
self.use_string_pattern = use_string_pattern
self.max_string_len = max_string_len
self.random = Random()
self.randomdict = RandomDict()
def parse_jeopardy(self):
path = "data/trivia/jeopardy.csv"
parsed_list = RandomDict()
print("Special routine for jeopardy parsing")
encoding = "ISO-8859-1"
print("encoding detected")
with open(path, "r", encoding=encoding) as f:
reader = csv.reader(f)
counter = 0
print("csv loop started")
for row in reader:
parsed_list[counter] = TriviaLine(question="\n".join(
row[:-1]).strip(),
answers=[row[-1]])
counter = counter + 1
if not parsed_list:
raise ValueError("Empty trivia list")
print("done")
return parsed_list
def parse_trivia_list(self, filename):
if filename == "jeopardy":
return self.parse_jeopardy()
print("parsing ordinary trivia txt")
path = "data/trivia/{}.txt".format(filename)
parsed_list = RandomDict()
with open(path, "rb") as f:
try:
encoding = chardet.detect(f.read())["encoding"]
except:
encoding = "ISO-8859-1"
with open(path, "r", encoding=encoding) as f:
trivia_list = f.readlines()
counter = 0
for line in trivia_list:
if "`" not in line:
continue
line = line.replace("\n", "")
line = line.split("`")
question = line[0]
answers = []
for l in line[1:]:
answers.append(l.strip())
if len(line) >= 2 and question and answers:
line = TriviaLine(question=question, answers=answers)
#parsed_list.append(line)
parsed_list[counter] = line
counter = counter + 1
if not parsed_list:
raise ValueError("Empty trivia list")
return parsed_list
def test_empty_random_value():
r = RandomDict()
r.random_value()
from anytree import Node, RenderTree
from anytree.importer import DictImporter
import treeTester
from randomdict import RandomDict
import io
import time
importer = DictImporter()
fNaive = io.open("./runs/TreeTest2.csv", 'a')
fTrees = io.open("./DictStore.pkl", 'r').read().splitlines()
singleRunResults = RandomDict()
key = "DEFAULT, SHOULD NEVER BE SEEN"
for line in fTrees:
if line[0] == '.':
key = line
singleRunResults[key] = []
else:
tDict = eval(line)
root = importer.import_(tDict)
singleRunResults[key].append(root)
done = 0
for x in singleRunResults.keys:
for y in range(0, 10):
t = time.time()
def genquery(genomeFile, jellyFile, totedits, medindel, insprob, delprob, queryfreq, querycount, outputFile):
#genome - path to genome
#totedits - total number of edits to make
#medindel - median (mean) size of indel edits. actual edit length determined from gaussian with mean medindel and std medindel/2
#insprob - probability of insertion
#delprob - probability of deletion
#outputs all edits into a text file called "sampleedits.txt"
if delprob + insprob > 1.0:
raise "Error, delprob = {} and insprob = {}. "\
"The sum is {} > 1.0".format(
delprob, insprob, delprob + insprob)
genome = genomeFile.readline()
genomeFile.close()
#mf = jellyfish.ReadMerFile(jellyFile)
qf = jellyfish.QueryMerFile(jellyFile)
numbases = len(genome)-1
genome = genome[0:numbases]
letters = ['A','C','G','T']
randr = []
allinds = []
snpProb = 1.0 - (insprob + delprob)
SNPrange = int(snpProb * totedits)
insrange = int(insprob * totedits)
delrange = int(delprob * totedits)
editTypes = (['S'] * SNPrange) +\
(['D'] * delrange) +\
(['I'] * insrange)
random.shuffle(editTypes)
qcount = 0
#effectedkmers = dict()
effectedkmers = RandomDict()
count=0
sample=10
for val in editTypes:
qcount += 1
if val == 'I':
p, s, seq = random_insertion(numbases, medindel)
numbases += s
outputFile.write('I %d %s\n' % (p, seq))
if ((qcount-1)%sample)==0:
add_kmers_in_seq(effectedkmers, seq)
add_kmers_in_seq(effectedkmers, genome[p-K+1:p+K])
elif val == 'D':
p, s = random_deletion(numbases, medindel)
numbases -= s
outputFile.write('D %d %d\n' % (p, p+s-1))
#add_kmers_in_seq(effectedkmers, genome[p-K+1:p+s-1+K])
else:
p, seq = random_snp(numbases)
outputFile.write('S %d %s\n' % (p, seq))
if ((qcount-1)%sample)==0:
add_kmers_in_seq(effectedkmers, genome[p-K+1:p+K-1])
# if it's time to output some queries
if qcount == queryfreq:
qcount = 0
for qlist in xrange(querycount):
dart = random.random()
if dart <= EDIT_QUERY_PROB:
kmer = effectedkmers.random_key()
#kmer = random.choice(effectedkmers.keys())
#kmer = random.sample(effectedkmers, 1)[0]
editflag = 'I'
else:
p = random.randrange(K*2, numbases - K*2)
kmer = genome[p:p+K].upper()
editflag = 'N'
kcount = 0
#kcount = int(qf[jellyfish.MerDNA(kmer)])
outputFile.write('Q %s %s %d\n' % (kmer, editflag, kcount))
outputFile.close()
def genquery(genomeFile, jellyFile, totedits, medindel, insprob, delprob,
queryfreq, querycount, outputFile):
#genome - path to genome
#totedits - total number of edits to make
#medindel - median (mean) size of indel edits. actual edit length determined from gaussian with mean medindel and std medindel/2
#insprob - probability of insertion
#delprob - probability of deletion
#outputs all edits into a text file called "sampleedits.txt"
if delprob + insprob > 1.0:
raise "Error, delprob = {} and insprob = {}. "\
"The sum is {} > 1.0".format(
delprob, insprob, delprob + insprob)
genome = genomeFile.readline()
genomeFile.close()
#mf = jellyfish.ReadMerFile(jellyFile)
qf = jellyfish.QueryMerFile(jellyFile)
numbases = len(genome) - 1
genome = genome[0:numbases]
letters = ['A', 'C', 'G', 'T']
randr = []
allinds = []
snpProb = 1.0 - (insprob + delprob)
SNPrange = int(snpProb * totedits)
insrange = int(insprob * totedits)
delrange = int(delprob * totedits)
editTypes = (['S'] * SNPrange) +\
(['D'] * delrange) +\
(['I'] * insrange)
random.shuffle(editTypes)
qcount = 0
#effectedkmers = dict()
effectedkmers = RandomDict()
count = 0
sample = 10
for val in editTypes:
qcount += 1
if val == 'I':
p, s, seq = random_insertion(numbases, medindel)
numbases += s
outputFile.write('I %d %s\n' % (p, seq))
if ((qcount - 1) % sample) == 0:
add_kmers_in_seq(effectedkmers, seq)
add_kmers_in_seq(effectedkmers, genome[p - K + 1:p + K])
elif val == 'D':
p, s = random_deletion(numbases, medindel)
numbases -= s
outputFile.write('D %d %d\n' % (p, p + s - 1))
#add_kmers_in_seq(effectedkmers, genome[p-K+1:p+s-1+K])
else:
p, seq = random_snp(numbases)
outputFile.write('S %d %s\n' % (p, seq))
if ((qcount - 1) % sample) == 0:
add_kmers_in_seq(effectedkmers, genome[p - K + 1:p + K - 1])
# if it's time to output some queries
if qcount == queryfreq:
qcount = 0
for qlist in xrange(querycount):
dart = random.random()
if dart <= EDIT_QUERY_PROB:
kmer = effectedkmers.random_key()
#kmer = random.choice(effectedkmers.keys())
#kmer = random.sample(effectedkmers, 1)[0]
editflag = 'I'
else:
p = random.randrange(K * 2, numbases - K * 2)
kmer = genome[p:p + K].upper()
editflag = 'N'
kcount = 0
#kcount = int(qf[jellyfish.MerDNA(kmer)])
outputFile.write('Q %s %s %d\n' % (kmer, editflag, kcount))
outputFile.close()
class Model():
def __init__(self, gui, num_agents):
self.GUI = gui
self.initial_population = num_agents
self.total_agents_created = 0
self.goods_params = {
"sugar": {
"min": 5,
"max": 25
},
"water": {
"min": 5,
"max": 25
}
}
self.max_vision = 1
# hash table that identifies possible moves relative to agent position
self.move_dict = {
v: {
i:
{j: True
for j in range(-v, v + 1) if (i**2 + j**2) <= (v**2)}
for i in range(-v, v + 1)
}
for v in range(1, self.max_vision + 1)
}
#sugarMap.shape calls the a tuple with dimensions
#of the dataframe
self.sugarMap = pd.read_csv('sugar-map.txt', header=None, sep=' ')
# add 1 to each max_Val
for key in self.sugarMap:
self.sugarMap[key] = self.sugarMap[key].add(1)
self.rows, self.cols = self.sugarMap.shape
#Use to efficiently track which patches are empty
self.initializePatches()
self.initializeAgents()
# self.aggregate_data = {"agent":}
def initializePatches(self):
#Instantiate Patches
#Create a list to hold the patches. We first fill these with
#zeros to hold the place for each Patch object
self.patch_dict = {
i: {
j: 0
}
for i in range(self.rows) for j in range(self.cols)
}
for i in range(self.rows):
for j in range(self.cols):
#replace zeros with actual Patch objects
good = "sugar" if i + j >= self.rows else "water"
self.patch_dict[i][j] = Patch(self, i, j, self.sugarMap[i][j],
good)
self.empty_patches = RandomDict({(i, j): self.patch_dict[i][j]
for i in range(self.rows)
for j in range(self.cols)})
def initializeAgents(self):
self.agent_dict = {}
# self.agentLocationDict = {}
for i in range(self.initial_population):
self.total_agents_created += 1
ID = self.total_agents_created
row, col = self.chooseRandomEmptyPatch()
del self.empty_patches[row, col]
self.agent_dict[ID] = Agent(self, row, col, ID)
# def recordAgentLocationInDict(self, agent):
# patchIndex = self.convert2dTo1d(agent.row, agent.col)
# self.agentLocationDict[patchIndex] = agent
def chooseRandomEmptyPatch(self):
i, j = self.empty_patches.random_key()
return i, j
def runModel(self, periods):
agent_list = list(self.agent_dict.values())
for period in range(1, periods + 1):
# print("period:", period)
self.growPatches()
random.shuffle(agent_list)
for agent in agent_list:
self.agentMove(agent)
if self.GUI.live_visual:
if period % self.GUI.every_t_frames == 0:
self.GUI.updatePatches()
self.GUI.moveAgents()
self.GUI.canvas.update()
def agentMove(self, agent):
# save agent coords to track agent movement, changes in (not) empty patches
curr_i, curr_j = agent.row, agent.col
max_patch, near_empty_patch = self.findMaxEmptyPatch(
agent, curr_i, curr_j)
if near_empty_patch[agent.target]:
target = agent.target
else:
target = agent.not_target
self.moveToMaxEmptyPatch(agent, curr_i, curr_j, max_patch,
near_empty_patch, target)
self.agentHarvest(agent)
def findMaxEmptyPatch(self, agent, curr_i, curr_j):
# dict to save empty patch with max q for each good
max_patch = {
good: {
"Q": 0,
"patch": None
}
for good in self.goods_params
}
patch_moves = [(curr_i + i, curr_j + j)
for i in self.move_dict[agent.vision]
for j in self.move_dict[agent.vision][i]]
# shuffle patches so not movement biased in one direction
random.shuffle(patch_moves)
near_empty_patch = {good: False for good in self.goods_params}
for coords in patch_moves:
if coords in self.empty_patches:
i, j = coords[0], coords[1]
empty_patch = self.patch_dict[i][j]
patch_q = empty_patch.Q
patch_good = empty_patch.good
near_empty_patch[patch_good] = True
if patch_q > max_patch[patch_good]["Q"]:
max_patch[patch_good]["patch"] = empty_patch
max_patch[patch_good]["Q"] = patch_q
return max_patch, near_empty_patch
def moveToMaxEmptyPatch(self, agent, curr_i, curr_j, max_patch,
near_empty_patch, target):
if near_empty_patch[target]:
# agent_move()
target_patch = max_patch[target]["patch"]
new_coords = target_patch.row, target_patch.col
agent.dx = target_patch.col - curr_j
agent.dy = target_patch.row - curr_i
agent.row, agent.col = new_coords
del self.empty_patches[new_coords]
self.empty_patches[curr_i,
curr_j] = self.patch_dict[curr_i][curr_j]
else:
agent.dx = 0
agent.dy = 0
def agentHarvest(self, agent):
agent_patch = self.patch_dict[agent.row][agent.col]
agent.good[agent_patch.good] += agent_patch.Q
agent_patch.Q = 0
def growPatches(self):
for i, vals in self.patch_dict.items():
for patch in vals.values():
if patch.Q < patch.maxQ:
patch.Q += 1
grid_col = current_col
grid_id = current_grid
if os.path.exists(os.path.join(output_folder, 'patches%04d.bmp' % grid_id)):
grid = cv2.imread(os.path.join(output_folder, 'patches%04d.bmp' % grid_id),
cv2.IMREAD_GRAYSCALE)
else:
grid = np.zeros(shape=(grid_sz * input_sz, grid_sz * input_sz),
dtype=np.uint8)
info_file = open(os.path.join(output_folder, 'info.txt'), 'a+')
#position_file = open(os.path.join(output_folder, 'position%04d.txt'%grid_id), 'w')
#
matches = RandomDict()
possible_n_matches = 0
nonmatches = {}
def init_choice_set(image_point):
choices = RandomDict()
n = len(image_point)
for i in range(n):
for j in range(i + 1, n):
image_id1, _ = image_point[i]
image_id2, _ = image_point[j]
choices[len(choices)] = (image_id1, image_id2)
return choices
def test_delete():
r = RandomDict({'a': 1})
del r['a']
assert len(r) == 0
class Simulation:
"""
Class for containing the elements of the simulation.
Attributes
----------
time : int
stores the time elapsed in the simulation
num_taxis : int
how many taxis there are
request_rate : float
rate of requests per time period
hard_limit : int
max distance from which a taxi is still assigned to a request
taxis : dict
storing all Taxi() instances in a dict
keys are `taxi_id`s
latest_taxi_id : int
shows latest given taxi_id
used or generating new taxis
taxis_available : list of int
stores `taxi_id`s of available taxis
taxis_to_request : list of int
stores `taxi_id`s of taxis moving to serve a request
taxis_to_destination : list of int
stores `taxi_id`s of taxis with passenger
requests : dict
storing all Request() instances in a dict
keys are `request_id`s
latest_request_id : int
shows latest given request_id
used or generating new requests
requests_pending : list of int
requests waiting to be served
requests_in_progress : list of int
requests with assigned taxis
requests_dropped : list of int
unsuccessful requests
city : City
geometry of class City() underlying the simulation
show_map_labels : bool
"""
def __init__(self, **config):
# initializing time
self.time = 0
self.num_taxis = config["num_taxis"]
self.request_rate = config["request_rate"]
if "price_fixed" in config:
# price that is used for every trip
self.price_fixed = config["price_fixed"]
else:
self.price_fixed = 0
if "price_per_dist" in config:
# price per unit distance while carrying a passenger
self.price_per_dist = config["price_per_dist"]
else:
self.price_per_dist = 1
if "cost_per_unit" in config:
# cost per unit distance (e.g. gas)
self.cost_per_unit = config["cost_per_unit"]
else:
self.cost_per_unit = 0
if "cost_per_time" in config:
# cost per time (e.g. amortization)
self.cost_per_time = config["cost_per_time"]
else:
self.cost_per_time = 0
if "matching" in config:
self.matching = config["matching"]
if "batch_size" in config:
self.batch_size = config["batch_size"]
if "max_time" in config:
self.max_time = config["max_time"]
# this is done now by the config generator in theory
if "max_request_waiting_time" in config:
self.max_request_waiting_time = config["max_request_waiting_time"]
else:
self.max_request_waiting_time = 10000
if ("batch_size" in config) and ("max_time" in config):
self.num_iter = int(np.ceil(self.max_time / self.batch_size))
else:
self.num_iter = None
if "behaviour" in config: # goback / stay / cruise
self.behaviour = config["behaviour"]
else:
self.behaviour = "go_back"
if "initial_conditions" in config: # base / home
self.initial_conditions = config["initial_conditions"]
else:
self.initial_conditions = "base"
if "reset_time" in config:
self.reset_time = config["reset_time"]
else:
self.reset_time = self.max_time + 1
# initializing counters
self.latest_taxi_id = 0
self.latest_request_id = 0
# initializing object storage
self.taxis = RandomDict()
self.taxis_available = RandomDict()
self.taxis_to_request = set()
self.taxis_to_destination = set()
self.requests = RandomDict()
self.requests_pending = set()
# speeding up going through requests in the order of waiting times
# they are pushed into a deque in the order of timestamps
self.requests_pending_deque = deque()
self.requests_pending_deque_batch = deque(
maxlen=self.max_request_waiting_time)
self.requests_pending_deque_temporary = deque()
self.requests_in_progress = set()
# city layout
self.city = City(**config)
# length of pregenerated random number storage
self.city.length = int(min(self.max_time * self.request_rate, 1e6))
# whether to log all movements for debugging purposes
self.log = config["log"]
self.city.log = self.log
# showing map of moving taxis in interactive jupyter notebook
self.show_plot = config["show_plot"]
# initializing simulation with taxis
for t in range(self.num_taxis):
self.add_taxi()
self.taxi_df = pd.DataFrame.from_dict(
[dict(v) for k, v in self.taxis.items()])
self.taxi_df.set_index('taxi_id', inplace=True)
if self.show_plot:
# plotting variables
self.canvas = plt.figure()
self.canvas_ax = self.canvas.add_subplot(1, 1, 1)
self.canvas_ax.set_aspect('equal', 'box')
self.cmap = plt.get_cmap('viridis')
self.taxi_colors = list(np.linspace(0, 0.85, self.num_taxis))
shuffle(self.taxi_colors)
self.show_map_labels = config["show_map_labels"]
self.show_pending = config["show_pending"]
self.init_canvas()
def init_canvas(self):
"""
Initialize plot.
"""
self.canvas_ax.clear()
self.canvas_ax.set_xlim(-0.5, self.city.n - 0.5)
self.canvas_ax.set_ylim(-0.5, self.city.m - 0.5)
self.canvas_ax.tick_params(length=0)
self.canvas_ax.xaxis.set_ticks(list(range(self.city.n)))
self.canvas_ax.yaxis.set_ticks(list(range(self.city.m)))
if not self.show_map_labels:
self.canvas_ax.xaxis.set_ticklabels([])
self.canvas_ax.yaxis.set_ticklabels([])
self.canvas_ax.set_aspect('equal', 'box')
self.canvas.tight_layout()
self.canvas_ax.grid()
def add_taxi(self):
"""
Create new taxi.
"""
# adding home coordinates, starting taxi
home = self.city.create_taxi_home_coords()
if self.initial_conditions == "base":
# create a taxi at the base
tx = Taxi(self.city.base_coords, self.latest_taxi_id)
elif self.initial_conditions == "home":
# create a taxi at home
tx = Taxi(home, self.latest_taxi_id)
tx.home = home
# add to taxi storage
self.taxis[self.latest_taxi_id] = tx
# add to available taxi matrix
self.city.A[self.city.coordinate_dict_ij_to_c[tx.x][tx.y]].add(
self.latest_taxi_id)
# add to available taxi storage
self.taxis_available[self.latest_taxi_id] = tx
# increase counter
self.latest_taxi_id += 1
def add_request(self):
"""
Create new request.
"""
# here we randomly choose a place for the request
# the random coordinates are pre-stored in a deque for faster access
# if there are no more pregenerated coordinates in the deque, we generate some more
# origin and destination coordinates
ox, oy, dx, dy = self.city.create_one_request_coord()
r = Request([ox, oy], [dx, dy], self.latest_request_id, self.time)
# add to request storage
self.requests[self.latest_request_id] = r
# add to free users
self.requests_pending_deque.append(self.latest_request_id)
# increase counter
self.latest_request_id += 1
def go_to_base(self, taxi_id, bcoords):
"""
This function sends the taxi to the base rom wherever it is.
"""
# fetch object
t = self.taxis[taxi_id]
# actual coordinates
acoords = [t.x, t.y]
# path between actual coordinates and destination
path = self.city.create_path(acoords, bcoords)
# erase path memory
t.with_passenger = False
t.to_request = False
t.available = True
# print("Erasing path memory, Taxi "+str(taxi_id)+".")
t.next_destination = deque()
# put path into taxi path queue
# print("Filling path memory, Taxi "+str(taxi_id)+". Path ",path)
t.next_destination.extend(path)
# put object back to its place
self.taxis[taxi_id] = t
def go_home_everybody(self):
"""
Drop requests that are currently executing, and set taxis as available at their home locations.
"""
for taxi_id in self.taxis:
tx = self.taxis[taxi_id]
if taxi_id in self.taxis_available:
# (magic wand) Apparate taxi home!
self.city.A[self.city.coordinate_dict_ij_to_c[tx.x][
tx.y]].remove(taxi_id)
self.city.A[self.city.coordinate_dict_ij_to_c[tx.home[0]][
tx.home[1]]].add(taxi_id)
tx.x, tx.y = tx.home
if taxi_id in self.taxis_to_destination:
# if somebody is sitting in it, finish request
self.dropoff_request(tx.actual_request_executing,
mode="going_home")
if taxi_id in self.taxis_to_request:
# if it was only going towards a request, cancel it
self.dropoff_request(tx.actual_request_executing,
mode="cancel")
self.taxis[taxi_id] = tx
def cruise(self, taxi_id):
return None
def assign_request(self, request_id, taxi_id):
"""
Given a request_id, taxi_id pair, this function makes the match.
It sets new state variables for the request and the taxi, updates path of the taxi etc.
"""
r = self.requests[request_id]
t = self.taxis[taxi_id]
# pair the match
t.actual_request_executing = request_id
r.taxi_id = taxi_id
# remove taxi from the available ones
self.city.A[self.city.coordinate_dict_ij_to_c[t.x][t.y]].remove(
taxi_id)
del self.taxis_available[taxi_id]
t.with_passenger = False
t.available = False
t.to_request = True
# mark taxi as moving to request
self.taxis_to_request.add(taxi_id)
# forget the path that has been assigned
t.next_destination = deque()
# create new path: to user, then to destination
path = self.city.create_path([t.x, t.y], [r.ox, r.oy]) + \
self.city.create_path([r.ox, r.oy], [r.dx, r.dy])[1:]
t.next_destination.extend(path)
# remove request from the pending ones, label it as "in progress"
self.requests_in_progress.add(request_id)
r.mode = 'waiting'
r.timestamps['assigned'] = self.time
# update taxi state in taxi storage
self.taxis[taxi_id] = t
# update request state
self.requests[request_id] = r
if self.log:
print("\tM request " + str(request_id) + " taxi " + str(taxi_id))
def matching_algorithm(self, mode="random_unlimited"):
"""
This function contains the possible matching functions which are selected by the mode keyword.
Parameters
----------
mode : str, default baseline
matching algorithm mode
* random_unlimited : assigning a random taxi to the user
* random_limited : assigning a random taxi to the user within the circle of a radius self.city.hard_limit
* nearest : sending the nearest available taxi for the user from within the circle of a radius self.city.hard_limit
* poorest : sending the least earning available taxi for the user from within the circle of a radius self.city.hard_limit
"""
if len(self.requests_pending_deque) == 0:
if self.log:
print("No pending requests.")
return
if self.log:
print('Matching algorithm.')
if mode == "random_unlimited":
while len(self.requests_pending_deque) > 0 and len(
self.taxis_available) > 0:
# select a random taxi
taxi_id = self.taxis_available.random_key()
# select oldest request from deque
request_id = self.requests_pending_deque.popleft()
# make assignment
self.assign_request(request_id, taxi_id)
elif mode == "random_limited":
while len(self.requests_pending_deque) > 0 and len(
self.taxis_available) > 0:
# select oldest request from deque
request_id = self.requests_pending_deque.popleft()
# fetch request
r = self.requests[request_id]
# search for nearest free taxis
possible_taxi_ids = self.city.find_nearest_available_taxis(
self.city.coordinate_dict_ij_to_c[r.ox][r.oy],
mode="circle",
radius=self.city.hard_limit)
# if there were any taxis near
if len(possible_taxi_ids) > 0:
# select taxi
taxi_id = choice(possible_taxi_ids)
self.assign_request(request_id, taxi_id)
else:
# mark request as still pending
self.requests_pending_deque_temporary.append(request_id)
elif mode == "nearest":
while len(self.requests_pending_deque) > 0 and len(
self.taxis_available) > 0:
# select oldest request from deque
request_id = self.requests_pending_deque.popleft()
# fetch request
r = self.requests[request_id]
# search for nearest free taxis
possible_taxi_ids = self.city.find_nearest_available_taxis(
self.city.coordinate_dict_ij_to_c[r.ox][r.oy])
# if there were any taxis near
if len(possible_taxi_ids) > 0:
# select taxi
taxi_id = choice(possible_taxi_ids)
self.assign_request(request_id, taxi_id)
else:
# mark request as still pending
self.requests_pending_deque_temporary.append(request_id)
elif mode == "poorest":
# always order taxi that has earned the least money so far
# but choose only from the nearest ones
# hard limiting: e.g. if there is no taxi within the radius, then quit
# evaulate the earnings of the available taxis so far
ta_list = list(self.taxis_available.keys)
# print('Available taxis ', self.taxis_available.keys)
taxi_earnings = [
self.eval_taxi_income(taxi_id) for taxi_id in ta_list
]
# print('Earnings ', taxi_earnings)
ta_list = list(np.array(ta_list)[np.argsort(taxi_earnings)])
pairs = 0
while len(self.requests_pending_deque) > 0 and len(
self.taxis_available) > 0:
# select oldest request from deque
request_id = self.requests_pending_deque.popleft()
# fetch request
r = self.requests[request_id]
# find nearest vehicles in a radius
possible_taxi_ids = self.city.find_nearest_available_taxis(
self.city.coordinate_dict_ij_to_c[r.ox][r.oy],
mode="circle",
radius=self.city.hard_limit)
hit = 0
for t in ta_list:
if t in possible_taxi_ids:
# on first hit
# make assignment
self.assign_request(request_id, t)
hit = 1
pairs += 1
break
if not hit:
self.requests_pending_deque_temporary.append(request_id)
else:
print(
"I know of no such assignment mode! Please provide a valid one!"
)
def pickup_request(self, request_id):
"""
Pick up passenger.
Parameters
----------
request_id : int
"""
# mark pickup timestamp
r = self.requests[request_id]
t = self.taxis[r.taxi_id]
self.taxis_to_request.remove(r.taxi_id)
self.taxis_to_destination.add(r.taxi_id)
# change taxi state to with passenger
t.to_request = False
t.with_passenger = True
t.available = False
r.timestamps['pickup'] = self.time
r.mode = 'serving'
# update request and taxi instances
self.requests[request_id] = r
self.taxis[r.taxi_id] = t
if self.log:
print('\tP ' + "request " + str(request_id) + ' taxi ' +
str(t.taxi_id))
def dropoff_request(self, request_id, mode="simple"):
"""
Drop off passenger, when taxi reached request destination.
"""
r = self.requests[request_id]
t = self.taxis[r.taxi_id]
if mode == "simple" or mode == "going_home":
# mark request as done
r.timestamps['dropoff'] = self.time
r.mode = 'done'
self.requests_in_progress.remove(request_id)
t.requests_completed.add(request_id)
# remove taxi from to_destination list
self.taxis_to_destination.remove(r.taxi_id)
elif mode == "cancel":
# mark request as dropped
r.mode = 'dropped'
# remove request from progressing ones
self.requests_in_progress.remove(request_id)
# clear taxi path
t.next_destination = deque()
# remove taxi from to_request list
self.taxis_to_request.remove(r.taxi_id)
# update taxi lists
if mode == "going_home":
# (magic wand) Apparate taxi home!
t.x, t.y = t.home
# update global availability containers
self.taxis_available[r.taxi_id] = t
self.city.A[self.city.coordinate_dict_ij_to_c[t.x][t.y]].add(r.taxi_id)
# update taxi internal states
t.with_passenger = False
t.available = True
t.actual_request_executing = None
# update request and taxi instances in global containers
self.requests[request_id] = r
self.taxis[r.taxi_id] = t
if self.log:
print("\tD request " + str(request_id) + ' taxi ' + str(t.taxi_id))
def eval_taxi_income(self, taxi_id):
"""
Parameters
----------
taxi_id : int
select taxi from self.taxis with id
Returns
-------
price : int
evaulated earnnigs of the taxi based on config
"""
t = self.taxis[taxi_id]
price =\
len(t.requests_completed) * self.price_fixed +\
int(not t.available) * self.price_fixed +\
t.time_serving*self.price_per_dist -\
(t.time_cruising+t.time_serving+t.time_to_request)*self.cost_per_unit -\
(t.time_serving+t.time_cruising+t.time_to_request+t.time_waiting)*self.cost_per_time
return price
def plot_simulation(self):
"""
Draws current state of the simulation on the predefined grid of the class.
Is based on the taxis and requests and their internal states.
"""
self.init_canvas()
for taxi_id, i in self.taxis.keys.items():
t = self.taxis[taxi_id]
# plot a circle at the place of the taxi
self.canvas_ax.plot(t.x,
t.y,
'o',
ms=10,
c=self.cmap(self.taxi_colors[i]))
if self.show_map_labels:
self.canvas_ax.annotate(str(i),
xy=(t.x, t.y),
xytext=(t.x, t.y),
ha='center',
va='center',
color='white')
# if the taxi has a path ahead of it, plot it
if len(t.next_destination) > 0:
path = np.array([[t.x, t.y]] + list(t.next_destination))
if len(path) > 1:
xp, yp = path.T
# plot path
self.canvas_ax.plot(xp,
yp,
'-',
c=self.cmap(self.taxi_colors[i]))
# plot a star at taxi destination
self.canvas_ax.plot(path[-1][0],
path[-1][1],
'*',
ms=5,
c=self.cmap(self.taxi_colors[i]))
# if a taxi serves a request, put request on the map
request_id = t.actual_request_executing
if (request_id is not None) and (not t.with_passenger):
r = self.requests[request_id]
self.canvas_ax.plot(r.ox, r.oy, 'ro', ms=3)
if self.show_map_labels:
self.canvas_ax.annotate(request_id,
xy=(r.ox, r.oy),
xytext=(r.ox - 0.2, r.oy - 0.2),
ha='center',
va='center')
# plot taxi base
self.canvas_ax.plot(self.city.base_coords[0],
self.city.base_coords[1],
'ks',
ms=15)
# plot pending requests
if self.show_pending:
for request_id in self.requests_pending_deque:
self.canvas_ax.plot(self.requests[request_id].ox,
self.requests[request_id].oy,
'ro',
ms=3,
alpha=0.5)
self.canvas.show()
def move_taxi(self, taxi_id):
"""
Move a taxi one step forward according to its path queue.
Update taxi position on availablity grid, if necessary.
Parameters
----------
plt.ion()
taxi_id : int
unique id of taxi that we want to move
"""
t = self.taxis[taxi_id]
try:
# move taxi one step forward
move = t.next_destination.popleft()
old_x = t.x
old_y = t.y
t.x = move[0]
t.y = move[1]
if t.with_passenger:
t.time_serving += 1
else:
if t.available:
t.time_cruising += 1
else:
t.time_to_request += 1
# move available taxis on availability grid
if t.available:
self.city.A[self.city.coordinate_dict_ij_to_c[old_x]
[old_y]].remove(taxi_id)
self.city.A[self.city.coordinate_dict_ij_to_c[t.x][t.y]].add(
taxi_id)
if self.log:
print("\tF moved taxi " + str(taxi_id) + " remaining path ",
list(t.next_destination),
"\n",
end="")
except:
t.time_waiting += 1
self.taxis[taxi_id] = t
def run_batch(self, run_id, data_path='results'):
"""
Create a batch run, where metrics are evaluated at every batch step and at the end.
Parameters
----------
run_id : str
id that stands for simulation
data_path : str
where to save the results
"""
# TODO check if data_path exists, if not, throw an error
measurement = Measurements(self)
if self.num_iter is None:
print(
"No batch run parameters were defined in the config file, please add them!"
)
return
print("Running simulation with run_id " + run_id + ".")
print("Batch time " + str(self.batch_size) + ".")
print("Number of items " + str(self.num_iter) + ".")
print("Total time simulated " + str(self.batch_size * self.num_iter) +
".")
print("Starting...")
results = []
time1 = time()
for i in range(self.num_iter):
# tick the clock
for k in range(self.batch_size):
self.step_time("")
ptm = measurement.read_per_taxi_metrics()
if i == 0:
# clearing future output files
f = open(
data_path + '/run_' + run_id + '_per_taxi_metrics.json',
'w')
f.close()
f = open(
data_path + '/run_' + run_id + '_per_request_metrics.json',
'w')
f.close()
# adding taxi homes to output
ptm['taxi_homes'] = [[
int(self.taxis[t].home[0]),
int(self.taxis[t].home[1])
] for t in self.taxis]
# dumping per taxi metrics out (per batch)
f = open(data_path + '/run_' + run_id + '_per_taxi_metrics.json',
'a')
json.dump(ptm, f)
f.write('\n')
f.close()
results.append(measurement.read_aggregated_metrics(ptm))
time2 = time()
print('Simulation batch ' + str(i + 1) + '/' + str(self.num_iter) +
' , %.2f sec/batch.' % (time2 - time1))
time1 = time2
# dumping batch results
f = open(data_path + '/run_' + run_id + '_aggregates.csv', 'w')
pd.DataFrame.from_dict(results).to_csv(f, float_format="%.4f")
f.write('\n')
f.close()
# dumping per request metrics out (only at the end)
f = open(data_path + '/run_' + run_id + '_per_request_metrics.json',
'a')
prm = measurement.read_per_request_metrics()
json.dump(prm, f)
f.write('\n')
f.close()
# compressing written objects
for file in [
'_per_taxi_metrics.json', '_per_request_metrics.json',
'_aggregates.csv'
]:
f1 = open(data_path + '/run_' + run_id + file, 'rb')
f2 = gzip.open(data_path + '/run_' + run_id + file + '.gz', 'wb')
shutil.copyfileobj(f1, f2)
f1.close()
f2.close()
os.remove(data_path + '/run_' + run_id + file)
print("Done.\n")
def step_time(self, handler):
"""
Ticks simulation time by 1.
"""
if self.log:
print("\n")
print("Timestamp " + str(self.time))
print("Taxis:\n")
print("\t Available: " + str(len(self.taxis_available)))
print("\t To request: " + str(len(self.taxis_to_request)))
print("\t To destination: " + str(len(self.taxis_to_destination)))
print("\n")
req_counter = {'TOTAL': self.latest_request_id}
for request_id in self.requests:
r = self.requests[request_id]
if r.mode in req_counter:
req_counter[r.mode] += 1
else:
req_counter[r.mode] = 1
print('Requests:')
for mode in req_counter:
print('\t' + mode + ': ' + str(req_counter[mode]))
print("\n")
print("Requests pending: ")
print(self.requests_pending)
print(self.requests_pending_deque)
print(self.requests_pending_deque_temporary)
if (self.time > 0) and (self.time % self.reset_time == 0):
# print("Going home!")
self.go_home_everybody()
else:
# move every taxi one step towards its destination
for taxi_id in self.taxis:
self.move_taxi(taxi_id)
t = self.taxis[taxi_id]
# if a taxi can pick up its passenger, do it
if taxi_id in self.taxis_to_request:
r = self.requests[t.actual_request_executing]
if (t.x == r.ox) and (t.y == r.oy):
try:
self.pickup_request(t.actual_request_executing)
except KeyError:
print(t)
print(r)
# if a taxi can drop off its passenger, do it
elif taxi_id in self.taxis_to_destination:
r = self.requests[t.actual_request_executing]
if (t.x == r.dx) and (t.y == r.dy):
self.dropoff_request(r.request_id)
if self.behaviour == "go_back":
if self.initial_conditions == "base":
self.go_to_base(taxi_id, self.city.base_coords)
elif self.initial_conditions == "home":
self.go_to_base(taxi_id, t.home)
elif self.behaviour == "stay":
pass
elif self.behaviour == "cruise":
self.cruise(taxi_id)
# make matchings
self.matching_algorithm(mode=self.matching)
# reunite pending requests
self.requests_pending_deque_temporary.reverse()
self.requests_pending_deque.extendleft(
self.requests_pending_deque_temporary)
self.requests_pending_deque_temporary = deque()
# delete old requests from pending ones
if self.time > self.max_request_waiting_time and len(
self.requests_pending_deque) > 0:
while len(self.requests_pending_deque) > 0 and (
self.requests_pending_deque[0]
in self.requests_pending_deque_batch[0]):
request_id = self.requests_pending_deque.popleft()
self.requests[request_id].mode = 'dropped'
self.requests_pending = set(self.requests_pending_deque)
# generate requests
new_requests = set()
rfrac, rint = np.modf(self.request_rate)
for i in range(int(rint)):
self.add_request()
new_requests.add(self.latest_request_id)
if rfrac > 1e-3:
try:
p = self.city.request_p.pop()
except IndexError:
self.city.request_p.extend(np.random.random(self.city.length))
p = self.city.request_p.pop()
if p < rfrac:
self.add_request()
new_requests.add(self.latest_request_id)
# this automatically pushes out requests that have been waiting for too long
self.requests_pending_deque_batch.append(new_requests)
if self.show_plot:
self.plot_simulation()
# step time
self.time += 1
def __init__(self, **config):
# initializing time
self.time = 0
self.num_taxis = config["num_taxis"]
self.request_rate = config["request_rate"]
if "price_fixed" in config:
# price that is used for every trip
self.price_fixed = config["price_fixed"]
else:
self.price_fixed = 0
if "price_per_dist" in config:
# price per unit distance while carrying a passenger
self.price_per_dist = config["price_per_dist"]
else:
self.price_per_dist = 1
if "cost_per_unit" in config:
# cost per unit distance (e.g. gas)
self.cost_per_unit = config["cost_per_unit"]
else:
self.cost_per_unit = 0
if "cost_per_time" in config:
# cost per time (e.g. amortization)
self.cost_per_time = config["cost_per_time"]
else:
self.cost_per_time = 0
if "matching" in config:
self.matching = config["matching"]
if "batch_size" in config:
self.batch_size = config["batch_size"]
if "max_time" in config:
self.max_time = config["max_time"]
# this is done now by the config generator in theory
if "max_request_waiting_time" in config:
self.max_request_waiting_time = config["max_request_waiting_time"]
else:
self.max_request_waiting_time = 10000
if ("batch_size" in config) and ("max_time" in config):
self.num_iter = int(np.ceil(self.max_time / self.batch_size))
else:
self.num_iter = None
if "behaviour" in config: # goback / stay / cruise
self.behaviour = config["behaviour"]
else:
self.behaviour = "go_back"
if "initial_conditions" in config: # base / home
self.initial_conditions = config["initial_conditions"]
else:
self.initial_conditions = "base"
if "reset_time" in config:
self.reset_time = config["reset_time"]
else:
self.reset_time = self.max_time + 1
# initializing counters
self.latest_taxi_id = 0
self.latest_request_id = 0
# initializing object storage
self.taxis = RandomDict()
self.taxis_available = RandomDict()
self.taxis_to_request = set()
self.taxis_to_destination = set()
self.requests = RandomDict()
self.requests_pending = set()
# speeding up going through requests in the order of waiting times
# they are pushed into a deque in the order of timestamps
self.requests_pending_deque = deque()
self.requests_pending_deque_batch = deque(
maxlen=self.max_request_waiting_time)
self.requests_pending_deque_temporary = deque()
self.requests_in_progress = set()
# city layout
self.city = City(**config)
# length of pregenerated random number storage
self.city.length = int(min(self.max_time * self.request_rate, 1e6))
# whether to log all movements for debugging purposes
self.log = config["log"]
self.city.log = self.log
# showing map of moving taxis in interactive jupyter notebook
self.show_plot = config["show_plot"]
# initializing simulation with taxis
for t in range(self.num_taxis):
self.add_taxi()
self.taxi_df = pd.DataFrame.from_dict(
[dict(v) for k, v in self.taxis.items()])
self.taxi_df.set_index('taxi_id', inplace=True)
if self.show_plot:
# plotting variables
self.canvas = plt.figure()
self.canvas_ax = self.canvas.add_subplot(1, 1, 1)
self.canvas_ax.set_aspect('equal', 'box')
self.cmap = plt.get_cmap('viridis')
self.taxi_colors = list(np.linspace(0, 0.85, self.num_taxis))
shuffle(self.taxi_colors)
self.show_map_labels = config["show_map_labels"]
self.show_pending = config["show_pending"]
self.init_canvas()
def __init__(self, size, time_out):
self.size = size
self.time_out = time_out
self.slot = RandomDict()
self.clock = {}
self.ver = '0.1'
for cell_barcode in cell_pool:
whitelist_file.write('{}\n'.format(cell_barcode))
#Generate some mutants
mutants = generate_mutants(cell_pool=cell_pool,
base_mutation_rate=0.001,
number_of_mutants=10)
print('Writing sim data to file.')
count = 1
with open(args.Outputname + "_R1.fastq",
"w") as handleR1, open(args.Outputname + "_R2.fastq",
"w") as handleR2:
for myfile in args.T:
record_dict = SeqIO.index(myfile.name, "fasta")
record_rand = RandomDict(record_dict)
for cell in range(0, nrCells):
cellSeq = random.sample(cell_pool, 1)[0]
cell_pool.discard(cellSeq)
umi_set = generate_umi_set(args.umi_bc_length, 100, 0.2)
for record in range(0, nrSeqs):
UMIseq = random.sample(umi_set, 1)[0]
myItem = record_rand.random_value()
if random.random() < 0.01:
myItem.seq = Seq(
mutate(sequence=myItem.seq, mutation_rate=0.001))
myItem += "A" * 150
start = 0
#print(myItem.id + " length: " + str(limit))
end = start + myLength
mySub = myItem[start:end]
def test_delete_missing():
r = RandomDict({'a': 1})
del r['b']
def test_init_with_update():
r = RandomDict({'a': 1})
assert 'a' in r
def test_len():
r = RandomDict({'a': 1})
assert len(r) == 1
def test_empty_random_item():
r = RandomDict()
r.random_item()
def test_empty_random_key():
r = RandomDict()
r.random_key()
