def __init__(self,
width,
height=None,
board=None,
difficulty=-1,
seed=randrange(sys.maxsize)):
self.board = board
self.width = width
self.height = height
if not height:
self.height = width
self.size = self.width * self.height
self.__difficulty = difficulty
assert self.width > 0, 'Width cannot be less than 1'
assert self.height > 0, 'Height cannot be less than 1'
assert self.size > 1, 'Board size cannot be 1 x 1'
if board:
blank_count = 0
for row in self.board:
for i in range(len(row)):
if type(row[i]) is not int or not 1 <= row[i] <= self.size:
row[i] = Sudoku._empty_cell_value
blank_count += 1
if difficulty == -1:
self.__difficulty = blank_count / self.size / self.size
else:
positions = list(range(self.size))
random_seed(seed)
shuffle(positions)
self.board = [[
(i + 1) if i == positions[j] else Sudoku._empty_cell_value
for i in range(self.size)
] for j in range(self.size)]
def __init__(self,
master_file_path, output_dir_path=None, adapter_str="rasa",
base_filepath=None, local=False, seed=None, force_overwriting=False
):
if local:
self.output_dir_path = os.path.dirname(master_file_path)
else:
self.output_dir_path = getcwd()
if output_dir_path is None:
self.output_dir_path = os.path.join(self.output_dir_path, "output")
else:
self.output_dir_path = os.path.join(self.output_dir_path,
output_dir_path)
self.force_overwriting = force_overwriting
# Initialize the random number generator
if seed is None:
seed = random_string()
print("Executing Chatette with random seed '" + seed + "'.")
else:
print("Executing Chatette with seed '" + seed + "'.")
random_seed(seed)
self.adapter = adapter_factory.create_adapter(
adapter_str, base_filepath
)
self.parser = Parser(master_file_path)
self.generator = None
def __init__(self, unique=False, seed=None):
assert (seed == None
or isinstance(seed, int)), 'Seed should be an integer.'
self.parameters = {}
if seed != None:
random_seed(seed)
def main():
global passComments
# parse the command line
seed = None
blockSize = None
lineLimit = None
passComments = None
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=", 1)[1]
if (arg.startswith("--seed=")):
seed = argVal
elif (arg.startswith("#")):
seed = arg[1:]
elif (arg.startswith("--block=")):
blockSize = int_with_unit(argVal)
elif (arg.startswith("--limit=")):
lineLimit = int_with_unit(argVal)
elif (arg == "--passcomments"):
passComments = ["#"]
elif (arg.startswith("--passcomments=")):
passComments = argVal
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
try:
seed = int(arg)
except:
# this is to prevent the common usage mistake of putting a filename
# in arg 1; the user who names her file as an integer unfortunately
# loses this protection
usage("Hey! Seed must be an integer")
if (seed != None): random_seed(seed)
# collect lines
if (blockSize == None):
data = read_block()
write_shuffled_block(data, lineLimit=lineLimit)
else:
while (True):
data = read_block(numLines=blockSize)
numLines = len(data)
if (numLines == 0): break
if (lineLimit == None): blockLimit = None
elif (lineLimit > numLines): blockLimit = numLines
else: blockLimit = lineLimit
write_shuffled_block(data, lineLimit=blockLimit)
if (lineLimit != None):
lineLimit -= blockLimit
if (lineLimit == 0): break
def show_channels_created(channels, messages, top=10):
channel_messages = Counter()
for record in messages:
channel_messages[record.channel.name] += 1
user_channels = Counter(_.creator.name for _ in channels)
users = [user for user, _ in user_channels.most_common()]
palette = sns.color_palette('deep', len(users))
fig, ax = plt.subplots()
X = []
Y = []
sizes = []
texts = []
colors = []
random_seed(42)
order = []
for record in sorted(channels, key=lambda _: _.created):
x = record.created
X.append(x)
user = record.creator.name
y = users.index(user)
if y >= top:
y = top
y += (random() - 0.5) / 2
Y.append(y)
size = channel_messages[record.name]
sizes.append(size)
text = record.name
order.append(text)
text = str(len(order))
text = ax.text(x, y, text, size=7)
texts.append(text)
color = palette[users.index(user)]
colors.append(color)
sizes = np.sqrt(sizes) + 1
ax.scatter(X, Y, s=sizes, c=colors, lw=0, alpha=0.75)
labels = users[:top] + [OTHER]
ax.set_yticks(range(top + 1))
ax.set_yticklabels(labels)
adjust_text(
texts,
arrowprops={
'arrowstyle': '-',
'color': 'silver',
'lw': 0.5
},
)
fig.autofmt_xdate()
# ax.set_axis_bgcolor('white')
ax.set_xlabel('created')
for chunk in wrap_sequence(list(enumerate(order, 1)), 5):
for index, channel in chunk:
print index, channel,
print
def main():
global debug
global oscType,gain,leftTempo,rightTempo,singleEar
# parse the command line
oscType = TriOsc
gain = 0.5
leftTempo = 100
rightTempo = 150
duration = 10.0
singleEar = None
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "SinOsc"):
oscType = SinOsc
elif (arg == "SawOsc"):
oscType = SawOsc
elif (arg == "TriOsc"):
oscType = TriOsc
elif (arg == "SqrOsc"):
oscType = SqrOsc
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("LB=")) or (arg.startswith("--leftbeat=")):
leftTempo = float_or_fraction(argVal)
elif (arg.startswith("RB=")) or (arg.startswith("--rightbeat=")):
rightTempo = float_or_fraction(argVal)
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--single=left"):
singleEar = "left"
elif (arg == "--single=right"):
singleEar = "right"
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(shred_per_ear(duration*zook.sec))
zook.run()
def main():
global debug
global oscType,gain,reverbGain,echos,quarterNote
# parse the command line
oscType = SinOsc
gain = 0.5
reverbGain = 0.6
echos = [60*zook.msec,80*zook.msec,100*zook.msec]
quarterNote = 0.3 * zook.sec
duration = 3.0
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "SinOsc"):
oscType = SinOsc
elif (arg == "SawOsc"):
oscType = SawOsc
elif (arg == "TriOsc"):
oscType = TriOsc
elif (arg == "SqrOsc"):
oscType = SqrOsc
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("RG=")) or (arg.startswith("--reverbgain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("E=")) or (arg.startswith("--echos=")):
echos = [float_or_fraction(d)*zook.msec for d in argVal.split(",")]
elif (arg.startswith("Q=")) or (arg.startswith("--quarter=")):
quarterNote = float_or_fraction(argVal) * zook.sec
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(inlet_test(duration*zook.sec))
zook.run()
def start_batch(batch, seed=None):
if seed is None:
random_seed(time.time())
# Seed for numpy randomisation is 0 to 2^32-1, inclusive.
seed = randint(0, (1 << 32) - 1)
print(f"Simulating with seed {seed}")
from ev3sim.batched_run import batched_run
batched_run(batch, seed)
def _trial_greedy_ssa_path_and_cost(r, inputs, output, size_dict, choose_fn, cost_fn):
"""A single, repeatable, greedy trial run. Returns ``ssa_path`` and cost.
"""
if r == 0:
# always start with the standard greedy approach
choose_fn = None
random_seed(r)
ssa_path = paths.ssa_greedy_optimize(inputs, output, size_dict, choose_fn, cost_fn)
cost, size = ssa_path_compute_cost(ssa_path, inputs, output, size_dict)
return ssa_path, cost, size
def main():
global debug
global gain,delay,numChannels,mix,quarterNote
# parse the command line
# try M=5/7 G=3
gain = 0.7
delay = 100 * zook.msec
numChannels = 1
mix = 0.5
quarterNote = 0.3 * zook.sec
duration = 3.0
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("D=")) or (arg.startswith("--delay=")):
delay = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("--channels=")):
numChannels = int(argVal)
elif (arg.startswith("M=")) or (arg.startswith("--mix=")):
mix = float_or_fraction(argVal)
elif (arg.startswith("Q=")) or (arg.startswith("--quarter=")):
quarterNote = float_or_fraction(argVal) * zook.sec
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(echo_test(duration*zook.sec))
zook.run()
def _evenly_split_it(it, seed=None):
# Determine the length of the sequence
cnt_it, it = tee(it, 2)
cnt = max(i for i, _ in enumerate(cnt_it, start=1))
if seed is not None:
random_seed(seed)
# Create a set with the elements to use for the first sequence
one_indices = set(sample(xrange(cnt), (cnt + 1) / 2))
# Create and filter two new iterators to give different elements
nfilt_one, nfilt_two = tee(it, 2)
return ((e for i, e in enumerate(nfilt_one) if i in one_indices),
(e for i, e in enumerate(nfilt_two) if i not in one_indices))
def _evenly_split_it(it, seed=None):
# Determine the length of the sequence
cnt_it, it = tee(it, 2)
cnt = max(i for i, _ in enumerate(cnt_it, start=1))
if seed is not None:
random_seed(seed)
# Create a set with the elements to use for the first sequence
one_indices = set(sample(xrange(cnt), (cnt + 1) / 2))
# Create and filter two new iterators to give different elements
nfilt_one, nfilt_two = tee(it, 2)
return ((e for i, e in enumerate(nfilt_one) if i in one_indices),
(e for i, e in enumerate(nfilt_two) if i not in one_indices))
def get_nlpba_down_set():
random_seed(0x8a3403ed)
paths = [d.path for d in _get_documents(NLPBA_DOWN_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
return (
(d for d in _get_documents(NLPBA_DOWN_DIR) if d.path in train_paths),
(d for d in _get_documents(NLPBA_DOWN_DIR) if d.path in dev_paths),
(d for d in _get_documents(NLPBA_DOWN_DIR) if d.path in test_paths),
)
def main():
global debug
global oscType,gain,tempo
# parse the command line
oscType = TriOsc
gain = 0.2
tempo = 400
duration = 10.0
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "SinOsc"):
oscType = SinOsc
elif (arg == "SawOsc"):
oscType = SawOsc
elif (arg == "TriOsc"):
oscType = TriOsc
elif (arg == "SqrOsc"):
oscType = SqrOsc
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("B=")) or (arg.startswith("--beat=")):
tempo = float_or_fraction(argVal)
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(slide_tones(duration*zook.sec))
zook.run()
def generate_move_process(generate_move: cu.GenMove, moves_q: mp.Queue):
from traceback import StackSummary
from random import seed as random_seed
import io
from time import time
import pickle
from contextlib import redirect_stderr, redirect_stdout
logger = logging.getLogger(__name__)
f_stderr, f_stdout = io.StringIO(), io.StringIO()
gma: GenMoveArgs = moves_q.get()
np.random.seed(gma.seed)
random_seed(gma.seed)
try:
with redirect_stdout(f_stdout), redirect_stderr(f_stderr):
t0 = time()
returned = generate_move(gma.board, gma.player, gma.state)
saved_state = None
if isinstance(returned, tuple):
action = returned[0]
if len(returned) > 1:
saved_state = returned[1]
else:
action = returned
move_time = time() - t0
stdout, stderr = f_stdout.getvalue(), f_stderr.getvalue()
result = GenMoveSuccess(stdout, stderr, move_time, action, saved_state)
except Exception as e:
logger.exception("An exception was thrown by the agent.")
error_msg = repr(e) + "\n"
extracted_list = traceback.extract_tb(e.__traceback__)
for item in StackSummary.from_list(extracted_list).format():
error_msg += str(item)
stdout, stderr = f_stdout.getvalue(), f_stderr.getvalue()
result = GenMoveFailure(stdout, stderr, error_msg)
try:
moves_q.put(result)
except pickle.PickleError:
logger.exception(
"Internal error in trying to send the result, probably caused by saved_state"
)
moves_q.put(GenMoveSuccess(stdout, stderr, move_time, action, None))
def get_nlpba_down_set():
random_seed(0x8a3403ed)
paths = [d.path for d in _get_documents(NLPBA_DOWN_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
return (
(d for d in _get_documents(NLPBA_DOWN_DIR)
if d.path in train_paths),
(d for d in _get_documents(NLPBA_DOWN_DIR)
if d.path in dev_paths),
(d for d in _get_documents(NLPBA_DOWN_DIR)
if d.path in test_paths),
)
def createVoroMan(boundingBox, grid, randomishness=1.0, seed=None):
pMin, pMax = boundingBox
size = Vec2(pMax) - Vec2(pMin)
if seed: save = True
else:
seed = iRand(0, maxint)
print 'seed:', seed
save = False
random_seed(seed)
centroids = randomGrid_square(grid, WindowSize, randomishness)
tooSmall = size.x / grid / TOOSMALL
print 'tooSmall:', tooSmall
polies = VoroMan(
centroids, boundingBox, tooSmall
) # bounding box not really implemented yet. Only size is used.
fp = "%i_%i.pickle" % (grid, seed)
if save: pickle.dump(polies, open(fp, "wb"))
return polies
def get_nlpba_set():
random_seed(0x25e453f9)
paths = [d.path for d in _get_documents(NLPBA_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return (
(d for d in _get_documents(NLPBA_DIR) if d.path in train_paths),
(d for d in _get_documents(NLPBA_DIR) if d.path in dev_paths),
(d for d in _get_documents(NLPBA_DIR) if d.path in test_paths),
)
def get_calbc_cii_set():
random_seed('0xc78e13c3')
#XXX: We down-sample to 250, 125, 125
paths = sample([d.path for d in _get_documents(CALBC_CII_DIR)], 500)
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return (
(d for d in _get_documents(CALBC_CII_DIR) if d.path in train_paths),
(d for d in _get_documents(CALBC_CII_DIR) if d.path in dev_paths),
(d for d in _get_documents(CALBC_CII_DIR) if d.path in test_paths),
)
def youtube_get_random_channel_video(channel_name):
global last_youtube_item_hash
channel_url = 'http://gdata.youtube.com/feeds/api/users/{0}/uploads?&v=2&max-results=6&alt=jsonc&hd'.format(channel_name)
json_data = urllib2.urlopen(channel_url)
data = json_load(json_data)
items = data['data']['items']
number_of_items = len(items)
random_seed()
random_item = None
if number_of_items <= 1:
random_item = items[0]
else:
while random_item == None or hash(random_item['player']['default']) == last_youtube_item_hash:
random_item = items[randrange(0, number_of_items)]
last_youtube_item_hash = hash(random_item['player']['default'])
item_url = random_item['player']['default']
find_and_play_video(item_url)
def __init__(self, master_file_path, output_dir_path, adapter_str="rasa",
local=False, seed=None):
if local:
self.output_dir_path = os.path.dirname(master_file_path)
else:
self.output_dir_path = os.getcwd()
if output_dir_path is None:
self.output_dir_path = os.path.join(self.output_dir_path, "output")
else:
self.output_dir_path = os.path.join(self.output_dir_path,
output_dir_path)
# Initialize the random number generator
if seed is not None:
random_seed(seed)
self.adapter = adapter_factory.create_adapter(adapter_str)
self.parser = Parser(master_file_path)
self.generator = None
def make_results_reproducible(model_is_convolutional: bool = False) -> None:
"""
Make the subsequent instructions produce purely deterministic outputs
by fixing all the relevant seeds:
"""
random_seed(0)
_ = numpy_seed(0)
_ = torch_manual_seed(0)
# since GPU computations may introduce additional stochasticity with
# their convolutional operation optimizations:
if model_is_convolutional:
if cuda_is_available():
# disabling benchmarking and choosing among convolution operation
# implementation alternatives, which is stochastic based due to
# noise and hardware:
cudnn.benchmark = False
# ansuring deterministic algorithms for colvolutional operations
# are employed:
cudnn.deterministic = True
def multi_game_experiment(num_games, num_players, seed=None):
"""
Returns durations of a number of games when playing with given seed.
Arguments
---------
num_games : int
Number of games to play
num_players : int
Number of players in the game
seed : int
Seed used to initialise the random number generator
Returns
-------
num_moves : list
List with the number of moves needed in each game.
"""
random_seed(seed)
return multiple_games(num_games, num_players)
def main(screen: Any, total_players: int, human_players: int, board_size: int, seed: int):
noecho()
cbreak()
start_color()
use_default_colors()
screen.keypad(True)
screen.border()
random_seed(seed)
ai_players = total_players - human_players
players = [
*[Human() for _ in range(human_players)],
*[AI() for _ in range(ai_players)],
]
game = Game(players, board_size, screen)
while game.winner is None:
game.next_turn()
endwin()
def get_grec_set():
random_seed(0x763f059c)
paths = [d.path for d in _get_documents(GREC_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return [
_filter_generic_grec_terms(it) for it in (
(d for d in _get_documents(GREC_DIR) if d.path in train_paths),
(d for d in _get_documents(GREC_DIR) if d.path in dev_paths),
(d for d in _get_documents(GREC_DIR) if d.path in test_paths),
)
]
def _trial_greedy_ssa_path_and_cost(
r: int,
inputs: List[ArrayIndexType],
output: ArrayIndexType,
size_dict: Dict[str, int],
choose_fn: Any,
cost_fn: Any,
) -> Tuple[PathType, int, int]:
"""A single, repeatable, greedy trial run. **Returns:** ``ssa_path`` and cost."""
if r == 0:
# always start with the standard greedy approach
choose_fn = None
random_seed(r)
ssa_path = paths.ssa_greedy_optimize(inputs, output, size_dict, choose_fn,
cost_fn)
cost, size = ssa_path_compute_cost(ssa_path, inputs, output, size_dict)
return ssa_path, cost, size
def get_nlpba_set():
random_seed(0x25e453f9)
paths = [d.path for d in _get_documents(NLPBA_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return (
(d for d in _get_documents(NLPBA_DIR)
if d.path in train_paths),
(d for d in _get_documents(NLPBA_DIR)
if d.path in dev_paths),
(d for d in _get_documents(NLPBA_DIR)
if d.path in test_paths),
)
def get_grec_set():
random_seed(0x763f059c)
paths = [d.path for d in _get_documents(GREC_DIR)]
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return [_filter_generic_grec_terms(it) for it in (
(d for d in _get_documents(GREC_DIR)
if d.path in train_paths),
(d for d in _get_documents(GREC_DIR)
if d.path in dev_paths),
(d for d in _get_documents(GREC_DIR)
if d.path in test_paths),
)]
def main():
args = parse_args()
with open(args.config) as f:
configs = json_load(f)
filenames = []
for n, config in enumerate(configs):
random_seed(args.seed)
fn_name = config.pop('_fn')
os_makedirs(args.path, exist_ok=True)
filename = '{:03}.{}.txt'.format(n, fn_name)
filename = path_join(args.path, filename)
with open(filename, 'w') as f:
REGISTERED_FUNCTIONS[fn_name](f, **config)
filenames.append(filename)
LOG.debug('# %s', filename)
return {
'outputs': filenames
}
def get_calbc_cii_set():
random_seed('0xc78e13c3')
#XXX: We down-sample to 250, 125, 125
paths = sample([d.path for d in _get_documents(CALBC_CII_DIR)], 500)
train_paths = set(sample(paths, len(paths) / 2))
paths = [p for p in paths if p not in train_paths]
dev_paths = set(sample(paths, len(paths) / 2))
test_paths = set([p for p in paths if p not in dev_paths])
del paths
#print 'train', train_paths
#print 'dev', dev_paths
#print 'test', test_paths
return (
(d for d in _get_documents(CALBC_CII_DIR)
if d.path in train_paths),
(d for d in _get_documents(CALBC_CII_DIR)
if d.path in dev_paths),
(d for d in _get_documents(CALBC_CII_DIR)
if d.path in test_paths),
)
def __init__(self,
player_field,
board=None,
seed=None,
randomize_players=True):
"""
Parameters
----------
player_field: list
board: Board
seed: int
randomize_players: bool
"""
self.board = board
random_seed(seed)
self.player_field = player_field
self.randomize = randomize_players
if board is None:
self.board = Board()
else:
self.board = board
self.results = []
def find_all_interterm_conversion_rates_start(pytrends_obj,
terms_list,
time_start,
time_end,
max_ratio=PYTRENDS_MAX_RATIO,
seed=1,
starting_term=None):
time_start = datetime.strptime(time_start, DT_FORMAT)
time_end = datetime.strptime(time_end, DT_FORMAT)
# Randomising the starting ref term or using the one provided by the user.
# This part could be changed to decrease the risk of getting stuck.
if starting_term is None:
random_seed(seed)
ref_term = terms_list[randint(0, len(terms_list) - 1)]
else:
ref_term = starting_term
# Removing the starting ref term from "rest".
rest = set(terms_list)
rest.remove(ref_term)
rest = list(rest)
ref_candidates = set()
conversion_ratio_list = list()
new_rest = set()
current_minimum = (None, max_ratio + 1)
current_maximum = (None, 1.0 / (max_ratio + 1))
return find_all_interterm_conversion_rates_continue(
pytrends_obj, rest, ref_term, 0, time_start, time_end, ref_candidates,
conversion_ratio_list, new_rest, max_ratio, current_minimum,
current_maximum)
def __init__(self, start_place, home, rand_seed=None):
"""
Initialise the simulation
Arguments
---------
start_place : int
The walker's initial position
home : int
The walk ends when the walker reaches home
rand_seed : int
Random generator seed
"""
self.start = start_place
self.home = home
self.seed = random_seed(rand_seed)
def mcmc(ks, geneToCases, num_patients, method, test, geneToP, seed, annotations=set(), verbose=0, step_len=100, nchains=1, niters=1000, alpha=1):
if verbose > 0: print '-' * 33, 'Running MCMC', '-' * 33
# Set up a local version of the weight function
if test == WRE:
def _test(M, X, T, Z, tbl):
return wre_test(T, X, [ geneToP[g] for g in M ], method=method)
elif test == RE:
def _test(M, X, T, Z, tbl):
return re_test(T, X, tbl, method=method)
else:
raise NotImplementedError('Test "{}" not implemented with MCMC'.format(testToName[test]))
def _weight(M):
if M not in setToPval:
setToPval[M] = _test(M, *setToObs[M])
return -np.log10(setToPval[M]**alpha)
def _valid_set(M):
# Compute or retrieve the observed statistics
M = frozenset(M)
if M not in setToObs:
setToObs[M] = observed_values(M, num_patients, geneToCases)
X, T, Z, tbl = setToObs[M]
# We don't allow sets with T <= Z or with multiple annotations
# (since these are often trivially exclusive)
if T <= Z or len(M & annotations) > 1:
return False
else:
return True
def _collection_weight(collection):
return sum( _weight(M) for M in collection )
def _to_collection(solution):
return frozenset( frozenset(M) for M in solution.values() )
# Compute the acceptance ratio
def _log_accept_ratio( W_current, W_next ): return W_next - W_current
# Set up PRNG, sample space, and output
random_seed(seed)
t = len(ks)
genespace = geneToCases.keys()
setsToFreq = [ defaultdict(int) for _ in xrange(nchains) ]
setToPval, setToObs = dict(), dict()
for c in xrange(nchains):
if verbose > 0: print '- Experiment', c+1
# Seed Markov chain
soln, assigned = choose_random_set(ks, genespace)
while not all( _valid_set(M) for M in _to_collection(soln) ):
soln, assigned = choose_random_set(ks, genespace)
weight = _collection_weight( _to_collection(soln) )
# Run MCMC
progress_step = float(np.ceil(niters / 72.))
itera = 0
while itera < niters:
# Simple progress bar
if verbose > 0 and (itera % progress_step == 0):
sys.stdout.write("\r[%-72s] %d%%" % ('='*int(np.ceil(itera / progress_step)) + '>', int(np.ceil(100.*itera / niters))))
sys.stdout.flush()
# Sample the next gene to swap in/around the set
next_soln = dict( (index, set(M)) for index, M in soln.iteritems() )
next_assigned = dict(assigned.items())
next_gene = choice(genespace)
# There are two possibilities for the next gene
# 1) The gene we sampled is already in the current solution. In this
# case, we swap the gene with another gene in a different set.
if next_gene in next_assigned:
# if we only have one set, we can't swap between sets
if t == 1: continue
i = next_assigned[next_gene]
swap_gene = choice([ g for g in next_assigned.keys() if g not in next_soln[i] ])
j = next_assigned[swap_gene]
next_assigned[swap_gene] = i
next_soln[i].add(swap_gene)
next_soln[i].remove( next_gene )
next_assigned[next_gene] = j
next_soln[j].remove(swap_gene)
next_soln[j].add(next_gene)
if not (_valid_set(next_soln[j]) and _valid_set(next_soln[i])):
continue
# 2) The gene is not in the current solution. In this case, we choose
# a random gene in the solution to remove, and add the next gene.
else:
swap_gene = choice(next_assigned.keys())
j = next_assigned[swap_gene]
del next_assigned[swap_gene]
next_assigned[next_gene] = j
next_soln[j].remove(swap_gene)
next_soln[j].add(next_gene)
if not _valid_set(next_soln[j]): continue
# Compare the current soln to the next soln
next_weight = _collection_weight(_to_collection(next_soln))
if _log_accept_ratio( weight, next_weight ) >= np.log10(random()):
soln, weight, assigned = next_soln, next_weight, next_assigned
# Freeze sets after thinning the chain a certain number of iterations
itera += 1
if (itera+1) % step_len == 0:
setsToFreq[c][_to_collection(soln)] += 1
if verbose > 0:
print '\r[' + ('='*71) + '>] 100%'
# Merge the various chains
setsToTotalFreq = defaultdict(int)
for counter in setsToFreq:
for sets, freq in counter.iteritems():
setsToTotalFreq[sets] += freq
return setsToTotalFreq, setToPval, setToObs
def main():
assert (len(argv) == 3), "need the sampleID and number of trials, and nothing else"
sampleId = argv[1]
numTrials = int(argv[2])
random_seed("acorn")
explainFailure = False
path = "kmer_histograms"
# ask the curve fitter what the default paramters are
fitter = EnrichedHapDipFitter(path+"/"+sampleId+".mixed.kmer_dist")
paramNames = fitter.paramNames
defaultParams = fitter.default_params()
if (defaultParams == None):
print "(failed to get default params)"
if (explainFailure):
print "... return code ..."
print hdFitter.retCode
print "... stdout ..."
print hdFitter.stdout
print "... stderr ..."
print hdFitter.stderr
assert (False)
defaultParams = params_to_float(defaultParams)
# read the "good" parameters (usually produced by explore3_hap_dip)
fitFilename = path+"/"+sampleId+".mixed.fit"
f = file(fitFilename,"rt")
goodParams = params_from_text([line for line in f])
f.close()
for name in defaultParams:
assert (name in goodParams), \
"parameter \"%s\" missing from %s" % (name,fitFilename)
for name in goodParams:
assert (name in defaultParams), \
"extra parameter \"%s\" in %s" % (name,fitFilename)
goodParams = params_to_float(goodParams)
print params_to_text(paramNames,goodParams,defaultParams,
prefix="good:",prefix2="dflt:")
# run the convergence trials
convergenceCount = 0
for trialNumber in xrange(numTrials):
print "=== trial %d of %d ===" \
% (1+trialNumber,numTrials)
# choose initial params as a random point in hypercube between "good"
# and "bad"
initParams = dict(goodParams)
norm2Init = 0.0
for (paramIx,name) in enumerate(paramNames):
step = unit_random()
initParams[name] += step*(defaultParams[name]-goodParams[name])
norm2Init += step*step
normInit = sqrt(norm2Init) / len(paramNames)
fitter.set_params(initParams)
fitParams = fitter.fit()
if (fitParams == None):
print params_to_text(paramNames,initParams,prefix="init-[%d]:" % trialNumber)
print "normInit: %.8f" % normInit
print "(failure or non-convergence)"
if (explainFailure):
print "... return code ..."
print fitter.retCode
print "... stdout ..."
print fitter.stdout
print "... stderr ..."
print fitter.stderr
continue
print params_to_text(paramNames,initParams,fitParams,
prefix="init+[%d]:" % trialNumber,
prefix2="cvrg[%d]:" % trialNumber)
fitParams = params_to_float(fitParams)
dGood = vector_distance(fitParams,goodParams)
print "normInit: %.8f" % normInit
print "dGood: %.8f" % dGood
convergenceCount += 1
print "%d of %d trials converged" % (convergenceCount,numTrials)
def main():
global debug
global showEnvelope,oscType,duty,gain
global envType,attack,decay,sustain,release
global numDelays,quarterNote
# parse the command line
showEnvelope = False
oscType = SinOsc
duty = None
gain = 0.5
envType = Envelope
attack = 10 * zook.msec
decay = 150 * zook.msec
sustain = 0.2
release = 150 * zook.msec
numDelays = None
quarterNote = 0.3 * zook.sec
duration = 5.0
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "--show:envelope"):
showEnvelope = True
elif (arg == "SinOsc"):
oscType = SinOsc
elif (arg == "SawOsc"):
oscType = SawOsc
elif (arg == "TriOsc"):
oscType = TriOsc
elif (arg == "SqrOsc"):
oscType = SqrOsc
elif (arg.startswith("--duty=")):
duty = float_or_fraction(argVal)
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg == "ADSR"):
envType = ADSR
elif (arg.startswith("ADSR=")):
envType = ADSR
(attack,decay,sustain,release) = argVal.split(",")
attack = float_or_fraction(attack) * zook.msec
decay = float_or_fraction(decay) * zook.msec
sustain = float_or_fraction(sustain)
release = float_or_fraction(release) * zook.msec
elif (arg.startswith("A=")) or (arg.startswith("--attack=")):
attack = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("D=")) or (arg.startswith("--decay=")):
decay = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("S=")) or (arg.startswith("--sustain=")):
sustain = float_or_fraction(argVal)
elif (arg.startswith("R=")) or (arg.startswith("--release=")):
release = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("N=")) or (arg.startswith("--delays=")):
numDelays = int(argVal)
elif (arg.startswith("Q=")) or (arg.startswith("--quarter=")):
quarterNote = float_or_fraction(argVal) * zook.sec
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(envelope_test(duration*zook.sec))
zook.run()
return process(ayat)
def draw(img_rgb, template, ayat, output = None):
w = template.shape[1]
h = template.shape[0]
for pt in ayat:
cv2.rectangle(img_rgb, pt, (pt[0] + w, pt[1] + h), (0,0,255,255), 2)
if output is not None:
cv2.imwrite(output, img_rgb)
def r(): return randint(128, 255)
# we want random colors, but same colors per page for every run
# because we want to be able to compare runs for faster reviewing
random_seed(10000)
aya_colors = [(r(), r(), r(), 255) for i in range(0, 100)]
last_page = None
last_aya = None
last_color = None
font = cv2.FONT_HERSHEY_SIMPLEX
def output_aya_segment(vals, image, file):
global last_page, last_aya, last_color
file.write(
'insert into glyphs values(NULL, %d, %d, %d, %d, %d, %d, %d, %d, %d);\n' % vals)
# schema: id, pageId, lineId, suraId, verseId, indexId, left, right, top, bottom
left_top = (int(vals[5]), int(vals[7]))
right_bottom = (int(vals[6]), int(vals[8]))
def main():
# parse the command line
numSequences = None
sequenceLength = None
nameTemplate = None
wrapLength = 100
alphabet = "ACGT"
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=", 1)[1]
if (arg.startswith("--name=")) or (arg.startswith("--names=")):
nameTemplate = argVal
elif (arg == "--noname") or (arg == "--nonames"):
nameTemplate = ""
elif (arg.startswith("--wrap=")):
wrapLength = int(argVal)
if (wrapLength <= 0): wrapLength = None
elif (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg.startswith("--n=")) or (arg.startswith("N=")):
numSequences = int_with_unit(argVal)
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
elif (arg.startswith("{")) and (arg.endswith("}")):
alphabet = arg[1:-1]
if (alphabet == ""):
usage("what am I supposed to do with an empty alphabet?")
elif (sequenceLength == None):
if ("x" in arg):
(n, l) = arg.split("x", 1)
numSequences = int_with_unit(n)
sequenceLength = l
else:
sequenceLength = arg
else:
usage("unrecognized option: %s" % arg)
if (sequenceLength == None):
usage("you must tell me the sequence length")
elif (sequenceLength == "stdin"):
pass
else:
sequenceLength = int_with_unit(sequenceLength)
if (numSequences == None): numSequences = 1
if (nameTemplate == None):
if (numSequences == None):
numDigits = 9
else:
numDigits = 1
biggestNumber = 9
while (biggestNumber < numSequences):
numDigits += 1
biggestNumber = 10 * biggestNumber + 9
nameTemplate = "RAND_[%d]" % numDigits
# generate the sequences
if (nameTemplate == ""): namer = None
else: namer = RandomNamer(nameTemplate)
lineNum = seqNum = 0
while (True):
lineNum += 1
if (numSequences != None) and (seqNum >= numSequences): break
seqLen = sequenceLength
if (sequenceLength == "stdin"):
line = stdin.readline()
if (line == ""): break
line = line.strip()
if (line == ""): continue
try:
seqLen = int_with_unit(line)
if (seqLen < 0): raise ValueError
except ValueError:
assert (False), "bad length (line %d): %s" % (lineNum, line)
if (seqLen == 0): continue
seqNum += 1
seq = random_sequence(seqLen, alphabet=alphabet)
if (namer != None):
print ">%s" % namer.next()
if (wrapLength == None):
print seq
else:
for i in range(0, seqLen, wrapLength):
print seq[i:i + wrapLength]
def main():
fragmentLength = 100
stepLength = 50
shuffleEm = False
origin = "one"
headLimit = None
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=", 1)[1]
if (arg.startswith("--fragment=")):
fragmentLength = int(argVal)
elif (arg.startswith("--step=")):
stepLength = int(argVal)
elif (arg == "--shuffle"):
shuffleEm = True
elif (arg.startswith("--shuffle=")):
shuffleEm = True
random_seed(argVal)
elif (arg.startswith("--origin=")):
origin = argVal
if (origin == "0"): origin = "zero"
if (origin == "1"): origin = "one"
assert (origin in ["zero", "one"]), "can't understand %s" % arg
elif (arg.startswith("--head=")):
headLimit = int_with_unit(argVal)
elif (arg.startswith("--")):
usage("can't understand %s" % arg)
else:
usage("can't understand %s" % arg)
allN = "N" * fragmentLength
# process the sequences
if (shuffleEm):
fragments = []
fragNum = 0
for (name, seq) in fasta_sequences(stdin):
if (headLimit != None) and (fragNum > headLimit): break
seq = seq.upper()
for ix in xrange(0, len(seq) - fragmentLength, stepLength):
frag = seq[ix:ix + fragmentLength]
if (frag == allN): continue
fragNum += 1
if (headLimit != None) and (fragNum > headLimit):
print >> stderr, "limit of %d emitted fragments reached" % headLimit
break
if (origin == "zero"): header = ">%s_%d" % (name, ix)
else: header = ">%s_%d" % (name, ix + 1)
if (shuffleEm):
fragments += [(header, frag)]
else:
print header
print frag
if (shuffleEm):
shuffle(fragments)
for (header, frag) in fragments:
print header
print frag
def main():
# parse the command line
arraysFilename = None
motifs = []
sequenceName = None
sequenceLen = 0
numRepeats = None
genNeighbors = 0.0
genMixture = 0.0
lengthsFilename = None
minFill = None
errorProfile = None
catalogFilename = None
wrapLength = 100
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--arrays=")):
arraysFilename = argVal
elif (arg.startswith("--name=")):
sequenceName = argVal
elif (arg.startswith("--length=")) or (arg.startswith("--len=")) or (arg.startswith("L=")):
if (argVal.endswith("%")):
sequenceLen = float(argVal[:-1]) / 100.0
assert (sequenceLen >= 1.0)
sequenceLen = ("%",sequenceLen)
elif (argVal.startswith("+")):
sequenceLen = int_with_unit(argVal[1:])
assert (sequenceLen >= 0)
sequenceLen = ("+",sequenceLen)
else:
sequenceLen = int_with_unit(argVal)
assert (sequenceLen >= 0)
elif (arg.startswith("--repeats=")) or (arg.startswith("N=")):
numRepeats = int_with_unit(argVal)
assert (numRepeats > 0)
elif (arg.startswith("--motif:neighbor=")):
genNeighbors = parse_probability(argVal)
elif (arg.startswith("--motif:mixture=")):
genMixture = parse_probability(argVal)
elif (arg.startswith("--lengths=")):
lengthsFilename = argVal
elif (arg.startswith("--minfill=")) or (arg.startswith("F=")):
minFill = int(argVal)
if (minFill < 0):
print >>stderr, "WARNING: \"%s\" interpreted as no minimum fill" % argVal
minFill = None
if (minFill == 0):
minFill = None
elif (arg.startswith("--errors=")):
errorProfile = None
if (argVal in ["pacbio","pacbio.v3","pacbio.GIAB","pacbio.giab"]):
errorProfile = errorProfilePacbioV3
elif (argVal == "pacbio.v2"): # for historical reasons, v2 is an alias for v3
errorProfile = errorProfilePacbioV3
elif (argVal in ["pacbio.v1","pacbio.Guiblet","pacbio.guiblet"]):
errorProfile = errorProfilePacbioV1
elif (argVal in ["pacbio.readsim"]):
errorProfile = errorProfilePacbioReadsim
elif (argVal in ["nanopore","nanopore.v3","nanopore.GIAB","nanopore.giab"]):
errorProfile = errorProfileNanoporeV3
elif (argVal == "nanopore.v2"): # for historical reasons, v2 is an alias for v3
errorProfile = errorProfileNanoporeV3
elif (argVal in ["nanopore.v1","nanopore.Jain","nanopore.jain"]):
errorProfile = errorProfileNanoporeV1
elif (argVal in ["nanopore.readsim"]):
errorProfile = errorProfileNanoporeReadSim
elif (":" in argVal):
try:
errorProfile = parse_error_spec(argVal)
except ValueError:
pass
else:
p = parse_probability(argVal)
errorProfile = {"mm":p, "i":p, "d":p }
if (errorProfile == None):
usage("\"%s\" is not a valid error spec" % argVal)
subProb = errorProfile["mm"]
insOpenProb = errorProfile["i"]
delOpenProb = errorProfile["d"]
insExtendProb = delExtendProb = 0.0
elif (arg.startswith("--catalog=")):
catalogFilename = argVal
elif (arg.startswith("--wrap=")):
wrapLength = int(argVal)
if (wrapLength <= 0): wrapLength = None
elif (arg.startswith("--seed=")):
# nota bene: if the seed is a number, use it as a number, since
# string seeds can produce different sequences on
# different versions/builds of python
seed = argVal
try:
seed = int(seed)
except ValueError:
try: seed = float(seed)
except ValueError: pass
random_seed(seed)
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
elif (is_nucleotide_string(arg)):
motifs += [arg.upper()]
else:
usage("unrecognized option: %s" % arg)
if (arraysFilename != None):
if (motifs != []):
usage("command line <motif>s cannot be used with --arrays")
if (numRepeats != None):
usage("--repeats cannot be used with --arrays")
if (lengthsFilename != None):
usage("--lengths cannot be used with --arrays")
if (genNeighbors != 0.0):
usage("--motif:neighbor cannot be used with --arrays")
if (genMixture != 0.0):
usage("--motif:mixture cannot be used with --arrays")
elif (motifs == []):
usage("you have to give me at least one motif")
if (numRepeats == None) and (arraysFilename != None):
numRepeats = 1
# read the arrays file, if we have one
repeatLengths = {}
haveSpecificArrays = False
if (arraysFilename != None):
haveSpecificArrays = True
f = file(arraysFilename,"rt")
numRepeats = 0
for (length,motif,_) in read_arrays(f,arraysFilename):
numRepeats += 1
if (motif not in repeatLengths):
motifs += [(motif)]
repeatLengths[motif] = [length]
else:
repeatLengths[motif] += [length]
f.close()
if (motifs == []):
usage("array file \"%s\" contains no arrays" % arraysFilename)
# read the lengths file
if (repeatLengths == {}):
if (lengthsFilename == None):
lengths = read_integers(stdin)
for motif in motifs:
repeatLengths[motif] = lengths
elif ("{motif}" not in lengthsFilename):
f = file(lengthsFilename,"rt")
lengths = read_integers(f,lengthsFilename)
f.close()
for motif in motifs:
repeatLengths[motif] = lengths
else:
for motif in motifs:
motifLengthsFilename = lengthsFilename.replace("{motif}",motif)
f = file(motifLengthsFilename,"rt")
lengths = read_integers(f,motifLengthsFilename)
f.close()
repeatLengths[motif] = lengths
# generate the number and type of motifs we'll embed
#
# note: to satisfy the requirement that the same seed generates the same
# pre-error sequence, we should have no variance in the use of the
# PRNG until after we've generated that sequence; see "point A" below
embeddings = []
if (haveSpecificArrays):
for motif in motifs:
for length in repeatLengths[motif]:
strand = choice(["+","-"])
offset = choice(xrange(len(motif)))
embeddings += [(1.0,motif,motif,strand,offset,length)]
shuffle(embeddings)
else:
for _ in xrange(numRepeats):
motif = choice(motifs)
length = choice(repeatLengths[motif])
u = unit_random()
if (genNeighbors > 0) and (u < genNeighbors):
motif = motif_neighbor(motif)
(mix,motif2) = (1.0,motif)
elif (genMixture > 0) and (u < genNeighbors+genMixture):
(mix,motif2) = (0.5,motif_neighbor(motif))
else:
(mix,motif2) = (1.0,motif)
strand = choice(["+","-"])
offset = choice(xrange(len(motif)))
embeddings += [(mix,motif,motif2,strand,offset,length)]
totalRepeatBp = sum([length for (_,_,_,_,_,length) in embeddings])
# assign each repeat a position within the "fill" sequence; note that we
# might have more than one repeat assigned to the same position, in which
# case they will be back-to-back with no fill between them
if (type(sequenceLen) == tuple):
(op,sequenceLen) = sequenceLen
if (op == "%"):
sequenceLen = int(round(totalRepeatBp*sequenceLen))
else: # if (op == "+"):
sequenceLen = totalRepeatBp + sequenceLen
if (totalRepeatBp > sequenceLen):
fillBp = 0
if (sequenceLen > 0):
print >>stderr, "WARNING: length of embedded repeats (%d) exceeds specified" % totalRepeatBp
print >>stderr, " sequence length (%d); there will be no fill DNA" % sequenceLen
elif (minFill != None):
fillBp = sequenceLen - totalRepeatBp
totalMinFill = (numRepeats+1) * minFill
if (totalMinFill > fillBp):
print >>stderr, "WARNING: minimum fill of %d cannot be achieved" % minFill
print >>stderr, " total minimum fill (%d) exceeds total fill (%d)" % (totalMinFill,fillBp)
minFill = fillBp / (numRepeats+1)
fillBp -= minFill * (numRepeats+1)
else:
fillBp = sequenceLen - totalRepeatBp
fillPositions = [randint(0,fillBp) for _ in xrange(numRepeats)]
fillPositions.sort()
if (minFill != None):
fillBp += minFill * (numRepeats+1)
for rptNum in xrange(numRepeats):
fillPositions[rptNum] += (rptNum+1) * minFill
# generate the sequence
catalog = None
if (catalogFilename != None):
catalog = []
fillSeq = str(EchyDna(fillBp))
seq = []
seqPos = 0
prevEnd = 0
fillPos = 0
for (ix,pos) in enumerate(fillPositions):
if (fillPos < pos):
seq += [fillSeq[fillPos:pos]]
seqPos += pos - fillPos
fillPos = pos
(mix,motif,motif2,strand,offset,length) = embeddings[ix]
if (catalog != None):
c = CatalogEntry()
c.start = seqPos
c.end = seqPos+length
c.mix = mix
c.motif = motif
c.motif2 = motif2
c.strand = strand
c.repeatLength = length
c.offset = offset
catalog += [c]
enoughCopies = (length+offset+len(motif)-1) / len(motif)
if (strand == "-"): motif = reverse_complement(motif)
if (mix >= 1.0):
repeat = motif * enoughCopies
else:
repeat = []
for _ in xrange(enoughCopies):
if (unit_random() < mix): repeat += [motif]
else: repeat += [motif2]
repeat = "".join(repeat)
seq += repeat[offset:offset+length]
seqPos += length
prevEnd = seqPos
if (fillPos < fillBp):
seq += [fillSeq[fillPos:fillBp]]
seq = "".join(seq)
#=== point A: it's now safe to make additional use of the PRNG ===
# apply error profile
events = profile = None
if (argVal in ["pacbio","pacbio.v3","pacbio.GIAB","pacbio.giab"]):
errorProfile = errorProfilePacbioV3
elif (argVal == "pacbio.v2"): # for historical reasons, v2 is an alias for v3
errorProfile = errorProfilePacbioV3
elif (argVal in ["pacbio.v1","pacbio.Guiblet","pacbio.guiblet"]):
errorProfile = errorProfilePacbioV1
elif (argVal in ["pacbio.readsim"]):
errorProfile = errorProfilePacbioReadsim
elif (argVal in ["nanopore","nanopore.v3","nanopore.GIAB","nanopore.giab"]):
errorProfile = errorProfileNanoporeV3
elif (argVal == "nanopore.v2"): # for historical reasons, v2 is an alias for v3
errorProfile = errorProfileNanoporeV3
elif (argVal in ["nanopore.v1","nanopore.Jain","nanopore.jain"]):
errorProfile = errorProfileNanoporeV1
elif (argVal in ["nanopore.readsim"]):
errorProfile = errorProfileNanoporeReadSim
elif (type(errorProfile) == float):
eRate = errorProfile / 3.0;
profile = {"mm":eRate, "i":eRate, "d":eRate }
elif (type(errorProfile) == dict):
profile = dict(errorProfile)
if (profile != None):
print >>stderr, "(applying error profile mm=%.2f%% i=%.2f%% d=%.2f%%)" \
% (100*profile["mm"],100*profile["i"],100*profile["d"])
(seq,catalog,events) = apply_errors(profile,seq,catalog)
# write the sequence
if (sequenceName != None):
print ">%s" % sequenceName
if (wrapLength == None):
print seq
else:
for i in range(0,len(seq),wrapLength):
print seq[i:i+wrapLength]
# write the catalog
if (catalogFilename != None):
catalogF = file(catalogFilename,"wt")
if (sequenceName in [None,""]): seqNameForCatalog = "seq"
else: seqNameForCatalog = sequenceName
if (events == None):
print >>catalogF, "#%s\t%s\t%s\t%s\t%s\t%s\t%s" \
% ("chrom","start","end","motif","rptLen","len","fill")
else:
print >>catalogF, "#%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" \
% ("chrom","start","end","motif","rptLen","len","fill",
"mRatio","m","mm","i","d")
prevEnd = 0
for (catIx,c) in enumerate(catalog):
motifStr = c.motif
if (c.mix < 1.0): motifStr += "," + c.motif2
motifStr += ".%s%s" % (c.offset,c.strand)
if (events == None):
print >>catalogF, "%s\t%s\t%s\t%s\t%s\t%s\t%s" \
% (seqNameForCatalog,c.start,c.end,motifStr,
c.repeatLength,c.end-c.start,c.start-prevEnd)
else:
if (catIx in events):
(m,mm,i,d) = events[catIx]
mRatio = "%.1f%%" % (100.0*m/(m+mm+i+d))
else:
mRatio = m = mm = i = d = "NA"
print >>catalogF, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" \
% (seqNameForCatalog,c.start,c.end,motifStr,
c.repeatLength,c.end-c.start,c.start-prevEnd,
mRatio,m,mm,i,d)
prevEnd = c.end
catalogF.close()
def main():
# parse the command line
numValues = None
mu = 0.0
sigma = 1.0
roundEm = None
precision = None
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--mu=")):
mu = float_or_fraction(argVal)
elif (arg.startswith("--sigma=")):
sigma = float_or_fraction(argVal)
elif (arg == "--round"):
roundEm = "round"
elif (arg == "--floor"):
roundEm = "floor"
elif (arg == "--ceiling"):
roundEm = "ceiling"
elif (arg.startswith("--precision=")):
precision = int(argVal)
elif (arg.startswith("--seed=")):
# nota bene: if the seed is a number, use it as a number, since
# string seeds can produce different sequences on
# different versions/builds of python
seed = argVal
try:
seed = int(seed)
except ValueError:
try: seed = float(seed)
except ValueError: pass
random_seed(seed)
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
elif (numValues == None):
numValues = int_with_unit(arg)
else:
usage("unrecognized option: %s" % arg)
if (numValues == None):
numValues = 1
if (roundEm != None) and (precision != None):
usage("can't use --precision with --%s" % roundEm)
if (precision == None): vFmt = "%s"
elif (precision <= 0): vFmt = "%d."
else: vFmt = "%%.%df" % precision
# generate the values
for _ in xrange(numValues):
v = gauss(mu,sigma)
if (roundEm == "round"): v = int(round(v))
elif (roundEm == "floor"): v = int(floor(v))
elif (roundEm == "ceiling"): v = int(ceil (v))
print vFmt % v
def main():
assert (len(argv) == 1), "give me no arguments"
numTrials = 1000
random_seed("acorn")
explainFailure = False
path = "kmer_histograms"
#sampleId = "mixedB"
#defaultParams = {"zp.copy.y" : 3.000,
# "zp.copy.hom" : 3.000,
# "zp.copy.het" : 3.000,
# "p.e" : 0.942,
# "shape.e" : 3.000,
# "scale.e" : 1.000,
# "p.y" : 0.900,
# "u.y" : 64.000,
# "sd.y" : 14.826,
# "shape.y" : 0.000,
# "p.hom" : 0.800,
# "u.hom" : 5.120,
# "sd.hom" : 1.186,
# "var.het" : 1.407}
#goodParams = {"zp.copy.y" : 2.042,
# "zp.copy.hom" : 3.157,
# "zp.copy.het" : 17.795,
# "p.e" : 0.935,
# "shape.e" : 0.096,
# "scale.e" : 0.465,
# "p.y" : 0.621,
# "u.y" : 68.084,
# "sd.y" : 8.626,
# "shape.y" : 0.057,
# "p.hom" : 0.853,
# "u.hom" : 11.101,
# "sd.hom" : 3.600,
# "var.het" : 10.916}
sampleId = "apple_E12_L150_D80_K25"
defaultParams = {"zp.copy.y" : 3.000,
"zp.copy.hom" : 3.000,
"zp.copy.het" : 3.000,
"p.e" : 0.940,
"shape.e" : 3.000,
"scale.e" : 1.000,
"p.y" : 0.900,
"u.y" : 62.000,
"sd.y" : 16.309,
"shape.y" : 0.000,
"p.hom" : 0.800,
"u.hom" : 4.960,
"sd.hom" : 1.305,
"var.het" : 1.702}
goodParams = {"zp.copy.y" : 2.047,
"zp.copy.hom" : 3.390,
"zp.copy.het" : 1.137,
"p.e" : 0.937,
"shape.e" : 0.114,
"scale.e" : 0.452,
"p.y" : 0.630,
"u.y" : 65.974,
"sd.y" : 8.666,
"shape.y" : 0.228,
"p.hom" : 0.818,
"u.hom" : 13.622,
"sd.hom" : 4.086,
"var.het" : 15.274}
fitter = EnrichedHapDipFitter(path+"/"+sampleId+".mixed.kmer_dist")
paramNames = fitter.paramNames
convergenceCount = 0
for trialNumber in xrange(numTrials):
print "=== trial %d of %d ===" \
% (1+trialNumber,numTrials)
# choose initial params as a random point in hypercube between "good"
# and "bad"
initParams = dict(goodParams)
norm2Init = 0.0
for (paramIx,name) in enumerate(paramNames):
step = unit_random()
initParams[name] += step*(defaultParams[name]-goodParams[name])
norm2Init += step*step
normInit = sqrt(norm2Init) / len(paramNames)
fitter.set_params(initParams)
fitParams = fitter.fit()
if (fitParams == None):
print params_to_text(paramNames,initParams,prefix="init-[%d]:" % trialNumber)
print "normInit: %.8f" % normInit
print "(failure or non-convergence)"
if (explainFailure):
print "... return code ..."
print fitter.retCode
print "... stdout ..."
print fitter.stdout
print "... stderr ..."
print fitter.stderr
continue
print params_to_text(paramNames,initParams,fitParams,
prefix="init+[%d]:" % trialNumber,
prefix2="cvrg[%d]:" % trialNumber)
fitParams = params_to_float(fitParams)
dGood = vector_distance(fitParams,goodParams)
print "normInit: %.8f" % normInit
print "dGood: %.8f" % dGood
convergenceCount += 1
print "%d of %d trials converged" % (convergenceCount,numTrials)
def main():
# parse the command line
symToCount = None
sequenceLen = None
wrap = None
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--length=")) or (arg.startswith("--len=")):
sequenceLen = int(argVal)
assert (sequenceLen > 0)
elif (arg.startswith("--wrap=")):
wrap = int(argVal)
assert (wrap > 0)
elif (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "--debug"):
debug += ["debug"]
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
elif (len(arg) > 2) and (arg[1] == ":"):
(sym,count) = (arg[0],arg[2:])
if (symToCount == None): symToCount = {}
assert (sym not in symToCount), \
"%s has more than one count (%d and %s)" \
% (sym,symToCount[sym],count)
try:
count = int(count)
if (count < 0): raise ValueError
except ValueError:
assert (False), \
"%s as not a valid count (%s:*d)" \
% (count,sym,count)
symToCount[sym] = count
else:
usage("unrecognized option: %s" % arg)
if (sequenceLen == None):
sequenceLen = 10000
# convert counts to probabilities, or use defaults if no counts given
if (symToCount != None):
totalCount = sum([symToCount[sym] for sym in symToCount])
assert (sum > 0), \
"you have to give me SOME positive counts!"
pMap = []
for sym in symToCount:
count = symToCount[sym]
if (count == 0): continue
pMap += [(count/float(totalCount),sym)]
else:
pMap = [(.077,"A"), (.093,"B"), (.044,"C"), (.091,"D"), (.126,"E"),
(.147,"F"), (.016,"G"), (.168,"H"), (.169,"I"), (.069,"J")]
# run the test
t = ProbabilityTable(pMap)
if ("table" in debug):
print >>stderr, t
if (wrap != None): line = []
for i in xrange(sequenceLen):
sym = t.choice()
if (wrap == None):
print sym
else:
line += [sym]
if (len(line) >= wrap):
print "".join(line)
line = []
if (wrap != None) and (len(line) > 0):
print "".join(line)
def main():
global debug
# parse the command line
formulas = None
requirements = None
prohibitions = None
useHeader = False
headerPrefix = "#"
removeColumns = None
nameColumns = None
divByZero = "NA"
separator = "\t"
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=", 1)[1]
if (arg.startswith("--require:")):
if (requirements == None): requirements = []
argVal = arg.split(":", 1)[1]
requirements += [argVal.strip()]
elif (arg.startswith("--prohibit:")) or (arg.startswith("--reject:")):
if (prohibitions == None): prohibitions = []
argVal = arg.split(":", 1)[1]
prohibitions += [argVal.strip()]
elif (arg.startswith("--remove=")):
if (removeColumns == None): removeColumns = []
removeColumns += argVal.split(",")
elif (arg.startswith("--names=")) or (arg.startswith("--name=")):
if (nameColumns == None): nameColumns = []
nameColumns += argVal.split(",")
elif (arg.startswith("~")):
argVal = arg[1:]
if (nameColumns == None): nameColumns = []
nameColumns += argVal.split(",")
elif (arg == "--header"):
useHeader = True
headerPrefix = None
elif (arg.startswith("--header=")):
useHeader = True
headerPrefix = argVal
elif (arg.startswith("--divbyzero=")):
divByZero = argVal
elif (arg == "--spaces"):
separator = " "
elif (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "--debug"):
debug += ["debug"]
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
if (formulas == None): formulas = []
formulas += [arg]
if (formulas == None) and (requirements == None) and (removeColumns
== None):
usage("you must give me some action to perform!")
if (requirements == None): requirements = []
if (prohibitions == None): prohibitions = []
# parse the formulas
formulaText = formulas if (formulas != None) else []
formulas = []
for arg in formulaText:
if (useHeader): formula = parse_formula(arg, destIsColumn=False)
else: formula = parse_formula(arg, destIsColumn=True)
if (formula == None):
usage("unrecognized option: %s" % arg)
(name, format, dest, expression) = formula
if (name == None): name = expression
formulas += [(name, format, dest, expression)]
if (nameColumns != None):
for spec in nameColumns:
(name, colName) = spec.split(":")
formulas = [(name, None, None, colName)] + formulas
if (removeColumns == None): continue
if (colName not in removeColumns): removeColumns += [colName]
if ("formulas" in debug):
print >> stderr, "=== formulas ==="
for (name, format, dest, expression) in formulas:
print >> stderr, "%s %s %s %s" % (name, format, dest, expression)
if ("formulas" in debug) and (requirements != []):
print >> stderr, "=== requirements ==="
for criterion in requirements:
print >> stderr, "%s" % (criterion)
if ("formulas" in debug) and (prohibitions != []):
print >> stderr, "=== prohibitions ==="
for criterion in prohibitions:
print >> stderr, "%s" % (criterion)
# determine the minimum number of input columns we'll need
maxCol = -1
if (removeColumns == None): removeColumns = []
if (useHeader):
for (ix, name) in enumerate(removeColumns):
assert (name not in removeColumns[:ix]), \
"%s appears more than once in --remove" % name
neededColumns = None # (we'll fill this in when we read the header)
else:
removalNames = removeColumns
removeColumns = []
for name in removalNames:
col = parse_column(name)
assert (col not in removeColumns), \
"%s appears more than once in --remove" % name
removeColumns += [col]
if (removeColumns != []):
maxCol = max(removeColumns)
neededColumns = []
for (name, format, dest, expression) in formulas:
if (dest != None) and (dest > maxCol): maxCol = dest
for col in columns_needed(expression):
if (col > maxCol): maxCol = col
neededColumns += [col]
columnsNeeded = maxCol + 1
# process the lines
numColumns = None
lineNum = 0
for line in stdin:
lineNum += 1
line = line.strip()
isHeader = False
if (useHeader) and (lineNum == 1):
isHeader = True
if (headerPrefix != None):
assert (line.startswith(headerPrefix)), \
"expected first line to begin with \"%s\"" % headerPrefix
line = line[len(headerPrefix):].strip()
elif (headerPrefix != None) and (line.startswith(headerPrefix)):
isHeader = True
line = line[len(headerPrefix):].strip()
if (line == ""):
if (not isHeader): print line
elif (headerPrefix != None): print headerPrefix + line
else: print line
continue
def pregame(self):
random_seed(self.game['player_seed'])
self.decider.start(self.game)
def main():
global debug
global oscType,lfoFreq,gain,attack,decay,sustain,release,tootTime
# parse the command line
oscType = TriOsc
lfoFreq = 3.0
gain = .8
attack = 50 * zook.msec
decay = 150 * zook.msec
sustain = 0.6
release = 150 * zook.msec
tootTime = 500 * zook.msec
duration = 5.0
debug = []
for arg in argv[1:]:
if ("=" in arg):
argVal = arg.split("=",1)[1]
if (arg.startswith("--seed=")):
random_seed(argVal)
elif (arg == "TriOsc"):
oscType = TriOsc
elif (arg == "SqrOsc"):
oscType = SqrOsc
elif (arg.startswith("LF=")) or (arg.startswith("--lfofreq=")):
lfoFreq = float_or_fraction(argVal)
elif (arg.startswith("G=")) or (arg.startswith("--gain=")):
gain = float_or_fraction(argVal)
elif (arg.startswith("ADSR=")):
(attack,decay,sustain,release) = argVal.split(",")
attack = float_or_fraction(attack) * zook.msec
decay = float_or_fraction(decay) * zook.msec
sustain = float_or_fraction(sustain)
release = float_or_fraction(release) * zook.msec
elif (arg.startswith("A=")) or (arg.startswith("--attack=")):
attack = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("D=")) or (arg.startswith("--decay=")):
decay = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("S=")) or (arg.startswith("--sustain=")):
sustain = float_or_fraction(argVal)
elif (arg.startswith("R=")) or (arg.startswith("--release=")):
release = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("Q=")) or (arg.startswith("--toot=")):
tootTime = float_or_fraction(argVal) * zook.msec
elif (arg.startswith("T=")) or (arg.startswith("--dur=")) or (arg.startswith("--duration=")):
duration = float_or_fraction(argVal)
elif (arg == "--help"):
usage()
elif (arg.startswith("--debug=")):
debug += argVal.split(",")
elif (arg.startswith("--")):
usage("unrecognized option: %s" % arg)
else:
usage("unrecognized option: %s" % arg)
# run the test
UGen.set_debug(debug)
Shreduler.set_debug(debug)
zook.spork(triangle_wave_tooter(duration*zook.sec))
zook.run()
def Seed():
if argv[1:]: seed = int(argv[1])
else:
seed = iRand(0, maxint)
print 'seed:', seed
random_seed(seed)
def Seed(): if argv[1:]: seed=int(argv[1]) else: seed = iRand(0, maxint ) print 'seed:',seed random_seed(seed)