def preload_data(self):
print('Pre-loading the data. This may take a while...')
t = Timer()
t.tic()
self.is_preload = False
npy_path = os.path.join(
self.image_path, self.image_files[self.num_samples - 1]).replace(
'images', 'gt_npy').replace('.jpg', '.npz')
if os.path.isfile(npy_path):
pass
else:
os.makedirs(self.image_path.replace('images', 'images_resized'))
os.makedirs(self.image_path.replace('images', 'gt_npy'))
self.blob_list = [_ for _ in range(self.num_samples)]
for i in range(self.num_samples):
img, den, count = self.load_index(i)
den = den.astype(np.float32)
image_path = os.path.join(self.image_path, self.image_files[i])
img.save(image_path.replace('images', 'images_resized'),
quality=100)
save_npy(
image_path.replace('images',
'gt_npy').replace('.jpg', '.npz'), den)
if i % 50 == 0:
print("loaded {}/{} samples".format(i, self.num_samples))
duration = t.toc(average=False)
print('Completed loading ', len(self.blob_list), ' files, time: ',
duration)
self.is_preload = True
def __init__(self, config=None):
self.field_width = 10
self.field_height = 10
self.field_objects = []
for j in range(self.field_height):
self.field_objects.append(
[Objects.empty_field for i in range(self.field_width)])
self.checker = Checker(self)
self.player1 = Player(config.white, 1)
self.player2 = Player(config.black, 2)
self.timer1 = Timer(config.timer, config.duration)
self.timer2 = Timer(config.timer, config.duration)
self.cur_timer = None
self.new_game = False
self.is_first_move = True
self.is_over = False
self.cur_figure = None
self.cur_player = self.player1
self.cur_move = None
self.init_field()
self.was_taken = False
self.logger = Logger()
self.dump_logger = Logger()
self.moves_quantity = 0
self.game_file = config.game_file
self.pause = 0.2
self.load = config.load_game
self.loaded_moves = queue.Queue()
self.loaded_moves_amount = 0
if self.load:
self.load_moves()
self.interface = Interface(self)
def preload(self):
self.ncls = {
k: np.random.choice([0, 1], p=pro)
for k, pro in self.pro_dict.items()
}
self.choices = {
k: random.randint(0,
len(self.sample_dict[k][ncl]) - 1)
for k, ncl in self.ncls.items()
}
self.loaded = {}
indexs = [self.sample_dict[self.patch_list[i][0] + 1]\
[self.ncls[self.patch_list[i][0] + 1]]\
[self.choices[self.patch_list[i][0] + 1]]\
[0] for i in range(self.get_num_samples())]
indexs = set(indexs)
load_timer = Timer()
load_timer.tic()
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
for blob in executor.map(lambda i: (i, self.dataloader[i]),
indexs):
self.loaded[blob[0]] = blob[1]
print("re-prior crop: %f s" % load_timer.toc(average=False))
def main():
(_, select_range, cc_type, antisense) = \
tuple(sys.argv)
antisense = antisense.lower() == 'true'
strand = 'sense' if not antisense else 'antisense'
select_range = select_range.split('_')
select_range = int(select_range[0]), int(select_range[1])
cross_corr_path = cross_corr_sense_path
orfs = paper_orfs
if antisense:
cross_corr_path = cross_corr_antisense_path
orfs = antisense_orfs
child_name = get_name(select_range, cc_type)
print_fl("Running entropy for %d-%d, %s, %s" %
(select_range[0], select_range[1], cc_type, strand))
name = 'entropy_%s' % strand
timer = Timer()
print_fl("Reading Cross correlation...", end='')
cross_correlation = pd.read_hdf(cross_corr_path, 'cross_correlation')
print_fl("Done.")
timer.print_time()
print_fl()
print_fl("Computing entropy")
calculate_cc_summary_measure_range_type(orfs, cross_correlation, cc_type,
select_range, strand, timer)
child_done(name, WATCH_TMP_DIR, child_name)
def get_p1_mnase_by_TSS(mnase_data, p1_shift, orfs, time):
timer = Timer()
p1_shift = p1_shift.loc[p1_shift.index.isin(orfs.index.values)]
# sort by chromosome and start, for MNase-seq caching speedup
p1_shift_sorted_idx = p1_shift[[]].join(
orfs[['chr', 'start']]).sort_values(['chr', 'start']).index.values
all_pos = np.arange(-500, 501)
orf_nuc_mid_counts = p1_shift[[]].copy()
orf_nuc_start_counts = p1_shift[[]].copy()
orf_nuc_stop_counts = p1_shift[[]].copy()
for pos in all_pos:
orf_nuc_mid_counts[pos] = 0
orf_nuc_start_counts[pos] = 0
orf_nuc_stop_counts[pos] = 0
mnase_data = mnase_data[mnase_data.time == time]
i = 0
# for each +1 nucleosome, sort by shiftedness
# get mnase_fragments f
for orf_name, row in p1_shift.loc[p1_shift_sorted_idx].iterrows():
orf = orfs.loc[orf_name]
span = orf.TSS - 500, orf.TSS + 500
chrom = orf.chr
orf_nuc_mnase = filter_mnase(mnase_data,
start=span[0],
end=span[1],
chrom=chrom,
translate_origin=orf.TSS,
flip=(orf.strand == '-'),
length_select=(144, 174),
sample=time)
mid_counts = get_binned_counts(orf_nuc_mnase, 'mid')
start_counts = get_binned_counts(orf_nuc_mnase, 'start')
stop_counts = get_binned_counts(orf_nuc_mnase, 'stop')
n = len(mid_counts)
orf_nuc_mid_counts.loc[orf_name, :] = mid_counts.values.reshape(n)
orf_nuc_start_counts.loc[orf_name, :] = start_counts.values.reshape(n)
orf_nuc_stop_counts.loc[orf_name] = stop_counts.values.reshape(n)
# get mnase-seq at this orf's TSS
# get the counts of the start, stop, and mids of nucleosome sized fragments
if i % 400 == 0:
print("%d/%d - %s" % (i, len(p1_shift), timer.get_time()))
i += 1
return (orf_nuc_mid_counts, orf_nuc_start_counts, orf_nuc_stop_counts)
def test_enrich_raw_ares():
with Timer('Enriching test article'):
enrich_raw_ares(file_paths=[
'/Users/harlaa04/workspace/Hack-work/HACK-2020-Q1/data/toy_dataset/14295731'
])
with Timer('Enriching test article 2nd time'):
enrich_raw_ares(file_paths=[
'/Users/harlaa04/workspace/Hack-work/HACK-2020-Q1/data/toy_dataset/14295731'
])
def __init__(self):
self.accepted_moves = {
"UP": self._up,
"DOWN": self._down,
"LEFT": self._left,
"RIGHT": self._right,
}
self._state = {'field_size': FIELD_SIZE, 'player_size': PLAYER_SIZE, 'players': {}, 'fruits': {}}
self.fruit_timer = Timer(3, self.add_fruit)
def fold_cross_validation(X,
Y,
k=3,
times=[0, 7.5, 15, 30, 60, 120],
l_scale=1.,
l_bounds=(1, 10),
time=False,
log=False):
np.random.seed(1)
original_orfs = X.index.values
shuffled_orfs_idx = X.index.values.copy()
np.random.shuffle(shuffled_orfs_idx)
Y_predict = pd.DataFrame(index=shuffled_orfs_idx)
for t in times:
Y_predict[t] = 0.
N = len(X)
fold_size = N / k
timer = Timer()
last_fold_models = {}
for time in times:
for fold in range(k):
if log and fold % 1 == 0: print_fl("%d/%d" % ((fold + 1), k))
X_train, Y_train, X_test, Y_test = get_fold_slice(
X, Y, k, fold, time)
test_orfs = X_test.index
model = fit_gp(X_train.values, Y_train.values, l_scale, l_bounds)
Y_pred = model.predict(X_test.values)
r2 = r2_score(Y_test.values, Y_pred)
Y_predict.loc[test_orfs, time] = Y_pred
if log:
print_fl(("\t%s - %s - r2 = %.3f" %
(str(time), timer.get_time(), r2)))
last_fold_models[time] = model
if log:
print_fl('')
mse = MSE(Y.loc[shuffled_orfs_idx], Y_predict.loc[shuffled_orfs_idx])
r2 = mse[[]].copy()
for time in times:
r2.loc[time] = r2_score(Y.loc[shuffled_orfs_idx][time],
Y_predict.loc[shuffled_orfs_idx][time])
return last_fold_models, mse, r2, Y_predict.loc[original_orfs]
def test_model_origin(net,
data_loader,
save_output=False,
save_path=None,
test_fixed_size=-1,
test_batch_size=1,
gpus=None):
timer = Timer()
timer.tic()
net.eval()
mae = 0.0
mse = 0.0
detail = ''
if save_output:
print save_path
for i, blob in enumerate(data_loader.get_loader(test_batch_size)):
if (i * len(gpus) + 1) % 100 == 0:
print "testing %d" % (i + 1)
if save_output:
index, fname, data, mask, gt_dens, gt_count = blob
else:
index, fname, data, mask, gt_count = blob
with torch.no_grad():
dens = net(data)
if save_output:
image = data.squeeze_().mul_(torch.Tensor([0.229,0.224,0.225]).view(3,1,1))\
.add_(torch.Tensor([0.485,0.456,0.406]).view(3,1,1)).data.cpu().numpy()
dgen.save_image(
image.transpose((1, 2, 0)) * 255.0, save_path,
fname[0].split('.')[0] + "_0_img.png")
gt_dens = gt_dens.data.cpu().numpy()
density_map = dens.data.cpu().numpy()
dgen.save_density_map(gt_dens.squeeze(), save_path,
fname[0].split('.')[0] + "_1_gt.png")
dgen.save_density_map(density_map.squeeze(), save_path,
fname[0].split('.')[0] + "_2_et.png")
_gt_count = gt_dens.sum().item()
del gt_dens
gt_count = gt_count.item()
et_count = dens.sum().item()
del data, dens
detail += "index: {}; fname: {}; gt: {}; et: {};\n".format(
i, fname[0].split('.')[0], gt_count, et_count)
mae += abs(gt_count - et_count)
mse += ((gt_count - et_count) * (gt_count - et_count))
mae = mae / len(data_loader)
mse = np.sqrt(mse / len(data_loader))
duration = timer.toc(average=False)
print "testing time: %d" % duration
return mae, mse, detail
def __init__(self,
screen_width: int,
screen_height: int,
player1_command: str,
player2_command: str,
graph_mode=True):
self.graph_mode = graph_mode
# Inicializando pygame
pygame.init()
if self.graph_mode:
self.screen = pygame.display.set_mode(
(screen_width, screen_height))
pygame.display.set_caption("Ajedrez")
self.screen_height = screen_height
self.screen_width = screen_width
s_size = (screen_width, screen_height)
# Limite de jugadas
self.play_count = 0
# Obtienendo directorio para la carga de recursos
dirname = os.path.dirname(__file__)
# Creando tablero
path = os.path.join(dirname, "assets/Tablero.jpg")
skin = pygame.image.load(path)
self.chessboard = Chessboard(skin, screen_width, screen_height)
# Cargando imagenes de victoria
path = os.path.join(dirname, "assets/Player 1 win.png")
self.player1_win = pygame.image.load(path)
self.player1_win = pygame.transform.scale(self.player1_win, s_size)
path = os.path.join(dirname, "assets/Player 2 win.png")
self.player2_win = pygame.image.load(path)
self.player2_win = pygame.transform.scale(self.player2_win, s_size)
path = os.path.join(dirname, "assets/Empate.png")
self.empate = pygame.image.load(path)
self.empate = pygame.transform.scale(self.empate, s_size)
# Creando jugadores
self.player1 = Player(Player.PLAYER_ONE, Controller(player1_command))
self.player2 = Player(Player.PLAYER_TWO, Controller(player2_command))
self._add_knights()
self.player1_turn = True
self.done = False
self.winner = False
self.play_number = 0
self.timer = Timer()
def collect_mnase(mnase_seq, window, pos_chr_df,
pos_key='position', chrom_key='chromosome',
strand=None, set_index=False, log=False):
collected_mnase_eq = pd.DataFrame()
win_2 = window/2
timer = Timer()
if log:
print_fl("Collecting MNase-seq fragments for %d entries" % len(pos_chr_df))
print_fl("around a %d window" % window)
i = 0
for chrom in range(1, 17):
# get chromosome specific nucleosoems and MNase-seq
chrom_entries = pos_chr_df[pos_chr_df[chrom_key] == chrom]
if len(chrom_entries) == 0: continue
chrom_mnase = filter_mnase(mnase_seq, chrom=chrom)
# for each element in the dataset
for idx, entry in chrom_entries.iterrows():
# get MNase-seq fragments at pos_chr_df
# and 0 center
center = entry[pos_key]
nuc_mnase = filter_mnase(chrom_mnase, start=center-win_2, end=center+win_2)
# orient properly left to right (upstream to downstream)
if strand is None or entry[strand] == '+':
nuc_mnase.loc[:, 'mid'] = nuc_mnase.mid - center
# crick strand, flip
else:
nuc_mnase.loc[:, 'mid'] = center - nuc_mnase.mid
select_columns = ['chr', 'length', 'mid', 'time']
if set_index:
nuc_mnase['parent'] = idx
select_columns.append('parent')
# append to MNase-seq
collected_mnase_eq = collected_mnase_eq.append(nuc_mnase[select_columns])
# print_fl progress
if log and i % 200 == 0: print_fl("%d/%d - %s" % (i, len(pos_chr_df),
timer.get_time()))
i += 1
if log: timer.print_time()
return collected_mnase_eq
def demo(symbol_name, params_path, img_dir):
timer = Timer()
model = get_symbol(symbol_name)()
restore_weights(model, params_path)
for img_path in Path(img_dir).glob('*.png'):
img = Image.open(img_path)
inputs = setup_data(img)
timer.tic()
num = eval(model, inputs)
costs = timer.toc()
print("the number in image %s is %d || forward costs %.4fs" %
(img_path, num, costs))
print("average costs: %.4fs" % (timer.average_time))
def preload_data(self):
print ('Pre-loading the data. This may take a while...')
t = Timer()
t.tic()
self.blob_list = [_ for _ in range(self.num_samples)]
self.is_preload = False
for i in range(self.num_samples):
self.blob_list[i] = (self.load_index(i))
if i % 50 == 0:
print ("loaded {}/{} samples".format(i, self.num_samples))
duration = t.toc(average=False)
print ('Completed loading ' ,len(self.blob_list), ' files, time: ', duration)
self.is_preload = True
def web_login():
if current_user.is_authenticated:
return render_template("index.html")
if request.method == "GET":
return render_template("login.html", form=request.form)
username = request.form.get("login", None)
password = request.form.get("password", None)
if check_permissions(username, 3):
return render_template("login.html", form=request.form)
if check_password(username, password):
user_to_login = None
for user in users:
if user.id == username:
user_to_login = user
break
if user_to_login is not None:
login_user(user_to_login)
timers.append(Timer(user_to_login.name))
# monitors.append(ResourceChecker(user_to_login.name))
return render_template("index.html")
else:
return render_template("login.html", form=request.form)
else:
return render_template("login.html", form=request.form)
def roman_login():
if current_user.is_authenticated:
return render_template("rhome.html")
if request.method == "GET":
return render_template("rauthenticate.html", form=request.form)
username = request.form.get("login", None)
password = request.form.get("password", None)
if check_permissions(username, 3):
flash(
"You don't have permission to use this application. If you think you should have it, please contact your administrator to change it."
)
return render_template("rauthenticate.html", form=request.form)
if check_password(username, password):
user_to_login = None
for user in users:
if user.id == username:
user_to_login = user
break
if user_to_login is not None:
login_user(user_to_login)
timer = get_timer_monitor(timers, user_to_login.name)
if not timer:
timers.append(Timer(user_to_login.name))
return render_template("rhome.html")
else:
flash("Something went wrong. Please try again.")
return render_template("rauthenticate.html", form=request.form)
else:
flash("Your credentials were incorrect. Please try again.")
return render_template("rauthenticate.html", form=request.form)
def __init__(self, width, height):
logging.info("Preparing fishtank (size %d, %d)..." % (width, height))
# Prepare window and tank dimensions
self.win_w = width - 1
self.win_h = height - 1
self.tank_w = self.win_w - 1
self.tank_h = self.win_h - 2
# Create a timer for the simulation
self.timer = Timer()
# Prepare empty lists for tank objects
self.fishes = []
self.predators = []
self.food = []
self.grass = []
# Generate some grass in the tank
logging.info("Generating grass...")
self.generateGrass(10)
# Generate an initial population of fish
logging.info("Generating fish population...")
self.generateFish(10)
# Prepare a help text
self.help = "[q]uit, [c]yan fish, [f]ood, [p]redator"
def main():
(_, chrom, antisense) = \
tuple(sys.argv)
antisense = antisense.lower() == 'true'
chrom = int(chrom)
print_fl("Running nucleosome calling on chromosome %d, antisense: %s" %
(chrom, str(antisense)))
name = task_name(antisense)
timer = Timer()
p123_orfs = paper_orfs
save_chrom_dir = sense_nuc_chrom_dir
cc_dir = cc_sense_chrom_dir
if antisense:
p123_orfs = antisense_orfs
save_chrom_dir = anti_nuc_chrom_dir
cc_dir = cc_antisense_chrom_dir
call_nucleosomes_p123_chrom(p123_orfs, chrom, antisense,
cc_dir, save_chrom_dir, timer)
child_done(name, WATCH_TMP_DIR, chrom)
def login():
log_data = request.get_json(force=True)
username = log_data['login']
password = log_data['password']
if check_permissions(username, 3):
return json.dumps({'success': False}), 401, {
'ContentType': 'application/json'
}
if check_password(username, password):
user_to_login = None
for user in users:
if user.id == username:
user_to_login = user
break
if user_to_login is not None:
login_user(user_to_login)
timer = get_timer_monitor(timers, user_to_login.name)
if not timer:
timers.append(Timer(user_to_login.name))
return json.dumps({'success': True}), 200, {
'ContentType': 'application/json'
}
else:
return json.dumps({'success': False}), 403, {
'ContentType': 'application/json'
}
else:
return json.dumps({'success': False}), 401, {
'ContentType': 'application/json'
}
def collect_motifs(self):
fimo = self.fimo
timer = Timer()
all_motifs = pd.DataFrame()
# filter peaks outside of ORFs promoter
promoters = load_calculated_promoters()
search_peaks = self.collected_peaks.reset_index(drop=True).copy()
# filter out peaks outside of promoters
print("Filtering peaks outside of promoters")
print(len(search_peaks))
for orf_name, row in promoters.iterrows():
cur_peaks = search_peaks[search_peaks.orf == orf_name]
if len(cur_peaks) > 0:
# remove if outside of promoter
remove_peaks = cur_peaks[(cur_peaks.original_mid > row.promoter_stop) |
(cur_peaks.original_mid < row.promoter_start)]
search_peaks = search_peaks.drop(remove_peaks.index)
self.prom_peaks = search_peaks
print(len(search_peaks))
for idx, peak in search_peaks.reset_index().iterrows():
search_window = (peak.original_mid-50, peak.original_mid+50)
try:
motifs = find_motif(fimo, None, peak.chr, search_window)
except KeyError:
continue
motifs['orf'] = peak.orf
motifs['peak'] = peak['name']
motifs = motifs[['orf', 'tf', 'score', 'p-value', 'q-value', 'motif_mid',
'strand', 'peak']]
all_motifs = all_motifs.append(motifs)
if idx % 100 == 0:
print("%d/%d - %s" % (idx, len(search_peaks), timer.get_time()))
all_motifs = all_motifs.reset_index(drop=True)
self.all_motifs = all_motifs
def main():
from src.kernel_fitter import compute_triple_kernel
(_, chrom, antisense) = \
tuple(sys.argv)
antisense = antisense.lower() == 'true'
chrom = int(chrom)
print_fl("Running cross correlation on chromosome %d, antisense: %s" %
(chrom, str(antisense)))
name = task_name(antisense)
timer = Timer()
nuc_kernel = MNaseSeqDensityKernel(filepath=nuc_kernel_path)
sm_kernel = MNaseSeqDensityKernel(filepath=sm_kernel_path)
triple_kernel = compute_triple_kernel(nuc_kernel)
print_fl("Reading MNase-seq...", end='')
all_mnase_data = pd.read_hdf(mnase_seq_path, 'mnase_data')
print_fl("Done.")
timer.print_time()
print_fl()
if not antisense:
cc_dir = cc_sense_chrom_dir
cc_orfs = read_orfs_data("%s/orfs_cd_paper_dataset.csv" % OUTPUT_DIR)
else:
cc_dir = cc_antisense_chrom_dir
cc_orfs = antisense_orfs
calculate_cross_correlation_chr(all_mnase_data,
cc_orfs,
chrom,
antisense,
nuc_kernel,
sm_kernel,
triple_kernel,
cc_dir,
log=True,
timer=timer)
child_done(name, WATCH_TMP_DIR, chrom)
def main(config_file_name):
"""
:param config_file_name: str, name of the configuration json file
:return:
"""
global push_notifier, settings
# read config file
settings = Settings(config_file_name)
# tell scraper about the settings
scraper.settings_ref = settings
# create notifier
push_notifier = PushBulletNotifier(settings.token)
# start timer
timer = Timer(settings.hours, unit_callback)
timer.start_counter()
def run_backtest(self):
print("Execution Mode: Rabbit MQ")
t = Timer()
t.start()
try:
self._run_backtest()
except KeyboardInterrupt as e:
quit(0)
t.end()
print("Total Process Execution Taken:", str(t.get()) + "s")
def __init__(self):
self.window = tkinter.Tk()
self.canvas = tkinter.Canvas(self.window, width=WIDTH, height=HEIGHT)
self.canvas.pack(expand=True, fill="both")
self.display_list = []
self.scroll = 0
self.window.bind("<Up>", self.scrollup)
self.window.bind("<Down>", self.scrolldown)
self.window.bind("<Configure>", self.windowresize)
self.window.bind("<Button-1>", self.handle_click)
self.window.bind("<Key>", self.keypress)
self.window.bind("<Return>", self.pressenter)
self.width = WIDTH
self.height = HEIGHT
self.hstep = HSTEP
self.vstep = VSTEP
self.scroll_step = SCROLL_STEP
self.history = []
self.focus = None
self.address_bar = ""
self.timer = Timer()
self.cookies = {}
def collect_small_peaks():
from src.small_peak_calling import call_orf_small_peaks
from src.timer import Timer
orfs = all_orfs_TSS_PAS()
timer = Timer()
all_peaks = pd.DataFrame()
for chrom in range(1, 17):
print("Chromosome %d" % chrom)
chr_orfs = orfs[orfs.chr == chrom]
# load relevant cross correlations
chrom_cross_correlation = pd.read_hdf(
'%s/cross_correlation_chr%d.h5.z' %
(cc_sense_chrom_dir, chrom))
small_cc = -1 * chrom_cross_correlation.loc['diff']
for idx, orf in chr_orfs.iterrows():
try:
peaks = call_orf_small_peaks(small_cc, orf)
except KeyError:
continue
all_peaks = all_peaks.append(peaks)
timer.print_time()
all_peaks = all_peaks.reset_index(drop=True)
all_peaks['name'] = all_peaks['orf'] + '_' + all_peaks['time'].astype(str) + '_' + \
all_peaks['chr'].astype(str) + '_' + all_peaks['original_mid'].astype(str)
all_peaks = all_peaks.set_index('name')
return all_peaks
def load_moves(self):
try:
with open(self.game_file, 'r') as f:
self.pause = 0
white = eval(f.readline().split()[1])
black = eval(f.readline().split()[1])
self.player1.person = white
self.player2.person = black
lines = []
for line in f:
lines.append(line)
for line in lines[:-2]:
move = line.split(':')
move = eval(move[1])
self.loaded_moves.put(move)
self.loaded_moves_amount += 1
enable_timer = eval(lines[-2])
duration = list(map(int, lines[-1].split()))
self.timer1 = Timer(enable_timer, duration[0])
self.timer2 = Timer(enable_timer, duration[1])
except:
raise Exception('Something wrong with file')
def run_model(name, save_dir, predict_TPM=True):
timer = Timer()
print_fl("Loading %s model" % name)
print_fl("Predicting TPM: %s" % predict_TPM)
sample_N = None
model_fun = get_model_funs()[name]
model = model_fun(sample_N=sample_N)
folds = 10
print_fl("Fitting %d folds.." % folds)
model.fit_cv(log=True, k=folds)
# save models to disk
res = model.Y.join(model.Y_predict, lsuffix='_true', rsuffix='_predicted')
res.to_csv('%s/%s_results.csv' % (save_dir, name))
res = pd.DataFrame({'r2': model.r2, 'mse': model.mse})
res.to_csv('%s/res_%s.csv' % (save_dir, name))
timer.print_time()
def plot_frag_len_dist(mnase_data,
title="Subsampled, merged fragment lengths",
normalize=True,
plt_legend=False):
from config import times
from src.plot_utils import plot_density, apply_global_settings
lengths = mnase_data.groupby(
['time', 'length']).count()[['chr']].rename(columns={'chr': 'count'})
from src.timer import Timer
timer = Timer()
apply_global_settings()
fig, ax = plt.subplots(figsize=(6, 4))
fig.tight_layout(rect=[0.1, 0.1, 0.825, 0.85])
colors = plt.get_cmap('magma_r')
i = 0
for time in times:
color = colors(float(i) * 0.8 / 5. + 1. / 5)
data = lengths.loc[time]
max_len = data.idxmax().values[0]
print("Most frequent length for %s: %d" % (str(time), max_len))
if normalize:
data = data / data.sum()
ax.plot(data, color=color, label="%s min" % str(time))
i += 1
ax.set_title(title, fontsize=20)
ax.set_xlabel('Fragment length (bp)')
ax.set_ylabel('Density')
ax.set_ylim(0, 0.02)
ax.set_xlim(0, 250)
if plt_legend:
ax.legend(bbox_to_anchor=(1.35, 1.), frameon=False)
def train(net, train_loader,optimizer, num_epochs):
log_file = open(args.SAVE_ROOT+"/"+args.Dataset+"_training.log","w",1)
log_print("Training ....", color='green', attrs=['bold'])
# training
train_loss = 0
step_cnt = 0
re_cnt = False
t = Timer()
t.tic()
for epoch in range(1,num_epochs+1):
step = -1
train_loss = 0
for blob in train_loader:
step = step + 1
im_data = blob['data']
gt_data = blob['gt_density']
density_map = net(im_data, gt_data)
loss = net.loss
train_loss += loss.data
step_cnt += 1
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % disp_interval == 0:
duration = t.toc(average=False)
fps = step_cnt / duration
gt_count = np.sum(gt_data)
density_map = density_map.data.cpu().numpy()
et_count = np.sum(density_map)
utils.save_results(im_data,gt_data,density_map, args.SAVE_ROOT)
log_text = 'epoch: %4d, step %4d, Time: %.4fs, gt_cnt: %4.1f, et_cnt: %4.1f' % (epoch,
step, 1./fps, gt_count,et_count)
log_print(log_text, color='green', attrs=['bold'])
re_cnt = True
if re_cnt:
t.tic()
re_cnt = False
return net
def main():
"""
Usage:
python transcript_boundaries <chrom> <save_dir>
<antisense> <pileup_path>
e.g.
python src/transcript_boundaries.py 1 True
"""
(_, chrom, antisense) = \
tuple(sys.argv)
orfs = load_park_boundaries()
# path loaded from config
pileup = pd.read_hdf(pileup_path, 'pileup')
chrom = int(chrom)
antisense = antisense.lower() == 'true'
print_fl("Running transcript boundaries on chromosome %d, antisense: %s" %
(chrom, str(antisense)))
save_dir = anti_chrom_dir if antisense else \
sense_chrom_dir
name = task_name(antisense)
timer = Timer()
compute_boundaries_chrom(orfs, pileup, chrom,
save_dir, antisense,
log=True, timer=timer)
child_done(name, WATCH_TMP_DIR, chrom)
def get_datascapes_features(parsed_article, mango_enrichment=False):
with Timer('Generated enriched article'):
enriched_article = NewsAresItem(ares_dict=parsed_article,
mango_enrichment=mango_enrichment)
datascapes_features = {}
if mango_enrichment:
with Timer('Generating score features'):
datascapes_features = decorate_article_with_score_features(
enriched_article, datascapes_features)
with Timer('Generating boolean features'):
datascapes_features = decorate_article_with_boolean_features(
parsed_article, datascapes_features)
with Timer('Generating sport featuress'):
datascapes_features = decorate_article_with_sport_features(
parsed_article, datascapes_features)
if enriched_article.combined_body_summary_headline:
# below code fails if no combined_body_summary_headline
with Timer('Generating entailment boolean features'):
datascapes_features = decorate_article_with_entailment_boolean_features(
enriched_article, datascapes_features)
with Timer('Generating tonality features'):
datascapes_features = decorate_article_with_tonality_features(
enriched_article, datascapes_features)
with Timer('Generating sarcasm features'):
datascapes_features = decorate_article_with_sarcasm_features(
enriched_article, datascapes_features)
else:
print('No combined_body_summary_headline for article: ',
enriched_article.asset_uri)
return datascapes_features
# scheduler = MultiStepLR(net.optimizer, milestones=[1], gamma=0.1)
net.train()
#training configuration
start_step = 0
end_step = opt.epochs
disp_interval = opt.disp_interval
save_interval = opt.save_interval
# training
train_loss = 0
step_cnt = 1
re_cnt = False
t = Timer()
t.tic()
print("Start training")
for epoch in range(start_step, end_step + 1):
step = -1
train_loss = 0
outer_timer = Timer()
outer_timer.tic()
'''regenerate crop patches'''
data_loader_train.shuffle_list()
load_timer = Timer()
load_time = 0.0
iter_timer = Timer()
iter_time = 0.0
class Tank:
def __init__(self, width, height):
logging.info("Preparing fishtank (size %d, %d)..." % (width, height))
# Prepare window and tank dimensions
self.win_w = width - 1
self.win_h = height - 1
self.tank_w = self.win_w - 1
self.tank_h = self.win_h - 2
# Create a timer for the simulation
self.timer = Timer()
# Prepare empty lists for tank objects
self.fishes = []
self.predators = []
self.food = []
self.grass = []
# Generate some grass in the tank
logging.info("Generating grass...")
self.generateGrass(10)
# Generate an initial population of fish
logging.info("Generating fish population...")
self.generateFish(10)
# Prepare a help text
self.help = "[q]uit, [c]yan fish, [f]ood, [p]redator"
def generateGrass(self, density):
# Compute how much grass to create
grassNum = int(self.tank_w / density)
# Create each grass
for i in range(0, grassNum):
x = random.randint(1, self.tank_w)
y = self.tank_h
grass = Grass("green")
grass.setPos((x, y))
grass.setHeight(random.randint(1, self.tank_h))
self.grass.append(grass)
def generateFish(self, density):
# Create each fish
for i in range (0, density):
smallFish = SmallFish("cyan")
self.initFish(smallFish)
self.fishes.append(smallFish)
def randPos(self):
x = random.randint(1, self.tank_w)
y = random.randint(1, self.tank_h)
return (x, y)
def initFish(self, fish):
fish.setBoundaries(1, 1, self.tank_w, self.tank_h)
fish.setPos(self.randPos())
def update(self, cio, key):
self.timer.update()
# 'c' key - creates new cyan small fish
if key == cio.key_c:
smallFish = SmallFish("cyan")
self.initFish(smallFish)
self.fishes.append(smallFish)
# 'p' key - creates new predator fish
if key == cio.key_p:
predatorFish = PredatorFish("red")
self.initFish(predatorFish)
self.predators.append(predatorFish)
# 'f' key - adds some food for the fishes
if key == cio.key_f:
food = Food("yellow")
food.setPos(self.randPos())
self.food.append(food)
# Update all the objects in the tank
for predator in self.predators:
predator.update(self.timer.getDelta(), self.fishes)
for fish in self.fishes:
newFish = fish.update(self.timer.getDelta(), self.fishes, self.food, self.predators)
if newFish:
smallFish = SmallFish("cyan")
smallFish.setBoundaries(1, 1, self.tank_w, self.tank_h)
smallFish.setPos(fish.getPos())
self.fishes.append(smallFish)
# Move the objects (movements were stored in update)
for predator in self.predators:
predator.postUpdate()
for fish in self.fishes:
fish.postUpdate()
def drawHelp(self, cio):
cio.drawAscii(0, self.win_h, self.help)
def draw(self, cio):
# Draw the tank outline and water
for y in range(0, self.win_h):
cio.drawAscii(0, y, '|')
cio.drawAscii(self.win_w, y, '|')
for x in range(1, self.win_w):
cio.drawAscii(x, 0, '~', "blue")
cio.drawAscii(x, self.win_h-1, '-')
# Draw objects in the tank
for grass in self.grass:
grass.draw(cio)
for food in self.food:
food.draw(cio)
for fish in self.fishes:
fish.draw(cio)
for predator in self.predators:
predator.draw(cio)
# Draw a help text
self.drawHelp(cio)