def make_ffmpeg_args(in_file: str, out_file: str, from_time: str,
to_time: str) -> t.Tuple[str, ...]:
args = DEFAULT_ARGS.copy()
from_time = parse_time(from_time)
from_time_formatted = format_time(from_time)
args[10] = f'-ss {from_time_formatted}'
args[20] = f'-i "{in_file}"'
try:
to_time = parse_time(to_time)
except ValueError:
pass
else:
duration_formatted = format_timedelta(get_duration(from_time, to_time))
args[30] = f'-to {duration_formatted}'
_, in_file_name = split_path(in_file)
in_file_extension = get_extension(in_file_name)
args[50] = f'"{add_extension(out_file, extension=in_file_extension)}"'
sorted_args = tuple(v for _, v in sorted(args.items()))
return sorted_args
def status(args):
# Transform selection criteria into test functions.
if args.time:
filter_time = lambda slot: \
all(map(lambda invl: slot.intersects(invl), args.time))
else:
filter_time = lambda _: True
if not args.all:
filter_open = lambda slot: slot.iscurrent()
else:
filter_open = lambda _: True
# Select work slots matching the selection criteria..
slots = [slot for slot in session.wslots \
if filter_time(slot) and filter_open(slot)]
if not slots:
# FIXME Update the message.
print("You don't seem to be working now.")
else:
# FIXME Update the message, especially in case when called with --all.
print("You have been working on the following tasks:")
now = datetime.now(session.config['TIMEZONE'])
task2slot = group_by(slots, "task", single_attr=True)
for task in task2slot:
task_slots = task2slot[task]
# Expected case: only working once on the task in parallel:
if len(task_slots) == 1:
end = task_slots[0].end or now
time_spent = format_timedelta(end - task_slots[0].start)
try:
print("\t{time: >18}: {task}".format(task=task.name,
time=time_spent))
except:
continue
else:
for slot in task_slots:
end = slot.end or now
time_spent = format_timedelta(end - slot.start)
print("M\t{time: >18}: {task}".format(task=task.name,
time=time_spent))
return 0
def list_timers(self, channel, user, message):
"""List all your timers"""
s = []
for msg, (cb, end_time) in self.timers[user].iteritems():
done = timedelta(seconds=int(end_time - time.time()))
out = "%s in %s" % (msg, format_timedelta(done))
s.append((done, out))
if s:
out = [e[1] for e in sorted(s, key=lambda e:e[0])]
return 'you have the following timers: %s' % (', '.join(out),)
else:
return 'you have no timers running.'
def _format_data(json):
date_format = '%d %b, %H:%M'
struct = namedtuple('Portable',
['author', 'last_updated', 'locations', 'time'])
# Populating portables
time = datetime.strptime(
json['values'][2][1],
date_format).replace(year=datetime.utcnow().year)
author = json['values'][2][3]
last_updated = util.format_timedelta(datetime.utcnow() - time,
short_names=True)
locations = {
'fletchers': {},
'crafters': {},
'braziers': {},
'sawmills': {},
'forges': {},
'ranges': {},
'wells': {}
}
# Finding all worlds for portables
for i in range(7):
worlds = locations[json['values'][0][i].strip().lower()]
locs = json['values'][1][i]
# Checking if no worlds
if 'host needed' in locs.lower() or 'n/a' in locs.lower():
continue
# Separating locations
for location in re.findall(
'\d+.+?(?:CA|MG|PRIFF|PRIF|P|BU|SP|CW|BA)', locs.upper()):
name = location.split(' ')[-1]
name = re.sub('(?:PRIFF|PRIF)', 'P', name, re.I)
worlds[name] = re.findall('\d+', location)
return struct(author=author,
locations=locations,
last_updated=last_updated,
time=time)
def add_timer(self, channel, user, message):
""" Add a timer"""
when, sep, msg = message.partition(' ')
delta = parse_time(when)
if delta is None:
return "'%s' does not parse as a time" % (when,)
if msg in self.timers[user]:
return "there is already a timer for %s" % (msg,)
end = "timer for " + choice(["%s is done", "%s is ready", "%s is finished"]) % msg
try:
end_time = datetime.now()+delta
except:
return "Invalid date. Stop that."
end_ts = time.mktime(end_time.timetuple())
def response():
self.reply(channel, user, end)
del self.timers[user][msg]
cb = self.io_loop.add_timeout(end_ts, response)
self.timers[user][msg] = (cb, end_ts)
return "okay! starting %s timer for %s." % (format_timedelta(delta), msg)
def genphantomdata(N_D, phantompath, ctfparfile):
mscope_params = {
'akv': 200,
'wgh': 0.07,
'cs': 2.0,
'psize': 2.8,
'bfactor': 500.0
}
N = 128
rad = 0.95
shift_sigma = 3.0
sigma_noise = 25.0
M_totalmass = 80000
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
shift_sigma = float(shift_sigma)
sigma_noise = float(sigma_noise)
M_totalmass = float(M_totalmass)
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(
mscope_params, parfields=['PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = n.sort(
n.random.random_integers(0,
srcctf_stack.get_num_ctfs() - 1, N_D))
M = mrc.readMRC(phantompath)
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
V = density.real_to_fspace(
premult.reshape((1, 1, -1)) * premult.reshape(
(1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
print "Generating data..."
sys.stdout.flush()
imgdata = n.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': [], 't': []}
prevctfI = None
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print "\r%.2f Percent.. (Elapsed: %s, Remaining: %s) " % (
i / float(N_D) * 100.0, format_timedelta(ellapse_time),
format_timedelta(remain_time)),
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N**2, ))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = n.random.randn(3)
pt /= n.linalg.norm(pt)
psi = 2 * n.pi * n.random.rand()
EA = geom.genEA(pt)[0]
EA[2] = psi
shift = n.random.randn(2) * shift_sigma
R = geom.rotmat3D_EA(*EA)[:, 0:2]
slop = cryoops.compute_projection_matrix([R], N, kernel, ksize, rad,
'rots')
S = cryoops.compute_shift_phases(shift.reshape((1, 2)), N, rad)[0]
D = slop.dot(V.reshape((-1, )))
D *= S
imgdata[i] = density.fspace_to_real((C * TtoF.dot(D)).reshape(
(N, N))) + n.require(n.random.randn(N, N) * sigma_noise,
dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / n.pi,
THETA=EA[1] * 180.0 / n.pi,
PSI=EA[2] * 180.0 / n.pi,
SHX=shift[0],
SHY=shift[1])
pardata['R'].append(R)
pardata['t'].append(shift)
print "\rDone in ", time.time() - tic, " seconds."
return imgdata, genctf_stack, pardata, mscope_params
def dowork(self):
"""Do one atom of work. I.E. Execute one minibatch"""
timing = {}
# Time each minibatch
tic_mini = time.time()
self.iteration += 1
# Fetch the current batches
trainbatch = self.cryodata.get_next_minibatch(self.cparams.get('shuffle_minibatches',True))
# Get the current epoch
cepoch = self.cryodata.get_epoch(frac=True)
epoch = self.cryodata.get_epoch()
num_data = self.cryodata.N_D_Train
# Evaluate the parameters
self.eval_params()
timing['setup'] = time.time() - tic_mini
# Do hyperparameter learning
if self.cparams.get('learn_params',False):
tic_learn = time.time()
if self.cparams.get('learn_prior_params',True):
tic_learn_prior = time.time()
self.prior_func.learn_params(self.params, self.cparams, M=self.M, fM=self.fM)
timing['learn_prior'] = time.time() - tic_learn_prior
if self.cparams.get('learn_likelihood_params',True):
tic_learn_like = time.time()
self.like_func.learn_params(self.params, self.cparams, M=self.M, fM=self.fM)
timing['learn_like'] = time.time() - tic_learn_like
if self.cparams.get('learn_prior_params',True) or self.cparams.get('learn_likelihood_params',True):
timing['learn_total'] = time.time() - tic_learn
# Time each epoch
if self.tic_epoch == None:
self.ostream("Epoch: %d" % epoch)
self.tic_epoch = (tic_mini,epoch)
elif self.tic_epoch[1] != epoch:
self.ostream("Epoch Total - %.6f seconds " % \
(tic_mini - self.tic_epoch[0]))
self.tic_epoch = (tic_mini,epoch)
sym = get_symmetryop(self.cparams.get('symmetry',None))
if sym is not None:
self.obj.ws[1] = 1.0/sym.get_order()
tic_mstats = time.time()
self.ostream(self.cparams['name']," Iteration:", self.iteration,\
" Epoch:", epoch, " Host:", socket.gethostname())
# Compute density statistics
N = self.cryodata.N
M_sum = self.M.sum(dtype=n.float64)
M_zeros = (self.M == 0).sum()
M_mean = M_sum/N**3
M_max = self.M.max()
M_min = self.M.min()
# self.ostream(" Density (min/max/avg/sum/zeros): " +
# "%.2e / %.2e / %.2e / %.2e / %g " %
# (M_min, M_max, M_mean, M_sum, M_zeros))
self.statout.output(total_density=[M_sum],
avg_density=[M_mean],
nonzero_density=[M_zeros],
max_density=[M_max],
min_density=[M_min])
timing['density_stats'] = time.time() - tic_mstats
# evaluate test batch if requested
if self.iteration <= 1 or self.cparams.get('evaluate_test_set',self.iteration%5):
tic_test = time.time()
testbatch = self.cryodata.get_testbatch()
self.obj.set_data(self.cparams,testbatch)
testLogP, res_test = self.obj.eval(M=self.M, fM=self.fM,
compute_gradient=False)
self.outputbatchinfo(testbatch, res_test, testLogP, 'test', 'Test')
timing['test_batch'] = time.time() - tic_test
else:
testLogP, res_test = None, None
# setup the wrapper for the objective function
tic_objsetup = time.time()
self.obj.set_data(self.cparams,trainbatch)
self.obj_wrapper.set_objective(self.obj)
x0 = self.obj_wrapper.set_density(self.M,self.fM)
evalobj = self.obj_wrapper.eval_obj
timing['obj_setup'] = time.time() - tic_objsetup
# Get step size
self.num_data_evals += trainbatch['N_M'] # at least one gradient
tic_objstep = time.time()
trainLogP, dlogP, v, res_train, extra_num_data = self.step.do_step(x0,
self.cparams,
self.cryodata,
evalobj,
batch=trainbatch)
# Apply the step
x = x0 + v
timing['step'] = time.time() - tic_objstep
# Convert from parameters to value
tic_stepfinalize = time.time()
prevM = n.copy(self.M)
self.M, self.fM = self.obj_wrapper.convert_parameter(x,comp_real=True)
apply_sym = sym is not None and self.cparams.get('perfect_symmetry',True) and self.cparams.get('apply_symmetry',True)
if apply_sym:
self.M = sym.apply(self.M)
# Truncate the density to bounds if they exist
if self.cparams['density_lb'] is not None:
n.maximum(self.M,self.cparams['density_lb']*self.cparams['modelscale'],out=self.M)
if self.cparams['density_ub'] is not None:
n.minimum(self.M,self.cparams['density_ub']*self.cparams['modelscale'],out=self.M)
# Compute net change
self.dM = prevM - self.M
# Convert to fourier space (may not be required)
if self.fM is None or apply_sym \
or self.cparams['density_lb'] != None \
or self.cparams['density_ub'] != None:
self.fM = density.real_to_fspace(self.M)
timing['step_finalize'] = time.time() - tic_stepfinalize
# Compute step statistics
tic_stepstats = time.time()
step_size = n.linalg.norm(self.dM)
grad_size = n.linalg.norm(dlogP)
M_norm = n.linalg.norm(self.M)
self.num_data_evals += extra_num_data
inc_ratio = step_size / M_norm
self.statout.output(step_size=[step_size],
inc_ratio=[inc_ratio],
grad_size=[grad_size],
norm_density=[M_norm])
timing['step_stats'] = time.time() - tic_stepstats
# Update import sampling distributions
tic_isupdate = time.time()
self.sampler_R.perform_update()
self.sampler_I.perform_update()
self.sampler_S.perform_update()
self.diagout.output(global_phi_R=self.sampler_R.get_global_dist())
self.diagout.output(global_phi_I=self.sampler_I.get_global_dist())
self.diagout.output(global_phi_S=self.sampler_S.get_global_dist())
timing['is_update'] = time.time() - tic_isupdate
# Output basic diagnostics
tic_diagnostics = time.time()
self.diagout.output(iteration=self.iteration, epoch=epoch, cepoch=cepoch)
if self.logpost_history.N_sum != self.cryodata.N_batches:
self.logpost_history.setup(trainLogP,self.cryodata.N_batches)
self.logpost_history.set_value(trainbatch['id'],trainLogP)
if self.like_history.N_sum != self.cryodata.N_batches:
self.like_history.setup(res_train['L'],self.cryodata.N_batches)
self.like_history.set_value(trainbatch['id'],res_train['L'])
self.outputbatchinfo(trainbatch, res_train, trainLogP, 'train', 'Train')
# Dump parameters here to catch the defaults used in evaluation
self.diagout.output(params=self.cparams,
envelope_mle=self.like_func.get_envelope_mle(),
sigma2_mle=self.like_func.get_sigma2_mle(),
hostname=socket.gethostname())
self.statout.output(num_data=[num_data],
num_data_evals=[self.num_data_evals],
iteration=[self.iteration],
epoch=[epoch],
cepoch=[cepoch],
logp=[self.logpost_history.get_mean()],
like=[self.like_history.get_mean()],
sigma=[self.like_func.get_rmse()],
time=[time.time()])
timing['diagnostics'] = time.time() - tic_diagnostics
checkpoint_it = self.iteration % self.cparams.get('checkpoint_frequency',50) == 0
save_it = checkpoint_it or self.cparams['save_iteration'] or \
time.time() - self.last_save > self.cparams.get('save_time',n.inf)
if save_it:
tic_save = time.time()
self.last_save = tic_save
if self.io_queue.qsize():
print "Warning: IO queue has become backlogged with {0} remaining, waiting for it to clear".format(self.io_queue.qsize())
self.io_queue.join()
self.io_queue.put(( 'pkl', self.statout.fname, copy(self.statout.outdict) ))
self.io_queue.put(( 'pkl', self.diagout.fname, deepcopy(self.diagout.outdict) ))
self.io_queue.put(( 'pkl', self.likeout.fname, deepcopy(self.likeout.outdict) ))
self.io_queue.put(( 'mrc', opj(self.outbase,'model.mrc'), \
(n.require(self.M,dtype=density.real_t),self.voxel_size) ))
self.io_queue.put(( 'mrc', opj(self.outbase,'dmodel.mrc'), \
(n.require(self.dM,dtype=density.real_t),self.voxel_size) ))
if checkpoint_it:
self.io_queue.put(( 'cp', self.diagout.fname, self.diagout.fname+'-{0:06}'.format(self.iteration) ))
self.io_queue.put(( 'cp', self.likeout.fname, self.likeout.fname+'-{0:06}'.format(self.iteration) ))
self.io_queue.put(( 'cp', opj(self.outbase,'model.mrc'), opj(self.outbase,'model-{0:06}.mrc'.format(self.iteration)) ))
timing['save'] = time.time() - tic_save
time_total = time.time() - tic_mini
self.ostream(" Minibatch Total - %.2f seconds Total Runtime - %s" %
(time_total, format_timedelta(datetime.now() - self.startdatetime) ))
return self.iteration < self.cparams.get('max_iterations',n.inf) and \
cepoch < self.cparams.get('max_epochs',n.inf)
def genphantomdata(N_D, phantompath):
mscope_params = {
'akv': 200,
'wgh': 0.07,
'cs': 2.0,
'psize': 3.0,
'bfactor': 500.0
}
M = mrc.readMRC(phantompath)
N = M.shape[0]
rad = 0.95
M_totalmass = 1000000
# M_totalmass = 1500000
kernel = 'lanczos'
ksize = 6
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
M_totalmass = float(M_totalmass)
ctfparfile = 'particle/examplectfs.par'
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(
mscope_params, parfields=['PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = np.sort(
np.random.random_integers(0,
srcctf_stack.get_num_ctfs() - 1, N_D))
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
# oversampling
oversampling_factor = 3
psize = psize * oversampling_factor
V = density.real_to_fspace_with_oversampling(M, oversampling_factor)
fM = V.real**2 + V.imag**2
# mrc.writeMRC('particle/EMD6044_fM_totalmass_{}_oversampling_{}.mrc'.format(str(int(M_totalmass)).zfill(5), oversampling_factor), fM, psz=psize)
print("Generating data...")
sys.stdout.flush()
imgdata = np.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': []}
prevctfI = None
coords = geometry.gencoords(N, 2, rad)
slicing_func = RegularGridInterpolator((np.arange(N), ) * 3,
fM,
bounds_error=False,
fill_value=0.0)
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print("\r%.2f Percent.. (Elapsed: %s, Remaining: %s)" %
(i / float(N_D) * 100.0, format_timedelta(ellapse_time),
format_timedelta(remain_time)),
end='')
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N, N))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
# Rotate coordinates and get slice image by interpolation
R = geometry.rotmat3D_EA(*EA)[:, 0:2]
rotated_coords = R.dot(coords.T).T + int(N / 2)
slice_data = slicing_func(rotated_coords)
intensity = TtoF.dot(slice_data)
np.maximum(intensity, 0.0, out=intensity)
# Add poisson noise
img = np.float_(np.random.poisson(intensity.reshape(N, N)))
np.maximum(1e-8, img, out=img)
imgdata[i] = np.require(img, dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / np.pi,
THETA=EA[1] * 180.0 / np.pi,
PSI=EA[2] * 180.0 / np.pi,
SHX=0.0,
SHY=0.0)
pardata['R'].append(R)
print("\n\rDone in ", time.time() - tic, " seconds.")
return imgdata, genctf_stack, pardata, mscope_params
def genphantomdata(N_D, phantompath, ctfparfile):
# mscope_params = {'akv': 200, 'wgh': 0.07,
# 'cs': 2.0, 'psize': 2.8, 'bfactor': 500.0}
mscope_params = {'akv': 200, 'wgh': 0.07,
'cs': 2.0, 'psize': 3.0, 'bfactor': 500.0}
M = mrc.readMRC(phantompath)
N = M.shape[0]
rad = 0.95
shift_sigma = 3.0
sigma_noise = 25.0
M_totalmass = 80000
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
shift_sigma = float(shift_sigma)
sigma_noise = float(sigma_noise)
M_totalmass = float(M_totalmass)
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(mscope_params, parfields=[
'PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = np.sort(np.random.random_integers(
0, srcctf_stack.get_num_ctfs() - 1, N_D))
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
V = density.real_to_fspace(
premult.reshape((1, 1, -1)) * premult.reshape((1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
print("Generating data...")
sys.stdout.flush()
imgdata = np.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': [], 't': []}
prevctfI = None
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print("\r%.2f Percent.. (Elapsed: %s, Remaining: %s)" % (i / float(N_D)
* 100.0, format_timedelta(ellapse_time), format_timedelta(remain_time)))
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N**2,))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
shift = np.random.randn(2) * shift_sigma
R = geometry.rotmat3D_EA(*EA)[:, 0:2]
slop = cryoops.compute_projection_matrix(
[R], N, kernel, ksize, rad, 'rots')
S = cryoops.compute_shift_phases(shift.reshape((1, 2)), N, rad)[0]
D = slop.dot(V.reshape((-1,)))
D *= S
imgdata[i] = density.fspace_to_real((C * TtoF.dot(D)).reshape((N, N))) + np.require(
np.random.randn(N, N) * sigma_noise, dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / np.pi, THETA=EA[1] * 180.0 / np.pi, PSI=EA[2] * 180.0 / np.pi,
SHX=shift[0], SHY=shift[1])
pardata['R'].append(R)
pardata['t'].append(shift)
print("\rDone in ", time.time() - tic, " seconds.")
return imgdata, genctf_stack, pardata, mscope_params
def delta(ts):
return util.format_timedelta(datetime.utcfromtimestamp(ts), now)
