def eval_best_model(self, epoch):
if self.metrics['val_loss_avg'][-1] < self.best_loss:
self.best_loss = self.metrics['val_loss_avg'][-1]
print(
'New best model at epoch {:0=3d} with val_loss {:.4f}'.format(
epoch, self.best_loss))
utils.flush()
def maybe_flush(self):
global prev_host
new_host = self.headers.get("Host", "")
if prev_host != new_host:
utils.flush()
prev_host = new_host
def maybe_flush(self):
global prev_host
new_host = self.headers.get("Host", "")
if prev_host != new_host:
utils.flush()
prev_host = new_host
def set_function(bot: Bot, update: Update):
print(TAG(), "called set: ", update.message)
chat_id = update.message.chat_id
args = update.message.text.split(" ")
if len(args) < 2:
bot.send_message(chat_id=chat_id,
text="Invalid message. Command pattern is:")
bot.send_message(chat_id=chat_id, text="/set name URL [CSS SELECTOR]")
return
name, url = args[1], args[2]
watcher = Watcher(name, url, update)
if len(args) > 3:
watcher.selector = " ".join(args[3:])
watcher.type = Selector.CSS
else:
watcher.type = Selector.NONE
ok = watcher_manager.add_watcher(chat_id, watcher)
if ok:
bot.send_message(
chat_id=chat_id,
text="Notifier {0} correctly created! (SELECTOR: '{1}' ({2}))".
format(watcher.name, watcher.selector, watcher.type))
print("{0}: watcher {1} created.".format(chat_id, name))
else:
bot.send_message(
chat_id=chat_id,
text="Notifier {0} already exists. Please delete it".format(name))
flush()
def log_execute(args, handler=None, env=None, cwd=None, shell=False):
utils.flush()
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=shell,
env=expand_variables(env),
cwd=cwd)
reader = _TailThread(fileInput=p.stdout,
handler=handler,
fromBegining=True)
rc = None
try:
reader.start()
# Wait until subprocess is done
rc = p.wait()
finally:
# Wait until we've processed all output
reader.stop(False)
if reader.is_alive(): reader.join()
utils.flush()
return rc
def clear_function(bot: Bot, update):
print(TAG(), "called clear: ", update.message)
chat_id = update.message.chat_id
watcher_manager.clear_watcher(chat_id)
bot.send_message(chat_id=chat_id, text="All notifiers are deleted")
flush()
def start_function(bot, update):
print(TAG(), "called start: ", update.message)
chat_id = update.message.chat_id
args = update.message.text.split(" ")
if len(args) > 1:
name = args[1]
ok = watcher_manager.start_watcher(chat_id, name)
if ok == "started":
bot.send_message(
chat_id=chat_id,
text="The notifier {0} now is running".format(name))
elif ok == "already":
bot.send_message(
chat_id=chat_id,
text="The notifier {0} is already running".format(name))
else:
bot.send_message(
chat_id=chat_id,
text="The notifier {0} doesn't exist".format(name))
else:
bot.send_message(
chat_id=chat_id,
text="Insert the notifier name that you want to start")
flush()
def create_summary_accs():
flush('Creating main accounts...')
query = db.session.query(
db.Hesaplar.ana_hesap, db.Hesaplar.lead_code, *[
func.sum(getattr(db.Hesaplar,
'{}'.format(period))).label('{}'.format(period))
for period in db.periodss
]).group_by('ana_hesap').all()
for k in query:
unmapped = None
if k.lead_code == 'Unmapped':
unmapped = db.session.query(db.Hesaplar).filter_by(
len=3, ana_hesap=k.ana_hesap).first() or db.session.query(
db.Hesaplar).filter_by(ana_hesap=k.ana_hesap).first()
source = db.session.query(
db.Lead).filter_by(account=k.ana_hesap).first()
main_source = unmapped or source
name = main_source.name
t = db.Hesaplar(number=k.ana_hesap,
ana_hesap=k.ana_hesap,
name=name,
lead_code=main_source.lead_code,
len=3)
for q in db.periodss:
[setattr(t, q, getattr(k, q))]
db.session.add(t)
db.session.commit()
def backup():
while True:
time.sleep(60)
print(TAG(), "thread_backup:", "START BACKUP ROUTINE")
watcher_manager.save_watchers()
print(TAG(), "thread_backup:", "BACKUP COMPLETED")
flush()
def print_output(name, src, toStdErr):
try:
while not finished and src is not None:
out(toStdErr, src.readline())
flush(toStdErr)
src.close()
except:
pass
def fix_mainaccs():
flush('Changing the names of main accounts...')
hesaplar = db.session.query(db.Hesaplar).filter_by(len=3)
for k in hesaplar.all():
item = db.session.query(db.Lead).filter_by(account=k.ana_hesap).first()
if item:
k.name = item.account_name
db.session.commit()
def prepare_mapping():
s = excel_file.sheet_by_name("Mapping")
for x in range(1, s.nrows):
db.session.add(
db.Lead(account=s.cell(x, 0).value,
account_name=s.cell(x, 1).value,
lead_code=s.cell(x, 2).value,
name=s.cell(x, 3).value))
flush('Parsing mappings...')
db.session.commit()
def define_variables():
flush('Defining Company names and audit periods...')
sheet = excel_file.sheet_by_name('Instruction')
db.tanimlar['company'] = sheet.cell_value(6, 2)
try:
db.tanimlar['optional'] = sheet.cell_value(7, 2) or "NO"
except IndexError:
flush('There is no optional column...')
for sheet in excel_file.sheets()[2:]:
db.periodss.append(sheet.name)
def list_function(bot: Bot, update):
print(TAG(), "called list: ", update.message)
chat_id = update.message.chat_id
watchers = watcher_manager.get_watchers(chat_id)
if len(watchers) > 0:
text = ""
for watcher in watchers:
text += str(watcher) + "\n"
else:
text = "No notifiers available"
bot.send_message(chat_id=chat_id, text=text)
flush()
def thread_function(watchers_manager: WatcherManager):
LOG = "thread_watchers:"
print(TAG(), LOG, "started")
while True:
# acquire lock
watchers_manager.watchers_lock.acquire()
print(TAG(), LOG, "start updating watchers")
try:
# start selenium browser
browser = webdriver.PhantomJS(executable_path=PATH_PHANTOM)
# for each watcher
for user_watchers in watchers_manager.watchers.values():
for watcher in user_watchers:
if watcher.isRunning:
browser.get(watcher.url)
if watcher.type == Watcher.Selector.CSS:
elements = browser.find_elements_by_css_selector(
watcher.selector)
text = "".join(
[element.text for element in elements])
else:
text = browser.find_element_by_css_selector(
'body').text
if watcher.old_text is None:
watcher.old_text = text
else:
if watcher.old_text != text:
watcher.old_text = text
watcher.update.message.\
reply_text("Notifier {0} has seen new changes! Go to see them:\n{1}"
.format(watcher.name, watcher.url))
print(
TAG(), LOG,
"updated watcher {0}: change saved!".
format(watcher.name))
else:
print(
TAG(), LOG,
"watcher {0} checked -> no changes".format(
watcher.name))
browser.close()
print(TAG(), LOG, "checked every running watcher")
except Exception as e:
print(LOG, e, file=sys.stderr)
finally:
# release lock
watchers_manager.watchers_lock.release()
# wait for next iteration
flush()
time.sleep(TIMER)
def delete_zeros(exceptions=None):
if exceptions is None:
exceptions = ['900']
flush('Deleting accounts with zero balances in all periods...')
query = db.session.query(
db.Hesaplar).filter_by(**{k: 0
for k in db.periodss}).all()
for item in query:
if item.number[:3] in exceptions:
continue
db.session.delete(item)
db.session.commit()
def execute_forward(build_cfg):
if build_cfg.forwarding_dir() is not None:
build_script = build_cfg.build_script()
utils.flush()
build_script.forward_buildresults()
utils.flush()
with open(os.path.join(build_cfg.genroot_dir(), 'forward.path'),
"w") as f:
f.write(build_cfg.forwarding_destination())
log.info("build result forwarding succeeded")
else:
log.info(
"build result forwarding step skipped because of missing option -F"
)
def exec_phase(name, method):
if name == 'PRELUDE':
log.info('='*100)
log.info('= {0:96} ='.format('entering xmake ' + name + ' phase'))
log.info('='*100)
try:
method(build_cfg)
notify_phase_ended(name, build_cfg)
utils.flush()
except SystemExit:
raise
except BaseException:
raise
log.info('end of xmake ' + name + ' phase')
log.info('='*100)
def handle_bootstrapper(xmake_inst_dir,argv):
f=join(xmake_inst_dir,'BOOTSTRAPPER_VERSION')
#log.info("looking for "+f)
if len(argv)>1 and argv[1]=='--select-bootstrapper':
select_bootstrapper(f,argv)
sys.exit(0)
if isfile(f):
log.info("determining bootstrapper")
v=get_first_line(f,'cannot read '+f)
if v!=None:
(v,l)=load_latest(v)
if (l!=None):
cmd=[sys.executable, join(l,'xmake','bootstrap.py'),'--bootstrap']
cmd.extend(argv[1:])
flush()
rc=subprocess.call(cmd)
sys.exit(rc)
def summarize_metrics(self, matrix, prefix):
for key in sorted(self.metrics_epoch.keys()):
self.metrics['{}_{}'.format(key, 'avg')].append(
self.metrics_epoch[key].avg)
self.metrics['{}_{}'.format(key, 'std')].append(
self.metrics_epoch[key].std)
print('epoch{:0=3d}_{}{:.4f}'.format(
self.metrics['epochs'][-1], '{}_{}'.format(key, 'avg'),
self.metrics['{}_{}'.format(key, 'avg')][-1]))
print('epoch{:0=3d}_{}{:.4f}'.format(
self.metrics['epochs'][-1], '{}_{}'.format(key, 'std'),
self.metrics['{}_{}'.format(key, 'std')][-1]))
print(matrix)
self.metrics['{}_acc'.format(prefix)].append(metrics.accuracy(matrix))
print('epoch{:0=3d}_{}{:.4f}'.format(
self.metrics['epochs'][-1], '{}_acc'.format(prefix),
self.metrics['{}_acc'.format(prefix)][-1]))
utils.flush()
def find_bds():
flush('Determining breakdowns...')
accounts = db.session.query(db.Hesaplar).all()
for k in accounts:
if db.tanimlar['optional'] != 'NO' and sum_var is False:
if len(k.number) > 3:
k.bd = True
elif db.tanimlar['optional'] != 'NO' and sum_var is True:
if len(db.session.query(db.Hesaplar).filter(and_(db.Hesaplar.number.startswith(k.number),
db.Hesaplar.optional == k.optional)).all()) <= 1 \
and len(k.number) > 3:
k.bd = True
else:
if len(
db.session.query(db.Hesaplar).filter(
db.Hesaplar.number.startswith(k.number)).all()) <= 1:
k.bd = True
db.session.commit()
flush('Breakdowns are set!')
def del_function(bot: Bot, update):
print(TAG(), "called del: ", update.message)
chat_id = update.message.chat_id
args = update.message.text.split(" ")
if len(args) < 2:
bot.send_message(chat_id=chat_id,
text="Invalid message. Command pattern is:")
bot.send_message(chat_id=chat_id, text="/del notifierName")
return
name = args[1]
ok = watcher_manager.delete_watcher(chat_id, name)
if ok:
bot.send_message(chat_id=chat_id,
text="Notifier {0} deleted!".format(name))
else:
bot.send_message(chat_id=chat_id,
text="Notifier {0} not found.".format(name))
flush()
def run_training(self):
self.smaller_validation_loss = float('inf')
for epoch in range(self.number_of_epochs):
print("\nPROGRESS: {0:02.2f}%\n".format(
epoch / self.number_of_epochs * 50),
flush=True)
self.train(epoch)
flush(self.logger)
if self.update_learning_rate:
self.update_lr(epoch)
print("\nPROGRESS: {0:02.2f}%\n".format(
(epoch + 0.9) / self.number_of_epochs * 50),
flush=True)
self.evaluate_learned_policy(self.number_of_epochs *
self.training_steps,
save_results=True,
number_of_epochs=1,
number_of_steps=100000)
self.best_policy.dump_network(self.cloned_network_path)
def output_timeseries_chis(
myfilename_prefix,
myreslist,
colnames,
nsims=6,
nchi=6,
):
#print "shape of matrix to ouput:"+str(mymatrix.shape)+"\n"
min_num_angles = min(myreslist[0].numangles)
timeseries_chis_matrix = zeros(
(nsims, len(myreslist) * nchi, min_num_angles), float64) #initialize
#self.angles = zeros((self.nchi,num_sims,max_angles),float64) # the dihedral angles
for res_ind1, myres1 in zip(range(len(myreslist)), myreslist):
print "\n#### Working on residue %s (%s):" % (
myres1.num, myres1.name), utils.flush()
for mychi1 in range(myres1.nchi):
#print "%s chi: %d/%d" % (myres1.name,int(myres1.num),mychi1+1)
#print "res_ind1: "+str(res_ind1)
#rint "mychi1: "+str(mychi1)
#print "nchi: " +str(nchi)
#print "min_num_angles: "+str(min_num_angles)
#print "res_ind1 * nchi + mychi1: "+str(res_ind1 * nchi + mychi1)
#print "myres1.angles: "
#print myres1.angles
#print "angle entries: "
#print myres1.angles[mychi1, :, :min_num_angles]
timeseries_chis_matrix[:, res_ind1 * nchi +
mychi1, :] = myres1.angles[
mychi1, :, :min_num_angles]
my_file_list = []
for mysim in range(nsims):
myfile = open(myfilename_prefix + "_" + str(mysim) + ".txt", 'w')
for col_num, col_name in zip(range(len(colnames)), colnames):
for col_chi in range(nchi):
myfile.write(col_name + "_" + str(col_chi) + " ")
myfile.write("\n")
for myrow in range(min_num_angles):
for col_num, col_name in zip(range(len(colnames)), colnames):
for col_chi in range(nchi):
myfile.write(
str(timeseries_chis_matrix[mysim,
col_num * nchi + col_chi,
myrow]))
myfile.write(" ")
myfile.write("\n")
myfile.close()
return timeseries_chis_matrix
def deploy_circuitpython_code():
print("========== DEPLOYING CODE ==========")
# Wait for the circuitpython drive to show up.
time.sleep(5)
cpy_drive = utils.find_drive_by_name("CIRCUITPY")
utils.clean_pycache(FIRMWARE_DIR)
utils.clean_pycache(EXAMPLES_DIR)
os.makedirs(os.path.join(cpy_drive, "lib"), exist_ok=True)
for src, dst in FILES_TO_DEPLOY.items():
if src.startswith("https://"):
http_src, zip_path = src.rsplit(':', 1)
zip_data = io.BytesIO(requests.get(http_src).content)
with zipfile.ZipFile(zip_data, "r") as zipfh:
file_data = zipfh.read(zip_path)
dst = os.path.join(dst, os.path.basename(zip_path))
with open(os.path.join(cpy_drive, dst), "wb") as fh:
fh.write(file_data)
else:
if os.path.isdir(src):
dst = os.path.join(cpy_drive, dst, os.path.basename(src))
if os.path.exists(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
else:
shutil.copy(src, os.path.join(cpy_drive, dst))
print(f"Copied {src} to {dst}")
utils.flush(cpy_drive)
def numerical_gradient_vectorized_v1(iobj, x):
check_vector4_vectorized(x)
if TEST_ON:
print "numerical_gradient1",
flush()
count = x.shape[0]
g = np.zeros((count, 4))
for i in range(x.shape[0]):
v = x[i, 0:4]
# inefficient: not vectorised
g[i, :] = numerical_gradient(iobj, v, is_vectorized=True)
assert not np.any(np.isnan(g), axis=None)
print "numerical_gradient2",
flush()
g2 = numerical_gradient_vectorized_v2(iobj, x.copy())
if TEST_ON:
assert np.allclose(g, g2)
print "done"
flush()
return g2
async def main():
await asyncio.gather(request("First"), request("Second"), request("Third"))
flush()
def parse_excel_file(file):
excel_file = open_workbook(file, encoding_override='utf-8')
flush('Opening excel file...')
def prepare_mapping():
s = excel_file.sheet_by_name("Mapping")
for x in range(1, s.nrows):
db.session.add(
db.Lead(account=s.cell(x, 0).value,
account_name=s.cell(x, 1).value,
lead_code=s.cell(x, 2).value,
name=s.cell(x, 3).value))
flush('Parsing mappings...')
db.session.commit()
def define_variables():
flush('Defining Company names and audit periods...')
sheet = excel_file.sheet_by_name('Instruction')
db.tanimlar['company'] = sheet.cell_value(6, 2)
try:
db.tanimlar['optional'] = sheet.cell_value(7, 2) or "NO"
except IndexError:
flush('There is no optional column...')
for sheet in excel_file.sheets()[2:]:
db.periodss.append(sheet.name)
# donus = dict()
define_variables()
db.Hesaplar = db.make_hesaplar()
db.create_db()
db.len_periods = len(db.periodss)
prepare_mapping()
def find_seperator():
from collections import defaultdict
pointer = defaultdict(int)
stop = 0
for item in excel_file.sheet_by_index(excel_file.nsheets - 1).col(0):
if stop == 5:
return max(pointer.keys(), key=lambda q: pointer[q])
if not str(item.value).isalnum():
for char in str(item.value):
if not char.isalnum():
pointer[char] += 1
stop += 1
for i in range(2, excel_file.nsheets):
sheet = excel_file.sheet_by_index(i)
key = sheet.name
flush('Parsing %s...' % key)
headers = [col.value.strip() for col in sheet.row(0)]
desc_col = 0
bal_col = 0
opt_col = 999
for y in range(len(headers)):
if headers[y].lower() in bakiye:
bal_col = y
if headers[y].lower() in hesap_adi:
desc_col = y
if headers[y] == db.tanimlar['optional']:
opt_col = y
for row in range(1, sheet.nrows):
temp = {}
# todo float diye gorulen kalemlerde seperator farkedilip aslinda str'ye cevrilip seperator replace edilebilir.
for col in range(sheet.ncols):
if col is 0:
temp["number"] = str(sheet.cell_value(
row, col)) # burasi duzeltilecek bug var.
elif col == bal_col:
temp[key] = sheet.cell_value(row, col)
elif col == opt_col:
temp['optional'] = sheet.cell_value(row, col)
elif col == desc_col:
temp['name'] = sheet.cell_value(row, col)
param = 'optional' if not db.tanimlar['optional'] == 'NO' else 'NO'
if param is 'NO':
gecici = db.session.query(
db.Hesaplar).filter_by(number=temp['number']).first()
else:
hink = db.session.query(db.Hesaplar).filter_by(
number=temp['number'], **{
param: temp[param]
}).first()
gecici = None
if hink is not None:
if int(getattr(hink, key)) is 0:
gecici = hink
ana_hesap = temp['number'][:3]
lead_cod = None
if db.session.query(
db.Lead).filter_by(account=ana_hesap).first() is not None:
lead_cod = db.session.query(
db.Lead).filter_by(account=ana_hesap).first().lead_code
if gecici is not None:
setattr(gecici, key, temp[key])
flush(temp['number'] + ' updated...', wait=0.0001, code='blue')
else:
db.session.add(
db.Hesaplar(**temp,
ana_hesap=ana_hesap,
lead_code=lead_cod,
len=len(temp['number'])))
flush(temp['number'] + ' added...', wait=0.0001, code='blue')
db.session.commit()
flush('Period %s parsed...' % key)
fake_samples = fake_samples[0,:]
print("==> (iters: "+str(iter_)+") Fake Samples Summary; mu "+str(mu)+"; sigma "+str(sigma)+" <===")
a1 = str(fake_samples.shape)
print("==> Shape: "+a1)
print("==> Vec: "+str(fake_samples))
print("==> Values Greater than mu="+str(mu)+"; n="+str(np.sum(fake_samples > mu)))
a2 = str(np.mean(fake_samples))
print("==> Mean: "+a2)
a3 = str(np.std(fake_samples))
print("==> STD: "+a3)
utils.flush(img_folder)
# generate_image(sess, batch_data, iter_)
if (np.mod(iter_, FREQ) == 0) or (iter_+1 == ITERS):
session_saver.save(sess, 'model/my_gan.ckpt')
utils.tick()
# In[12]:
def execute_build(build_cfg):
# set configured tool resolutions
if not build_cfg.suppress_variant_handling():
p = build_cfg.variant_cosy()
cosy = CoSy(p)
log.info('using variant coordinate system ' + p)
log.info(' coordinate dimensions: ' + str(cosy.get_dimensions()))
for d in cosy.get_dimensions():
log.info(' dimension ' + d + ': ' + str(cosy.get_dimension(d)))
coords = build_cfg.variant_coords()
# projection of variant coordinates according to actual coordinate system
for c in coords.keys():
if not c in cosy.get_dimensions():
log.warning("invalid dimension '" + c + "' -> adjusted")
del coords[c]
cosy.check_coords(coords)
build_cfg._variant_vector = cosy.variant_coord_vector(coords)
ct = build_cfg.configured_tools()
if len(ct) != 0:
tools = build_cfg.tools()
for n in ct.keys():
tid = ct[n].toolid()
ct[n]._inst_dir = tools[tid][ct[n].version()]
#as a fallback, use built-in vmake build plugin
if not build_cfg.skip_build():
if build_cfg._externalplugin_setup:
log.info('-' * 100)
log.info('| {0:15} {1:80} |'.format('Build plugin:',
build_cfg._build_script_name))
log.info('| {0:96} |'.format('-' * 96))
log.info('| {0:15} {1:80} |'.format(
'version:', build_cfg._build_script_version))
try:
log.info('| {0:15} {1:80} |'.format(
'description:',
build_cfg._externalplugin_setup.get_description()))
except AttributeError:
log.info('| {0:15} {1:80} |'.format('description:', ''))
log.info('|{0:98}|'.format(''))
try:
log.info('| {0:15} {1:80} |'.format(
'author:', build_cfg._externalplugin_setup.get_author()))
except AttributeError:
log.info('| {0:15} {1:80} |'.format('author:', ''))
try:
log.info('| {0:15} {1:80} |'.format(
'contact:',
build_cfg._externalplugin_setup.get_contact_email()))
except AttributeError:
log.info('| {0:15} {1:80} |'.format('contact:', ''))
log.info('|{0:98}|'.format(''))
log.info('-' * 100)
build_script = build_cfg.build_script()
utils.flush()
build_script.run()
utils.flush()
log.info("build succeeded")
else:
log.info("build step skipped because of explicitly given option -B")
def __init__(self,myname,mynum,mychain,xvg_resnum,basedir,num_sims,max_angles,xvgorpdb,binwidth,sigalpha=1,
permutations=0,phipsi=0,backbone_only=0,adaptive_partitioning=0,which_runs=None,pair_runs=None,bootstrap_choose=3,
calc_variance=False, all_angle_info=None, xvg_chidir = "/dihedrals/g_chi/", skip=1, skip_over_steps=0, last_step=None, calc_mutinf_between_sims="yes", max_num_chis=99,
sequential_res_num = 0, pdbfile = None, xtcfile = None, output_timeseries = "no", minmax=None, bailout_early = False, lagtime_interval = None, markov_samples = 250, num_convergence_points=1):
global master_angles_matrix
global test_reslist
global xtc_coords
global last_good_numangles # last good value for number of dihedrals
global NumChis, NumChis_Safe
self.name = myname
self.num = mynum
self.chain = mychain
self.xvg_basedir = basedir
self.xvg_chidir = xvg_chidir
self.xvg_resnum = xvg_resnum
self.sequential_res_num = sequential_res_num
self.backbone_only, self.phipsi = backbone_only, phipsi
self.max_num_chis = max_num_chis
self.markov_samples = markov_samples
coarse_discretize = None
split_main_side = None
# we will look at mutual information convergence by taking linear subsets of the data instead of bootstraps, but use the bootstraps data structures and machinery. The averages over bootstraps then won't be meaningful
# however the highest number bootstrap will contain the desired data -- this could be fixed later at the bottom of the code if desired
# I also had to change some things in routines above that this code references in order to change numangles_bootstrap. We will essentially look at convergence by only looking at subsets of the data
# in the weaves below, numangles will vary with
if(phipsi >= 0):
try:
self.nchi = self.get_num_chis(myname) * (1 - backbone_only) + phipsi * self.has_phipsi(myname)
except:
NumChis = NumChis_Safe #don't use Ser/Thr hydroxyls for pdb trajectories
self.nchi = NumChis[myname] * (1 - backbone_only) + phipsi * self.has_phipsi(myname)
elif(phipsi == -2):
split_main_side = True
if(self.chain == "S"):
self.nchi = self.get_num_chis(myname)
else:
self.nchi = 2 * self.has_phipsi(myname)
elif(phipsi == -3):
self.nchi = 3 #C-alpha x, y, z
elif(phipsi == -4):
print "doing analysis of stress data"
self.nchi = 1 # just phi as a placeholder for a single variable
else: #coarse discretize phi/psi into 4 bins: alpha, beta, turn, other
self.nchi = self.get_num_chis(myname) * (1 - backbone_only) + 1 * self.has_phipsi(myname)
coarse_discretize = 1
phipsi = 1
if(xtcfile != None):
self.nchi = 3 # x, y, z
self.symmetry = ones((self.nchi),int16)
self.numangles = zeros((num_sims),int32)
self.num_sims = num_sims
self.which_runs = array(which_runs)
which_runs = self.which_runs
#which_runs=array(self.which_runs)
self.pair_runs = pair_runs
self.permutations= permutations
self.calc_mutinf_between_sims = calc_mutinf_between_sims
if(bootstrap_choose == 0):
bootstrap_choose = num_sims
#print "bootstrap set size: "+str(bootstrap_choose)+"\n"
#print "num_sims: "+str(num_sims)+"\n"
#print self.which_runs
#print "\n number of bootstrap sets: "+str(len(self.which_runs))+"\n"
#check for free memory at least 15%
#check_for_free_mem()
#allocate stuff
bootstrap_sets = self.which_runs.shape[0]
#check num convergence points
if num_convergence_points > 1:
assert(num_convergence_points == bootstrap_sets)
self.entropy = zeros((bootstrap_sets,self.nchi), float64)
self.entropy2 = zeros((bootstrap_sets,self.nchi), float64) #entropy w/fewer bins
self.entropy3 = zeros((bootstrap_sets,self.nchi), float64) #entropy w/fewer bins
self.entropy4 = zeros((bootstrap_sets,self.nchi), float64) #entropy adaptive
self.var_ent = zeros((bootstrap_sets,self.nchi), float64)
self.numangles_bootstrap = zeros((bootstrap_sets),int32)
print "\n#### Residue: "+self.name+" "+self.num+" "+self.chain+" torsions: "+str(self.nchi), utils.flush()
binwidth = float(binwidth)
bins = arange(0,360, binwidth) # bin edges global variable
nbins=len(bins) # number of bins
nbins_cor = int(nbins * FEWER_COR_BTW_BINS);
self.nbins=nbins
self.nbins_cor=nbins_cor
sqrt_num_sims=sqrt(num_sims)
self.chi_pop_hist=zeros((bootstrap_sets, self.nchi,nbins),float64)
self.chi_counts=zeros((bootstrap_sets, self.nchi, nbins), float64) #since these can be weighted in advanced sampling
#self.chi_var_pop=zeros((bootstrap_sets, self.nchi,nbins),float64)
self.chi_pop_hist_sequential=zeros((num_sims, self.nchi, nbins_cor), float64)
num_histogram_sizes_to_try = 2 # we could try more and pick the optimal size
self.chi_counts_sequential=zeros((num_sims, self.nchi, nbins_cor), float64) #half bin size
self.chi_counts_sequential_varying_bin_size=zeros((num_histogram_sizes_to_try, num_sims, self.nchi, int(nbins*(num_histogram_sizes_to_try/2)) ), float64) #varying bin size
self.angles_input = zeros((self.nchi,num_sims,max_angles),float64) # the dihedral angles, with a bigger array than will be needed later
#self.sorted_angles = zeros((self.nchi,num_sims,max_angles),float64) # the dihedral angles sorted
self.ent_hist_left_breaks = zeros((self.nchi, nbins * MULT_1D_BINS + 1),float64)
self.adaptive_hist_left_breaks = zeros((bootstrap_sets, nbins + 1),float64) #nbins plus one to define the right side of the last bin
self.adaptive_hist_left_breaks_sequential = zeros(( num_sims, nbins_cor + 1 ),float64) #nbins_cor plus one to define the right side of the last bin
self.adaptive_hist_binwidths = zeros((bootstrap_sets, nbins ),float64)
self.adaptive_hist_binwidths_sequential = zeros(( num_sims, nbins_cor ),float64)
self.ent_hist_binwidths = zeros((bootstrap_sets, self.nchi, nbins * MULT_1D_BINS),float64)
self.ent_from_sum_log_nn_dists = zeros((bootstrap_sets, self.nchi, MAX_NEAREST_NEIGHBORS),float64)
self.minmax = zeros((2,self.nchi))
self.minmax[1,:] += 1 #to avoid zero in divide in expand_contract angles
if(phipsi >= 0):
self.nchi = self.get_num_chis(myname) * (1 - backbone_only) + phipsi * self.has_phipsi(myname)
elif(phipsi == -2):
split_main_side = True
if(self.chain == "S"):
self.nchi = self.get_num_chis(myname)
else:
self.nchi = 2 * self.has_phipsi(myname)
elif(phipsi == -3):
self.nchi = 3 #C-alpha x, y, z
elif(phipsi == -4):
print "doing analysis of stress data"
self.nchi = 1 # just phi as a placeholder for a single variable
else: #coarse discretize phi/psi into 4 bins: alpha, beta, turn, other
self.nchi = self.get_num_chis(myname) * (1 - backbone_only) + 1 * self.has_phipsi(myname)
coarse_discretize = 1
phipsi = 1
if(xtcfile != None):
self.nchi = 3 # x, y, z
self.symmetry = ones((self.nchi),int16)
self.numangles = zeros((num_sims),int32)
self.num_sims = num_sims
self.which_runs = array(which_runs)
which_runs = self.which_runs
#which_runs=array(self.which_runs)
self.pair_runs = pair_runs
self.permutations= permutations
self.calc_mutinf_between_sims = calc_mutinf_between_sims
if(bootstrap_choose == 0):
bootstrap_choose = num_sims
#print "bootstrap set size: "+str(bootstrap_choose)+"\n"
#print "num_sims: "+str(num_sims)+"\n"
#print self.which_runs
#print "\n number of bootstrap sets: "+str(len(self.which_runs))+"\n"
#check for free memory at least 15%
#check_for_free_mem()
#allocate stuff
bootstrap_sets = self.which_runs.shape[0]
#check num convergence points
if num_convergence_points > 1:
assert(num_convergence_points == bootstrap_sets)
self.numangles_bootstrap = zeros((bootstrap_sets),int32)
print "\n#### Residue: "+self.name+" "+self.num+" "+self.chain+" torsions: "+str(self.nchi), utils.flush()
if(xvgorpdb == "xvg"):
self._load_xvg_data(basedir, num_sims, max_angles, xvg_chidir, skip,skip_over_steps,last_step, coarse_discretize, split_main_side)
if(xvgorpdb == "pdb"):
self._load_pdb_data(all_angle_info, max_angles)
if(xvgorpdb == "xtc"):
self.load_xtc_data(basedir, num_sims, max_angles, xvg_chidir, skip, skip_over_steps, pdbfile, xtcfile)
#resize angles array to get rid of trailing zeros, use minimum number
print "weights"
print self.weights
#print "resizing angles array, and creating arrays for adaptive partitioning"
min_num_angles = int(min(self.numangles))
max_angles = int(min_num_angles)
if(min_num_angles > 0):
last_good_numangles = min_num_angles
self.angles = zeros((self.nchi, num_sims, min_num_angles))
angles_autocorrelation = zeros((self.nchi, bootstrap_sets, min_num_angles), float64)
#bins_autocorrelation = zeros((self.nchi, bootstrap_sets, min_num_angles), float64)
self.boot_sorted_angles = zeros((self.nchi,bootstrap_sets,bootstrap_choose*max_angles),float64)
self.boot_ranked_angles = zeros((self.nchi,bootstrap_sets,bootstrap_choose*max_angles),int32)
self.boot_weights = zeros((bootstrap_sets,bootstrap_choose*max_angles),float64)
self.rank_order_angles = zeros((self.nchi,num_sims,max_angles),int32) # the dihedral angles
# rank-ordered with respect to all sims together
self.rank_order_angles_sequential = zeros((self.nchi,num_sims,max_angles),int32) # the dihedral angles
# rank-ordered for each sim separately
# for mutinf between sims
#max_num_angles = int(max(self.numangles))
max_num_angles = int(min(self.numangles)) #to avoid bugs
#counts_marginal=zeros((bootstrap_sets,self.nchi,nbins),float32) # normalized number of counts per bin,
#print "initialized angles_new array"
self.numangles[:] = min(self.numangles)
#print "new numangles"
#print self.numangles
for mychi in range(self.nchi):
for num_sim in range(num_sims):
self.angles[mychi,num_sim,:min_num_angles] = self.angles_input[mychi,num_sim,:min_num_angles]
#print "done copying angles over"
del self.angles_input #clear up memory space
#print self.angles
if ((self.name in ("GLY", "ALA")) and (phipsi == 0 or phipsi == -2)):
# First prepare chi pop hist and chi pop hist sequential needed for mutual information -- just dump everything into one bin,
# giving entropy zero, which should also give MI zero, but serve as a placeholder in the mutual information matrix
if(last_good_numangles > 0):
self.numangles[:] = last_good_numangles # dummy value
self.numangles_bootstrap[:] = bootstrap_choose * int(min(self.numangles))
return #if the side chains don't have torsion angles, drop out
self.sorted_angles = zeros((self.nchi, sum(self.numangles)),float64)
if(xvgorpdb == "xvg" or (xvgorpdb == "pdb" and phipsi != -3)): #if not using C-alphas from pdb
self.correct_and_shift_angles(num_sims,bootstrap_sets,bootstrap_choose, coarse_discretize)
elif(xvgorpdb == "xtc" or (xvgorpdb == "pdb" and phipsi == -3)) : #if using xtc cartesians or pdb C-alphas
self.correct_and_shift_carts(num_sims,bootstrap_sets,bootstrap_choose, num_convergence_points)
if(minmax == None):
print "getting min/max values"
mymin = zeros(self.nchi)
mymax = zeros(self.nchi)
for mychi in range(self.nchi):
mymin[mychi] = min((self.angles[mychi,:,:min(self.numangles)]).flatten())
mymax[mychi] = max((self.angles[mychi,:,:min(self.numangles)]).flatten())
print "__init__ mymin: "
print mymin
print "__init__ mymax: "
print mymax
self.minmax[0, :] = mymin
self.minmax[1, :] = mymax
for mychi in range(self.nchi):
if(self.minmax[1,mychi] - self.minmax[0,mychi] <= 0):
self.minmax[1,mychi] = self.minmax[0,mychi] + 1
print self.minmax
else:
self.minmax = minmax
self.expand_contract_data(num_sims,bootstrap_sets,bootstrap_choose)
if(master_angles_matrix == None):
master_angles_matrix=zeros((run_params.num_sims, len(test_reslist), min_num_angles),float64) #nchi is 1 for stress analysis
master_angles_matrix[:,sequential_res_num,:] = self.angles[0,:,:]
print "master angles matrix: "
print master_angles_matrix[0]
del self.angles #cleanup
del self.rank_order_angles
del self.rank_order_angles_sequential
del self.sorted_angles
del self.boot_sorted_angles
del self.boot_ranked_angles
print(TAG(), "thread_backup:", "START BACKUP ROUTINE")
watcher_manager.save_watchers()
print(TAG(), "thread_backup:", "BACKUP COMPLETED")
flush()
updater = Updater(token=TOKEN)
updater.dispatcher.add_handler(CommandHandler('set', set_function))
updater.dispatcher.add_handler(CommandHandler('del', del_function))
updater.dispatcher.add_handler(CommandHandler('clear', clear_function))
updater.dispatcher.add_handler(CommandHandler('list', list_function))
updater.dispatcher.add_handler(CommandHandler('start', start_function))
updater.dispatcher.add_handler(CommandHandler('stop', stop_function))
# set watchers backup
t = Thread(target=backup)
t.start()
# start watchers thread
watcher_manager.start()
# start telegram
updater.start_polling()
print(TAG(), "Bot started!")
flush()
# start idle
updater.idle()
# !/usr/bin/python
# -*- coding: utf-8 -*-
from itertools import islice
import neo4j
import utils
if __name__ == '__main__':
driver = neo4j.GraphDatabase.driver(uri='bolt://35.204.164.2:7687', auth=('neo4j', 'cdert_433450'))
query = utils.clear()
# stage 1
utils.flush(query, driver=driver)
# stage 2
with open('C:\Users\user\PycharmProjects\untitled1\wikidata\data.json') as f:
bufer = f.read()
driver.close()
def output_timeseries_chis(myfilename_prefix,myreslist,colnames, nsims = 6, nchi=6, ):
#print "shape of matrix to ouput:"+str(mymatrix.shape)+"\n"
min_num_angles = min(myreslist[0].numangles)
timeseries_chis_matrix = zeros((nsims, len(myreslist) * nchi, min_num_angles), float64) #initialize
#self.angles = zeros((self.nchi,num_sims,max_angles),float64) # the dihedral angles
for res_ind1, myres1 in zip(range(len(myreslist)), myreslist):
print "\n#### Working on residue %s (%s):" % (myres1.num, myres1.name) , utils.flush()
for mychi1 in range(myres1.nchi):
#print "%s chi: %d/%d" % (myres1.name,int(myres1.num),mychi1+1)
#print "res_ind1: "+str(res_ind1)
#rint "mychi1: "+str(mychi1)
#print "nchi: " +str(nchi)
#print "min_num_angles: "+str(min_num_angles)
#print "res_ind1 * nchi + mychi1: "+str(res_ind1 * nchi + mychi1)
#print "myres1.angles: "
#print myres1.angles
#print "angle entries: "
#print myres1.angles[mychi1, :, :min_num_angles]
timeseries_chis_matrix[:, res_ind1 * nchi + mychi1, :] = myres1.angles[mychi1, :, :min_num_angles]
my_file_list = []
for mysim in range(nsims):
myfile = open(myfilename_prefix + "_" + str(mysim) + ".txt",'w')
for col_num, col_name in zip(range(len(colnames)), colnames):
for col_chi in range(nchi):
myfile.write(col_name + "_" +str(col_chi) + " ")
myfile.write("\n")
for myrow in range(min_num_angles):
for col_num, col_name in zip(range(len(colnames)), colnames):
for col_chi in range(nchi):
myfile.write(str(timeseries_chis_matrix[mysim,col_num * nchi + col_chi, myrow]))
myfile.write(" ")
myfile.write("\n")
myfile.close()
return timeseries_chis_matrix
info = engineInfo.read_info_file(engine_path)
executor = executors.make_executor(info)
for config_name in options.configs:
config = configs.getConfig(config_name, info)
if config.omit():
continue
try:
results = executor.run(benchmark, config)
if not results:
continue
except Exception as e:
log('Failed to run ' + engine_path + ' - ' + benchmark.version + ' - ' + config_name + '!')
log('Exception: ' + repr(e))
import traceback
traceback.print_exc()
continue
mode = submitter.mode(info["engine_type"], config_name)
submitter.add_tests(results, benchmark.suite, benchmark.version, mode)
# Try to preserve order of logs.
utils.flush()
if not options.session:
submitter.finish()
log("my work is done here!")
utils.flush()
