def minimize2(atoms, bonds, angles, dihedrals, name='', restrained=None, restraint_value=None): run_name = utils.unique_filename('lammps/', 'minimize2_'+name, '.data') write_data_file2(atoms, bonds, angles, dihedrals, run_name) run_minimize(run_name, restrained=restrained, restraint_value=restraint_value) tail = subprocess.Popen('tail lammps/'+run_name+'.xyz -n '+str(len(atoms)), shell=True, stdout=subprocess.PIPE).communicate()[0] if not tail: raise Exception('Minimize failed') return [[float(xyz) for xyz in line.split()[1:]] for line in tail.splitlines()]
def onStart(self, e): path = self.dirpicker.GetPath() raw_url = self.urlinput.GetValue() isvalid = True msg = "" manifest = utils.get_manifest(raw_url) if path == "": isvalid = False msg = "A directory must be selected!" elif not os.path.exists(path): isvalid = False msg = "\"{}\" does not exist!".format(path) elif not os.path.isdir(path): isvalid = False msg = "\"{}\" is not a directory!".format(path) elif manifest is None: isvalid = False msg = "Could not download IIIF Presentation Manifest at URL!" elif len(manifest.sequences) == 0: isvalid = False msg = "No sequences in this manifest!" if not isvalid: msgdlg = wx.MessageDialog(self, msg, caption="IIIF Downloader - Error!", style=wx.OK | wx.CENTRE | wx.ICON_ERROR) msgdlg.ShowModal() else: path = os.path.join(path, manifest.label) path = utils.unique_filename(path, "") os.makedirs(path) sequence = manifest.sequences[0] canvases = sequence.canvases progressstr = "Downloading Canvas {} of {}" progressdlg = wx.ProgressDialog( title="Downloading \"{}\"".format(manifest.label), message="Starting...", maximum=len(canvases), style=wx.PD_CAN_ABORT | wx.PD_ELAPSED_TIME) for canvas, num in zip(canvases, range(len(canvases))): ok, _ = progressdlg.Update(num, newmsg=progressstr.format( num + 1, len(canvases))) if not ok: break runthread = Thread(target=utils.download_canvas, args=(canvas, path, "{}_".format(num))) runthread.start() while runthread.is_alive(): progressdlg.Update(num) runthread.join(0.200) progressdlg.Destroy() print("Pressed")
def minimize2(atoms, bonds, angles, dihedrals, name="", restrained=None, restraint_value=None): run_name = utils.unique_filename("lammps/", "minimize2_" + name, ".data") write_data_file2(atoms, bonds, angles, dihedrals, run_name) run_minimize(run_name, restrained=restrained, restraint_value=restraint_value) tail = subprocess.Popen( "tail lammps/" + run_name + ".xyz -n " + str(len(atoms)), shell=True, stdout=subprocess.PIPE ).communicate()[0] if not tail: raise Exception("Minimize failed") return [[float(xyz) for xyz in line.split()[1:]] for line in tail.splitlines()]
def anneal(atoms, bonds, angles, dihedrals, params, name=''): run_name = utils.unique_filename('lammps/', 'anneal_'+name, '.data') write_data_file(atoms, bonds, angles, dihedrals, params, run_name) run_anneal(run_name) jsub.wait(run_name) tail = subprocess.Popen('tail lammps/'+run_name+'.xyz -n '+str(len(atoms)), shell=True, stdout=subprocess.PIPE).communicate()[0] if 'No such file or directory' in tail: raise Exception('Anneal '+run_name+' failed') for i,line in enumerate(tail.splitlines()): atoms[i].x, atoms[i].y, atoms[i].z = [float(s) for s in line.split()[1:]] return atoms
def create_symlink(orig_path, new_path): cmd = 'ln -s "%s" "%s"' print cmd % (orig_path, new_path) if not os.path.exists(orig_path): print '\tpath does not exist, skipping:',orig_path return 'skipped' if os.path.exists(new_path): new_path = utils.unique_filename(new_path) print '\tlink exists, renamed to:',new_path res = os.system(cmd % (quote(orig_path), quote(new_path))) print '\tresult:',res return res
def scatter_plot_3d(input_data, hover_info=[], marker_size=6, title="", plot_save_path=""): """Generate and open an interactive 3D scatter plot as html. Args: input_data (dict): Dictionary of (user: feature_vector). hover_info (list): List of labels/info to display for each user in input_data in 3D scatter plot. May be Hastags or Mentions. marker_size (int, optional): Point size in plot. Defaults to 6. title (str, optional): Title of the plot. Defaults to "". plot_save_path (str, optional): Save path for plot html file. Defaults to "". TODO: Add marker labels. """ # Get data to plot from input_data dict feature_vectors, labels = zip(*list(input_data.values())) x, y, z = zip(*[tuple(x) for x in feature_vectors]) trace = [ go.Scatter3d(x=x, y=y, z=z, mode='markers', hovertemplate='%{text}', text=[" ".join(x for x in item) for item in hover_info], marker=dict(size=marker_size, color=labels, colorscale='Viridis', opacity=0.8)) ] fig = go.Figure(data=trace) title += INFO.LOAD_PARAMS_USED + INFO.DIM_RED_USED + INFO.CLUSTERING_USED fig.update_layout(title=title, xaxis_title="", yaxis_title="", legend_title="") if plot_save_path: extra_info = INFO.LOAD_PARAMS_USED + INFO.DIM_RED_USED + INFO.CLUSTERING_USED plot_save_path = unique_filename(plot_save_path, extra_info) fig.write_html(plot_save_path) fig.show() return
def move_file(orig_path, new_path): cmd = 'mv %s %s' # % (quote(orig_path), quote(new_path)) if not os.path.exists(orig_path): print '\tpath does not exist, skipping:',orig_path return 'skipped' if os.path.exists(new_path): new_path = utils.unique_filename(new_path) print '\tlink exists, renamed to:',new_path cmd = cmd % (quote(orig_path), quote(new_path)) print cmd res = os.system(cmd) print '\tresult:',res return res
def anneal(atoms, bonds, angles, dihedrals, params, name=""): run_name = utils.unique_filename("lammps/", "anneal_" + name, ".data") write_data_file(atoms, bonds, angles, dihedrals, params, run_name) run_anneal(run_name) jsub.wait(run_name) tail = subprocess.Popen( "tail lammps/" + run_name + ".xyz -n " + str(len(atoms)), shell=True, stdout=subprocess.PIPE ).communicate()[0] if "No such file or directory" in tail: raise Exception("Anneal " + run_name + " failed") for i, line in enumerate(tail.splitlines()): atoms[i].x, atoms[i].y, atoms[i].z = [float(s) for s in line.split()[1:]] return atoms
def chelpg(atoms, theory, queue='batch', chkfile_run_name=None, name=''): run_name = utils.unique_filename('gaussian/', 'chelpg_'+name+'_'+theory[:8].translate( string.maketrans('/(),*', '_____') ), '.log') if chkfile_run_name: shutil.copyfile('gaussian/'+chkfile_run_name+'.chk', 'gaussian/'+run_name+'.chk') job(atoms, theory, queue, run_name, 'Pop=CHelpG') jsub.wait(run_name) try: charges = parse_chelpg('gaussian/'+run_name+'.log') except: #sometimes file is not written yet time.sleep(10) charges = parse_chelpg('gaussian/'+run_name+'.log') if charges: for i,charge in enumerate(charges): atoms[i].charge = charge return charges
def save_and_get_text(files): """ Save image, perform ocr, save result to mongo :param files: :return: """ print files text = [] times = [] filenames = [] first_image = None storage = GoogleCloudStorage() for i, file in enumerate(files): # Check if the file is one of the allowed types/extensions if file and allowed_file(file.filename): start_time = time.time() # Make the filename safe, remove unsupported chars filename = unique_filename(file.filename) filenames.append(filename) # Image object img = Image.open(file) # Perform ocr to get text result = perform_ocr(img) text.append(result) # Save file to temporary folder filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) img.convert('RGB').save(filepath, optimize=True, quality=85) if i == 0: first_image = img else: img.close() # Save file to Cloud Storage storage.upload_to_cloud_storage(filepath) # Delete tmp file delete_file(filename) # Get time delta in seconds end_time = time.time() times.append(end_time - start_time) if first_image is None: return "", [] # Join in unique text text = "\n".join(text) # Create thumbnail of first image thumbnail = create_thumbnail(first_image) first_image.close() # Save to mongo save_history(text, thumbnail, filenames) return text, times
def main_stem( ag, ckpt, archtype, shape_x, dim_y, tr_src_loader, val_src_loader, ls_ts_tgt_loader=None, # for ood tr_tgt_loader=None, ts_tgt_loader=None, testdom=None # for da ): print(ag) print_infrstru_info() IS_OOD = is_ood(ag.mode) device = tc.device("cuda:" + str(ag.gpu) if tc.cuda.is_available() else "cpu") # Datasets dim_x = tc.tensor(shape_x).prod().item() if IS_OOD: n_per_epk = len(tr_src_loader) else: n_per_epk = max(len(tr_src_loader), len(tr_tgt_loader)) # Models res = get_models(archtype, edic(locals()) | vars(ag), ckpt, device) if ag.mode.endswith("-da2"): discr, gen, frame, discr_src = res discr_src.train() else: discr, gen, frame = res discr.train() if gen is not None: gen.train() # Methods and Losses if IS_OOD: lossfn = ood_methods( discr, frame, ag, dim_y, cnbb_actv="Sigmoid" ) # Actually the activation is ReLU, but there is no `is_treat` rule for ReLU in CNBB. domdisc = None else: lossfn, domdisc, dalossobj = da_methods( discr, frame, ag, dim_x, dim_y, device, ckpt, discr_src if ag.mode.endswith("-da2") else None) # Optimizer pgc = ParamGroupsCollector(ag.lr) pgc.collect_params(discr) if ag.mode.endswith("-da2"): pgc.collect_params(discr_src) if gen is not None: pgc.collect_params(gen, frame) if domdisc is not None: pgc.collect_params(domdisc) if ag.optim == "SGD": opt = getattr(tc.optim, ag.optim)(pgc.param_groups, weight_decay=ag.wl2, momentum=ag.momentum, nesterov=ag.nesterov) shrink_opt = ShrinkRatio(w_iter=ag.lr_wdatum * ag.n_bat, decay_rate=ag.lr_expo) lrsched = tc.optim.lr_scheduler.LambdaLR(opt, shrink_opt) auto_load(locals(), 'lrsched', ckpt) else: opt = getattr(tc.optim, ag.optim)(pgc.param_groups, weight_decay=ag.wl2) auto_load(locals(), 'opt', ckpt) # Training epk0 = 1 i_bat0 = 1 if ckpt is not None: epk0 = ckpt['epochs'][-1] + 1 if ckpt['epochs'] else 1 i_bat0 = ckpt['i_bat'] res = ResultsContainer( len(ag.testdoms) if IS_OOD else None, frame, ag, dim_y == 1, device, ckpt) print(f"Run in mode '{ag.mode}' for {ag.n_epk:3d} epochs:") try: for epk in range(epk0, ag.n_epk + 1): pbar = tqdm.tqdm(total=n_per_epk, desc=f"Train epoch = {epk:3d}", ncols=80, leave=False) for i_bat, data_bat in enumerate( tr_src_loader if IS_OOD else zip_longer( tr_src_loader, tr_tgt_loader), start=1): if i_bat < i_bat0: continue if IS_OOD: x, y = data_bat data_args = (x.to(device), y.to(device)) else: (x, y), (xt, yt) = data_bat data_args = (x.to(device), y.to(device), xt.to(device)) opt.zero_grad() if ag.mode in MODES_GEN: n_iter_tot = (epk - 1) * n_per_epk + i_bat - 1 loss = lossfn(*data_args, n_iter_tot) else: loss = lossfn(*data_args) loss.backward() opt.step() if ag.optim == "SGD": lrsched.step() pbar.update(1) # end for pbar.close() i_bat = 1 i_bat0 = 1 if epk % ag.eval_interval == 0: res.update(epk=epk, loss=loss.item()) print(f"Mode '{ag.mode}': Epoch {epk:.1f}, Loss = {loss:.3e},") discr.eval() if ag.mode.endswith("-da2"): discr_src.eval() true_discr = discr_src elif ag.mode in MODES_TWIST and ag.true_sup_val: true_discr = partial(frame.logit_y1x_src, n_mc_q=ag.n_mc_q) else: true_discr = discr res.evaluate(true_discr, "val " + str(ag.traindom), 'val', val_src_loader, 'src') if IS_OOD: for i, (testdom, ts_tgt_loader) in enumerate( zip(ag.testdoms, ls_ts_tgt_loader)): res.evaluate(discr, "test " + str(testdom), 'ts', ts_tgt_loader, 'tgt', i) else: res.evaluate(discr, "test " + str(testdom), 'ts', ts_tgt_loader, 'tgt') print() discr.train() if ag.mode.endswith("-da2"): discr_src.train() # end for except (KeyboardInterrupt, SystemExit): pass res.summary("val " + str(ag.traindom), 'val') if IS_OOD: for i, testdom in enumerate(ag.testdoms): res.summary("test " + str(testdom), 'ts', i) else: res.summary("test " + str(testdom), 'ts') if not ag.no_save: dirname = "ckpt_" + ag.mode + "/" os.makedirs(dirname, exist_ok=True) for i, testdom in enumerate(ag.testdoms if IS_OOD else [testdom]): filename = unique_filename( dirname + ("ood" if IS_OOD else "da"), ".pt", n_digits=3) if ckpt is None else ckpt['filename'] dc_vars = edic(locals()).sub([ 'dirname', 'filename', 'testdom', 'shape_x', 'dim_x', 'dim_y', 'i_bat' ]) | (edic(vars(ag)) - {'testdoms'}) | dc_state_dict( locals(), "discr", "opt") if ag.mode.endswith("-da2"): dc_vars.update(dc_state_dict(locals(), "discr_src")) if ag.optim == "SGD": dc_vars.update(dc_state_dict(locals(), "lrsched")) if ag.mode in MODES_GEN: dc_vars.update(dc_state_dict(locals(), "gen", "frame")) elif ag.mode in {"dann", "cdan", "dan", "mdd"}: dc_vars.update(dc_state_dict(locals(), "domdisc", "dalossobj")) else: pass if IS_OOD: dc_vars.update( edic({ k: v for k, v in res.dc.items() if not k.startswith('ls_') }) | { k[3:]: v[i] for k, v in res.dc.items() if k.startswith('ls_') }) else: dc_vars.update(res.dc) tc.save(dc_vars, filename) print(f"checkpoint saved to '{filename}'.")
def dynamics(atoms, bonds, angles, dihedrals, params, name=""): run_name = utils.unique_filename("lammps/", "dynamics_" + name, ".data")
def evaluate(mol_atoms, mol_bonds, mol_angles, mol_dihedrals, N=1): run_name = utils.unique_filename("lammps/", "eval", ".data") import copy atoms = [] bonds = [] angles = [] dihedrals = [] for mol in range(N): for a in mol_atoms: aa = copy.copy(a) aa.y += mol * 10.0 aa.index += mol * len(mol_atoms) atoms.append(aa) for b in mol_bonds: bb = copy.copy(b) bb.atoms = [atoms[x.index - 1 + mol * len(mol_atoms)] for x in b.atoms] bonds.append(bb) for b in mol_angles: bb = copy.copy(b) bb.atoms = [atoms[x.index - 1 + mol * len(mol_atoms)] for x in b.atoms] angles.append(bb) for b in mol_dihedrals: bb = copy.copy(b) bb.atoms = [atoms[x.index - 1 + mol * len(mol_atoms)] for x in b.atoms] dihedrals.append(bb) box_size = ( max(atoms, key=lambda a: a.x).x * 2 + 100, max(atoms, key=lambda a: a.y).y * 2 + 100, max(atoms, key=lambda a: a.z).z * 2 + 100, ) os.chdir("lammps") f = open(run_name + ".data", "w") f.write( """LAMMPS Description """ + str(len(atoms)) + """ atoms """ + str(len(bonds)) + """ bonds """ + str(len(angles)) + """ angles """ + str(len(dihedrals)) + """ dihedrals 0 impropers """ + str(len(atoms)) + """ atom types """ + str(len(bonds)) + """ bond types """ + str(len(angles)) + """ angle types """ + str(len(dihedrals)) + """ dihedral types 0 improper types -""" + str(box_size[0] / 2) + """ """ + str(box_size[0] / 2) + """ xlo xhi -""" + str(box_size[1] / 2) + """ """ + str(box_size[1] / 2) + """ ylo yhi -""" + str(box_size[2] / 2) + """ """ + str(box_size[2] / 2) + """ zlo zhi Masses """ + ("\n".join(["%d\t%f" % (atom.index, atom.type.mass) for atom in atoms])) + """ Pair Coeffs """ + ("\n".join(["%d\t%f\t%f" % (atom.index, atom.type.vdw_e, atom.type.vdw_r) for atom in atoms])) ) if bonds: f.write( "\n\nBond Coeffs\n\n" + "\n".join(["%d\t%f\t%f" % (i + 1, bond.e, bond.d) for i, bond in enumerate(bonds)]) ) if angles: f.write( "\n\nAngle Coeffs\n\n" + "\n".join(["%d\t%f\t%f" % (i + 1, angle.e, angle.theta) for i, angle in enumerate(angles)]) ) if dihedrals: f.write( "\n\nDihedral Coeffs\n\n" + "\n".join(["%d\t%f\t%f\t%f\t%f" % ((i + 1,) + tuple(dihedral.e)) for i, dihedral in enumerate(dihedrals)]) ) f.write( "\n\nAtoms\n\n" + "\n".join( ["\t".join([str(q) for q in (a.index, 1, a.index, a.charge, a.x, a.y, a.z)]) for i, a in enumerate(atoms)] ) ) # atom (molecule type charge x y z) if bonds: f.write( "\n\nBonds\n\n" + "\n".join( [ "\t".join([str(q) for q in [i + 1, i + 1, b.atoms[0].index, b.atoms[1].index]]) for i, b in enumerate(bonds) ] ) ) # bond (type a b) if angles: f.write( "\n\nAngles\n\n" + "\n".join( [ "\t".join([str(q) for q in [i + 1, i + 1] + [atom.index for atom in a.atoms]]) for i, a in enumerate(angles) ] ) ) # ID type atom1 atom2 atom3 if dihedrals: f.write( "\n\nDihedrals\n\n" + "\n".join( [ "\t".join([str(q) for q in [i + 1, i + 1] + [atom.index for atom in d.atoms]]) for i, d in enumerate(dihedrals) ] ) ) # ID type a b c d f.write("\n\n") f.close() f = open(run_name + ".in", "w") f.write( """units real atom_style full #bonds, angles, dihedrals, impropers, charges #pair_style lj/cut/coul/long 10.0 8.0 pair_style lj/cut/coul/cut 10.0 8.0 bond_style harmonic angle_style harmonic dihedral_style opls #kspace_style pppm 1.0e-6 special_bonds lj/coul 0.0 0.0 0.5 read_data """ + run_name + """.data thermo_style custom temp press vol pe etotal tpcpu thermo 300 dump 1 all xyz 100 """ + run_name + """.xyz minimize 0.0 1.0e-8 1000 100000 velocity all create 5000 1 rot yes dist gaussian timestep 1.0 fix anneal all nvt temp 5000 1 10 print "anneal" run 30000 unfix anneal minimize 0.0 1.0e-8 1000 100000 """ ) f.close() os.system("lammps < " + run_name + ".in") os.chdir("..")
def main(): """ A command-line REPORT runner, that uses the optargs library, especially the cut-out passthrough from the query-def to define the qualifiers """ #QUERIES=[x.require_query() for x in REPORTS] parser = init_parser(PROG, REPORTS) args_obj = parse_commandline_args(parser, sys.argv) report_obj = args_obj.func #Build Query: query_obj = build_qualified_query(report_obj, args_obj); #at this point the query_obj is fully qualified successfully, and ready for throwing at the db. if DEBUG: print query_obj.debug_qualifiers() # return #args = vars(args_obj) if args_obj.sqlonly: print query_obj.sql return #### #Load DataSet: result_set = query_obj.load_dataset() #print result_set #print arguments #### META = build_meta(args_obj, report_obj, query_obj) #Render Report: addr_from='*****@*****.**' subject_prefix = '[TipprReportServer] ' try: formatter = formatter_jumptable[args_obj.format] DOC_OUT, MIME_TYPE, CON_TYPE = (formatter)(result_set["ROWS"], report_obj.COLS_OUT, CONTEXT=META) #### #Store it: if not args_obj.storage == 'STDOUT': file_locator = None file_name = utils.unique_filename(format=args_obj.format, report=report_obj.slug, by=args_obj.requestor) if args_obj.storage =='S3': file_locator = utils.store_doc_to_s3(DOC_OUT, file_name, headers = {'Content-Type':CON_TYPE}) #can't get it to work yet: # file_locator = utils.url_to_tiny(file_locator) if args_obj.storage =='LOCAL': file_locator = utils.store_doc_to_local(DOC_OUT, file_name) META['file_locator'] = file_locator #### #Send it out to audience: if args_obj.audience: #TODO: add in requestor in here or in notify. deliver_to = args_obj.audience if args_obj.storage=='EMAIL-ATTACH': subject = 'Here is your Report' body = Template(email_body_template('attach')).render(DOC_OUT=DOC_OUT, **META) utils.email_senddoc(addr_from=addr_from, addr_to=deliver_to, subject=subject_prefix+subject, body=body, ctype=CON_TYPE, doc=DOC_OUT, fname=file_name) else: if args_obj.storage=='EMAIL-EMBED': subject = 'Here is your Report' body = Template(email_body_template('embed')).render(DOC_OUT=DOC_OUT, **META) elif file_locator: #No point in sending if we can't send it (ATTACH, EMBED) # or send a link to it subject = 'Your Report is ready to be viewed' body = Template(email_body_template('link')).render(DOC_OUT=DOC_OUT, **META) utils.email_simple(addr_from=addr_from, addr_to=deliver_to, subject=subject_prefix+subject , body=body) #### #Output Report-run info to STDOUT print Template(email_body_template('link')).render(DOC_OUT=DOC_OUT, **META) else: #if STDOUT: print Template(email_body_template('embed')).render(DOC_OUT=DOC_OUT, **META) #### #Notify interested parties: if (args_obj.requestor not in args_obj.audience): #if i'm not already sending the requestor the report, then send him/her the notification. args_obj.notify = (args_obj.notify or []) + [args_obj.requestor,] #Notify Requestor (+notify) of success: if args_obj.notify: #TODO: add in requestor too. subject = 'Notification: Report has run' body = Template(email_body_template('notify')).render(DOC_OUT=DOC_OUT, **META) utils.email_simple(addr_from=addr_from, addr_to=args_obj.notify, subject=subject_prefix+subject, body= body) except: exc_info = sys.exc_info() error_text = """ ERROR: %s: %s Traceback: %s """%(exc_info[0],exc_info[1], traceback.format_exc()) subject = 'ERROR in Report run' body = Template(email_body_template('embed')).render(DOC_OUT=error_text, **META) utils.email_simple(addr_from=addr_from, addr_to=[args_obj.requestor,], subject=subject_prefix+subject, body= body) print body #Echo any errors locally to STDOUT
with open(input_file) as inp: contents = inp.read() if 'Normal termination of Gaussian 09' not in contents: return None start = contents.rindex('Fitting point charges to electrostatic potential') end = contents.index('-----------------', start) charges = [] for line in contents[start:end].splitlines(): columns = line.split() if len(columns)==3: charges.append( float(columns[2]) ) return charges def minimize(atoms, theory, queue='batch', name='', restrained=None, async=False): #blocks until done run_name = utils.unique_filename('gaussian/', 'min_'+name+'_'+theory[:8].translate( string.maketrans('/(),*', '_____') ), '.inp') if not restrained: job(atoms, theory, queue, run_name, 'Opt=CalcFC') #SCRF(Solvent=n-Hexane) else: dihedral = restrained job(atoms, theory, queue, run_name, 'Opt=(ModRedundant,Loose,CalcFC)', extra_section='D %d %d %d %d F'%tuple([a.index for a in dihedral.atoms])) if async: return run_name else: jsub.wait(run_name) energy, coords = parse_coords('gaussian/'+run_name+'.log') if coords: for i,xyz in enumerate(coords): atoms[i].x, atoms[i].y, atoms[i].z = xyz return run_name, energy
def dynamics(atoms, bonds, angles, dihedrals, params, name=''): run_name = utils.unique_filename('lammps/', 'dynamics_'+name, '.data')