def upload(sure=False,part=None,bulk=False): """ Upload the most recent CPT and TPR file to a cluster for continuation. Need to re-write the step/part-specific uploader. """ serial_number() default_fns,default_dirs = ['makefile'],['amx'] default_fns += [os.path.join(root,fn) for root,dirnames,fns in os.walk('./amx') for fn in fns for dn in default_dirs if not re.match('.+\.pyc$',fn)!=None] default_fns = [i for i in default_fns if not re.match('.+\/amx\/docs',i)] last_step,part_num = detect_last() if part: part_num = int(part) last_step, = [i for i in glob.glob('s%02d-*'%part_num)] if not last_step and not bulk: raise Exception('\n[ERROR] no steps to upload (try "bulk" instead)') elif last_step and not bulk: if not part_num: raise Exception('\n[ERROR] cannot find a part number (did you mean "bulk"?)') restart_fns = [last_step+'/md.part%04d.%s'%(part_num,suf) for suf in ['cpt','tpr']] restart_fns += [last_step+'/script-continue.sh'] if not all([os.path.isfile(fn) for fn in restart_fns]): error = '[STATUS] could not find necessary upload files (part number %04d)'%part_num error += '\n[ERROR] upload only works if there is a TPR for the last CPT part' error += "\n[ERROR] missing: %s"%str([fn for fn in restart_fns if not os.path.isfile(fn)]) raise Exception(error) with open('uploads.txt','w') as fp: for fn in restart_fns+default_fns: fp.write(fn+'\n') sshname = raw_input('[QUESTION] enter ssh alias for destination machine: ') subfolder = raw_input('[QUESTION] enter subfolder on remote machine (default is ~/): ') cwd = os.path.basename(os.path.abspath(os.getcwd())) if not sure: cmd = 'rsync -%s%s ../%s %s:~/%s/%s'%( 'avin',' --files-from=uploads.txt' if not bulk else ' --exclude=.git',cwd, sshname,subfolder,cwd if not bulk else '') p = subprocess.Popen(cmd,shell=True,cwd=os.path.abspath(os.getcwd()),executable='/bin/bash') log = p.communicate() if sure or raw_input('\n[QUESTION] continue [y/N]? ')[:1] not in 'nN': cmd = 'rsync -%s%s ../%s %s:~/%s/%s'%( 'avi',' --files-from=uploads.txt' if not bulk else ' --exclude=.git',cwd, sshname,subfolder,cwd if not bulk else '') p = subprocess.Popen(cmd,shell=True,cwd=os.path.abspath(os.getcwd()),executable='/bin/bash') log = p.communicate() if not bulk: os.remove('uploads.txt') if p.returncode == 0 and last_step: with open('script-%s.log'%last_step.rstrip('/'),'a') as fp: destination = '%s:~/%s/%s'%(sshname,subfolder,cwd) ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y.%m.%d.%H%M') fp.write("[FUNCTION] upload () {'destination': '%s', 'time': '%s', 'sure': %s}\n"%( destination,ts,str(sure))) elif p.returncode != 0: print "[STATUS] upload failure (not logged)" sys.exit(1)
def download(): """ Synchronize uploaded files according to log-uploads. """ regex_upload = '^\[FUNCTION]\s+upload\s+\(\)\s+(\{[^\}]+\})' last_step,part_num = detect_last() #----infer the log from the number of the last step last_step_code = re.search('^([a-z][0-9]+)-',last_step).group(1) last_log = [f for f in glob.glob('script-%s-*'%last_step_code)][0] with open(last_log) as fp: loglines = fp.readlines() upload_records = [i for i in loglines if re.match('^\[FUNCTION]\s+upload',i)] if upload_records == []: raise Exception("\n[ERROR] cannot download that which has not been uploaded") last_upload = upload_records[-1] upload_dict = eval(re.findall(regex_upload,last_upload)[0]) destination = upload_dict['destination'] print "[STATUS] log at %s says that this simulation is located at %s"%(last_log,destination) try: cmd = 'rsync -avin --progress %s/* ./'%destination print '[STATUS] running: "%s"'%cmd p = subprocess.Popen(cmd,shell=True,cwd=os.path.abspath(os.getcwd())) log = p.communicate() if p.returncode != 0: raise if raw_input('\n[QUESTION] continue [y/N]? ')[:1] not in 'nN': cmd = 'rsync -avi --progress %s/* ./'%destination print '[STATUS] running "%s"'%cmd p = subprocess.Popen(cmd,shell=True,cwd=os.path.abspath(os.getcwd())) log = p.communicate() except Exception as e: import traceback #---from omnicalc s = traceback.format_exc() print "[TRACE] > "+"\n[TRACE] > ".join(s.split('\n')) print "[ERROR] failed to find simulation" print "[NOTE] find the data on the remote machine via \"find ./ -name serial-%s\""%serial_number() sys.exit(1)
def cluster(**kwargs): """ Write a cluster header according to the machine configuration. Note that we do not log this operation because it only changes the BASH scripts """ if not 'cluster_header' in machine_configuration: print '[STATUS] no cluster information' return head = machine_configuration['cluster_header'] for key,val in machine_configuration.items(): head = re.sub(key.upper(),str(val),head) with open('cluster-header.sh','w') as fp: fp.write(head) print '[STATUS] wrote cluster-header.sh' #---get the most recent step (possibly duplicate code from base) last_step,part_num = detect_last() if last_step: #---code from base.functions.write_continue_script to rewrite the continue script with open('amx/procedures/scripts/script-continue.sh','r') as fp: lines = fp.readlines() tl = [float(j) if j else 0.0 for j in re.match('^([0-9]+)\:?([0-9]+)?\:?([0-9]+)?', machine_configuration['walltime']).groups()] maxhours = tl[0]+float(tl[1])/60+float(tl[2])/60/60 settings = { 'maxhours':maxhours, 'nprocs':machine_configuration['nprocs'], 'tpbconv':gmxpaths['tpbconv'], 'mdrun':gmxpaths['mdrun'], } #---! how should we parse multiple modules from the machine_configuration? if 'modules' in machine_configuration: need_modules = machine_configuration['modules'] need_modules = [need_modules] if type(need_modules)==str else need_modules for m in need_modules: head += "module load %s\n"%m for key in ['extend','until']: if key in machine_configuration: settings[key] = machine_configuration[key] #---! must intervene above to come up with the correct executables setting_text = '\n'.join([ str(key.upper())+'='+('"' if type(val)==str else '')+str(val)+('"' if type(val)==str else '') for key,val in settings.items()]) lines = map(lambda x: re.sub('#---SETTINGS OVERRIDES HERE$',setting_text,x),lines) script_fn = 'script-continue.sh' cont_fn = last_step+script_fn print '[STATUS] %swriting %s'%('over' if os.path.isfile(last_step+script_fn) else '',cont_fn) with open(last_step+script_fn,'w') as fp: for line in lines: fp.write(line) os.chmod(last_step+script_fn,0744) #---code above from base.functions.write_continue_script with open(cont_fn,'r') as fp: continue_script = fp.read() continue_script = re.sub('#!/bin/bash\n','',continue_script) cluster_continue = last_step+'/cluster-continue.sh' print '[STATUS] writing %s'%cluster_continue with open(cluster_continue,'w') as fp: fp.write(head+continue_script) #---for each python script in the root directory we write an equivalent cluster script pyscripts = glob.glob('script-*.py') if len(pyscripts)>0: with open('cluster-header.sh','r') as fp: header = fp.read() for script in pyscripts: name = re.findall('^script-([\w-]+)\.py$',script)[0] with open('cluster-%s.sh'%name,'w') as fp: fp.write(header+'\n') fp.write('python script-%s.py &> log-%s\n'%(name,name)) print '[STATUS] wrote cluster-%s.sh'%name