def main(): """Loads an update from json buffer in midas.""" odb = midas.ODB('gm2-nmr') key = odb.get_value('Custom/Data/json-buffer/key').rstrip() fdir = odb.get_value('Custom/Path').rstrip() fname = os.path.join(fdir, odb.get_value('Custom/Data/json-buffer/file').rstrip()) value = odb.get_value('Custom/Data/json-buffer/value').rstrip() if key == '' or value == '': return 0 if os.path.exists(fname): json_data = json.loads(open(fname).read(), object_pairs_hook=OrderedDict) else: json_data = {} json_data[key] = json.loads(value, object_pairs_hook=OrderedDict) f = open(fname, 'w') f.write(json.dumps(json_data, indent=2)) f.close() key = odb.set_value('Custom/Data/json-buffer/key', '') fname = odb.set_value('Custom/Data/json-buffer/file', '') value = odb.set_value('Custom/Data/json-buffer/value', '') return 0
def main(): # Get the data directory odb = midas.ODB('gm2-nmr') datadir = odb.get_value('/Logger/Data dir').rstrip() + '/' metafiles = glob.glob(datadir + '/*/.processing_metadata.json') for metafile in metafiles: with open(metafile) as f: mf_string = f.read() try: # Load the json file. metadata = json.loads(mf_string) # Save it if not corrupted. with open(metafile + '.old', 'w') as f: f.write(json.dumps(metadata)) except: pass
import time import copy import logging as log import hashlib from subprocess import call, Popen import numpy as np import setproctitle import zmq import gevent import midas # Create a zmq context, global since you only need one. context = zmq.Context() odb = midas.ODB('gm2-nmr') nworkers = 4 maxcount = 20 datadir = odb.get_value('/Logger/Data dir').rstrip() offline_dir = os.path.abspath(os.path.dirname(__file__) + '/../..') # Set up the zmq socket for data. crunch_sck = context.socket(zmq.PULL) crunch_sck.bind('tcp://127.0.1.1:44444') # Set up the zmq socket for logging. logger_sck = context.socket(zmq.PULL) logger_sck.bind('tcp://127.0.1.1:44445') # Worker socket to send job results. worker_sck = context.socket(zmq.PUSH)
def main(): # Open the ZMQ socket that talks to the job scheduler. ctx = zmq.Context() job_sck = ctx.socket(zmq.PUSH) job_sck.connect('tcp://127.0.1.1:44444') req_sck = ctx.socket(zmq.REQ) req_sck.connect('tcp://127.0.1.1:44446') # Get the data directory odb = midas.ODB('gm2-nmr') datadir = odb.get_value('/Logger/Data dir').rstrip() + '/' # Intialize a standard job submission. info = {'run': 1, 'type': 'standard'} # Update only new runs. if sys.argv[1] == 'update': metafile = datadir + 'crunched/.crunchd_metadata.json' crunched_runs = json.load(open(metafile)).keys() crunched_runs.sort() last_crunch = 1 while (('%05i' % last_crunch) in crunched_runs): last_crunch += 1 current_run = int(odb.get_value('/Runinfo/Run number')) current_status = int(odb.get_value('/Runinfo/State')) # If the experiment isn't running, bump the run number if current_status == 1: current_run += 1 print last_crunch, current_run if last_crunch >= current_run - 1: print 'Runs are all crunched.' else: for i in xrange(last_crunch, current_run): info['cmd'] = 'crunch' info['run'] = i rc = job_sck.send_json(info) # Update all runs given elif sys.argv[1] == 'crunch': info['cmd'] = 'crunch' for i in xrange(2, len(sys.argv)): try: info['run'] = int(sys.argv[i]) except (ValueError): print 'Not a valid run number.' continue rc = job_sck.send_json(info) elif sys.argv[1] == 'reset': info['cmd'] = 'reset' for i in xrange(2, len(sys.argv)): try: info['run'] = int(sys.argv[i]) except (ValueError): print 'Not a valid run number.' continue rc = job_sck.send_json(info) elif sys.argv[1] == 'status': msg = {'type': 'status'} rc = req_sck.send_json(msg) print req_sck.recv_json() else: print 'Unrecognized command.'
def main(): # Create the ZMQ sockets that talk to the job scheduler. ctx = zmq.Context() job_sck = ctx.socket(zmq.PUSH) job_sck.connect('tcp://127.0.1.1:44444') req_sck = ctx.socket(zmq.REQ) req_sck.connect('tcp://127.0.1.1:44446') # Grab the data directory from the experiment's ODB. odb = midas.ODB('gm2-nmr') # datadir = odb.get_value('/Logger/Data dir').rstrip() + '/' datadir = '/home/newg2/data/' if len(sys.argv) < 2: print "Insufficient arguments." print "usage: python update_full_scans.py <command> [runs ...]" sys.exit(1) if sys.argv[1] == 'add': print "Adding new full scan." run_num = 1 runfile_glob = datadir + '/bundles/run_list_full_scan*%03i.txt' while len(glob.glob(runfile_glob % run_num)): run_num += 1 if len(sys.argv) < 3: print 'Cannot create a full scan with no runs.' sys.exit(1) runs = [] for i in range(2, len(sys.argv)): runs.append(int(sys.argv[i])) print 'Next full scan is run number %i.' % run_num print 'Writing the run list.' runfile = datadir + '/bundles/run_list_full_scan_%03i.txt' runfile = runfile % run_num with open(runfile, 'w') as f: runlist = [str(run) for run in runs] f.write(' '.join(runlist) + '\n') print 'The run list is complete. To process the full scan now, type:' print '' print 'python scripts/full_scan_helper.py update %i' % run_num print '' elif sys.argv[1] == 'update': print "Rebundling full scans." runfiles = [] # If runs were supplied, check those. if len(sys.argv) > 2: for i in range(2, len(sys.argv)): run = int(sys.argv[i]) runfile = datadir + 'bundles/run_list_full_scan_%03i.txt' % run runfiles.append(runfile) # If not, check all of them. else: runfiles = glob.glob(datadir + 'bundles/run_list*.txt') runfiles.sort() runfiles.reverse() for runfile in runfiles: scan_idx = int(runfile[-7:-4]) runlist = np.genfromtxt(runfile, dtype=np.int).tolist() # Submit all the jobs that the full scan depends on. for run in runlist: job_sck.send_json({'type': 'normal', 'run': run}) print 'Ensuring all prerequisite jobs have run.' while len(runlist) != 0: tmplist = copy.copy(runlist) print '%i job completions unverified.' % len(tmplist) print tmplist for run in runlist: req = {} req['type'] = 'check' req['body'] = {} req['body']['type'] = 'normal' req['body']['msg'] = {'run': run, 'type': 'normal'} req_sck.send_json(req) msg = req_sck.recv_json() if msg['result'] == False: runlist.remove(run) job_sck.send_json({'type': 'bundle-full_scan', 'run': scan_idx}) else: print "Unrecognized argument." print "usage: python update_full_scans.py <command> [runs ...]"
def main(): attr_file = '/home/newg2/Applications/gm2-nmr/resources/log/midas_runlog.json' arch_dir = '/home/newg2/Applications/gm2-nmr/resources/history' try: run_attr = json.loads(open(attr_file).read(), object_pairs_hook=OrderedDict) except: run_attr = json.loads('{}', object_pairs_hook=OrderedDict) odb = midas.ODB('gm2-nmr') if len(sys.argv) > 1: start = int(sys.argv[1]) else: start = 1 if len(sys.argv) > 2: stop = int(sys.argv[2]) else: stop = int(odb.get_value('/Runinfo/Run number').rstrip()) if (odb.get_value('/Runinfo/State').rstrip() != '1'): stop -= 1 for run_num in range(start, stop + 1): run_files = glob.glob(arch_dir + '/*%05i*' % run_num) key = "run_%05i" % run_num run_attr[key] = OrderedDict() # Set the order and initialize. run_attr[key]['comment'] = '' run_attr[key]['tags'] = [] run_attr[key]['start_time'] = 0 run_attr[key]['stop_time'] = 0 run_attr[key]['field_on'] = False run_attr[key]['step_size'] = 0.0 run_attr[key]['laser_point'] = '' run_attr[key]['laser_swap'] = False run_attr[key]['laser_phi_offset'] = 0.0 run_attr[key]['metrolab_angle'] = -1.0 for f in run_files: got_step_size = False for line in open(f): if 'xml' in f: if '"Comment" type="STRING" size="80"' in line: comment = line.split('>')[1].split('<')[0].rstrip() run_attr[key]['comment'] = comment if '"Tags" type="STRING"' in line: tags = line.split('>')[1].split('<')[0].rstrip() run_attr[key]['tags'] = tags.split(',') if '"Start time binary" type="DWORD"' in line: start_time = int(line.split('>')[1].split('<')[0]) run_attr[key]['start_time'] = start_time if '"Stop time binary" type="DWORD"' in line: stop_time = int(line.split('>')[1].split('<')[0]) run_attr[key]['stop_time'] = stop_time if '"Field On" type="BOOL"' in line: field_on = bool(line.split('>')[1].split('<')[0]) run_attr[key]['field_on'] = field_on if '"Step Size" type="DOUBLE"' in line: step_size = float(line.split('>')[1].split('<')[0]) if not got_step_size: run_attr[key]['step_size'] = step_size got_step_size = True if '"Laser Tracker Point" type="STRING"' in line: laser_point = line.split('>')[1].split('<')[0].upper() laser_point = laser_point.strip() if laser_point != 'P1' and laser_point != 'P2': laser_point = 'N' run_attr[key]['laser_point'] = laser_point if '"Laser Phi Offset" type="DOUBLE"' in line: laser_phi_offset = float( line.split('>')[1].split('<')[0]) run_attr[key]['laser_phi_offset'] = laser_phi_offset if '"Metrolab Angle" type="DOUBLE"' in line: metrolab_angle = float( line.split('>')[1].split('<')[0]) run_attr[key]['metrolab_angle'] = metrolab_angle else: if 'Comment = STRING : [80] ' in line: comment = line.split(': [80] ')[1].rstrip() run_attr[key]['comment'] = comment if 'Tags = STRING : [128] ' in line: tags = line.split(': [128] ')[1].rstrip() run_attr[key]['tags'] = tags.split(',') if 'Start time binary = DWORD' in line: start_time = int(line.split(':')[1]) run_attr[key]['start_time'] = start_time if 'Stop time binary = DWORD' in line: stop_time = int(line.split(':')[1]) run_attr[key]['stop_time'] = stop_time if 'Field On = BOOL' in line: field_on = bool(line.split(':')[1]) run_attr[key]['field_on'] = field_on if 'Step Size = DOUBLE' in line: step_size = float(line.split(':')[1]) if not got_step_size: run_attr[key]['step_size'] = step_size got_step_size = True if 'Laser Tracker Point = STRING' in line: laser_point = line.split(':')[1].split(']')[1].upper() laser_point = laser_point.strip() if laser_point != 'P1' and laser_point != 'P2': laser_point = 'N' run_attr[key]['laser_point'] = laser_point if 'Laser Phi Offset = DOUBLE' in line: laser_phi_offset = float(line.split(':')[1]) run_attr[key]['laser_phi_offset'] = laser_phi_offset if 'Metrolab Angle = FLOAT' in line: metrolab_angle = float(line.split(':')[1]) run_attr[key]['metrolab_angle'] = metrolab_angle try: run_attr[key]['step_size'] except: run_attr[key]['step_size'] = -1.0 try: run_attr[key]['laser_point'] except: run_attr[key]['laser_point'] = 'N' try: run_attr[key]['laser_swap'] except: run_attr[key]['laser_swap'] = False print run_attr[key] with open(attr_file, 'w') as f: f.write(json.dumps(run_attr, indent=2))