def _icopy_files(root_names, dst_dir): """ For each of the root file names `root_names` copy <file name>.fits from iRODS to `dst_dir`. Return 0 is all the copy operations succeeded, n where n is the number of files that failed otherwise. """ # FIXME: make this more generic by fetching the iRODS env from a config file err = 0 URL = "irods://dadsops.hlspZone:[email protected]:1247" URL += "/hlspZone/home/rods/dads/pipeline/inputs/" cmd = which("irods.py") if not dst_dir.endswith("/"): dst_dir += "/" # The iRODS directory structure for a POD file called # lz_bdd9_143_0001748545.pod # is # /hlspZone/home/rods/dads/pipeline/inputs/lz_bdd9/143_0001/ # and the actual POD file is stored there. for root_name in root_names: root_name = root_name.lower() tokens = root_name.split("_") full_url = os.path.join( URL, "%s_%s" % (tokens[0], tokens[1]), "%s_%s" % (tokens[2], tokens[3][:4]), "%s.pod" % root_name ) proc = subprocess.Popen([cmd, full_url, dst_dir], shell=False) _error = proc.wait() if _error: err += 1 return err
def process_index(request, instrument, mode, dataset, exposure, repo_root=REPO_ROOT, exe=EXE): """ Display available dataset exposures for the given instrument and mode. """ # Load the template. t = loader.get_template('control/process.html') # Special handling for modeD which uses iRODS. This is a hack :-( if(str(mode) == 'modeD'): repo_root = repo_root.replace('/jwdmsdevvm1/data1/', '', 1) exe = which('process_idataset.py') name = str(instrument) + '/' + str(mode) # Find the instrument. instr_dict = {'name': '%s not found' % (name), 'datasets': ()} for i in INSTRUMENTS: if(i['name'] == name): instr_dict = i break # Get the repository path. repo_path = os.path.join(repo_root, dataset) # Compose the command. args = (exe, '-r', repo_path, '-i', instrument, '-m', mode, exposure) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) err = proc.wait() # Render the template and exit. cmd_str = ' '.join(args) c = Context({'command': cmd_str, 'exit_code': err, 'instrument': instr_dict, 'dataset': dataset, 'exposure': exposure}) return(HttpResponse(t.render(c)))
def create_job_template(drmaa_session, dag_name, work_dir): """ Create a DRMAA Condor DAGMan job given the already instantiated OWL Workflow templates for `dag_name` found in `work_dir`. Return the DRMAA job as a DRMAA template (confusing terminology, I know). """ dagman = which('condor_dagman') if(not dagman): raise(Exception('Unable to find condor_dagman in $PATH')) drmaa_tmplt = drmaa_session.createJobTemplate() drmaa_tmplt.remoteCommand = dagman drmaa_tmplt.workingDirectory = work_dir drmaa_tmplt.nativeSpecification = NATIVE_SPEC_TMPL.render(dag_name=dag_name) drmaa_tmplt.blockEmail = True drmaa_tmplt.jobName = dag_name drmaa_tmplt.outputPath = ':' + os.path.join(work_dir, dag_name + '.lib.out') drmaa_tmplt.errorPath = ':' + os.path.join(work_dir, dag_name + '.lib.err') drmaa_tmplt.joinFiles = False return(drmaa_tmplt)
def create_job_template(drmaa_session, dag_name, work_dir): """ Create a DRMAA Condor DAGMan job given the already instantiated OWL Workflow templates for `dag_name` found in `work_dir`. Return the DRMAA job as a DRMAA template (confusing terminology, I know). """ dagman = which('condor_dagman') if (not dagman): raise (Exception('Unable to find condor_dagman in $PATH')) drmaa_tmplt = drmaa_session.createJobTemplate() drmaa_tmplt.remoteCommand = dagman drmaa_tmplt.workingDirectory = work_dir drmaa_tmplt.nativeSpecification = NATIVE_SPEC_TMPL.render( dag_name=dag_name) drmaa_tmplt.blockEmail = True drmaa_tmplt.jobName = dag_name drmaa_tmplt.outputPath = ':' + os.path.join(work_dir, dag_name + '.lib.out') drmaa_tmplt.errorPath = ':' + os.path.join(work_dir, dag_name + '.lib.err') drmaa_tmplt.joinFiles = False return (drmaa_tmplt)
import os import subprocess from django.template import Context, loader from django.http import HttpResponse from owl.utils import which # Constants # TODO: We should be getting these from, say a DB. REPO_ROOT = '/data2/jwst/repository/raw' EXE = which('process_dataset.py') dataset001 = {'name': 'dataset_001', 'exposures': ('raw-000001', 'raw-000002', 'raw-000003', 'raw-000004')} dataset002 = {'name': 'dataset_002', 'exposures': ('raw-000001', 'raw-000002')} INSTRUMENTS = ({'name': 'instrument1/modeA', 'link_name': 'Regular BCW', 'datasets': (dataset001, dataset002)}, {'name': 'instrument1/modeD', 'link_name': 'iRODS BCW', 'datasets': (dataset001, dataset002)}, )
import os import shutil import socket import time from owl import dag from owl import blackboard from owl.utils import which from spread import client from spread_keyclient import get_clusterid # Constants # TODO: infer these fron the condor/owl config JOB_HOOK = os.path.join(os.path.dirname(which('owld.py')), 'owl_job_hook.py') HOSTNAME = socket.gethostname() class DAG(dag.DAG): """ This is a spread DAG. We assume that the comman-line argument do not have spaces! """ def execute(self, work_dir): err = 0 running = _enqueue(self.roots, work_dir) # Now wait. A node is done if and only if al its instances are done. total_time = 0.
def show(self, format='ascii'): """ Print out the plot to STDOUT, specifying the plot format. At the moment ASCII plots are the only ones which are supported. """ if (format.upper() != 'ASCII'): print('Warning: %s format not supported. Reverting to ASCII' \ % (format)) # Convert the datetime instances to UNIX timestamps. We assume that # self.data is a list of 2 element tuples [(x, y), (x, y), ...] and is # homogeneous, i.e. all x are of teh same type and all y are of the same # (usually different) type. xistime = False yistime = False convert_x = lambda x: x convert_y = lambda y: y if (isinstance(self.data[0][0], datetime)): convert_x = lambda x: time.mktime(x.timetuple()) xistime = True if (isinstance(self.data[0][1], datetime)): convert_y = lambda y: time.mktime(y.timetuple()) yistime = True # write the (converted) data to a temp file. (fid, file_name) = tempfile.mkstemp() os.close(fid) data_file = open(file_name, 'w') for (x, y) in self.data: data_file.write('%s,%s\n' % (str(convert_x(x)), str(convert_y(y)))) data_file.close() gnuplot = which('gnuplot') if (not gnuplot): gnuplot = '/Users/fpierfed/NoBalckup/gnuplot-4.6.0/src/gnuplot' proc = subprocess.Popen([ gnuplot, ], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write('set terminal dumb size 100,24\n') proc.stdin.write('set title "%s/%s"\n' % (self.yfield, self.xfield)) proc.stdin.write('set ylabel "%s"\n' % (self.yfield)) proc.stdin.write('set xlabel "%s"\n' % (self.xfield)) proc.stdin.write('set timefmt "%s"\n') if (xistime): proc.stdin.write('set xdata time\n') proc.stdin.write('set format x "%m/%y"\n') if (yistime): proc.stdin.write('set ydata time\n') proc.stdin.write('set format y "%m/%y"\n') proc.stdin.write('set datafile separator ","\n') proc.stdin.write('plot "%s" using 1:2 with lines title ""\n' \ % (file_name)) proc.stdin.write('quit\n') done = False while (not done): err = proc.poll() if (err is not None): done = True print(proc.stdout.read()) proc.stdout.close() proc.stdin.close() del (proc) os.remove(file_name) return
def show(self, format='ascii'): """ Print out the plot to STDOUT, specifying the plot format. At the moment ASCII plots are the only ones which are supported. """ if(format.upper() != 'ASCII'): print('Warning: %s format not supported. Reverting to ASCII' \ % (format)) # Convert the datetime instances to UNIX timestamps. We assume that # self.data is a list of 2 element tuples [(x, y), (x, y), ...] and is # homogeneous, i.e. all x are of teh same type and all y are of the same # (usually different) type. xistime = False yistime = False convert_x = lambda x: x convert_y = lambda y: y if(isinstance(self.data[0][0], datetime)): convert_x = lambda x: time.mktime(x.timetuple()) xistime = True if(isinstance(self.data[0][1], datetime)): convert_y = lambda y: time.mktime(y.timetuple()) yistime = True # write the (converted) data to a temp file. (fid, file_name) = tempfile.mkstemp() os.close(fid) data_file = open(file_name, 'w') for (x, y) in self.data: data_file.write('%s,%s\n' % (str(convert_x(x)), str(convert_y(y)))) data_file.close() gnuplot = which('gnuplot') if(not gnuplot): gnuplot = '/Users/fpierfed/NoBalckup/gnuplot-4.6.0/src/gnuplot' proc = subprocess.Popen([gnuplot, ], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write('set terminal dumb size 100,24\n') proc.stdin.write('set title "%s/%s"\n' % (self.yfield, self.xfield)) proc.stdin.write('set ylabel "%s"\n' % (self.yfield)) proc.stdin.write('set xlabel "%s"\n' % (self.xfield)) proc.stdin.write('set timefmt "%s"\n') if(xistime): proc.stdin.write('set xdata time\n') proc.stdin.write('set format x "%m/%y"\n') if(yistime): proc.stdin.write('set ydata time\n') proc.stdin.write('set format y "%m/%y"\n') proc.stdin.write('set datafile separator ","\n') proc.stdin.write('plot "%s" using 1:2 with lines title ""\n' \ % (file_name)) proc.stdin.write('quit\n') done = False while(not done): err = proc.poll() if(err is not None): done = True print(proc.stdout.read()) proc.stdout.close() proc.stdin.close() del(proc) os.remove(file_name) return