def _icopy_files(root_names, dst_dir):
"""
For each of the root file names `root_names` copy <file name>.fits from
iRODS to `dst_dir`. Return 0 is all the copy operations succeeded, n where n
is the number of files that failed otherwise.
"""
# FIXME: make this more generic by fetching the iRODS env from a config file
err = 0
URL = "irods://dadsops.hlspZone:[email protected]:1247"
URL += "/hlspZone/home/rods/dads/pipeline/inputs/"
cmd = which("irods.py")
if not dst_dir.endswith("/"):
dst_dir += "/"
# The iRODS directory structure for a POD file called
# lz_bdd9_143_0001748545.pod
# is
# /hlspZone/home/rods/dads/pipeline/inputs/lz_bdd9/143_0001/
# and the actual POD file is stored there.
for root_name in root_names:
root_name = root_name.lower()
tokens = root_name.split("_")
full_url = os.path.join(
URL, "%s_%s" % (tokens[0], tokens[1]), "%s_%s" % (tokens[2], tokens[3][:4]), "%s.pod" % root_name
)
proc = subprocess.Popen([cmd, full_url, dst_dir], shell=False)
_error = proc.wait()
if _error:
err += 1
return err
def process_index(request, instrument, mode, dataset, exposure,
repo_root=REPO_ROOT, exe=EXE):
"""
Display available dataset exposures for the given instrument and mode.
"""
# Load the template.
t = loader.get_template('control/process.html')
# Special handling for modeD which uses iRODS. This is a hack :-(
if(str(mode) == 'modeD'):
repo_root = repo_root.replace('/jwdmsdevvm1/data1/', '', 1)
exe = which('process_idataset.py')
name = str(instrument) + '/' + str(mode)
# Find the instrument.
instr_dict = {'name': '%s not found' % (name),
'datasets': ()}
for i in INSTRUMENTS:
if(i['name'] == name):
instr_dict = i
break
# Get the repository path.
repo_path = os.path.join(repo_root, dataset)
# Compose the command.
args = (exe, '-r', repo_path, '-i', instrument, '-m', mode, exposure)
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
err = proc.wait()
# Render the template and exit.
cmd_str = ' '.join(args)
c = Context({'command': cmd_str,
'exit_code': err,
'instrument': instr_dict,
'dataset': dataset,
'exposure': exposure})
return(HttpResponse(t.render(c)))
def create_job_template(drmaa_session, dag_name, work_dir):
"""
Create a DRMAA Condor DAGMan job given the already instantiated OWL Workflow
templates for `dag_name` found in `work_dir`. Return the DRMAA job as a
DRMAA template (confusing terminology, I know).
"""
dagman = which('condor_dagman')
if(not dagman):
raise(Exception('Unable to find condor_dagman in $PATH'))
drmaa_tmplt = drmaa_session.createJobTemplate()
drmaa_tmplt.remoteCommand = dagman
drmaa_tmplt.workingDirectory = work_dir
drmaa_tmplt.nativeSpecification = NATIVE_SPEC_TMPL.render(dag_name=dag_name)
drmaa_tmplt.blockEmail = True
drmaa_tmplt.jobName = dag_name
drmaa_tmplt.outputPath = ':' + os.path.join(work_dir, dag_name + '.lib.out')
drmaa_tmplt.errorPath = ':' + os.path.join(work_dir, dag_name + '.lib.err')
drmaa_tmplt.joinFiles = False
return(drmaa_tmplt)
def create_job_template(drmaa_session, dag_name, work_dir):
"""
Create a DRMAA Condor DAGMan job given the already instantiated OWL Workflow
templates for `dag_name` found in `work_dir`. Return the DRMAA job as a
DRMAA template (confusing terminology, I know).
"""
dagman = which('condor_dagman')
if (not dagman):
raise (Exception('Unable to find condor_dagman in $PATH'))
drmaa_tmplt = drmaa_session.createJobTemplate()
drmaa_tmplt.remoteCommand = dagman
drmaa_tmplt.workingDirectory = work_dir
drmaa_tmplt.nativeSpecification = NATIVE_SPEC_TMPL.render(
dag_name=dag_name)
drmaa_tmplt.blockEmail = True
drmaa_tmplt.jobName = dag_name
drmaa_tmplt.outputPath = ':' + os.path.join(work_dir,
dag_name + '.lib.out')
drmaa_tmplt.errorPath = ':' + os.path.join(work_dir, dag_name + '.lib.err')
drmaa_tmplt.joinFiles = False
return (drmaa_tmplt)
import os
import subprocess
from django.template import Context, loader
from django.http import HttpResponse
from owl.utils import which
# Constants
# TODO: We should be getting these from, say a DB.
REPO_ROOT = '/data2/jwst/repository/raw'
EXE = which('process_dataset.py')
dataset001 = {'name': 'dataset_001',
'exposures': ('raw-000001', 'raw-000002', 'raw-000003', 'raw-000004')}
dataset002 = {'name': 'dataset_002',
'exposures': ('raw-000001', 'raw-000002')}
INSTRUMENTS = ({'name': 'instrument1/modeA',
'link_name': 'Regular BCW',
'datasets': (dataset001, dataset002)},
{'name': 'instrument1/modeD',
'link_name': 'iRODS BCW',
'datasets': (dataset001, dataset002)}, )
import os
import shutil
import socket
import time
from owl import dag
from owl import blackboard
from owl.utils import which
from spread import client
from spread_keyclient import get_clusterid
# Constants
# TODO: infer these fron the condor/owl config
JOB_HOOK = os.path.join(os.path.dirname(which('owld.py')), 'owl_job_hook.py')
HOSTNAME = socket.gethostname()
class DAG(dag.DAG):
"""
This is a spread DAG. We assume that the comman-line argument do not have
spaces!
"""
def execute(self, work_dir):
err = 0
running = _enqueue(self.roots, work_dir)
# Now wait. A node is done if and only if al its instances are done.
total_time = 0.
def show(self, format='ascii'):
"""
Print out the plot to STDOUT, specifying the plot format. At the moment
ASCII plots are the only ones which are supported.
"""
if (format.upper() != 'ASCII'):
print('Warning: %s format not supported. Reverting to ASCII' \
% (format))
# Convert the datetime instances to UNIX timestamps. We assume that
# self.data is a list of 2 element tuples [(x, y), (x, y), ...] and is
# homogeneous, i.e. all x are of teh same type and all y are of the same
# (usually different) type.
xistime = False
yistime = False
convert_x = lambda x: x
convert_y = lambda y: y
if (isinstance(self.data[0][0], datetime)):
convert_x = lambda x: time.mktime(x.timetuple())
xistime = True
if (isinstance(self.data[0][1], datetime)):
convert_y = lambda y: time.mktime(y.timetuple())
yistime = True
# write the (converted) data to a temp file.
(fid, file_name) = tempfile.mkstemp()
os.close(fid)
data_file = open(file_name, 'w')
for (x, y) in self.data:
data_file.write('%s,%s\n' % (str(convert_x(x)), str(convert_y(y))))
data_file.close()
gnuplot = which('gnuplot')
if (not gnuplot):
gnuplot = '/Users/fpierfed/NoBalckup/gnuplot-4.6.0/src/gnuplot'
proc = subprocess.Popen([
gnuplot,
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
proc.stdin.write('set terminal dumb size 100,24\n')
proc.stdin.write('set title "%s/%s"\n' % (self.yfield, self.xfield))
proc.stdin.write('set ylabel "%s"\n' % (self.yfield))
proc.stdin.write('set xlabel "%s"\n' % (self.xfield))
proc.stdin.write('set timefmt "%s"\n')
if (xistime):
proc.stdin.write('set xdata time\n')
proc.stdin.write('set format x "%m/%y"\n')
if (yistime):
proc.stdin.write('set ydata time\n')
proc.stdin.write('set format y "%m/%y"\n')
proc.stdin.write('set datafile separator ","\n')
proc.stdin.write('plot "%s" using 1:2 with lines title ""\n' \
% (file_name))
proc.stdin.write('quit\n')
done = False
while (not done):
err = proc.poll()
if (err is not None):
done = True
print(proc.stdout.read())
proc.stdout.close()
proc.stdin.close()
del (proc)
os.remove(file_name)
return
def show(self, format='ascii'):
"""
Print out the plot to STDOUT, specifying the plot format. At the moment
ASCII plots are the only ones which are supported.
"""
if(format.upper() != 'ASCII'):
print('Warning: %s format not supported. Reverting to ASCII' \
% (format))
# Convert the datetime instances to UNIX timestamps. We assume that
# self.data is a list of 2 element tuples [(x, y), (x, y), ...] and is
# homogeneous, i.e. all x are of teh same type and all y are of the same
# (usually different) type.
xistime = False
yistime = False
convert_x = lambda x: x
convert_y = lambda y: y
if(isinstance(self.data[0][0], datetime)):
convert_x = lambda x: time.mktime(x.timetuple())
xistime = True
if(isinstance(self.data[0][1], datetime)):
convert_y = lambda y: time.mktime(y.timetuple())
yistime = True
# write the (converted) data to a temp file.
(fid, file_name) = tempfile.mkstemp()
os.close(fid)
data_file = open(file_name, 'w')
for (x, y) in self.data:
data_file.write('%s,%s\n' % (str(convert_x(x)),
str(convert_y(y))))
data_file.close()
gnuplot = which('gnuplot')
if(not gnuplot):
gnuplot = '/Users/fpierfed/NoBalckup/gnuplot-4.6.0/src/gnuplot'
proc = subprocess.Popen([gnuplot, ],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
proc.stdin.write('set terminal dumb size 100,24\n')
proc.stdin.write('set title "%s/%s"\n' % (self.yfield, self.xfield))
proc.stdin.write('set ylabel "%s"\n' % (self.yfield))
proc.stdin.write('set xlabel "%s"\n' % (self.xfield))
proc.stdin.write('set timefmt "%s"\n')
if(xistime):
proc.stdin.write('set xdata time\n')
proc.stdin.write('set format x "%m/%y"\n')
if(yistime):
proc.stdin.write('set ydata time\n')
proc.stdin.write('set format y "%m/%y"\n')
proc.stdin.write('set datafile separator ","\n')
proc.stdin.write('plot "%s" using 1:2 with lines title ""\n' \
% (file_name))
proc.stdin.write('quit\n')
done = False
while(not done):
err = proc.poll()
if(err is not None):
done = True
print(proc.stdout.read())
proc.stdout.close()
proc.stdin.close()
del(proc)
os.remove(file_name)
return
