def runTest(NJOBS, NRUN, rootDir, output_dir, rep_type):
if DEBUG:
print 'from runTest: rootDir %s, output_dir %s'%(rootDir, output_dir)
if rep_type == "Remote":
repository = repositoryFactory(repositoryType = rep_type,
root_dir = rootDir,
streamer = SimpleJobStreamer(),
host = 'gangamd.cern.ch',
port = 8822,
login = os.getlogin(),
keepalive = True)
elif rep_type == "Local":
repository = repositoryFactory(repositoryType = rep_type,
root_dir = rootDir,
streamer = SimpleJobStreamer(),
local_root = '/tmp')
else:
print "Wrong type of repository..."
print "Exiting ..."
return
nn = tempfile.mktemp(suffix = '.test')
nn = os.path.join(output_dir, os.path.basename(nn))
ff = file(nn, 'w')
try:
jjj = []
for n in range(NRUN):
ff.write("NUMBER of jobs in the repository %d \n" %len(jjj))
jj = []
for i in range(NJOBS):
j = Job()
#j.application = Gaudi()
j.name = "MyJob" + str(i)
jj.append(j)
jjj.extend(jj)
t1 = _startText(ff, 'registering %d jobs...' % NJOBS)
repository.registerJobs(jj)
if DEBUG:
print "--->command status", "OK", "\n"
_endText(ff, t1)
t1 = _startText(ff, 'deleting jobs...')
repository.deleteJobs(map(lambda j: j.id, jjj))
if DEBUG:
print "--->command status", "OK", "\n"
_endText(ff, t1)
finally:
ff.close()
def bootstrap():
import os.path
global names, regs
names = ["jobs", "templates"]
docstrings = ["default job registry", "default registry of job templates"]
# debug
for c in config:
logger.debug("%s = %s", c, config[c])
from Ganga.Core.JobRepository.ARDA import repositoryFactory
from Ganga.Core.exceptions import RepositoryError
def print_error(x):
for c in config:
logger.error("%s = %s", c, config[c])
if config["repositorytype"] == "RemoteAMGA":
logger.error("check login name, host and port number")
s = "Cannot connect to the repository: " + str(x)
logger.error(s)
return s
reps = []
try:
for n in names:
reps.append(repositoryFactory(subpath=n))
except RepositoryError, x:
s = print_error(x)
raise
def runTest(NTEST, LEN, rootDir, output_dir, rep_type):
if DEBUG:
print 'from runTest: rootDir %s, output_dir %s'%(rootDir, output_dir)
if rep_type == "Remote":
repository = repositoryFactory(repositoryType = rep_type,
root_dir = rootDir,
streamer = SimpleJobStreamer(),
host = 'gangamd.cern.ch',
#host = 'lxgate41.cern.ch',
port = 8822,
login = os.getlogin(),
keepalive = True)
elif rep_type == "Local":
repository = repositoryFactory(repositoryType = rep_type,
root_dir = rootDir,
streamer = SimpleJobStreamer(),
#local_root = os.path.expanduser('~'),
local_root = os.path.join('/afs/cern.ch/sw/ganga/workdir',
os.getlogin(), 'gangadir/repository'))
else:
print "Wrong type of repository..."
print "Exiting ..."
return
nn = tempfile.mktemp(suffix = '.test')
nn = os.path.join(output_dir, os.path.basename(nn))
ff = file(nn, 'w')
try:
jj = []
for i in range(NTEST):
j = getSplitJob(LEN)
j.name = "MyJob" + str(i)
j.application.args = ['/' + ARG_LEN*'abcd']
jj.append(j)
#----------------------------------------------------
t1 = _startText(ff, 'registering %d jobs...' % NTEST)
if DEBUG:
print 'registering %d jobs...' % NTEST
try:
repository.registerJobs(jj)
except Exception, e:
print "EXCEPTION in registerJobs", str(e)
if DEBUG:
print "--->command status", "FAIL", "\n"
else:
def runTest(NTEST, rootDir, output_dir, rep_type):
if DEBUG:
print "from runTest: rootDir %s, output_dir %s" % (rootDir, output_dir)
if rep_type == "Remote":
repository = repositoryFactory(
repositoryType=rep_type,
root_dir=rootDir,
streamer=SimpleJobStreamer(),
host="lxgate41.cern.ch",
port=8822,
login=os.getlogin(),
keepalive=True,
)
elif rep_type == "Local":
repository = repositoryFactory(
repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), local_root=os.path.expanduser("~")
)
else:
print "Wrong type of repository..."
print "Exiting ..."
return
nn = tempfile.mktemp(suffix=".test")
nn = os.path.join(output_dir, os.path.basename(nn))
ff = file(nn, "w")
try:
jj = []
for i in range(NTEST):
j = Job()
# j.application = Gaudi()
j.name = "MyJob" + str(i)
j.application.args = 1000 * ["/abc"]
jj.append(j)
# ----------------------------------------------------
t1 = _startText(ff, "registering %d jobs..." % NTEST)
if DEBUG:
print "registering %d jobs..." % NTEST
try:
repository.registerJobs(jj)
except Exception, e:
print "EXCEPTION in registerJobs", str(e)
if DEBUG:
print "--->command status", "FAIL", "\n"
else:
def getOldJobs():
salvaged_jobs = {'jobs':[],'templates':[]}
basepath = os.path.join(expandfilename(config['gangadir']),'repository',config['user'])
names = ['jobs','templates']
path = os.path.join(basepath,"LocalAMGA")
if os.path.exists(path) and not os.path.exists(os.path.join(path,"converted.to.XML.6.0")):
from Ganga.Core.JobRepository.ARDA import repositoryFactory
for name in names:
try:
rep = repositoryFactory(subpath = name)
co_jobs = rep.checkoutJobs({})
salvaged_jobs[name].extend(co_jobs)
file(os.path.join(path,"converted.to.XML.6.0"),"w").close()
rep.releaseAllLocks()
if len(co_jobs) > 0:
logger.warning("Converted %i jobs from old AMGA repository" % len(co_jobs))
except Exception,x:
logger.error("Could not load old AMGA repository: %s" % x)
raise
def runTest(NTEST, rootDir, output_dir, rep_type):
logger.debug('from runTest: rootDir %s, output_dir %s' %
(rootDir, output_dir))
if rep_type == "Remote":
repository = repositoryFactory(repositoryType=rep_type,
root_dir=rootDir,
streamer=SimpleJobStreamer(),
host='lxgate41.cern.ch',
port=8822,
login=os.getlogin(),
keepalive=True)
elif rep_type == "Local":
repository = repositoryFactory(repositoryType=rep_type,
root_dir=rootDir,
streamer=SimpleJobStreamer(),
local_root=os.path.expanduser('~'))
else:
logger.error("Wrong type of repository...")
logger.error("Exiting ...")
return
nn = tempfile.mktemp(suffix='.test')
nn = os.path.join(output_dir, os.path.basename(nn))
ff = file(nn, 'w')
try:
jj = []
for i in range(NTEST):
j = Job()
#j.application = Gaudi()
j.name = "MyJob" + str(i)
j.application.args = 1000 * ['/abc']
jj.append(j)
#----------------------------------------------------
t1 = _startText(ff, 'registering %d jobs...' % NTEST)
logger.debug('registering %d jobs...' % NTEST)
try:
repository.registerJobs(jj)
except Exception as e:
logger.error("EXCEPTION in registerJobs " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command status OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'testing splitting of %d jobs...' % NTEST)
logger.debug('testing splitting of %d jobs...' % NTEST)
try:
for i in range(NTEST):
testSplitting(repository, LEN=10)
except Exception as e:
logger.error("EXCEPTION in testSplitting " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command status OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'retrieving info about first 10 jobs...')
logger.debug('retrieving info about first 10 jobs...')
try:
rjj = repository.checkoutJobs(map(lambda j: j.id, jj[:10]))
except Exception as e:
logger.error("EXCEPTION in checkoutJobs " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->checkout jobs " + map(lambda j: j.id, rjj))
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'retrieving info about ALL jobs')
logger.debug('retrieving info about ALL jobs')
try:
rjj = repository.checkoutJobs({})
except Exception as e:
logger.error("EXCEPTION in checkoutJobs " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug(
"--->checkout jobs " + len(rjj), map(lambda j: j.id, rjj))
_endText(ff, t1)
for j in jj:
j.application = Executable()
try:
j.updateStatus('submitting')
except:
pass
#----------------------------------------------------
t1 = _startText(ff, 'commiting %d jobs...' % NTEST)
logger.debug('commiting %d jobs...' % NTEST)
try:
repository.commitJobs(jj)
except Exception as e:
logger.error("EXCEPTION in commitJobs " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command status OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'setting status for %d jobs...' % NTEST)
logger.debug('setting status for %d jobs...' % NTEST)
try:
repository.setJobsStatus(map(lambda j: (j.id, 'submitted'), jj))
except Exception as e:
logger.error("EXCEPTION in setJobsStatus " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command status OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'getting status of first 10 jobs...')
logger.debug('getting status of first 10 jobs...')
try:
rjj = repository.getJobsStatus(map(lambda j: j.id, jj[:10]))
except Exception as e:
logger.error("EXCEPTION in getJobsStatus " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command output " + str(len(rjj)) + str(rjj))
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'getting id of jobs with particular attributes...')
logger.debug('getting id of jobs with particular attributes...')
try:
rjj = repository.getJobIds(
{'status': 'submitted', 'application': 'Executable'})
except Exception as e:
logger.error("EXCEPTION in getJobIds " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command output" + str(len(rjj)) + str(rjj))
_endText(ff, t1)
t1 = _startText(ff, 'retrieving info about ALL jobs')
rjj = repository.checkoutJobs({})
if DEBUG:
logger.debug('retrieving info about ALL jobs')
jj_id = map(lambda j: j.id, jj)
st_lst = []
for j in rjj:
if j.id in jj_id:
st_lst.append((j.id, j.status))
logger.error(
"--->command output " + str(len(st_lst)) + ' ' + str(st_lst))
_endText(ff, t1)
t1 = _startText(ff, 'deleting %d jobs...' % NTEST)
logger.debug('deleting %d jobs...' % NTEST)
try:
repository.deleteJobs(map(lambda j: j.id, jj))
except Exception as e:
logger.error("EXCEPTION in deleteJobs " + str(e))
logger.debug("--->command status FAIL")
else:
logger.debug("--->command status OK")
_endText(ff, t1)
finally:
ff.close()
def runTest(NTEST, LEN, rootDir, output_dir, rep_type):
logger.debug('from runTest: rootDir %s, output_dir %s' %
(rootDir, output_dir))
if rep_type == "Remote":
repository = repositoryFactory(repositoryType=rep_type,
root_dir=rootDir,
streamer=SimpleJobStreamer(),
host='gangamd.cern.ch',
#host = 'lxgate41.cern.ch',
port=8822,
login=os.getlogin(),
keepalive=True)
elif rep_type == "Local":
repository = repositoryFactory(repositoryType=rep_type,
root_dir=rootDir,
streamer=SimpleJobStreamer(),
#local_root = os.path.expanduser('~'),
local_root=os.path.join('/afs/cern.ch/sw/ganga/workdir',
os.getlogin(), 'gangadir/repository'))
else:
logger.error("Wrong type of repository...")
logger.error("Exiting ...")
return
nn = tempfile.mktemp(suffix='.test')
nn = os.path.join(output_dir, os.path.basename(nn))
ff = file(nn, 'w')
try:
jj = []
for i in range(NTEST):
j = getSplitJob(LEN)
j.name = "MyJob" + str(i)
j.application.args = ['/' + ARG_LEN * 'abcd']
jj.append(j)
#----------------------------------------------------
t1 = _startText(ff, 'registering %d jobs...' % NTEST)
logger.debug('registering %d jobs...' % NTEST)
try:
repository.registerJobs(jj)
except Exception as e:
logger.error("EXCEPTION in registerJobs " + str(e))
logger.debug("--->command status", "FAIL")
else:
logger.debug("--->command status", "OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'retrieving info about ALL jobs')
logger.debug('retrieving info about ALL jobs')
try:
#rjj = repository.checkoutJobs(map(lambda j: j.id, jj))
rjj = repository.checkoutJobs({})
except Exception as e:
logger.error("EXCEPTION in checkoutJobs " + str(e))
logger.debug("--->command status", "FAIL")
else:
logger.debug(
"--->checkout jobs", len(rjj), map(lambda j: j.id, rjj))
_endText(ff, t1)
# some job modification
for j in jj:
# j.application = Executable()
j.name = j.name + 'Changed'
for sj in j.subjobs:
sj.application.exe = '/bin/ls'
#----------------------------------------------------
t1 = _startText(ff, 'commiting %d jobs...' % NTEST)
logger.debug('commiting %d jobs...' % NTEST)
try:
repository.commitJobs(jj)
except Exception as e:
logger.error("EXCEPTION in commitJobs " + str(e))
logger.debug("--->command status", "FAIL")
else:
logger.debug("--->command status", "OK")
_endText(ff, t1)
#----------------------------------------------------
t1 = _startText(ff, 'deleting %d jobs...' % NTEST)
logger.debug('deleting %d jobs...' % NTEST)
try:
repository.deleteJobs(map(lambda j: j.id, jj))
except Exception as e:
logger.error("EXCEPTION in deleteJobs " + str(e))
logger.debug("--->command status", "FAIL")
else:
logger.debug("--->command status", "OK")
_endText(ff, t1)
finally:
ff.close()