def runTest(NJOBS, NRUN, rootDir, output_dir, rep_type): if DEBUG: print 'from runTest: rootDir %s, output_dir %s'%(rootDir, output_dir) if rep_type == "Remote": repository = repositoryFactory(repositoryType = rep_type, root_dir = rootDir, streamer = SimpleJobStreamer(), host = 'gangamd.cern.ch', port = 8822, login = os.getlogin(), keepalive = True) elif rep_type == "Local": repository = repositoryFactory(repositoryType = rep_type, root_dir = rootDir, streamer = SimpleJobStreamer(), local_root = '/tmp') else: print "Wrong type of repository..." print "Exiting ..." return nn = tempfile.mktemp(suffix = '.test') nn = os.path.join(output_dir, os.path.basename(nn)) ff = file(nn, 'w') try: jjj = [] for n in range(NRUN): ff.write("NUMBER of jobs in the repository %d \n" %len(jjj)) jj = [] for i in range(NJOBS): j = Job() #j.application = Gaudi() j.name = "MyJob" + str(i) jj.append(j) jjj.extend(jj) t1 = _startText(ff, 'registering %d jobs...' % NJOBS) repository.registerJobs(jj) if DEBUG: print "--->command status", "OK", "\n" _endText(ff, t1) t1 = _startText(ff, 'deleting jobs...') repository.deleteJobs(map(lambda j: j.id, jjj)) if DEBUG: print "--->command status", "OK", "\n" _endText(ff, t1) finally: ff.close()
def bootstrap(): import os.path global names, regs names = ["jobs", "templates"] docstrings = ["default job registry", "default registry of job templates"] # debug for c in config: logger.debug("%s = %s", c, config[c]) from Ganga.Core.JobRepository.ARDA import repositoryFactory from Ganga.Core.exceptions import RepositoryError def print_error(x): for c in config: logger.error("%s = %s", c, config[c]) if config["repositorytype"] == "RemoteAMGA": logger.error("check login name, host and port number") s = "Cannot connect to the repository: " + str(x) logger.error(s) return s reps = [] try: for n in names: reps.append(repositoryFactory(subpath=n)) except RepositoryError, x: s = print_error(x) raise
def runTest(NTEST, LEN, rootDir, output_dir, rep_type): if DEBUG: print 'from runTest: rootDir %s, output_dir %s'%(rootDir, output_dir) if rep_type == "Remote": repository = repositoryFactory(repositoryType = rep_type, root_dir = rootDir, streamer = SimpleJobStreamer(), host = 'gangamd.cern.ch', #host = 'lxgate41.cern.ch', port = 8822, login = os.getlogin(), keepalive = True) elif rep_type == "Local": repository = repositoryFactory(repositoryType = rep_type, root_dir = rootDir, streamer = SimpleJobStreamer(), #local_root = os.path.expanduser('~'), local_root = os.path.join('/afs/cern.ch/sw/ganga/workdir', os.getlogin(), 'gangadir/repository')) else: print "Wrong type of repository..." print "Exiting ..." return nn = tempfile.mktemp(suffix = '.test') nn = os.path.join(output_dir, os.path.basename(nn)) ff = file(nn, 'w') try: jj = [] for i in range(NTEST): j = getSplitJob(LEN) j.name = "MyJob" + str(i) j.application.args = ['/' + ARG_LEN*'abcd'] jj.append(j) #---------------------------------------------------- t1 = _startText(ff, 'registering %d jobs...' % NTEST) if DEBUG: print 'registering %d jobs...' % NTEST try: repository.registerJobs(jj) except Exception, e: print "EXCEPTION in registerJobs", str(e) if DEBUG: print "--->command status", "FAIL", "\n" else:
def runTest(NTEST, rootDir, output_dir, rep_type): if DEBUG: print "from runTest: rootDir %s, output_dir %s" % (rootDir, output_dir) if rep_type == "Remote": repository = repositoryFactory( repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), host="lxgate41.cern.ch", port=8822, login=os.getlogin(), keepalive=True, ) elif rep_type == "Local": repository = repositoryFactory( repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), local_root=os.path.expanduser("~") ) else: print "Wrong type of repository..." print "Exiting ..." return nn = tempfile.mktemp(suffix=".test") nn = os.path.join(output_dir, os.path.basename(nn)) ff = file(nn, "w") try: jj = [] for i in range(NTEST): j = Job() # j.application = Gaudi() j.name = "MyJob" + str(i) j.application.args = 1000 * ["/abc"] jj.append(j) # ---------------------------------------------------- t1 = _startText(ff, "registering %d jobs..." % NTEST) if DEBUG: print "registering %d jobs..." % NTEST try: repository.registerJobs(jj) except Exception, e: print "EXCEPTION in registerJobs", str(e) if DEBUG: print "--->command status", "FAIL", "\n" else:
def getOldJobs(): salvaged_jobs = {'jobs':[],'templates':[]} basepath = os.path.join(expandfilename(config['gangadir']),'repository',config['user']) names = ['jobs','templates'] path = os.path.join(basepath,"LocalAMGA") if os.path.exists(path) and not os.path.exists(os.path.join(path,"converted.to.XML.6.0")): from Ganga.Core.JobRepository.ARDA import repositoryFactory for name in names: try: rep = repositoryFactory(subpath = name) co_jobs = rep.checkoutJobs({}) salvaged_jobs[name].extend(co_jobs) file(os.path.join(path,"converted.to.XML.6.0"),"w").close() rep.releaseAllLocks() if len(co_jobs) > 0: logger.warning("Converted %i jobs from old AMGA repository" % len(co_jobs)) except Exception,x: logger.error("Could not load old AMGA repository: %s" % x) raise
def runTest(NTEST, rootDir, output_dir, rep_type): logger.debug('from runTest: rootDir %s, output_dir %s' % (rootDir, output_dir)) if rep_type == "Remote": repository = repositoryFactory(repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), host='lxgate41.cern.ch', port=8822, login=os.getlogin(), keepalive=True) elif rep_type == "Local": repository = repositoryFactory(repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), local_root=os.path.expanduser('~')) else: logger.error("Wrong type of repository...") logger.error("Exiting ...") return nn = tempfile.mktemp(suffix='.test') nn = os.path.join(output_dir, os.path.basename(nn)) ff = file(nn, 'w') try: jj = [] for i in range(NTEST): j = Job() #j.application = Gaudi() j.name = "MyJob" + str(i) j.application.args = 1000 * ['/abc'] jj.append(j) #---------------------------------------------------- t1 = _startText(ff, 'registering %d jobs...' % NTEST) logger.debug('registering %d jobs...' % NTEST) try: repository.registerJobs(jj) except Exception as e: logger.error("EXCEPTION in registerJobs " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command status OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'testing splitting of %d jobs...' % NTEST) logger.debug('testing splitting of %d jobs...' % NTEST) try: for i in range(NTEST): testSplitting(repository, LEN=10) except Exception as e: logger.error("EXCEPTION in testSplitting " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command status OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'retrieving info about first 10 jobs...') logger.debug('retrieving info about first 10 jobs...') try: rjj = repository.checkoutJobs(map(lambda j: j.id, jj[:10])) except Exception as e: logger.error("EXCEPTION in checkoutJobs " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->checkout jobs " + map(lambda j: j.id, rjj)) _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'retrieving info about ALL jobs') logger.debug('retrieving info about ALL jobs') try: rjj = repository.checkoutJobs({}) except Exception as e: logger.error("EXCEPTION in checkoutJobs " + str(e)) logger.debug("--->command status FAIL") else: logger.debug( "--->checkout jobs " + len(rjj), map(lambda j: j.id, rjj)) _endText(ff, t1) for j in jj: j.application = Executable() try: j.updateStatus('submitting') except: pass #---------------------------------------------------- t1 = _startText(ff, 'commiting %d jobs...' % NTEST) logger.debug('commiting %d jobs...' % NTEST) try: repository.commitJobs(jj) except Exception as e: logger.error("EXCEPTION in commitJobs " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command status OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'setting status for %d jobs...' % NTEST) logger.debug('setting status for %d jobs...' % NTEST) try: repository.setJobsStatus(map(lambda j: (j.id, 'submitted'), jj)) except Exception as e: logger.error("EXCEPTION in setJobsStatus " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command status OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'getting status of first 10 jobs...') logger.debug('getting status of first 10 jobs...') try: rjj = repository.getJobsStatus(map(lambda j: j.id, jj[:10])) except Exception as e: logger.error("EXCEPTION in getJobsStatus " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command output " + str(len(rjj)) + str(rjj)) _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'getting id of jobs with particular attributes...') logger.debug('getting id of jobs with particular attributes...') try: rjj = repository.getJobIds( {'status': 'submitted', 'application': 'Executable'}) except Exception as e: logger.error("EXCEPTION in getJobIds " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command output" + str(len(rjj)) + str(rjj)) _endText(ff, t1) t1 = _startText(ff, 'retrieving info about ALL jobs') rjj = repository.checkoutJobs({}) if DEBUG: logger.debug('retrieving info about ALL jobs') jj_id = map(lambda j: j.id, jj) st_lst = [] for j in rjj: if j.id in jj_id: st_lst.append((j.id, j.status)) logger.error( "--->command output " + str(len(st_lst)) + ' ' + str(st_lst)) _endText(ff, t1) t1 = _startText(ff, 'deleting %d jobs...' % NTEST) logger.debug('deleting %d jobs...' % NTEST) try: repository.deleteJobs(map(lambda j: j.id, jj)) except Exception as e: logger.error("EXCEPTION in deleteJobs " + str(e)) logger.debug("--->command status FAIL") else: logger.debug("--->command status OK") _endText(ff, t1) finally: ff.close()
def runTest(NTEST, LEN, rootDir, output_dir, rep_type): logger.debug('from runTest: rootDir %s, output_dir %s' % (rootDir, output_dir)) if rep_type == "Remote": repository = repositoryFactory(repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), host='gangamd.cern.ch', #host = 'lxgate41.cern.ch', port=8822, login=os.getlogin(), keepalive=True) elif rep_type == "Local": repository = repositoryFactory(repositoryType=rep_type, root_dir=rootDir, streamer=SimpleJobStreamer(), #local_root = os.path.expanduser('~'), local_root=os.path.join('/afs/cern.ch/sw/ganga/workdir', os.getlogin(), 'gangadir/repository')) else: logger.error("Wrong type of repository...") logger.error("Exiting ...") return nn = tempfile.mktemp(suffix='.test') nn = os.path.join(output_dir, os.path.basename(nn)) ff = file(nn, 'w') try: jj = [] for i in range(NTEST): j = getSplitJob(LEN) j.name = "MyJob" + str(i) j.application.args = ['/' + ARG_LEN * 'abcd'] jj.append(j) #---------------------------------------------------- t1 = _startText(ff, 'registering %d jobs...' % NTEST) logger.debug('registering %d jobs...' % NTEST) try: repository.registerJobs(jj) except Exception as e: logger.error("EXCEPTION in registerJobs " + str(e)) logger.debug("--->command status", "FAIL") else: logger.debug("--->command status", "OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'retrieving info about ALL jobs') logger.debug('retrieving info about ALL jobs') try: #rjj = repository.checkoutJobs(map(lambda j: j.id, jj)) rjj = repository.checkoutJobs({}) except Exception as e: logger.error("EXCEPTION in checkoutJobs " + str(e)) logger.debug("--->command status", "FAIL") else: logger.debug( "--->checkout jobs", len(rjj), map(lambda j: j.id, rjj)) _endText(ff, t1) # some job modification for j in jj: # j.application = Executable() j.name = j.name + 'Changed' for sj in j.subjobs: sj.application.exe = '/bin/ls' #---------------------------------------------------- t1 = _startText(ff, 'commiting %d jobs...' % NTEST) logger.debug('commiting %d jobs...' % NTEST) try: repository.commitJobs(jj) except Exception as e: logger.error("EXCEPTION in commitJobs " + str(e)) logger.debug("--->command status", "FAIL") else: logger.debug("--->command status", "OK") _endText(ff, t1) #---------------------------------------------------- t1 = _startText(ff, 'deleting %d jobs...' % NTEST) logger.debug('deleting %d jobs...' % NTEST) try: repository.deleteJobs(map(lambda j: j.id, jj)) except Exception as e: logger.error("EXCEPTION in deleteJobs " + str(e)) logger.debug("--->command status", "FAIL") else: logger.debug("--->command status", "OK") _endText(ff, t1) finally: ff.close()