def fill_qms(template, queue_name, count, num_events=EV_PER_JD, start_id=10): assert os.path.exists(template) with open(template) as fh: t = json.load(fh) if num_events > 0: t['args']['--num-events'] = num_events t0 = datetime.datetime.now() queue = QueueMS(queue_name, api_url=API_URL) count0 = queue.qsize() for i in range(count): jd = copy(t) jd["job_id"] = start_id + i queue.put(jd) t1 = datetime.datetime.now() print "Time taken:", t1 - t0
def unlock_qms(lockfile, name): assert os.path.exists(lockfile) jds = _lock_ids(lockfile) assert len(jds) > 0 queue_orig = QueueMS(name, api_url=API_URL) queue_succ = QueueMS(name + ".success", api_url=API_URL) host_id = int(re.match("[^\d]*(\d*).*", lockfile).groups()[0]) for job_id, jd in jds.iteritems(): if _has_output("mc%02d" % host_id, jd): queue_succ.put(jd) logger.info("%d -> success" % job_id) else: queue_orig.put(jd) logger.info("%d -> orig" % job_id) os.remove(lockfile)
def test_queue_MS(api_url="http://mc03.h.cern.yandex.net:5000"): """ deprecated, use test in libscheduler """ from libscheduler.queue import QueueMS # from libscheduler.common import ms_delete dirs = ["_d1", "_d2"] q1 = QueueMS(dirs[0], api_url=api_url) print "size:", q1.qsize() # ms_delete(q1.queue_url) # q1.clear() assert q1.empty(), "not empty" # q1.qsize() item = q1.get() assert item is None assert q1.qsize() == 0 assert q1.empty() i1 = {"key": 1} q1.put(i1) assert q1.qsize() == 1 o1 = q1.get() print o1.status, o1.description assert i1 == o1.description, "i1: %s, o1: %s" % (i1, o1.description) assert q1.qsize() == 0 q2 = QueueMS(dirs[1], api_url=api_url) assert q2.empty() print ">>>", i1 q2.put(i1) q2.put({"key": 2}) assert q2.qsize() == 2 l = [] for i in q2: logger.info("iter: %s" % i) l.append(i.description) assert q2.empty() q1.extend(l) assert q1.qsize() == 2 i1 = q1.get() q1.put(i1.description) assert q1.qsize() == 2, q1.qsize()
def f(x): q = QueueMS('test_queue', api_url="http://test02cern.vs.os.yandex.net:5000/") for i in xrange(1000): q.put({"value": i}) if i % 100 == 0: print "%d: written %d!" % (x, i)