def fill_qms(template, queue_name, count, num_events=EV_PER_JD, start_id=10):
assert os.path.exists(template)
with open(template) as fh:
t = json.load(fh)
if num_events > 0:
t['args']['--num-events'] = num_events
t0 = datetime.datetime.now()
queue = QueueMS(queue_name, api_url=API_URL)
count0 = queue.qsize()
for i in range(count):
jd = copy(t)
jd["job_id"] = start_id + i
queue.put(jd)
t1 = datetime.datetime.now()
print "Time taken:", t1 - t0
def unlock_qms(lockfile, name):
assert os.path.exists(lockfile)
jds = _lock_ids(lockfile)
assert len(jds) > 0
queue_orig = QueueMS(name, api_url=API_URL)
queue_succ = QueueMS(name + ".success", api_url=API_URL)
host_id = int(re.match("[^\d]*(\d*).*", lockfile).groups()[0])
for job_id, jd in jds.iteritems():
if _has_output("mc%02d" % host_id, jd):
queue_succ.put(jd)
logger.info("%d -> success" % job_id)
else:
queue_orig.put(jd)
logger.info("%d -> orig" % job_id)
os.remove(lockfile)
def dump_qms(queue_name):
queue = QueueMS(queue_name, api_url=API_URL)
print queue_name, API_URL, queue.qsize()
def test_queue_MS(api_url="http://mc03.h.cern.yandex.net:5000"):
""" deprecated, use test in libscheduler """
from libscheduler.queue import QueueMS
# from libscheduler.common import ms_delete
dirs = ["_d1", "_d2"]
q1 = QueueMS(dirs[0], api_url=api_url)
print "size:", q1.qsize()
# ms_delete(q1.queue_url)
# q1.clear()
assert q1.empty(), "not empty" # q1.qsize()
item = q1.get()
assert item is None
assert q1.qsize() == 0
assert q1.empty()
i1 = {"key": 1}
q1.put(i1)
assert q1.qsize() == 1
o1 = q1.get()
print o1.status, o1.description
assert i1 == o1.description, "i1: %s, o1: %s" % (i1, o1.description)
assert q1.qsize() == 0
q2 = QueueMS(dirs[1], api_url=api_url)
assert q2.empty()
print ">>>", i1
q2.put(i1)
q2.put({"key": 2})
assert q2.qsize() == 2
l = []
for i in q2:
logger.info("iter: %s" % i)
l.append(i.description)
assert q2.empty()
q1.extend(l)
assert q1.qsize() == 2
i1 = q1.get()
q1.put(i1.description)
assert q1.qsize() == 2, q1.qsize()
def f(x):
q = QueueMS('test_queue', api_url="http://test02cern.vs.os.yandex.net:5000/")
for i in xrange(1000):
q.put({"value": i})
if i % 100 == 0:
print "%d: written %d!" % (x, i)
def queue_size(queue_name):
queue = QueueMS(queue_name, api_url=API_URL)
return queue.qsize()
