def storeJob(jobDoc, bucket, first_pass = True):
client = McdClient(HOST, PORT)
client.sasl_auth_plain(bucket, "")
doc = jobDoc
url = doc["url"]
res = getJS(url, {"depth" : 0}).json()
if res is None:
return
buildHist = {}
if res["lastBuild"]:
bids = [b["number"] for b in res["builds"]]
lastTotalCount = -1
idx=0
for bid in bids:
idx = idx + 1
i = 1
if idx < len(bids):
while bids[idx] != bid-i:
key = "%s-%s" % (doc["name"], bid-i)
key = hashlib.md5(key).hexdigest()
try:
client.delete(key, vbucket=0)
except:
pass
i = i + 1
if bid in JOBS[doc["name"]]:
continue # job already stored
else:
if first_pass == False:
JOBS[doc["name"]].append(bid)
doc["build_id"] = bid
res = getJS(url+str(bid), {"depth" : 0}).json()
if res is None:
return
if "result" not in res:
continue
doc["result"] = res["result"]
doc["duration"] = res["duration"]
if bucket == "server":
if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]:
continue # unknown result state
actions = res["actions"]
totalCount = getAction(actions, "totalCount") or 0
failCount = getAction(actions, "failCount") or 0
skipCount = getAction(actions, "skipCount") or 0
if totalCount == 0:
if lastTotalCount == -1:
continue # no tests ever passed for this build
else:
totalCount = lastTotalCount
failCount = totalCount
else:
lastTotalCount = totalCount
doc["failCount"] = failCount
doc["totalCount"] = totalCount - skipCount
params = getAction(actions, "parameters")
if params is None:
doc["priority"] = P1
doc["build"] = DEFAULT_BUILD
else:
doc["build"] = getAction(params, "name", "version_number") or getAction(params, "name", "cluster_version") or DEFAULT_BUILD
doc["priority"] = getAction(params, "name", "priority") or P1
if doc["priority"].upper() not in [P0, P1, P2]:
doc["priority"] = P1
doc["build"] = doc["build"].replace("-rel","").split(",")[0]
try:
_build= doc["build"].split("-")
rel, bno = _build[0], _build[1]
# check partial rel #'s
rlen = len(rel.split("."))
while rlen < 3:
rel = rel+".0"
rlen+=1
# verify rel, build
m=re.match("^\d\.\d\.\d{1,5}", rel)
if m is None:
print "unsupported version_number: "+doc["build"]
continue
m=re.match("^\d{1,10}", bno)
if m is None:
print "unsupported version_number: "+doc["build"]
continue
doc["build"] = "%s-%s" % (rel, bno.zfill(4))
except:
print "unsupported version_number: "+doc["build"]
continue
else:
# use date as version for sdk and mobile
if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]:
continue
actions = res["actions"]
totalCount = getAction(actions, "totalCount") or 0
failCount = getAction(actions, "failCount") or 0
skipCount = getAction(actions, "skipCount") or 0
if totalCount == 0:
if lastTotalCount == -1:
continue # no tests ever passed for this build
else:
totalCount = lastTotalCount
failCount = totalCount
else:
lastTotalCount = totalCount
doc["failCount"] = failCount
doc["totalCount"] = totalCount - skipCount
doc["priority"] = P0
ts = res["timestamp"]/1000;
month = int(datetime.datetime.fromtimestamp(ts).strftime("%m"))
_ts = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m%d")
yr, md = _ts.split("-")
doc["build"] = "%s-%s%s" % (MOBILE_VERSION, yr[-1], md)
if doc["build"] in buildHist:
#print "REJECTED- doc already in build results: %s" % doc
#print buildHist
# attempt to delete if this record has been stored in couchbase
try:
oldKey = "%s-%s" % (doc["name"], doc["build_id"])
oldKey = hashlib.md5(oldKey).hexdigest()
client.delete(oldKey, vbucket = 0)
#print "DELETED- %s:%s" % (doc["build"],doc["build_id"])
except:
pass
continue # already have this build results
key = "%s-%s" % (doc["name"], doc["build_id"])
key = hashlib.md5(key).hexdigest()
val = json.dumps(doc)
try:
#print val
client.set(key, 0, 0, val, 0)
buildHist[doc["build"]] = doc["build_id"]
except:
print "set failed, couchbase down?: %s:%s" % (HOST,PORT)
if first_pass:
storeJob(jobDoc, bucket, first_pass = False)
mc = MemcachedClient("127.0.0.1", 11211)
keys = ["{0}-{1}".format(prefix, i) for i in range(0, number_of_items)]
info("inserting {0} items".format(number_of_items))
for k in keys:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
while True:
info(
"now remove 3 chars from 80% of keys - if < 3 chars delete the key - if key does not exist create it"
)
for i in range(0, 3):
for k in keys:
try:
a, b, value = mc.get(k)
if len(value) < 3:
mc.delete(k)
else:
mc.set(k, 0, 0, value[0:len(value) - 7])
except:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
time.sleep(sleep_time)
for k in keys:
try:
mc.prepend(k, "two")
except:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
time.sleep(sleep_time)
number_of_items = int(options.items)
mc = MemcachedClient("127.0.0.1", 11211)
keys = ["{0}-{1}".format(prefix, i) for i in range(0, number_of_items)]
info("inserting {0} items".format(number_of_items))
for k in keys:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
while True:
info("now remove 3 chars from 80% of keys - if < 3 chars delete the key - if key does not exist create it")
for i in range(0, 3):
for k in keys:
try:
a, b, value = mc.get(k)
if len(value) < 3:
mc.delete(k)
else:
mc.set(k, 0, 0, value[0:len(value) - 7])
except:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
time.sleep(sleep_time)
for k in keys:
try:
mc.prepend(k, "two")
except:
mc.set(k, 0, 0, str(uuid.uuid4())[0:16])
time.sleep(sleep_time)
def storeJob(jobDoc, bucket, first_pass=True):
client = McdClient(HOST, PORT)
client.sasl_auth_plain(bucket, "")
doc = jobDoc
url = doc["url"]
res = getJS(url, {"depth": 0}).json()
if res is None:
return
buildHist = {}
if res["lastBuild"]:
bids = [b["number"] for b in res["builds"]]
lastTotalCount = -1
idx = 0
for bid in bids:
idx = idx + 1
i = 1
if idx < len(bids):
while bids[idx] != bid - i:
key = "%s-%s" % (doc["name"], bid - i)
key = hashlib.md5(key).hexdigest()
try:
client.delete(key, vbucket=0)
except:
pass
i = i + 1
if bid in JOBS[doc["name"]]:
continue # job already stored
else:
if first_pass == False:
JOBS[doc["name"]].append(bid)
doc["build_id"] = bid
res = getJS(url + str(bid), {"depth": 0}).json()
if res is None:
return
if "result" not in res:
continue
doc["result"] = res["result"]
doc["duration"] = res["duration"]
if bucket == "server":
if res["result"] not in [
"SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"
]:
continue # unknown result state
actions = res["actions"]
totalCount = getAction(actions, "totalCount") or 0
failCount = getAction(actions, "failCount") or 0
skipCount = getAction(actions, "skipCount") or 0
if totalCount == 0:
if lastTotalCount == -1:
continue # no tests ever passed for this build
else:
totalCount = lastTotalCount
failCount = totalCount
else:
lastTotalCount = totalCount
doc["failCount"] = failCount
doc["totalCount"] = totalCount - skipCount
params = getAction(actions, "parameters")
if params is None:
doc["priority"] = P1
doc["build"] = DEFAULT_BUILD
else:
doc["build"] = getAction(
params, "name", "version_number") or getAction(
params, "name", "cluster_version") or DEFAULT_BUILD
doc["priority"] = getAction(params, "name",
"priority") or P1
if doc["priority"].upper() not in [P0, P1, P2]:
doc["priority"] = P1
doc["build"] = doc["build"].replace("-rel", "").split(",")[0]
try:
_build = doc["build"].split("-")
rel, bno = _build[0], _build[1]
# check partial rel #'s
rlen = len(rel.split("."))
while rlen < 3:
rel = rel + ".0"
rlen += 1
# verify rel, build
m = re.match("^\d\.\d\.\d{1,5}", rel)
if m is None:
print "unsupported version_number: " + doc["build"]
continue
m = re.match("^\d{1,10}", bno)
if m is None:
print "unsupported version_number: " + doc["build"]
continue
doc["build"] = "%s-%s" % (rel, bno.zfill(4))
except:
print "unsupported version_number: " + doc["build"]
continue
else:
# use date as version for sdk and mobile
if res["result"] not in [
"SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"
]:
continue
actions = res["actions"]
totalCount = getAction(actions, "totalCount") or 0
failCount = getAction(actions, "failCount") or 0
skipCount = getAction(actions, "skipCount") or 0
if totalCount == 0:
if lastTotalCount == -1:
continue # no tests ever passed for this build
else:
totalCount = lastTotalCount
failCount = totalCount
else:
lastTotalCount = totalCount
doc["failCount"] = failCount
doc["totalCount"] = totalCount - skipCount
doc["priority"] = P0
ts = res["timestamp"] / 1000
month = int(datetime.datetime.fromtimestamp(ts).strftime("%m"))
_ts = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m%d")
yr, md = _ts.split("-")
doc["build"] = "%s-%s%s" % (MOBILE_VERSION, yr[-1], md)
if doc["build"] in buildHist:
#print "REJECTED- doc already in build results: %s" % doc
#print buildHist
# attempt to delete if this record has been stored in couchbase
try:
oldKey = "%s-%s" % (doc["name"], doc["build_id"])
oldKey = hashlib.md5(oldKey).hexdigest()
client.delete(oldKey, vbucket=0)
#print "DELETED- %s:%s" % (doc["build"],doc["build_id"])
except:
pass
continue # already have this build results
key = "%s-%s" % (doc["name"], doc["build_id"])
key = hashlib.md5(key).hexdigest()
val = json.dumps(doc)
try:
#print val
client.set(key, 0, 0, val, 0)
buildHist[doc["build"]] = doc["build_id"]
except:
print "set failed, couchbase down?: %s:%s" % (HOST, PORT)
if first_pass:
storeJob(jobDoc, bucket, first_pass=False)
