def get_total_fail_count(document):
totalCount = 0
failCount = 0
for OS, os in document['os'].items():
for COMPONENT, component in os.items():
for JOBNAME, jobName in component.items():
build = pydash.find(jobName, {"olderBuild": False})
if build:
totalCount += build['totalCount']
failCount += build['failCount']
document['totalCount'] = totalCount
document['failCount'] = failCount
def sanitize():
client = CLIENTS['builds']
query = "select meta().id from `builds` where `build` is not null"
for row in client.n1ql_query(N1QLQuery(query)):
build_id = row['id']
document = client.get(build_id).value
for OS, os in document['os'].items():
for COMPONENT, component in os.items():
for JOBNAME, jobName in component.items():
pydash.sort(jobName,
key=lambda item: item['build_id'],
reverse=True)
for build in jobName[1:]:
build['olderBuild'] = True
get_total_fail_count(document)
client.upsert(build_id, document)
def print_statistics(name, settings):
global TOTAL_COMPILER
global TOTAL_ARCH
global TOTAL_OS
global TOTAL_DOWNLOADS
arch = defaultdict(int)
compiler = defaultdict(int)
os = defaultdict(int)
total = 0
for data in settings:
for value in data.values():
# in case of installer package
arch_key = "arch_build" if "arch_build" in value else "arch"
os_key = "os_build" if "os_build" in value else "os"
downloads = value["downloads"]
total += downloads
TOTAL_DOWNLOADS += downloads
# header-only
if "arch" not in value and \
"compiler" not in value and \
"arch_build" not in value and \
"os" not in value and \
"os_build" not in value:
continue
if arch_key in value:
arch[value[arch_key]] += downloads
TOTAL_ARCH[value[arch_key]] += downloads
if os_key in value:
os[value[os_key]] += downloads
TOTAL_OS[value[os_key]] += downloads
if "compiler" in value:
compiler_name = "{} {}".format(value["compiler"],
value["compiler.version"])
compiler[compiler_name] += downloads
TOTAL_COMPILER[compiler_name] += downloads
print("===== %s =====" % name.upper())
generic_list = []
for key, value in arch.items():
generic_list.append([key, value])
if generic_list:
print(tabulate(generic_list, ["Arch", "Downloads"], tablefmt="grid"))
generic_list = []
for key, value in compiler.items():
generic_list.append([key, value])
if generic_list:
print(
tabulate(generic_list, ["Compiler", "Downloads"], tablefmt="grid"))
generic_list = []
for key, value in os.items():
generic_list.append([key, value])
if generic_list:
print(tabulate(generic_list, ["OS", "Downloads"], tablefmt="grid"))
print("TOTAL: {}".format(total))
def flush_crp_to_cmdb():
Log.logger.info(
'----------------flush_crp_to_cmdb job/5min----------------')
osid_status = []
now = datetime.datetime.now()
with db.app.app_context():
try:
envs = CRP_URL.items()
env_list = []
url_list = []
for env, url in envs:
if url not in url_list:
env_list.append(env)
url_list.append(url)
for env in env_list:
if not env:
continue
try:
K8sInfos = ConfigureK8sModel.objects.filter(env=env)
if K8sInfos:
for info in K8sInfos:
namespace = info.namespace_name
env_ = get_CRP_url(env)
crp_url = '%s%s' % (
env_, 'api/openstack/nova/states?namespace={}'.
format(namespace))
ret = requests.get(
crp_url,
timeout=290).json()["result"]["vm_info_dict"]
osid_status.append(ret)
else:
env_ = get_CRP_url(env)
crp_url = '%s%s' % (env_, 'api/openstack/nova/states')
ret = requests.get(
crp_url).json()["result"]["vm_info_dict"]
osid_status.append(ret)
except Exception as e:
Log.logger.error("Get vm info from crp error {}".format(e))
if osid_status:
for os in osid_status:
for k, v in os.items():
vms = Statusvm.objects.filter(osid=str(k))
if not vms:
q = "-".join(str(k).split("-")[:-2])
vms = Statusvm.objects.filter(
resource_name__icontains=q,
update_time__ne=now)
if vms:
vms[0].update(status=v[1],
osid=k,
ip=v[0],
update_time=now,
physical_server=v[-1])
if CMDB_URL:
cmdb_url = CMDB_URL + "cmdb/api/vmdocker/status/"
ret = requests.put(
cmdb_url,
data=json.dumps({"osid_status": osid_status})).json()
else:
Log.logger.info(
"flush_crp_to_cmdb crp->openstack result is null")
except Exception as exc:
Log.logger.error("flush_crp_to_cmdb error:{}".format(exc))
def get_supported_targets():
for (os_name, os) in SUPPORTED_TARGETS.items():
for (suite_name, suite) in os.items():
for arch in suite:
yield (os_name, suite_name, arch)