'vendor': [get_curr_dir(__file__) + '/../../vendor/cisco']
}
LOGGER.info('Starting to iterate through files')
for key in stats_list:
search_dirs = stats_list[key]
if key == 'sdo':
sdo = True
prepare_sdo = prepare.Prepare("prepare", yangcatalog_api_prefix)
for search_dir in search_dirs:
LOGGER.info('Found directory for sdo {}'.format(search_dir))
integrity = statistics.Statistics(search_dir)
capability = cap.Capability(search_dir, index, prepare_sdo,
integrity, args.api, sdo,
args.json_dir,
args.result_html_dir,
args.save_file_dir,
private_credentials)
LOGGER.info('Starting to parse files in sdo directory')
capability.parse_and_dump_sdo()
index += 1
prepare_sdo.dump_modules(args.json_dir)
else:
sdo = False
prepare_vendor = prepare.Prepare("prepare", yangcatalog_api_prefix)
for search_dir in search_dirs:
patterns = ['*ietf-yang-library*.xml', '*capabilit*.xml']
for pattern in patterns:
for filename in find_files(search_dir, pattern):
update = True
if not args.api and args.save_modification_date:
'sdo': ['../../experimental', '../../standard'],
'vendor': ['../../vendor']
}
statistics_in_catalog = statisticsInCatalog.StatisticsInCatalog()
LOGGER.info('Starting to iterate through files')
for key in stats_list:
search_dirs = stats_list[key]
if key == 'sdo':
sdo = True
prepare_sdo = prepare.Prepare("prepare")
for search_dir in search_dirs:
LOGGER.info('Found directory for sdo {}'.format(search_dir))
integrity = statistics.Statistics(search_dir)
capability = cap.Capability(search_dir, index, prepare_sdo,
integrity, args.api, sdo,
args.json_dir,
statistics_in_catalog)
LOGGER.info('Starting to parse files in sdo directory')
capability.parse_and_dump_sdo()
index += 1
prepare_sdo.dump_sdo(args.json_dir)
else:
sdo = False
prepare_vendor = prepare.Prepare("prepare")
for search_dir in search_dirs:
for filename in find_files(search_dir, '*capabilit*.xml'):
update = True
if not args.api and args.save_modification_date:
try:
file_modification = open(
'fileModificationDate/' +
if time.strptime(time_in_file, '%a %b %d %H:%M:%S %Y') == time.ctime(
os.path.getmtime(filename)):
update = False
file_modification.close()
else:
file_modification.seek(0)
file_modification.write(time.ctime(os.path.getmtime(filename)))
file_modification.truncate()
file_modification.close()
except IOError:
file_modification = open(
'fileModificationDate/' + '-'.join(filename.split('/')[-4:]) +
'.txt', 'w')
file_modification.write(str(time.ctime(os.path.getmtime(filename))))
file_modification.close()
if update:
integrity = integrityChecker.Integrity(filename)
print('Found xml source:' + filename)
capability = cap.Capability(filename, index, prepare, integrity)
capability.parse_and_dump()
index += 1
prepare.dump()
# for filename in find_files('../', '*restconf-capabilit*.xml'):
# print('Found xml source:' + filename)
# capability = cap.Capability(filename)
# capability.parse_and_dump()
do_stats()
end = time.time()
print(end - start)
'sdo': ['../../experimental', '../../standard'],
'vendor': ['../../vendor']
}
LOGGER.info('Starting to iterate through files')
for key in stats_list:
search_dirs = stats_list[key]
if key == 'sdo':
sdo = True
prepare_sdo = prepare.Prepare("prepare", args.result_html_dir)
for search_dir in search_dirs:
LOGGER.info('Found directory for sdo {}'.format(search_dir))
integrity = statistics.Statistics(search_dir)
capability = cap.Capability(search_dir, index, prepare_sdo,
integrity, args.api, sdo,
args.json_dir,
args.result_html_dir)
LOGGER.info('Starting to parse files in sdo directory')
capability.parse_and_dump_sdo()
index += 1
prepare_sdo.dump_modules(args.json_dir)
else:
sdo = False
prepare_vendor = prepare.Prepare("prepare", args.result_html_dir)
for search_dir in search_dirs:
patterns = ['*ietf-yang-library*.xml', '*capabilit*.xml']
for pattern in patterns:
for filename in find_files(search_dir, pattern):
update = True
if not args.api and args.save_modification_date:
try:
def main(self, get_kcc_version):
self.verify_bundle_name()
# Read in the existing build info for the release
logging.info("Loading release build info...")
ktools_path = os.path.join(
get_kcc_version.kcc_version().get_kcc_path(self.os_type), "bin")
rel_binfo = download_build_info.build_info(self.build_dir,
self.build_name,
ktools_path,
self.kcs_output_dir)
logging.info("DONE")
# Get the build id from Kymera ELF
self.build_id = rel_binfo.get_build_id()
chip = chip_info.chip_info(rel_binfo.get_chipname())
if chip.is_supported() == False:
raise Exception("Chip NOT supported: " + str(chip.get_string()))
try:
os.chdir(self.kcsmaker_path)
mod = imp.load_source("mod", self.bdl_file)
except IOError:
logging.error("Can't find file " + str(self.bdl_file))
raise
try:
if len(mod.KDCs) > 0:
logging.info("Building KCS " + str(self.bundle_name))
else:
logging.error("Error. No KDCs to be built")
exit()
except IOError:
logging.error("Error. KDCs variable NOT found in " +
str(self.bdl_file))
raise
if self.linkscript_path == "":
self.linkscript_path = os.path.join(
self.kcsmaker_path, "templates",
chip.get_linkscript_template_name())
logging.info("DOWNLOAD_CONFIG = %s" % self.download_config)
logging.info("RELEASE_FLAG = %s" % self.release_flag)
logging.info("CAPABILITIES_PATH = %s" % self.capabilities_path)
logging.info("KYMERA_SRC_PATH = %s" % self.kymera_src_path)
logging.info("BUNDLE_NAME = %s" % self.bundle_name)
logging.info("BUILD_DIR = %s" % self.build_dir)
logging.info("PATCH_DIR = %s" % self.patch_dir)
logging.info("BUILD_NAME = %s" % self.build_name)
logging.info("BUILD_ROOT = %s" % self.build_root)
logging.info("BUILD_ID = %s" % self.build_id)
logging.info("KCS_OUTPUT_DIR = %s" % self.kcs_output_dir)
logging.info("KCSMAKER_PATH = %s" % self.kcsmaker_path)
logging.info("LINKSCRIPT_PATH = %s" % self.linkscript_path)
logging.info("LABEL_NAME = %s" % self.label_name)
kcsFileRoot = os.path.join(self.kcs_output_dir, "download", "debugbin",
self.bundle_name)
kcsFileT = kcsFileRoot + ".kcs"
kcsFileB = kcsFileRoot + ".dkcs"
kcsFileD = kcsFileRoot + ".kcs_DRAM"
makeKCS = make_kcs.make_kcs(chip, self.build_id)
logging.info("Generating KCS file " + kcsFileT + " ...")
# Create output file if it doesn't exist
if not os.path.exists(self.kcs_output_dir):
os.makedirs(self.kcs_output_dir)
# Check if we are building on the same bundle. If not then force a clean build
if (os.path.exists(os.path.join(self.kcs_output_dir,
"last_bundle.txt"))):
last_bundle_file = open(
os.path.join(self.kcs_output_dir, "last_bundle.txt"), "r")
prev_bundle = last_bundle_file.readline()
if prev_bundle != self.bundle_name:
self.clean_build = True
logging.info(
"Previous build detected: {prev_build}, now building: {cur_build}. \
Forcing clean build".format(prev_build=prev_bundle,
cur_build=self.bundle_name))
last_bundle_file.close()
else:
self.clean_build = True
if self.clean_build == None:
self.clean_build = self.release_flag
kdc_count = 1
for kdc_name in mod.KDCs.keys():
logging.info("Generating KDC " + kdc_name)
logging.info("Building " + str(mod.KDCs[kdc_name]) +
" capabilities ...")
cap_list = []
for cap_tuple in mod.KDCs[kdc_name]:
cap_list.append(
capability.Capability(cap_tuple[0], cap_tuple[1],
cap_tuple[2], cap_tuple[3],
self.capabilities_path))
# Build the elf and object files for each capability
# Create a Build_Controller object to control the process
if self.clean_build:
# If Make sure the KDC file is created with an empty output directory
for dir_or_file in os.listdir(self.kcs_output_dir):
dir_or_file_abs = os.path.join(self.kcs_output_dir,
dir_or_file)
if os.path.isdir(dir_or_file_abs):
# Don't delete this temp directory used to store kmap files
if dir_or_file != "kmap_tmp":
shutil.rmtree(dir_or_file_abs)
else:
# It's a file, remove it
os.remove(dir_or_file_abs)
# Write current bundle into a file
last_bundle_file = open(
os.path.join(self.kcs_output_dir, "last_bundle.txt"), "w")
last_bundle_file.write(self.bundle_name)
last_bundle_file.close()
if self.make_jobs is None:
self.make_jobs = True
build = build_controller.Build_Controller(
self.build_root, self.build_dir, self.kcs_output_dir,
self.kcsmaker_path, self.download_config,
rel_binfo.get_elf_name(), self.extra_include, self.build_name,
self.kymera_src_path, self.os_type, self.patch_dir)
build.build(rel_binfo, cap_list, kdc_name, self.linkscript_path,
self.make_path, self.clean_build, self.make_jobs)
logging.info(
str(mod.KDCs[kdc_name]) + " capabilities successfully built.")
# Copy kmap file to temp folder (it will get deleted on next KDC build)
if not os.path.exists(os.path.join(self.kcs_output_dir,
"kmap_tmp")):
os.makedirs(os.path.join(self.kcs_output_dir, "kmap_tmp"))
shutil.copy(
os.path.join(self.kcs_output_dir, "download", "debugbin",
kdc_name + ".kmap"),
os.path.join(self.kcs_output_dir, "kmap_tmp",
kdc_name + "(KDC).kmap"))
objDumpPath = os.path.join(
get_kcc_version.kcc_version().get_kcc_path(self.os_type),
"bin", "kobjdump")
oFiles = self.get_obj_files(chip, get_kcc_version)
logging.info("oFiles" + str(oFiles))
self.check_lib_release_files(oFiles)
dnldElfFile = os.path.join(self.kcs_output_dir, "download",
"debugbin", kdc_name + ".elf")
mapOutFile = os.path.join(self.kcs_output_dir, "download",
"debugbin", "map_out.txt")
makeKDC = make_kdc.make_kdc(dnldElfFile, rel_binfo.get_elf_name(),
chip, objDumpPath, oFiles, cap_list,
self.os_type, mapOutFile)
kdc = makeKDC.generateKDC(kdc_count, rel_binfo.get_rom_kalaccess())
print(("Sizes for KDC {}:".format(dnldElfFile)))
kdc.print_memory_usage()
makeKCS.addKDC(kdc)
kdc_count = kdc_count + 1
# Copy kmap files to the output folder where the KCS files are
files = glob.iglob(
os.path.join(self.kcs_output_dir, "kmap_tmp", "*.kmap"))
for file in files:
if os.path.isfile(file):
debugbin_dir = os.path.join(self.kcs_output_dir, "download",
"debugbin")
if self.clean_build:
shutil.move(file, debugbin_dir)
else:
shutil.copy(file, debugbin_dir)
if self.clean_build:
os.rmdir(os.path.join(self.kcs_output_dir, "kmap_tmp"))
kcs = makeKCS.generateKCS()
kcs.dumpToTextFile(kcsFileT)
kcs.dumpToBinaryFile(kcsFileB)
kcs.dumpToDRAMFile(kcsFileD)
if self.release_flag:
import release
this_rel = release.release(self.release_flag, self.kcsmaker_path,
self.build_id, self.bundle_name,
self.download_config, self.user)
release.get_lock()
# Generate new release ID
this_rel.generate_new_version_id()
self.version_build_id = this_rel.get_version_id()
logging.info("New BUILD ID: " + str(this_rel.get_version_id()))
if self.kymera_src_id == "":
self.kymera_src_id = self.get_base_build_id_from_kymera_src_path(
)
this_rel.create_log()
label = this_rel.label()
this_rel.write_log("Label for released code " + str(label))
this_rel.write_log("Base Build ID\t" + str(self.kymera_src_id))
this_rel.end_log()
this_rel.archive_KCS(
this_rel.obtain_product_name(self.download_config))
release.release_lock()
else:
# Copy binaries to a common folder so they don't get erased when a new build is triggered
if not os.path.exists(
os.path.join(self.kymera_src_path, "output_bundles",
self.download_config, self.bundle_name)):
os.makedirs(
os.path.join(self.kymera_src_path, "output_bundles",
self.download_config, self.bundle_name))
file_ext_list = [".dkcs", ".kcs", ".elf", ".kmap", ".lst", ".sym"]
for file_ext in file_ext_list:
shutil.copy(
os.path.join(self.kcs_output_dir, "download", "debugbin",
self.bundle_name + file_ext),
os.path.join(self.kymera_src_path, "output_bundles",
self.download_config, self.bundle_name,
self.bundle_name + file_ext))
print("Bundle built successfully and available at: " +
"{0}".format(
os.path.join(self.kymera_src_path, "output_bundles",
self.download_config, self.bundle_name,
self.bundle_name + ".dkcs")))
clean_build = True
logging.info("Previous build detected: {prev_build}, now building: {cur_build}. Forcing clean build".format(prev_build=prev_bundle, cur_build=BUNDLE_NAME))
last_bundle_file.close()
else:
clean_build = True
if clean_build == None:
clean_build = RELEASE_FLAG
kdc_count = 1
for kdc_name in mod.KDCs.keys():
logging.info("Generating KDC " + kdc_name)
logging.info("Building " + str(mod.KDCs[kdc_name]) + " capabilities ...")
cap_list = []
for cap_tuple in mod.KDCs[kdc_name]:
cap_list.append(capability.Capability(cap_tuple[0], cap_tuple[1], cap_tuple[2], cap_tuple[3], CAPABILITIES_PATH))
# Build the elf and object files for each capability
# Create a Build_Controller object to control the process
if clean_build:
# If Make sure the KDC file is created with an empty output directory
for dir_or_file in os.listdir(KCS_OUTPUT_DIR):
dir_or_file_abs = os.path.join(KCS_OUTPUT_DIR,dir_or_file)
if os.path.isdir(dir_or_file_abs):
# Don't delete this temp directory used to store kmap files
if dir_or_file != "kmap_tmp":
shutil.rmtree(dir_or_file_abs)
else:
# It's a file, remove it
os.remove(dir_or_file_abs)
import time
import capability as cap
import prepare
def find_files(directory, pattern):
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern):
filename = os.path.join(root, basename)
yield filename
start = time.time()
i = 1
prepare = prepare.Prepare("prepare")
for filename in find_files('../../', '*capabilit*.xml'):
print('Found xml source:' + filename)
capability = cap.Capability(filename, i, prepare)
capability.parse_and_dump()
i += 1
prepare.dump()
# for filename in find_files('../', '*restconf-capabilit*.xml'):
# print('Found xml source:' + filename)
# capability = cap.Capability(filename)
# capability.parse_and_dump()
end = time.time()
print(end - start)
default=False,
help='if we are doing sdos')
parser.add_argument('--run-statistics',
action='store_true',
default=False,
help='if we are running statistics')
args = parser.parse_args()
start = time.time()
index = 1
prepare = prepare.Prepare("prepare")
integrity = None
if args.sdo:
integrity = statistics.Statistics(args.dir)
print('Found dir:' + args.dir)
capability = cap.Capability(args.dir, index, prepare, integrity,
args.api, args.sdo, args.run_statistics)
capability.parse_and_dump_sdo()
index += 1
else:
for filename in find_files(args.dir, '*capabilit*.xml'):
update = True
if not args.api and args.save_modification_date:
try:
file_modification = open(
'fileModificationDate/' +
'-'.join(filename.split('/')[-4:]) + '.txt', 'rw')
time_in_file = file_modification.readline()
if time_in_file in str(
time.ctime(os.path.getmtime(filename))):
update = False
print(filename +
