'vendor': [get_curr_dir(__file__) + '/../../vendor/cisco'] } LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare", yangcatalog_api_prefix) for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, args.result_html_dir, args.save_file_dir, private_credentials) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_modules(args.json_dir) else: sdo = False prepare_vendor = prepare.Prepare("prepare", yangcatalog_api_prefix) for search_dir in search_dirs: patterns = ['*ietf-yang-library*.xml', '*capabilit*.xml'] for pattern in patterns: for filename in find_files(search_dir, pattern): update = True if not args.api and args.save_modification_date:
'sdo': ['../../experimental', '../../standard'], 'vendor': ['../../vendor'] } statistics_in_catalog = statisticsInCatalog.StatisticsInCatalog() LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare") for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, statistics_in_catalog) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_sdo(args.json_dir) else: sdo = False prepare_vendor = prepare.Prepare("prepare") for search_dir in search_dirs: for filename in find_files(search_dir, '*capabilit*.xml'): update = True if not args.api and args.save_modification_date: try: file_modification = open( 'fileModificationDate/' +
if time.strptime(time_in_file, '%a %b %d %H:%M:%S %Y') == time.ctime( os.path.getmtime(filename)): update = False file_modification.close() else: file_modification.seek(0) file_modification.write(time.ctime(os.path.getmtime(filename))) file_modification.truncate() file_modification.close() except IOError: file_modification = open( 'fileModificationDate/' + '-'.join(filename.split('/')[-4:]) + '.txt', 'w') file_modification.write(str(time.ctime(os.path.getmtime(filename)))) file_modification.close() if update: integrity = integrityChecker.Integrity(filename) print('Found xml source:' + filename) capability = cap.Capability(filename, index, prepare, integrity) capability.parse_and_dump() index += 1 prepare.dump() # for filename in find_files('../', '*restconf-capabilit*.xml'): # print('Found xml source:' + filename) # capability = cap.Capability(filename) # capability.parse_and_dump() do_stats() end = time.time() print(end - start)
'sdo': ['../../experimental', '../../standard'], 'vendor': ['../../vendor'] } LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare", args.result_html_dir) for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, args.result_html_dir) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_modules(args.json_dir) else: sdo = False prepare_vendor = prepare.Prepare("prepare", args.result_html_dir) for search_dir in search_dirs: patterns = ['*ietf-yang-library*.xml', '*capabilit*.xml'] for pattern in patterns: for filename in find_files(search_dir, pattern): update = True if not args.api and args.save_modification_date: try:
def main(self, get_kcc_version): self.verify_bundle_name() # Read in the existing build info for the release logging.info("Loading release build info...") ktools_path = os.path.join( get_kcc_version.kcc_version().get_kcc_path(self.os_type), "bin") rel_binfo = download_build_info.build_info(self.build_dir, self.build_name, ktools_path, self.kcs_output_dir) logging.info("DONE") # Get the build id from Kymera ELF self.build_id = rel_binfo.get_build_id() chip = chip_info.chip_info(rel_binfo.get_chipname()) if chip.is_supported() == False: raise Exception("Chip NOT supported: " + str(chip.get_string())) try: os.chdir(self.kcsmaker_path) mod = imp.load_source("mod", self.bdl_file) except IOError: logging.error("Can't find file " + str(self.bdl_file)) raise try: if len(mod.KDCs) > 0: logging.info("Building KCS " + str(self.bundle_name)) else: logging.error("Error. No KDCs to be built") exit() except IOError: logging.error("Error. KDCs variable NOT found in " + str(self.bdl_file)) raise if self.linkscript_path == "": self.linkscript_path = os.path.join( self.kcsmaker_path, "templates", chip.get_linkscript_template_name()) logging.info("DOWNLOAD_CONFIG = %s" % self.download_config) logging.info("RELEASE_FLAG = %s" % self.release_flag) logging.info("CAPABILITIES_PATH = %s" % self.capabilities_path) logging.info("KYMERA_SRC_PATH = %s" % self.kymera_src_path) logging.info("BUNDLE_NAME = %s" % self.bundle_name) logging.info("BUILD_DIR = %s" % self.build_dir) logging.info("PATCH_DIR = %s" % self.patch_dir) logging.info("BUILD_NAME = %s" % self.build_name) logging.info("BUILD_ROOT = %s" % self.build_root) logging.info("BUILD_ID = %s" % self.build_id) logging.info("KCS_OUTPUT_DIR = %s" % self.kcs_output_dir) logging.info("KCSMAKER_PATH = %s" % self.kcsmaker_path) logging.info("LINKSCRIPT_PATH = %s" % self.linkscript_path) logging.info("LABEL_NAME = %s" % self.label_name) kcsFileRoot = os.path.join(self.kcs_output_dir, "download", "debugbin", self.bundle_name) kcsFileT = kcsFileRoot + ".kcs" kcsFileB = kcsFileRoot + ".dkcs" kcsFileD = kcsFileRoot + ".kcs_DRAM" makeKCS = make_kcs.make_kcs(chip, self.build_id) logging.info("Generating KCS file " + kcsFileT + " ...") # Create output file if it doesn't exist if not os.path.exists(self.kcs_output_dir): os.makedirs(self.kcs_output_dir) # Check if we are building on the same bundle. If not then force a clean build if (os.path.exists(os.path.join(self.kcs_output_dir, "last_bundle.txt"))): last_bundle_file = open( os.path.join(self.kcs_output_dir, "last_bundle.txt"), "r") prev_bundle = last_bundle_file.readline() if prev_bundle != self.bundle_name: self.clean_build = True logging.info( "Previous build detected: {prev_build}, now building: {cur_build}. \ Forcing clean build".format(prev_build=prev_bundle, cur_build=self.bundle_name)) last_bundle_file.close() else: self.clean_build = True if self.clean_build == None: self.clean_build = self.release_flag kdc_count = 1 for kdc_name in mod.KDCs.keys(): logging.info("Generating KDC " + kdc_name) logging.info("Building " + str(mod.KDCs[kdc_name]) + " capabilities ...") cap_list = [] for cap_tuple in mod.KDCs[kdc_name]: cap_list.append( capability.Capability(cap_tuple[0], cap_tuple[1], cap_tuple[2], cap_tuple[3], self.capabilities_path)) # Build the elf and object files for each capability # Create a Build_Controller object to control the process if self.clean_build: # If Make sure the KDC file is created with an empty output directory for dir_or_file in os.listdir(self.kcs_output_dir): dir_or_file_abs = os.path.join(self.kcs_output_dir, dir_or_file) if os.path.isdir(dir_or_file_abs): # Don't delete this temp directory used to store kmap files if dir_or_file != "kmap_tmp": shutil.rmtree(dir_or_file_abs) else: # It's a file, remove it os.remove(dir_or_file_abs) # Write current bundle into a file last_bundle_file = open( os.path.join(self.kcs_output_dir, "last_bundle.txt"), "w") last_bundle_file.write(self.bundle_name) last_bundle_file.close() if self.make_jobs is None: self.make_jobs = True build = build_controller.Build_Controller( self.build_root, self.build_dir, self.kcs_output_dir, self.kcsmaker_path, self.download_config, rel_binfo.get_elf_name(), self.extra_include, self.build_name, self.kymera_src_path, self.os_type, self.patch_dir) build.build(rel_binfo, cap_list, kdc_name, self.linkscript_path, self.make_path, self.clean_build, self.make_jobs) logging.info( str(mod.KDCs[kdc_name]) + " capabilities successfully built.") # Copy kmap file to temp folder (it will get deleted on next KDC build) if not os.path.exists(os.path.join(self.kcs_output_dir, "kmap_tmp")): os.makedirs(os.path.join(self.kcs_output_dir, "kmap_tmp")) shutil.copy( os.path.join(self.kcs_output_dir, "download", "debugbin", kdc_name + ".kmap"), os.path.join(self.kcs_output_dir, "kmap_tmp", kdc_name + "(KDC).kmap")) objDumpPath = os.path.join( get_kcc_version.kcc_version().get_kcc_path(self.os_type), "bin", "kobjdump") oFiles = self.get_obj_files(chip, get_kcc_version) logging.info("oFiles" + str(oFiles)) self.check_lib_release_files(oFiles) dnldElfFile = os.path.join(self.kcs_output_dir, "download", "debugbin", kdc_name + ".elf") mapOutFile = os.path.join(self.kcs_output_dir, "download", "debugbin", "map_out.txt") makeKDC = make_kdc.make_kdc(dnldElfFile, rel_binfo.get_elf_name(), chip, objDumpPath, oFiles, cap_list, self.os_type, mapOutFile) kdc = makeKDC.generateKDC(kdc_count, rel_binfo.get_rom_kalaccess()) print(("Sizes for KDC {}:".format(dnldElfFile))) kdc.print_memory_usage() makeKCS.addKDC(kdc) kdc_count = kdc_count + 1 # Copy kmap files to the output folder where the KCS files are files = glob.iglob( os.path.join(self.kcs_output_dir, "kmap_tmp", "*.kmap")) for file in files: if os.path.isfile(file): debugbin_dir = os.path.join(self.kcs_output_dir, "download", "debugbin") if self.clean_build: shutil.move(file, debugbin_dir) else: shutil.copy(file, debugbin_dir) if self.clean_build: os.rmdir(os.path.join(self.kcs_output_dir, "kmap_tmp")) kcs = makeKCS.generateKCS() kcs.dumpToTextFile(kcsFileT) kcs.dumpToBinaryFile(kcsFileB) kcs.dumpToDRAMFile(kcsFileD) if self.release_flag: import release this_rel = release.release(self.release_flag, self.kcsmaker_path, self.build_id, self.bundle_name, self.download_config, self.user) release.get_lock() # Generate new release ID this_rel.generate_new_version_id() self.version_build_id = this_rel.get_version_id() logging.info("New BUILD ID: " + str(this_rel.get_version_id())) if self.kymera_src_id == "": self.kymera_src_id = self.get_base_build_id_from_kymera_src_path( ) this_rel.create_log() label = this_rel.label() this_rel.write_log("Label for released code " + str(label)) this_rel.write_log("Base Build ID\t" + str(self.kymera_src_id)) this_rel.end_log() this_rel.archive_KCS( this_rel.obtain_product_name(self.download_config)) release.release_lock() else: # Copy binaries to a common folder so they don't get erased when a new build is triggered if not os.path.exists( os.path.join(self.kymera_src_path, "output_bundles", self.download_config, self.bundle_name)): os.makedirs( os.path.join(self.kymera_src_path, "output_bundles", self.download_config, self.bundle_name)) file_ext_list = [".dkcs", ".kcs", ".elf", ".kmap", ".lst", ".sym"] for file_ext in file_ext_list: shutil.copy( os.path.join(self.kcs_output_dir, "download", "debugbin", self.bundle_name + file_ext), os.path.join(self.kymera_src_path, "output_bundles", self.download_config, self.bundle_name, self.bundle_name + file_ext)) print("Bundle built successfully and available at: " + "{0}".format( os.path.join(self.kymera_src_path, "output_bundles", self.download_config, self.bundle_name, self.bundle_name + ".dkcs")))
clean_build = True logging.info("Previous build detected: {prev_build}, now building: {cur_build}. Forcing clean build".format(prev_build=prev_bundle, cur_build=BUNDLE_NAME)) last_bundle_file.close() else: clean_build = True if clean_build == None: clean_build = RELEASE_FLAG kdc_count = 1 for kdc_name in mod.KDCs.keys(): logging.info("Generating KDC " + kdc_name) logging.info("Building " + str(mod.KDCs[kdc_name]) + " capabilities ...") cap_list = [] for cap_tuple in mod.KDCs[kdc_name]: cap_list.append(capability.Capability(cap_tuple[0], cap_tuple[1], cap_tuple[2], cap_tuple[3], CAPABILITIES_PATH)) # Build the elf and object files for each capability # Create a Build_Controller object to control the process if clean_build: # If Make sure the KDC file is created with an empty output directory for dir_or_file in os.listdir(KCS_OUTPUT_DIR): dir_or_file_abs = os.path.join(KCS_OUTPUT_DIR,dir_or_file) if os.path.isdir(dir_or_file_abs): # Don't delete this temp directory used to store kmap files if dir_or_file != "kmap_tmp": shutil.rmtree(dir_or_file_abs) else: # It's a file, remove it os.remove(dir_or_file_abs)
import time import capability as cap import prepare def find_files(directory, pattern): for root, dirs, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern): filename = os.path.join(root, basename) yield filename start = time.time() i = 1 prepare = prepare.Prepare("prepare") for filename in find_files('../../', '*capabilit*.xml'): print('Found xml source:' + filename) capability = cap.Capability(filename, i, prepare) capability.parse_and_dump() i += 1 prepare.dump() # for filename in find_files('../', '*restconf-capabilit*.xml'): # print('Found xml source:' + filename) # capability = cap.Capability(filename) # capability.parse_and_dump() end = time.time() print(end - start)
default=False, help='if we are doing sdos') parser.add_argument('--run-statistics', action='store_true', default=False, help='if we are running statistics') args = parser.parse_args() start = time.time() index = 1 prepare = prepare.Prepare("prepare") integrity = None if args.sdo: integrity = statistics.Statistics(args.dir) print('Found dir:' + args.dir) capability = cap.Capability(args.dir, index, prepare, integrity, args.api, args.sdo, args.run_statistics) capability.parse_and_dump_sdo() index += 1 else: for filename in find_files(args.dir, '*capabilit*.xml'): update = True if not args.api and args.save_modification_date: try: file_modification = open( 'fileModificationDate/' + '-'.join(filename.split('/')[-4:]) + '.txt', 'rw') time_in_file = file_modification.readline() if time_in_file in str( time.ctime(os.path.getmtime(filename))): update = False print(filename +