def dir_walk(target_dir=None, quiet=None): '''recursively walk a directory containing cti and return the stats''' files = find_files('*.xml', resolve_path(target_dir)) if not quiet: widgets = ['Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA()] progress = ProgressBar(widgets=widgets, maxval=len(files)).start() cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = dict() for file_ in files: try: stix_package = file_to_stix(file_) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): if not k in cooked_cybox_objs.keys(): cooked_cybox_objs[k] = set() cooked_cybox_objs[k].update(raw_cybox_objs[k]) if not quiet: progress.update(i) except: next if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)
def framework_test(code_root, output_dir, cli_config): print("Starting framework test run") abs_out_dir = util.resolve_path(output_dir, root_path=code_root, env=os.environ) try: log_str = shell_command_wrapper('./mdtf -f {input_file}'.format( input_file=os.path.join(code_root, cli_config['config_out'])), cwd=code_root) log_str = util.coerce_to_iter(log_str) # write to most recent directory in output_dir runs = [ d for d in glob.glob(os.path.join(abs_out_dir, '*')) if os.path.isdir(d) ] if not runs: raise IOError( "Can't find framework output in {}".format(abs_out_dir)) run_output = max(runs, key=os.path.getmtime) with open(os.path.join(run_output, 'mdtf_test.log'), 'w') as f: f.write('\n'.join(log_str)) except Exception as exc: fatal_exception_handler(exc, "ERROR: framework test run failed.") print("Finished framework test run at {}".format(run_output)) return run_output
def write_one_sheet(self, writer_info, outfile, data_module): writer_name = writer_info["class"] ns = { "FILE_PATH" : outfile, "FILE_DIR" : os.path.dirname(outfile), "FILE_NAME" : os.path.basename(outfile), } full_path = util.resolve_path(writer_info["file_path"], (ns, xlsconfig, )) util.ensure_folder_exist(full_path) writer_class = getattr(writers, writer_name) wt = writer_class(full_path, data_module, writer_info) wt.write_comment("此文件由导表工具自动生成,禁止手动修改。") wt.write_comment("from " + data_module.info["infile"]) wt.begin_write() wt.write_sheet("main_sheet", data_module.main_sheet) extra_sheets = getattr(data_module, "extra_sheets", None) if extra_sheets is not None: assert(isinstance(extra_sheets, dict)) wt.write_module(extra_sheets) wt.end_write() wt.close() return
def safe_parse_path(option, option_key, cfg_key): path = getattr(option, option_key, None) if path is None: path = getattr(xlsconfig, cfg_key) path = resolve_path(path) setattr(xlsconfig, cfg_key, path)
def dir_walk(target_dir=None, quiet=None): '''recursively walk a directory containing cti and return the stats''' files = find_files('*.xml', resolve_path(target_dir)) if not quiet: widgets = [ 'Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA() ] progress = ProgressBar(widgets=widgets, maxval=len(files)).start() cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = dict() for file_ in files: try: stix_package = file_to_stix(file_) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): if not k in cooked_cybox_objs.keys(): cooked_cybox_objs[k] = set() cooked_cybox_objs[k].update(raw_cybox_objs[k]) if not quiet: progress.update(i) except: next if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)
def resolve_path_in_config(config): for info in config: keys = info.keys() for k in keys: if k.endswith("_path"): info[k] = resolve_path(info[k]) return
def _init_path(self, key, d, env=None): if self._unittest_flag: # use in unit testing only return 'TEST_' + key else: # need to check existence in case we're being called directly assert key in d, 'Error: {} not initialized.'.format(key) return util.resolve_path(util.coerce_from_iter(d[key]), root_path=self.CODE_ROOT, env=env)
def parse_env_vars(self, cli_obj, config): super(GFDLMDTFFramework, self).parse_env_vars(cli_obj, config) # set temp directory according to where we're running if gfdl.running_on_PPAN(): gfdl_tmp_dir = cli_obj.config.get('GFDL_PPAN_TEMP', '$TMPDIR') else: gfdl_tmp_dir = cli_obj.config.get('GFDL_WS_TEMP', '$TMPDIR') gfdl_tmp_dir = util.resolve_path( gfdl_tmp_dir, root_path=self.code_root, env=config.global_envvars ) if not os.path.isdir(gfdl_tmp_dir): gfdl.make_remote_dir(gfdl_tmp_dir) tempfile.tempdir = gfdl_tmp_dir os.environ['MDTF_GFDL_TMPDIR'] = gfdl_tmp_dir config.global_envvars['MDTF_GFDL_TMPDIR'] = gfdl_tmp_dir
def run(self): file_path = util.resolve_path(self.generator_info["file_path"]) util.ensure_folder_exist(file_path) print "生成文件列表", file_path only_name = os.path.splitext(os.path.basename(file_path))[0] wt = JsonWriter(file_path, None) wt.begin_write() package = to_utf8( self.generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE)) wt.write_value("package", to_utf8(package)) class_name_format = to_utf8( self.generator_info.get("class_name_format")) enum_name_format = to_utf8(self.generator_info.get("enum_name_format")) keys = self.exporter.data_modules.keys() keys.sort() ret = [] for key in keys: data_module = self.exporter.data_modules[key] converter_name = data_module.info["parser"] file_name = os.path.splitext(key)[0] enum_name = file_name.replace('\\', '_').replace('/', '_').upper() if enum_name_format: enum_name = enum_name_format % enum_name fpath, fname = os.path.split(file_name) file_name = os.path.join(only_name, fpath, "%s.wg" % fname) file_name = format_slash(file_name) class_name = to_class_name(converter_name.split('.')[-1]) if class_name_format: class_name = class_name_format % class_name ret.append([enum_name, file_name, class_name]) wt.write_value(only_name, ret) wt.end_write() wt.close()
def parse_config(self): d = self.config # abbreviation # determine downloads d.downloads_list = ['obs'] if not d.no_cesm: d.downloads_list.append('model_cesm') if not d.no_am4: d.downloads_list.append('model_am4') # determine runtime setup d.pods = 'all' d.conda_envmgr = True if d.env_setup == 'conda-basic': d.conda_envs = [self.settings.conda['framework_env'], 'NCL_base'] d.environment_manager = "Conda" d.pods = [ "Wheeler_Kiladis", "EOF_500hPa", "MJO_suite", "MJO_teleconnection" ] if 'model_am4' in d.downloads_list: d.downloads_list.remove('model_am4') elif d.env_setup == 'conda-full': d.conda_envs = ['all'] d.environment_manager = "Conda" elif d.env_setup == 'no-conda': d.conda_envmgr = False d.conda_envs = [] d.environment_manager = "VirtualEnv" if d.conda_install_dev and not d.conda_envmgr: d.conda_envs.append('dev') # make settings consistent with config ordered_data = collections.OrderedDict() for k in d.downloads_list: ordered_data[k] = self.settings.data[k] self.settings.data = ordered_data for k in self._shared_conda_keys: self.settings.conda[k] = d[k] # convert relative paths to absolute for key in (self._env_paths + self._data_paths): if d[key]: d[key] = util.resolve_path(d[key], root_path=self.code_root, env=os.environ)
def dir_walk(target_dir): '''recursively walk a directory containing cti and return the stats''' files = gen_find('*.xml', resolve_path(target_dir)) cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = dict() for file_ in files: try: stix_package = file_to_stix(file_) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): if not k in cooked_cybox_objs.keys(): cooked_cybox_objs[k] = set() cooked_cybox_objs[k].update(raw_cybox_objs[k]) except: next return (cooked_stix_objs, cooked_cybox_objs)
def run(self): file_path = util.resolve_path(self.generator_info["file_path"]) util.ensure_folder_exist(file_path) print "生成枚举类", file_path wt = BaseWriter(file_path, None) indent = 0 wt._output_line(indent, "// 此文件由导表工具自动生成,禁止手动修改。") wt._output_line() package = util.to_utf8( self.generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE)) wt._output_line(indent, "package %s;" % package) wt._output_line() class_name = os.path.splitext(os.path.basename(file_path))[0] wt._output_line(indent, "public enum %s {" % class_name) indent += 1 value_pairs = [] for outfile, data_module in self.exporter.data_modules.iteritems(): enume_name = to_enum_name(outfile) comment = data_module.info["arguments"].get("describe") value_pairs.append((enume_name, comment)) value_pairs.sort(key=lambda v: v[0]) for enum_name, comment in value_pairs: if comment: wt._output_line(indent, "%-20s // %s" % (enum_name + ",", comment)) else: wt._output_line(indent, enum_name, ",") indent -= 1 wt._output_line(indent, "};") wt.close()
def run(self): src_name = self.module_name.split('.')[-1] self.class_name = util.to_class_name(src_name) generator_info = self.generator_info name_format = util.to_utf8(generator_info.get("name_format")) if name_format: self.class_name = name_format % self.class_name ns = { "FILE_PATH": self.class_name, } self.file_path = util.resolve_path(generator_info["file_path"], ( ns, xlsconfig, )) util.ensure_folder_exist(self.file_path) self.write_line(0, "// 此文件由导表工具自动生成,禁止手动修改。") self.write_line() package = util.to_utf8( generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE)) self.write_line(0, "package %s;" % package) self.write_line() imports = generator_info.get("imports") if imports: for imp in imports: self.write_line(0, "import %s;" % imp.encode("utf-8")) self.write_line() items = self.collect_members(self.module) self.gen_class(items, 0) self.save_to_file(self.file_path)
def load_configure(cfg_file, option): cfg_file = os.path.abspath(cfg_file) print "load configure file", cfg_file if not os.path.exists(cfg_file): raise RuntimeError, "配置文件不存在: %s" % cfg_file cfg = imp.load_source("custom_configure", cfg_file) for k, v in cfg.__dict__.iteritems(): if k.startswith('_'): continue setattr(xlsconfig, k, v) file_path = os.path.dirname(cfg_file) xlsconfig.CONFIG_PATH = file_path xlsconfig.pre_init_method(xlsconfig) safe_parse_path(option, "project", "PROJECT_PATH") safe_parse_path(option, "input", "INPUT_PATH") safe_parse_path(option, "output", "OUTPUT_PATH") safe_parse_path(option, "temp", "TEMP_PATH") safe_parse_path(option, "converter", "CONVERTER_PATH") for k in xlsconfig.DEPENDENCIES.keys(): path = xlsconfig.DEPENDENCIES[k] xlsconfig.DEPENDENCIES[k] = resolve_path(path) # 所有已知的配置中,名字叫'file_path'的路径,会自动转义 resolve_path_in_config(xlsconfig.CODE_GENERATORS) resolve_path_in_config(xlsconfig.DATA_WRITERS) resolve_path_in_config(xlsconfig.POSTPROCESSORS) # 加载完毕回调 xlsconfig.post_init_method(xlsconfig) return
def dir_walk(target_dir=None, quiet=None): '''recursively walk a directory containing cti and return the stats''' files = find_files('*.xml', resolve_path(target_dir)) if not quiet: widgets = ['Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA()] progress = ProgressBar(widgets=widgets, maxval=len(files)).start() cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = {'AccountObjectType': set(), 'AddressObjectType': set(), 'APIObjectType': set(), 'ArchiveFileObjectType': set(), 'ARPCacheObjectType': set(), 'ArtifactObjectType': set(), 'ASObjectType': set(), 'CodeObjectType': set(), 'CustomObjectType': set(), 'DeviceObjectType': set(), 'DiskObjectType': set(), 'DiskPartitionObjectType': set(), 'DNSCacheObjectType': set(), 'DNSQueryObjectType': set(), 'DNSRecordObjectType': set(), 'DomainNameObjectType': set(), 'EmailMessageObjectType': set(), 'FileObjectType': set(), 'GUIDialogboxObjectType': set(), 'GUIObjectType': set(), 'GUIWindowObjectType': set(), 'HostnameObjectType': set(), 'HTTPSessionObjectType': set(), 'ImageFileObjectType': set(), 'LibraryObjectType': set(), 'LinkObjectType': set(), 'LinuxPackageObjectType': set(), 'MemoryObjectType': set(), 'MutexObjectType': set(), 'NetworkConnectionObjectType': set(), 'NetworkFlowObjectType': set(), 'NetworkPacketObjectType': set(), 'NetworkRouteEntryObjectType': set(), 'NetRouteObjectType': set(), 'NetworkSocketObjectType': set(), 'NetworkSubnetObjectType': set(), 'PDFFileObjectType': set(), 'PipeObjectType': set(), 'PortObjectType': set(), 'ProcessObjectType': set(), 'ProductObjectType': set(), 'SemaphoreObjectType': set(), 'SMSMessageObjectType': set(), 'SocketAddressObjectType': set(), 'SystemObjectType': set(), 'UnixFileObjectType': set(), 'UnixNetworkRouteEntryObjectType': set(), 'UnixPipeObjectType': set(), 'UnixProcessObjectType': set(), 'UnixUserAccountObjectType': set(), 'UnixVolumeObjectType': set(), 'URIObjectType': set(), 'URLHistoryObjectType': set(), 'UserAccountObjectType': set(), 'UserSessionObjectType': set(), 'VolumeObjectType': set(), 'WhoisObjectType': set(), 'WindowsComputerAccountObjectType': set(), 'WindowsCriticalSectionObjectType': set(), 'WindowsDriverObjectType': set(), 'WindowsEventLogObjectType': set(), 'WindowsEventObjectType': set(), 'WindowsExecutableFileObjectType': set(), 'WindowsFilemappingObjectType': set(), 'WindowsFileObjectType': set(), 'WindowsHandleObjectType': set(), 'WindowsHookObjectType': set(), 'WindowsKernelHookObjectType': set(), 'WindowsKernelObjectType': set(), 'WindowsMailslotObjectType': set(), 'WindowsMemoryPageRegionObjectType': set(), 'WindowsMutexObjectType': set(), 'WindowsNetworkRouteEntryObjectType': set(), 'WindowsNetworkShareObjectType': set(), 'WindowsPipeObjectType': set(), 'WindowsPrefetchObjectType': set(), 'WindowsProcessObjectType': set(), 'WindowsRegistryKeyObjectType': set(), 'WindowsSemaphoreObjectType': set(), 'WindowsServiceObjectType': set(), 'WindowsSystemObjectType': set(), 'WindowsSystemRestoreObjectType': set(), 'WindowsTaskObjectType': set(), 'WindowsThreadObjectType': set(), 'WindowsUserAccountObjectType': set(), 'WindowsVolumeObjectType': set(), 'WindowsWaitableTimerObjectType': set(), 'X509CertificateObjectType': set(), } for file_ in files: try: stix_package = file_to_stix(file_) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): cooked_cybox_objs[k].update(raw_cybox_objs[k]) if not quiet: progress.update(i) except: next if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)
def dir_walk(target_dir=None, quiet=None): '''recursively walk a directory containing cti and return the stats''' files = find_files('*.xml', resolve_path(target_dir)) if not quiet: widgets = [ 'Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA() ] progress = ProgressBar(widgets=widgets, maxval=len(files)).start() cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = { 'AccountObjectType': set(), 'AddressObjectType': set(), 'APIObjectType': set(), 'ArchiveFileObjectType': set(), 'ARPCacheObjectType': set(), 'ArtifactObjectType': set(), 'ASObjectType': set(), 'CodeObjectType': set(), 'CustomObjectType': set(), 'DeviceObjectType': set(), 'DiskObjectType': set(), 'DiskPartitionObjectType': set(), 'DNSCacheObjectType': set(), 'DNSQueryObjectType': set(), 'DNSRecordObjectType': set(), 'DomainNameObjectType': set(), 'EmailMessageObjectType': set(), 'FileObjectType': set(), 'GUIDialogboxObjectType': set(), 'GUIObjectType': set(), 'GUIWindowObjectType': set(), 'HostnameObjectType': set(), 'HTTPSessionObjectType': set(), 'ImageFileObjectType': set(), 'LibraryObjectType': set(), 'LinkObjectType': set(), 'LinuxPackageObjectType': set(), 'MemoryObjectType': set(), 'MutexObjectType': set(), 'NetworkConnectionObjectType': set(), 'NetworkFlowObjectType': set(), 'NetworkPacketObjectType': set(), 'NetworkRouteEntryObjectType': set(), 'NetRouteObjectType': set(), 'NetworkSocketObjectType': set(), 'NetworkSubnetObjectType': set(), 'PDFFileObjectType': set(), 'PipeObjectType': set(), 'PortObjectType': set(), 'ProcessObjectType': set(), 'ProductObjectType': set(), 'SemaphoreObjectType': set(), 'SMSMessageObjectType': set(), 'SocketAddressObjectType': set(), 'SystemObjectType': set(), 'UnixFileObjectType': set(), 'UnixNetworkRouteEntryObjectType': set(), 'UnixPipeObjectType': set(), 'UnixProcessObjectType': set(), 'UnixUserAccountObjectType': set(), 'UnixVolumeObjectType': set(), 'URIObjectType': set(), 'URLHistoryObjectType': set(), 'UserAccountObjectType': set(), 'UserSessionObjectType': set(), 'VolumeObjectType': set(), 'WhoisObjectType': set(), 'WindowsComputerAccountObjectType': set(), 'WindowsCriticalSectionObjectType': set(), 'WindowsDriverObjectType': set(), 'WindowsEventLogObjectType': set(), 'WindowsEventObjectType': set(), 'WindowsExecutableFileObjectType': set(), 'WindowsFilemappingObjectType': set(), 'WindowsFileObjectType': set(), 'WindowsHandleObjectType': set(), 'WindowsHookObjectType': set(), 'WindowsKernelHookObjectType': set(), 'WindowsKernelObjectType': set(), 'WindowsMailslotObjectType': set(), 'WindowsMemoryPageRegionObjectType': set(), 'WindowsMutexObjectType': set(), 'WindowsNetworkRouteEntryObjectType': set(), 'WindowsNetworkShareObjectType': set(), 'WindowsPipeObjectType': set(), 'WindowsPrefetchObjectType': set(), 'WindowsProcessObjectType': set(), 'WindowsRegistryKeyObjectType': set(), 'WindowsSemaphoreObjectType': set(), 'WindowsServiceObjectType': set(), 'WindowsSystemObjectType': set(), 'WindowsSystemRestoreObjectType': set(), 'WindowsTaskObjectType': set(), 'WindowsThreadObjectType': set(), 'WindowsUserAccountObjectType': set(), 'WindowsVolumeObjectType': set(), 'WindowsWaitableTimerObjectType': set(), 'X509CertificateObjectType': set(), } for file_ in files: try: stix_package = file_to_stix(file_) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): cooked_cybox_objs[k].update(raw_cybox_objs[k]) if not quiet: progress.update(i) except: next if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)