def dir_walk(target_dir=None, quiet=None):
'''recursively walk a directory containing cti and return the stats'''
files = find_files('*.xml', resolve_path(target_dir))
if not quiet:
widgets = ['Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA()]
progress = ProgressBar(widgets=widgets, maxval=len(files)).start()
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
if not quiet:
progress.update(i)
except:
next
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
def framework_test(code_root, output_dir, cli_config):
print("Starting framework test run")
abs_out_dir = util.resolve_path(output_dir,
root_path=code_root,
env=os.environ)
try:
log_str = shell_command_wrapper('./mdtf -f {input_file}'.format(
input_file=os.path.join(code_root, cli_config['config_out'])),
cwd=code_root)
log_str = util.coerce_to_iter(log_str)
# write to most recent directory in output_dir
runs = [
d for d in glob.glob(os.path.join(abs_out_dir, '*'))
if os.path.isdir(d)
]
if not runs:
raise IOError(
"Can't find framework output in {}".format(abs_out_dir))
run_output = max(runs, key=os.path.getmtime)
with open(os.path.join(run_output, 'mdtf_test.log'), 'w') as f:
f.write('\n'.join(log_str))
except Exception as exc:
fatal_exception_handler(exc, "ERROR: framework test run failed.")
print("Finished framework test run at {}".format(run_output))
return run_output
def write_one_sheet(self, writer_info, outfile, data_module):
writer_name = writer_info["class"]
ns = {
"FILE_PATH" : outfile,
"FILE_DIR" : os.path.dirname(outfile),
"FILE_NAME" : os.path.basename(outfile),
}
full_path = util.resolve_path(writer_info["file_path"], (ns, xlsconfig, ))
util.ensure_folder_exist(full_path)
writer_class = getattr(writers, writer_name)
wt = writer_class(full_path, data_module, writer_info)
wt.write_comment("此文件由导表工具自动生成，禁止手动修改。")
wt.write_comment("from " + data_module.info["infile"])
wt.begin_write()
wt.write_sheet("main_sheet", data_module.main_sheet)
extra_sheets = getattr(data_module, "extra_sheets", None)
if extra_sheets is not None:
assert(isinstance(extra_sheets, dict))
wt.write_module(extra_sheets)
wt.end_write()
wt.close()
return
def safe_parse_path(option, option_key, cfg_key):
path = getattr(option, option_key, None)
if path is None:
path = getattr(xlsconfig, cfg_key)
path = resolve_path(path)
setattr(xlsconfig, cfg_key, path)
def dir_walk(target_dir=None, quiet=None):
'''recursively walk a directory containing cti and return the stats'''
files = find_files('*.xml', resolve_path(target_dir))
if not quiet:
widgets = [
'Directory Walk: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=len(files)).start()
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
if not quiet:
progress.update(i)
except:
next
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
def resolve_path_in_config(config):
for info in config:
keys = info.keys()
for k in keys:
if k.endswith("_path"):
info[k] = resolve_path(info[k])
return
def _init_path(self, key, d, env=None):
if self._unittest_flag: # use in unit testing only
return 'TEST_' + key
else:
# need to check existence in case we're being called directly
assert key in d, 'Error: {} not initialized.'.format(key)
return util.resolve_path(util.coerce_from_iter(d[key]),
root_path=self.CODE_ROOT,
env=env)
def parse_env_vars(self, cli_obj, config):
super(GFDLMDTFFramework, self).parse_env_vars(cli_obj, config)
# set temp directory according to where we're running
if gfdl.running_on_PPAN():
gfdl_tmp_dir = cli_obj.config.get('GFDL_PPAN_TEMP', '$TMPDIR')
else:
gfdl_tmp_dir = cli_obj.config.get('GFDL_WS_TEMP', '$TMPDIR')
gfdl_tmp_dir = util.resolve_path(
gfdl_tmp_dir, root_path=self.code_root, env=config.global_envvars
)
if not os.path.isdir(gfdl_tmp_dir):
gfdl.make_remote_dir(gfdl_tmp_dir)
tempfile.tempdir = gfdl_tmp_dir
os.environ['MDTF_GFDL_TMPDIR'] = gfdl_tmp_dir
config.global_envvars['MDTF_GFDL_TMPDIR'] = gfdl_tmp_dir
def run(self):
file_path = util.resolve_path(self.generator_info["file_path"])
util.ensure_folder_exist(file_path)
print "生成文件列表", file_path
only_name = os.path.splitext(os.path.basename(file_path))[0]
wt = JsonWriter(file_path, None)
wt.begin_write()
package = to_utf8(
self.generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE))
wt.write_value("package", to_utf8(package))
class_name_format = to_utf8(
self.generator_info.get("class_name_format"))
enum_name_format = to_utf8(self.generator_info.get("enum_name_format"))
keys = self.exporter.data_modules.keys()
keys.sort()
ret = []
for key in keys:
data_module = self.exporter.data_modules[key]
converter_name = data_module.info["parser"]
file_name = os.path.splitext(key)[0]
enum_name = file_name.replace('\\', '_').replace('/', '_').upper()
if enum_name_format: enum_name = enum_name_format % enum_name
fpath, fname = os.path.split(file_name)
file_name = os.path.join(only_name, fpath, "%s.wg" % fname)
file_name = format_slash(file_name)
class_name = to_class_name(converter_name.split('.')[-1])
if class_name_format: class_name = class_name_format % class_name
ret.append([enum_name, file_name, class_name])
wt.write_value(only_name, ret)
wt.end_write()
wt.close()
def parse_config(self):
d = self.config # abbreviation
# determine downloads
d.downloads_list = ['obs']
if not d.no_cesm:
d.downloads_list.append('model_cesm')
if not d.no_am4:
d.downloads_list.append('model_am4')
# determine runtime setup
d.pods = 'all'
d.conda_envmgr = True
if d.env_setup == 'conda-basic':
d.conda_envs = [self.settings.conda['framework_env'], 'NCL_base']
d.environment_manager = "Conda"
d.pods = [
"Wheeler_Kiladis", "EOF_500hPa", "MJO_suite",
"MJO_teleconnection"
]
if 'model_am4' in d.downloads_list:
d.downloads_list.remove('model_am4')
elif d.env_setup == 'conda-full':
d.conda_envs = ['all']
d.environment_manager = "Conda"
elif d.env_setup == 'no-conda':
d.conda_envmgr = False
d.conda_envs = []
d.environment_manager = "VirtualEnv"
if d.conda_install_dev and not d.conda_envmgr:
d.conda_envs.append('dev')
# make settings consistent with config
ordered_data = collections.OrderedDict()
for k in d.downloads_list:
ordered_data[k] = self.settings.data[k]
self.settings.data = ordered_data
for k in self._shared_conda_keys:
self.settings.conda[k] = d[k]
# convert relative paths to absolute
for key in (self._env_paths + self._data_paths):
if d[key]:
d[key] = util.resolve_path(d[key],
root_path=self.code_root,
env=os.environ)
def dir_walk(target_dir):
'''recursively walk a directory containing cti and return the stats'''
files = gen_find('*.xml', resolve_path(target_dir))
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
return (cooked_stix_objs, cooked_cybox_objs)
def run(self):
file_path = util.resolve_path(self.generator_info["file_path"])
util.ensure_folder_exist(file_path)
print "生成枚举类", file_path
wt = BaseWriter(file_path, None)
indent = 0
wt._output_line(indent, "// 此文件由导表工具自动生成，禁止手动修改。")
wt._output_line()
package = util.to_utf8(
self.generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE))
wt._output_line(indent, "package %s;" % package)
wt._output_line()
class_name = os.path.splitext(os.path.basename(file_path))[0]
wt._output_line(indent, "public enum %s {" % class_name)
indent += 1
value_pairs = []
for outfile, data_module in self.exporter.data_modules.iteritems():
enume_name = to_enum_name(outfile)
comment = data_module.info["arguments"].get("describe")
value_pairs.append((enume_name, comment))
value_pairs.sort(key=lambda v: v[0])
for enum_name, comment in value_pairs:
if comment:
wt._output_line(indent,
"%-20s // %s" % (enum_name + ",", comment))
else:
wt._output_line(indent, enum_name, ",")
indent -= 1
wt._output_line(indent, "};")
wt.close()
def run(self):
src_name = self.module_name.split('.')[-1]
self.class_name = util.to_class_name(src_name)
generator_info = self.generator_info
name_format = util.to_utf8(generator_info.get("name_format"))
if name_format:
self.class_name = name_format % self.class_name
ns = {
"FILE_PATH": self.class_name,
}
self.file_path = util.resolve_path(generator_info["file_path"], (
ns,
xlsconfig,
))
util.ensure_folder_exist(self.file_path)
self.write_line(0, "// 此文件由导表工具自动生成，禁止手动修改。")
self.write_line()
package = util.to_utf8(
generator_info.get("package", xlsconfig.DEFAULT_JAVA_PACKAGE))
self.write_line(0, "package %s;" % package)
self.write_line()
imports = generator_info.get("imports")
if imports:
for imp in imports:
self.write_line(0, "import %s;" % imp.encode("utf-8"))
self.write_line()
items = self.collect_members(self.module)
self.gen_class(items, 0)
self.save_to_file(self.file_path)
def load_configure(cfg_file, option):
cfg_file = os.path.abspath(cfg_file)
print "load configure file", cfg_file
if not os.path.exists(cfg_file):
raise RuntimeError, "配置文件不存在: %s" % cfg_file
cfg = imp.load_source("custom_configure", cfg_file)
for k, v in cfg.__dict__.iteritems():
if k.startswith('_'): continue
setattr(xlsconfig, k, v)
file_path = os.path.dirname(cfg_file)
xlsconfig.CONFIG_PATH = file_path
xlsconfig.pre_init_method(xlsconfig)
safe_parse_path(option, "project", "PROJECT_PATH")
safe_parse_path(option, "input", "INPUT_PATH")
safe_parse_path(option, "output", "OUTPUT_PATH")
safe_parse_path(option, "temp", "TEMP_PATH")
safe_parse_path(option, "converter", "CONVERTER_PATH")
for k in xlsconfig.DEPENDENCIES.keys():
path = xlsconfig.DEPENDENCIES[k]
xlsconfig.DEPENDENCIES[k] = resolve_path(path)
# 所有已知的配置中，名字叫'file_path'的路径，会自动转义
resolve_path_in_config(xlsconfig.CODE_GENERATORS)
resolve_path_in_config(xlsconfig.DATA_WRITERS)
resolve_path_in_config(xlsconfig.POSTPROCESSORS)
# 加载完毕回调
xlsconfig.post_init_method(xlsconfig)
return
def dir_walk(target_dir=None, quiet=None):
'''recursively walk a directory containing cti and return the stats'''
files = find_files('*.xml', resolve_path(target_dir))
if not quiet:
widgets = ['Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA()]
progress = ProgressBar(widgets=widgets, maxval=len(files)).start()
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
cooked_cybox_objs[k].update(raw_cybox_objs[k])
if not quiet:
progress.update(i)
except:
next
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
def dir_walk(target_dir=None, quiet=None):
'''recursively walk a directory containing cti and return the stats'''
files = find_files('*.xml', resolve_path(target_dir))
if not quiet:
widgets = [
'Directory Walk: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=len(files)).start()
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {
'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
cooked_cybox_objs[k].update(raw_cybox_objs[k])
if not quiet:
progress.update(i)
except:
next
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
