def run(self):
client = self.link_cls()
client.link()
Oval_definition.init_oval(client)
for root, dirs, files in os.walk(self.path):
for filepath in files:
try:
if not filepath.endswith('.xml'):
continue
self.results.update(Oval_definition(root+'/'+filepath).result())
except LinkBase.LinkError, e:
Output('Link').error(e)
except Exception, e:
Output(filepath).error(e)
def __init__(self, arg_names):
quiet = False
args = [a.strip() for a in sys.argv[1:]]
args = [a for a in args if len(a) > 0]
if len(args) > 0 and args[0] == '-q':
quiet = True
args = args[1:]
self.name = os.path.basename(sys.argv[0])
self.output = Output(quiet)
self.arg_names = arg_names.split()
self.arg_values = args
self.arg_db = {}
self.args_tail = self.arg_values[len(self.arg_names):]
required_num = 0
optional_num = 0
for name in self.arg_names:
if name[0] == '[' and name[-1] == ']':
optional_num += 1
else:
required_num += 1
# Optional args are only allowed after all required args
if optional_num > 0:
self.usage()
if len(self.arg_values) < required_num:
self.usage()
for name, value in zip(self.arg_names, self.arg_values):
self.arg_db[name] = value
current_dir = os.path.dirname(__file__)
self.dir_a2x = os.path.abspath(os.path.join(current_dir, '..', '..'))
self.dir_bin = os.path.join(self.dir_a2x, 'bin')
self.dir_cfg = os.path.join(os.environ['HOME'], '.config', 'a2x')
self.dir_make = os.path.join(self.dir_a2x, 'make')
self.dir_src = os.path.join(self.dir_a2x, 'src')
if not os.path.exists(self.dir_cfg):
os.makedirs(self.dir_cfg)
elif not os.path.isdir(self.dir_cfg):
self.output.error('{} is not a dir'.format(self.dir_cfg))
def __init__(self, filepath, source='AXTX'):
self.__root__ = TagNode.load(filepath)
self.source = source
self.output = Output()
super(Oval_definition, self).__init__(self.__root__)
for element in self.key_element:
res = getattr(self, element+'s')
for items in self.all(element+'s'):
for item in items.get_childtag_list():
itemid = item.get_attr('id')
res[itemid] = item
if itemid not in self.records[element]:
self.records[element][itemid] = None
for key, value in self.variables.items():
if isinstance(value, list):
continue
self.variables[key] = Variable(value).result()
def main():
with codecs.open('data/syntaxTree.json', 'r', 'utf-8') as fin:
data = json.load(fin)
result = hierarchy(data)
if result[0]:
file_map = check_file_set(result[2])
root = result[1]
print(str_stat(result[2]))
output = Output(root, file_map)
output.addOutput(Output.DEBUG_LEVEL, 'data/out/debug.txt',
tag_iobes)
output.addOutput(Output.CHAR_LEVEL, 'data/out/char.txt', tag_iobes)
output.addOutput(Output.WORD_LEVEL, 'data/out/word.txt', tag_iobes)
output.addOutput(Output.SENTENCE_LEVEL, 'data/out/sentence.txt',
tag_iobes)
output.generate(100)
with codecs.open('data/out/tree.json', 'w', 'utf-8') as fout:
root = generate(root)
children = root['children']
root['children'] = [data['children'][0]]
root['children'].extend(children)
fout.write(json.dumps(root, ensure_ascii=False))
from utils.hierarchy import hierarchy, link_entity, str_stat
from utils.output import Output
random.seed(0)
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file', required=True, help='input file')
parser.add_argument('-w', '--lv_word', help='word level output')
parser.add_argument('-s', '--lv_sentence', help='sentence level')
parser.add_argument('-c',
'--count',
type=int,
help='sentence count',
default=1000)
args = parser.parse_args()
with codecs.open(args.file, 'r', encoding='utf-8') as fin:
setting = json.load(fin)
result = hierarchy(setting['rule'])
if result[0]:
root = result[1]
entity_map = link_entity(result[2], setting['entity'])
print(str_stat(result[2], entity_map))
output = Output(root, entity_map)
if args.lv_word is not None:
output.addOutput(Output.WORD_LEVEL, args.lv_word, tag_iob2)
if args.lv_sentence is not None:
output.addOutput(Output.SENTENCE_LEVEL, args.lv_sentence, tag_iob2)
output.generate(args.count)
def __init__(self, ArgNames, FlagNames=''):
required_num = 0
optional_num = 0
has_tail = False
arg_names = ArgNames.split()
flag_names = ['-q'] + FlagNames.split()
self.name = os.path.basename(sys.argv[0])
self.out = Output(self)
self.arg_names = arg_names
self.arg_db = {}
self.flag_names = flag_names
self.flag_db = {}
arg_values = [a.strip() for a in sys.argv[1:]]
arg_values = [a for a in arg_values if len(a) > 0]
for value in arg_values:
if value in flag_names:
self.flag_db[value] = True
arg_values = arg_values[1:]
else:
break
for name in arg_names:
if has_tail:
self.out.error('... must be the last argument')
elif name == '...':
if optional_num > 0:
self.out.error('Cannot have both optional args and ...')
has_tail = True
elif name[0] == '[' and name[-1] == ']':
optional_num += 1
elif optional_num > 0:
self.out.error('Cannot have normal args after optional args')
else:
required_num += 1
if len(arg_values) < required_num:
self.usage('All required arguments must be present')
if has_tail:
arg_names = arg_names[:-1]
self.arg_db['...'] = arg_values[required_num:]
for name, value in zip(arg_names, arg_values):
self.arg_db[name] = value
if 'FAUR_PATH' in os.environ:
faur_path = os.environ['FAUR_PATH']
else:
faur_path = f'{os.path.dirname(__file__)}/../..'
self.dir_faur = os.path.realpath(
os.path.expandvars(os.path.expanduser(faur_path)))
self.dir_bin = f'{self.dir_faur}/bin'
self.dir_cfg = f'{os.environ["HOME"]}/.config/faur'
self.dir_make = f'{self.dir_faur}/make'
self.dir_src = f'{self.dir_faur}/src'
if not os.path.exists(self.dir_cfg):
os.makedirs(self.dir_cfg)
elif not os.path.isdir(self.dir_cfg):
self.out.error(f'{self.dir_cfg} is not a dir')
if type_pattern.search(line):
type_parse = eval(f"self.{_type}.parse(asn, line)")
return type_parse
def starts_with_community_or_remarks(self, line):
starts_with_remarks = self.patterns.remarks.match(line)
starts_with_community = self.patterns.community.match(line)
invalid_community_format = self.patterns.invalid_community.search(line)
if (starts_with_remarks or starts_with_community)\
and not invalid_community_format:
return True
return False
if __name__ == "__main__":
args = GetArgs()
irr_file_list, json_file, csv_file = args.get_system_paths()
output_template = LoadCommunitiesDict(json_file)
communities_dict = output_template.get_data_structure()
irr_parser = IrrCommunitiesParser(irr_file_list, communities_dict)
collected_data = irr_parser.get_communities_from_irr_files()
output = Output(collected_data, json_file, csv_file)
output.save_files_and_summarize_results()
