def setUp(self): self.settings = Settings(u'settings.json') self.display = ParseDisplayOutput(u"trace-display.output") self.cmdgen = CommandGenerator(self.display, self.settings) self.trace(4, u'setup done') print self.cmdgen.add_indv(u'csta')
def __init__(self, program_name, argv: [str], settings: 'Settings' = None) -> None: self.command_line = CommandLineParser(program_name, argv) self.settings = settings or Main.find_settings( self.command_line.settings_file) self.parsed_display = ParseDisplayOutput("") self.command_generator = CommandGenerator(self.parsed_display, self.settings)
class TestCommendGenerator(unittest.TestCase): def trace(self, lvl, *argv): argv = (u"TestCommendGenerator: ", ) + argv return tools.trace(lvl, argv) def setUp(self): self.settings = Settings(u'settings.json') self.display = ParseDisplayOutput(u"trace-display.output") self.cmdgen = CommandGenerator(self.display, self.settings) self.trace(4, u'setup done') print self.cmdgen.add_indv(u'csta') def test_expand(self): self.assertListEqual([u'1', u'2', u'3'], self.cmdgen.expand_names(u'1,2,3')) self.assertListEqual([u'1', u'2', u'3', u'5'], self.cmdgen.expand_names(u'1-3,5')) self.assertListEqual([u'SIPLP', u'RMP', u'CMP'], self.cmdgen.expand_names(u'usual')) self.assertListEqual([u'SIPLP', u'RMP', u'CMP', u'extra'], self.cmdgen.expand_names(u'usual,extra')) self.assertListEqual([u'SIPLP', u'RMP', u'CMP', u'extra'], self.cmdgen.expand_names([u'usual,extra'])) self.assertListEqual([u'SIPLP', u'RMP', u'CMP', u'extra'], self.cmdgen.expand_names([u'usual', u'extra'])) def test_add_ISUS(self): isus = self.display.get_individual(u'ISUS') self.assertIsNone(isus) isus_cmd = self.cmdgen.add_indv(u'ISUS', u'4') self.trace(u"add(ISUS) cmd ", isus_cmd) self.assertTrue(isinstance(isus_cmd, list)) self.assertEqual(1, len(isus_cmd)) self.assertEqual(u' '.join(isus_cmd[0]), u'-lim 4 -unit ISUS') def test_add_csta_gang(self): gang_cmd = self.cmdgen.add_indv(u'csta') self.assertIn(u'-lim 1 -unit ISUS'.split(u' '), gang_cmd) # self.assertIn('trace -lim 1 -unit CSTServer', gang_cmd) self.assertNotIn([u'-lim', u'1', u'-unit', u'SIPLP'], gang_cmd) def test_set_textlevel(self): set_cmd = self.cmdgen.set_textlevel(u'SIPLP', u'full') self.trace(3, set_cmd) def test_print_usual(self): self.trace(3, u'Printing buffers for usual gang: SIPLP, RMP, CMP') cmd = self.cmdgen.save_cmd(self.settings.expand_to_ids([u'usual']), u'sample_') for args, fname in cmd: self.trace(3, u" " + u", ".join(args) + u" filename:" + fname) self.assertEqual(3, len(cmd)) self.assertEqual(u'sample_SIPLP.log', cmd[0][1])
def set_parsed_display(self, val): self.parsed_display = val self.command_generator = CommandGenerator(val, self.settings)
class Main: help_str = \ """ trace-helper, can call and group calls to MX-trace. If starting something not yet defined, it is auto-defined. Possible to start/stop/print on name, not only ID. Possible to configure groups if units, all listening to the same name Switches not used by this program should be passed down to trace Many settings have default values, change ~/.mx-trace.json -display_settings to output current settings-file -gangs Prints which gangs are defined -add indivuduals_or_gang (same as -unit in trace) -remove indivuduals_or_gang -start indivuduals_or_gang -stop indivuduals_or_gang -clear indivuduals_or_gang -print indivuduals_or_gang -save indivudual_or_gang -prefix trace -postfix log Print all the individuals to separate files, with specified prefix/postfix. (Prefix might include paths) -zip output_file.zip indivudual_or_gang Like save, but generate 1 zip file with all traces If specifying many individuals/gangs, separate with space """ def __init__(self, program_name, argv: [str], settings: 'Settings' = None) -> None: self.command_line = CommandLineParser(program_name, argv) self.settings = settings or Main.find_settings( self.command_line.settings_file) self.parsed_display = ParseDisplayOutput("") self.command_generator = CommandGenerator(self.parsed_display, self.settings) def set_parsed_display(self, val: ParseDisplayOutput): self.parsed_display = val self.command_generator = CommandGenerator(val, self.settings) def execute(self, args: [str]) -> 'Executor': return Executor(self.settings.trace_cmd, self.settings.trace_args + args, trace_cmd_level=self.settings.debug_trace_commands) def execute_all(self, list_of_args: [[str]]) -> str: if not isinstance(list_of_args, list): raise ValueError( "args should be list of command-line-lists, was just " + str(type(list_of_args))) if not isinstance(list_of_args[0], list): raise ValueError( "args should be list of command-line-lists, was just " + str(type(list_of_args))) return "\n".join( [self.execute(arg).str_result for arg in list_of_args]) @staticmethod def find_settings(file: str = None) -> str: ex = None files = [ file, os.path.join(os.path.dirname(os.path.realpath(__file__)), 'settings.json'), os.path.join(os.path.expanduser("~"), '.mx-trace', 'settings.json'), os.path.join(os.path.expanduser("~"), '.mx-trace.json') ] for f in files: if not f is None and os.path.exists(f): try: return Settings(f) except BaseException as ex: trace( 3, "Failed to open settings from " + f + ": " + str(ex)) trace( 3, "Failed to open settings from any known file " + "\n " + "\n ".join(files)) return Settings("{\n}") # Return empty settings def main(self) -> str: trace( 7, "main method: " + self.command_line.program_name + " args: [ " + ", ".join(self.command_line.argv) + " ]") display_settings = self.command_line.display_settings if display_settings: return self.call_display_settings() if self.command_line.gangs_request: return self.call_display_gangs() help_args = self.command_line.help if help_args: return self.call_help() display_args = self.command_line.display if not display_args is None: trace(3, "display " + " ".join(display_args), file=sys.stderr) self.call_display(display_args + self.command_line.display_extra_args().argv) print(self.parsed_display) return add_args = self.command_line.add if not add_args is None: trace(3, "add " + " ".join(add_args)) self.call_display() add_extra_args = self.command_line.add_extra_args().argv self.call_add( add_args.split(','), self.command_line.lim, self.command_line.textlevel or self.settings.default_textlevel, add_extra_args) return remove_args = self.command_line.remove if not remove_args is None: trace(3, "remove " + " ".join(remove_args)) self.call_display() remove_extra_args = self.command_line.remove_extra_args().argv self.call_remove(remove_args.split(','), remove_extra_args) return start_args = self.command_line.start if not start_args is None: trace(3, "start " + " ".join(start_args)) self.call_display() start_extra_args = self.command_line.start_extra_args().argv self.call_start( start_args.split(','), self.command_line.lim, self.command_line.textlevel or self.settings.default_textlevel, start_extra_args) return stop_args = self.command_line.stop if not stop_args is None: trace(3, "stop " + " ".join(stop_args)) self.call_display() self.call_stop(stop_args.split(','), self.command_line.stop_extra_args().argv) return print_args = self.command_line.print_args if not print_args is None: trace(3, "print " + print_args) self.call_display() extra_args = self.command_line.print_extra_args().argv printout = self.call_print([print_args], extra_args) print(printout) return printout save_args = self.command_line.save if not save_args is None: prefix = self.command_line.file_prefix or self.settings.file_prefix postfix = self.command_line.file_postfix or self.settings.file_postfix trace(3, "save " + save_args, ", prefix=", prefix, ", postfix=", postfix) self.call_display() self.call_save(save_args.split(','), prefix, postfix) return zip_args = self.command_line.zip if not zip_args is None: prefix = self.command_line.file_prefix or self.settings.zip_prefix postfix = self.command_line.file_postfix or self.settings.zip_postfix zip_file = zip_args[0] _ignored, file_extension = os.path.splitext(zip_file) if len(file_extension) == 0: zip_file = zip_file.rstrip('.') + '.zip' individuals = zip_args[1:] trace(3, "zip ", individuals, " to ", zip_file, ", prefix=", prefix, ", postfix=", postfix) self.call_display() self.call_zip(individuals, zip_file, prefix, postfix) return trace( 2, 'Unknown command, calling trace verbatim for all individuals') self.call_display() self.call_unknown_command() return def expand_to_ids(self, ids_or_gangs: [str]) -> str: if isinstance(ids_or_gangs, str): # Handle if list forgotten ids_or_gangs = [ids_or_gangs] list_of_lists = [iog.split(',') for iog in ids_or_gangs] ids_or_gangs = [ val for sublist in list_of_lists for val in sublist ] if len(ids_or_gangs) == 0 or ids_or_gangs[0].lower() == 'all': return [str(indv) for indv in self.parsed_display.individuals] else: return self.settings.expand_to_ids(ids_or_gangs) def get_existing_ids(self, id_names: [str]) -> [str]: return map(lambda indv: indv.id, self.get_existing_indivuduals(id_names)) # if len(id_names) == 0 or id_names[0].lower() == 'all': # id_names = [str(indv) for indv in self.parsed_display.individuals] # else: # id_names = self.settings.expand_to_ids(id_names) # return list(filter(lambda id: not self.parsed_display.get_individual(id) is None, id_names)) def get_existing_indivuduals( self, id_names: [str]) -> ['ParseDisplayOutput.Individual']: if len(id_names) == 0 or id_names[0].lower() == 'all': id_names = [ str(indv) for indv in self.parsed_display.individuals ] else: id_names = self.settings.expand_to_ids(id_names) individuals = map( lambda id: self.parsed_display.get_individual(id), id_names) return [x for x in individuals if x is not None] #return list(filter(lambda indv: not self.parsed_display.get_individual(id) is None, id_names)) def get_non_existing_individuals(self, id_names: [str]) -> [str]: if len(id_names) == 0 or id_names[0].lower() == 'all': id_names = [ str(indv) for indv in self.parsed_display.individuals ] else: id_names = self.settings.expand_to_ids(id_names) return list( filter( lambda id: self.parsed_display.get_individual(id) is None, id_names)) def ensure_individuals_exists(self, id_names: [str], lim: str, textlevel: str, extra_args: [str]): self.add_individuals(self.get_non_existing_individuals(id_names), lim, textlevel, extra_args) def call_display_gangs(self) -> str: res = "\n".join([(g['name'] + ": " + ", ".join(g['members'])) for g in self.settings.gangs]) print(res) return res def call_display_settings(self) -> str: print(self.settings.raw_data) return self.settings.raw_data def call_help(self) -> str: print(Main.help_str) return Main.help_str def call_display(self, args: [str] = []) -> 'ParseDisplayOutput': disp_output = self.execute(['-display'] + args).str_result self.set_parsed_display(ParseDisplayOutput(disp_output)) def add_individuals(self, individuals: [str], lim: str, textlevel: str, extra_args: [str]): trace(4, "Adding individuals ", individuals) self.execute_all( self.command_generator.add_indv(individuals, lim, extra_args)) self.call_display() for id in individuals: indv = self.parsed_display.get_individual(id) if indv is None: trace(2, 'Failed to create induvidual ' + id) sys.exit(17) if indv.textlevel != textlevel: self.execute_all( self.command_generator.set_textlevel(id, textlevel)) def call_add(self, args: [str], lim: str, textlevel: str, extra_args: [str] = []): if self.parsed_display is None: raise ValueError("Called start when no display parser yet!") individuals = self.expand_to_ids(args) self.ensure_individuals_exists(individuals, lim, textlevel, extra_args) def call_remove(self, args: [str], extra_args: [str] = []): if self.parsed_display is None: raise ValueError("Called start when no display parser yet!") existing = self.get_existing_ids(self.expand_to_ids(args)) remove_cmds = self.command_generator.remove(existing) for indv_stop in remove_cmds: self.execute(indv_stop + extra_args) def call_start(self, args: [str], lim: str, textlevel: str, extra_args: [str] = []): self.call_add(args, lim, textlevel) individuals = self.expand_to_ids(args) start_cmds = self.command_generator.start(individuals) for indv_start in start_cmds: self.execute(indv_start + extra_args) def call_stop(self, args: [str], extra_args: [str] = []): if self.parsed_display is None: raise ValueError("Called stop when no display parser yet!") individuals = self.expand_to_ids(args) existing = self.get_existing_ids(individuals) stop_cmds = self.command_generator.stop(existing) for indv_stop in stop_cmds: self.execute(indv_stop + extra_args) def call_print(self, args: [str], extra_args: [str] = []) -> str: if self.parsed_display is None: raise ValueError("Called print when no display parser yet!") individuals = self.expand_to_ids(args) existing = filter( lambda id: not self.parsed_display.get_individual(id) is None, individuals) print_cmds = self.command_generator.print_cmd(existing) return self.execute_all(print_cmds + extra_args) def call_save(self, args: [str], prefix, postfix, extra_args: [str] = []) -> str: if self.parsed_display is None: raise ValueError("Called print when no display parser yet!") individuals_names = self.expand_to_ids(args) individuals = map( lambda id: self.parsed_display.get_individual(id), individuals_names) existing_individuals = self.get_existing_ids(individuals) # extra_args = self.command_line.get_non_save() for indv in existing_individuals: (print_cmd, filename) = self.command_generator.save_cmd([indv.unit_name], prefix, postfix)[0] trace( 3, 'printing ' + indv.id + "/" + indv.unit_name + " to " + filename) ex = self.execute(print_cmd) with io.open(filename, "w", encoding="latin-1") as fil: fil.write(ex.str_result + extra_args) def call_zip(self, args: [str], zipfilename, prefix, postfix, extra_args: [str] = []) -> str: if not isinstance(args, list): raise ValueError('Should be called with list of individuals') if self.parsed_display is None: raise ValueError("Called print when no display parser yet!") individuals_names = self.expand_to_ids(args) existing_individuals = self.get_existing_indivuduals( individuals_names) # extra_args = self.command_line.get_non_zip() with zipfile.ZipFile(zipfilename, "w") as z: for indv in existing_individuals: (print_cmd, filename) = self.command_generator.save_cmd( [indv.unit_name], prefix, postfix)[0] trace( 3, 'printing ' + indv.id + "/" + indv.unit_name + " to " + filename) ex = self.execute(print_cmd) z.writestr(filename, ex.result) trace(5, "Wrote to " + zipfilename) try: trace(5, "Size became " + str(os.path.getsize(zipfilename))) except: trace(1, "Failed to save data to " + zipfilename) def expand_first_gang_in_commandline(self, args): for i, a in enumerate(args): expanded = self.expand_to_ids(a) if len(expanded) > 1: pre = args[:i] if i > 0 else [] post = args[i + 1:] if i + 1 < len(args) else [] return [pre + [indv] + post for indv in expanded] return [args] def call_unknown_command(self): def find_gang() -> (int, [str]): for i, a in enumerate(self.command_line.argv[1:]): gang = self.expand_to_ids(a) if len(gang) > 0: pre = self.command_line.argv[:i] if i > 0 else [] post = self.command_line.argv[i + 1:] if i + 1 < len( self.command_line.argv) else [] return [pre + [indv] + post for indv in gang] return None gang_expanded = find_gang() or self.command_line.argv[1:] for a in gang_expanded: trace(4, a) self.execute(a)
def connect_components(self): for data_source in self.data_sources: print 'connecting data sources:' data_source > self.alligator (self.alligator > self.logic_validator > command_generator > self.dependency_checker > self.command_executor) if __name__ == '__main__': etl_pipeline = ETLPipeline() ontology.instantiate_data_sources() alligator = Alligator() logic_validator = LogicValidator() dependency_checker = DependencyChecker() command_generator = CommandGenerator() command_executor = CommandExecutor() data_stores_dict = get_data_stores_dict() etl_pipeline.add(alligator) etl_pipeline.add(logic_validator) etl_pipeline.add(dependency_checker) etl_pipeline.add(command_generator) etl_pipeline.add(command_executor) etl_pipeline.add(alligator) for data_source in ontology.SOURCES_STORE.itervalues(): etl_pipeline.add(data_source) for data_store in data_stores_dict.itervalues(): etl_pipeline.add(data_store)