def register_script(self): """ Registers a pig scripts with its variables substituted. raises: IOException If a temp file containing the pig script could not be created. raises: ParseException The pig script could not have all its variables substituted. todo: Refactor this processes that result in calling this method. This method gets called twice for a single assert as every method that needs the data assumes no one else has called it (even methods that call other methods that call it (assertOutput() calls get_alias() which both call this method). """ pigIStream = BufferedReader(StringReader(self.orig_pig_code)) pigOStream = StringWriter() ps = ParameterSubstitutionPreprocessor(50) # Where does 50 come from? ps.genSubstitutedFile(pigIStream, pigOStream, self.args, self.arg_files) substitutedPig = pigOStream.toString() f = File.createTempFile("tmp", "pigunit") pw = PrintWriter(f) pw.println(substitutedPig) pw.close() pigSubstitutedFile = f.getCanonicalPath() self._temp_pig_script = pigSubstitutedFile self.pig.registerScript(pigSubstitutedFile, self.alias_overrides)
def dm_dump_dot(dmuid, filename): dmm = project.getDUM() uid = DMUID.parse(dmuid) dm = dmm.getDM(uid) if dm == None: printf('DM %s not found', dmuid) else: printf('Dumping %s...', dm) dot = AST2DOT(dm, project.getZDB()) dot.blacklistField("fParent") # dot.blacklistField("fSource") # dot.blacklistField("fStartCol") # dot.blacklistField("fStartLine") dot.blacklistField("fEndCol") dot.blacklistField("fEndLine") dot.blacklistField("fDeclarationMap") out = PrintWriter(BufferedWriter(FileWriter(filename))) dot.convert(out) out.close() printf("python: wrote dot file to %s", filename)
def ig_dump_dot(dmuid, filename): igm = project.getIGM() uid = DMUID.parse(dmuid) module = igm.findModule(Toplevel(uid, None)) if module == None: printf('DM %s not found', dmuid) else: printf('Dumping %s...', module) dot = IG2DOT(module) dot.blacklistField("fImportedLibs") dot.blacklistField("fImportedPackages") dot.blacklistField("fZPrjID") dot.blacklistField("fSFDBID") dot.blacklistField("fLine") dot.blacklistField("fCol") dot.blacklistField("fScore") dot.blacklistField("fFailed") dot.blacklistField("fReject") dot.blacklistField("fInertial") dot.blacklistField("fDelay") out = PrintWriter(BufferedWriter(FileWriter(filename))) dot.convert(out) out.close() printf("python: wrote dot file to %s", filename)
def rtl_dump_svg(dmuid,filename): rtlmanager = project.getRTLM() uid = DMUID.parse(dmuid) rtlm = rtlmanager.findModule(Toplevel(uid, None)) if rtlm == None: printf('RTLM %s not found', dmuid) else: printf('Dumping %s...', rtlm) out = PrintWriter(BufferedWriter(FileWriter(filename))) gc = VGGCSVG(out) contentProvider = RTLVisualGraphContentProvider(rtlm) labelProvider = RTLVisualGraphLabelProvider(rtlm) selectionProvider = RTLVisualGraphSelectionProvider() layout = VGLayout(contentProvider, labelProvider, gc) layout.paint(selectionProvider); out.close() printf("python: wrote svg file to %s", filename)
def write_ordered_variables(program_name, variable_map, file_path, append=False): """ Write variables to file while preserving order of the variables. :param program_name: name of the calling program :param variable_map: map or variable properties to write to file :param file_path: the file to which to write the properties :param append: defaults to False. Append properties to the end of file :raises VariableException if an error occurs while storing the variables in the file """ _method_name = 'write_ordered_variables' _logger.entering(program_name, file_path, append, class_name=_class_name, method_name=_method_name) pw = None try: pw = PrintWriter(FileOutputStream(File(file_path), Boolean(append)), Boolean('true')) for key, value in variable_map.iteritems(): formatted = '%s=%s' % (key, value) pw.println(formatted) pw.close() except IOException, ioe: _logger.fine('WLSDPLY-20007', file_path, ioe.getLocalizedMessage()) ex = exception_helper.create_variable_exception( 'WLSDPLY-20007', file_path, ioe.getLocalizedMessage(), error=ioe) _logger.throwing(ex, class_name=_class_name, method_name=_method_name) if pw is not None: pw.close() raise ex
def dm_dump_dot(dmuid,filename): dmm = project.getDUM() uid = DMUID.parse(dmuid) dm = dmm.getDM(uid) if dm == None: printf('DM %s not found', dmuid) else: printf('Dumping %s...', dm) dot = AST2DOT(dm, project.getZDB()) dot.blacklistField("fParent") # dot.blacklistField("fSource") # dot.blacklistField("fStartCol") # dot.blacklistField("fStartLine") dot.blacklistField("fEndCol") dot.blacklistField("fEndLine") dot.blacklistField("fDeclarationMap") out = PrintWriter(BufferedWriter(FileWriter(filename))); dot.convert(out); out.close() printf("python: wrote dot file to %s", filename)
def ig_dump_dot(dmuid,filename): igm = project.getIGM() uid = DMUID.parse(dmuid) module = igm.findModule(Toplevel(uid, None)) if module == None: printf('DM %s not found', dmuid) else: printf('Dumping %s...', module) dot = IG2DOT(module); dot.blacklistField("fImportedLibs"); dot.blacklistField("fImportedPackages"); dot.blacklistField("fZPrjID"); dot.blacklistField("fSFDBID"); dot.blacklistField("fLine"); dot.blacklistField("fCol"); dot.blacklistField("fScore"); dot.blacklistField("fFailed"); dot.blacklistField("fReject"); dot.blacklistField("fInertial"); dot.blacklistField("fDelay"); out = PrintWriter(BufferedWriter(FileWriter(filename))); dot.convert(out); out.close() printf("python: wrote dot file to %s", filename)
def testCSVPipe(self): """testing the CSV pipe""" from java.io import PrintWriter, FileWriter from com.ziclix.python.sql.pipe import Pipe from com.ziclix.python.sql.pipe.db import DBSource from com.ziclix.python.sql.pipe.csv import CSVSink try: src = self.connect() fn = tempfile.mktemp(suffix="csv") writer = PrintWriter(FileWriter(fn)) csvSink = CSVSink(writer) c = self.cursor() try: c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1000, 'this,has,a,comma', 'and a " quote')]) c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1001, 'this,has,a,comma and a "', 'and a " quote')]) # ORACLE has a problem calling stmt.setObject(index, null) c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1010, '"this,has,a,comma"', None)], {2:zxJDBC.VARCHAR}) self.db.commit() finally: self.db.rollback() c.close() dbSource = DBSource(src, c.datahandler.__class__, "zxtesting", None, None, None) cnt = Pipe().pipe(dbSource, csvSink) - 1 # ignore the header row finally: writer.close() src.close() os.remove(fn)
def register_script(self): """ Registers a pig scripts with its variables substituted. raises: IOException If a temp file containing the pig script could not be created. raises: ParseException The pig script could not have all its variables substituted. todo: Refactor this processes that result in calling this method. This method gets called twice for a single assert as every method that needs the data assumes no one else has called it (even methods that call other methods that call it (assertOutput() calls get_alias() which both call this method). """ pigIStream = BufferedReader(StringReader(self.orig_pig_code)) pigOStream = StringWriter() ps = ParameterSubstitutionPreprocessor(50) # Where does 50 come from? ps.genSubstitutedFile(pigIStream, pigOStream, self.args, self.arg_files) substitutedPig = pigOStream.toString() f = File.createTempFile("tmp", "pigunit") pw = PrintWriter(f) pw.println(substitutedPig) pw.close() pigSubstitutedFile = f.getCanonicalPath() self._temp_pig_script = pigSubstitutedFile self.pig.registerScript(pigSubstitutedFile, self.alias_overrides)
def rtl_dump_svg(dmuid, filename): rtlmanager = project.getRTLM() uid = DMUID.parse(dmuid) rtlm = rtlmanager.findModule(Toplevel(uid, None)) if rtlm == None: printf('RTLM %s not found', dmuid) else: printf('Dumping %s...', rtlm) out = PrintWriter(BufferedWriter(FileWriter(filename))) gc = VGGCSVG(out) contentProvider = RTLVisualGraphContentProvider(rtlm) labelProvider = RTLVisualGraphLabelProvider(rtlm) selectionProvider = RTLVisualGraphSelectionProvider() layout = VGLayout(contentProvider, labelProvider, gc) layout.paint(selectionProvider) out.close() printf("python: wrote svg file to %s", filename)
def createInputFile(self, fs, fileName, input_data): if(fs.exists(Path(fileName))): raise IOException("File " + fileName + " already exists on the minicluster") stream = fs.create(Path(fileName)) pw = PrintWriter(OutputStreamWriter(stream, "UTF-8")) for i in xrange(len(input_data)): pw.println(input_data[i]) pw.close();
def createInputFile(self, fs, fileName, input_data): if (fs.exists(Path(fileName))): raise IOException("File " + fileName + " already exists on the minicluster") stream = fs.create(Path(fileName)) pw = PrintWriter(OutputStreamWriter(stream, "UTF-8")) for i in xrange(len(input_data)): pw.println(input_data[i]) pw.close()
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path): write_str = "" file_reader = FileReader(svg_input_path) buffered_reader = BufferedReader(file_reader) read_line = "" check = False while True: read_line = buffered_reader.readLine() if read_line is None: break if "viewBox" in read_line: view_box_content = _get_viewbox_content(read_line) view_box_values = _get_viewbox_values(view_box_content) if view_box_values[0] != 0: view_box_values[2] = abs(view_box_values[2]) + abs( view_box_values[0]) view_box_values[0] = 0 if view_box_values[1] != 0: view_box_values[3] = abs(view_box_values[3]) + abs( view_box_values[1]) view_box_values[1] = 0 read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "", read_line, 1) read_line = re.sub(r"width=\"[0-9]+\"", "width=\"" + str(view_box_values[2]) + "\"", read_line, 1) read_line = re.sub(r"height=\"[0-9]+\"", "height=\"" + str(view_box_values[3]) + "\"", read_line, 1) check = True if "g id=\"ID" in read_line and not check: if "transform=" in read_line: _log.info(read_line) read_line = read_line[0:read_line.find("transform")] + ">" check = True write_str += read_line + "\n" buffered_reader.close() file_reader.close() file_writer = PrintWriter(svg_output_path) file_writer.print(write_str) file_writer.close()
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path): write_str = "" file_reader = FileReader(svg_input_path) buffered_reader = BufferedReader(file_reader) read_line = "" check = False while True: read_line = buffered_reader.readLine() if read_line is None: break if "viewBox" in read_line: view_box_content = _get_viewbox_content(read_line) view_box_values = _get_viewbox_values(view_box_content) if view_box_values[0] != 0: view_box_values[2] = abs(view_box_values[2]) + abs(view_box_values[0]) view_box_values[0] = 0 if view_box_values[1] != 0: view_box_values[3] = abs(view_box_values[3]) + abs(view_box_values[1]) view_box_values[1] = 0 read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "", read_line, 1) read_line = re.sub(r"width=\"[0-9]+\"", "width=\""+ str(view_box_values[2]) + "\"", read_line, 1) read_line = re.sub(r"height=\"[0-9]+\"", "height=\""+ str(view_box_values[3]) + "\"", read_line, 1) check = True if "g id=\"ID" in read_line and not check: if "transform=" in read_line: _log.info(read_line) read_line = read_line[0:read_line.find("transform")] + ">" check = True write_str += read_line + "\n" buffered_reader.close() file_reader.close() file_writer = PrintWriter(svg_output_path) file_writer.print(write_str) file_writer.close()
def _testXMLPipe(self): """testing the XML pipe""" from java.io import PrintWriter, FileWriter from com.ziclix.python.sql.pipe import Pipe from com.ziclix.python.sql.pipe.db import DBSource from com.ziclix.python.sql.pipe.xml import XMLSink try: src = self.connect() fn = tempfile.mktemp(suffix="csv") writer = PrintWriter(FileWriter(fn)) xmlSink = XMLSink(writer) dbSource = DBSource(src, self.datahandler, "zxtesting", None, None, None) cnt = Pipe().pipe(dbSource, xmlSink) - 1 # ignore the header row finally: writer.close() src.close() os.remove(fn)
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path): write_str = "" file_reader = FileReader(svg_input_path) buffered_reader = BufferedReader(file_reader) read_line = "" while True: read_line = buffered_reader.readLine() if read_line is None: break if "viewBox" in read_line: view_box_content = _get_viewbox_content(read_line) view_box_values = _get_viewbox_values(view_box_content) if view_box_values[0] != 0: view_box_values[2] += view_box_values[0] view_box_values[0] = 0 if view_box_values[1] != 0: view_box_values[3] += view_box_values[1] view_box_values[1] = 0 new_view_box = str(view_box_values[0]) + " " + str(view_box_values[1]) + " " + \ str(view_box_values[2]) + " " + str(view_box_values[3]) read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "viewBox=\"" + new_view_box + "\"", read_line, 1) read_line = re.sub(r"width=\"[0-9]+\"", "width=\""+ str(view_box_values[2]) + "\"", read_line, 1) read_line = re.sub(r"height=\"[0-9]+\"", "height=\""+ str(view_box_values[3]) + "\"", read_line, 1) write_str += read_line + "\n" buffered_reader.close() file_reader.close() file_writer = PrintWriter(svg_output_path) file_writer.print(write_str) file_writer.close()
def _testXMLPipe(self): """testing the XML pipe""" from java.io import PrintWriter, FileWriter from com.ziclix.python.sql.pipe import Pipe from com.ziclix.python.sql.pipe.db import DBSource from com.ziclix.python.sql.pipe.xml import XMLSink try: src = self.connect() fn = tempfile.mktemp(suffix="csv") writer = PrintWriter(FileWriter(fn)) xmlSink = XMLSink(writer) dbSource = DBSource(src, self.datahandler, "zxtesting", None, None, None) cnt = Pipe().pipe(dbSource, xmlSink) - 1 # ignore the header row finally: writer.close() src.close() os.remove(fn)
def main(): """ The main entry point for the discoverDomain tool. :param args: the command-line arguments """ _method_name = 'main' _logger.entering(class_name=_class_name, method_name=_method_name) for index, arg in enumerate(sys.argv): _logger.finer('sys.argv[{0}] = {1}', str(index), str(arg), class_name=_class_name, method_name=_method_name) _outputdir = None try: model_context = __process_args(sys.argv) _outputdir = model_context.get_compare_model_output_dir() model1 = model_context.get_trailing_argument(0) model2 = model_context.get_trailing_argument(1) for f in [model1, model2]: if not os.path.exists(f): raise CLAException("Model %s does not exists" % f) if os.path.isdir(f): raise CLAException("Model %s is a directory" % f) model1_file = JFile(model1) model2_file = JFile(model2) if not (FileUtils.isYamlFile(model1_file) or FileUtils.isJsonFile(model1_file)): raise CLAException("Model extension must be either yaml or json") if not (FileUtils.isYamlFile(model1_file) and FileUtils.isYamlFile(model2_file) or FileUtils.isJsonFile(model1_file) and FileUtils.isJsonFile(model2_file)): ext = os.path.splitext(model1)[1] raise CLAException("Model %s is not a %s file " % (model2, ext)) obj = ModelFileDiffer(model1, model2, model_context, _outputdir) rc = obj.compare() if rc == VALIDATION_FAIL: System.exit(2) if _outputdir: fos = None writer = None file_name = None if len(compare_msgs) > 0: try: file_name = _outputdir + '/compare_model_stdout' fos = JFileOutputStream(file_name, False) writer = JPrintWriter(fos, True) writer.println(BLANK_LINE) writer.println(BLANK_LINE) index = 1 for line in compare_msgs: msg_key = line[0] msg_value = line[1] writer.println("%s. %s" % (index, format_message(msg_key,msg_value.replace(PATH_TOKEN, "-->")))) index = index + 1 writer.println(BLANK_LINE) fos.close() writer.close() except JIOException, ioe: if fos: fos.close() if writer: writer.close() _logger.severe('WLSDPLY-05708', file_name, ioe.getLocalizedMessage(), error=ioe, class_name=_class_name, method_name=_method_name) else:
print BLANK_LINE return 0 if self.output_dir: fos = None writer = None file_name = None try: print format_message('WLSDPLY-05711', self.output_dir) print BLANK_LINE file_name = self.output_dir + '/diffed_model.json' fos = JFileOutputStream(file_name, False) writer = JPrintWriter(fos, True) json_object = PythonToJson(net_diff) json_object._write_dictionary_to_json_file(net_diff, writer) writer.close() file_name = self.output_dir + '/diffed_model.yaml' fos = JFileOutputStream(file_name, False) writer = JPrintWriter(fos, True) pty = PythonToYaml(net_diff) pty._write_dictionary_to_yaml_file(net_diff, writer) writer.close() except JIOException, ioe: if fos: fos.close() if writer: writer.close() _logger.severe('WLSDPLY-05708', file_name, ioe.getLocalizedMessage(), error=ioe, class_name=_class_name, method_name=_method_name) return 2 else:
""" download_and_save_csv.py From: http://wiki.cmci.info/documents/120206pyip_cooking/python_imagej_cookbook """ from ij import IJ from java.io import PrintWriter content = IJ.openUrlAsString('http://cmci.info/imgdata/tenFrameResults.csv') out = PrintWriter('/Users/jrminter/tmp/test1.csv') out.print(content) out.close()
def walk(self): _method_name = "walk" model_file_name = None try: model_file_list = self.model_files.split(',') for model_file in model_file_list: self.cache.clear() if os.path.splitext(model_file)[1].lower() == ".yaml": model_file_name = model_file FileToPython(model_file_name, True).parse() aliases = Aliases(model_context=self.model_context, wlst_mode=WlstModes.OFFLINE) validator = Validator(self.model_context, aliases, wlst_mode=WlstModes.OFFLINE) # Just merge and validate but without substitution model_dictionary = cla_helper.merge_model_files(model_file_name, None) variable_file = self.model_context.get_variable_file() if not os.path.exists(variable_file): variable_file=None return_code = validator.validate_in_tool_mode(model_dictionary, variables_file_name=variable_file, archive_file_name=None) if return_code == Validator.ReturnCode.STOP: __logger.severe('WLSDPLY-05705', model_file_name) return VALIDATION_FAIL self.current_dict = model_dictionary self.__walk_model_section(model.get_model_domain_info_key(), self.current_dict, aliases.get_model_section_top_level_folder_names(DOMAIN_INFO)) self.__walk_model_section(model.get_model_topology_key(), self.current_dict, aliases.get_model_topology_top_level_folder_names()) self.__walk_model_section(model.get_model_resources_key(), self.current_dict, aliases.get_model_resources_top_level_folder_names()) self.current_dict = self._apply_filter_and_inject_variable(self.current_dict, self.model_context, validator) file_name = os.path.join(self.output_dir, os.path.basename(model_file_name)) fos = JFileOutputStream(file_name, False) writer = JPrintWriter(fos, True) pty = PythonToYaml(self.current_dict) pty._write_dictionary_to_yaml_file(self.current_dict, writer) writer.close() self.cache.clear() for key in self.secrets_to_generate: self.cache[key] = '' target_configuration_helper.generate_k8s_script(self.model_context.get_kubernetes_variable_file(), self.cache) except ValidateException, te: __logger.severe('WLSDPLY-20009', _program_name, model_file_name, te.getLocalizedMessage(), error=te, class_name=_class_name, method_name=_method_name) ex = exception_helper.create_compare_exception(te.getLocalizedMessage(), error=te) __logger.throwing(ex, class_name=_class_name, method_name=_method_name) return VALIDATION_FAIL
def walk(self): """ Replace password attributes in each model file with secret tokens, and write each model. Generate a script to create the required secrets. Create any additional output specified for the target environment. """ _method_name = "walk" model_file_name = None try: model_file_list = self.model_files.split(',') for model_file in model_file_list: self.cache.clear() if os.path.splitext(model_file)[1].lower() == ".yaml": model_file_name = model_file FileToPython(model_file_name, True).parse() aliases = Aliases(model_context=self.model_context, wlst_mode=WlstModes.OFFLINE) validator = Validator(self.model_context, aliases, wlst_mode=WlstModes.OFFLINE) # Just merge and validate but without substitution model_dictionary = cla_helper.merge_model_files( model_file_name, None) variable_file = self.model_context.get_variable_file() if not os.path.exists(variable_file): variable_file = None return_code = validator.validate_in_tool_mode( model_dictionary, variables_file_name=variable_file, archive_file_name=None) if return_code == Validator.ReturnCode.STOP: self._logger.severe('WLSDPLY-05705', model_file_name) return VALIDATION_FAIL self.current_dict = model_dictionary self.__walk_model_section( model.get_model_domain_info_key(), self.current_dict, aliases.get_model_section_top_level_folder_names( DOMAIN_INFO)) self.__walk_model_section( model.get_model_topology_key(), self.current_dict, aliases.get_model_topology_top_level_folder_names()) self.__walk_model_section( model.get_model_resources_key(), self.current_dict, aliases.get_model_resources_top_level_folder_names()) self.current_dict = self._apply_filter_and_inject_variable( self.current_dict, self.model_context, validator) file_name = os.path.join(self.output_dir, os.path.basename(model_file_name)) fos = JFileOutputStream(file_name, False) writer = JPrintWriter(fos, True) pty = PythonToYaml(self.current_dict) pty._write_dictionary_to_yaml_file(self.current_dict, writer) writer.close() self.cache.clear() for key in self.secrets_to_generate: self.cache[key] = '' # use a merged, substituted, filtered model to get domain name and create additional target output. full_model_dictionary = cla_helper.load_model( _program_name, self.model_context, self._aliases, "discover", WlstModes.OFFLINE) target_configuration_helper.generate_k8s_script( self.model_context, self.cache, full_model_dictionary) # create any additional outputs from full model dictionary target_configuration_helper.create_additional_output( Model(full_model_dictionary), self.model_context, self._aliases, ExceptionType.VALIDATE) except ValidateException, te: self._logger.severe('WLSDPLY-20009', _program_name, model_file_name, te.getLocalizedMessage(), error=te, class_name=_class_name, method_name=_method_name) ex = exception_helper.create_compare_exception( te.getLocalizedMessage(), error=te) self._logger.throwing(ex, class_name=_class_name, method_name=_method_name) return VALIDATION_FAIL