def register_script(self):
"""
Registers a pig scripts with its variables substituted.
raises: IOException If a temp file containing the pig script could not be created.
raises: ParseException The pig script could not have all its variables substituted.
todo: Refactor this processes that result in calling this method. This method gets
called twice for a single assert as every method that needs the data assumes no one
else has called it (even methods that call other methods that call it (assertOutput()
calls get_alias() which both call this method).
"""
pigIStream = BufferedReader(StringReader(self.orig_pig_code))
pigOStream = StringWriter()
ps = ParameterSubstitutionPreprocessor(50) # Where does 50 come from?
ps.genSubstitutedFile(pigIStream, pigOStream, self.args,
self.arg_files)
substitutedPig = pigOStream.toString()
f = File.createTempFile("tmp", "pigunit")
pw = PrintWriter(f)
pw.println(substitutedPig)
pw.close()
pigSubstitutedFile = f.getCanonicalPath()
self._temp_pig_script = pigSubstitutedFile
self.pig.registerScript(pigSubstitutedFile, self.alias_overrides)
def dm_dump_dot(dmuid, filename):
dmm = project.getDUM()
uid = DMUID.parse(dmuid)
dm = dmm.getDM(uid)
if dm == None:
printf('DM %s not found', dmuid)
else:
printf('Dumping %s...', dm)
dot = AST2DOT(dm, project.getZDB())
dot.blacklistField("fParent")
# dot.blacklistField("fSource")
# dot.blacklistField("fStartCol")
# dot.blacklistField("fStartLine")
dot.blacklistField("fEndCol")
dot.blacklistField("fEndLine")
dot.blacklistField("fDeclarationMap")
out = PrintWriter(BufferedWriter(FileWriter(filename)))
dot.convert(out)
out.close()
printf("python: wrote dot file to %s", filename)
def ig_dump_dot(dmuid, filename):
igm = project.getIGM()
uid = DMUID.parse(dmuid)
module = igm.findModule(Toplevel(uid, None))
if module == None:
printf('DM %s not found', dmuid)
else:
printf('Dumping %s...', module)
dot = IG2DOT(module)
dot.blacklistField("fImportedLibs")
dot.blacklistField("fImportedPackages")
dot.blacklistField("fZPrjID")
dot.blacklistField("fSFDBID")
dot.blacklistField("fLine")
dot.blacklistField("fCol")
dot.blacklistField("fScore")
dot.blacklistField("fFailed")
dot.blacklistField("fReject")
dot.blacklistField("fInertial")
dot.blacklistField("fDelay")
out = PrintWriter(BufferedWriter(FileWriter(filename)))
dot.convert(out)
out.close()
printf("python: wrote dot file to %s", filename)
def rtl_dump_svg(dmuid,filename):
rtlmanager = project.getRTLM()
uid = DMUID.parse(dmuid)
rtlm = rtlmanager.findModule(Toplevel(uid, None))
if rtlm == None:
printf('RTLM %s not found', dmuid)
else:
printf('Dumping %s...', rtlm)
out = PrintWriter(BufferedWriter(FileWriter(filename)))
gc = VGGCSVG(out)
contentProvider = RTLVisualGraphContentProvider(rtlm)
labelProvider = RTLVisualGraphLabelProvider(rtlm)
selectionProvider = RTLVisualGraphSelectionProvider()
layout = VGLayout(contentProvider, labelProvider, gc)
layout.paint(selectionProvider);
out.close()
printf("python: wrote svg file to %s", filename)
def write_ordered_variables(program_name,
variable_map,
file_path,
append=False):
"""
Write variables to file while preserving order of the variables.
:param program_name: name of the calling program
:param variable_map: map or variable properties to write to file
:param file_path: the file to which to write the properties
:param append: defaults to False. Append properties to the end of file
:raises VariableException if an error occurs while storing the variables in the file
"""
_method_name = 'write_ordered_variables'
_logger.entering(program_name,
file_path,
append,
class_name=_class_name,
method_name=_method_name)
pw = None
try:
pw = PrintWriter(FileOutputStream(File(file_path), Boolean(append)),
Boolean('true'))
for key, value in variable_map.iteritems():
formatted = '%s=%s' % (key, value)
pw.println(formatted)
pw.close()
except IOException, ioe:
_logger.fine('WLSDPLY-20007', file_path, ioe.getLocalizedMessage())
ex = exception_helper.create_variable_exception(
'WLSDPLY-20007', file_path, ioe.getLocalizedMessage(), error=ioe)
_logger.throwing(ex, class_name=_class_name, method_name=_method_name)
if pw is not None:
pw.close()
raise ex
def dm_dump_dot(dmuid,filename):
dmm = project.getDUM()
uid = DMUID.parse(dmuid)
dm = dmm.getDM(uid)
if dm == None:
printf('DM %s not found', dmuid)
else:
printf('Dumping %s...', dm)
dot = AST2DOT(dm, project.getZDB())
dot.blacklistField("fParent")
# dot.blacklistField("fSource")
# dot.blacklistField("fStartCol")
# dot.blacklistField("fStartLine")
dot.blacklistField("fEndCol")
dot.blacklistField("fEndLine")
dot.blacklistField("fDeclarationMap")
out = PrintWriter(BufferedWriter(FileWriter(filename)));
dot.convert(out);
out.close()
printf("python: wrote dot file to %s", filename)
def ig_dump_dot(dmuid,filename):
igm = project.getIGM()
uid = DMUID.parse(dmuid)
module = igm.findModule(Toplevel(uid, None))
if module == None:
printf('DM %s not found', dmuid)
else:
printf('Dumping %s...', module)
dot = IG2DOT(module);
dot.blacklistField("fImportedLibs");
dot.blacklistField("fImportedPackages");
dot.blacklistField("fZPrjID");
dot.blacklistField("fSFDBID");
dot.blacklistField("fLine");
dot.blacklistField("fCol");
dot.blacklistField("fScore");
dot.blacklistField("fFailed");
dot.blacklistField("fReject");
dot.blacklistField("fInertial");
dot.blacklistField("fDelay");
out = PrintWriter(BufferedWriter(FileWriter(filename)));
dot.convert(out);
out.close()
printf("python: wrote dot file to %s", filename)
def testCSVPipe(self):
"""testing the CSV pipe"""
from java.io import PrintWriter, FileWriter
from com.ziclix.python.sql.pipe import Pipe
from com.ziclix.python.sql.pipe.db import DBSource
from com.ziclix.python.sql.pipe.csv import CSVSink
try:
src = self.connect()
fn = tempfile.mktemp(suffix="csv")
writer = PrintWriter(FileWriter(fn))
csvSink = CSVSink(writer)
c = self.cursor()
try:
c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1000, 'this,has,a,comma', 'and a " quote')])
c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1001, 'this,has,a,comma and a "', 'and a " quote')])
# ORACLE has a problem calling stmt.setObject(index, null)
c.execute("insert into zxtesting (id, name, state) values (?, ?, ?)", [(1010, '"this,has,a,comma"', None)], {2:zxJDBC.VARCHAR})
self.db.commit()
finally:
self.db.rollback()
c.close()
dbSource = DBSource(src, c.datahandler.__class__, "zxtesting", None, None, None)
cnt = Pipe().pipe(dbSource, csvSink) - 1 # ignore the header row
finally:
writer.close()
src.close()
os.remove(fn)
def register_script(self):
"""
Registers a pig scripts with its variables substituted.
raises: IOException If a temp file containing the pig script could not be created.
raises: ParseException The pig script could not have all its variables substituted.
todo: Refactor this processes that result in calling this method. This method gets
called twice for a single assert as every method that needs the data assumes no one
else has called it (even methods that call other methods that call it (assertOutput()
calls get_alias() which both call this method).
"""
pigIStream = BufferedReader(StringReader(self.orig_pig_code))
pigOStream = StringWriter()
ps = ParameterSubstitutionPreprocessor(50) # Where does 50 come from?
ps.genSubstitutedFile(pigIStream, pigOStream, self.args, self.arg_files)
substitutedPig = pigOStream.toString()
f = File.createTempFile("tmp", "pigunit")
pw = PrintWriter(f)
pw.println(substitutedPig)
pw.close()
pigSubstitutedFile = f.getCanonicalPath()
self._temp_pig_script = pigSubstitutedFile
self.pig.registerScript(pigSubstitutedFile, self.alias_overrides)
def rtl_dump_svg(dmuid, filename):
rtlmanager = project.getRTLM()
uid = DMUID.parse(dmuid)
rtlm = rtlmanager.findModule(Toplevel(uid, None))
if rtlm == None:
printf('RTLM %s not found', dmuid)
else:
printf('Dumping %s...', rtlm)
out = PrintWriter(BufferedWriter(FileWriter(filename)))
gc = VGGCSVG(out)
contentProvider = RTLVisualGraphContentProvider(rtlm)
labelProvider = RTLVisualGraphLabelProvider(rtlm)
selectionProvider = RTLVisualGraphSelectionProvider()
layout = VGLayout(contentProvider, labelProvider, gc)
layout.paint(selectionProvider)
out.close()
printf("python: wrote svg file to %s", filename)
def createInputFile(self, fs, fileName, input_data):
if(fs.exists(Path(fileName))):
raise IOException("File " + fileName + " already exists on the minicluster")
stream = fs.create(Path(fileName))
pw = PrintWriter(OutputStreamWriter(stream, "UTF-8"))
for i in xrange(len(input_data)):
pw.println(input_data[i])
pw.close();
def createInputFile(self, fs, fileName, input_data):
if (fs.exists(Path(fileName))):
raise IOException("File " + fileName +
" already exists on the minicluster")
stream = fs.create(Path(fileName))
pw = PrintWriter(OutputStreamWriter(stream, "UTF-8"))
for i in xrange(len(input_data)):
pw.println(input_data[i])
pw.close()
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path):
write_str = ""
file_reader = FileReader(svg_input_path)
buffered_reader = BufferedReader(file_reader)
read_line = ""
check = False
while True:
read_line = buffered_reader.readLine()
if read_line is None:
break
if "viewBox" in read_line:
view_box_content = _get_viewbox_content(read_line)
view_box_values = _get_viewbox_values(view_box_content)
if view_box_values[0] != 0:
view_box_values[2] = abs(view_box_values[2]) + abs(
view_box_values[0])
view_box_values[0] = 0
if view_box_values[1] != 0:
view_box_values[3] = abs(view_box_values[3]) + abs(
view_box_values[1])
view_box_values[1] = 0
read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "", read_line, 1)
read_line = re.sub(r"width=\"[0-9]+\"",
"width=\"" + str(view_box_values[2]) + "\"",
read_line, 1)
read_line = re.sub(r"height=\"[0-9]+\"",
"height=\"" + str(view_box_values[3]) + "\"",
read_line, 1)
check = True
if "g id=\"ID" in read_line and not check:
if "transform=" in read_line:
_log.info(read_line)
read_line = read_line[0:read_line.find("transform")] + ">"
check = True
write_str += read_line + "\n"
buffered_reader.close()
file_reader.close()
file_writer = PrintWriter(svg_output_path)
file_writer.print(write_str)
file_writer.close()
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path):
write_str = ""
file_reader = FileReader(svg_input_path)
buffered_reader = BufferedReader(file_reader)
read_line = ""
check = False
while True:
read_line = buffered_reader.readLine()
if read_line is None:
break
if "viewBox" in read_line:
view_box_content = _get_viewbox_content(read_line)
view_box_values = _get_viewbox_values(view_box_content)
if view_box_values[0] != 0:
view_box_values[2] = abs(view_box_values[2]) + abs(view_box_values[0])
view_box_values[0] = 0
if view_box_values[1] != 0:
view_box_values[3] = abs(view_box_values[3]) + abs(view_box_values[1])
view_box_values[1] = 0
read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "", read_line, 1)
read_line = re.sub(r"width=\"[0-9]+\"", "width=\""+ str(view_box_values[2]) + "\"",
read_line, 1)
read_line = re.sub(r"height=\"[0-9]+\"", "height=\""+ str(view_box_values[3]) + "\"",
read_line, 1)
check = True
if "g id=\"ID" in read_line and not check:
if "transform=" in read_line:
_log.info(read_line)
read_line = read_line[0:read_line.find("transform")] + ">"
check = True
write_str += read_line + "\n"
buffered_reader.close()
file_reader.close()
file_writer = PrintWriter(svg_output_path)
file_writer.print(write_str)
file_writer.close()
def _testXMLPipe(self): """testing the XML pipe""" from java.io import PrintWriter, FileWriter from com.ziclix.python.sql.pipe import Pipe from com.ziclix.python.sql.pipe.db import DBSource from com.ziclix.python.sql.pipe.xml import XMLSink try: src = self.connect() fn = tempfile.mktemp(suffix="csv") writer = PrintWriter(FileWriter(fn)) xmlSink = XMLSink(writer) dbSource = DBSource(src, self.datahandler, "zxtesting", None, None, None) cnt = Pipe().pipe(dbSource, xmlSink) - 1 # ignore the header row finally: writer.close() src.close() os.remove(fn)
def _parse_and_rewrite_svg_file(svg_input_path, svg_output_path):
write_str = ""
file_reader = FileReader(svg_input_path)
buffered_reader = BufferedReader(file_reader)
read_line = ""
while True:
read_line = buffered_reader.readLine()
if read_line is None:
break
if "viewBox" in read_line:
view_box_content = _get_viewbox_content(read_line)
view_box_values = _get_viewbox_values(view_box_content)
if view_box_values[0] != 0:
view_box_values[2] += view_box_values[0]
view_box_values[0] = 0
if view_box_values[1] != 0:
view_box_values[3] += view_box_values[1]
view_box_values[1] = 0
new_view_box = str(view_box_values[0]) + " " + str(view_box_values[1]) + " " + \
str(view_box_values[2]) + " " + str(view_box_values[3])
read_line = re.sub(r"viewBox=\"[\-|0-9| ]+\"", "viewBox=\""
+ new_view_box + "\"", read_line, 1)
read_line = re.sub(r"width=\"[0-9]+\"", "width=\""+ str(view_box_values[2]) + "\"",
read_line, 1)
read_line = re.sub(r"height=\"[0-9]+\"", "height=\""+ str(view_box_values[3]) + "\"",
read_line, 1)
write_str += read_line + "\n"
buffered_reader.close()
file_reader.close()
file_writer = PrintWriter(svg_output_path)
file_writer.print(write_str)
file_writer.close()
def _testXMLPipe(self):
"""testing the XML pipe"""
from java.io import PrintWriter, FileWriter
from com.ziclix.python.sql.pipe import Pipe
from com.ziclix.python.sql.pipe.db import DBSource
from com.ziclix.python.sql.pipe.xml import XMLSink
try:
src = self.connect()
fn = tempfile.mktemp(suffix="csv")
writer = PrintWriter(FileWriter(fn))
xmlSink = XMLSink(writer)
dbSource = DBSource(src, self.datahandler, "zxtesting", None, None,
None)
cnt = Pipe().pipe(dbSource, xmlSink) - 1 # ignore the header row
finally:
writer.close()
src.close()
os.remove(fn)
def main():
"""
The main entry point for the discoverDomain tool.
:param args: the command-line arguments
"""
_method_name = 'main'
_logger.entering(class_name=_class_name, method_name=_method_name)
for index, arg in enumerate(sys.argv):
_logger.finer('sys.argv[{0}] = {1}', str(index), str(arg), class_name=_class_name, method_name=_method_name)
_outputdir = None
try:
model_context = __process_args(sys.argv)
_outputdir = model_context.get_compare_model_output_dir()
model1 = model_context.get_trailing_argument(0)
model2 = model_context.get_trailing_argument(1)
for f in [model1, model2]:
if not os.path.exists(f):
raise CLAException("Model %s does not exists" % f)
if os.path.isdir(f):
raise CLAException("Model %s is a directory" % f)
model1_file = JFile(model1)
model2_file = JFile(model2)
if not (FileUtils.isYamlFile(model1_file) or FileUtils.isJsonFile(model1_file)):
raise CLAException("Model extension must be either yaml or json")
if not (FileUtils.isYamlFile(model1_file) and FileUtils.isYamlFile(model2_file)
or FileUtils.isJsonFile(model1_file) and FileUtils.isJsonFile(model2_file)):
ext = os.path.splitext(model1)[1]
raise CLAException("Model %s is not a %s file " % (model2, ext))
obj = ModelFileDiffer(model1, model2, model_context, _outputdir)
rc = obj.compare()
if rc == VALIDATION_FAIL:
System.exit(2)
if _outputdir:
fos = None
writer = None
file_name = None
if len(compare_msgs) > 0:
try:
file_name = _outputdir + '/compare_model_stdout'
fos = JFileOutputStream(file_name, False)
writer = JPrintWriter(fos, True)
writer.println(BLANK_LINE)
writer.println(BLANK_LINE)
index = 1
for line in compare_msgs:
msg_key = line[0]
msg_value = line[1]
writer.println("%s. %s" % (index, format_message(msg_key,msg_value.replace(PATH_TOKEN, "-->"))))
index = index + 1
writer.println(BLANK_LINE)
fos.close()
writer.close()
except JIOException, ioe:
if fos:
fos.close()
if writer:
writer.close()
_logger.severe('WLSDPLY-05708', file_name, ioe.getLocalizedMessage(),
error=ioe, class_name=_class_name, method_name=_method_name)
else:
print BLANK_LINE
return 0
if self.output_dir:
fos = None
writer = None
file_name = None
try:
print format_message('WLSDPLY-05711', self.output_dir)
print BLANK_LINE
file_name = self.output_dir + '/diffed_model.json'
fos = JFileOutputStream(file_name, False)
writer = JPrintWriter(fos, True)
json_object = PythonToJson(net_diff)
json_object._write_dictionary_to_json_file(net_diff, writer)
writer.close()
file_name = self.output_dir + '/diffed_model.yaml'
fos = JFileOutputStream(file_name, False)
writer = JPrintWriter(fos, True)
pty = PythonToYaml(net_diff)
pty._write_dictionary_to_yaml_file(net_diff, writer)
writer.close()
except JIOException, ioe:
if fos:
fos.close()
if writer:
writer.close()
_logger.severe('WLSDPLY-05708', file_name, ioe.getLocalizedMessage(),
error=ioe, class_name=_class_name, method_name=_method_name)
return 2
else:
"""
download_and_save_csv.py
From:
http://wiki.cmci.info/documents/120206pyip_cooking/python_imagej_cookbook
"""
from ij import IJ
from java.io import PrintWriter
content = IJ.openUrlAsString('http://cmci.info/imgdata/tenFrameResults.csv')
out = PrintWriter('/Users/jrminter/tmp/test1.csv')
out.print(content)
out.close()
def walk(self):
_method_name = "walk"
model_file_name = None
try:
model_file_list = self.model_files.split(',')
for model_file in model_file_list:
self.cache.clear()
if os.path.splitext(model_file)[1].lower() == ".yaml":
model_file_name = model_file
FileToPython(model_file_name, True).parse()
aliases = Aliases(model_context=self.model_context, wlst_mode=WlstModes.OFFLINE)
validator = Validator(self.model_context, aliases, wlst_mode=WlstModes.OFFLINE)
# Just merge and validate but without substitution
model_dictionary = cla_helper.merge_model_files(model_file_name, None)
variable_file = self.model_context.get_variable_file()
if not os.path.exists(variable_file):
variable_file=None
return_code = validator.validate_in_tool_mode(model_dictionary,
variables_file_name=variable_file,
archive_file_name=None)
if return_code == Validator.ReturnCode.STOP:
__logger.severe('WLSDPLY-05705', model_file_name)
return VALIDATION_FAIL
self.current_dict = model_dictionary
self.__walk_model_section(model.get_model_domain_info_key(), self.current_dict,
aliases.get_model_section_top_level_folder_names(DOMAIN_INFO))
self.__walk_model_section(model.get_model_topology_key(), self.current_dict,
aliases.get_model_topology_top_level_folder_names())
self.__walk_model_section(model.get_model_resources_key(), self.current_dict,
aliases.get_model_resources_top_level_folder_names())
self.current_dict = self._apply_filter_and_inject_variable(self.current_dict, self.model_context,
validator)
file_name = os.path.join(self.output_dir, os.path.basename(model_file_name))
fos = JFileOutputStream(file_name, False)
writer = JPrintWriter(fos, True)
pty = PythonToYaml(self.current_dict)
pty._write_dictionary_to_yaml_file(self.current_dict, writer)
writer.close()
self.cache.clear()
for key in self.secrets_to_generate:
self.cache[key] = ''
target_configuration_helper.generate_k8s_script(self.model_context.get_kubernetes_variable_file(),
self.cache)
except ValidateException, te:
__logger.severe('WLSDPLY-20009', _program_name, model_file_name, te.getLocalizedMessage(),
error=te, class_name=_class_name, method_name=_method_name)
ex = exception_helper.create_compare_exception(te.getLocalizedMessage(), error=te)
__logger.throwing(ex, class_name=_class_name, method_name=_method_name)
return VALIDATION_FAIL
def walk(self):
"""
Replace password attributes in each model file with secret tokens, and write each model.
Generate a script to create the required secrets.
Create any additional output specified for the target environment.
"""
_method_name = "walk"
model_file_name = None
try:
model_file_list = self.model_files.split(',')
for model_file in model_file_list:
self.cache.clear()
if os.path.splitext(model_file)[1].lower() == ".yaml":
model_file_name = model_file
FileToPython(model_file_name, True).parse()
aliases = Aliases(model_context=self.model_context,
wlst_mode=WlstModes.OFFLINE)
validator = Validator(self.model_context,
aliases,
wlst_mode=WlstModes.OFFLINE)
# Just merge and validate but without substitution
model_dictionary = cla_helper.merge_model_files(
model_file_name, None)
variable_file = self.model_context.get_variable_file()
if not os.path.exists(variable_file):
variable_file = None
return_code = validator.validate_in_tool_mode(
model_dictionary,
variables_file_name=variable_file,
archive_file_name=None)
if return_code == Validator.ReturnCode.STOP:
self._logger.severe('WLSDPLY-05705', model_file_name)
return VALIDATION_FAIL
self.current_dict = model_dictionary
self.__walk_model_section(
model.get_model_domain_info_key(), self.current_dict,
aliases.get_model_section_top_level_folder_names(
DOMAIN_INFO))
self.__walk_model_section(
model.get_model_topology_key(), self.current_dict,
aliases.get_model_topology_top_level_folder_names())
self.__walk_model_section(
model.get_model_resources_key(), self.current_dict,
aliases.get_model_resources_top_level_folder_names())
self.current_dict = self._apply_filter_and_inject_variable(
self.current_dict, self.model_context, validator)
file_name = os.path.join(self.output_dir,
os.path.basename(model_file_name))
fos = JFileOutputStream(file_name, False)
writer = JPrintWriter(fos, True)
pty = PythonToYaml(self.current_dict)
pty._write_dictionary_to_yaml_file(self.current_dict, writer)
writer.close()
self.cache.clear()
for key in self.secrets_to_generate:
self.cache[key] = ''
# use a merged, substituted, filtered model to get domain name and create additional target output.
full_model_dictionary = cla_helper.load_model(
_program_name, self.model_context, self._aliases, "discover",
WlstModes.OFFLINE)
target_configuration_helper.generate_k8s_script(
self.model_context, self.cache, full_model_dictionary)
# create any additional outputs from full model dictionary
target_configuration_helper.create_additional_output(
Model(full_model_dictionary), self.model_context,
self._aliases, ExceptionType.VALIDATE)
except ValidateException, te:
self._logger.severe('WLSDPLY-20009',
_program_name,
model_file_name,
te.getLocalizedMessage(),
error=te,
class_name=_class_name,
method_name=_method_name)
ex = exception_helper.create_compare_exception(
te.getLocalizedMessage(), error=te)
self._logger.throwing(ex,
class_name=_class_name,
method_name=_method_name)
return VALIDATION_FAIL
