def begin_class(self, classname, extends, doc, abstract): cls = self.interface_name(classname) self.current_class = cls self.current_class_is_abstract = abstract self.current_loader = cStringIO() self.current_fields = cStringIO() with open(os.path.join(self.outdir, "%s.java" % cls), "w") as f: if extends: ext = "extends " + ", ".join(self.interface_name(e) for e in extends) else: ext = "" f.write("""package {package}; public interface {cls} {ext} {{ """. format(package=self.package, cls=cls, ext=ext)) if self.current_class_is_abstract: return with open(os.path.join(self.outdir, "%sImpl.java" % cls), "w") as f: f.write("""package {package}; public class {cls}Impl implements {cls} {{ """. format(package=self.package, cls=cls, ext=ext)) self.current_loader.write(""" void Load() { """)
def debuginfo_parser(adebug_package, filename): """ return the contents of filename """ try: # dfd = open(adebug_package, "rb") da = libarchive.Archive(adebug_package) # da = libarchive.Archive(dfd) for entry in da: size = entry.size # skip 0 byte files only, size can be 0 due to compression also! if size == 0: continue # skip directories if stat.S_ISDIR(entry.mode): continue # .dwz stuff is special if filename == "dwz" and entry.pathname.startswith("./usr/lib/debug/.dwz/"): data = da.read(entry.size) return cStringIO(data) elif entry.pathname.endswith(filename): data = da.read(entry.size) return cStringIO(data) except Exception, exc: print(adebug_package, str(exc)) traceback.print_exc()
def test_get_strict_formula(self): smtlib_single = """ (set-logic UF_LIRA) (declare-fun x () Bool) (declare-fun y () Bool) (declare-fun r () Real) (assert (> r 0.0)) (assert x) (check-sat) """ smtlib_double = smtlib_single + """ (assert (not y)) (check-sat) """ r = Symbol("r", REAL) x, y = Symbol("x"), Symbol("y") target_one = And(GT(r, Real(0)), x) target_two = And(GT(r, Real(0)), x, Not(y)) stream_in = cStringIO(smtlib_single) f = get_formula(stream_in) self.assertEqual(f, target_one) stream_in = cStringIO(smtlib_double) f = get_formula(stream_in) self.assertEqual(f, target_two) stream_in = cStringIO(smtlib_double) with self.assertRaises(PysmtValueError): f = get_formula_strict(stream_in)
def test_run_sysargv(self): bmodules = sys.modules bargv = sys.argv bpath = sys.path bget_executable = runner.get_executable try: sys.modules = { '__main__': MockModule('/path/to/cwd/afile.py', '__main__', '') } sys.argv = ['afile.py', '...'] sys.path = ['', sys.path[1]] runner.get_executable = get_executable def func(arg=1): raise NotImplementedError out = cStringIO() err = cStringIO() runner.run(func, exit=False, out=out, err=err) self.assertFalse(out.getvalue()) self.assertEqual(err.getvalue(), "python -m afile: Bad value for arg: '...'\n" "Usage: python -m afile [arg]\n") finally: sys.modules = bmodules sys.argv = bargv sys.path = bpath runner.get_executable = bget_executable
def test_from_formula(self): x, y = Symbol("x"), Symbol("y") f = And(x, Or(y, x)) script = smtlibscript_from_formula(f) self.assertIsNotNone(script) outstream = cStringIO() script.serialize(outstream) output = outstream.getvalue() self.assertIn("(set-logic ", output) self.assertIn("(declare-fun x () Bool)", output) self.assertIn("(declare-fun y () Bool)", output) self.assertIn("(check-sat)", output) # Use custom logic (as str) script2 = smtlibscript_from_formula(f, logic="BOOL") outstream = cStringIO() script2.serialize(outstream) output = outstream.getvalue() self.assertIn("(set-logic BOOL)", output) # Use custom logic (as Logic obj) script3 = smtlibscript_from_formula(f, logic=QF_UFLIRA) outstream = cStringIO() script3.serialize(outstream) output = outstream.getvalue() self.assertIn("(set-logic QF_UFLIRA)", output) # Custom logic must be a Logic or Str with self.assertRaises(UndefinedLogicError): smtlibscript_from_formula(f, logic=4)
def test_capture(): from syn.base_utils import capture, assign from six.moves import cStringIO oout = cStringIO() oerr = cStringIO() with assign(sys, 'stdout', oout): with assign(sys, 'stderr', oerr): print("Outside") sys.stderr.write('Err1\n') with capture() as (out, err): print("Inside") sys.stderr.write('Err!\n') assert out.getvalue() == 'Inside\n' assert err.getvalue() == 'Err!\n' print("Outside2") sys.stderr.write('Err2\n') assert out.getvalue() == 'Inside\n' assert err.getvalue() == 'Err!\n' print("Outside") assert oout.getvalue() == 'Outside\nOutside2\n' assert oerr.getvalue() == 'Err1\nErr2\n'
def test_dumped_logic(self): # Dumped logic matches the logic in the example fs = get_example_formulae() for (f_out, _, _, logic) in fs: buf_out = cStringIO() script_out = smtlibscript_from_formula(f_out) script_out.serialize(outstream=buf_out) buf_in = cStringIO(buf_out.getvalue()) parser = SmtLibParser() script_in = parser.get_script(buf_in) for cmd in script_in: if cmd.name == "set-logic": logic_in = cmd.args[0] if logic == logics.QF_BOOL: self.assertEqual(logic_in, logics.QF_UF) elif logic == logics.BOOL: self.assertEqual(logic_in, logics.LRA) else: self.assertEqual(logic_in, logic, script_in) break else: # Loops exited normally print("-"*40) print(script_in)
def test_iprint(self): top = Problem() top.root = SellarStateConnection() top.setup(check=False) base_stdout = sys.stdout try: ostream = cStringIO() sys.stdout = ostream top.run() finally: sys.stdout = base_stdout printed = ostream.getvalue() self.assertEqual(printed, '') # Turn on all iprints top.print_all_convergence() try: ostream = cStringIO() sys.stdout = ostream top.run() finally: sys.stdout = base_stdout printed = ostream.getvalue() self.assertEqual(printed.count('NEWTON'), 3) self.assertEqual(printed.count('GMRES'), 4) self.assertTrue('[root] NL: NEWTON 0 | ' in printed) self.assertTrue(' [root.sub] LN: GMRES 0 | ' in printed)
def myexec(self, s, with_exec=True): if with_exec and not s.startswith("exec"): s = "exec " + s if not s.endswith("\n"): s += "\n" # http://stackoverflow.com/questions/5342402/can-i-get-the-exit-code-of-a-command-executed-in-a-subshell-via-ssh self.sendall(s) #stdout = self.makefile() #stderr = self.makefile_stderr() # Wait for it..... #count = 0 #while not self.recv_ready(): # time.sleep(1) # count += 1 # if count >= 6: # print('time out') #TODO: add exeption here #time.sleep(10) #return_code = self.recv_exit_status() #print(return_code) #result = "" #while self.recv_ready(): ##while self.recv_ready() and not self.exit_status_ready(): # o = self.recv(1024) # #if not len(o): break # #if o: # print("o", o) # result += o #self.settimeout(30) stdout, stderr = cStringIO(), cStringIO() #try: #TODO: add exeption here # Need to read the entire buffer to caputure output while not self.exit_status_ready(): if self.recv_ready(): out_buff = self.recv(1024) while out_buff: #print("Indside stdout", out_buff) stdout.write(out_buff) out_buff = self.recv(1024) if self.recv_stderr_ready(): error_buff = self.recv_stderr(1024) while error_buff: #print("Indside stderr", error_buff) stderr.write(error_buff) error_buff = self.recv_stderr(1024) #except socket.timeout: # raise socket.timeout stdout.seek(0), stderr.seek(0) return SSHResult(stdout, stderr, return_code=self.recv_exit_status())
def _parseINI(text): from six.moves.configparser import ConfigParser from six.moves import cStringIO parser = ConfigParser() try: parser.read_file(cStringIO(text)) except AttributeError: # Python 2 parser.readfp(cStringIO(text)) return parser
def write_main(argv): """ write FILENAME Write a local copy of FILENAME using FILENAME_tweaks for local tweaks. """ if len(argv) != 1: print("Please provide the name of a file to write.") return 1 filename = argv[0] resource_name = "files/" + filename tweaks_name = amend_filename(filename, "_tweaks") if not pkg_resources.resource_exists("edx_lint", resource_name): print("Don't have file %r to write." % filename) return 2 if os.path.exists(filename): print("Checking existing copy of %s" % filename) tef = TamperEvidentFile(filename) if not tef.validate(): bak_name = amend_filename(filename, "_backup") print("Your copy of %s seems to have been edited, renaming it to %s" % (filename, bak_name)) if os.path.exists(bak_name): print("A previous %s exists, deleting it" % bak_name) os.remove(bak_name) os.rename(filename, bak_name) print("Reading edx_lint/files/%s" % filename) cfg = configparser.RawConfigParser() resource_string = pkg_resources.resource_string("edx_lint", resource_name).decode("utf8") # pkg_resources always reads binary data (in both python2 and python3). # ConfigParser.read_string only exists in python3, so we have to wrap the string # from pkg_resources in a cStringIO so that we can pass it into ConfigParser.readfp. cfg.readfp(cStringIO(resource_string), resource_name) # pylint: disable=deprecated-method if os.path.exists(tweaks_name): print("Applying local tweaks from %s" % tweaks_name) cfg_tweaks = configparser.RawConfigParser() cfg_tweaks.read([tweaks_name]) merge_configs(cfg, cfg_tweaks) print("Writing %s" % filename) output_text = cStringIO() output_text.write(WARNING_HEADER.format(filename=filename, tweaks_name=tweaks_name)) cfg.write(output_text) out_tef = TamperEvidentFile(filename) if six.PY2: output_bytes = output_text.getvalue() else: output_bytes = output_text.getvalue().encode("utf8") out_tef.write(output_bytes) return 0
def capture_stdout(): oldout, olderr = sys.stdout, sys.stderr try: out = [cStringIO(), cStringIO()] sys.stdout, sys.stderr = out yield out finally: sys.stdout, sys.stderr = oldout, olderr out[0] = out[0].getvalue() out[1] = out[1].getvalue()
def test_run_fail_exit(self): def func(): raise errors.ArgumentError("test_run_fail_exit") stdout = cStringIO() stderr = cStringIO() self.assert_systemexit( 2, runner.run, func, args=['test'], out=stdout, err=stderr) self.assertFalse(stdout.getvalue()) self.assertTrue(stderr.getvalue()) runner.run(func, args=['test'], out=stdout, err=stderr, exit=False)
def test_run_success_exit(self): def func(): return "test_run_success_exit" stdout = cStringIO() stderr = cStringIO() self.assert_systemexit( None, runner.run, func, args=['test'], out=stdout, err=stderr) self.assertFalse(stderr.getvalue()) self.assertEqual(stdout.getvalue(), 'test_run_success_exit\n') runner.run(func, args=['test'], out=stdout, err=stderr, exit=False)
def _generate_legend_images(gene): cache = gene.caches[gene.options.cache] for layer in gene.layers.values(): if "legend_mime" in layer and "legend_extention" in layer: if layer["type"] == "wms": session = requests.session() session.headers.update(layer["headers"]) previous_hash = None for zoom, resolution in enumerate(layer["grid_ref"]["resolutions"]): legends = [] for l in layer["layers"]: response = session.get( layer["url"] + "?" + urlencode( { "SERVICE": "WMS", "VERSION": "1.1.1", "REQUEST": "GetLegendGraphic", "LAYER": l, "FORMAT": layer["legend_mime"], "TRANSPARENT": "TRUE" if layer["legend_mime"] == "image/png" else "FALSE", "STYLE": layer["wmts_style"], "SCALE": resolution / 0.00028, } ) ) try: legends.append(Image.open(cStringIO(response.content))) except: # pragma: nocover logger.warn( "Unable to read legend image for layer '%s', resolution '%i': %r" % (layer["name"], resolution, response.content) ) width = max(i.size[0] for i in legends) height = sum(i.size[1] for i in legends) image = Image.new("RGBA", (width, height)) y = 0 for i in legends: image.paste(i, (0, y)) y += i.size[1] string_io = cStringIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]]) result = string_io.getvalue() new_hash = sha1(result).hexdigest() if new_hash != previous_hash: previous_hash = new_hash _send( result, "1.0.0/%s/%s/legend%s.%s" % (layer["name"], layer["wmts_style"], zoom, layer["legend_extention"]), layer["legend_mime"], cache, )
def run_cmd(self, cmd, main_func): old_stdout = sys.stdout sys.stdout = mystdout = cStringIO() old_stderr = sys.stderr sys.stderr = mystderr = cStringIO() self.assert_main_equals(cmd, main_func, []) sys.stdout = old_stdout sys.stderr = old_stderr log.info(mystdout.getvalue()) log.info(mystderr.getvalue()) return mystdout.getvalue(), mystderr.getvalue()
def run_cli(self, func, args): orig_out = sys.stdout orig_err = sys.stderr try: sys.stdout = out = cStringIO() sys.stderr = err = cStringIO() ret = func(*args) finally: sys.stdout = orig_out sys.stderr = orig_err return ret, out, err
def crun(self, func, args, stdin=None, **kwargs): orig = sys.stdin, sys.stdout, sys.stderr if stdin is None: stdin = cStringIO() sys.stdin = stdin sys.stdout = stdout = cStringIO() sys.stderr = stderr = cStringIO() try: runner.run(func, args=args, exit=False, out=stdout, err=stderr, **kwargs) return stdout, stderr finally: sys.stdin, sys.stdout, sys.stderr = orig
def test_wrong(self): txt = """ (declare-fun A () Bool) (declare-fun B () Bool) (assert (and A B)) (check-sat) (exit) """ with self.assertRaises(UnknownSmtLibCommandError): self.ts_parser.get_ts(cStringIO(txt)) script = self.smt_parser.get_script(cStringIO(txt)) self.assertIsNotNone(script)
def test_basic(self): txt = """ (declare-fun A () Bool) (declare-fun B () Bool) (init (and A B)) (trans (=> A (next A))) (exit) """ with self.assertRaises(UnknownSmtLibCommandError): self.smt_parser.get_script(cStringIO(txt)) ts = self.ts_parser.get_ts(cStringIO(txt)) self.assertIsNotNone(ts)
def test_run_multi(self): def func1(): return '1' def func2(): return '2' stdout, stderr = util.run([func1, func2], args=['test', 'func1']) self.assertFalse(stderr.getvalue()) self.assertEqual(stdout.getvalue(), '1\n') stdout, stderr = util.run([func1, func2], args=['test', 'func2']) self.assertFalse(stderr.getvalue()) self.assertEqual(stdout.getvalue(), '2\n') stdout = cStringIO() stderr = cStringIO() runner.run(func1, func2, args=['test', 'func1'], out=stdout, err=stderr, exit=False) self.assertFalse(stderr.getvalue()) self.assertEqual(stdout.getvalue(), '1\n')
def put_ini(self, text): """ """ context = self.context parser = ConfigParser() try: parser.read_file(cStringIO(text)) except AttributeError: # Python 2 parser.readfp(cStringIO(text)) for option, value in parser.defaults().items(): prop_type = context.getPropertyType(option) if prop_type is None: context._setProperty(option, value, 'string') else: context._updateProperty(option, value)
def test_parse_examples_daggified_bv(self): fs = get_example_formulae() for (f_out, _, _, logic) in fs: if logic != logics.QF_BV: # See test_parse_examples_daggified continue buf_out = cStringIO() script_out = smtlibscript_from_formula(f_out) script_out.serialize(outstream=buf_out, daggify=True) buf_in = cStringIO(buf_out.getvalue()) parser = SmtLibParser() script_in = parser.get_script(buf_in) f_in = script_in.get_last_formula() self.assertValid(Iff(f_in, f_out), f_in.serialize())
def _get_aws_cli_completions(self, document): """Get completions from the official AWS CLI for the current scope. Args: * document: An instance of prompt_toolkit's Document. Returns: A list of string completions. """ text = self.replace_shortcut(document.text) # Redirect stdout to a string so we can capture the AWS CLI # autocompleter results # See: http://stackoverflow.com/a/1218951 old_stdout = sys.stdout sys.stdout = mystdout = cStringIO() try: self.aws_completer.complete(text, len(text)) except Exception as e: self.log_exception(e, traceback) sys.stdout = old_stdout aws_completer_results = mystdout.getvalue() # Tidy up the completions and store it in a list aws_completer_results = re.sub("\n", "", aws_completer_results) aws_completer_results_list = aws_completer_results.split() return aws_completer_results_list
def df_to_db(df, table_name, schema=None, pk='id'): # Does not sanitize DataFrame input. Will fail if values contain # the escape character, backslash (\). Binary format COPY would be # faster and might be more robust. if schema: schema_name = schema.__name__ qualified_name = "{}.{}".format(schema_name, table_name) else: schema_name = None qualified_name = table_name df.columns = [s.lower() for s in df.columns] empty_df = df.iloc[[0]] with db.cursor() as cur: empty_df.to_sql(table_name, db._engine, schema=schema_name, index=True, if_exists='replace') cur.execute("DELETE FROM {}".format(qualified_name)) buf = cStringIO() df.to_csv(buf, sep='\t', na_rep=r'\N', header=False, index=True) buf.seek(0) cur.copy_from(buf, qualified_name, columns=tuple([df.index.name] + df.columns.values.tolist())) if pk: cur.execute(""" ALTER TABLE {} ADD COLUMN {} serial PRIMARY KEY; """.format(qualified_name, pk)) db.refresh()
def _send_email(self, msg, tag): if self.mailto is None: return -1 header = msg.splitlines() app = header.append app("Submitted on: %s" % time.ctime(self.start_time)) app("Completed on: %s" % time.asctime()) app("Elapsed time: %s" % str(self.get_delta_etime())) app("Number of errored tasks: %d" % self.flow.num_errored_tasks) app("Number of unconverged tasks: %d" % self.flow.num_unconverged_tasks) strio = cStringIO() strio.writelines("\n".join(header) + 4 * "\n") # Add the status of the flow. self.flow.show_status(stream=strio) if self.exceptions: # Report the list of exceptions. strio.writelines(self.exceptions) if tag is None: tag = " [ALL OK]" if self.flow.all_ok else " [WARNING]" return sendmail(subject=self.flow.name + tag, text=strio.getvalue(), mailto=self.mailto)
def test_update_image(self): # Updates an image by image_id # Create image image_name = data_utils.rand_name('image') container_format = CONF.image.container_formats[0] disk_format = CONF.image.disk_formats[0] body = self.client.create_image(name=image_name, container_format=container_format, disk_format=disk_format, visibility='private') self.addCleanup(self.client.delete_image, body['id']) self.assertEqual('queued', body['status']) image_id = body['id'] # Now try uploading an image file image_file = moves.cStringIO(data_utils.random_bytes()) self.client.store_image_file(image_id, image_file) # Update Image new_image_name = data_utils.rand_name('new-image') body = self.client.update_image(image_id, [ dict(replace='/name', value=new_image_name)]) # Verifying updating body = self.client.show_image(image_id) self.assertEqual(image_id, body['id']) self.assertEqual(new_image_name, body['name'])
def csv_report(query, delim=None): try: res = query.execute_query_only() return write_csv(res.headers, res.data, delim) except DatabaseError as e: resp = cStringIO() return resp.write(str(e)) # consistent return type
def test_both(self): captured = cStringIO() json = """[{"FromPort":8000,"ToPort":9000,"IpProtocol":"tcp","IpRanges":[{"CidrIp":"192.168.100.0/24"}]}]""" args = ' --group-name foobar --port 100 --ip-permissions %s' % json args_list = (self.prefix + args).split() with mock.patch('sys.stderr', captured): self.assert_params_for_cmd(args_list, {}, expected_rc=255)
def getInterfacesFromXML( xmlStr, replaceKnownInterfaces = False ): """ Parses the supplied Introspection XML string and returns a list of L{interface.DBusInerface} instances representing the XML interface definitions. @type replaceKnownInterfaces: C{bool} @param replaceKnownInterfaces: If true, pre-existing interface definitions will be replaced by the contents of the interfaces defined within the XML string @rtype: C{list} of L{interface.DBusInerface} """ handler = IntrospectionHandler( replaceKnownInterfaces ) xmlStr = xmlStr.strip() if xmlStr.startswith('<!DOCTYPE'): xmlStr = xmlStr[ xmlStr.find('>')+1 : ] #xml.sax.parseString( xmlStr, handler ) p = xml.sax.make_parser() p.setFeature(xml.sax.handler.feature_validation, False) p.setFeature(xml.sax.handler.feature_external_ges, False) p.setContentHandler(handler) p.parse(cStringIO(xmlStr)) return handler.interfaces
def test_declare_sort(self): class SmtLibIgnore(SmtLibIgnoreMixin): declare_sort_history = [] def declare_sort(self, name, arity): self.declare_sort_history.append((name, arity)) mock = SmtLibIgnore() parser = SmtLibParser() smtlib_script = '\n'.join(['(declare-sort s0 0)', \ '(declare-sort s1 1)', \ '(declare-const c0 s0)', \ '(declare-const c1 (s1 Int))']) outstream = cStringIO(smtlib_script) script = parser.get_script(outstream) script.evaluate(solver=mock) self.assertEqual(len(mock.declare_sort_history), 2) s0_name, s0_arity = mock.declare_sort_history[0] s1_name, s1_arity = mock.declare_sort_history[1] self.assertEqual(s0_name, "s0") self.assertEqual(s0_arity, 0) self.assertEqual(s1_name, "s1") self.assertEqual(s1_arity, 1)
def test_import_flow(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = cStringIO("TEST_IMAGE") with mock.patch.object(putils, 'trycmd') as tmock: tmock.return_value = (json.dumps({ 'format': 'qcow2', }), None) executor.begin_processing(self.task.task_id) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path)) self.assertEqual(1, len(list(self.image.locations))) self.assertEqual( "file://%s/%s" % (self.test_dir, self.image.image_id), self.image.locations[0]['url'])
def _indented_print(f_locals, d, indent, excludes=('__init__',), file=sys.stdout): """ Print trace info, indenting based on call depth. """ sindent = tab * indent sep = '=' if d is f_locals else ':' for name in sorted(d, key=lambda a: str(a)): if name not in excludes: if isinstance(d[name], (dict, OrderedDict)): f = cStringIO() _indented_print(f_locals, d[name], 0, file=f) s = " %s%s%s{%s}" % (sindent, name, sep, f.getvalue()) else: s = " %s%s%s%s" % (sindent, name, sep, d[name]) if ' object at ' in s: s = addr_regex.sub('', s) linelen = len(s) leneq = len(s.split(sep, 1)[0]) if linelen > MAXLINE: if '\n' in s: # change indent s = s.replace("\n", "\n%s" % (' '*leneq)) print(s, file=file)
def df_to_csv_string(df, col_order): # if six.PY3: # # convert any byte strings to string to avoid 'b' prefix bug in pandas # # https://github.com/pandas-dev/pandas/issues/9712 # str_df = df.select_dtypes([np.object]) # str_df = str_df.stack().str.decode('utf-8').unstack() # for col in str_df: # df[col] = str_df[col] cs = cStringIO() cs.writelines( df[col_order].to_csv( index=False, header=True, na_rep='NaN', sep='\t', encoding='UTF-8', quoting=csv.QUOTE_MINIMAL, escapechar='"' ) ) # logger.info('----- CSV DATA DIRECTLY FROM PANDAS---- {}'.format(cs.getvalue())) return cs.getvalue()
def _generate_doc_string_(op_proto, additional_args_lines=None, skip_attrs_set=None): """ Generate docstring by OpProto Args: op_proto (framework_pb2.OpProto): a protobuf message typed OpProto Returns: str: the document string """ if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError("OpProto should be `framework_pb2.OpProto`") buf = cStringIO() buf.write(escape_math(op_proto.comment)) buf.write('\nArgs:\n') for each_input in op_proto.inputs: line_begin = ' {0}'.format(_convert_(each_input.name)) buf.write(line_begin) buf.write(" (Tensor): ") buf.write(escape_math(each_input.comment)) if each_input.duplicable: buf.write(" Duplicatable.") if each_input.dispensable: buf.write(" Optional.") buf.write('\n') skip_attrs = OpProtoHolder.generated_op_attr_names() # attr use_mkldnn and is_test also should not be visible to users. skip_attrs.add("use_mkldnn") skip_attrs.add("is_test") skip_attrs.add("use_cudnn") if skip_attrs_set: for t in skip_attrs_set: skip_attrs.add(t) for each_attr in op_proto.attrs: if each_attr.name in skip_attrs: continue buf.write(' ') buf.write(each_attr.name) buf.write(' (') buf.write(_type_to_str_(each_attr.type)) buf.write('): ') buf.write(escape_math(each_attr.comment)) buf.write('\n') if additional_args_lines is not None: for line in additional_args_lines: line = line.strip() buf.write(' ') buf.write(line) buf.write('\n') if len(op_proto.outputs) != 0: buf.write('\nReturns:\n') buf.write(' ') for each_opt in op_proto.outputs: if not each_opt.intermediate: break buf.write(_convert_(each_opt.name)) buf.write(' (Tensor): ') buf.write(escape_math(each_opt.comment)) return buf.getvalue()
def test_closing(self): obj = cStringIO("".join(self.text)) ns = util.NamedStream(obj, self.textname, close=True) assert_equal(ns.closed, False) ns.close() assert_equal(ns.closed, True)
def get_and_run_test(method_path): """ Return desired source code for a single feature after testing it. Used by embed_test. 1. Get the source code for a unit test method 2. Replace the asserts with prints 3. Insert extra print statements to indicate start and end of print Out blocks 4. Run the test using source_with_out_start_stop_indicators -> run_outputs 5. Split method_source up into groups of "In" blocks -> input_blocks 6. Extract from run_outputs, the Out blocks -> output_blocks 7. Return method_source, input_blocks, output_blocks, skipped Parameters ---------- method_path : str Module hiearchy path to the test. Returns ------- str Cleaned source code, ready for inclusion in doc. str Reason that the test failed or was skipped. list of str List of input code blocks list of str List of Python output blocks bool True if test was skipped """ #---------------------------------------------------------- # 1. Get the source code for a unit test method. #---------------------------------------------------------- module_path = '.'.join(method_path.split('.')[:-2]) class_name = method_path.split('.')[-2] method_name = method_path.split('.')[-1] test_module = importlib.import_module(module_path) cls = getattr(test_module, class_name) try: import mpi4py except ImportError: use_mpi = False else: N_PROCS = getattr(cls, 'N_PROCS', 1) use_mpi = N_PROCS > 1 method = getattr(cls, method_name) method_source = inspect.getsource(method) method_source = strip_header(method_source) method_source = remove_docstrings(method_source) method_source = replace_asserts_with_prints(method_source) method_source = remove_initial_empty_lines(method_source) #----------------------------------------------------------------------------------- # 3. Insert extra print statements to indicate start and end of print Out blocks #----------------------------------------------------------------------------------- source_with_output_start_stop_indicators = insert_output_start_stop_indicators(method_source) #------------------------------------------------------------------------------------ # Get all the pieces of code needed to run the unit test method #----------------------------------------------------------------------------------- global_imports = globals_for_imports(method_source) # make 'self' available to test code (as an instance of the test case) self_code = "from %s import %s\nself = %s('%s')\n" % \ (module_path, class_name, class_name, method_name) # get setUp and tearDown but don't duplicate if it is the method being tested setup_code = '' if method_name == 'setUp' else \ get_method_body(inspect.getsource(getattr(cls, 'setUp'))) teardown_code = '' if method_name == 'tearDown' else \ get_method_body(inspect.getsource(getattr(cls, 'tearDown'))) code_to_run = '\n'.join([global_imports, self_code, setup_code, source_with_output_start_stop_indicators, teardown_code]) #----------------------------------------------------------------------------------- # 4. Run the test using source_with_out_start_stop_indicators -> run_outputs #----------------------------------------------------------------------------------- skipped = False failed = False try: if use_mpi: # use subprocess to run test with `mpirun` # write code to a file so we can run it. fd, code_to_run_path = tempfile.mkstemp() with os.fdopen(fd, 'w') as tmp: tmp.write(code_to_run) tmp.close() # output will be written to one file per process env = os.environ.copy() env['USE_PROC_FILES'] = '1' p = subprocess.Popen(['mpirun', '-n', str(N_PROCS), 'python', code_to_run_path], env=env) p.wait() # extract output blocks from all output files & merge them multi_out_blocks = [] for i in range(N_PROCS): with open('%d.out' % i) as f: multi_out_blocks.append(extract_output_blocks(f.read())) os.remove('%d.out' % i) output_blocks = [] for i in range(len(multi_out_blocks[0])): output_blocks.append('\n'.join(["(rank %d) %s" % (j, m[i]) for j, m in enumerate(multi_out_blocks) if m[i]])) else: # just exec() the code for serial tests. # capture all output stdout = sys.stdout stderr = sys.stderr strout = cStringIO() sys.stdout = strout sys.stderr = strout # set all the loggers to write to our captured stream from openmdao.utils.logger_utils import _loggers for name in _loggers: _loggers[name]['logger'].handlers[0].stream = strout # We need more precision from numpy save_opts = np.get_printoptions() np.set_printoptions(precision=8) # Move to the test directory in case there are files to read. save_dir = os.getcwd() os.chdir('/'.join(test_module.__file__.split('/')[:-1])) try: exec(code_to_run, {}) finally: os.chdir(save_dir) np.set_printoptions(precision=save_opts['precision']) run_outputs = strout.getvalue() except subprocess.CalledProcessError as e: # Get a traceback. if 'raise unittest.SkipTest' in e.output.decode('utf-8'): reason_for_skip = e.output.splitlines()[-1][len('unittest.case.SkipTest: '):] run_outputs = reason_for_skip skipped = True else: run_outputs = "Running of embedded test {} in docs failed due to: \n\n{}".format(method_path, e.output.decode('utf-8')) failed = True except Exception as err: if 'SkipTest' in code_to_run: txt1 = code_to_run.split('SkipTest(')[1] run_outputs = txt1.split(')')[0] skipped = True else: msg = "Running of embedded test {} in docs failed due to: \n\n{}" run_outputs = msg.format(method_path, str(err)) failed = True finally: if use_mpi: os.remove(code_to_run_path) else: sys.stdout = stdout sys.stderr = stderr if PY3 and not use_mpi and not isinstance(run_outputs, str): run_outputs = "".join(map(chr, run_outputs)) # in Python 3, run_outputs is of type bytes! if skipped: input_blocks = output_blocks = None skipped_output = run_outputs elif failed: raise SphinxError(run_outputs) else: ##################### ### 5. Split method_source up into groups of "In" blocks -> input_blocks ### ##################### input_blocks = split_source_into_input_blocks(source_with_output_start_stop_indicators) ##################### ### 6. Extract from run_outputs, the Out blocks -> output_blocks ### ##################### if not use_mpi: output_blocks = extract_output_blocks(run_outputs) # the last input block may not produce any output if len(output_blocks) == len(input_blocks) - 1: output_blocks.append('') # Need to deal with the cases when there is no output for a given input block # Merge an input block with the previous block and throw away the output block input_blocks, output_blocks = clean_up_empty_output_blocks(input_blocks, output_blocks) skipped_output = None return method_source, skipped_output, input_blocks, output_blocks, skipped
def main_func(): global opt_unwrap global opt_check global opt_diffs global opt_format global orig_opt_format # Purposefully undocumented; just like -f. env_format = os.environ.get("PKGFMT_OUTPUT") if env_format: opt_format = orig_opt_format = env_format ret = 0 opt_set = set() try: opts, pargs = getopt.getopt(sys.argv[1:], "cdf:u?", ["help"]) for opt, arg in opts: opt_set.add(opt) if opt == "-c": opt_check = True elif opt == "-d": opt_diffs = True elif opt == "-f": opt_format = orig_opt_format = arg elif opt == "-u": opt_unwrap = True elif opt in ("--help", "-?"): usage(exitcode=EXIT_OK) except getopt.GetoptError as e: usage(_("illegal global option -- {0}").format(e.opt)) if len(opt_set - set(["-f"])) > 1: usage(_("only one of [cdu] may be specified")) if opt_format not in (FMT_V1, FMT_V2): usage(_("unsupported format '{0}'").format(opt_format)) def difference(in_file): whole_f1 = in_file.readlines() f2 = cStringIO() fmt_file(cStringIO("".join(whole_f1)), f2) f2.seek(0) whole_f2 = f2.readlines() if whole_f1 == whole_f2: if opt_diffs: return 0, "" return 0, "".join(whole_f2) elif opt_diffs: return 1, "".join(unified_diff(whole_f2, whole_f1)) return 1, "".join(whole_f2) flist = pargs if not flist: try: in_file = cStringIO() in_file.write(sys.stdin.read()) in_file.seek(0) ret, formatted = difference(in_file) if ret == 1 and opt_check: # Manifest was different; if user didn't specify # a format explicitly, try V1 format. if not orig_opt_format: opt_format = FMT_V1 in_file.seek(0) rcode, formatted = difference(in_file) opt_format = FMT_V2 if rcode == 0: # Manifest is in V1 format. return 0 error(_("manifest is not in pkgfmt form")) elif ret == 1 and not opt_diffs: # Treat as successful exit if not checking # formatting or displaying diffs. ret = 0 # Display formatted version (explicit 'end' needed to # prevent output of extra newline) even if manifest # didn't need formatting for the stdin case. (The # assumption is that it might be used in a pipeline.) if formatted: print(formatted, end="") except EnvironmentError as e: if e.errno == errno.EPIPE: # User closed input or output (i.e. killed piped # program before all input was read or output # was written). return 1 return ret ret = 0 tname = None for fname in flist: try: # force path to be absolute; gives better diagnostics if # something goes wrong. path = os.path.abspath(fname) with open(fname, "r") as f: rcode, formatted = difference(f) if rcode == 0: continue if opt_check: # Manifest was different; if user didn't specify # a format explicitly, try V1 format. if not orig_opt_format: opt_format = FMT_V1 with open(fname, "r") as f: rcode, formatted = difference(f) opt_format = FMT_V2 if rcode == 0: # Manifest is in V1 format. continue ret = 1 error(_("{0} is not in pkgfmt form; run pkgfmt " "on file without -c or -d to reformat " "manifest in place").format(fname), exitcode=None) continue elif opt_diffs: # Display differences (explicit 'end' needed to # prevent output of extra newline). ret = 1 print(formatted, end=" ") continue elif ret != 1: # Treat as successful exit if not checking # formatting or displaying diffs. ret = 0 # Replace manifest with formatted version. pathdir = os.path.dirname(path) tfd, tname = tempfile.mkstemp(dir=pathdir) with os.fdopen(tfd, "w") as t: t.write(formatted) try: # Ensure existing mode is preserved. mode = os.stat(fname).st_mode os.chmod(tname, mode) os.rename(tname, fname) except EnvironmentError as e: error(str(e), exitcode=EXIT_OOPS) except (EnvironmentError, IOError) as e: error(str(e), exitcode=EXIT_OOPS) finally: if tname: try: pkg.portable.remove(tname) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return ret
def serialize_to_string(self, daggify=True): buf = cStringIO() self.serialize(buf, daggify=daggify) return buf.getvalue()
c.add('fn') c.fn.value = u.full_name() le = c.add('email', 'kn') le.value = u.email le.type_paramlist = ['INTERNET'] c.add('X-ABLabel', 'kn').value = 'kn' if u.telephone is not None: c.add('tel', 'kn') c.tel.value = u.telephone c.tel.type_param = 'CELL' if (u.addr_street is not None and u.addr_city is not None and u.addr_number is not None and u.addr_zipCode is not None): le = c.add('adr', 'kn') le.value = vobject.vcard.Address( ' '.join((u.addr_street, u.addr_number)), u.addr_city, '', u.addr_zipCode, 'Nederland') c.add('x-abadr', 'kn').value = 'nl' return c.serialize() if __name__ == '__main__': tb = cStringIO() tf = tarfile.open(mode='w', fileobj=tb) for u in args_to_users(sys.argv[1:]): ti = tarfile.TarInfo(u.username + ".vcf") td = vcard(u) ti.size = len(td) tf.addfile(ti, cStringIO(td)) tf.close() print(tb.getvalue())
def test_both(self): captured = cStringIO() json = """[{"FromPort":8000,"ToPort":9000,"IpProtocol":"tcp","IpRanges":[{"CidrIp":"192.168.100.0/24"}]}]""" args = ' --group-name foobar --port 100 --ip-permissions %s' % json args_list = (self.prefix + args).split() self.assert_params_for_cmd(args_list, {}, expected_rc=255)
def run_code(code_to_run, path, module=None, cls=None, shows_plot=False, imports_not_required=False): """ Run the given code chunk and collect the output. """ skipped = False failed = False if cls is None: use_mpi = False else: try: import mpi4py except ImportError: use_mpi = False else: N_PROCS = getattr(cls, 'N_PROCS', 1) use_mpi = N_PROCS > 1 try: # use subprocess to run code to avoid any nasty interactions between codes # Move to the test directory in case there are files to read. save_dir = os.getcwd() if module is None: code_dir = os.path.dirname(os.path.abspath(path)) else: code_dir = os.path.dirname(os.path.abspath(module.__file__)) os.chdir(code_dir) if use_mpi: env = os.environ.copy() # output will be written to one file per process env['USE_PROC_FILES'] = '1' env['OPENMDAO_CURRENT_MODULE'] = module.__name__ env['OPENMDAO_CODE_TO_RUN'] = code_to_run p = subprocess.Popen( ['mpirun', '-n', str(N_PROCS), sys.executable, _sub_runner], env=env) p.wait() # extract output blocks from all output files & merge them output = [] for i in range(N_PROCS): with open('%d.out' % i) as f: output.append(f.read()) os.remove('%d.out' % i) elif shows_plot: if module is None: # write code to a file so we can run it. fd, code_to_run_path = tempfile.mkstemp() with os.fdopen(fd, 'w') as tmp: tmp.write(code_to_run) try: p = subprocess.Popen([sys.executable, code_to_run_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=os.environ) output, _ = p.communicate() if p.returncode != 0: failed = True finally: os.remove(code_to_run_path) else: env = os.environ.copy() env['OPENMDAO_CURRENT_MODULE'] = module.__name__ env['OPENMDAO_CODE_TO_RUN'] = code_to_run p = subprocess.Popen([sys.executable, _sub_runner], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) output, _ = p.communicate() if p.returncode != 0: failed = True output = output.decode('utf-8', 'ignore') else: # just exec() the code for serial tests. # capture all output stdout = sys.stdout stderr = sys.stderr strout = cStringIO() sys.stdout = strout sys.stderr = strout # We need more precision from numpy with printoptions(precision=8): if module is None: globals_dict = { '__file__': path, '__name__': '__main__', '__package__': None, '__cached__': None, } else: if imports_not_required: # code does not need to include all imports # Get from module globals_dict = module.__dict__ else: globals_dict = {} try: exec(code_to_run, globals_dict) except Exception as err: # for actual errors, print code (with line numbers) to facilitate debugging if not isinstance(err, unittest.SkipTest): for n, line in enumerate(code_to_run.split('\n')): print('%4d: %s' % (n, line), file=stderr) raise finally: sys.stdout = stdout sys.stderr = stderr output = strout.getvalue() except subprocess.CalledProcessError as e: output = e.output.decode('utf-8', 'ignore') # Get a traceback. if 'raise unittest.SkipTest' in output: reason_for_skip = output.splitlines( )[-1][len('unittest.case.SkipTest: '):] output = reason_for_skip skipped = True else: output = "Running of embedded code {} in docs failed due to: \n\n{}".format( path, output) failed = True except unittest.SkipTest as skip: output = str(skip) skipped = True except Exception as exc: output = "Running of embedded code {} in docs failed due to: \n\n{}".format( path, traceback.format_exc()) failed = True finally: os.chdir(save_dir) return skipped, failed, output
def __init__(self, srcdir, confdir, outdir, doctreedir, buildername, confoverrides=None, status=sys.stdout, warning=sys.stderr, freshenv=False, warningiserror=False, tags=None, verbosity=0, parallel=0): # type: (unicode, unicode, unicode, unicode, unicode, Dict, IO, IO, bool, bool, List[unicode], int, int) -> None # NOQA self.verbosity = verbosity self.extensions = {} # type: Dict[unicode, Extension] self._setting_up_extension = ['?'] # type: List[unicode] self.builder = None # type: Builder self.env = None # type: BuildEnvironment self.registry = SphinxComponentRegistry() self.enumerable_nodes = {} # type: Dict[nodes.Node, Tuple[unicode, Callable]] # NOQA self.post_transforms = [] # type: List[Transform] self.html_themes = {} # type: Dict[unicode, unicode] self.srcdir = srcdir self.confdir = confdir self.outdir = outdir self.doctreedir = doctreedir self.parallel = parallel if status is None: self._status = cStringIO() # type: IO self.quiet = True else: self._status = status self.quiet = False if warning is None: self._warning = cStringIO() # type: IO else: self._warning = warning self._warncount = 0 self.warningiserror = warningiserror logging.setup(self, self._status, self._warning) self.events = EventManager() # keep last few messages for traceback # This will be filled by sphinx.util.logging.LastMessagesWriter self.messagelog = deque(maxlen=10) # type: deque # say hello to the world logger.info(bold('Running Sphinx v%s' % sphinx.__display_version__)) # status code for command-line application self.statuscode = 0 if not path.isdir(outdir): logger.info('making output directory...') os.makedirs(outdir) # read config self.tags = Tags(tags) self.config = Config(confdir, CONFIG_FILENAME, confoverrides or {}, self.tags) self.config.check_unicode() # defer checking types until i18n has been initialized # initialize some limited config variables before initialize i18n and loading # extensions self.config.pre_init_values() # set up translation infrastructure self._init_i18n() # check the Sphinx version if requested if self.config.needs_sphinx and self.config.needs_sphinx > sphinx.__display_version__: raise VersionRequirementError( _('This project needs at least Sphinx v%s and therefore cannot ' 'be built with this version.') % self.config.needs_sphinx) # set confdir to srcdir if -C given (!= no confdir); a few pieces # of code expect a confdir to be set if self.confdir is None: self.confdir = self.srcdir # load all built-in extension modules for extension in builtin_extensions: self.setup_extension(extension) # load all user-given extension modules for extension in self.config.extensions: self.setup_extension(extension) # preload builder module (before init config values) self.preload_builder(buildername) # the config file itself can be an extension if self.config.setup: self._setting_up_extension = ['conf.py'] # py31 doesn't have 'callable' function for below check if hasattr(self.config.setup, '__call__'): self.config.setup(self) else: raise ConfigError( _("'setup' as currently defined in conf.py isn't a Python callable. " "Please modify its definition to make it a callable function. This is " "needed for conf.py to behave as a Sphinx extension.") ) # now that we know all config values, collect them from conf.py self.config.init_values() # check extension versions if requested verify_required_extensions(self, self.config.needs_extensions) # check primary_domain if requested primary_domain = self.config.primary_domain if primary_domain and not self.registry.has_domain(primary_domain): logger.warning(_('primary_domain %r not found, ignored.'), primary_domain) # create the builder self.builder = self.create_builder(buildername) # check all configuration values for permissible types self.config.check_types() # set up source_parsers self._init_source_parsers() # set up the build environment self._init_env(freshenv) # set up the builder self._init_builder() # set up the enumerable nodes self._init_enumerable_nodes()
def script_copy(self): parser = SmtLibParser() return parser.get_script(cStringIO(self.script_string))
def test_progiter(): from ubelt import Timer # Define a function that takes some time def is_prime(n): return n >= 2 and not any(n % i == 0 for i in range(2, n)) N = 500 if False: file = cStringIO() prog = ProgIter(range(N), clearline=False, file=file, freq=N // 10, adjust=False) file.seek(0) print(file.read()) prog = ProgIter(range(N), clearline=False) for n in prog: was_prime = is_prime(n) prog.set_extra('n=%r, was_prime=%r' % ( n, was_prime, )) if (n + 1) % 128 == 0 and was_prime: prog.set_extra('n=%r, was_prime=%r EXTRA' % ( n, was_prime, )) file.seek(0) print(file.read()) total = 200 # N = 5000 N = 500 N0 = N - total print('N = %r' % (N, )) print('N0 = %r' % (N0, )) print('\n-----') print('Demo #0: progress can be disabled and incur essentially 0 overhead') print('However, the overhead of enabled progress is minimal and typically ' 'insignificant') print('this is verbosity mode verbose=0') sequence = (is_prime(n) for n in range(N0, N)) with Timer('demo0'): psequence = ProgIter(sequence, total=total, desc='demo0', enabled=False) list(psequence) print('\n-----') print('Demo #1: progress is shown by default in the same line') print('this is verbosity mode verbose=1') sequence = (is_prime(n) for n in range(N0, N)) with Timer('demo1'): psequence = ProgIter(sequence, total=total, desc='demo1') list(psequence) # Default behavior adjusts frequency of progress reporting so # the performance of the loop is minimally impacted print('\n-----') print('Demo #2: clearline=False prints multiple lines.') print('Progress is only printed as needed') print('Notice the adjustment behavior of the print frequency') print('this is verbosity mode verbose=2') with Timer('demo2'): sequence = (is_prime(n) for n in range(N0, N)) psequence = ProgIter(sequence, total=total, clearline=False, desc='demo2') list(psequence) # import utool as ut # print(ut.repr4(psequence.__dict__)) print('\n-----') print('Demo #3: Adjustments can be turned off to give constant feedback') print('this is verbosity mode verbose=3') sequence = (is_prime(n) for n in range(N0, N)) with Timer('demo3'): psequence = ProgIter(sequence, total=total, adjust=False, clearline=False, freq=100, desc='demo3') list(psequence)
def test_cStringIO_read(self): with open(datafiles.PSF, "r") as f: obj = cStringIO(f.read()) assert_equal(util.isstream(obj), True) obj.close()
def pp(value): """ Utility method used to print the data nicely. """ output = cStringIO() PrettyPrinter(stream=output).pprint(value) return output.getvalue()
def __init__(self, srcdir, confdir, outdir, doctreedir, buildername, confoverrides=None, status=sys.stdout, warning=sys.stderr, freshenv=False, warningiserror=False, tags=None, verbosity=0, parallel=0): self.verbosity = verbosity self.next_listener_id = 0 self._extensions = {} self._extension_metadata = {} self._additional_source_parsers = {} self._listeners = {} self._setting_up_extension = ['?'] self.domains = {} self.buildername = buildername self.builderclasses = {} self.builder = None self.env = None self.enumerable_nodes = {} self.srcdir = srcdir self.confdir = confdir self.outdir = outdir self.doctreedir = doctreedir self.parallel = parallel if status is None: self._status = cStringIO() self.quiet = True else: self._status = status self.quiet = False if warning is None: self._warning = cStringIO() else: self._warning = warning self._warncount = 0 self.warningiserror = warningiserror self._events = events.copy() self._translators = {} # keep last few messages for traceback self.messagelog = deque(maxlen=10) # say hello to the world self.info(bold('Running Sphinx v%s' % sphinx.__display_version__)) # status code for command-line application self.statuscode = 0 if not path.isdir(outdir): self.info('making output directory...') os.makedirs(outdir) # read config self.tags = Tags(tags) self.config = Config(confdir, CONFIG_FILENAME, confoverrides or {}, self.tags) self.config.check_unicode(self.warn) # defer checking types until i18n has been initialized # initialize some limited config variables before loading extensions self.config.pre_init_values(self.warn) # check the Sphinx version if requested if self.config.needs_sphinx and self.config.needs_sphinx > sphinx.__display_version__: raise VersionRequirementError( 'This project needs at least Sphinx v%s and therefore cannot ' 'be built with this version.' % self.config.needs_sphinx) # force preload html_translator_class if self.config.html_translator_class: translator_class = self.import_object( self.config.html_translator_class, 'html_translator_class setting') self.set_translator('html', translator_class) # set confdir to srcdir if -C given (!= no confdir); a few pieces # of code expect a confdir to be set if self.confdir is None: self.confdir = self.srcdir # load all built-in extension modules for extension in builtin_extensions: self.setup_extension(extension) # extension loading support for alabaster theme # self.config.html_theme is not set from conf.py at here # for now, sphinx always load a 'alabaster' extension. if 'alabaster' not in self.config.extensions: self.config.extensions.append('alabaster') # load all user-given extension modules for extension in self.config.extensions: self.setup_extension(extension) # the config file itself can be an extension if self.config.setup: self._setting_up_extension = ['conf.py'] # py31 doesn't have 'callable' function for below check if hasattr(self.config.setup, '__call__'): self.config.setup(self) else: raise ConfigError( "'setup' that is specified in the conf.py has not been " + "callable. Please provide a callable `setup` function " + "in order to behave as a sphinx extension conf.py itself.") # now that we know all config values, collect them from conf.py self.config.init_values(self.warn) # check extension versions if requested if self.config.needs_extensions: for extname, needs_ver in self.config.needs_extensions.items(): if extname not in self._extensions: self.warn( 'needs_extensions config value specifies a ' 'version requirement for extension %s, but it is ' 'not loaded' % extname) continue has_ver = self._extension_metadata[extname]['version'] if has_ver == 'unknown version' or needs_ver > has_ver: raise VersionRequirementError( 'This project needs the extension %s at least in ' 'version %s and therefore cannot be built with the ' 'loaded version (%s).' % (extname, needs_ver, has_ver)) # check primary_domain if requested if self.config.primary_domain and self.config.primary_domain not in self.domains: self.warn('primary_domain %r not found, ignored.' % self.config.primary_domain) # set up translation infrastructure self._init_i18n() # check all configuration values for permissible types self.config.check_types(self.warn) # set up source_parsers self._init_source_parsers() # set up the build environment self._init_env(freshenv) # set up the builder self._init_builder(self.buildername) # set up the enumerable nodes self._init_enumerable_nodes()
def test_distributed_array_list_vars(self): size = 100 # how many items in the array prob = om.Problem() prob.model.add_subsystem('des_vars', om.IndepVarComp('x', np.ones(size)), promotes=['x']) prob.model.add_subsystem('plus', DistributedAdder(size=size), promotes=['x', 'y']) prob.model.add_subsystem('summer', Summer(size=size), promotes=['y', 'sum']) prob.setup() prob['x'] = np.arange(size) prob.run_driver() stream = cStringIO() inputs = sorted( prob.model.list_inputs(values=True, print_arrays=True, out_stream=stream)) self.assertEqual(inputs[0][0], 'plus.x') self.assertEqual(inputs[1][0], 'summer.y') self.assertEqual(inputs[0][1]['value'].size, 50) # should only return half that is local self.assertEqual(inputs[1][1]['value'].size, 100) text = stream.getvalue() if prob.comm.rank: # Only rank 0 prints self.assertEqual(len(text), 0) else: self.assertEqual(text.count('value'), 3) self.assertEqual(text.count('\nmodel'), 1) self.assertEqual(text.count('\n plus'), 1) self.assertEqual(text.count('\n x'), 1) self.assertEqual(text.count('\n summer'), 1) self.assertEqual(text.count('\n y'), 1) # make sure all the arrays written have 100 elements in them self.assertEqual(len(text.split('[')[1].split(']')[0].split()), 100) self.assertEqual(len(text.split('[')[2].split(']')[0].split()), 100) stream = cStringIO() outputs = sorted( prob.model.list_outputs(values=True, units=True, shape=True, bounds=True, residuals=True, scaling=True, hierarchical=True, print_arrays=True, out_stream=stream)) self.assertEqual(outputs[0][0], 'des_vars.x') self.assertEqual(outputs[1][0], 'plus.y') self.assertEqual(outputs[2][0], 'summer.sum') self.assertEqual(outputs[0][1]['value'].size, 100) self.assertEqual(outputs[1][1]['value'].size, 50) self.assertEqual(outputs[2][1]['value'].size, 1) text = stream.getvalue() if prob.comm.rank: # Only rank 0 prints self.assertEqual(len(text), 0) else: self.assertEqual(text.count('value'), 3) self.assertEqual(text.count('\n des_vars'), 1) self.assertEqual(text.count('\n x'), 1) self.assertEqual(text.count('\n plus'), 1) self.assertEqual(text.count('\n y'), 1) self.assertEqual(text.count('\n summer'), 1) self.assertEqual(text.count('\n sum'), 1) # make sure all the arrays written have 100 elements in them self.assertEqual(len(text.split('[')[1].split(']')[0].split()), 100) self.assertEqual(len(text.split('[')[2].split(']')[0].split()), 100) self.assertEqual(len(text.split('[')[3].split(']')[0].split()), 100) self.assertEqual(len(text.split('[')[4].split(']')[0].split()), 100)
def test_distribcomp_list_vars(self): from openmdao.test_suite.components.distributed_components import DistribComp, Summer print_opts = {'linewidth': 1024} from distutils.version import LooseVersion if LooseVersion(np.__version__) >= LooseVersion("1.14"): print_opts['legacy'] = '1.13' size = 15 model = om.Group() model.add_subsystem("indep", om.IndepVarComp('x', np.zeros(size))) model.add_subsystem("C2", DistribComp(size=size)) model.add_subsystem("C3", Summer(size=size)) model.connect('indep.x', 'C2.invec') model.connect('C2.outvec', 'C3.invec') prob = om.Problem(model) prob.setup() # prior to model execution, the global shape of a distributed variable is not available # and only the local portion of the value is available stream = cStringIO() with printoptions(**print_opts): model.C2.list_inputs(hierarchical=False, shape=True, global_shape=True, print_arrays=True, out_stream=stream) if prob.comm.rank == 0: text = stream.getvalue().split('\n') expected = [ "1 Input(s) in 'C2'", '------------------', '', 'varname value shape global_shape', '------- --------------- ----- ------------', 'invec |2.82842712475| (8,) Unavailable ', ' value:', ' array([1., 1., 1., 1., 1., 1., 1., 1.])' ] for i, line in enumerate(expected): if line and not line.startswith('-'): self.assertEqual( remove_whitespace(text[i]), remove_whitespace(line), '\nExpected: %s\nReceived: %s\n' % (line, text[i])) stream = cStringIO() with printoptions(**print_opts): model.C2.list_outputs(hierarchical=False, shape=True, global_shape=True, print_arrays=True, out_stream=stream) if prob.comm.rank == 0: text = stream.getvalue().split('\n') expected = [ "1 Explicit Output(s) in 'C2'", '----------------------------', '', 'varname value shape global_shape', '------- --------------- ----- ------------', 'outvec |2.82842712475| (8,) Unavailable ', ' value:', ' array([1., 1., 1., 1., 1., 1., 1., 1.])' ] for i, line in enumerate(expected): if line and not line.startswith('-'): self.assertEqual( remove_whitespace(text[i]), remove_whitespace(line), '\nExpected: %s\nReceived: %s\n' % (line, text[i])) # run the model prob['indep.x'] = np.ones(size) prob.run_model() # after model execution, the global shape of a distributed variable is available # and the complete global value is available stream = cStringIO() with printoptions(**print_opts): model.C2.list_inputs(hierarchical=False, shape=True, global_shape=True, print_arrays=True, out_stream=stream) if prob.comm.rank == 0: text = stream.getvalue().split('\n') expected = [ "1 Input(s) in 'C2'", '------------------', '', 'varname value shape global_shape', '-------- --------------- ----- ------------', 'C2.invec |3.87298334621| (8,) (15,) ', ' value:', ' array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])' ] for i, line in enumerate(expected): if line and not line.startswith('-'): self.assertEqual( remove_whitespace(text[i]), remove_whitespace(line), '\nExpected: %s\nReceived: %s\n' % (line, text[i])) stream = cStringIO() with printoptions(**print_opts): model.C2.list_outputs(hierarchical=False, shape=True, global_shape=True, print_arrays=True, out_stream=stream) if prob.comm.rank == 0: text = stream.getvalue().split('\n') expected = [ "1 Explicit Output(s) in 'C2'", '----------------------------', '', 'varname value shape global_shape', '--------- -------------- ----- ------------', 'C2.outvec |9.74679434481| (8,) (15,) ', ' value:', ' array([ 2., 2., 2., 2., 2., 2., 2., 2., -3., -3., -3., -3., -3., -3., -3.])' ] for i, line in enumerate(expected): if line and not line.startswith('-'): self.assertEqual( remove_whitespace(text[i]), remove_whitespace(line), '\nExpected: %s\nReceived: %s\n' % (line, text[i])) # note that the shape of the input variable for the non-distributed Summer component # is different on each processor, use the all_procs argument to display on all processors stream = cStringIO() with printoptions(**print_opts): model.C3.list_inputs(hierarchical=False, shape=True, global_shape=True, all_procs=True, print_arrays=True, out_stream=stream) text = stream.getvalue().split('\n') if prob.comm.rank == 0: norm = '|5.65685424949|' shape = (8, ) value = '[2., 2., 2., 2., 2., 2., 2., 2.]' else: norm = '|7.93725393319|' shape = (7, ) value = '[-3., -3., -3., -3., -3., -3., -3.]' expected = [ "1 Input(s) in 'C3'", '------------------', '', 'varname value shape global_shape', '-------- ------------------- ----- ------------', 'C3.invec {} {} {} '.format(norm, shape, shape), ' value:', ' array({})'.format(value), ] for i, line in enumerate(expected): if line and not line.startswith('-'): self.assertEqual( remove_whitespace(text[i]), remove_whitespace(line), '\nExpected: %s\nReceived: %s\n' % (line, text[i])) assert_rel_error(self, prob['C3.out'], -5.)
def test_define_funs_arg_and_fun(self): smtlib_script = "\n".join( ['(define-fun f ((n Int)) Int n)', '(declare-fun n () Real)']) stream = cStringIO(smtlib_script) parser = SmtLibParser() script = parser.get_script(stream)
def write_main(argv): """ write FILENAME Write a local copy of FILENAME using FILENAME_tweaks for local tweaks. """ if len(argv) != 1: print("Please provide the name of a file to write.") return 1 filename = argv[0] resource_name = "files/" + filename tweaks_name = amend_filename(filename, "_tweaks") if not pkg_resources.resource_exists("edx_lint", resource_name): print("Don't have file %r to write." % filename) return 2 if os.path.exists(filename): print("Checking existing copy of %s" % filename) tef = TamperEvidentFile(filename) if not tef.validate(): bak_name = amend_filename(filename, "_backup") print( "Your copy of %s seems to have been edited, renaming it to %s" % (filename, bak_name)) if os.path.exists(bak_name): print("A previous %s exists, deleting it" % bak_name) os.remove(bak_name) os.rename(filename, bak_name) print("Reading edx_lint/files/%s" % filename) cfg = configparser.RawConfigParser() resource_string = pkg_resources.resource_string( "edx_lint", resource_name).decode("utf8") # pkg_resources always reads binary data (in both python2 and python3). # ConfigParser.read_string only exists in python3, so we have to wrap the string # from pkg_resources in a cStringIO so that we can pass it into ConfigParser.readfp. cfg.readfp(cStringIO(resource_string), resource_name) # pylint: disable=deprecated-method if os.path.exists(tweaks_name): print("Applying local tweaks from %s" % tweaks_name) cfg_tweaks = configparser.RawConfigParser() cfg_tweaks.read([tweaks_name]) merge_configs(cfg, cfg_tweaks) print("Writing %s" % filename) output_text = cStringIO() output_text.write( WARNING_HEADER.format(filename=filename, tweaks_name=tweaks_name)) cfg.write(output_text) out_tef = TamperEvidentFile(filename) if six.PY2: output_bytes = output_text.getvalue() else: output_bytes = output_text.getvalue().encode("utf8") out_tef.write(output_bytes) return 0
import sys import pysmt from pysmt.rewritings import PrenexNormalizer, Ackermannizer from pysmt.smtlib.script import SmtLibScript from pysmt.smtlib.parser import SmtLibParser from pysmt.shortcuts import to_smtlib, write_smtlib from six.moves import cStringIO parser = SmtLibParser() with open("/home/yoniz/git/hermes/dispatcher/dispatcher/examples/Assessment2/nodtbbg.smt2", 'r') as f: smtlib_str = f.read(); stream = cStringIO(smtlib_str) script = parser.get_script(stream) formula = script.get_last_formula() ackermanization = Ackermannizer() ackermized_formula = ackermanization.do_ackermannization(formula) write_smtlib(ackermized_formula, "/home/yoniz/git/hermes/dispatcher/dispatcher/examples/Assessment2/nodtbbg_ack.smt2" )
"""See setuptools' test_runner setup argument for information.""" # only run test cases with id starting with given prefix testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER') filtered_cases = [] for case in _loader.iterate_suite_cases(suite): if not testcase_filter or case.id().startswith(testcase_filter): filtered_cases.append(case) # Ensure that every test case has no collision with any other test case in # the augmented results. augmented_cases = [ AugmentedCase(case, uuid.uuid4()) for case in filtered_cases ] case_id_by_case = dict((augmented_case.case, augmented_case.id) for augmented_case in augmented_cases) result_out = moves.cStringIO() result = _result.TerminalResult( result_out, id_map=lambda case: case_id_by_case[case]) stdout_pipe = CaptureFile(sys.stdout.fileno()) stderr_pipe = CaptureFile(sys.stderr.fileno()) kill_flag = [False] def sigint_handler(signal_number, frame): if signal_number == signal.SIGINT: kill_flag[0] = True # Python 2.7 not having 'local'... :-( signal.signal(signal_number, signal.SIG_DFL) def fault_handler(signal_number, frame): stdout_pipe.write_bypass( 'Received fault signal {}\nstdout:\n{}\n\nstderr:{}\n'.format( signal_number, stdout_pipe.output(), stderr_pipe.output()))
import sys import time import json from six.moves import cStringIO from pysmt.shortcuts import Solver from pysmt.smtlib.parser import SmtLibParser if __name__ == '__main__': filename = sys.argv[1] solver_name = sys.argv[2] smt_file = open(filename, 'r').read().replace('\r', "") parser = SmtLibParser() script = parser.get_script(cStringIO(smt_file)) # with Solver(name=solver_name) as solver: solver = Solver(name=solver_name) error = False s = time.time() try: log = script.evaluate(solver) e = time.time() except: e = time.time() error = True log = [] """ print json.dumps({
def __init__(self, *args, **kwds): self._buffer = cStringIO(*args, **kwds) self.read_only = bool(args)
def create_NamedStream(self, name=None): if name is None: name = self.textname obj = cStringIO() return util.NamedStream(obj, name)
def test_cStringIO_write(self): obj = cStringIO() assert_equal(util.isstream(obj), True) obj.close()
def exercise(): from libtbx.test_utils import show_diff, Exception_expected from six.moves import cPickle as pickle # from libtbx.str_utils import split_keeping_spaces assert split_keeping_spaces(s="") == [] assert split_keeping_spaces(s=" ") == [" "] assert split_keeping_spaces(s="a") == ["a"] assert split_keeping_spaces(s="abc") == ["abc"] assert split_keeping_spaces(s=" a") == [" ", "a"] assert split_keeping_spaces(s=" a") == [" ", "a"] assert split_keeping_spaces(s=" abc") == [" ", "abc"] assert split_keeping_spaces(s=" abc ") == [" ", "abc", " "] assert split_keeping_spaces(s=" abc ") == [" ", "abc", " "] assert split_keeping_spaces(s="a ") == ["a", " "] assert split_keeping_spaces(s="a ") == ["a", " "] assert split_keeping_spaces(s="abc ") == ["abc", " "] assert split_keeping_spaces(s="a b") == ["a", " ", "b"] assert split_keeping_spaces(s="a b") == ["a", " ", "b"] assert split_keeping_spaces(s=" a b c d ") == [ " ", "a", " ", "b", " ", "c", " ", "d", " " ] # from libtbx.str_utils import size_as_string_with_commas assert size_as_string_with_commas(0) == "0" assert size_as_string_with_commas(1) == "1" assert size_as_string_with_commas(-1) == "-1" assert size_as_string_with_commas(10) == "10" assert size_as_string_with_commas(100) == "100" assert size_as_string_with_commas(1000) == "1,000" assert size_as_string_with_commas(12345) == "12,345" assert size_as_string_with_commas(12345678) == "12,345,678" assert size_as_string_with_commas(-12345678) == "-12,345,678" # from libtbx.str_utils import show_string assert show_string("abc") == '"abc"' assert show_string("a'c") == '"a\'c"' assert show_string('a"c') == "'a\"c'" assert show_string('\'"c') == '"\'\\"c"' # from libtbx.str_utils import prefix_each_line assert prefix_each_line(prefix="^", lines_as_one_string="""\ hello world""") == """\ ^hello ^world""" # from libtbx.str_utils import prefix_each_line_suffix assert prefix_each_line_suffix(prefix="^", lines_as_one_string="""\ hello world""", suffix=" ") == """\ ^hello ^world""" assert prefix_each_line_suffix(prefix="^", lines_as_one_string="""\ hello world""", suffix=" ", rstrip=False) == """\ ^hello%s ^world """ % " " # from libtbx.str_utils import show_sorted_by_counts from six.moves import cStringIO out = cStringIO() assert show_sorted_by_counts(label_count_pairs=[("b", 3), ("a", 3), ("c", -2)], out=out, prefix="%") assert not show_diff(out.getvalue(), """\ %"a" 3 %"b" 3 %"c" -2 """) out = cStringIO() assert show_sorted_by_counts(label_count_pairs=[("b", -3), ("a", -3), ("c", 2)], reverse=False, out=out, prefix="%", annotations=[None, "", "x"]) assert not show_diff(out.getvalue(), """\ %"a" -3 %"b" -3 %"c" 2 x """) # from libtbx.str_utils import line_breaker for string, expected_result in [ ("", [""]), ("this is", ["this is"]), ("this is a", ["this is", "a"]), ("this is a sentence", ["this is", "a", "sentence"]), ("this is a longer sentence", ["this is", "a", "longer", "sentence"]), ("this is a very long sentence indeed", ["this is", "a very", "long", "sentence", "indeed"]) ]: assert [block for block in line_breaker(string, width=7)] == expected_result # from libtbx.str_utils import StringIO out1 = cStringIO() out2 = StringIO() out3 = StringIO("Hello world!\n") print("Hello world!", file=out1) print("Hello world!", file=out2) try: print("Hello world!", file=out3) except AttributeError: pass else: raise Exception_expected out4 = pickle.loads(pickle.dumps(out2)) out5 = pickle.loads(pickle.dumps(out3)) assert out4.getvalue() == out1.getvalue() == out2.getvalue( ) == out5.getvalue() # from libtbx.str_utils import reformat_terminal_text txt1 = """ This is some terminal-formatted text which needs to be reset. """ assert (reformat_terminal_text(txt1) == "This is some terminal-formatted text which needs to be reset.") txt2 = """ This is more terminal-formatted text which needs to be reset. """ # from libtbx.str_utils import strip_lines, rstrip_lines lines = [" This is more ", " terminal-formatted ", " text "] assert ( strip_lines(txt2) == "\nThis is more\nterminal-formatted\ntext which needs\nto be reset.") assert ( rstrip_lines(txt2) == "\n This is more\n terminal-formatted\n text which needs\n to be reset." ) # from libtbx.str_utils import expandtabs_track_columns def check(s): es, js = expandtabs_track_columns(s=s) assert len(js) == len(s) assert es == s.expandtabs() sr = "".join([es[j] for j in js]) assert sr == s.replace("\t", " ") check("") check("\t") check("\t\t") check("\ty") check("x\ty") check("x\ty\tz") check("\txy\t\tz") check("abcdefg\txy\t\tz") check("ab defgh\txyz\t\tu") # from libtbx.str_utils import format_value assert format_value("%.4f", 1.2345678) == "1.2346" assert format_value("%.4f", None) == " None" assert format_value("%.4f", None, replace_none_with="---") == " ---" # from libtbx.str_utils import make_header out = StringIO() make_header("Header 1", out=out) assert (out.getvalue() == """ =================================== Header 1 ================================== """) out = StringIO() make_header("Header 2", out=out) assert (out.getvalue() == """ =================================== Header 2 ================================== """) # import sys from libtbx.str_utils import string_representation iset = list(range(130)) + list(range(250, 256)) for i in iset: s = chr(i) for j in iset: ss = s + chr(j) sr = string_representation(string=ss, preferred_quote="'", alternative_quote='"') if sys.hexversion < 0x03000000: assert sr == repr(ss) else: assert eval(sr) == ss from libtbx.str_utils import framed_output out = StringIO() box = framed_output(out, frame='#') print("Hello, world!", file=box) box.close() assert (out.getvalue() == """ ################# # Hello, world! # ################# """) out = StringIO() box = framed_output(out, frame='-', width=80, center=True, title="Refinement stats") box.write("r_free = 0.1234") box.write(" ") box.write("r_work = 0.1567") box.close() assert (out.getvalue() == """ |--------------------------------Refinement stats------------------------------| | r_free = 0.1234 r_work = 0.1567 | |------------------------------------------------------------------------------| """) out = StringIO() box = framed_output(out, frame='-', width=72, prefix=" ", title="Validation summary") print("Overall MolProbity score: 2.56", file=box) box.add_separator() print("""\ Ramachandran favored: 97.5 % outliers: 2.5 % Rotamer outliers: 5.9 % Clashscore: 10.9""", file=box) assert (out.getvalue() == "") del box assert (out.getvalue() == """ |-Validation summary---------------------------------------------------| | Overall MolProbity score: 2.56 | |----------------------------------------------------------------------| | Ramachandran favored: 97.5 % | | outliers: 2.5 % | | Rotamer outliers: 5.9 % | | Clashscore: 10.9 | |----------------------------------------------------------------------| """) from libtbx.str_utils import print_message_in_box out = StringIO() print_message_in_box( message="This is some terminal-formatted text which needs to be reset.", out=out, width=32, center=True, prefix=" ", frame='*') assert (out.getvalue() == """ ******************************** * This is some * * terminal-formatted text * * which needs to be reset. * ******************************** """) from libtbx.str_utils import make_big_header out = StringIO() make_big_header("Section title", out=out) assert (out.getvalue() == """ ################################################################################ # Section title # ################################################################################ """)
def as_cStringIO(self, name): return cStringIO(self.buffers[name])