def test_start_logger(self): sio = StringIO() logger = logging.getLogger('CONT #abcdef: test') logger.addHandler(logging.StreamHandler(sio)) # set log level as INFO logger = start_logger('test', 'INFO', 'abcdef') self.assertEqual(logging.INFO, logger.level) # INFO message is recorded with INFO leg level logger.info('test1') self.assertEqual(sio.getvalue(), 'test1\n') # DEBUG message is not recorded with INFO leg level logger.debug('test2') self.assertEqual(sio.getvalue(), 'test1\n') # set log level as DEBUG logger = start_logger('test', 'DEBUG', 'abcdef') self.assertEqual(logging.DEBUG, logger.level) # DEBUG message is recorded with DEBUG leg level logger.debug('test3') self.assertEqual(sio.getvalue(), 'test1\ntest3\n') # If the level parameter is unkown, use ERROR as log level logger = start_logger('test', 'foo', 'abcdef') self.assertEqual(logging.ERROR, logger.level)
def test_json_format_one(): sf = json_format.JSONFormatter() c = ('a', 'b', 'c', 'd') d = ('A', 'B', 'C', '"escape me"') expected = { 'a': 'A', 'b': 'B', 'c': 'C', 'd': '"escape me"' } args = mock.Mock() sf.add_argument_group(args) args.noindent = True output = StringIO() sf.emit_one(c, d, output, args) value = output.getvalue() print(len(value.splitlines())) assert 1 == len(value.splitlines()) actual = json.loads(value) assert expected == actual args.noindent = False output = StringIO() sf.emit_one(c, d, output, args) value = output.getvalue() assert 6 == len(value.splitlines()) actual = json.loads(value) assert expected == actual
def test_any_sequence_to_fasta(self): # store writer function, sequence object to write, expected # fasta filepath for default parameters, expected fasta filepath for # non-defaults, and expected qual filepath for non-defaults id_ = 'f o o' desc = 'b\na\nr' test_data = ( (_biological_sequence_to_fasta, Sequence('ACGT', id=id_, description=desc, quality=range(1, 5)), ('fasta_single_bio_seq_defaults', 'fasta_single_bio_seq_non_defaults', 'qual_single_bio_seq_non_defaults')), (_dna_sequence_to_fasta, DNA('TACG', id=id_, description=desc, quality=range(4)), ('fasta_single_dna_seq_defaults', 'fasta_single_dna_seq_non_defaults', 'qual_single_dna_seq_non_defaults')), (_rna_sequence_to_fasta, RNA('UACG', id=id_, description=desc, quality=range(2, 6)), ('fasta_single_rna_seq_defaults', 'fasta_single_rna_seq_non_defaults', 'qual_single_rna_seq_non_defaults')), (_protein_sequence_to_fasta, Protein('PQQ', id=id_, description=desc, quality=[42, 41, 40]), ('fasta_single_prot_seq_defaults', 'fasta_single_prot_seq_non_defaults', 'qual_single_prot_seq_non_defaults'))) for fn, obj, fps in test_data: defaults_fp, non_defaults_fasta_fp, non_defaults_qual_fp = fps # test writing with default parameters fh = StringIO() fn(obj, fh) obs = fh.getvalue() fh.close() with open(get_data_path(defaults_fp), 'U') as fh: exp = fh.read() self.assertEqual(obs, exp) # test writing with non-defaults fasta_fh = StringIO() qual_fh = StringIO() fn(obj, fasta_fh, id_whitespace_replacement='-', description_newline_replacement='_', max_width=1, qual=qual_fh) obs_fasta = fasta_fh.getvalue() obs_qual = qual_fh.getvalue() fasta_fh.close() qual_fh.close() with open(get_data_path(non_defaults_fasta_fp), 'U') as fh: exp_fasta = fh.read() with open(get_data_path(non_defaults_qual_fp), 'U') as fh: exp_qual = fh.read() self.assertEqual(obs_fasta, exp_fasta) self.assertEqual(obs_qual, exp_qual)
def test_prints_signature(self): the_time = 1406143563.020043 key = 'secret squirrel' expires = 3600 path = '/v1/a/c/o' redirect = 'https://example.com/done.html' max_file_size = str(int(1024 * 1024 * 1024 * 3.14159)) # π GiB max_file_count = '3' expected_signature = hmac.new( key, "\n".join(( path, redirect, max_file_size, max_file_count, str(int(the_time + expires)))), hashlib.sha1).hexdigest() out = StringIO() with mock.patch('swift.cli.form_signature.time', lambda: the_time): with mock.patch('sys.stdout', out): exitcode = form_signature.main([ '/path/to/swift-form-signature', path, redirect, max_file_size, max_file_count, str(expires), key]) self.assertEqual(exitcode, 0) self.assertTrue("Signature: %s" % expected_signature in out.getvalue()) self.assertTrue("Expires: %d" % (the_time + expires,) in out.getvalue()) sig_input = ('<input type="hidden" name="signature" value="%s" />' % expected_signature) self.assertTrue(sig_input in out.getvalue())
def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir): load_tool.loaders = {} document_loader, workflowobj, uri = fetch_document( get_data(wf_path), resolver=tool_resolver) document_loader, _, processobj, metadata, uri = validate_document( document_loader, workflowobj, uri, [], {}) packed = json.loads(print_pack(document_loader, processobj, uri, metadata)) assert not namespaced or "$namespaces" in packed wf_packed_handle, wf_packed_path = tempfile.mkstemp() with open(wf_packed_path, 'w') as temp_file: json.dump(packed, temp_file) normal_output = StringIO() packed_output = StringIO() normal_params = ['--outdir', str(tmpdir), get_data(wf_path), get_data(job_path)] packed_params = ['--outdir', str(tmpdir), '--debug', get_data(wf_packed_path), get_data(job_path)] assert main(normal_params, stdout=normal_output) == 0 assert main(packed_params, stdout=packed_output) == 0 assert json.loads(packed_output.getvalue()) == json.loads(normal_output.getvalue()) os.close(wf_packed_handle) os.remove(wf_packed_path)
class TestUI(object): def setup(self): self.out = StringIO() self.err = StringIO() self.ui = utils.UI(self.out, self.err) def test_write_goes_to_out_obj(self): self.ui.write("Foo") assert self.out.getvalue() == 'Foo' assert self.err.getvalue() == '' def test_error_goes_to_err_obj(self): self.ui.error("Foo") assert self.err.getvalue() == 'Foo' assert self.out.getvalue() == '' def test_confirm_raises_own_exception(self): confirm = mock.Mock(spec=click.confirm) confirm.side_effect = click.Abort() ui = utils.UI(self.out, self.err, confirm) with pytest.raises(utils.AbortedError): ui.confirm("Confirm?") def test_confirm_returns_value(self): confirm = mock.Mock(spec=click.confirm) confirm.return_value = 'foo' ui = utils.UI(self.out, self.err, confirm) return_value = ui.confirm("Confirm?") assert return_value == 'foo'
class _Buffer(object): def __init__(self, stream): self._stream = stream self._buffer = StringIO() def fileno(self): return self._stream.fileno() def __getattr__(self, attr): # this happens on unpickling if attr == '_buffer': raise AttributeError("No _buffer yet") return getattr(self._buffer, attr) def __le__(self, obj): return self._buffer.getvalue() == obj def __eq__(self, obj): return self._buffer.getvalue() == obj def __str__(self): return self._buffer.getvalue() def __repr__(self): return repr(self._buffer.getvalue())
def _build_tag(tag, hide_attributes): tag_el = _find_tag_el(tag) attributes = _find_attributes(tag) tag_help = StringIO() annotation_el = tag_el.find("{http://www.w3.org/2001/XMLSchema}annotation") text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text for line in text.splitlines(): if line.startswith("$attribute_list:"): attributes_str, header_level = line.split(":")[1:3] attribute_names = attributes_str.split(",") header_level = int(header_level) text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) if line.startswith("$assertions"): assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']") assertions_buffer = StringIO() assertions_buffer.write(_doc_or_none(assertions_tag)) assertions_buffer.write("\n\n") assertions_buffer.write("Child Element/Assertion | Details \n") assertions_buffer.write("--- | ---\n") elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") for element in elements: doc = _doc_or_none(element).strip() assertions_buffer.write("``%s`` | %s\n" % (element.attrib["name"], doc)) text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) best_practices = _get_bp_link(annotation_el) if best_practices: tag_help.write("\n\n### Best Practices\n") tag_help.write(""" Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices) tag_help.write(_build_attributes_table(tag, attributes, hide_attributes)) return tag_help.getvalue()
def test_invalid_project_name(self): #with patch('sys.stdout', self.stdout): stderr_tmp = StringIO() with patch('sys.stderr', stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse([ '-q', '--db=postgres://user:pwd@host/dbname', '-p'+self.project_dir, 'test']) self.assertTrue(stderr_tmp.getvalue().find("Project name 'test' is not valid") > -1) stderr_tmp = StringIO() with patch('sys.stderr', stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse([ '-q', '--db=postgres://user:pwd@host/dbname', '-p'+self.project_dir, 'assert']) self.assertTrue(stderr_tmp.getvalue().find("Project name 'assert' is not valid") > -1) stderr_tmp = StringIO() with patch('sys.stderr', stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse([ '-q', '--db=postgres://user:pwd@host/dbname', '-p'+self.project_dir, 'values']) self.assertTrue(stderr_tmp.getvalue().find("Project name 'values' is not valid") > -1)
def test_info_unicode(self, depfile): output = StringIO() task = Task("t1", [], file_dep=[six.u('tests/data/dependency1')]) cmd = Info(outstream=output, dep_file=depfile.name, task_list=[task]) cmd._execute(['t1']) assert """name:'t1'""" in output.getvalue() assert """'tests/data/dependency1'""" in output.getvalue()
def _in_process_execute(self, cli_ctx, command, expect_failure=False): from six import StringIO from vcr.errors import CannotOverwriteExistingCassetteException if command.startswith('az '): command = command[3:] stdout_buf = StringIO() logging_buf = StringIO() try: # issue: stderr cannot be redirect in this form, as a result some failure information # is lost when command fails. self.exit_code = cli_ctx.invoke(shlex.split(command), out_file=stdout_buf) or 0 self.output = stdout_buf.getvalue() self.applog = logging_buf.getvalue() except CannotOverwriteExistingCassetteException as ex: raise AssertionError(ex) except CliExecutionError as ex: if expect_failure: self.exit_code = 1 self.output = stdout_buf.getvalue() self.applog = logging_buf.getvalue() elif ex.exception: raise ex.exception else: raise ex except Exception as ex: # pylint: disable=broad-except self.exit_code = 1 self.output = stdout_buf.getvalue() self.process_error = ex finally: stdout_buf.close() logging_buf.close()
def test_var(self): m = ConcreteModel() m.x = Var(bounds=(-1, 1), initialize=3) mc_var = mc(m.x) self.assertEqual(mc_var.lower(), -1) self.assertEqual(mc_var.upper(), 1) m.no_ub = Var(bounds=(0, None), initialize=3) output = StringIO() with LoggingIntercept(output, 'pyomo.contrib.mcpp', logging.WARNING): mc_var = mc(m.no_ub) self.assertIn("Var no_ub missing upper bound.", output.getvalue().strip()) self.assertEqual(mc_var.lower(), 0) self.assertEqual(mc_var.upper(), 500000) m.no_lb = Var(bounds=(None, -3), initialize=-1) output = StringIO() with LoggingIntercept(output, 'pyomo.contrib.mcpp', logging.WARNING): mc_var = mc(m.no_lb) self.assertIn("Var no_lb missing lower bound.", output.getvalue().strip()) self.assertEqual(mc_var.lower(), -500000) self.assertEqual(mc_var.upper(), -3) m.no_val = Var(bounds=(0, 1)) output = StringIO() with LoggingIntercept(output, 'pyomo.contrib.mcpp', logging.WARNING): mc_var = mc(m.no_val) mc_var.subcv() self.assertIn("Var no_val missing value.", output.getvalue().strip()) self.assertEqual(mc_var.lower(), 0) self.assertEqual(mc_var.upper(), 1)
def test_roundtrip_generators(self): # test that fasta and qual files can be streamed into memory and back # out to disk using generator reader and writer fps = list(map(lambda e: list(map(get_data_path, e)), [('empty', 'empty'), ('fasta_multi_seq_roundtrip', 'qual_multi_seq_roundtrip')])) for fasta_fp, qual_fp in fps: with open(fasta_fp, 'U') as fh: exp_fasta = fh.read() with open(qual_fp, 'U') as fh: exp_qual = fh.read() fasta_fh = StringIO() qual_fh = StringIO() _generator_to_fasta(_fasta_to_generator(fasta_fp, qual=qual_fp), fasta_fh, qual=qual_fh) obs_fasta = fasta_fh.getvalue() obs_qual = qual_fh.getvalue() fasta_fh.close() qual_fh.close() self.assertEqual(obs_fasta, exp_fasta) self.assertEqual(obs_qual, exp_qual)
class TestRexlexTraceState(unittest.TestCase): expected = 'rexlex: test' def setUp(self): self.stderr = StringIO() self.real_stderr = sys.stderr logger.handlers[0].stream = self.stderr logger.setLevel(rexlex.log_config.REXLEX_TRACE_STATE) def tearDown(self): logger.handlers[0].stream = self.real_stderr def test_trace_result(self): logger.rexlex_trace_result("test") self.assertIn(self.expected, self.stderr.getvalue()) def test_trace_meta(self): logger.rexlex_trace_meta("test") self.assertIn(self.expected, self.stderr.getvalue()) def test_trace_state(self): logger.rexlex_trace_state("test") self.assertIn(self.expected, self.stderr.getvalue()) def test_trace_rule(self): logger.rexlex_trace_rule("test") self.assertNotIn(self.expected, self.stderr.getvalue()) def test_trace(self): logger.rexlex_trace("test") self.assertNotIn(self.expected, self.stderr.getvalue())
def write_to_db(self, db, transaction=None, commit=True): if transaction is None: transaction = db fp = StringIO() if len(self) < Timeseries.MAX_ALL_BOTTOM: top = '' middle = None self.write(fp) bottom = fp.getvalue() else: dates = sorted(self.keys()) self.write(fp, end=dates[Timeseries.ROWS_IN_TOP_BOTTOM - 1]) top = fp.getvalue() fp.truncate(0) fp.seek(0) self.write(fp, start=dates[Timeseries.ROWS_IN_TOP_BOTTOM], end=dates[-(Timeseries.ROWS_IN_TOP_BOTTOM + 1)]) middle = self.blob_create( zlib.compress(fp.getvalue().encode('ascii'))) fp.truncate(0) fp.seek(0) self.write(fp, start=dates[-Timeseries.ROWS_IN_TOP_BOTTOM]) bottom = fp.getvalue() fp.close() c = db.cursor() c.execute("DELETE FROM ts_records WHERE id=%d" % (self.id)) c.execute("""INSERT INTO ts_records (id, top, middle, bottom) VALUES (%s, %s, %s, %s)""", (self.id, top, middle, bottom)) c.close() if commit: transaction.commit()
def toXml(self, filename='', compress=False): xml = StringIO() xml.write("<?xml version='1.0' encoding='UTF-8'?>\n") xml.write( "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd \">\n") self.svg.toXml(0, xml) if not filename: if compress: import gzip f = StringIO() zf = gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) return f.read() else: return xml.getvalue() else: if filename[-4:] == 'svgz': import gzip f = gzip.GzipFile( filename=filename, mode="wb", compresslevel=9) f.write(xml.getvalue()) f.close() else: f = file(filename, 'w') f.write(xml.getvalue()) f.close()
def test_commands(self): out = StringIO() call_command('billing_check', stdout=out) self.assertEqual(out.getvalue(), '') self.add_project() self.add_project() out = StringIO() call_command('billing_check', stdout=out) self.assertEqual( out.getvalue(), 'Following billings are over limit:\n' ' * test0, test1 (test)\n' ) out = StringIO() call_command('billing_check', '--valid', stdout=out) self.assertEqual(out.getvalue(), '') self.invoice.delete() out = StringIO() call_command('billing_check', stdout=out) self.assertEqual( out.getvalue(), 'Following billings are over limit:\n' ' * test0, test1 (test)\n' 'Following billings are past due date:\n' ' * test0, test1 (test)\n' ) call_command('billing_check', '--notify', stdout=out) self.assertEqual(len(mail.outbox), 1)
def decide_notebook_merge(base, local, remote, args=None): # Build merge strategies for each document path from arguments strategies = notebook_merge_strategies(args) # Compute notebook specific diffs local_diffs = diff_notebooks(base, local) remote_diffs = diff_notebooks(base, remote) # Debug outputs if args and args.log_level == "DEBUG": nbdime.log.debug("In merge, base-local diff:") buf = StringIO() pretty_print_notebook_diff("<base>", "<local>", base, local_diffs, buf) nbdime.log.debug(buf.getvalue()) nbdime.log.debug("In merge, base-remote diff:") buf = StringIO() pretty_print_notebook_diff("<base>", "<remote>", base, remote_diffs, buf) nbdime.log.debug(buf.getvalue()) # Execute a generic merge operation decisions = decide_merge_with_diff( base, local, remote, local_diffs, remote_diffs, strategies) # Debug outputs if args and args.log_level == "DEBUG": nbdime.log.debug("In merge, decisions:") buf = StringIO() pretty_print_merge_decisions(base, decisions, buf) nbdime.log.debug(buf.getvalue()) return decisions
def test_any_sequences_to_fasta(self): for fn, obj in ((_sequence_collection_to_fasta, self.seq_coll), (_alignment_to_fasta, self.align)): # test writing with default parameters fh = StringIO() fn(obj, fh) obs = fh.getvalue() fh.close() with open(get_data_path('fasta_3_seqs_defaults'), 'U') as fh: exp = fh.read() self.assertEqual(obs, exp) # test writing with non-defaults fasta_fh = StringIO() qual_fh = StringIO() fn(obj, fasta_fh, id_whitespace_replacement='*', description_newline_replacement='+', max_width=3, qual=qual_fh) obs_fasta = fasta_fh.getvalue() obs_qual = qual_fh.getvalue() fasta_fh.close() qual_fh.close() with open(get_data_path('fasta_3_seqs_non_defaults'), 'U') as fh: exp_fasta = fh.read() with open(get_data_path('qual_3_seqs_non_defaults'), 'U') as fh: exp_qual = fh.read() self.assertEqual(obs_fasta, exp_fasta) self.assertEqual(obs_qual, exp_qual)
def _in_process_execute(self, command): # from azure.cli import as cli_main from azure.cli.main import main as cli_main from six import StringIO from vcr.errors import CannotOverwriteExistingCassetteException if command.startswith('az '): command = command[3:] output_buffer = StringIO() try: # issue: stderr cannot be redirect in this form, as a result some failure information # is lost when command fails. self.exit_code = cli_main(shlex.split(command), file=output_buffer) or 0 self.output = output_buffer.getvalue() except CannotOverwriteExistingCassetteException as ex: raise AssertionError(ex) except CliExecutionError as ex: if ex.exception: raise ex.exception else: raise ex except Exception as ex: # pylint: disable=broad-except self.exit_code = 1 self.output = output_buffer.getvalue() self.process_error = ex finally: output_buffer.close()
def help_cmd_test(self): client = TestClient() try: old_stdout = sys.stdout result = StringIO() sys.stdout = result client.run("help new") finally: sys.stdout = old_stdout self.assertIn("Creates a new package recipe template with a 'conanfile.py'", result.getvalue()) try: old_stdout = sys.stdout result = StringIO() sys.stdout = result client.run("help build") finally: sys.stdout = old_stdout self.assertIn("Calls your local conanfile.py 'build()' method", result.getvalue()) client.run("help") self.assertIn("Creator commands", client.out)
def no_configuration_test(self): dummy = """GlobalSection EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection """ folder = temp_folder() path = os.path.join(folder, "dummy.sln") save(path, dummy) new_out = StringIO() tools.set_global_instances(ConanOutput(new_out), None) command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False, build_type='Debug', arch="x86", parallel=False) self.assertIn('/p:Configuration=Debug /p:Platform="x86"', command) self.assertIn("WARN: ***** The configuration Debug|x86 does not exist in this solution *****", new_out.getvalue()) # use platforms new_out = StringIO() tools.set_global_instances(ConanOutput(new_out), None) command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False, build_type='Debug', arch="x86", parallel=False, platforms={"x86": "Win32"}) self.assertIn('/p:Configuration=Debug /p:Platform="Win32"', command) self.assertNotIn("WARN", new_out.getvalue()) self.assertNotIn("ERROR", new_out.getvalue())
def test_trap_output(self): class BorkedOutput(outputs.ListOutput): def _write(self, x): raise RuntimeError("BORK") out = BorkedOutput(close_atexit=False) sio = StringIO() def cleanup(stderr, output): sys.stderr = stderr sio.close() self.log.output = output out.close() self.addCleanup(cleanup, sys.stderr, self.log.output) sys.stderr = sio self.log.output = out self.log.fields().info('hi') assert "BORK" in sio.getvalue() assert "Offending message: <twiggy.message.Message object" in sio.getvalue() assert "Error in twiggy internal log! Something is seriously broken." in sio.getvalue() assert "Traceback" in sio.getvalue()
class TestBuild(unittest.TestCase): def setUp(self): self.out = StringIO() os.mkdir('/tmp/resu') # Append uuid os.chdir('/tmp/resu') def test_basic(self): resu.cli.run(args=['-g'], out=self.out) resu.cli.run(args=[], out=self.out) assert os.path.isfile('resu.pdf') self.assertEquals(self.out.getvalue(), '') def test_specify_output_file(self): resu.cli.run(args=['-g'], out=self.out) resu.cli.run(args=['-o', 'other.html'], out=self.out) assert os.path.isfile('other.html') self.assertEquals(self.out.getvalue(), '') def test_non_existant_parser(self): resu.cli.run(args=['-g'], out=self.out) with self.assertRaises(FeatureNotFound): resu.cli.run(args=['-p', 'gibberish'], out=self.out) assert not os.path.isfile('resu.pdf') def tearDown(self): shutil.rmtree('/tmp/resu')
def understand_buffer_usage_and_performance(): multiplier = 1000 * 100 big_data = data * multiplier # no buffer st = time.time() big_data_json = json.dumps(big_data) elapse = time.time() - st print("%.6f" % elapse) # write to text, then create buffer st = time.time() buffer = StringIO() big_data_json = json.dumps(big_data) buffer = StringIO(big_data_json) value = buffer.getvalue() elapse = time.time() - st print("%.6f" % elapse) # write to buffer st = time.time() buffer = StringIO() json.dump(big_data, buffer) value = buffer.getvalue() elapse = time.time() - st print("%.6f" % elapse) print("%s KB" % sys.getsizeof(big_data_json))
def test_display(self): model = ConcreteModel() model.e = Expression() with capture_output() as out: model.e.display() self.assertEqual(out.getvalue().strip(), """ e : Size=1 Key : Value None : Undefined """.strip()) model.e.set_value(1.0) with capture_output() as out: model.e.display() self.assertEqual(out.getvalue().strip(), """ e : Size=1 Key : Value None : 1.0 """.strip()) out = StringIO() with capture_output() as no_out: model.e.display(ostream=out) self.assertEqual(no_out.getvalue(), "") self.assertEqual(out.getvalue().strip(), """ e : Size=1 Key : Value None : 1.0 """.strip()) model.E = Expression([1,2]) with capture_output() as out: model.E.display() self.assertEqual(out.getvalue().strip(), """ E : Size=2 Key : Value 1 : Undefined 2 : Undefined """.strip()) model.E[1].set_value(1.0) with capture_output() as out: model.E.display() self.assertEqual(out.getvalue().strip(), """ E : Size=2 Key : Value 1 : 1.0 2 : Undefined """.strip()) out = StringIO() with capture_output() as no_out: model.E.display(ostream=out) self.assertEqual(no_out.getvalue(), "") self.assertEqual(out.getvalue().strip(), """ E : Size=2 Key : Value 1 : 1.0 2 : Undefined """.strip())
def test_invalid_project_name(self): with patch("sys.stdout", self.stdout): stderr_tmp = StringIO() with patch("sys.stderr", stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse( ["-q", "--db=postgres://user:pwd@host/dbname", "-p" + self.project_dir, "test"] ) self.assertTrue(stderr_tmp.getvalue().find('Project name "test" is not a valid app name') > -1) stderr_tmp = StringIO() with patch("sys.stderr", stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse( ["-q", "--db=postgres://user:pwd@host/dbname", "-p" + self.project_dir, "assert"] ) self.assertTrue(stderr_tmp.getvalue().find('Project name "assert" is not a valid app name') > -1) stderr_tmp = StringIO() with patch("sys.stderr", stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse( ["-q", "--db=postgres://user:pwd@host/dbname", "-p" + self.project_dir, "values"] ) self.assertTrue(stderr_tmp.getvalue().find('Project name "values" is not a valid app name') > -1) stderr_tmp = StringIO() with patch("sys.stderr", stderr_tmp): with self.assertRaises(SystemExit) as error: conf_data = config.parse( ["-q", "--db=postgres://user:pwd@host/dbname", "-p" + self.project_dir, "project-name"] ) self.assertTrue(stderr_tmp.getvalue().find('Project name "project-name" is not a valid app name') > -1)
def test_declared(self): b = PySPConfigBlock() safe_register_common_option(b, "verbose") b.display() b.display() out = StringIO() b.display(ostream=out) self.assertEqual(out.getvalue(), "verbose: false\n") self.assertEqual(b.check_usage(), True) self.assertEqual(b.check_usage(error=False), True) b.verbose = True out = StringIO() b.display(ostream=out) self.assertEqual(out.getvalue(), "verbose: true\n") with self.assertRaises(ValueError): b.check_usage() with self.assertRaises(ValueError): b.check_usage() self.assertEqual(b.check_usage(error=False), False) b.verbose self.assertEqual(b.check_usage(), True) self.assertEqual(b.check_usage(error=False), True) verbose_about = \ """PySPConfigValue: verbose - type: <%s 'bool'> - default: False - doc: Generate verbose output for both initialization and execution.""" % ('class' if six.PY3 else 'type') self.assertEqual(b.about("verbose"), verbose_about)
def test_info_unicode(self, depfile): output = StringIO() task = Task("t1", [], file_dep=[six.u("tests/data/dependency1")]) cmd = CmdFactory(Info, outstream=output, dep_file=depfile.name, task_list=[task]) cmd._execute(["t1"]) assert """t1""" in output.getvalue() assert """tests/data/dependency1""" in output.getvalue()
def test_import_ignore(self): output = StringIO() call_command( 'import_json', '--main-component', 'test', '--project', 'test', TEST_COMPONENTS, stdout=output ) self.assertIn( 'Imported Test/Gettext PO with 3 translations', output.getvalue() ) output.truncate() call_command( 'import_json', '--main-component', 'test', '--project', 'test', '--ignore', TEST_COMPONENTS, stderr=output ) self.assertIn( 'Component Test/Gettext PO already exists', output.getvalue() )
rownames=["Actual Result"], colnames=["Predicted Result"]) from six import StringIO from IPython.display import Image import graphviz from sklearn.tree import export_graphviz import pydotplus dot_data = StringIO() export_graphviz(Dmodel, out_file=dot_data, filled=True, rounded=True, special_characters=True, feature_names=feature_cols, class_names=['Good', 'Bad']) graph = pydotplus.graph_from_dot_data(dot_data.getvalue()) graph.write_png('company_gini.png') Image(graph.create_png()) Dclassifier = DecisionTreeClassifier(criterion="entropy") Dmodel = Dclassifier.fit(X_train, np.ravel(Y_train)) Dpreds = Dmodel.predict(X_test) Dresult = confusion_matrix(Y_test, Dpreds) print("Confusion Matrix:") print(Dresult) Dresult1 = classification_report(Y_test, Dpreds) print("Classification Report:", ) print(Dresult1) Dresult2 = accuracy_score(Y_test, Dpreds) print("Accuracy:", Dresult2)
def changelog(since, to, write, force): """ Generates a markdown file containing the list of checks that changed for a given Agent release. Agent version numbers are derived inspecting tags on `integrations-core` so running this tool might provide unexpected results if the repo is not up to date with the Agent release process. If neither `--since` or `--to` are passed (the most common use case), the tool will generate the whole changelog since Agent version 6.3.0 (before that point we don't have enough information to build the log). """ agent_tags = get_agent_tags(since, to) # store the changes in a mapping {agent_version --> {check_name --> current_version}} changes_per_agent = OrderedDict() # to keep indexing easy, we run the loop off-by-one for i in range(1, len(agent_tags)): req_file_name = os.path.basename(get_agent_release_requirements()) current_tag = agent_tags[i - 1] # Requirements for current tag file_contents = git_show_file(req_file_name, current_tag) catalog_now = parse_agent_req_file(file_contents) # Requirements for previous tag file_contents = git_show_file(req_file_name, agent_tags[i]) catalog_prev = parse_agent_req_file(file_contents) changes_per_agent[current_tag] = OrderedDict() for name, ver in iteritems(catalog_now): # at some point in the git history, the requirements file erroneusly # contained the folder name instead of the package name for each check, # let's be resilient old_ver = ( catalog_prev.get(name) or catalog_prev.get(get_folder_name(name)) or catalog_prev.get(get_package_name(name)) ) # normalize the package name to the check_name if name.startswith(DATADOG_PACKAGE_PREFIX): name = get_folder_name(name) if old_ver and old_ver != ver: # determine whether major version changed breaking = old_ver.split('.')[0] < ver.split('.')[0] changes_per_agent[current_tag][name] = (ver, breaking) elif not old_ver: # New integration changes_per_agent[current_tag][name] = (ver, False) # store the changelog in memory changelog_contents = StringIO() # prepare the links agent_changelog_url = 'https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#{}' check_changelog_url = 'https://github.com/DataDog/integrations-core/blob/master/{}/CHANGELOG.md' # go through all the agent releases for agent, version_changes in iteritems(changes_per_agent): url = agent_changelog_url.format(agent.replace('.', '')) # Github removes dots from the anchor changelog_contents.write('## Datadog Agent version [{}]({})\n\n'.format(agent, url)) if not version_changes: changelog_contents.write('* There were no integration updates for this version of the Agent.\n\n') else: for name, ver in iteritems(version_changes): # get the "display name" for the check manifest_file = os.path.join(get_root(), name, 'manifest.json') if os.path.exists(manifest_file): decoded = json.loads(read_file(manifest_file).strip(), object_pairs_hook=OrderedDict) display_name = decoded.get('display_name') else: display_name = name breaking_notice = " **BREAKING CHANGE**" if ver[1] else "" changelog_url = check_changelog_url.format(name) changelog_contents.write( '* {} [{}]({}){}\n'.format(display_name, ver[0], changelog_url, breaking_notice) ) # add an extra line to separate the release block changelog_contents.write('\n') # save the changelog on disk if --write was passed if write: dest = get_agent_changelog() # don't overwrite an existing file if os.path.exists(dest) and not force: msg = "Output file {} already exists, run the command again with --force to overwrite" abort(msg.format(dest)) write_file(dest, changelog_contents.getvalue()) else: echo_info(changelog_contents.getvalue())
class TestOutput(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_out_json_valid(self): """ The JSON output when the input is a dict should be the dict serialized to JSON """ output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_byte(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "0b1f6472" } """)) def test_out_json_byte_empty(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "" } """)) def test_out_boolean_valid(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out(CommandResultItem(True)) self.assertEqual(util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""True\n\n\n""")) # TABLE output tests def test_out_table_valid_query1(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem( [{ 'name': 'qwerty', 'id': '0b1f6472qwerty' }, { 'name': 'asdf', 'id': '0b1f6472asdf' }], simple_output_query='[*].{Name:name, Id:id}') output_producer.out(result_item) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Name | Id -------|--------------- qwerty | 0b1f6472qwerty asdf | 0b1f6472asdf """)) def test_out_table_no_query(self): output_producer = OutputProducer(formatter=format_table, file=self.io) with self.assertRaises(util.CLIError): output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) def test_out_table_valid_query2(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem([{ 'name': 'qwerty', 'id': '0b1f6472qwerty' }, { 'name': 'asdf', 'id': '0b1f6472asdf' }], simple_output_query='[*].{Name:name}') output_producer.out(result_item) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Name ------ qwerty asdf """)) def test_out_table_bad_query(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem([{ 'name': 'qwerty', 'id': '0b1f6472qwerty' }, { 'name': 'asdf', 'id': '0b1f6472asdf' }], simple_output_query='[*].{Name:name') with self.assertRaises(util.CLIError): output_producer.out(result_item) def test_out_table_complex_obj(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem([{ 'name': 'qwerty', 'id': '0b1f6472qwerty', 'sub': {'1'} }]) with self.assertRaises(util.CLIError): output_producer.out(result_item) def test_out_table_complex_obj_with_query_ok(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem([{ 'name': 'qwerty', 'id': '0b1f6472qwerty', 'sub': {'1'} }], simple_output_query='[*].{Name:name}') output_producer.out(result_item) self.assertEqual(util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Name ------ qwerty """)) def test_out_table_complex_obj_with_query_still_complex(self): output_producer = OutputProducer(formatter=format_table, file=self.io) result_item = CommandResultItem( [{ 'name': 'qwerty', 'id': '0b1f6472qwerty', 'sub': {'1'} }], simple_output_query='[*].{Name:name, Sub:sub}') with self.assertRaises(util.CLIError): output_producer.out(result_item) # LIST output tests def test_out_list_valid(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 0b1f6472 """)) def test_out_list_valid_none_val(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': None, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : None Id : 0b1f6472 """)) def test_out_list_valid_empty_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': None, 'id': '0b1f6472', 'hosts': [] })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : None Id : 0b1f6472 Hosts : None """)) def test_out_list_valid_array_complex(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem([{ 'active': True, 'id': '783yesdf' }, { 'active': False, 'id': '3hjnme32' }, { 'active': False, 'id': '23hiujbs' }])) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 783yesdf Active : False Id : 3hjnme32 Active : False Id : 23hiujbs """)) def test_out_list_valid_str_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem(['location', 'id', 'host', 'server'])) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""location id host server """)) def test_out_list_valid_complex_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472', 'myarray': ['1', '2', '3', '4'] })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 0b1f6472 Myarray : 1 2 3 4 """)) def test_out_list_format_key_simple(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('locationId') self.assertEqual(lo._formatted_keys_cache, {'locationId': 'Location Id'}) def test_out_list_format_key_single(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('location') self.assertEqual(lo._formatted_keys_cache, {'location': 'Location'}) def test_out_list_format_key_multiple_caps(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('fooIDS') self.assertEqual(lo._formatted_keys_cache, {'fooIDS': 'Foo I D S'}) def test_out_list_format_key_multiple_words(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('locationIdState') self.assertEqual(lo._formatted_keys_cache, {'locationIdState': 'Location Id State'}) # TSV output tests def test_output_format_dict(self): obj = {} obj['A'] = 1 obj['B'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_dict_sort(self): obj = {} obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '2\t1\n') def test_output_format_ordereddict_not_sorted(self): from collections import OrderedDict obj = OrderedDict() obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_ordereddict_list_not_sorted(self): from collections import OrderedDict obj1 = OrderedDict() obj1['B'] = 1 obj1['A'] = 2 obj2 = OrderedDict() obj2['A'] = 3 obj2['B'] = 4 result = format_tsv(CommandResultItem([obj1, obj2])) self.assertEqual(result, '1\t2\n3\t4\n')
def _mol_writer(data, fmt='sdf', filepath_or_buffer=None, update_properties=True, molecule_column=None, columns=None): """Universal writing function for private use. .. versionadded:: 0.3 Parameters ---------- fmt : string The format of molecular file filepath_or_buffer : string or None File path update_properties : bool, optional (default=True) Switch to update properties from the DataFrames to the molecules while writting. molecule_column : string or None, optional (default='mol') Name of molecule column. If None the molecules will be skipped. columns : list or None, optional (default=None) A list of columns to write to file. If None then all available fields are written. """ if filepath_or_buffer is None: out = StringIO() elif hasattr(filepath_or_buffer, 'write'): out = filepath_or_buffer else: out = oddt.toolkit.Outputfile(fmt, filepath_or_buffer, overwrite=True) if isinstance(data, pd.DataFrame): molecule_column = molecule_column or data._molecule_column for ix, row in data.iterrows(): mol = row[molecule_column].clone if update_properties: new_data = row.to_dict() del new_data[molecule_column] mol.data.update(new_data) if columns: for k in mol.data.keys(): if k not in columns: del mol.data[k] if filepath_or_buffer is None or hasattr(filepath_or_buffer, 'write'): out.write(mol.write(fmt)) else: out.write(mol) elif isinstance(data, pd.Series): for mol in data: if filepath_or_buffer is None or hasattr(filepath_or_buffer, 'write'): out.write(mol.write(fmt)) else: out.write(mol) if filepath_or_buffer is None: return out.getvalue() elif not hasattr(filepath_or_buffer, 'write'): # dont close foreign buffer out.close()
def test_get_arg_vals_for_call_stack(self): """Exercise SBFrame.GetVariables() API to get argument vals.""" self.build() exe = os.path.join(os.getcwd(), "a.out") # Create a target by the debugger. target = self.dbg.CreateTarget(exe) self.assertTrue(target, VALID_TARGET) # Now create a breakpoint on main.c by name 'c'. breakpoint = target.BreakpointCreateByName('c', 'a.out') #print("breakpoint:", breakpoint) self.assertTrue(breakpoint and breakpoint.GetNumLocations() == 1, VALID_BREAKPOINT) # Now launch the process, and do not stop at the entry point. process = target.LaunchSimple(None, None, self.get_process_working_directory()) process = target.GetProcess() self.assertTrue(process.GetState() == lldb.eStateStopped, PROCESS_STOPPED) # Keeps track of the number of times 'a' is called where it is within a # depth of 3 of the 'c' leaf function. callsOfA = 0 from six import StringIO as SixStringIO session = SixStringIO() while process.GetState() == lldb.eStateStopped: thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint) self.assertIsNotNone(thread) # Inspect at most 3 frames. numFrames = min(3, thread.GetNumFrames()) for i in range(numFrames): frame = thread.GetFrameAtIndex(i) if self.TraceOn(): print("frame:", frame) name = frame.GetFunction().GetName() if name == 'a': callsOfA = callsOfA + 1 # We'll inspect only the arguments for the current frame: # # arguments => True # locals => False # statics => False # in_scope_only => True valList = frame.GetVariables(True, False, False, True) argList = [] for val in valList: argList.append( "(%s)%s=%s" % (val.GetTypeName(), val.GetName(), val.GetValue())) print("%s(%s)" % (name, ", ".join(argList)), file=session) # Also check the generic pc & stack pointer. We can't test their absolute values, # but they should be valid. Uses get_GPRs() from the lldbutil # module. gpr_reg_set = lldbutil.get_GPRs(frame) pc_value = gpr_reg_set.GetChildMemberWithName("pc") self.assertTrue(pc_value, "We should have a valid PC.") pc_value_int = int(pc_value.GetValue(), 0) # Make sure on arm targets we dont mismatch PC value on the basis of thumb bit. # Frame PC will not have thumb bit set in case of a thumb # instruction as PC. if self.getArchitecture() in ['arm']: pc_value_int &= ~1 self.assertTrue( pc_value_int == frame.GetPC(), "PC gotten as a value should equal frame's GetPC") sp_value = gpr_reg_set.GetChildMemberWithName("sp") self.assertTrue(sp_value, "We should have a valid Stack Pointer.") self.assertTrue( int(sp_value.GetValue(), 0) == frame.GetSP(), "SP gotten as a value should equal frame's GetSP") print("---", file=session) process.Continue() # At this point, the inferior process should have exited. self.assertTrue(process.GetState() == lldb.eStateExited, PROCESS_EXITED) # Expect to find 'a' on the call stacks two times. self.assertTrue(callsOfA == 2, "Expect to find 'a' on the call stacks two times") # By design, the 'a' call frame has the following arg vals: # o a((int)val=1, (char)ch='A') # o a((int)val=3, (char)ch='A') if self.TraceOn(): print("Full stack traces when stopped on the breakpoint 'c':") print(session.getvalue()) self.expect(session.getvalue(), "Argugment values displayed correctly", exe=False, substrs=[ "a((int)val=1, (char)ch='A')", "a((int)val=3, (char)ch='A')" ])
def build_example(label, param_dict_real): """Build the model with parameter values set in param_dict_real. Args: label: Label of the model (i.e. the filename in the zip). param_dict_real: Parameter dictionary (arguments to the factories make_graph and make_test_inputs) Returns: (tflite_model_binary, report) where tflite_model_binary is the serialized flatbuffer as a string and report is a dictionary with keys `toco_log` (log of toco conversion), `tf_log` (log of tf conversion), `toco` (a string of success status of the conversion), `tf` (a string success status of the conversion). """ np.random.seed(RANDOM_SEED) report = {"toco": report_lib.NOTRUN, "tf": report_lib.FAILED} # Build graph report["tf_log"] = "" report["toco_log"] = "" tf.compat.v1.reset_default_graph() with tf.device("/cpu:0"): try: inputs, outputs = make_graph(param_dict_real) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report sess = tf.compat.v1.Session() try: baseline_inputs, baseline_outputs = (make_test_inputs( param_dict_real, sess, inputs, outputs)) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report report["toco"] = report_lib.FAILED report["tf"] = report_lib.SUCCESS # Convert graph to toco input_tensors = [(input_tensor.name.split(":")[0], input_tensor.shape, input_tensor.dtype) for input_tensor in inputs] output_tensors = [ _normalize_output_name(out.name) for out in outputs ] # pylint: disable=g-long-ternary graph_def = freeze_graph( sess, tf.global_variables() + inputs + outputs) if use_frozen_graph else sess.graph_def if "split_tflite_lstm_inputs" in param_dict_real: extra_toco_options.split_tflite_lstm_inputs = param_dict_real[ "split_tflite_lstm_inputs"] tflite_model_binary, toco_log = options.tflite_convert_function( options, graph_def, input_tensors, output_tensors, extra_toco_options=extra_toco_options, test_params=param_dict_real) report["toco"] = (report_lib.SUCCESS if tflite_model_binary is not None else report_lib.FAILED) report["toco_log"] = toco_log if options.save_graphdefs: archive.writestr(label + ".pbtxt", text_format.MessageToString(graph_def), zipfile.ZIP_DEFLATED) if tflite_model_binary: if options.make_edgetpu_tests: # Set proper min max values according to input dtype. baseline_inputs, baseline_outputs = generate_inputs_outputs( tflite_model_binary, min_value=0, max_value=255) archive.writestr(label + ".bin", tflite_model_binary, zipfile.ZIP_DEFLATED) example = { "inputs": baseline_inputs, "outputs": baseline_outputs } example_fp = StringIO() write_examples(example_fp, [example]) archive.writestr(label + ".inputs", example_fp.getvalue(), zipfile.ZIP_DEFLATED) example_fp2 = StringIO() write_test_cases(example_fp2, label + ".bin", [example]) archive.writestr(label + "_tests.txt", example_fp2.getvalue(), zipfile.ZIP_DEFLATED) zip_manifest.append(label + "\n") return tflite_model_binary, report
def make_zip_of_tests(options, test_parameters, make_graph, make_test_inputs, extra_toco_options=ExtraTocoOptions(), use_frozen_graph=False, expected_tf_failures=0): """Helper to make a zip file of a bunch of TensorFlow models. This does a cartestian product of the dictionary of test_parameters and calls make_graph() for each item in the cartestian product set. If the graph is built successfully, then make_test_inputs() is called to build expected input/output value pairs. The model is then converted to tflite with toco, and the examples are serialized with the tflite model into a zip file (2 files per item in the cartesian product set). Args: options: An Options instance. test_parameters: Dictionary mapping to lists for each parameter. e.g. `{"strides": [[1,3,3,1], [1,2,2,1]], "foo": [1.2, 1.3]}` make_graph: function that takes current parameters and returns tuple `[input1, input2, ...], [output1, output2, ...]` make_test_inputs: function taking `curr_params`, `session`, `input_tensors`, `output_tensors` and returns tuple `(input_values, output_values)`. extra_toco_options: Additional toco options. use_frozen_graph: Whether or not freeze graph before toco converter. expected_tf_failures: Number of times tensorflow is expected to fail in executing the input graphs. In some cases it is OK for TensorFlow to fail because the one or more combination of parameters is invalid. Raises: RuntimeError: if there are converter errors that can't be ignored. """ zip_path = os.path.join(options.output_path, options.zip_to_output) parameter_count = 0 for parameters in test_parameters: parameter_count += functools.reduce( operator.mul, [len(values) for values in parameters.values()]) all_parameter_count = parameter_count if options.multi_gen_state: all_parameter_count += options.multi_gen_state.parameter_count if not options.no_tests_limit and all_parameter_count > _MAX_TESTS_PER_ZIP: raise RuntimeError( "Too many parameter combinations for generating '%s'.\n" "There are at least %d combinations while the upper limit is %d.\n" "Having too many combinations will slow down the tests.\n" "Please consider splitting the test into multiple functions.\n" % (zip_path, all_parameter_count, _MAX_TESTS_PER_ZIP)) if options.multi_gen_state: options.multi_gen_state.parameter_count = all_parameter_count # TODO(aselle): Make this allow multiple inputs outputs. if options.multi_gen_state: archive = options.multi_gen_state.archive else: archive = zipfile.PyZipFile(zip_path, "w") zip_manifest = [] convert_report = [] toco_errors = 0 processed_labels = set() if options.make_edgetpu_tests: extra_toco_options.inference_input_type = tf.uint8 extra_toco_options.inference_output_type = tf.uint8 # Only count parameters when fully_quantize is True. parameter_count = 0 for parameters in test_parameters: if True in parameters.get("fully_quantize", []): parameter_count += functools.reduce(operator.mul, [ len(values) for key, values in parameters.items() if key != "fully_quantize" ]) label_base_path = zip_path if options.multi_gen_state: label_base_path = options.multi_gen_state.label_base_path for parameters in test_parameters: keys = parameters.keys() for curr in itertools.product(*parameters.values()): label = label_base_path.replace(".zip", "_") + (",".join( "%s=%r" % z for z in sorted(zip(keys, curr))).replace(" ", "")) if label[0] == "/": label = label[1:] if label in processed_labels: # Do not populate data for the same label more than once. It will cause # errors when unzipping. continue processed_labels.add(label) param_dict = dict(zip(keys, curr)) if options.make_edgetpu_tests and not param_dict.get( "fully_quantize", False): continue def generate_inputs_outputs(tflite_model_binary, min_value=0, max_value=255): """Generate input values and output values of the given tflite model. Args: tflite_model_binary: A serialized flatbuffer as a string. min_value: min value for the input tensor. max_value: max value for the input tensor. Returns: (input_values, output_values): input values and output values built. """ interpreter = tf.lite.Interpreter( model_content=tflite_model_binary) interpreter.allocate_tensors() input_details = interpreter.get_input_details() input_values = [] for input_detail in input_details: input_value = create_tensor_data(input_detail["dtype"], input_detail["shape"], min_value=min_value, max_value=max_value) interpreter.set_tensor(input_detail["index"], input_value) input_values.append(input_value) interpreter.invoke() output_details = interpreter.get_output_details() output_values = [] for output_detail in output_details: output_values.append( interpreter.get_tensor(output_detail["index"])) return input_values, output_values def build_example(label, param_dict_real): """Build the model with parameter values set in param_dict_real. Args: label: Label of the model (i.e. the filename in the zip). param_dict_real: Parameter dictionary (arguments to the factories make_graph and make_test_inputs) Returns: (tflite_model_binary, report) where tflite_model_binary is the serialized flatbuffer as a string and report is a dictionary with keys `toco_log` (log of toco conversion), `tf_log` (log of tf conversion), `toco` (a string of success status of the conversion), `tf` (a string success status of the conversion). """ np.random.seed(RANDOM_SEED) report = {"toco": report_lib.NOTRUN, "tf": report_lib.FAILED} # Build graph report["tf_log"] = "" report["toco_log"] = "" tf.compat.v1.reset_default_graph() with tf.device("/cpu:0"): try: inputs, outputs = make_graph(param_dict_real) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report sess = tf.compat.v1.Session() try: baseline_inputs, baseline_outputs = (make_test_inputs( param_dict_real, sess, inputs, outputs)) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report report["toco"] = report_lib.FAILED report["tf"] = report_lib.SUCCESS # Convert graph to toco input_tensors = [(input_tensor.name.split(":")[0], input_tensor.shape, input_tensor.dtype) for input_tensor in inputs] output_tensors = [ _normalize_output_name(out.name) for out in outputs ] # pylint: disable=g-long-ternary graph_def = freeze_graph( sess, tf.global_variables() + inputs + outputs) if use_frozen_graph else sess.graph_def if "split_tflite_lstm_inputs" in param_dict_real: extra_toco_options.split_tflite_lstm_inputs = param_dict_real[ "split_tflite_lstm_inputs"] tflite_model_binary, toco_log = options.tflite_convert_function( options, graph_def, input_tensors, output_tensors, extra_toco_options=extra_toco_options, test_params=param_dict_real) report["toco"] = (report_lib.SUCCESS if tflite_model_binary is not None else report_lib.FAILED) report["toco_log"] = toco_log if options.save_graphdefs: archive.writestr(label + ".pbtxt", text_format.MessageToString(graph_def), zipfile.ZIP_DEFLATED) if tflite_model_binary: if options.make_edgetpu_tests: # Set proper min max values according to input dtype. baseline_inputs, baseline_outputs = generate_inputs_outputs( tflite_model_binary, min_value=0, max_value=255) archive.writestr(label + ".bin", tflite_model_binary, zipfile.ZIP_DEFLATED) example = { "inputs": baseline_inputs, "outputs": baseline_outputs } example_fp = StringIO() write_examples(example_fp, [example]) archive.writestr(label + ".inputs", example_fp.getvalue(), zipfile.ZIP_DEFLATED) example_fp2 = StringIO() write_test_cases(example_fp2, label + ".bin", [example]) archive.writestr(label + "_tests.txt", example_fp2.getvalue(), zipfile.ZIP_DEFLATED) zip_manifest.append(label + "\n") return tflite_model_binary, report _, report = build_example(label, param_dict) if report["toco"] == report_lib.FAILED: ignore_error = False if not options.known_bugs_are_errors: for pattern, bug_number in options.known_bugs.items(): if re.search(pattern, label): print("Ignored converter error due to bug %s" % bug_number) ignore_error = True if not ignore_error: toco_errors += 1 print( "-----------------\nconverter error!\n%s\n-----------------\n" % report["toco_log"]) convert_report.append((param_dict, report)) if not options.no_conversion_report: report_io = StringIO() report_lib.make_report_table(report_io, zip_path, convert_report) if options.multi_gen_state: archive.writestr( "report_" + options.multi_gen_state.test_name + ".html", report_io.getvalue()) else: archive.writestr("report.html", report_io.getvalue()) if options.multi_gen_state: options.multi_gen_state.zip_manifest.extend(zip_manifest) else: archive.writestr("manifest.txt", "".join(zip_manifest), zipfile.ZIP_DEFLATED) # Log statistics of what succeeded total_conversions = len(convert_report) tf_success = sum(1 for x in convert_report if x[1]["tf"] == report_lib.SUCCESS) toco_success = sum(1 for x in convert_report if x[1]["toco"] == report_lib.SUCCESS) percent = 0 if tf_success > 0: percent = float(toco_success) / float(tf_success) * 100. tf.compat.v1.logging.info( ("Archive %s Considered %d graphs, %d TF evaluated graphs " " and %d TOCO converted graphs (%.1f%%"), zip_path, total_conversions, tf_success, toco_success, percent) tf_failures = parameter_count - tf_success if tf_failures / parameter_count > 0.8: raise RuntimeError( ("Test for '%s' is not very useful. " "TensorFlow fails in %d percent of the cases.") % (zip_path, int(100 * tf_failures / parameter_count))) if not options.make_edgetpu_tests and tf_failures != expected_tf_failures: raise RuntimeError( ("Expected TF to fail %d times while generating '%s', " "but that happened %d times") % (expected_tf_failures, zip_path, tf_failures)) if not options.ignore_converter_errors and toco_errors > 0: raise RuntimeError("Found %d errors while generating toco models" % toco_errors)
def _write_equations_section(self, model, output_file, all_blocks_list, active_components_data_var, symbol_map, c_labeler, output_fixed_variable_bounds, skip_trivial_constraints, sorter): referenced_variable_ids = set() def _skip_trivial(constraint_data): if skip_trivial_constraints: if constraint_data._linear_canonical_form: repn = constraint_data.canonical_form() if (repn.variables is None) or \ (len(repn.variables) == 0): return True elif constraint_data.body.polynomial_degree() == 0: return True return False # # Check for active suffixes to export # if isinstance(model, IBlock): suffix_gen = lambda b: ((suf.storage_key, suf) \ for suf in pyomo.core.kernel.suffix.\ export_suffix_generator(b, active=True, descend_into=False)) else: suffix_gen = lambda b: pyomo.core.base.suffix.\ active_export_suffix_generator(b) r_o_eqns = [] c_eqns = [] l_eqns = [] branching_priorities_suffixes = [] for block in all_blocks_list: for name, suffix in suffix_gen(block): if name == 'branching_priorities': branching_priorities_suffixes.append(suffix) elif name == 'constraint_types': for constraint_data, constraint_type in iteritems(suffix): if not _skip_trivial(constraint_data): if constraint_type.lower() == 'relaxationonly': r_o_eqns.append(constraint_data) elif constraint_type.lower() == 'convex': c_eqns.append(constraint_data) elif constraint_type.lower() == 'local': l_eqns.append(constraint_data) else: raise ValueError( "A suffix '%s' contained an invalid value: %s\n" "Choices are: [relaxationonly, convex, local]" % (suffix.name, constraint_type)) else: raise ValueError( "The BARON writer can not export suffix with name '%s'. " "Either remove it from block '%s' or deactivate it." % (block.name, name)) non_standard_eqns = r_o_eqns + c_eqns + l_eqns # # EQUATIONS # #Equation Declaration n_roeqns = len(r_o_eqns) n_ceqns = len(c_eqns) n_leqns = len(l_eqns) eqns = [] # Alias the constraints by declaration order since Baron does not # include the constraint names in the solution file. It is important # that this alias not clash with any real constraint labels, hence # the use of the ".c<integer>" template. It is not possible to declare # a component having this type of name when using standard syntax. # There are ways to do it, but it is unlikely someone will. order_counter = 0 alias_template = ".c%d" output_file.write('EQUATIONS ') output_file.write("c_e_FIX_ONE_VAR_CONST__") order_counter += 1 for block in all_blocks_list: for constraint_data in block.component_data_objects( Constraint, active=True, sort=sorter, descend_into=False): if (not constraint_data.has_lb()) and \ (not constraint_data.has_ub()): assert not constraint_data.equality continue # non-binding, so skip if (not _skip_trivial(constraint_data)) and \ (constraint_data not in non_standard_eqns): eqns.append(constraint_data) con_symbol = symbol_map.createSymbol( constraint_data, c_labeler) assert not con_symbol.startswith('.') assert con_symbol != "c_e_FIX_ONE_VAR_CONST__" symbol_map.alias(constraint_data, alias_template % order_counter) output_file.write(", " + str(con_symbol)) order_counter += 1 output_file.write(";\n\n") if n_roeqns > 0: output_file.write('RELAXATION_ONLY_EQUATIONS ') for i, constraint_data in enumerate(r_o_eqns): con_symbol = symbol_map.createSymbol(constraint_data, c_labeler) assert not con_symbol.startswith('.') assert con_symbol != "c_e_FIX_ONE_VAR_CONST__" symbol_map.alias(constraint_data, alias_template % order_counter) if i == n_roeqns - 1: output_file.write(str(con_symbol) + ';\n\n') else: output_file.write(str(con_symbol) + ', ') order_counter += 1 if n_ceqns > 0: output_file.write('CONVEX_EQUATIONS ') for i, constraint_data in enumerate(c_eqns): con_symbol = symbol_map.createSymbol(constraint_data, c_labeler) assert not con_symbol.startswith('.') assert con_symbol != "c_e_FIX_ONE_VAR_CONST__" symbol_map.alias(constraint_data, alias_template % order_counter) if i == n_ceqns - 1: output_file.write(str(con_symbol) + ';\n\n') else: output_file.write(str(con_symbol) + ', ') order_counter += 1 if n_leqns > 0: output_file.write('LOCAL_EQUATIONS ') for i, constraint_data in enumerate(l_eqns): con_symbol = symbol_map.createSymbol(constraint_data, c_labeler) assert not con_symbol.startswith('.') assert con_symbol != "c_e_FIX_ONE_VAR_CONST__" symbol_map.alias(constraint_data, alias_template % order_counter) if i == n_leqns - 1: output_file.write(str(con_symbol) + ';\n\n') else: output_file.write(str(con_symbol) + ', ') order_counter += 1 # Create a dictionary of baron variable names to match to the # strings that constraint.to_string() prints. An important # note is that the variable strings are padded by spaces so # that whole variable names are recognized, and simple # variable names are not identified inside longer names. # Example: ' x[1] ' -> ' x3 ' #FIXME: 7/18/14 CLH: This may cause mistakes if spaces in # variable names are allowed if isinstance(model, IBlock): mutable_param_gen = lambda b: \ b.components(ctype=Param, descend_into=False) else: def mutable_param_gen(b): for param in block.component_objects(Param): if param._mutable and param.is_indexed(): param_data_iter = \ (param_data for index, param_data in iteritems(param)) elif not param.is_indexed(): param_data_iter = iter([param]) else: param_data_iter = iter([]) for param_data in param_data_iter: yield param_data if False: # # This was part of a merge from master that caused # test failures. But commenting this out didn't cause additional failures!?! # vstring_to_var_dict = {} vstring_to_bar_dict = {} pstring_to_bar_dict = {} _val_template = ' %' + self._precision_string + ' ' for block in all_blocks_list: for var_data in active_components_data_var[id(block)]: variable_stream = StringIO() var_data.to_string(ostream=variable_stream, verbose=False) variable_string = variable_stream.getvalue() variable_string = ' ' + variable_string + ' ' vstring_to_var_dict[variable_string] = var_data if output_fixed_variable_bounds or (not var_data.fixed): vstring_to_bar_dict[variable_string] = \ ' '+object_symbol_dictionary[id(var_data)]+' ' else: assert var_data.value is not None vstring_to_bar_dict[variable_string] = \ (_val_template % (var_data.value,)) for param_data in mutable_param_gen(block): param_stream = StringIO() param_data.to_string(ostream=param_stream, verbose=False) param_string = param_stream.getvalue() param_string = ' ' + param_string + ' ' pstring_to_bar_dict[param_string] = \ (_val_template % (param_data(),)) # Equation Definition string_template = '%' + self._precision_string output_file.write('c_e_FIX_ONE_VAR_CONST__: ONE_VAR_CONST__ == 1;\n') for constraint_data in itertools.chain(eqns, r_o_eqns, c_eqns, l_eqns): variables = set() #print(symbol_map.byObject.keys()) eqn_body = expression_to_string(constraint_data.body, variables, smap=symbol_map) #print(symbol_map.byObject.keys()) referenced_variable_ids.update(variables) if len(variables) == 0: assert not skip_trivial_constraints eqn_body += " + 0 * ONE_VAR_CONST__ " # 7/29/14 CLH: #FIXME: Baron doesn't handle many of the # intrinsic_functions available in pyomo. The # error message given by baron is also very # weak. Either a function here to re-write # unallowed expressions or a way to track solver # capability by intrinsic_expression would be # useful. ########################## con_symbol = symbol_map.byObject[id(constraint_data)] output_file.write(str(con_symbol) + ': ') # Fill in the left and right hand side (constants) of # the equations # Equality constraint if constraint_data.equality: eqn_lhs = '' eqn_rhs = ' == ' + \ str(string_template % self._get_bound(constraint_data.upper)) # Greater than constraint elif not constraint_data.has_ub(): eqn_rhs = ' >= ' + \ str(string_template % self._get_bound(constraint_data.lower)) eqn_lhs = '' # Less than constraint elif not constraint_data.has_lb(): eqn_rhs = ' <= ' + \ str(string_template % self._get_bound(constraint_data.upper)) eqn_lhs = '' # Double-sided constraint elif constraint_data.has_lb() and \ constraint_data.has_ub(): eqn_lhs = str(string_template % self._get_bound(constraint_data.lower)) + \ ' <= ' eqn_rhs = ' <= ' + \ str(string_template % self._get_bound(constraint_data.upper)) eqn_string = eqn_lhs + eqn_body + eqn_rhs + ';\n' output_file.write(eqn_string) # # OBJECTIVE # output_file.write("\nOBJ: ") n_objs = 0 for block in all_blocks_list: for objective_data in block.component_data_objects( Objective, active=True, sort=sorter, descend_into=False): n_objs += 1 if n_objs > 1: raise ValueError( "The BARON writer has detected multiple active " "objective functions on model %s, but " "currently only handles a single objective." % (model.name)) # create symbol symbol_map.createSymbol(objective_data, c_labeler) symbol_map.alias(objective_data, "__default_objective__") if objective_data.is_minimizing(): output_file.write("minimize ") else: output_file.write("maximize ") variables = set() #print(symbol_map.byObject.keys()) obj_string = expression_to_string(objective_data.expr, variables, smap=symbol_map) #print(symbol_map.byObject.keys()) referenced_variable_ids.update(variables) output_file.write(obj_string + ";\n\n") #referenced_variable_ids.update(symbol_map.byObject.keys()) return referenced_variable_ids, branching_priorities_suffixes
def test_merge_model_instances(self, keep_first_or_last_instance, get_field_names, get_model_to_deduplicate): get_model_to_deduplicate.return_value = Person get_field_names.return_value = ["name"] keep_first_or_last_instance.return_value = "first" name = Name.objects.create(name="Name") note = Note.objects.create(note="This is a note.") personality_1 = Personality.objects.create(description="Child 1's personality.") personality_2 = Personality.objects.create(description="Child 2's personality.") child_1 = Person.objects.create( name=Name.objects.create(name="Child1"), age=10, personality=personality_1, ) child_1.notes.add(note) child_2 = Person.objects.create( name=Name.objects.create(name="Child2"), age=10, personality=personality_2, ) child_2.notes.add(note) club1 = Club.objects.create(name="Club one") club2 = Club.objects.create(name="Club two") person_1 = Person.objects.create( name=name, age=50, personality=Personality.objects.create(description="First personality"), ) person_1.children.add(child_1) person_1.notes.add(note) Permission.objects.create(text="Permission", person=person_1) person_2 = Person.objects.create( name=name, age=50, personality=Personality.objects.create(description="Second personality"), ) person_2.children.add(child_2) new_note = Note.objects.create(note="This is a new note") person_2.notes.add(new_note) Membership.objects.create(club=club1, person=person_2) Membership.objects.create(club=club1, person=person_2) Permission.objects.create(text="Permission", person=person_2) person_3 = Person.objects.create( name=name, age=50, personality=Personality.objects.create(description="Third personality"), ) person_3.children.add(child_2) person_3.notes.add(new_note) Membership.objects.create(club=club2, person=person_3) Membership.objects.create(club=club2, person=person_3) Permission.objects.create(text="Permission", person=person_3) self.assertEqual(Person.objects.count(), 5) self.assertEqual(Membership.objects.count(), 4) out = StringIO() call_command('merge_model_instances', stdout=out) self.ouptput = out.getvalue() self.assertEqual(Person.objects.count(), 3) person = Person.objects.get(name__name="Name") self.assertRaises( Person.DoesNotExist, lambda: Person.objects.get(personality__description="Second personality"), ) self.assertEqual(person.notes.count(), 2) self.assertEqual(person.clubs.distinct().count(), 2) self.assertEqual(person.permission_set.count(), 3) self.assertRaises( Personality.DoesNotExist, lambda: Personality.objects.get(description="Second personality"), )
def __call__(self, model, output_filename, solver_capability, io_options): # Make sure not to modify the user's dictionary, they may be # reusing it outside of this call io_options = dict(io_options) # NOTE: io_options is a simple dictionary of keyword-value # pairs specific to this writer. symbolic_solver_labels = \ io_options.pop("symbolic_solver_labels", False) labeler = io_options.pop("labeler", None) # How much effort do we want to put into ensuring the # LP file is written deterministically for a Pyomo model: # 0 : None # 1 : sort keys of indexed components (default) # 2 : sort keys AND sort names (over declaration order) file_determinism = io_options.pop("file_determinism", 1) sorter = SortComponents.unsorted if file_determinism >= 1: sorter = sorter | SortComponents.indices if file_determinism >= 2: sorter = sorter | SortComponents.alphabetical output_fixed_variable_bounds = \ io_options.pop("output_fixed_variable_bounds", False) # Skip writing constraints whose body section is fixed (i.e., # no variables) skip_trivial_constraints = \ io_options.pop("skip_trivial_constraints", False) # Note: Baron does not allow specification of runtime # option outside of this file, so we add support # for them here solver_options = io_options.pop("solver_options", {}) if len(io_options): raise ValueError( "ProblemWriter_baron_writer passed unrecognized io_options:\n\t" + "\n\t".join("%s = %s" % (k, v) for k, v in iteritems(io_options))) if symbolic_solver_labels and (labeler is not None): raise ValueError("Baron problem writer: Using both the " "'symbolic_solver_labels' and 'labeler' " "I/O options is forbidden") # Make sure there are no strange ActiveComponents. The expression # walker will handle strange things in constraints later. model_ctypes = model.collect_ctypes(active=True) invalids = set() for t in (model_ctypes - valid_active_ctypes_minlp): if issubclass(t, ActiveComponent): invalids.add(t) if len(invalids): invalids = [t.__name__ for t in invalids] raise RuntimeError( "Unallowable active component(s) %s.\nThe BARON writer cannot " "export models with this component type." % ", ".join(invalids)) if output_filename is None: output_filename = model.name + ".bar" output_file = open(output_filename, "w") # Process the options. Rely on baron to catch # and reset bad option values output_file.write("OPTIONS {\n") summary_found = False if len(solver_options): for key, val in iteritems(solver_options): if (key.lower() == 'summary'): summary_found = True if key.endswith("Name"): output_file.write(key + ": \"" + str(val) + "\";\n") else: output_file.write(key + ": " + str(val) + ";\n") if not summary_found: # The 'summary option is defaulted to 0, so that no # summary file is generated in the directory where the # user calls baron. Check if a user explicitly asked for # a summary file. output_file.write("Summary: 0;\n") output_file.write("}\n\n") if symbolic_solver_labels: v_labeler = AlphaNumericTextLabeler() c_labeler = AlphaNumericTextLabeler() elif labeler is None: v_labeler = NumericLabeler('x') c_labeler = NumericLabeler('c') symbol_map = SymbolMap() symbol_map.default_labeler = v_labeler #sm_bySymbol = symbol_map.bySymbol # Cache the list of model blocks so we don't have to call # model.block_data_objects() many many times, which is slow # for indexed blocks all_blocks_list = list( model.block_data_objects(active=True, sort=sorter, descend_into=True)) active_components_data_var = {} #for block in all_blocks_list: # tmp = active_components_data_var[id(block)] = \ # list(obj for obj in block.component_data_objects(Var, # sort=sorter, # descend_into=False)) # create_symbols_func(symbol_map, tmp, labeler) # GAH: Not sure this is necessary, and also it would break for # non-mutable indexed params so I am commenting out for now. #for param_data in active_components_data(block, Param, sort=sorter): #instead of checking if param_data._mutable: #if not param_data.is_constant(): # create_symbol_func(symbol_map, param_data, labeler) #symbol_map_variable_ids = set(symbol_map.byObject.keys()) #object_symbol_dictionary = symbol_map.byObject # # Go through the objectives and constraints and generate # the output so that we can obtain the set of referenced # variables. # equation_section_stream = StringIO() referenced_variable_ids, branching_priorities_suffixes = \ self._write_equations_section( model, equation_section_stream, all_blocks_list, active_components_data_var, symbol_map, c_labeler, output_fixed_variable_bounds, skip_trivial_constraints, sorter) # # BINARY_VARIABLES, INTEGER_VARIABLES, POSITIVE_VARIABLES, VARIABLES # BinVars = [] IntVars = [] PosVars = [] Vars = [] for vid in referenced_variable_ids: name = symbol_map.byObject[vid] var_data = symbol_map.bySymbol[name]() if var_data.is_continuous(): if var_data.has_lb() and \ (self._get_bound(var_data.lb) >= 0): TypeList = PosVars else: TypeList = Vars elif var_data.is_binary(): TypeList = BinVars elif var_data.is_integer(): TypeList = IntVars else: assert False TypeList.append(name) if len(BinVars) > 0: BinVars.sort() output_file.write('BINARY_VARIABLES ') output_file.write(", ".join(BinVars)) output_file.write(';\n\n') if len(IntVars) > 0: IntVars.sort() output_file.write('INTEGER_VARIABLES ') output_file.write(", ".join(IntVars)) output_file.write(';\n\n') PosVars.append('ONE_VAR_CONST__') PosVars.sort() output_file.write('POSITIVE_VARIABLES ') output_file.write(", ".join(PosVars)) output_file.write(';\n\n') if len(Vars) > 0: Vars.sort() output_file.write('VARIABLES ') output_file.write(", ".join(Vars)) output_file.write(';\n\n') # # LOWER_BOUNDS # lbounds = {} for vid in referenced_variable_ids: name = symbol_map.byObject[vid] var_data = symbol_map.bySymbol[name]() if var_data.fixed: if output_fixed_variable_bounds: var_data_lb = var_data.value else: var_data_lb = None else: var_data_lb = None if var_data.has_lb(): var_data_lb = self._get_bound(var_data.lb) if var_data_lb is not None: name_to_output = symbol_map.getSymbol(var_data) lb_string_template = '%s: %' + self._precision_string + ';\n' lbounds[name_to_output] = lb_string_template % (name_to_output, var_data_lb) if len(lbounds) > 0: output_file.write("LOWER_BOUNDS{\n") output_file.write("".join(lbounds[key] for key in sorted(lbounds.keys()))) output_file.write("}\n\n") lbounds = None # # UPPER_BOUNDS # ubounds = {} for vid in referenced_variable_ids: name = symbol_map.byObject[vid] var_data = symbol_map.bySymbol[name]() if var_data.fixed: if output_fixed_variable_bounds: var_data_ub = var_data.value else: var_data_ub = None else: var_data_ub = None if var_data.has_ub(): var_data_ub = self._get_bound(var_data.ub) if var_data_ub is not None: name_to_output = symbol_map.getSymbol(var_data) ub_string_template = '%s: %' + self._precision_string + ';\n' ubounds[name_to_output] = ub_string_template % (name_to_output, var_data_ub) if len(ubounds) > 0: output_file.write("UPPER_BOUNDS{\n") output_file.write("".join(ubounds[key] for key in sorted(ubounds.keys()))) output_file.write("}\n\n") ubounds = None # # BRANCHING_PRIORITIES # # Specifying priorities requires that the pyomo model has established an # EXTERNAL, float suffix called 'branching_priorities' on the model # object, indexed by the relevant variable BranchingPriorityHeader = False for suffix in branching_priorities_suffixes: for var_data, priority in iteritems(suffix): if id(var_data) not in referenced_variable_ids: continue if priority is not None: if not BranchingPriorityHeader: output_file.write('BRANCHING_PRIORITIES{\n') BranchingPriorityHeader = True name_to_output = symbol_map.getSymbol(var_data) output_file.write(name_to_output + ': ' + str(priority) + ';\n') if BranchingPriorityHeader: output_file.write("}\n\n") # # Now write the objective and equations section # output_file.write(equation_section_stream.getvalue()) # # STARTING_POINT # output_file.write('STARTING_POINT{\nONE_VAR_CONST__: 1;\n') tmp = {} string_template = '%s: %' + self._precision_string + ';\n' for vid in referenced_variable_ids: name = symbol_map.byObject[vid] var_data = symbol_map.bySymbol[name]() starting_point = var_data.value if starting_point is not None: var_name = symbol_map.getSymbol(var_data) tmp[var_name] = string_template % (var_name, starting_point) output_file.write("".join(tmp[key] for key in sorted(tmp.keys()))) output_file.write('}\n\n') output_file.close() return output_filename, symbol_map
class SSHSession(Session): "Implements a :rfc:`4742` NETCONF session over SSH." def __init__(self, device_handler): capabilities = Capabilities(device_handler.get_capabilities()) Session.__init__(self, capabilities) self._host_keys = paramiko.HostKeys() self._transport = None self._connected = False self._channel = None self._channel_id = None self._channel_name = None self._buffer = StringIO() # parsing-related, see _parse() self._device_handler = device_handler self._parsing_state10 = 0 self._parsing_pos10 = 0 self._parsing_pos11 = 0 self._parsing_state11 = 0 self._expchunksize = 0 self._curchunksize = 0 self._inendpos = 0 self._size_num_list = [] self._message_list = [] def _parse(self): "Messages ae delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a byte has been read it will not be considered again." return self._parse10() def _parse10(self): """Messages are delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a chunk has been read it will not be considered again.""" logger.debug("parsing netconf v1.0") buf = self._buffer buf.seek(self._parsing_pos10) if MSG_DELIM in buf.read().decode('UTF-8'): buf.seek(0) msg, _, remaining = buf.read().decode('UTF-8').partition(MSG_DELIM) msg = msg.strip() if sys.version < '3': self._dispatch_message(msg.encode()) else: self._dispatch_message(msg) # create new buffer which contains remaining of old buffer self._buffer = StringIO() self._buffer.write(remaining.encode()) self._parsing_pos10 = 0 if len(remaining) > 0: # There could be another entire message in the # buffer, so we should try to parse again. logger.debug( 'Trying another round of parsing since there is still data' ) self._parse10() else: # handle case that MSG_DELIM is split over two chunks self._parsing_pos10 = buf.tell() - MSG_DELIM_LEN if self._parsing_pos10 < 0: self._parsing_pos10 = 0 def _parse11(self): logger.debug("parsing netconf v1.1") expchunksize = self._expchunksize curchunksize = self._curchunksize idle, instart, inmsg, inbetween, inend = range(5) state = self._parsing_state11 inendpos = self._inendpos num_list = self._size_num_list MAX_STARTCHUNK_SIZE = 12 # \#+4294967295+\n pre = 'invalid base:1:1 frame' buf = self._buffer buf.seek(self._parsing_pos11) message_list = self._message_list # a message is a list of chunks chunk_list = [] # a chunk is a list of characters should_recurse = False while True: x = buf.read(1) if not x: logger.debug('No more data to read') # Store the current chunk to the message list chunk = b''.join(chunk_list) message_list.append(textify(chunk)) break # done reading logger.debug('x: %s', x) if state == idle: if x == b'\n': state = instart inendpos = 1 else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception elif state == instart: if inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception elif inendpos == 2: if x.isdigit(): inendpos += 1 # == 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)' % (pre, state)) raise Exception else: if inendpos == MAX_STARTCHUNK_SIZE: logger.debug('%s (%s: no. too long)' % (pre, state)) raise Exception elif x == b'\n': num = b''.join(num_list) num_list = [] # Reset num_list try: num = int(num) except: logger.debug('%s (%s: invalid no.)' % (pre, state)) raise Exception else: state = inmsg expchunksize = num logger.debug('response length: %d' % expchunksize) curchunksize = 0 inendpos += 1 elif x.isdigit(): inendpos += 1 # > 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)' % (pre, state)) raise Exception elif state == inmsg: chunk_list.append(x) curchunksize += 1 chunkleft = expchunksize - curchunksize if chunkleft == 0: inendpos = 0 state = inbetween chunk = b''.join(chunk_list) message_list.append(textify(chunk)) chunk_list = [] # Reset chunk_list logger.debug('parsed new chunk: %s' % (chunk)) elif state == inbetween: if inendpos == 0: if x == b'\n': inendpos += 1 else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception elif inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception else: inendpos += 1 # == 3 now # if x == b'#': state = inend elif x.isdigit(): # More trunks state = instart num_list = [] num_list.append(x) else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception elif state == inend: if inendpos == 3: if x == b'\n': inendpos = 0 state = idle logger.debug('dispatching message') self._dispatch_message(''.join(message_list)) # reset rest = buf.read() buf = BytesIO() buf.write(rest) buf.seek(0) message_list = [] self._message_list = message_list chunk_list = [] expchunksize = chunksize = 0 parsing_state11 = idle inendpos = parsing_pos11 = 0 # There could be another entire message in the # buffer, so we should try to parse again. should_recurse = True break else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception else: logger.debug('%s (%s invalid state)' % (pre, state)) raise Exception self._expchunksize = expchunksize self._curchunksize = curchunksize self._parsing_state11 = state self._inendpos = inendpos self._size_num_list = num_list self._buffer = buf self._parsing_pos11 = self._buffer.tell() logger.debug('parse11 ending ...') if should_recurse: logger.debug( 'Trying another round of parsing since there is still data') self._parse11() def load_known_hosts(self, filename=None): """Load host keys from an openssh :file:`known_hosts`-style file. Can be called multiple times. If *filename* is not specified, looks in the default locations i.e. :file:`~/.ssh/known_hosts` and :file:`~/ssh/known_hosts` for Windows. """ if filename is None: filename = os.path.expanduser('~/.ssh/known_hosts') try: self._host_keys.load(filename) except IOError: # for windows filename = os.path.expanduser('~/ssh/known_hosts') try: self._host_keys.load(filename) except IOError: pass else: self._host_keys.load(filename) def close(self): if self._transport.is_active(): self._transport.close() # Wait for the transport thread to close. while self.is_alive() and (self is not threading.current_thread()): self.join(10) self._channel = None self._connected = False # REMEMBER to update transport.rst if sig. changes, since it is hardcoded there def connect(self, host, port=830, timeout=None, unknown_host_cb=default_unknown_host_cb, username=None, password=None, key_filename=None, allow_agent=True, hostkey_verify=True, look_for_keys=True, ssh_config=None, sock_fd=None): """Connect via SSH and initialize the NETCONF session. First attempts the publickey authentication method and then password authentication. To disable attempting publickey authentication altogether, call with *allow_agent* and *look_for_keys* as `False`. *host* is the hostname or IP address to connect to *port* is by default 830, but some devices use the default SSH port of 22 so this may need to be specified *timeout* is an optional timeout for socket connect *unknown_host_cb* is called when the server host key is not recognized. It takes two arguments, the hostname and the fingerprint (see the signature of :func:`default_unknown_host_cb`) *username* is the username to use for SSH authentication *password* is the password used if using password authentication, or the passphrase to use for unlocking keys that require it *key_filename* is a filename where a the private key to be used can be found *allow_agent* enables querying SSH agent (if found) for keys *hostkey_verify* enables hostkey verification from ~/.ssh/known_hosts *look_for_keys* enables looking in the usual locations for ssh keys (e.g. :file:`~/.ssh/id_*`) *ssh_config* enables parsing of an OpenSSH configuration file, if set to its path, e.g. :file:`~/.ssh/config` or to True (in this case, use :file:`~/.ssh/config`). *sock_fd* is an already open socket which shall be used for this connection. Useful for NETCONF outbound ssh. Use host=None together with a valid sock_fd number """ if not (host or sock_fd): raise SSHError("Missing host or socket fd") # Optionally, parse .ssh/config config = {} if ssh_config is True: ssh_config = "~/.ssh/config" if sys.platform != "win32" else "~/ssh/config" if ssh_config is not None: config = paramiko.SSHConfig() config.parse(open(os.path.expanduser(ssh_config))) config = config.lookup(host) host = config.get("hostname", host) if username is None: username = config.get("user") if key_filename is None: key_filename = config.get("identityfile") if hostkey_verify: userknownhostsfile = config.get("userknownhostsfile") if userknownhostsfile: self.load_known_hosts( os.path.expanduser(userknownhostsfile)) if username is None: username = getpass.getuser() if sock_fd is None: if config.get("proxycommand"): sock = paramiko.proxy.ProxyCommand(config.get("proxycommand")) else: for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.settimeout(timeout) except socket.error: continue try: sock.connect(sa) except socket.error: sock.close() continue break else: raise SSHError("Could not open socket to %s:%s" % (host, port)) else: if sys.version_info[0] < 3: s = socket.fromfd(int(sock_fd), socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, _sock=s) else: sock = socket.fromfd(int(sock_fd), socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) t = self._transport = paramiko.Transport(sock) t.set_log_channel(logger.name) if config.get("compression") == 'yes': t.use_compression() try: t.start_client() except paramiko.SSHException: raise SSHError('Negotiation failed') # host key verification server_key = t.get_remote_server_key() fingerprint = _colonify(hexlify(server_key.get_fingerprint())) if hostkey_verify: known_host = self._host_keys.check(host, server_key) if not known_host and not unknown_host_cb(host, fingerprint): raise SSHUnknownHostError(host, fingerprint) if key_filename is None: key_filenames = [] elif isinstance(key_filename, (str, bytes)): key_filenames = [key_filename] else: key_filenames = key_filename self._auth(username, password, key_filenames, allow_agent, look_for_keys) self._connected = True # there was no error authenticating # TODO: leopoul: Review, test, and if needed rewrite this part subsystem_names = self._device_handler.get_ssh_subsystem_names() for subname in subsystem_names: c = self._channel = self._transport.open_session() self._channel_id = c.get_id() channel_name = "%s-subsystem-%s" % (subname, str(self._channel_id)) c.set_name(channel_name) try: c.invoke_subsystem(subname) except paramiko.SSHException as e: logger.info("%s (subsystem request rejected)", e) handle_exception = self._device_handler.handle_connection_exceptions( self) # Ignore the exception, since we continue to try the different # subsystem names until we find one that can connect. #have to handle exception for each vendor here if not handle_exception: continue self._channel_name = c.get_name() self._post_connect() return raise SSHError( "Could not open connection, possibly due to unacceptable" " SSH subsystem name.") def _auth(self, username, password, key_filenames, allow_agent, look_for_keys): saved_exception = None for key_filename in key_filenames: for cls in (paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey): try: key = cls.from_private_key_file(key_filename, password) logger.debug( "Trying key %s from %s" % (hexlify(key.get_fingerprint()), key_filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if allow_agent: for key in paramiko.Agent().get_keys(): try: logger.debug("Trying SSH agent key %s" % hexlify(key.get_fingerprint())) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) keyfiles = [] if look_for_keys: rsa_key = os.path.expanduser("~/.ssh/id_rsa") dsa_key = os.path.expanduser("~/.ssh/id_dsa") ecdsa_key = os.path.expanduser("~/.ssh/id_ecdsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) if os.path.isfile(ecdsa_key): keyfiles.append((paramiko.ECDSAKey, ecdsa_key)) # look in ~/ssh/ for windows users: rsa_key = os.path.expanduser("~/ssh/id_rsa") dsa_key = os.path.expanduser("~/ssh/id_dsa") ecdsa_key = os.path.expanduser("~/ssh/id_ecdsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) if os.path.isfile(ecdsa_key): keyfiles.append((paramiko.ECDSAKey, ecdsa_key)) for cls, filename in keyfiles: try: key = cls.from_private_key_file(filename, password) logger.debug("Trying discovered key %s in %s" % (hexlify(key.get_fingerprint()), filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if password is not None: try: self._transport.auth_password(username, password) return except Exception as e: saved_exception = e logger.debug(e) if saved_exception is not None: # need pep-3134 to do this right raise AuthenticationError(repr(saved_exception)) raise AuthenticationError("No authentication methods available") def run(self): chan = self._channel q = self._q def start_delim(data_len): return '\n#%s\n' % (data_len) try: while True: # select on a paramiko ssh channel object does not ever return it in the writable list, so channels don't exactly emulate the socket api r, w, e = select([chan], [], [], TICK) # will wakeup evey TICK seconds to check if something to send, more if something to read (due to select returning chan in readable list) if r: data = chan.recv(BUF_SIZE) if data: self._buffer.write(data) if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities and 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: logger.debug( "Selecting netconf:base:1.1 for encoding") self._parse11() elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:netconf:base:1.0' in self._client_capabilities: logger.debug( "Selecting netconf:base:1.0 for encoding") self._parse10() else: raise Exception else: self._parse10() # HELLO msg uses EOM markers. else: raise SessionCloseError(self._buffer.getvalue()) if not q.empty() and chan.send_ready(): logger.debug("Sending message") data = q.get() try: # send a HELLO msg using v1.0 EOM markers. validated_element( data, tags= '{urn:ietf:params:xml:ns:netconf:base:1.0}hello') data = "%s%s" % (data, MSG_DELIM) except XMLError: # this is not a HELLO msg # we publish v1.1 support if 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities: # send using v1.1 chunked framing data = "%s%s%s" % (start_delim( len(data)), data, END_DELIM) elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities: # send using v1.0 EOM markers data = "%s%s" % (data, MSG_DELIM) else: raise Exception else: logger.debug( 'HELLO msg was sent, but server capabilities are still not known' ) raise Exception # we publish only v1.0 support else: # send using v1.0 EOM markers data = "%s%s" % (data, MSG_DELIM) finally: logger.debug("Sending: %s", data) while data: n = chan.send(data) if n <= 0: raise SessionCloseError( self._buffer.getvalue(), data) data = data[n:] except Exception as e: logger.debug("Broke out of main loop, error=%r", e) self._dispatch_error(e) self.close() @property def transport(self): "Underlying `paramiko.Transport <http://www.lag.net/paramiko/docs/paramiko.Transport-class.html>`_ object. This makes it possible to call methods like :meth:`~paramiko.Transport.set_keepalive` on it." return self._transport
import pyutilib.misc from six import StringIO output = StringIO() pyutilib.misc.setup_redirect(output) try: # Run the runner from run_path_constraint import results, m as model finally: pyutilib.misc.reset_redirect() # Report the result for line in output.getvalue().splitlines(): if line.startswith('EXIT'): print(line) model.obj.display() model.u.display() model.x1.display() model.x2.display() model.x3.display()
def c_extract(self, name, sub, check_input=True, check_broadcast=True): sio = StringIO() fail = sub['fail'] nd = self.ndim print(""" assert(py_%(name)s->ob_refcnt >= 2); // There should be at least one ref from the container object, // and one ref from the local scope. if (CudaNdarray_Check(py_%(name)s)) { //fprintf(stderr, "c_extract CNDA object w refcnt %%p %%i\\n", py_%(name)s, (py_%(name)s->ob_refcnt)); %(name)s = (CudaNdarray*)py_%(name)s; //std::cerr << "c_extract " << %(name)s << '\\n'; """ % locals(), file=sio) if (check_input): print(""" if (%(name)s->nd != %(nd)s) { PyErr_Format(PyExc_RuntimeError, "c_extract: Some CudaNdarray has rank %%i, it was supposed to have rank %(nd)s", %(name)s->nd); %(name)s = NULL; %(fail)s; } //std::cerr << "c_extract " << %(name)s << " nd check passed\\n"; """ % locals(), file=sio) for i, b in enumerate(self.broadcastable): if b and check_broadcast: print(""" if (CudaNdarray_HOST_DIMS(%(name)s)[%(i)s] != 1) { PyErr_Format(PyExc_RuntimeError, "c_extract: Some CudaNdarray has dim %%i on broadcastable dimension %%i", CudaNdarray_HOST_DIMS(%(name)s)[%(i)s], %(i)s); %(name)s = NULL; %(fail)s; } //std::cerr << "c_extract " << %(name)s << "dim check %(i)s passed\\n"; //std::cerr << "c_extract " << %(name)s << "checking bcast %(i)s <" << %(name)s->str<< ">\\n"; //std::cerr << "c_extract " << %(name)s->str[%(i)s] << "\\n"; if (CudaNdarray_HOST_STRIDES(%(name)s)[%(i)s]) { //std::cerr << "c_extract bad stride detected...\\n"; PyErr_Format(PyExc_RuntimeError, "c_extract: Some CudaNdarray has a nonzero stride %%i on a broadcastable dimension %%i", CudaNdarray_HOST_STRIDES(%(name)s)[%(i)s], %(i)s); %(name)s = NULL; %(fail)s; } //std::cerr << "c_extract " << %(name)s << "bcast check %(i)s passed\\n"; """ % locals(), file=sio) print(""" assert(%(name)s); Py_INCREF(py_%(name)s); } else if (py_%(name)s == Py_None) { PyErr_SetString(PyExc_TypeError, "expected a CudaNdarray, not None"); %(name)s = NULL; %(fail)s; } else { //fprintf(stderr, "FAILING c_extract CNDA object w refcnt %%p %%i\\n", py_%(name)s, (py_%(name)s->ob_refcnt)); PyErr_SetString(PyExc_TypeError, "Argument not a CudaNdarray"); %(name)s = NULL; %(fail)s; } //std::cerr << "c_extract done " << %(name)s << '\\n'; """ % locals(), file=sio) else: print(""" assert(%(name)s); Py_INCREF(py_%(name)s); } """ % locals(), file=sio) # print sio.getvalue() return sio.getvalue()
def test_no_color(self): with force_color_support: out = StringIO() call_command('show_urls', '--no-color', stdout=out) self.output = out.getvalue() self.assertNotIn('\x1b', self.output)
class SympyGenerator(with_metaclass(MetaClass, VariableAnalyzer)): def __init__(self, model, **kwargs): self.has_random = False VariableAnalyzer.__init__(self, model, **kwargs) # for attr in ['state', 'parameter', 'input']: # lst = [k for k, v in self.variables.items() if v.type == attr] # lst.sort() # setattr(self, attr+'s', lst) self.ode_src = StringIO() self.symbol_src = StringIO() self.ostream = self.ode_src self.equations = [] for key in dir(self): if key[:8] == "_handle_": setattr(self, key[1:], getattr(self, key)) self.generate() self.get_symbols() self.sympy_dct = {} self.latex_src = None self.compile_sympy() self.to_latex() @property def sympy_src(self): return self.symbol_src.getvalue() + self.ode_src.getvalue() def get_symbols(self): if self.has_random: self.symbol_src.write("N = Function('N')%c" % self.newline) self.symbol_src.write("t = Symbol('t')%c" % self.newline) for key, val in self.variables.items(): src = "{0} = Symbol('{0}'){1}".format(key, self.newline) self.symbol_src.write(src) for key in self.signature: src = "{0} = Symbol('{0}'){1}".format(key, self.newline) self.symbol_src.write(src) def compile_sympy(self): try: exec(self.sympy_src, globals(), self.sympy_dct) except: for n, line in enumerate(self.sympy_src.split("\n")): try: exec(line, globals(), self.sympy_dct) except IndentationError as e: raise err.NeuralSymPyCodeGenError( "SymPy Compilation Failed for model" f" '{self.model.__class__.__name__}' on:" f" Line {n}: \n{line}" "\n This is likely an issue with using 'elif' statement" " , avoid 'elif' and prefer binary masking operators" " like '(x>0)*x' in general.") from e except Exception as e: raise err.NeuralSymPyCodeGenError( "SymPy Compilation Failed for model" f" '{self.model.__class__.__name__}' on:" f" Line {n}: \n{line}") from e def to_latex(self): cond = lambda v: v.type == "state" and v.integral is None states = [k for k, v in self.variables.items() if cond(v)] states.sort() states_src = ",~".join([latex(self.sympy_dct[x]) for x in states]) params = [ k for k, v in self.variables.items() if v.type == "parameter" ] params.sort() params_src = ",~".join([latex(self.sympy_dct[x]) for x in params]) template_src = r"\mbox{State Variables: }%s\\\mbox{Parameters: }%s\\" self.latex_src = template_src % (states_src, params_src) self.latex_src = "" self.latex_src += r"\begin{eqnarray}" for eq in self.equations: try: tmp = latex(self.sympy_dct[eq], mul_symbol="dot") except Exception as e: raise err.NeuralSymPyCodeGenError( "Failed to Generate Sympy Code for model" f" {self.model.__class__.__name__}") from e self.latex_src += tmp.replace("=", " &=& ") + r"\\" self.latex_src += r"\end{eqnarray}" def _handle_call_function(self, ins): narg = int(ins.arg) # hacky way to handle keyword arguments if self.kwargs and self.var[-(narg + 1)] == (self.kwargs + ".pop"): arg = self.var[-narg][1:-1] self.var[-(narg + 1)] = arg new_arg = "%s" % arg self.signature.append(new_arg) else: args = [] if narg == 0 else [str(x) for x in self.var[-narg:]] func_name = self.var[-(narg + 1)] func_globals = get_function_globals(self.model.ode) pyfunc = eval(func_name, func_globals) sympyfunc = self.pyfunc_to_sympyfunc.get(pyfunc) if sympyfunc is not None: self.var[-(narg + 1)] = sympyfunc(self, args) else: self.var[-(narg + 1)] = "{}({})".format( func_name, ",".join(args)) if narg: del self.var[-narg:] def _handle_load_attr(self, ins): key = ins.argval if self.var[-1] == "self": if key[:2] == "d_": key = key.split("d_")[-1] depth = 1 else: depth = 0 if self.variables[key].integral: while self.variables[key].integral is not None: depth += 1 key = self.variables[key].integral if depth > 0: key = "Derivative(%s%s)" % (key, ", t" * depth) self.var[-1] = key else: self.var[-1] += "." + key def _handle_store_attr(self, ins): self.handle_load_attr(ins) rval, lval = self.var[-2:] del self.var[-1] if rval != lval: eqn = "Eq(%s, %s)" % (lval, rval) self.equations.append("eqn_%d" % len(self.equations)) self.var[-1] = "%s = %s" % (self.equations[-1], eqn) else: del self.var[-1] def _handle_store_fast(self, ins): key = ins.argval prefix, indent = "", "" if ins.argval == self.var[-1]: del self.var[-1] return elif self.variables[key].type == "local": eqn = "Eq(%s, %s)" % (key, self.var[-1]) self.equations.append("eqn_%d" % len(self.equations)) indent = " " * (self.space + self.indent) prefix = "with evaluate(False):" + self.newline self.var[-1] = "{}{}{} = {}".format(prefix, indent, self.equations[-1], eqn) else: self.var[-1] = "{} = {}".format(key, self.var[-1]) def handle_pop_jump_if_true(self, ins): self.jump_targets.append(ins.arg) self.enter_indent = True self.var[-1] = "if not UnevaluatedExpr({0}):".format(self.var[-1]) def handle_pop_jump_if_false(self, ins): self.jump_targets.append(ins.arg) self.enter_indent = True self.var[-1] = "if UnevaluatedExpr({0}):".format(self.var[-1]) def handle_jump_forward(self, ins): self.leave_indent = True self.output_statement() target, old_target = ins.argval, self.jump_targets.pop() if target != old_target: self.var.append("") self.enter_indent = True self.jump_targets.append(target) else: self.var.append("") self.output_statement() def handle_compare_op(self, ins): """Convert Comparison Operation to Heaviside Expressions""" op = ins.argval if op in [">", ">="]: diff = f"{self.var[-2]} - {self.var[-1]}" elif op in ["<", "<="]: diff = f"{self.var[-1]} - {self.var[-2]}" else: raise ValueError( f"Comparison with Operator '{op}' not understood.") thres = 1 if "=" in op else 0 self.var[-2] = f"Heaviside({diff}, {thres})" del self.var[-1] def _handle_return_value(self, ins): self.var[-1] = "" def _py2sympy(*source_funcs): def wrapper(func): func._source_funcs = source_funcs return func return wrapper def _random_func(func): """ A decorator for registering random functions """ @wraps(func) def wrap(self, args): self.has_random = True return func(self, args) return wrap def _generate_sympy_func(self, func, args): return "%s(%s)" % (func, ", ".join(args)) @_py2sympy(np.exp) def _np_exp(self, args): return self._generate_sympy_func("exp", args) @_py2sympy(np.log) def _np_log(self, args): return self._generate_sympy_func("log", args) @_py2sympy(np.power) def _np_power(self, args): return self._generate_sympy_func("pow", args) @_py2sympy(np.cbrt) def _np_cbrt(self, args): return self._generate_sympy_func("cbrt", args) @_py2sympy(np.sqrt) def _np_sqrt(self, args): return self._generate_sympy_func("sqrt", args) @_py2sympy(random.gauss, np.random.normal) @_random_func def _random_gauss(self, args): return "N({0}, {1})".format(args[0], args[1])
def rst_table(elts): """Print out a RST-style table.""" cols = StringIO() ncol, widths = colify(elts, output=cols, tty=True) header = ' '.join('=' * (w - 1) for w in widths) return '%s\n%s%s' % (header, cols.getvalue(), header)
def test_list_languages(self): output = StringIO() call_command('list_addons', stdout=output) self.assertIn('msgmerge', output.getvalue())
def do_update_changelog(ctx, target, cur_version, new_version, dry_run=False): """ Actually perform the operations needed to update the changelog, this method is supposed to be used by other tasks and not directly. """ # get the name of the current release tag target_tag = get_release_tag_string(target, cur_version) # get the diff from HEAD target_path = os.path.join(ROOT, target) cmd = 'git log --pretty=%s {}... {}'.format(target_tag, target_path) diff_lines = ctx.run(cmd, hide='out').stdout.split('\n') # for each PR get the title, we'll use it to populate the changelog endpoint = GITHUB_API_URL + '/repos/DataDog/integrations-core/pulls/{}' pr_numbers = parse_pr_numbers(diff_lines) print("Found {} PRs merged since tag: {}".format(len(pr_numbers), target_tag)) entries = [] for pr_num in pr_numbers: try: response = urlopen(endpoint.format(pr_num)) except Exception as e: sys.stderr.write("Unable to fetch info for PR #{}\n: {}".format( pr_num, e)) continue payload = json.loads(response.read()) if NO_CHANGELOG_LABEL in (l.get('name') for l in payload.get('labels', [])): # No changelog entry for this PR print("Skipping PR #{} from changelog".format(pr_num)) continue author = payload.get('user', {}).get('login') author_url = payload.get('user', {}).get('html_url') entry = ChangelogEntry(pr_num, payload.get('title'), payload.get('html_url'), author, author_url, is_contributor(payload)) entries.append(entry) # store the new changelog in memory new_entry = StringIO() # the header contains version and date header = "### {} / {}\n".format(new_version, datetime.now().strftime("%Y-%m-%d")) new_entry.write(header) # one bullet point for each PR new_entry.write("\n") for entry in entries: thanknote = "" if entry.is_contributor: thanknote = " Thanks [{}]({}).".format(entry.author, entry.author_url) new_entry.write("* {}. See [#{}]({}).{}\n".format( entry.title, entry.number, entry.url, thanknote)) new_entry.write("\n") # read the old contents changelog_path = os.path.join(ROOT, target, "CHANGELOG.md") with open(changelog_path, 'r') as f: old = f.readlines() # write the new changelog in memory changelog = StringIO() # preserve the title changelog.write("".join(old[:2])) # prepend the new changelog to the old contents # make the command idempotent if header not in old: changelog.write(new_entry.getvalue()) # append the rest of the old changelog changelog.write("".join(old[2:])) # print on the standard out in case of a dry run if dry_run: print(changelog.getvalue()) sys.exit(0) # overwrite the old changelog with open(changelog_path, 'w') as f: f.write(changelog.getvalue())
def test_list_command(self): add_document() output = StringIO() call_command('list_memory', stdout=output) self.assertIn('test', output.getvalue())
def solve(self, *args, **kwds): """ Solve a model via the GAMS Python API. Keyword Arguments ----------------- tee=False: bool Output GAMS log to stdout. logfile=None: str Filename to output GAMS log to a file. load_solutions=True: bool Load solution into model. If False, the results object will contain the solution data. keepfiles=False: bool Keep temporary files. Equivalent of DebugLevel.KeepFiles. Summary of temp files can be found in _gams_py_gjo0.pf tmpdir=None: str Specify directory path for storing temporary files. A directory will be created if one of this name doesn't exist. By default uses the system default temporary path. report_timing=False: bool Print timing reports for presolve, solver, postsolve, etc. io_options: dict Options that get passed to the writer. See writer in pyomo.repn.plugins.gams_writer for details. Updated with any other keywords passed to solve method. """ # Make sure available() doesn't crash self.available() from gams import GamsWorkspace, DebugLevel from gams.workspace import GamsExceptionExecution if len(args) != 1: raise ValueError('Exactly one model must be passed ' 'to solve method of GAMSSolver.') model = args[0] # self.options are default for each run, overwritten by kwds options = dict() options.update(self.options) options.update(kwds) load_solutions = options.pop("load_solutions", True) tee = options.pop("tee", False) logfile = options.pop("logfile", None) keepfiles = options.pop("keepfiles", False) tmpdir = options.pop("tmpdir", None) report_timing = options.pop("report_timing", False) io_options = options.pop("io_options", {}) # Pass remaining keywords to writer, which will handle # any unrecognized arguments io_options.update(options) initial_time = time.time() #################################################################### # Presolve #################################################################### # Create StringIO stream to pass to gams_writer, on which the # model file will be written. The writer also passes this StringIO # back, but output_file is defined in advance for clarity. output_file = StringIO() if isinstance(model, IBlock): # Kernel blocks have slightly different write method smap_id = model.write(filename=output_file, format=ProblemFormat.gams, _called_by_solver=True, **io_options) symbolMap = getattr(model, "._symbol_maps")[smap_id] else: (_, smap_id) = model.write(filename=output_file, format=ProblemFormat.gams, io_options=io_options) symbolMap = model.solutions.symbol_map[smap_id] presolve_completion_time = time.time() if report_timing: print(" %6.2f seconds required for presolve" % (presolve_completion_time - initial_time)) #################################################################### # Apply solver #################################################################### # IMPORTANT - only delete the whole tmpdir if the solver was the one # that made the directory. Otherwise, just delete the files the solver # made, if not keepfiles. That way the user can select a directory # they already have, like the current directory, without having to # worry about the rest of the contents of that directory being deleted. newdir = True if tmpdir is not None and os.path.exists(tmpdir): newdir = False ws = GamsWorkspace(debug=DebugLevel.KeepFiles if keepfiles else DebugLevel.Off, working_directory=tmpdir) t1 = ws.add_job_from_string(output_file.getvalue()) try: with OutputStream(tee=tee, logfile=logfile) as output_stream: t1.run(output=output_stream) except GamsExceptionExecution as e: try: if e.rc == 3: # Execution Error check_expr_evaluation(model, symbolMap, 'direct') finally: # Always name working directory or delete files, # regardless of any errors. if keepfiles: print("\nGAMS WORKING DIRECTORY: %s\n" % ws.working_directory) elif tmpdir is not None: # Garbage collect all references to t1.out_db # So that .gdx file can be deleted t1 = rec = rec_lo = rec_hi = None file_removal_gams_direct(tmpdir, newdir) raise except: # Catch other errors and remove files first if keepfiles: print("\nGAMS WORKING DIRECTORY: %s\n" % ws.working_directory) elif tmpdir is not None: # Garbage collect all references to t1.out_db # So that .gdx file can be deleted t1 = rec = rec_lo = rec_hi = None file_removal_gams_direct(tmpdir, newdir) raise solve_completion_time = time.time() if report_timing: print(" %6.2f seconds required for solver" % (solve_completion_time - presolve_completion_time)) #################################################################### # Postsolve #################################################################### # import suffixes must be on the top-level model if isinstance(model, IBlock): model_suffixes = list(comp.storage_key for comp \ in pyomo.core.kernel.suffix.\ import_suffix_generator(model, active=True, descend_into=False)) else: model_suffixes = list(name for (name,comp) \ in pyomo.core.base.suffix.\ active_import_suffix_generator(model)) extract_dual = ('dual' in model_suffixes) extract_rc = ('rc' in model_suffixes) results = SolverResults() results.problem.name = t1.name results.problem.lower_bound = t1.out_db["OBJEST"].find_record().value results.problem.upper_bound = t1.out_db["OBJEST"].find_record().value results.problem.number_of_variables = \ t1.out_db["NUMVAR"].find_record().value results.problem.number_of_constraints = \ t1.out_db["NUMEQU"].find_record().value results.problem.number_of_nonzeros = \ t1.out_db["NUMNZ"].find_record().value results.problem.number_of_binary_variables = None # Includes binary vars: results.problem.number_of_integer_variables = \ t1.out_db["NUMDVAR"].find_record().value results.problem.number_of_continuous_variables = \ t1.out_db["NUMVAR"].find_record().value \ - t1.out_db["NUMDVAR"].find_record().value results.problem.number_of_objectives = 1 # required by GAMS writer obj = list(model.component_data_objects(Objective, active=True)) assert len(obj) == 1, 'Only one objective is allowed.' obj = obj[0] objctvval = t1.out_db["OBJVAL"].find_record().value if obj.is_minimizing(): results.problem.sense = ProblemSense.minimize results.problem.upper_bound = objctvval else: results.problem.sense = ProblemSense.maximize results.problem.lower_bound = objctvval results.solver.name = "GAMS " + str(self.version()) # Init termination condition to None to give preference to this first # block of code, only set certain TC's below if it's still None results.solver.termination_condition = None results.solver.message = None solvestat = t1.out_db["SOLVESTAT"].find_record().value if solvestat == 1: results.solver.status = SolverStatus.ok elif solvestat == 2: results.solver.status = SolverStatus.ok results.solver.termination_condition = TerminationCondition.maxIterations elif solvestat == 3: results.solver.status = SolverStatus.ok results.solver.termination_condition = TerminationCondition.maxTimeLimit elif solvestat == 5: results.solver.status = SolverStatus.ok results.solver.termination_condition = TerminationCondition.maxEvaluations elif solvestat == 7: results.solver.status = SolverStatus.aborted results.solver.termination_condition = TerminationCondition.licensingProblems elif solvestat == 8: results.solver.status = SolverStatus.aborted results.solver.termination_condition = TerminationCondition.userInterrupt elif solvestat == 10: results.solver.status = SolverStatus.error results.solver.termination_condition = TerminationCondition.solverFailure elif solvestat == 11: results.solver.status = SolverStatus.error results.solver.termination_condition = TerminationCondition.internalSolverError elif solvestat == 4: results.solver.status = SolverStatus.warning results.solver.message = "Solver quit with a problem (see LST file)" elif solvestat in (9, 12, 13): results.solver.status = SolverStatus.error elif solvestat == 6: results.solver.status = SolverStatus.unknown results.solver.return_code = 0 # Not sure if this value is actually user time # "the elapsed time it took to execute a solve statement in total" results.solver.user_time = t1.out_db["ETSOLVE"].find_record().value results.solver.system_time = None results.solver.wallclock_time = None results.solver.termination_message = None soln = Solution() modelstat = t1.out_db["MODELSTAT"].find_record().value if modelstat == 1: results.solver.termination_condition = TerminationCondition.optimal soln.status = SolutionStatus.optimal elif modelstat == 2: results.solver.termination_condition = TerminationCondition.locallyOptimal soln.status = SolutionStatus.locallyOptimal elif modelstat in [3, 18]: results.solver.termination_condition = TerminationCondition.unbounded soln.status = SolutionStatus.unbounded elif modelstat in [4, 5, 6, 10, 19]: results.solver.termination_condition = TerminationCondition.infeasible soln.status = SolutionStatus.infeasible elif modelstat == 7: results.solver.termination_condition = TerminationCondition.feasible soln.status = SolutionStatus.feasible elif modelstat == 8: # 'Integer solution model found' results.solver.termination_condition = TerminationCondition.optimal soln.status = SolutionStatus.optimal elif modelstat == 9: results.solver.termination_condition = TerminationCondition.intermediateNonInteger soln.status = SolutionStatus.other elif modelstat == 11: # Should be handled above, if modelstat and solvestat both # indicate a licensing problem if results.solver.termination_condition is None: results.solver.termination_condition = TerminationCondition.licensingProblems soln.status = SolutionStatus.error elif modelstat in [12, 13]: if results.solver.termination_condition is None: results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error elif modelstat == 14: if results.solver.termination_condition is None: results.solver.termination_condition = TerminationCondition.noSolution soln.status = SolutionStatus.unknown elif modelstat in [15, 16, 17]: # Having to do with CNS models, # not sure what to make of status descriptions results.solver.termination_condition = TerminationCondition.optimal soln.status = SolutionStatus.unsure else: # This is just a backup catch, all cases are handled above soln.status = SolutionStatus.error soln.gap = abs(results.problem.upper_bound \ - results.problem.lower_bound) for sym, ref in iteritems(symbolMap.bySymbol): obj = ref() if isinstance(model, IBlock): # Kernel variables have no 'parent_component' if obj.ctype is IObjective: soln.objective[sym] = {'Value': objctvval} if obj.ctype is not IVariable: continue else: if obj.parent_component().type() is Objective: soln.objective[sym] = {'Value': objctvval} if obj.parent_component().type() is not Var: continue rec = t1.out_db[sym].find_record() # obj.value = rec.level soln.variable[sym] = {"Value": rec.level} if extract_rc and not math.isnan(rec.marginal): # Do not set marginals to nan # model.rc[obj] = rec.marginal soln.variable[sym]['rc'] = rec.marginal if extract_dual: for c in model.component_data_objects(Constraint, active=True): if c.body.is_fixed() or \ (not (c.has_lb() or c.has_ub())): # the constraint was not sent to GAMS continue sym = symbolMap.getSymbol(c) if c.equality: rec = t1.out_db[sym].find_record() if not math.isnan(rec.marginal): # model.dual[c] = rec.marginal soln.constraint[sym] = {'dual': rec.marginal} else: # Solver didn't provide marginals, # nothing else to do here break else: # Inequality, assume if 2-sided that only # one side's marginal is nonzero # Negate marginal for _lo equations marg = 0 if c.lower is not None: rec_lo = t1.out_db[sym + '_lo'].find_record() marg -= rec_lo.marginal if c.upper is not None: rec_hi = t1.out_db[sym + '_hi'].find_record() marg += rec_hi.marginal if not math.isnan(marg): # model.dual[c] = marg soln.constraint[sym] = {'dual': marg} else: # Solver didn't provide marginals, # nothing else to do here break results.solution.insert(soln) if keepfiles: print("\nGAMS WORKING DIRECTORY: %s\n" % ws.working_directory) elif tmpdir is not None: # Garbage collect all references to t1.out_db # So that .gdx file can be deleted t1 = rec = rec_lo = rec_hi = None file_removal_gams_direct(tmpdir, newdir) #################################################################### # Finish with results #################################################################### results._smap_id = smap_id results._smap = None if isinstance(model, IBlock): if len(results.solution) == 1: results.solution(0).symbol_map = \ getattr(model, "._symbol_maps")[results._smap_id] results.solution(0).default_variable_value = \ self._default_variable_value if load_solutions: model.load_solution(results.solution(0)) else: assert len(results.solution) == 0 # see the hack in the write method # we don't want this to stick around on the model # after the solve assert len(getattr(model, "._symbol_maps")) == 1 delattr(model, "._symbol_maps") del results._smap_id if load_solutions and \ (len(results.solution) == 0): logger.error("No solution is available") else: if load_solutions: model.solutions.load_from(results) results._smap_id = None results.solution.clear() else: results._smap = model.solutions.symbol_map[smap_id] model.solutions.delete_symbol_map(smap_id) postsolve_completion_time = time.time() if report_timing: print(" %6.2f seconds required for postsolve" % (postsolve_completion_time - solve_completion_time)) print(" %6.2f seconds required total" % (postsolve_completion_time - initial_time)) return results
def __call__(self, _, __, event_dict): # Initialize lazily to prevent import side-effects. if self._init_colorama: if self._force_colors: colorama.deinit() colorama.init(strip=False) else: colorama.init() self._init_colorama = False sio = StringIO() ts = event_dict.pop("timestamp", None) if ts is not None: sio.write( # can be a number if timestamp is UNIXy self._styles.timestamp + str(ts) + self._styles.reset + " " ) level = event_dict.pop("level", None) if level is not None: sio.write( "[" + self._level_to_color[level] + _pad(level, self._longest_level) + self._styles.reset + "] " ) # force event to str for compatibility with standard library event = event_dict.pop("event") if not PY2 or not isinstance(event, string_types): event = str(event) if event_dict: event = _pad(event, self._pad_event) + self._styles.reset + " " else: event += self._styles.reset sio.write(self._styles.bright + event) logger_name = event_dict.pop("logger", None) if logger_name is not None: sio.write( "[" + self._styles.logger_name + self._styles.bright + logger_name + self._styles.reset + "] " ) stack = event_dict.pop("stack", None) exc = event_dict.pop("exception", None) sio.write( " ".join( self._styles.kv_key + key + self._styles.reset + "=" + self._styles.kv_value + self._repr(event_dict[key]) + self._styles.reset for key in sorted(event_dict.keys()) ) ) if stack is not None: sio.write("\n" + stack) if exc is not None: sio.write("\n\n" + "=" * 79 + "\n") if exc is not None: sio.write("\n" + exc) return sio.getvalue()
def pformat_msg(obj): fp = StringIO() pprint.pprint(obj, fp) return fp.getvalue()
def _rest_request(self, path, method='GET', args=None, body=None, \ headers=None): """Rest request main function :param path: path within tree :type path: str :param method: method to be implemented :type method: str :param args: the arguments for method :type args: dict :param body: body payload for the rest call :type body: dict :param headers: provide additional headers :type headers: dict :returns: returns a RestResponse object """ headers = self._get_req_headers(headers) reqpath = path.replace('//', '/') if body is not None: if isinstance(body, dict) or isinstance(body, list): headers['Content-Type'] = 'application/json' body = json.dumps(body) elif isinstance(body, bytes): headers['Content-Type'] = 'application/octet-stream' body = body else: headers['Content-Type'] = 'application/x-www-form-urlencoded' body = urlencode(body) if method == 'PUT': resp = self._rest_request(path=path) try: if resp.getheader('content-encoding') == 'gzip': buf = StringIO() gfile = gzip.GzipFile(mode='wb', fileobj=buf) try: gfile.write(str(body)) finally: gfile.close() compresseddata = buf.getvalue() if compresseddata: data = bytearray() data.extend(buffer(compresseddata)) body = data except BaseException as excp: LOGGER.error('Error occur while compressing body: %s', excp) raise headers['Content-Length'] = len(body) if args: if method == 'GET': reqpath += '?' + urlencode(args) elif method == 'PUT' or method == 'POST' or method == 'PATCH': LOGGER.warning('For POST, PUT and PATCH methods, the provided "args" parameter "{}" is ignored.' .format(args)) if not body: LOGGER.warning('Use the "body" parameter to supply the request payload.') restreq = RestRequest(reqpath, method=method, body=body) attempts = 0 restresp = None cause_exception = None while attempts <= self._max_retry: if LOGGER.isEnabledFor(logging.DEBUG): try: logbody = None if restreq.body: if restreq.body[0] == '{': logbody = restreq.body else: raise ValueError('Body of message is binary') LOGGER.debug('HTTP REQUEST: %s\n\tPATH: %s\n\tBODY: %s'% \ (restreq.method, restreq.path, logbody)) except: LOGGER.debug('HTTP REQUEST: %s\n\tPATH: %s\n\tBODY: %s'% \ (restreq.method, restreq.path, 'binary body')) attempts = attempts + 1 LOGGER.info('Attempt %s of %s', attempts, path) try: while True: if self._conn is None: self.__init_connection() self._conn.request(method.upper(), reqpath, body=body, \ headers=headers) self._conn_count += 1 if sys.version_info < (3, 3): inittime = time.clock() else: inittime = time.perf_counter() resp = self._conn.getresponse() if sys.version_info < (3, 3): endtime = time.clock() else: endtime = time.perf_counter() LOGGER.info('Response Time for %s to %s: %s seconds.' % (method, reqpath, str(endtime-inittime))) if resp.getheader('Connection') == 'close': self.__destroy_connection() # redirect handling if resp.status not in list(range(300, 399)) or \ resp.status == 304: break newloc = resp.getheader('location') newurl = urlparse(newloc) if resp.status == 303: method = 'GET' body = None for h in ['Content-Type', 'Content-Length']: if h in headers: del headers[h] reqpath = newurl.path self.__init_connection(newurl) restresp = RestResponse(restreq, resp) try: if restresp.getheader('content-encoding') == "gzip": compressedfile = BytesIO(restresp.read) decompressedfile = gzip.GzipFile(fileobj=compressedfile) restresp.text = decompressedfile.read().decode("utf-8") except Exception as excp: LOGGER.error('Error occur while decompressing body: %s', \ excp) raise DecompressResponseError() except Exception as excp: if isinstance(excp, DecompressResponseError): raise if not cause_exception: cause_exception = excp LOGGER.info('Retrying %s [%s]'% (path, excp)) time.sleep(1) self.__init_connection() continue else: break if attempts <= self._max_retry: if LOGGER.isEnabledFor(logging.DEBUG): headerstr = '' if restresp is not None: for header in restresp.getheaders(): headerstr += '\t' + header[0] + ': ' + header[1] + '\n' try: LOGGER.debug('HTTP RESPONSE for %s:\nCode: %s\nHeaders:\n' \ '%s\nBody Response of %s: %s'%\ (restresp.request.path, str(restresp._http_response.status)+ ' ' + \ restresp._http_response.reason, headerstr, restresp.request.path, restresp.read)) except: LOGGER.debug('HTTP RESPONSE:\nCode:%s', (restresp)) else: LOGGER.debug('HTTP RESPONSE: <No HTTP Response obtained>') return restresp else: raise_from(RetriesExhaustedError(), cause_exception)
def test_dump_command(self): add_document() output = StringIO() call_command('dump_memory', stdout=output) data = json.loads(output.getvalue()) self.assertEqual(data, [TEST_DOCUMENT])
class TestOutput(unittest.TestCase): def setUp(self): self.mock_ctx = MockContext() self.io = StringIO() def tearDown(self): self.io.close() def test_cli_ctx_type_error(self): with self.assertRaises(TypeError): OutputProducer(cli_ctx=object()) def test_out_json_valid(self): """ The JSON output when the input is a dict should be the dict serialized to JSON """ output_producer = OutputProducer(cli_ctx=self.mock_ctx) output_producer.out(CommandResultItem({ 'active': True, 'id': '0b1f6472' }), formatter=format_json, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_from_ordered_dict(self): """ The JSON output when the input is OrderedDict should be serialized to JSON """ output_producer = OutputProducer(cli_ctx=self.mock_ctx) output_producer.out(CommandResultItem( OrderedDict({ 'active': True, 'id': '0b1f6472' })), formatter=format_json, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_byte(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) output_producer.out(CommandResultItem({ 'active': True, 'contents': b'0b1f6472' }), formatter=format_json, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""{ "active": true, "contents": "0b1f6472" } """)) def test_out_json_byte_empty(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) output_producer.out(CommandResultItem({ 'active': True, 'contents': b'' }), formatter=format_json, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""{ "active": true, "contents": "" } """)) # TABLE output tests def test_out_table(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = OrderedDict() obj['active'] = True obj['val'] = '0b1f6472' obj['lun'] = 0 output_producer.out(CommandResultItem(obj), formatter=format_table, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Active Lun Val -------- ----- -------- True 0 0b1f6472 """)) def test_out_table_list_of_lists(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = [['a', 'b'], ['c', 'd']] output_producer.out(CommandResultItem(obj), formatter=format_table, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Column1 Column2 --------- --------- a b c d """)) def test_out_table_complex_obj(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = OrderedDict() obj['name'] = 'qwerty' obj['val'] = '0b1f6472qwerty' obj['sub'] = {'1'} result_item = CommandResultItem(obj) output_producer.out(result_item, formatter=format_table, out_file=self.io) self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Name Val ------ -------------- qwerty 0b1f6472qwerty """)) def test_out_table_no_query_no_transformer_order(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } result_item = CommandResultItem(obj, table_transformer=None, is_query_active=False) output_producer.out(result_item, formatter=format_table, out_file=self.io) # Should be alphabetical order as no table transformer and query is not active. self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Active Name Sub Val -------- ------ -------- -------------- True qwerty 0b1f6472 0b1f6472qwerty """)) def test_out_table_no_query_yes_transformer_order(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } def transformer(r): return OrderedDict([('Name', r['name']), ('Val', r['val']), ('Active', r['active']), ('Sub', r['sub'])]) result_item = CommandResultItem(obj, table_transformer=transformer, is_query_active=False) output_producer.out(result_item, formatter=format_table, out_file=self.io) # Should be table transformer order self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Name Val Active Sub ------ -------------- -------- -------- qwerty 0b1f6472qwerty True 0b1f6472 """)) def test_out_table_no_query_yes_jmespath_table_transformer(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } result_item = CommandResultItem( obj, table_transformer='{Name:name, Val:val, Active:active}', is_query_active=False) output_producer.out(result_item, formatter=format_table, out_file=self.io) # Should be table transformer order self.assertEqual( normalize_newlines(self.io.getvalue()), normalize_newlines("""Name Val Active ------ -------------- -------- qwerty 0b1f6472qwerty True """)) def test_out_table_with_number(self): output_producer = OutputProducer(cli_ctx=self.mock_ctx) obj = OrderedDict() obj['Sku'] = '6.10' output_producer.out(CommandResultItem(obj), formatter=format_table, out_file=self.io) self.assertEqual(normalize_newlines(self.io.getvalue()), normalize_newlines("""Sku ----- 6.10 """)) # TSV output tests def test_output_format_dict(self): obj = {} obj['A'] = 1 obj['B'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_dict_sort(self): obj = {} obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '2\t1\n') def test_output_format_ordereddict_not_sorted(self): obj = OrderedDict() obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_ordereddict_list_not_sorted(self): obj1 = OrderedDict() obj1['B'] = 1 obj1['A'] = 2 obj2 = OrderedDict() obj2['A'] = 3 obj2['B'] = 4 result = format_tsv(CommandResultItem([obj1, obj2])) self.assertEqual(result, '1\t2\n3\t4\n')
def do_request(self, req, status=None, expect_errors=None): """ Executes the given webob Request (``req``), with the expected ``status``. Generally :meth:`~webtest.TestApp.get` and :meth:`~webtest.TestApp.post` are used instead. To use this:: req = webtest.TestRequest.blank('url', ...args...) resp = app.do_request(req) .. note:: You can pass any keyword arguments to ``TestRequest.blank()``, which will be set on the request. These can be arguments like ``content_type``, ``accept``, etc. """ errors = StringIO() req.environ['wsgi.errors'] = errors script_name = req.environ.get('SCRIPT_NAME', '') if script_name and req.path_info.startswith(script_name): req.path_info = req.path_info[len(script_name):] # set framework hooks req.environ['paste.testing'] = True req.environ['paste.testing_variables'] = {} # set request cookies self.cookiejar.add_cookie_header(utils._RequestCookieAdapter(req)) # verify wsgi compatibility app = lint.middleware(self.app) if self.lint else self.app # FIXME: should it be an option to not catch exc_info? res = req.get_response(app, catch_exc_info=True) # be sure to decode the content res.decode_content() # set a few handy attributes res._use_unicode = self.use_unicode res.request = req res.app = app res.test_app = self # We do this to make sure the app_iter is exhausted: try: res.body except TypeError: # pragma: no cover pass res.errors = errors.getvalue() for name, value in req.environ['paste.testing_variables'].items(): if hasattr(res, name): raise ValueError( "paste.testing_variables contains the variable %r, but " "the response object already has an attribute by that " "name" % name) setattr(res, name, value) if not expect_errors: self._check_status(status, res) self._check_errors(res) # merge cookies back in self.cookiejar.extract_cookies(utils._ResponseCookieAdapter(res), utils._RequestCookieAdapter(req)) return res
def c_code(self, node, nodename, inp, out, sub): itemsize_x = np.dtype(node.inputs[0].dtype).itemsize worksize_x = np.dtype(work_dtype(node.inputs[0].dtype)).itemsize itemsize_b = np.dtype(node.inputs[1].dtype).itemsize itemsize_y_idx = np.dtype(node.inputs[2].dtype).itemsize itemsize_nll = np.dtype(node.outputs[0].dtype).itemsize itemsize_sm = np.dtype(node.outputs[1].dtype).itemsize itemsize_am = np.dtype(node.outputs[2].dtype).itemsize x, b, y_idx = inp nll, sm, am = out fail = sub['fail'] ctx = sub['params'] k_var = "k_xent_sm_1hot_bias_%(nodename)s" % locals() err_check = """ if (err != GA_NO_ERROR) { PyErr_Format(PyExc_RuntimeError, "gpuarray error: %(k_var)s: %%s.", GpuKernel_error(&%(k_var)s, err)); %(fail)s; } """ % locals() sync = "" if config.gpuarray.sync: sync = """ err = GpuArray_sync(&%(z)s->ga); %(err_check)s """ % locals() sio = StringIO() print(""" if (PyGpuArray_DIMS(%(x)s)[0] != PyGpuArray_DIMS(%(y_idx)s)[0]) { PyErr_SetString(PyExc_ValueError, "dimension mismatch in x,y_idx arguments"); %(fail)s; } if (PyGpuArray_DIMS(%(x)s)[1] != PyGpuArray_DIMS(%(b)s)[0]) { PyErr_SetString(PyExc_ValueError, "dimension mismatch in x,b arguments"); %(fail)s; } if (theano_prep_output(&%(nll)s, 1, PyGpuArray_DIMS(%(y_idx)s), %(x)s->ga.typecode, GA_C_ORDER, %(ctx)s)) %(fail)s if (theano_prep_output(&%(sm)s, 2, PyGpuArray_DIMS(%(x)s), %(x)s->ga.typecode, GA_C_ORDER, %(ctx)s)) %(fail)s if (theano_prep_output(&%(am)s, 1, PyGpuArray_DIMS(%(y_idx)s), %(y_idx)s->ga.typecode, GA_C_ORDER, %(ctx)s)) %(fail)s { size_t n_blocks = std::min(PyGpuArray_DIM(%(x)s, 0), (size_t)4096); size_t n_threads = std::min(PyGpuArray_DIM(%(x)s, 1), (size_t)256); size_t n_shared = n_threads * %(worksize_x)s; //TODO: launch more threads per row and do parallel sum and max reductions int err = k_xent_sm_1hot_bias_call( 1, &n_blocks, &n_threads, n_shared, PyGpuArray_DIMS(%(x)s)[0], PyGpuArray_DIMS(%(x)s)[1], %(x)s->ga.data, %(x)s->ga.offset, PyGpuArray_STRIDE(%(x)s, 0) / %(itemsize_x)s, PyGpuArray_STRIDE(%(x)s, 1) / %(itemsize_x)s, %(b)s->ga.data, %(b)s->ga.offset, PyGpuArray_STRIDE(%(b)s, 0) / %(itemsize_b)s, %(y_idx)s->ga.data, %(y_idx)s->ga.offset, PyGpuArray_STRIDE(%(y_idx)s, 0) / %(itemsize_y_idx)s, %(nll)s->ga.data, %(nll)s->ga.offset, PyGpuArray_STRIDE(%(nll)s, 0) / %(itemsize_nll)s, %(sm)s->ga.data, %(sm)s->ga.offset, PyGpuArray_STRIDE(%(sm)s, 0) / %(itemsize_sm)s, PyGpuArray_STRIDE(%(sm)s, 1) / %(itemsize_sm)s, %(am)s->ga.data, %(am)s->ga.offset, PyGpuArray_STRIDE(%(am)s, 0) / %(itemsize_am)s); %(err_check)s %(sync)s } """ % locals(), file=sio) return sio.getvalue()
def cgitb_error_log(monitor): string_buf = StringIO() cgitb_hook(file=string_buf, format="text") monitor.logger.log(string_buf.getvalue(), level=SandeshLevel.SYS_ERR)
def test_called_command_with_no_jobs_wrong_date(self): out = StringIO() sys.stdout = out call_command("send_weekly_mailing", stdout=out) self.assertEqual("False\n", out.getvalue())
def gpu_kernels(self, node, nodename): dtype_x = node.inputs[0].dtype dtype_b = node.inputs[1].dtype dtype_y_idx = node.inputs[2].dtype work_x = work_dtype(dtype_x) work_b = work_dtype(dtype_b) load_x = load_w(dtype_x) load_b = load_w(dtype_b) write_x = write_w(dtype_x) write_b = write_w(dtype_b) flags = Kernel.get_flags(dtype_x, dtype_b, dtype_y_idx) type_x = gpuarray.dtype_to_ctype(dtype_x) type_b = gpuarray.dtype_to_ctype(dtype_b) work_x = gpuarray.dtype_to_ctype(work_x) type_y_idx = gpuarray.dtype_to_ctype(dtype_y_idx) kname = "k_xent_sm_1hot_bias" k_var = "k_xent_sm_1hot_bias_" + nodename if node.inputs[0].type.context.kind != b'cuda': f = '' else: f = '' if dtype_x == 'float64' else 'f' params = [ gpuarray.SIZE, gpuarray.SIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE, gpuarray.SSIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE, gpuarray.SSIZE, gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SSIZE ] sio = StringIO() print("""#include "cluda.h" KERNEL void %(kname)s(const ga_size M, const ga_size N, GLOBAL_MEM const %(type_x)s* x_data, const ga_size offset_x, const ga_ssize xs0, const ga_ssize xs1, GLOBAL_MEM const %(type_b)s* b, const ga_size offset_b, const ga_ssize bs0, GLOBAL_MEM const %(type_y_idx)s* y_idx_data, const ga_size offset_y_idx, const ga_ssize y_idxs0, GLOBAL_MEM %(type_x)s* nll_data, const ga_size offset_nll, const ga_ssize nlls0, GLOBAL_MEM %(type_x)s* sm_data, const ga_size offset_sm, const ga_ssize sms0, const ga_ssize sms1, GLOBAL_MEM %(type_y_idx)s* am_data, const ga_size offset_am, const ga_ssize ams0 GA_DECL_SHARED_PARAM(%(work_x)s, per_thread_values)) { x_data = (GLOBAL_MEM const %(type_x)s *)(((GLOBAL_MEM char *)x_data)+offset_x); b = (GLOBAL_MEM const %(type_b)s *)(((GLOBAL_MEM char *)b)+offset_b); y_idx_data = (GLOBAL_MEM const %(type_y_idx)s *)(((GLOBAL_MEM char *)y_idx_data)+offset_y_idx); nll_data = (GLOBAL_MEM %(type_x)s *)(((GLOBAL_MEM char *)nll_data)+offset_nll); sm_data = (GLOBAL_MEM %(type_x)s *)(((GLOBAL_MEM char *)sm_data)+offset_sm); am_data = (GLOBAL_MEM %(type_y_idx)s *)(((GLOBAL_MEM char *)am_data)+offset_am); for (ga_int row = GID_0; row < M; row += GDIM_0){ GLOBAL_MEM const %(type_x)s* x = x_data + xs0 * row; GLOBAL_MEM %(type_x)s* sm = sm_data + sms0 * row; GA_DECL_SHARED_BODY(%(work_x)s, per_thread_values); LOCAL_MEM %(work_x)s row_max, sum, sum_inv; LOCAL_MEM ga_int row_max_threadIdx; %(work_x)s per_thread_row_max, per_thread_sum; ga_int per_thread_row_max_j; // COMPUTE ROW MAX AND ARGMAX // compute separate per-thread maximums and argmaxes per_thread_row_max = NAN; per_thread_row_max_j = 0; for (ga_int j = LID_0; j < N; j += LDIM_0) { %(work_x)s row_ij = %(load_x)s(x[j * xs1]) + %(load_b)s(b[j * bs0]); per_thread_row_max_j = (row_ij > per_thread_row_max) ? j : per_thread_row_max_j; per_thread_row_max = fmax%(f)s(row_ij, per_thread_row_max); } per_thread_values[LID_0] = per_thread_row_max; local_barrier(); if (LID_0 == 0) { row_max = NAN; row_max_threadIdx = 0; for (ga_int j = 0; j < LDIM_0; j++) { %(work_x)s per_thread_max = per_thread_values[j]; row_max_threadIdx = (per_thread_max > row_max) ? j : row_max_threadIdx; row_max = fmax%(f)s(per_thread_max, row_max); } } local_barrier(); // The thread with the higest max writes out which of its // values was the winner. if (LID_0 == row_max_threadIdx) am_data[row * ams0] = per_thread_row_max_j; // COMPUTE SOFTMAX per_thread_sum = 0.0; for (ga_int j = LID_0; j < N; j += LDIM_0) { %(work_x)s row_ij = %(load_x)s(x[j * xs1]) + %(load_b)s(b[j * bs0]); %(work_x)s sm_ij = exp%(f)s(row_ij - row_max); per_thread_sum += sm_ij; sm[j * sms1] = %(write_x)s(sm_ij); } per_thread_values[LID_0] = per_thread_sum; local_barrier(); if (LID_0 == 0) { sum = 0.0; for (ga_int j = 0; j < LDIM_0; j++) { sum += per_thread_values[j]; } sum_inv = 1.0 / sum; } local_barrier(); for (ga_int j = LID_0; j < N; j += LDIM_0) { sm[j * sms1] = %(write_x)s(%(load_x)s(sm[j * sms1]) * sum_inv); } if (LID_0 == 0) { const %(type_y_idx)s y_idx = (ga_int)y_idx_data[row * y_idxs0]; if ((y_idx >= N || y_idx < 0)) { // raise some suspicion. nll_data[row * nlls0] = %(write_x)s(0.0); } else { nll_data[row * nlls0] = %(write_x)s( - %(load_x)s(x[y_idx * xs1]) - %(load_b)s(b[y_idx * bs0]) + row_max + log%(f)s(sum)); } } } } """ % locals(), file=sio) return [ Kernel(code=sio.getvalue(), name=kname, params=params, flags=flags, objvar=k_var) ]