def test_subprocess(self): """Instead of the ``argv`` shortcut, subclasses can also use the ``subprocess`` helper manually. """ class Filter(ExternalTool): pass # Without stdin data self.popen.return_value.returncode = 0 self.popen.return_value.communicate.return_value = ['stdout', 'stderr'] out = StringIO() Filter.subprocess(['test'], out) assert out.getvalue() == 'stdout' self.popen.return_value.communicate.assert_called_with(None) # With stdin data self.popen.reset_mock() self.popen.return_value.returncode = 0 self.popen.return_value.communicate.return_value = ['stdout', 'stderr'] out = StringIO() Filter.subprocess(['test'], out, data='data') assert out.getvalue() == 'stdout' self.popen.return_value.communicate.assert_called_with('data') # With error self.popen.return_value.returncode = 1 self.popen.return_value.communicate.return_value = ['stdout', 'stderr'] assert_raises(FilterError, Filter.subprocess, ['test'], StringIO())
def toXml(self, filename=u''): """ converts the document to a valid Xml format. @param filename unicode string: the name of a file, defaults to an empty string. @return if filename is not empty, the XML code will be written into it and the method returns None; otherwise the method returns a StringIO containing valid XML. Then a ".getvalue()" should return a unicode string. """ assert(type(filename)==type(u"")) result=None xml=StringIO() if sys.version_info[0]==2: xml.write(_XMLPROLOGUE) else: xml.write(_XMLPROLOGUE) self.body.toXml(0, xml) if not filename: result=xml.getvalue() else: f=codecs.open(filename,'w', encoding='utf-8') f.write(xml.getvalue()) f.close() return result
def test_low_mapq(self): """ We no longer fail reads because of low mapq. When we use more than one reference, reads can receive low mapq if they are in a conserved region that matches more than one reference. """ remap_file = StringIO("""\ qname,flag,rname,pos,mapq,cigar,rnext,pnext,tlen,seq,qual Example_read_1,99,V3LOOP,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Example_read_1,147,V3LOOP,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Example_read_2,99,INT,1,8,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Example_read_2,147,INT,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA """) expected_aligned_csv = """\ refname,qcut,rank,count,offset,seq INT,15,0,1,0,TGTACAAGACCCAACAACAATACAAGAAAAAG V3LOOP,15,0,1,0,TGTACAAGACCCAACAACAATACAAGAAAAAG """ expected_failed_csv = """\ qname,cause """ actual_aligned_csv = StringIO() actual_failed_csv = StringIO() sam2aln(remap_file, actual_aligned_csv, failed_csv=actual_failed_csv) self.assertMultiLineEqual(expected_aligned_csv, actual_aligned_csv.getvalue()) self.assertMultiLineEqual(expected_failed_csv, actual_failed_csv.getvalue())
def test_info_max_cols(self): df = DataFrame(np.random.randn(10, 5)) for len_, verbose in [(5, None), (5, False), (10, True)]: # For verbose always ^ setting ^ summarize ^ full output with option_context('max_info_columns', 4): buf = StringIO() df.info(buf=buf, verbose=verbose) res = buf.getvalue() assert len(res.strip().split('\n')) == len_ for len_, verbose in [(10, None), (5, False), (10, True)]: # max_cols no exceeded with option_context('max_info_columns', 5): buf = StringIO() df.info(buf=buf, verbose=verbose) res = buf.getvalue() assert len(res.strip().split('\n')) == len_ for len_, max_cols in [(10, 5), (5, 4)]: # setting truncates with option_context('max_info_columns', 4): buf = StringIO() df.info(buf=buf, max_cols=max_cols) res = buf.getvalue() assert len(res.strip().split('\n')) == len_ # setting wouldn't truncate with option_context('max_info_columns', 5): buf = StringIO() df.info(buf=buf, max_cols=max_cols) res = buf.getvalue() assert len(res.strip().split('\n')) == len_
def test_low_read_quality(self): remap_file = StringIO("""\ qname,flag,rname,pos,mapq,cigar,rnext,pnext,tlen,seq,qual Example_read_1,99,V3LOOP,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,000000000000000000AAAAAAAAAAAAAA Example_read_1,147,V3LOOP,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,000000000000000000AAAAAAAAAAAAAA Example_read_2,99,INT,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Example_read_2,147,INT,1,44,32M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA """) expected_aligned_csv = """\ refname,qcut,rank,count,offset,seq INT,15,0,1,0,TGTACAAGACCCAACAACAATACAAGAAAAAG """ expected_failed_csv = """\ qname,cause Example_read_1,manyNs """ actual_aligned_csv = StringIO() actual_failed_csv = StringIO() sam2aln(remap_file, actual_aligned_csv, failed_csv=actual_failed_csv) self.assertMultiLineEqual(expected_aligned_csv, actual_aligned_csv.getvalue()) self.assertMultiLineEqual(expected_failed_csv, actual_failed_csv.getvalue())
def assertBlock(self, python, java): self.maxDiff = None dump = False py_block = PyBlock(parent=PyModule('test', 'test.py')) if python: python = adjust(python) code = compile(python, '<test>', 'exec') py_block.extract(code, debug=dump) java_code = py_block.transpile() out = BytesIO() constant_pool = ConstantPool() java_code.resolve(constant_pool) constant_pool.add(Utf8('test')) constant_pool.add(Utf8('Code')) constant_pool.add(Utf8('LineNumberTable')) writer = ClassFileWriter(out, constant_pool) java_code.write(writer) debug = StringIO() reader = ClassFileReader(BytesIO(out.getbuffer()), constant_pool, debug=debug) JavaCode.read(reader, dump=0) if dump: print(debug.getvalue()) java = adjust(java) self.assertEqual(debug.getvalue(), java[1:])
def test_insertion(self): remap_file = StringIO("""\ qname,flag,rname,pos,mapq,cigar,rnext,pnext,tlen,seq,qual Example_read_1,99,V3LOOP,1,44,12M6I14M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Example_read_1,147,V3LOOP,1,44,12M6I14M,=,1,-32,TGTACAAGACCCAACAACAATACAAGAAAAAG,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA """) expected_aligned_csv = """\ refname,qcut,rank,count,offset,seq V3LOOP,15,0,1,0,TGTACAAGACCCAATACAAGAAAAAG """ expected_insert_csv = """\ qname,fwd_rev,refname,pos,insert,qual Example_read_1,F,V3LOOP,12,AACAAC,AAAAAA Example_read_1,R,V3LOOP,12,AACAAC,AAAAAA """ actual_aligned_csv = StringIO() actual_insert_csv = StringIO() sam2aln(remap_file, actual_aligned_csv, actual_insert_csv) self.assertMultiLineEqual(expected_aligned_csv, actual_aligned_csv.getvalue()) self.assertMultiLineEqual(expected_insert_csv, actual_insert_csv.getvalue())
class TFramedTransport(TTransportBase, CReadableTransport): """Class that wraps another transport and frames its I/O when writing.""" def __init__(self, trans,): self.__trans = trans self.__rbuf = StringIO() self.__wbuf = StringIO() def isOpen(self): return self.__trans.isOpen() def open(self): return self.__trans.open() def close(self): return self.__trans.close() def read(self, sz): ret = self.__rbuf.read(sz) if len(ret) != 0: return ret self.readFrame() return self.__rbuf.read(sz) def readFrame(self): buff = self.__trans.readAll(4) sz, = unpack('!i', buff) self.__rbuf = StringIO(self.__trans.readAll(sz)) def write(self, buf): self.__wbuf.write(buf) def flush(self): wout = self.__wbuf.getvalue() wsz = len(wout) # reset wbuf before write/flush to preserve state on underlying failure self.__wbuf = StringIO() # N.B.: Doing this string concatenation is WAY cheaper than making # two separate calls to the underlying socket object. Socket writes in # Python turn out to be REALLY expensive, but it seems to do a pretty # good job of managing string buffer operations without excessive copies buf = pack("!i", wsz) + wout self.__trans.write(buf) self.__trans.flush() # Implement the CReadableTransport interface. @property def cstringio_buf(self): return self.__rbuf def cstringio_refill(self, prefix, reqlen): # self.__rbuf will already be empty here because fastbinary doesn't # ask for a refill until the previous buffer is empty. Therefore, # we can start reading new frames immediately. while len(prefix) < reqlen: self.readFrame() prefix += self.__rbuf.getvalue() self.__rbuf = StringIO(prefix) return self.__rbuf
def cmd(self, *args, **kw): exit_code = kw.get('exit_code', 0) fork = kw.get('fork', False) if fork: try: output = subprocess.check_output((sys.executable, '-m', 'borg.archiver') + args) ret = 0 except subprocess.CalledProcessError as e: output = e.output ret = e.returncode output = os.fsdecode(output) if ret != exit_code: print(output) self.assert_equal(exit_code, ret) return output args = list(args) stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr try: sys.stdin = StringIO() output = StringIO() sys.stdout = sys.stderr = output ret = self.archiver.run(args) sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr if ret != exit_code: print(output.getvalue()) self.assert_equal(exit_code, ret) return output.getvalue() finally: sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
def test_ecsv_mixins_as_one(table_cls): """Test write/read all cols at once and validate intermediate column names""" names = sorted(mixin_cols) serialized_names = ['ang', 'dt', 'el.x', 'el.y', 'el.z', 'lat', 'lon', 'q', 'sc.ra', 'sc.dec', 'scc.x', 'scc.y', 'scc.z', 'scd.ra', 'scd.dec', 'scd.distance', 'scd.obstime', 'tm', # serialize_method is formatted_value 'tm2', # serialize_method is formatted_value 'tm3.jd1', 'tm3.jd2', # serialize is jd1_jd2 'tm3.location.x', 'tm3.location.y', 'tm3.location.z'] t = table_cls([mixin_cols[name] for name in names], names=names) out = StringIO() t.write(out, format="ascii.ecsv") t2 = table_cls.read(out.getvalue(), format='ascii.ecsv') assert t.colnames == t2.colnames # Read as a ascii.basic table (skip all the ECSV junk) t3 = table_cls.read(out.getvalue(), format='ascii.basic') assert t3.colnames == serialized_names
def scenario_parse_args_exits(self, argv): with self.assertRaises(SystemExit) as cm: stdout = StringIO() stderr = StringIO() with Redirect(stdout=stdout, stderr=stderr): parsed_args = args.parse(argv) return (stdout.getvalue(), stderr.getvalue(), cm.exception.code)
def test_include_partitions(self): """inspectdb --include-partitions creates models for partitions.""" with connection.cursor() as cursor: cursor.execute('''\ CREATE TABLE inspectdb_partition_parent (name text not null) PARTITION BY LIST (left(upper(name), 1)) ''') cursor.execute('''\ CREATE TABLE inspectdb_partition_child PARTITION OF inspectdb_partition_parent FOR VALUES IN ('A', 'B', 'C') ''') out = StringIO() partition_model_parent = 'class InspectdbPartitionParent(models.Model):' partition_model_child = 'class InspectdbPartitionChild(models.Model):' partition_managed = 'managed = False # Created from a partition.' try: call_command('inspectdb', table_name_filter=inspectdb_tables_only, stdout=out) no_partitions_output = out.getvalue() self.assertIn(partition_model_parent, no_partitions_output) self.assertNotIn(partition_model_child, no_partitions_output) self.assertNotIn(partition_managed, no_partitions_output) call_command('inspectdb', table_name_filter=inspectdb_tables_only, include_partitions=True, stdout=out) with_partitions_output = out.getvalue() self.assertIn(partition_model_parent, with_partitions_output) self.assertIn(partition_model_child, with_partitions_output) self.assertIn(partition_managed, with_partitions_output) finally: with connection.cursor() as cursor: cursor.execute('DROP TABLE IF EXISTS inspectdb_partition_child') cursor.execute('DROP TABLE IF EXISTS inspectdb_partition_parent')
def runTest(self): # noqa try: out = StringIO() sys.stdout = out logger = Mann(console=True) logger.log('foo') output = out.getvalue().strip() self.assertEqual(output, 'foo') finally: sys.stdout = sys.__stdout__ try: out = StringIO() sys.stdout = out logger = Mann() logger.log('foo') output = out.getvalue().strip() self.assertEqual(output, '') finally: sys.stdout = sys.__stdout__
def test_handler_args_kwargs(self): out = StringIO() err = StringIO() stream = StringIO() file_name = 'test_logging-test_handler_args_kwargs' with replace_attr(sys, 'stdout', out, 'stderr', err): common_config = finish_common_config('utf-8', {'log_file': [ ['RotatingFileHandler', [[file_name], {'maxBytes': 1, 'backupCount': 1}]] ]}) try: try: logger, pl, get_module_attr = create_logger(common_config, stream=stream) pl.error('Foo') pl.error('Bar') close_handlers(logger) with codecs.open(file_name, encoding='utf-8') as fp: self.assertMatches(fp.read(), '^' + TIMESTAMP_RE + ':ERROR:__unknown__:Bar\n$') with codecs.open(file_name + '.1', encoding='utf-8') as fp: self.assertMatches(fp.read(), '^' + TIMESTAMP_RE + ':ERROR:__unknown__:Foo\n$') finally: os.unlink(file_name + '.1') finally: os.unlink(file_name) self.assertEqual(stream.getvalue(), '') self.assertEqual(err.getvalue(), '') self.assertEqual(out.getvalue(), '')
def test_merge_two_csv(self): # This test is not fully implemented yet. print("\n") _tmpstdout = StringIO() _tmpstderr = StringIO() _call = sys.executable + " " + os.path.abspath(os.path.join(Test_Script_Dir,"..", "optimal_sync.py") + " -d " + os.path.join(Test_Resource_Dir,"test_merge_two_files.json")) print("Call: " + _call) try: _res = check_output(_call, shell=True, cwd=Test_Script_Dir) print("Result : " + str(_res)) except UnsupportedOperation as e: print("Error: " + str(e.args)) except Exception as e: print("Error: " + str(e)) with open(os.path.join(Test_Resource_Dir, "csv_out.csv")) as _f_out: _output = _f_out.read() with open(os.path.join(Test_Resource_Dir, "csv_cmp.csv")) as _f_cmp: _cmp = _f_cmp.read() self.assertEqual(_output, _cmp, diff_strings(_output, _cmp)) print("out : " + str(_tmpstdout.getvalue())) print("err : " + str(_tmpstderr.getvalue())) print("Done") self.assertEqual(True, True)
def test_fail_gracefully_on_bogus__qualname__and__name__(): # Test that we correctly repr types that have non-string values for both # __qualname__ and __name__ class Meta(type): __name__ = 5 class Type(object): __metaclass__ = Meta __qualname__ = 5 stream = StringIO() printer = pretty.RepresentationPrinter(stream) printer.pretty(Type) printer.flush() output = stream.getvalue() # If we can't find __name__ or __qualname__ just use a sentinel string. expected = '.'.join([__name__, '<unknown type>']) assert_equal(output, expected) # Clear stream buffer. stream.buf = '' # Test repring of an instance of the type. instance = Type() printer.pretty(instance) printer.flush() output = stream.getvalue() # Should look like: # <IPython.lib.tests.test_pretty.<unknown type> at 0x7f7658ae07d0> prefix = '<' + '.'.join([__name__, '<unknown type>']) + ' at 0x' assert_true(output.startswith(prefix))
def test_write_multiple_dict_rows(self): fileobj = StringIO() writer = csv.DictWriter(fileobj, fieldnames=["f1", "f2", "f3"]) writer.writeheader() self.assertEqual(fileobj.getvalue(), "f1,f2,f3\r\n") writer.writerows([{"f1": 1, "f2": "abc", "f3": "f"}, {"f1": 2, "f2": 5, "f3": "xyz"}]) self.assertEqual(fileobj.getvalue(), "f1,f2,f3\r\n1,abc,f\r\n2,5,xyz\r\n")
def multiple_alignment(fasta_dict, alignment_type=SeqTypeData().TYPE_DEFAULT): in_handle = StringIO() fasta_tools.write_fasta_handle(in_handle, fasta_dict) muscle_cmd = SeqTypeData().type2cmd[alignment_type] child = subprocess.Popen(str(muscle_cmd), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=(sys.platform != "win32")) if not child: print("Process was not created!") return if sys.version_info[0] == 3: child.stdin.write(bytes(in_handle.getvalue(), 'utf-8')) child.stdin.close() align = AlignIO.read(StringIO("".join(line.decode() for line in child.stdout)), "clustal") else: child.stdin.write(in_handle.getvalue()) child.stdin.close() align = AlignIO.read(child.stdout, "clustal") fd = copy.deepcopy(fasta_dict) for a in align: fd.set(a.id, str(a.seq)) return fd
def test_fallback_to__name__on_type(): # Test that we correctly repr types that have non-string values for # __qualname__ by falling back to __name__ class Type(object): __qualname__ = 5 # Test repring of the type. stream = StringIO() printer = pretty.RepresentationPrinter(stream) printer.pretty(Type) printer.flush() output = stream.getvalue() # If __qualname__ is malformed, we should fall back to __name__. expected = '.'.join([__name__, Type.__name__]) assert_equal(output, expected) # Clear stream buffer. stream.buf = '' # Test repring of an instance of the type. instance = Type() printer.pretty(instance) printer.flush() output = stream.getvalue() # Should look like: # <IPython.lib.tests.test_pretty.Type at 0x7f7658ae07d0> prefix = '<' + '.'.join([__name__, Type.__name__]) + ' at 0x' assert_true(output.startswith(prefix))
def test_serialization(self): "m2m-through models aren't serialized as m2m fields. Refs #8134" pks = {"p_pk": self.bob.pk, "g_pk": self.roll.pk, "m_pk": self.bob_roll.pk} out = StringIO() management.call_command("dumpdata", "m2m_through_regress", format="json", stdout=out) self.assertJSONEqual( out.getvalue().strip(), '[{"pk": %(m_pk)s, "model": "m2m_through_regress.membership", "fields": {"person": %(p_pk)s, "price": ' '100, "group": %(g_pk)s}}, {"pk": %(p_pk)s, "model": "m2m_through_regress.person", "fields": {"name": ' '"Bob"}}, {"pk": %(g_pk)s, "model": "m2m_through_regress.group", "fields": {"name": "Roll"}}]' % pks ) out = StringIO() management.call_command("dumpdata", "m2m_through_regress", format="xml", indent=2, stdout=out) self.assertXMLEqual(out.getvalue().strip(), """ <?xml version="1.0" encoding="utf-8"?> <django-objects version="1.0"> <object pk="%(m_pk)s" model="m2m_through_regress.membership"> <field to="m2m_through_regress.person" name="person" rel="ManyToOneRel">%(p_pk)s</field> <field to="m2m_through_regress.group" name="group" rel="ManyToOneRel">%(g_pk)s</field> <field type="IntegerField" name="price">100</field> </object> <object pk="%(p_pk)s" model="m2m_through_regress.person"> <field type="CharField" name="name">Bob</field> </object> <object pk="%(g_pk)s" model="m2m_through_regress.group"> <field type="CharField" name="name">Roll</field> </object> </django-objects> """.strip() % pks)
def test_dict(self): d = SqliteDict() self.run_dict_tests( d) d = SqliteDict(index=False) self.run_dict_tests(d) d["extra"] = "item" d2 = SqliteDict({1:"a", 2:"b", 3:"c", "1":5}) d.update(d2) self.assertEqual(7, len(d)) self.assertIn(1, d) self.assertIn(2, d) self.assertEqual(5, d["1"]) d = SqliteDict({1:"a", 2:"b", 3:"c", "1":5}, filename="mydict.sqlite") d2 = SqliteDict(filename="mydict.sqlite") self.assertEqual(set([x for x in d.items()]), set([x for x in d2.items()])) d = SqliteDict({1:"a", 2:"b", 3:"c", "1":5}) self.assertEqual(4, len(d)) d.clear() self.assertEqual(0, len(d)) d = SqliteDict({1:"a", 2:"b", 3:"c"}) io = StringIO() d.write(io) self.assertEqual({"1":"a", "2":"b", "3":"c"}, json.loads(io.getvalue())) d = SqliteDict({1:"a", 2:"b", 3:"c"}) io = StringIO() d.write_lines(io) values = sorted([line.split("\t") for line in io.getvalue().strip().split("\n")]) self.assertEqual([["1", '"a"'], ["2", '"b"'], ['3', '"c"']], values)
def test_printer(self): # Test the printer class m1 = dsl_mock('m1') m2 = dsl_mock('m2') m3 = dsl_mock('m3') o1 = other_mock('o1') f = StringIO() p = printer(endl=';',output_file=f,tab=2) p.out( m1 ) self.assertEqual( f.getvalue(), 'm1;\n' ) p.indent() p.out( 199 ) self.assertEqual( f.getvalue(), 'm1;\n 199;\n' ) p.indent() p.out( m2 ) self.assertEqual( f.getvalue(), 'm1;\n 199;\n m2;\n') p.outdent() p.outdent() p.out( m3 ) self.assertEqual( f.getvalue(), 'm1;\n 199;\n m2;\nm3;\n') p.out( o1 ) self.assertEqual( f.getvalue(), 'm1;\n 199;\n m2;\nm3;\n__o1__;\n') # Negative indent not allowed self.assertRaises(AssertionError, p.outdent)
def cbFun(sendRequesthandle, errorIndication, errorStatus, errorIndex, varBindTable, cbCtx): global oid_list global maxRepetitions if errorIndication: print(errorIndication) return # stop on error if errorStatus: print('%s at %s' % ( errorStatus.prettyPrint(), errorIndex and varBindTable[-1][int(errorIndex)-1] or '?' ) ) return # stop on error #print(varBindTable) for varBindRow in varBindTable: if maxRepetitions == 0:#如果为0 return#停止,并且返回 else: for oid, val in varBindTable[0]: o = StringIO() print(oid,file=o) oid_get = o.getvalue().strip()#通过print到StringIO进行转码,然后读回 o.close() v = StringIO() print(val,file=v) val_get = v.getvalue().strip()#通过print到StringIO进行转码,然后读回 v.close() oid_list.append((oid_get,val_get))#把oid和val的对添加到全局清单oid_list maxRepetitions -= 1#数量减一 return True # signal dispatcher to continue walking#返回一个型号,继续往下查询!
def test_detect_wig_pos(self): si.replicate_comparison = self.mock.mock_replicate_comparison nums = {'pro': 3, 'tss': 3, 'uni': 0, 'cds': 3, 'ta': 3} out_table = StringIO() output = StringIO() args = self.mock_args.mock() args.texs = "texs" args.replicates = "rep" args.max_len = 300 args.min_len = 30 args.decrease_inter = 50 args.fuzzy_inter = 5 args.tex_notex = "tex_notex" args.pros = copy.deepcopy(self.example.pros) tas = copy.deepcopy(self.example.tas) args.table_best = True args.nums = nums args.out_table = out_table args.output = output args.tolerance = 5 si.detect_wig_pos(self.example.wigs, tas[0], 20, 70, "TSS_160+", 10, 20, args) self.assertEqual(output.getvalue(), ("aaa\tANNOgesic\tncRNA\t20\t190\t.\t+\t.\t" "ID=aaa_srna0;Name=sRNA_00000;sRNA_type=intergenic;" "with_TSS=TSS_160+;end_cleavage=Cleavage:190_+;" "best_avg_coverage=40;best_high_coverage=50;" "best_low_coverage=10\n")) self.assertEqual(out_table.getvalue(), ("aaa\t00000\t20\t190\t+\tcond1\t" "test1\t40\t50\t10\t\n"))
class BaseReporter(object): """Reporter Object to collate the failed tests""" def __init__(self): self.report_logger = logging.getLogger('servercheck') self.failed_tests = StringIO() report_handler = logging.StreamHandler(self.failed_tests) report_handler.addFilter(FailFilter()) report_handler.setLevel(logging.WARNING) self.report_logger.addHandler(report_handler) def report(self): failed_tests = self.failed_tests.getvalue() msg = '----- Report: -----\n\n' if failed_tests: msg += self.failed_tests.getvalue() log_level = 'warn' else: msg += '\033[1;32mAll tests passed, YAY!\033[0m\n' log_level = 'info' msg += '\n-------------------' getattr(self.report_logger, log_level)(msg)
def render(self, data, accepted_media_type=None, renderer_context=None): """ Renders `data` into serialized XML. """ if data is None: return '' stream = StringIO() xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() # If we do not have users or request_user then we have errors if not data.get('users', False) and not data.get('request_user', False): self._to_errors(data, xml) xml.endDocument() return stream.getvalue() # If users are a list, deal with that if type(data['users']) is QuerySet or type(data['users']) is list: xml.startElement('people', {'type': 'array'}) self._to_xml(data['users'], data['request_user'], xml) xml.endElement('people') # Otherwise just render a person else: self.render_person(data['users'], data['request_user'], xml) xml.endDocument() return stream.getvalue()
def test_print_file(self): string = "aaa\tintergenic\tsRNA\t10\t15\t.\t+\t." nums = {'pro': 3, 'tss': 3, 'uni': 0, 'cds': 3, 'ta': 3} out_table = StringIO() output = StringIO() srna_datas = {"high": 20, "low": 5, "best": 13, "conds": {"cond1": "test1"}, "detail": [{"track": "test1", "high": 30, "low": 10, "avg": 15}, {"track": "test2", "high": 25, "low": 13, "avg": 20}]} args = self.mock_args.mock() args.nums = nums args.out_table = out_table args.output = output args.table_best = False si.print_file(string, "TSS_160+", srna_datas, "intergenic", args, "aaa") self.assertEqual(out_table.getvalue(), ("aaa\t00000\t10\t15\t+\tcond1\ttest1\t13\t20\t5\t" "TSS_160+\ttest1(avg=15;high=30;low=10);" "test2(avg=20;high=25;low=13)\n")) self.assertEqual(output.getvalue(), ("aaa\tintergenic\tsRNA\t10\t15\t.\t+\t.\t" "ID=aaa_srna0;Name=sRNA_00000;sRNA_type=intergenic;" "with_TSS=TSS_160+;best_avg_coverage=13;" "best_high_coverage=20;best_low_coverage=5\n"))
class DumpOptionsTests(unittest.TestCase): stream = None def setUp(self): self.stream = StringIO() def test_indent_four(self): rc = yajl.dump({'foo' : 'bar'}, self.stream, indent=4) expected = '{\n "foo": "bar"\n}\n' self.assertEquals(self.stream.getvalue(), expected) def test_indent_zero(self): rc = yajl.dump({'foo' : 'bar'}, self.stream, indent=0) expected = '{\n"foo": "bar"\n}\n' self.assertEquals(self.stream.getvalue(), expected) def test_indent_str(self): self.failUnlessRaises(TypeError, yajl.dump, {'foo' : 'bar'}, self.stream, indent='4') def test_negative_indent(self): ''' Negative `indent` should not result in pretty printing ''' rc = yajl.dump({'foo' : 'bar'}, self.stream, indent=-1) self.assertEquals(self.stream.getvalue(), '{"foo":"bar"}') def test_none_indent(self): ''' None `indent` should not result in pretty printing ''' rc = yajl.dump({'foo' : 'bar'}, self.stream, indent=None) self.assertEquals(self.stream.getvalue(), '{"foo":"bar"}')
def test_info_memory_usage_qualified(self): buf = StringIO() df = DataFrame(1, columns=list('ab'), index=[1, 2, 3]) df.info(buf=buf) assert '+' not in buf.getvalue() buf = StringIO() df = DataFrame(1, columns=list('ab'), index=list('ABC')) df.info(buf=buf) assert '+' in buf.getvalue() buf = StringIO() df = DataFrame(1, columns=list('ab'), index=pd.MultiIndex.from_product( [range(3), range(3)])) df.info(buf=buf) assert '+' not in buf.getvalue() buf = StringIO() df = DataFrame(1, columns=list('ab'), index=pd.MultiIndex.from_product( [range(3), ['foo', 'bar']])) df.info(buf=buf) assert '+' in buf.getvalue()
def run_level36(): f = """\ 4 2 1 1 2 3 3 3 . . 2 3 3 . 4 . . 2 . 2 4 3 2 2 2 . . . 2 4 3 4 . . 3 2 3 3 """ order = DESCENDING strategy = Done.FIRST_STRATEGY output = StringIO() solve_file(f, strategy, order, output) expected = """\ 3 4 3 2 3 4 4 . 3 2 . . 3 4 3 2 . 1 . 3 . 2 3 3 . 2 . 2 3 . 2 . 2 2 2 . 1 """ if output.getvalue() != expected: raise AssertionError("got a wrong answer:\n%s" % output.getvalue())
def test_split_file_writer_with_json(s3, folder, filename, compress, filewriter_type): """Test jsonline string and bytes writer""" bucket_name = "test" ext = "jsonl.gz" if compress else "jsonl" s3.meta.client.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "eu-west-1"}, ) jsonl_data = [ { "i": 1, "x1": "a", "x2": "b" }, { "i": 2, "x1": "a", "x2": "b" }, { "i": 3, "x1": "a", "x2": "b" }, { "i": 4, "x1": "a", "x2": "b" }, { "i": 5, "x1": "a", "x2": "b" }, ] if filewriter_type == "string": f = StringSplitFileWriter( f"s3://{bucket_name}/{folder}", filename, max_bytes=60, compress_on_upload=compress, file_extension=ext, ) elif filewriter_type == "bytes": f = BytesSplitFileWriter( f"s3://{bucket_name}/{folder}", filename, max_bytes=60, compress_on_upload=compress, file_extension=ext, ) else: raise ValueError( "Input filewriter_type must be either 'string' or 'bytes'") # Write data j_writer = jsonlines.Writer(f) expected_file = StringIO() e_j_writer = jsonlines.Writer(expected_file) for row in jsonl_data: j_writer.write(row) e_j_writer.write(row) f.close() actual_s3_objects = sorted( [o.key for o in s3.Bucket(bucket_name).objects.all()]) # Test files written to s3 expected_s3_objects = [ f"{folder}{filename}-0.{ext}", f"{folder}{filename}-1.{ext}", f"{folder}{filename}-2.{ext}", ] assert expected_s3_objects == actual_s3_objects # Test file contents expected = expected_file.getvalue() actual = "" for expeceted_object in expected_s3_objects: file_object = BytesIO() s3.Object(bucket_name, expeceted_object).download_fileobj(file_object) if compress: actual += gzip.decompress(file_object.getvalue()).decode("utf-8") else: actual += file_object.getvalue().decode("utf-8") file_object.close() assert actual == expected
wrongmovefrom = 0 while not b.is_game_over(): print("Referee Board:") b.prettyPrint() print("Before move", nbmoves) legals = b.legal_moves() print("Legal Moves: ", legals) nbmoves += 1 otherplayer = (nextplayer + 1) % 2 othercolor = Goban.Board.flip(nextplayercolor) currentTime = time.time() sys.stdout = stringio move = players[nextplayer].getPlayerMove() sys.stdout = sysstdout playeroutput = "\r" + stringio.getvalue() stringio.truncate(0) print(("[Player "+str(nextplayer) + "] ").join(playeroutput.splitlines(True))) outputs[nextplayer] += playeroutput totalTime[nextplayer] += time.time() - currentTime print("Player ", nextplayercolor, players[nextplayer].getPlayerName(), "plays" + str(move)) if not move in legals: print(otherplayer, nextplayer, nextplayercolor) print("Problem: illegal move") wrongmovefrom = nextplayercolor break b.push(move) players[otherplayer].playOpponentMove(move) nextplayer = otherplayer nextplayercolor = othercolor
def put_df_to_S3(df, bucket, path): from io import StringIO csv_buffer = StringIO() df.to_csv(csv_buffer) s3_resource = boto3.resource('s3') s3_resource.Object(bucket, path).put(Body=csv_buffer.getvalue())
class OutputEventFilter(object): ''' File-like object that looks for encoded job events in stdout data. ''' EVENT_DATA_RE = re.compile( r'\x1b\[K((?:[A-Za-z0-9+/=]+\x1b\[\d+D)+)\x1b\[K') def __init__(self, event_callback): self._event_callback = event_callback self._counter = 0 self._start_line = 0 self._buffer = StringIO() self._last_chunk = '' self._current_event_data = None def flush(self): # pexpect wants to flush the file it writes to, but we're not # actually capturing stdout to a raw file; we're just # implementing a custom `write` method to discover and emit events from # the stdout stream pass def write(self, data): data = smart_str(data) self._buffer.write(data) # keep a sliding window of the last chunk written so we can detect # event tokens and determine if we need to perform a search of the full # buffer should_search = '\x1b[K' in (self._last_chunk + data) self._last_chunk = data # Only bother searching the buffer if we recently saw a start/end # token (\x1b[K) while should_search: value = self._buffer.getvalue() match = self.EVENT_DATA_RE.search(value) if not match: break try: base64_data = re.sub(r'\x1b\[\d+D', '', match.group(1)) event_data = json.loads(base64.b64decode(base64_data)) except ValueError: event_data = {} self._emit_event(value[:match.start()], event_data) remainder = value[match.end():] self._buffer = StringIO() self._buffer.write(remainder) self._last_chunk = remainder def close(self): value = self._buffer.getvalue() if value: self._emit_event(value) self._buffer = StringIO() self._event_callback(dict(event='EOF', final_counter=self._counter)) def _emit_event(self, buffered_stdout, next_event_data=None): next_event_data = next_event_data or {} if self._current_event_data: event_data = self._current_event_data stdout_chunks = [buffered_stdout] elif buffered_stdout: event_data = dict(event='verbose') stdout_chunks = buffered_stdout.splitlines(True) else: stdout_chunks = [] for stdout_chunk in stdout_chunks: self._counter += 1 event_data['counter'] = self._counter event_data['stdout'] = stdout_chunk[:-2] if len( stdout_chunk) > 2 else "" n_lines = stdout_chunk.count('\n') event_data['start_line'] = self._start_line event_data['end_line'] = self._start_line + n_lines self._start_line += n_lines if self._event_callback: self._event_callback(event_data) if next_event_data.get('uuid', None): self._current_event_data = next_event_data else: self._current_event_data = None
def test_command_start(self): out = StringIO() call_command('directord', daemon_action='start', stdout=out) self.assertIn('Expected output', out.getvalue())
def generate_c_docstrings(): from astropy.wcs import docstrings docstrings = docstrings.__dict__ keys = [ key for key in docstrings.keys() if not key.startswith('__') and type(key) in string_types ] keys.sort() docs = {} for key in keys: docs[key] = docstrings[key].encode('utf8').lstrip() + b'\0' h_file = StringIO() h_file.write("""/* DO NOT EDIT! This file is autogenerated by astropy/wcs/setup_package.py. To edit its contents, edit astropy/wcs/docstrings.py */ #ifndef __DOCSTRINGS_H__ #define __DOCSTRINGS_H__ #if defined(_MSC_VER) void fill_docstrings(void); #endif """) for key in keys: val = docs[key] h_file.write('extern char doc_{0}[{1}];\n'.format(key, len(val))) h_file.write("\n#endif\n\n") setup_helpers.write_if_different(join(WCSROOT, 'include', 'docstrings.h'), h_file.getvalue().encode('utf-8')) c_file = StringIO() c_file.write("""/* DO NOT EDIT! This file is autogenerated by astropy/wcs/setup_package.py. To edit its contents, edit astropy/wcs/docstrings.py The weirdness here with strncpy is because some C compilers, notably MSVC, do not support string literals greater than 256 characters. */ #include <string.h> #include "docstrings.h" #if defined(_MSC_VER) """) for key in keys: val = docs[key] c_file.write('char doc_{0}[{1}];\n'.format(key, len(val))) c_file.write("\nvoid fill_docstrings(void)\n{\n") for key in keys: val = docs[key] # For portability across various compilers, we need to fill the # docstrings in 256-character chunks for i in range(0, len(val), 256): chunk = string_escape(val[i:i + 256]).replace('"', '\\"') c_file.write(' strncpy(doc_{0} + {1}, "{2}", {3});\n'.format( key, i, chunk, min(len(val) - i, 256))) c_file.write("\n") c_file.write("\n}\n\n") c_file.write("#else /* UNIX */\n") for key in keys: val = docs[key] c_file.write('char doc_{0}[{1}] = "{2}";\n\n'.format( key, len(val), string_escape(val).replace('"', '\\"'))) c_file.write("#endif\n") setup_helpers.write_if_different(join(WCSROOT, 'src', 'docstrings.c'), c_file.getvalue().encode('utf-8'))
def runjob(self, db, job_uuid, **kw): http.request.session.db = db env = http.request.env(user=odoo.SUPERUSER_ID) def retry_postpone(job, message, seconds=None): job.env.clear() with odoo.api.Environment.manage(): with odoo.registry(job.env.cr.dbname).cursor() as new_cr: job.env = job.env(cr=new_cr) job.postpone(result=message, seconds=seconds) job.set_pending(reset_retry=False) job.store() new_cr.commit() # ensure the job to run is in the correct state and lock the record env.cr.execute( "SELECT state FROM queue_job WHERE uuid=%s AND state=%s FOR UPDATE", (job_uuid, ENQUEUED), ) if not env.cr.fetchone(): _logger.warn( "was requested to run job %s, but it does not exist, " "or is not in state %s", job_uuid, ENQUEUED, ) return "" job = Job.load(env, job_uuid) assert job and job.state == ENQUEUED try: try: self._try_perform_job(env, job) except OperationalError as err: # Automatically retry the typical transaction serialization # errors if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: raise retry_postpone( job, tools.ustr(err.pgerror, errors="replace"), seconds=PG_RETRY ) _logger.debug("%s OperationalError, postponed", job) except NothingToDoJob as err: if str(err): msg = str(err) else: msg = _("Job interrupted and set to Done: nothing to do.") job.set_done(msg) job.store() env.cr.commit() except RetryableJobError as err: # delay the job later, requeue retry_postpone(job, str(err), seconds=err.seconds) _logger.debug("%s postponed", job) except (FailedJobError, Exception): buff = StringIO() traceback.print_exc(file=buff) _logger.error(buff.getvalue()) job.env.clear() with odoo.api.Environment.manage(): with odoo.registry(job.env.cr.dbname).cursor() as new_cr: job.env = job.env(cr=new_cr) job.set_failed(exc_info=buff.getvalue()) job.store() new_cr.commit() raise return ""
def test_check_estimator(): # tests that the estimator actually fails on "bad" estimators. # not a complete test of all checks, which are very extensive. # check that we have a set_params and can clone msg = "it does not implement a 'get_params' methods" assert_raises_regex(TypeError, msg, check_estimator, object) assert_raises_regex(TypeError, msg, check_estimator, object()) # check that values returned by get_params match set_params msg = "get_params result does not match what was passed to set_params" assert_raises_regex(AssertionError, msg, check_estimator, ModifiesValueInsteadOfRaisingError()) assert_warns(UserWarning, check_estimator, RaisesErrorInSetParams()) assert_raises_regex(AssertionError, msg, check_estimator, ModifiesAnotherValue()) # check that we have a fit method msg = "object has no attribute 'fit'" assert_raises_regex(AttributeError, msg, check_estimator, BaseEstimator) assert_raises_regex(AttributeError, msg, check_estimator, BaseEstimator()) # check that fit does input validation msg = "ValueError not raised" assert_raises_regex(AssertionError, msg, check_estimator, BaseBadClassifier) assert_raises_regex(AssertionError, msg, check_estimator, BaseBadClassifier()) # check that sample_weights in fit accepts pandas.Series type try: from pandas import Series # noqa msg = ("Estimator NoSampleWeightPandasSeriesType raises error if " "'sample_weight' parameter is of type pandas.Series") assert_raises_regex(ValueError, msg, check_estimator, NoSampleWeightPandasSeriesType) except ImportError: pass # check that predict does input validation (doesn't accept dicts in input) msg = "Estimator doesn't check for NaN and inf in predict" assert_raises_regex(AssertionError, msg, check_estimator, NoCheckinPredict) assert_raises_regex(AssertionError, msg, check_estimator, NoCheckinPredict()) # check that estimator state does not change # at transform/predict/predict_proba time msg = 'Estimator changes __dict__ during predict' assert_raises_regex(AssertionError, msg, check_estimator, ChangesDict) # check that `fit` only changes attribures that # are private (start with an _ or end with a _). msg = ('Estimator ChangesWrongAttribute should not change or mutate ' 'the parameter wrong_attribute from 0 to 1 during fit.') assert_raises_regex(AssertionError, msg, check_estimator, ChangesWrongAttribute) check_estimator(ChangesUnderscoreAttribute) # check that `fit` doesn't add any public attribute msg = (r'Estimator adds public attribute\(s\) during the fit method.' ' Estimators are only allowed to add private attributes' ' either started with _ or ended' ' with _ but wrong_attribute added') assert_raises_regex(AssertionError, msg, check_estimator, SetsWrongAttribute) # check for invariant method name = NotInvariantPredict.__name__ method = 'predict' msg = ("{method} of {name} is not invariant when applied " "to a subset.").format(method=method, name=name) assert_raises_regex(AssertionError, msg, check_estimator, NotInvariantPredict) # check for sparse matrix input handling name = NoSparseClassifier.__name__ msg = "Estimator %s doesn't seem to fail gracefully on sparse data" % name # the check for sparse input handling prints to the stdout, # instead of raising an error, so as not to remove the original traceback. # that means we need to jump through some hoops to catch it. old_stdout = sys.stdout string_buffer = StringIO() sys.stdout = string_buffer try: check_estimator(NoSparseClassifier) except: pass finally: sys.stdout = old_stdout assert msg in string_buffer.getvalue() # Large indices test on bad estimator msg = ('Estimator LargeSparseNotSupportedClassifier doesn\'t seem to ' r'support \S{3}_64 matrix, and is not failing gracefully.*') assert_raises_regex(AssertionError, msg, check_estimator, LargeSparseNotSupportedClassifier) # non-regression test for estimators transforming to sparse data check_estimator(SparseTransformer()) # doesn't error on actual estimator check_estimator(AdaBoostClassifier) check_estimator(AdaBoostClassifier()) check_estimator(MultiTaskElasticNet) check_estimator(MultiTaskElasticNet()) # doesn't error on binary_only tagged estimator check_estimator(TaggedBinaryClassifier) # does error on binary_only untagged estimator msg = 'Only 2 classes are supported' assert_raises_regex(ValueError, msg, check_estimator, UntaggedBinaryClassifier)
def distribute_task(table, func_to_run, bucket=None, func_kwargs=None, func_class=None, func_class_kwargs=None, catch=False, group_count=100, storage='s3'): """ Distribute processing rows in a table across multiple AWS Lambda invocations. If you are running the processing of a table inside AWS Lambda, then you are limited by how many rows can be processed within the Lambda's time limit (at time-of-writing, maximum 15min). Based on experience and some napkin math, with the same data that would allow 1000 rows to be processed inside a single AWS Lambda instance, this method allows 10 MILLION rows to be processed. Rather than converting the table to SQS or other options, the fastest way is to upload the table to S3, and then invoke multiple Lambda sub-invocations, each of which can be sent a byte-range of the data in the S3 CSV file for which to process. Using this method requires some setup. You have three tasks: 1. Define the function to process rows, the first argument, must take your table's data (though only a subset of rows will be passed) (e.g. `def task_for_distribution(table, **kwargs):`) 2. Where you would have run `task_for_distribution(my_table, **kwargs)` instead call `distribute_task(my_table, task_for_distribution, func_kwargs=kwargs) (either setting env var S3_TEMP_BUCKET or passing a bucket= parameter) 3. Setup your Lambda handler to include :py:meth:`parsons.aws.event_command` (or run and deploy your lambda with `Zappa <https://github.com/Miserlou/Zappa>`_) To test locally, include the argument `storage="local"` which will test the distribute_task function, but run the task sequentially and in local memory. A minimalistic example Lambda handler might look something like this: .. code-block:: python :emphasize-lines: 5,6 from parsons.aws import event_command, distribute_task def process_table(table, foo, bar=None): for row in table: do_sloooooow_thing(row, foo, bar) def handler(event, context): ## ADD THESE TWO LINES TO TOP OF HANDLER: if event_command(event, context): return table = FakeDatasource.load_to_table(username='******', password='******') # table is so big that running # process_table(table, foo=789, bar='baz') would timeout # so instead we: distribute_task(table, process_table, bucket='my-temp-s3-bucket', func_kwargs={'foo': 789, 'bar': 'baz'}) `Args:` table: Parsons Table Table of data you wish to distribute processing across Lambda invocations of `func_to_run` argument. func_to_run: function The function you want to run whose first argument will be a subset of table bucket: str The bucket name to use for s3 upload to process the whole table Not required if you set environment variable ``S3_TEMP_BUCKET`` func_kwargs: dict If the function has other arguments to pass along with `table` then provide them as a dict here. They must all be JSON-able. func_class: class If the function is a classmethod or function on a class, then pass the pure class here. E.g. If you passed `ActionKit.bulk_upload_table`, then you would pass `ActionKit` here. func_class_kwargs: dict If it is a class function, and the class must be instantiated, then pass the kwargs to instantiate the class here. E.g. If you passed `ActionKit.bulk_upload_table` as the function, then you would pass {'domain': ..., 'username': ... etc} here. This must all be JSON-able data. catch: bool Lambda will retry running an event three times if there's an exception -- if you want to prevent this, set `catch=True` and then it will catch any errors and stop retries. The error will be in CloudWatch logs with string "Distribute Error" This might be important if row-actions are not idempotent and your own function might fail causing repeats. group_count: int Set this to how many rows to process with each Lambda invocation (Default: 100) storage: str Debugging option: Defaults to "s3". To test distribution locally without s3, set to "local". `Returns:` Debug information -- do not rely on the output, as it will change depending on how this method is invoked. """ if storage not in ('s3', 'local'): raise DistributeTaskException(f'storage argument must be s3 or local') bucket = check('S3_TEMP_BUCKET', bucket) csvdata = StringIO() outcsv = csv.writer(csvdata) outcsv.writerows(table.table.data()) return distribute_task_csv(csvdata.getvalue().encode('utf-8-sig'), func_to_run, bucket, header=table.columns, func_kwargs=func_kwargs, func_class=func_class, func_class_kwargs=func_class_kwargs, catch=catch, group_count=group_count, storage=storage)
entry_points={ 'console_scripts': [ 'cocotb-config=cocotb.config:main', ] }, platforms='any', classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "License :: OSI Approved :: BSD License", "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", ], # these appear in the sidebar on PyPI project_urls={ "Bug Tracker": "https://github.com/cocotb/cocotb/issues", "Source Code": "https://github.com/cocotb/cocotb", "Documentation": "https://docs.cocotb.org", }, extras_require={ "bus": ["cocotb_bus"] } ) print(log_stream.getvalue())
def test_print3 (self) : w = StringIO() collatz_print(w, 23832, 23234, 284) self.assertEqual(w.getvalue(), "23832 23234 284\n")
finally: if f: f.close() with open('d:/dev-test', 'r') as f: print(f.read()) with open('d:/dev-test', 'r') as f: for line in f.readlines(): print(line.strip()) with open('d:/dev-test', 'r', encoding='gbk', errors='ignore') as f: for line in f.readlines(): print(line.strip()) with open('d:/1.txt', 'w', encoding='gbk') as f: f.write('abc') f = StringIO('Hello!\nHi!\nGoodbye!') while True: s = f.readline() if s == '': break print(s.strip()) f = BytesIO() f.write('中文'.encode('utf-8')) print(f.getvalue())
def test_print1 (self) : w = StringIO() collatz_print(w, 1, 10, 20) self.assertEqual(w.getvalue(), "1 10 20\n")
class TestConditionalVariants(unittest.TestCase): def setUp(self): self.lgr = LGR() # Configure log system to redirect validation logs to local attribute self.log_output = StringIO() ch = logging.StreamHandler(self.log_output) ch.setLevel(logging.DEBUG) logging.getLogger('lgr.validate').addHandler(ch) def test_empty_lgr(self): success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertEqual(len(log_content), 0) self.assertTrue(success) self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': []}) def test_no_variants(self): self.lgr.add_cp([0x0061]) self.lgr.add_cp([0x0062]) success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertEqual(len(log_content), 0) self.assertTrue(success) self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': []}) def test_no_rule_when(self): self.lgr.add_cp([0x0061]) self.lgr.add_variant([0x0061], [0x0062], when="when-rule") success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertGreater(len(log_content), 0) self.assertEqual(log_content, "CP U+0061: Variant 'U+0062' \"when\" attribute " "'when-rule' is not an existing rule name.\n") self.assertFalse(success) var = self.lgr.get_variant([0x0061], (0x0062, ))[0] self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': [{'char': self.lgr.get_char([0x0061]), 'variant': var, 'rule_type': 'when', 'rule': var.when}]}) def test_no_rule_not_when(self): self.lgr.add_cp([0x0061]) self.lgr.add_variant([0x0061], [0x0062], not_when="not-when-rule") success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertGreater(len(log_content), 0) self.assertEqual(log_content, "CP U+0061: Variant 'U+0062' \"not-when\" attribute " "'not-when-rule' is not an existing rule name.\n") self.assertFalse(success) var = self.lgr.get_variant([0x0061], (0x0062, ))[0] self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': [{'char': self.lgr.get_char([0x0061]), 'variant': var, 'rule_type': 'not-when', 'rule': var.not_when}]}) def test_no_rule_when_not_when(self): self.lgr.add_cp([0x0061]) self.lgr.add_variant([0x0061], [0x0062], when="when-rule", not_when="not-when-rule", force=True) success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertGreater(len(log_content), 0) self.assertEqual(log_content, "CP U+0061: Variant 'U+0062' \"when\" attribute " "'when-rule' is not an existing rule name.\n" "CP U+0061: Variant 'U+0062' \"not-when\" attribute " "'not-when-rule' is not an existing rule name.\n") self.assertFalse(success) var = self.lgr.get_variant([0x0061], (0x0062, ))[0] self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': [{'char': self.lgr.get_char([0x0061]), 'variant': var, 'rule_type': 'when', 'rule': var.when}, {'char': self.lgr.get_char([0x0061]), 'variant': var, 'rule_type': 'not-when', 'rule': var.not_when} ]}) def test_conditional_when_ok(self): self.lgr.add_cp([0x0061]) self.lgr.add_variant([0x0061], [0x0062], when="when-rule") self.lgr.rules.append("when-rule") success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertEqual(len(log_content), 0) self.assertTrue(success) self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': []}) def test_conditional_not_when_ok(self): self.lgr.add_cp([0x0061]) self.lgr.add_variant([0x0061], [0x0062], not_when="not-when-rule") self.lgr.rules.append("not-when-rule") success, result = check_conditional_variants(self.lgr, {}) log_content = self.log_output.getvalue() self.assertEqual(len(log_content), 0) self.assertTrue(success) self.assertDictEqual(result, {'description': 'Testing conditional variants', 'repertoire': []})
def test_solve2 (self): read = StringIO("491 2460\n950 2853\n325 1296\n") write = StringIO() collatz_solve(read, write) self.assertEqual( write.getvalue(), "491 2460 183\n950 2853 209\n325 1296 182\n")
def test_print2 (self) : w = StringIO() collatz_print(w, 1, 100, 119) self.assertEqual(w.getvalue(), "1 100 119\n")
def test_command_success(self, ctx, config): result = StringIO() ctx.cluster.run(args=["python3", "-c", "print('hi')"], stdout=result) assert result.getvalue().strip() == "hi"
def test_solve3 (self): reader = StringIO("1 1\n8493 358846\n8473 27\n11234 1378\n") writer = StringIO() collatz_solve(reader, writer) self.assertEqual(writer.getvalue(), \ "1 1 1\n8493 358846 443\n8473 27 262\n11234 1378 268\n")
def save_ihelp_to_file(function,save_help=False,save_code=True, as_md=False,as_txt=True, folder='readme_resources/ihelp_outputs/', filename=None,file_mode='w'): """Saves the string representation of the ihelp source code as markdown. Filename should NOT have an extension. .txt or .md will be added based on as_md/as_txt. If filename is None, function name is used.""" if as_md & as_txt: raise Exception('Only one of as_md / as_txt may be true.') import sys from io import StringIO ## save original output to restore orig_output = sys.stdout ## instantiate io stream to capture output io_out = StringIO() ## Redirect output to output stream sys.stdout = io_out if save_code: print('### SOURCE:') help_md = get_source_code_markdown(function) ## print output to io_stream print(help_md) if save_help: print('### HELP:') help(function) ## Get printed text from io stream text_to_save = io_out.getvalue() ## MAKE FULL FILENAME if filename is None: ## Find the name of the function import re func_names_exp = re.compile(r'def (\w*)\(') func_name = func_names_exp.findall(text_to_save)[0] print(f'Found code for {func_name}') save_filename = folder+func_name#+'.txt' else: save_filename = folder+filename if as_md: ext = '.md' elif as_txt: ext='.txt' full_filename = save_filename + ext with open(full_filename,file_mode) as f: f.write(text_to_save) print(f'Output saved as {full_filename}') sys.stdout = orig_output
def test_solve11 (self) : r = StringIO("1 10\n100 200\n201 210\n900 1000\n") w = StringIO() collatz_solve(r, w) self.assertEqual(w.getvalue(), "1 10 20\n100 200 125\n201 210 89\n900 1000 174\n")
# Train Decision Tree Classifer clf = clf.fit(X_train, y_train) #Predict the response for test dataset y_pred = clf.predict(X_test) # Model Accuracy, how often is the classifier correct? print("Unpruned Accuracy:", metrics.accuracy_score(y_test, y_pred)) dot_data = StringIO() export_graphviz(clf, out_file=dot_data, filled=True, rounded=True, special_characters=True) graph = pydotplus.graph_from_dot_data(dot_data.getvalue()) graph.write_pdf("DT.pdf") # Create Decision Tree classifer object clfl2 = DecisionTreeClassifier(max_depth=2, criterion="entropy") # Train Decision Tree Classifer clfl2 = clfl2.fit(X_train, y_train) #Predict the response for test dataset y_pred = clfl2.predict(X_test) # Model Accuracy, how often is the classifier correct? print("Accuracy (2 layer):", metrics.accuracy_score(y_test, y_pred)) dot_data = StringIO()
def make_job(self, scene): imp.reload(export_cycles) imp.reload(crowdprocess) fp = StringIO() export_cycles.export_cycles(fp=fp, scene=scene, inline_textures=True) scene_xml = fp.getvalue() emcycles_core = open( path.join(path.dirname(__file__), 'emcycles/cloudrender_core.js')).read() pako = open(path.join(path.dirname(__file__), 'pako_deflate.js')).read() crowdprocess_func = ''' function Run(data) { var console = { log: function(){}, error: function(){}, warn: function(){}, time: function(){}, timeEnd: function(){}, assert: function(){} }; var Module = { print: function () {}, tileX: data.x, tileY: data.y, tileH: data.h, tileW: data.w } var SCENE = %s; // scene_xml var INCLUDES = []; ;%s; // emcycles data.tile = Module.imageData ;%s; // pako ;%s; // base64 var deflated = pako.deflate(data.tile) if (deflated.length < data.tile.length) { data.tile = deflated data.deflated = true } data.tile = base64(data.tile) return data } ''' % (json.dumps(scene_xml), emcycles_core, pako, base_64_enc) # open('/tmp/wow.js', 'w').write(crowdprocess_func) crp = crowdprocess.CrowdProcess(scene.ore_render.username, scene.ore_render.password) self.job = crp.job(crowdprocess_func)
def __str__(self): buf = StringIO() self.save(buf) return buf.getvalue()
def ihelp_menu(function_list,box_style='warning', to_embed=False):#, to_file=False):#, json_file='ihelp_output.txt' ): """ Creates a widget menu of the source code and and help documentation of the functions in function_list. Args: function_list (list): list of function object or string names of loaded function. to_embed (bool, optional): Returns interface (layout,output) if True. Defaults to False. to_file (bool, optional): Save . Defaults to False. json_file (str, optional): [description]. Defaults to 'ihelp_output.txt'. Returns: full_layout (ipywidgets GridBox): Layout of interface. output () """ # Accepts a list of string names for loaded modules/functions to save the `help` output and # inspect.getsource() outputs to dictionary for later reference and display ## One way using sys to write txt file import pandas as pd import sys import inspect from io import StringIO from IPython.display import display,Markdown notebook_output = sys.stdout result = StringIO() sys.stdout=result ## Turn single input into a list if isinstance(function_list,list)==False: function_list = [function_list] ## Make a dictionary of{function_name : function_object} functions_dict = dict() for fun in function_list: ## if input is a string, save string as key, and eval(function) as value if isinstance(fun, str): functions_dict[fun] = eval(fun) ## if input is a function, get the name of function using inspect and make key, function as value elif inspect.isfunction(fun): members= inspect.getmembers(fun) member_df = pd.DataFrame(members,columns=['param','values']).set_index('param') fun_name = member_df.loc['__name__'].values[0] functions_dict[fun_name] = fun ## Create an output dict to store results for functions output_dict = {} for fun_name, real_func in functions_dict.items(): output_dict[fun_name] = {} ## First save help help(real_func) output_dict[fun_name]['help'] = result.getvalue() ## Clear contents of io stream result.truncate(0) try: ## Next save source source_DF = inspect.getsource(real_func) # # if markdown == True: # output = "```python" +'\n'+source_DF+'\n'+"```" # display(Markdown(output)) # else: # output=source_DF print(source_DF) # output_dict[fun_name]['source'] = source_DF # print(inspect.getsource(real_func)) #eval(fun)))###f"{eval(fun)}")) except: # print("Source code for object was not found") print("Source code for object was not found") # finally: output_dict[fun_name]['source'] = result.getvalue() ## clear contents of io stream result.truncate(0) ## Get file location try: file_loc = inspect.getfile(real_func) print(file_loc) except: print("File location not found") output_dict[fun_name]['file_location'] =result.getvalue() ## clear contents of io stream result.truncate(0) ## Reset display back to notebook sys.stdout = notebook_output # if to_file==True: # with open(json_file,'w') as f: # import json # json.dump(output_dict,f) ## CREATE INTERACTIVE MENU from ipywidgets import interact, interactive, interactive_output import ipywidgets as widgets from IPython.display import display # from functions_combined_BEST import ihelp # import functions_combined_BEST as ji ## Check boxes check_help = widgets.Checkbox(description="Show 'help(func)'",value=True) check_source = widgets.Checkbox(description="Show source code",value=True) check_fileloc=widgets.Checkbox(description="Show source filepath",value=False) check_boxes = widgets.HBox(children=[check_help,check_source,check_fileloc]) ## dropdown menu (dropdown, label, button) dropdown = widgets.Dropdown(options=list(output_dict.keys())) label = widgets.Label('Function Menu') button = widgets.ToggleButton(description='Show/hide',value=False) ## Putting it all together title = widgets.Label('iHelp Menu: View Help and/or Source Code') menu = widgets.HBox(children=[label,dropdown,button]) titled_menu = widgets.VBox(children=[title,menu]) full_layout = widgets.GridBox(children=[titled_menu,check_boxes],box_style=box_style) ## Define output manager # show_output = widgets.Output() def dropdown_event(change): new_key = change.new output_display = output_dict[new_key] dropdown.observe(dropdown_event,names='values') def show_ihelp(display_help=button.value,function=dropdown.value, show_help=check_help.value,show_code=check_source.value, show_file=check_fileloc.value,ouput_dict=output_dict): from IPython.display import Markdown # import functions_combined_BEST as ji from IPython.display import display page_header = '---'*28 # import json # with open(json_file,'r') as f: # output_dict = json.load(f) func_dict = output_dict[function] source_code=None if display_help: if show_help: # display(print(func_dict['help'])) print(page_header) banner = ''.join(["---"*2,' HELP ',"---"*24,'\n']) print(banner) print(func_dict['help']) if show_code: print(page_header) banner = ''.join(["---"*2,' SOURCE -',"---"*23]) print(banner) source_code = func_dict['source']#.encode('utf-8') if source_code.startswith('`'): source_code = source_code.replace('`',"").encode('utf-8') if 'google.colab' in sys.modules: print(source_code) else: md_source = "```python\n"+source_code md_source += "```" display(Markdown(md_source)) if show_file: print(page_header) banner = ''.join(["---"*2,' FILE LOCATION ',"---"*21]) print(banner) file_loc = func_dict['file_location'] print(file_loc) if show_help==False & show_code==False & show_file==False: display('Check at least one "Show" checkbox for output.') else: display('Press "Show/hide" for display') ## Fully integrated output output = widgets.interactive_output(show_ihelp,{'display_help':button, 'function':dropdown, 'show_help':check_help, 'show_code':check_source, 'show_file':check_fileloc}) if to_embed: return full_layout, output else: display(full_layout, output)
def test_main(self): capturedOutput = StringIO() sys.stdout = capturedOutput QP_main() sys.stdout = sys.__stdout__ self.assertIn("-59231", capturedOutput.getvalue())
class UMCmdTest(unittest.TestCase): def setUp(self): self.output = StringIO() sys.stdout = self.output v = um_view.UMView() d = data_util.DataUtil() a = analysis_util.AnalysisUtil() c = um_controller.UMController(v, d, a) self.cmd = um_cmd.UMCmd(c) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_search_1(self): self.cmd.do_search('blueprints') expected = 'Displaying the first 10 assets (max 10):' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_search_2(self): self.cmd.do_search('fire effects') expected = 'Displaying the first 6 assets (max 10):' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_category_1(self): self.cmd.do_category('blueprints') expected = 'Displaying the first 10 assets (max 10):' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_category_2(self): self.cmd.do_category('fire effects') expected = 'The category "fire effects" was not found' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_results_1(self): self.cmd.do_results(1) self.cmd.do_search('blueprints') expected = 'Displaying the first 1 assets (max 1):' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_view_asset_image_1(self): self.cmd.do_search('fire effects') self.reset_output() self.cmd.do_view_asset_image(1) expected = '' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_view_asset_image_2(self): self.cmd.do_search('fire effects') self.reset_output() self.cmd.do_view_asset_image(99) expected = 'Value must be within the previous search results' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_view_asset_image_3(self): self.cmd.do_view_asset_image(1) expected = 'You must run a query first' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_analyse_results_1(self): self.cmd.do_search('fire effects') self.cmd.do_analyse_results('') expected = '' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_analyse_results_2(self): self.cmd.do_analyse_results('') expected = 'You must run a query first' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_save_1(self): self.cmd.do_search('fire effects') self.cmd.do_save('') expected = '' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_save_2(self): self.cmd.do_save('') expected = 'No results to save' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) # @unittest.skip("Skipping test to speed up execution while writing tests") def test_load_1(self): self.cmd.do_search('fire effects') self.cmd.do_save('') self.reset_output() self.cmd.do_load('') expected = 'Displaying the first 6 assets (max 6):' actual = self.output.getvalue().strip()[:len(expected)] self.assertEqual(actual, expected) def reset_output(self): """Clears the output buffer""" self.output.seek(0) self.output.truncate(0)
class RepoSyncTest(unittest.TestCase): def setUp(self): self.reposync = spacewalk.satellite_tools.reposync # kill logging self.reposync.rhnLog.initLOG = Mock() # catching stdout # this could be assertRaisesRegexp in python>=2.7. just sayin' self.saved_stdout = sys.stdout self.stdout = StringIO() sys.stdout = self.stdout # catching stderr self.saved_stderr = sys.stderr self.stderr = StringIO() sys.stderr = self.stderr self.reposync.os = Mock() self.reposync.rhnSQL.initDB = Mock() self.reposync.rhnSQL.commit = Mock() _mock_rhnsql( self.reposync, [ [{'id': 'id1', 'repo_label': 'label1', 'source_url': 'http://url.one', 'metadata_signed': 'Y', 'repo_type': 'yum'}], [{'id': 'id2', 'repo_label': 'label2', 'source_url': 'http://url.two', 'metadata_signed': 'N', 'repo_type': 'yum'}], ] ) def tearDown(self): self.stdout.close() sys.stdout = self.saved_stdout self.stderr.close() sys.stderr = self.saved_stderr imp.reload(spacewalk.satellite_tools.reposync) def test_init_succeeds_with_correct_attributes(self): rs = _init_reposync(self.reposync, 'Label', RTYPE) self.assertEqual(rs.channel_label, 'Label') # these should have been set automatically self.assertEqual(rs.fail, False) self.assertEqual(rs.interactive, True) def test_init_with_custom_url(self): rs = _init_reposync(self.reposync, 'Label', RTYPE, url='http://example.com') self.assertEqual(rs.urls, [{'source_url': 'http://example.com', 'repo_label': None, 'id': None, 'metadata_signed': 'N', 'repo_type': 'yum' }]) def test_init_with_custom_flags(self): rs = _init_reposync(self.reposync, 'Label', RTYPE, fail=True, noninteractive=True) self.assertEqual(rs.fail, True) self.assertEqual(rs.interactive, False) def test_init_wrong_url(self): """Test generates empty metadata via taskomatic and quits""" # the channel shouldn't be found in the database _mock_rhnsql(self.reposync, False) self.reposync.taskomatic.add_to_repodata_queue_for_channel_package_subscription = Mock() channel = {'org_id':1, 'id':1, 'arch': 'arch1'} self.reposync.RepoSync.load_channel = Mock(return_value=channel) self.assertRaises(SystemExit, self.reposync.RepoSync, 'WrongLabel', RTYPE) self.assertTrue(self.reposync.taskomatic. add_to_repodata_queue_for_channel_package_subscription. called) def test_init_rhnlog(self): """Init rhnLog successfully""" rs = _init_reposync(self.reposync, 'Label', RTYPE) self.assertTrue(self.reposync.rhnLog.initLOG.called) def test_init_channel(self): self.reposync.rhnChannel.channel_info = Mock(return_value= {'name': 'mocked Channel', 'id': 1, 'org_id': 1}) self.reposync.RepoSync.get_compatible_arches = Mock(return_value=['arch1', 'arch2']) self.reposync.RepoSync.get_channel_arch = Mock('arch1') rs = self.reposync.RepoSync('Label', RTYPE) self.assertEqual(rs.channel, {'name': 'mocked Channel', 'id': 1, 'org_id': 1}) def test_init_bad_channel(self): self.reposync.rhnChannel.channel_info = Mock(return_value=None) self.assertRaises(SystemExit, self.reposync.RepoSync, 'Label', RTYPE) def test_bad_repo_type(self): rs = _init_reposync(self.reposync, 'Label', RTYPE) self.assertRaises(SystemExit, rs.load_plugin, 'bad-repo-type') self.assertIn("Repository type bad-repo-type is not supported. " "Could not import " "spacewalk.satellite_tools." "repo_plugins.bad-repo-type_src.\n", self.stderr.getvalue()) def test_sync_success_no_regen(self): rs = _init_reposync(self.reposync) rs.urls = [ {"source_url": ["http://none.host/bogus-url"], "id": 42, "metadata_signed": "N", "repo_label": None, 'repo_type': 'yum'}] _mock_rhnsql(self.reposync, None) rs = _mock_sync(self.reposync, rs) rs.sync() self.assertEqual(rs.repo_plugin.call_args[0], (('http://none.host/bogus-url', 'bogus-url', True, True))) self.assertEqual(rs.import_packages.call_args, ((rs.mocked_plugin, 42, "http://none.host/bogus-url", 1), {})) self.assertEqual(rs.import_updates.call_args, ((rs.mocked_plugin,), {})) self.assertEqual(rs.import_products.call_args, ((rs.mocked_plugin,), {})) # for the rest just check if they were called or not self.assertTrue(rs.update_date.called) # these aren't supposed to be called unless self.regen is True self.assertFalse(self.reposync.taskomatic.add_to_repodata_queue_for_channel_package_subscription.called) self.assertFalse(self.reposync.taskomatic.add_to_erratacache_queue.called) def test_sync_success_regen(self): rs = _init_reposync(self.reposync) rs.urls = [{"source_url": ["http://none.host/bogus-url"], "id": 42, "metadata_signed": "N", "repo_label": None, 'repo_type': 'yum'}] _mock_rhnsql(self.reposync, {}) rs = _mock_sync(self.reposync, rs) rs.regen = True rs.sync() self.assertEqual(self.reposync.taskomatic.add_to_repodata_queue_for_channel_package_subscription.call_args, ((["Label"], [], "server.app.yumreposync"), {})) self.assertEqual(self.reposync.taskomatic.add_to_erratacache_queue.call_args, (("Label", ), {})) def test_sync_raises_channel_timeout(self): rs = self._create_mocked_reposync() exception = self.reposync.ChannelTimeoutException("anony-error") rs.load_plugin = Mock(return_value=Mock(side_effect=exception)) rs.sendErrorMail = Mock() etime, ret = rs.sync() self.assertEqual(-1, ret) self.assertEqual(rs.sendErrorMail.call_args, (("anony-error", ), {})) self.assertEqual(self.reposync.log.call_args[0][1], exception) def test_sync_raises_unexpected_error(self): rs = self._create_mocked_reposync() rs.load_plugin = Mock(return_value=Mock(side_effect=TypeError)) rs.sendErrorMail = Mock() etime, ret = rs.sync() self.assertEqual(-1, ret) error_string = self.reposync.log.call_args[0][1] assert (error_string.startswith('Traceback') and 'TypeError' in error_string), ( "The error string does not contain the keywords " "'Traceback' and 'TypeError':\n %s\n---end of assert" % error_string) def test_update_bugs(self): notice = {'references': [{'type': 'bugzilla', 'id': '12345', 'title': 'title1', 'href': 'href1'}, {'type': 'bugzilla', 'id': 'string_id', 'title': 'title2', 'href': 'href2', 'this': 'non-integer bz ids should be skipped'}, {'type': 'bugzilla', 'id': 'string_id', 'title': 'title3', 'href': 'http://dummyhost/show_bug.cgi?id=11111', 'this': 'bz id parsed from href'}, {'type': 'bugzilla', 'id': '54321', 'title': 'title2', 'href': 'href2'}, {'type': 'bugzilla', 'id': '54321', 'title': 'duplicate_id', 'href': 'duplicate_id'}, {'type': 'godzilla', 'this': 'should be skipped'}]} bugs = self.reposync.RepoSync._update_bugs(notice) bug_values = [set(['12345', 'title1', 'href1']), set(['54321', 'title2', 'href2']), set(['11111', 'title3', 'http://dummyhost/show_bug.cgi?id=11111'])] self.assertEqual(len(bugs), 3) for bug in bugs: self.assertCountEqual(list(bug.keys()), ['bug_id', 'href', 'summary']) assert set(bug.values()) in bug_values, ( "Bug set(%s) not in %s" % (list(bug.values()), bug_values)) def test_update_cves(self): notice = {'references': [{'type': 'cve', 'id': "CVE-1234-5678"}, {'type': 'cve', 'id': "CVE-1234-123456"}, {'type': 'cve', 'id': "CVE-1234-5678"}, {'type': 'this should be skipped'}], 'description': None} cves = self.reposync.RepoSync._update_cve(notice) self.assertCountEqual(cves, ["CVE-1234-5678", "CVE-1234-123456"]) def test_update_cves_with_description(self): notice = {'references': [{'type': 'cve', 'id': "CVE-1234-5678"}, {'type': 'cve', 'id': "CVE-1234-1234"}, {'type': 'cve', 'id': "CVE-1234-5678"}, {'type': 'this should be skipped'}], 'description': 'This is a text with two CVE numbers CVE-1234-5678, CVE-1234-567901'} cves = self.reposync.RepoSync._update_cve(notice) self.assertCountEqual(cves, ["CVE-1234-567901", "CVE-1234-5678", "CVE-1234-1234"]) def test_update_keywords_reboot(self): notice = {'reboot_suggested': True, 'restart_suggested': False} keyword = self.reposync.importLib.Keyword() keyword.populate({'keyword': 'reboot_suggested'}) self.assertEqual(self.reposync.RepoSync._update_keywords(notice), [keyword]) def test_update_keywords_restart(self): notice = {'reboot_suggested': False, 'restart_suggested': True} keyword = self.reposync.importLib.Keyword() keyword.populate({'keyword': 'restart_suggested'}) self.assertEqual(self.reposync.RepoSync._update_keywords(notice), [keyword]) def test_update_keywords_restart_and_reboot(self): notice = {'reboot_suggested': True, 'restart_suggested': True} keyword_restart = self.reposync.importLib.Keyword() keyword_restart.populate({'keyword': 'restart_suggested'}) keyword_reboot = self.reposync.importLib.Keyword() keyword_reboot.populate({'keyword': 'reboot_suggested'}) self.assertEqual(self.reposync.RepoSync._update_keywords(notice), [keyword_reboot, keyword_restart]) def test_update_keywords_both_false(self): notice = {'reboot_suggested': False, 'restart_suggested': False} self.assertEqual(self.reposync.RepoSync._update_keywords(notice), []) @patch("uyuni.common.context_managers.initCFG", Mock()) def test_send_error_mail(self): rs = self._create_mocked_reposync() self.reposync.rhnMail.send = Mock() self.reposync.hostname = 'testhost' CFG = Mock() CFG.TRACEBACK_MAIL = 'recipient' with patch("uyuni.common.context_managers.CFG", CFG): rs.sendErrorMail('email body') self.assertEqual(self.reposync.rhnMail.send.call_args, ( ({'To': 'recipient', 'From': 'testhost <recipient>', 'Subject': "SUSE Manager repository sync failed (testhost)"}, "Syncing Channel 'Label' failed:\n\nemail body"), {})) def test_updates_process_packages_simple(self): rs = self._create_mocked_reposync() packages = [{'name': 'n1', 'version': 'v1', 'release': 'r1', 'arch': 'arch1', 'channel_label': 'l1', 'epoch': []}, {'name': 'n2', 'version': 'v2', 'release': 'r2', 'arch': 'arch2', 'channel_label': 'l2', 'epoch': 'e2'}] checksum = {'epoch': None, 'checksum_type': None, 'checksum': None, 'id': None} _mock_rhnsql(self.reposync, checksum) processed = rs._updates_process_packages(packages, 'a name', []) for p in processed: self.assertTrue(isinstance(p, self.reposync.importLib.IncompletePackage)) def test_updates_process_packages_returns_the_right_values(self): rs = self._create_mocked_reposync() packages = [{'name': 'n1', 'version': 'v1', 'release': 'r1', 'arch': 'arch1', 'epoch': []}, {'name': 'n2', 'version': 'v2', 'release': 'r2', 'arch': 'arch2', 'epoch': 'e2'}] checksum = {'epoch': 'cs_epoch', 'checksum_type': 'md5', 'checksum': '12345', 'id': 'cs_package_id'} _mock_rhnsql(self.reposync, checksum) processed = rs._updates_process_packages(packages, 'patchy', []) p1 = self.reposync.importLib.IncompletePackage() p1.populate({'package_size': None, 'name': 'n1', 'checksum_list': None, 'md5sum': None, 'org_id': 1, 'epoch': 'cs_epoch', 'channels': None, 'package_id': 'cs_package_id', 'last_modified': None, 'version': 'v1', 'checksum_type': 'md5', 'release': 'r1', 'checksums': {'md5': '12345'}, 'checksum': '12345', 'arch': 'arch1'}) p2 = self.reposync.importLib.IncompletePackage() p2.populate({'package_size': None, 'name': 'n2', 'checksum_list': None, 'md5sum': None, 'org_id': 1, 'epoch': 'cs_epoch', 'channels': None, 'package_id': 'cs_package_id', 'last_modified': None, 'version': 'v2', 'checksum_type': 'md5', 'release': 'r2', 'checksums': {'md5': '12345'}, 'checksum': '12345', 'arch': 'arch2'}) fixtures = [p1, p2] for pkg, fix in zip(processed, fixtures): self.assertEqual(pkg, fix) def test_updates_process_packages_checksum_not_found(self): rs = self._create_mocked_reposync() packages = [{'name': 'n2', 'version': 'v2', 'release': 'r2', 'arch': 'arch2', 'channel_label': 'l2', 'epoch': 'e2'}] ident = "%(name)s-%(epoch)s:%(version)s-%(release)s.%(arch)s" % packages[0] rs.available_packages[ident] = 1 _mock_rhnsql(self.reposync, []) self.assertEqual(rs._updates_process_packages(packages, 'patchy', []), []) self.assertEqual(self.reposync.log.call_args[0][1], "The package n2-e2:v2-r2.arch2 " "which is referenced by patch patchy was not found " "in the database. This patch has been skipped.") def test_updates_process_packages_checksum_not_found_no_epoch(self): rs = self._create_mocked_reposync() packages = [{'name': 'n1', 'version': 'v1', 'release': 'r1', 'arch': 'arch1', 'channel_label': 'l1', 'epoch': '' }] ident = "%(name)s-%(epoch)s%(version)s-%(release)s.%(arch)s" % packages[0] rs.available_packages[ident] = 1 _mock_rhnsql(self.reposync, []) self.assertEqual(rs._updates_process_packages(packages, 'patchy', []), []) self.assertEqual(self.reposync.log.call_args[0][1], "The package n1-v1-r1.arch1 " "which is referenced by patch patchy was not found " "in the database. This patch has been skipped.") def test_updates_process_packages_checksum_not_found_but_not_available(self): rs = self._create_mocked_reposync() packages = [{'name': 'n1', 'version': 'v1', 'release': 'r1', 'arch': 'arch1', 'channel_label': 'l1', 'epoch': '' }] _mock_rhnsql(self.reposync, []) self.assertEqual(rs._updates_process_packages(packages, 'patchy', []), []) self.assertEqual(self.reposync.log.call_args, None) # RedHat has errata with empty package list # they removed the check - therefor this is disabled too #def test_upload_updates_referenced_package_not_found(self): # timestamp1 = datetime.now().isoformat(' ') # notices = [{'from': 'from1', # 'update_id': 'update_id1', # 'version': 'version1', # 'type': 'security', # 'severity': 'Low', # 'release': 'release1', # 'description': 'description1', # 'title': 'title1', # 'issued': timestamp1, # we mock _to_db_date anyway # 'updated': timestamp1, # 'pkglist': [{'packages': []}], # 'reboot_suggested': False, # 'restart_suggested': False, # 'references': None, # }] # self.reposync._to_db_date = Mock(return_value=timestamp1) # # no packages related to this errata makes the ErrataImport be called # # with an empty list # self.reposync.RepoSync._updates_process_packages = Mock(return_value=[]) # self.reposync.get_errata = Mock(return_value=None) # mocked_backend = Mock() # self.reposync.SQLBackend = Mock(return_value=mocked_backend) # self.reposync.ErrataImport = Mock() # rs = self._create_mocked_reposync() # rs._patch_naming = Mock(return_value='package-name') # rs.upload_updates(notices) # self.assertEqual(self.reposync.ErrataImport.call_args, None) def test_associate_package(self): pack = ContentPackage() pack.setNVREA('name1', 'version1', 'release1', 'epoch1', 'arch1') pack.unique_id = 1 pack.a_pkg = rhn_rpm.RPM_Package(None) pack.a_pkg.checksum = 'checksum1' pack.a_pkg.checksum_type = 'c_type1' pack.a_pkg.header = {'epoch': 'epoch1'} pack.checksums[1] = 'checksum1' mocked_backend = Mock() self.reposync.SQLBackend = Mock(return_value=mocked_backend) rs = self._create_mocked_reposync() rs._importer_run = Mock() rs.channel_label = 'Label1' rs.channel = {'id': 'channel1', 'org_id': 1} package = {'name': 'name1', 'version': 'version1', 'release': 'release1', 'epoch': 'epoch1', 'arch': 'arch1', 'checksum': 'checksum1', 'checksum_type': 'c_type1', 'org_id': 1, 'channels': [{'label': 'Label1', 'id': 'channel1'}]} refpack = importLib.IncompletePackage().populate(package) ipack = rs.associate_package(pack) self.assertEqual(ipack, refpack) def test_get_errata_no_advisories_found(self): rs = self._create_mocked_reposync() _mock_rhnsql(self.reposync, None) self.assertEqual(rs.get_errata('bogus'), None) def test_get_errata_advisories_but_no_channels(self): rs = self._create_mocked_reposync() _mock_rhnsql(self.reposync, [{'id': 42}, []]) self.assertEqual(rs.get_errata('bogus'), {'channels': [], 'id': 42, 'packages': []}) def test_get_errata_success(self): rs = self._create_mocked_reposync() _mock_rhnsql(self.reposync, [{'id': 42}, ['channel1', 'channel2']]) self.assertEqual(rs.get_errata('bogus'), {'id': 42, 'channels': ['channel1', 'channel2'], 'packages': []}) def test_get_compat_arches(self): _mock_rhnsql(self.reposync, ({'label': 'a1'}, {'label':'a2'})) self.assertEqual(self.reposync.RepoSync.get_compatible_arches(None), ['a1', 'a2']) def test_get_channel_arch(self): _mock_rhnsql(self.reposync, {'label': 'channel-amd64-deb'}) self.assertEqual(self.reposync.RepoSync.get_channel_arch(None), 'amd64') def test_set_repo_credentials_no_credentials(self): url = {'source_url': "http://example.com"} rs = self._create_mocked_reposync() rs.set_repo_credentials(url) self.assertEqual(url['source_url'], "http://example.com") def test_set_repo_credentials_old_default_credentials_bad(self): url = { "source_url": [ "http://example.com/?credentials=testcreds" ] } rs = self._create_mocked_reposync() self.assertRaises(SystemExit, rs.set_repo_credentials, url) def test_set_repo_credentials_bad_credentials(self): rs = _init_reposync(self.reposync) rs.error_msg = Mock() url = { "source_url": [ "http://example.com/?credentials=bad_creds_with_underscore" ] } self.assertRaises(SystemExit, rs.set_repo_credentials, url) def test_set_repo_credentials_number_credentials(self): rs = self._create_mocked_reposync() url = { "source_url": [ "http://example.com/?credentials=testcreds_42" ] } _mock_rhnsql(self.reposync, [{ 'username' : 'foo', 'password': '******' , 'extra_auth': memoryview(b'{\"my_header\": \"my_value\"}')}]) self.assertEqual( rs.set_repo_credentials(url), [{"url":"http://*****:*****@example.com/", "http_headers": {"my_header": "my_value"}}]) def test_is_old_style(self): """ Test for _is_old_suse_style """ notice = {'from': '*****@*****.**', 'version': '1111', 'update_id': 'sles-kernel-default'} self.assertTrue(self.reposync.RepoSync._is_old_suse_style(notice)) notice = {'from': '*****@*****.**', 'version': '7', 'update_id': 'res5ct-kernel-default'} self.assertTrue(self.reposync.RepoSync._is_old_suse_style(notice)) notice = {'from': '*****@*****.**', 'version': '1', 'update_id': 'sles-kernel-default'} self.assertFalse(self.reposync.RepoSync._is_old_suse_style(notice)) notice = {'from': '*****@*****.**', 'version': '6', 'update_id': 'res5ct-kernel-default'} self.assertFalse(self.reposync.RepoSync._is_old_suse_style(notice)) def test_to_db_date(self): """ Test for _to_db_date """ # Unsure datetime.fromtimestamp is always returning UTC times class DateTimeMock(datetime): @classmethod def fromtimestamp(cls, timestamp): return cls.utcfromtimestamp(timestamp) os.environ['TZ'] = 'UTC' time.tzset() with patch("spacewalk.satellite_tools.reposync.datetime", DateTimeMock): self.assertEqual(self.reposync.RepoSync._to_db_date('2015-01-02 01:02:03'), '2015-01-02 01:02:03') self.assertEqual(self.reposync.RepoSync._to_db_date('1420160523'), '2015-01-02 01:02:03') self.assertEqual(self.reposync.RepoSync._to_db_date('2015-01-02'), '2015-01-02 00:00:00') self.assertEqual(self.reposync.RepoSync._to_db_date('2015-09-02 13:39:49 UTC'), '2015-09-02 13:39:49') self.assertEqual(self.reposync.RepoSync._to_db_date('2015-01-02T02:02:03+0100'), '2015-01-02 01:02:03') self.assertRaises(ValueError, self.reposync.RepoSync._to_db_date, '2015-01-02T01:02:03+nonsense') def _create_mocked_reposync(self): """Create a fully mocked RepoSync""" rs = _init_reposync(self.reposync) rs.urls = [{'id': None, "source_url": ["http://none.host/bogus-url"], "metadata_signed": "N", "repo_label": None, 'repo_type': 'yum'}] rs = _mock_sync(self.reposync, rs) return rs
def __str__(self, print_doc=True): sio = StringIO() self.config_print(buf=sio, print_doc=print_doc) return sio.getvalue()
from io import StringIO sIO = StringIO('Hello\nWorld\nMartian') print(sIO.getvalue()) while True: s = sIO.readline() if s == '': break print(s.strip())
df_train = dd.read_csv(pathToFile) df_train = df_train.compute() tfs = pd.read_csv('src/tissue_tfs.csv', header=None) genes = pd.read_csv('src/tissue_genes.csv', header=None) tfs.columns = ['', ''] genes.columns = ['', ''] for sampleid in range(1, nSamples[k] + 1): sampleNet = pd.DataFrame( index=tfs.iloc[1:, 1], columns=genes.iloc[:, 1], data=df_train.iloc[:, sampleid].values.reshape((644, 30243))) fileName = tissuesCase[k] + '_sample' + '_' + df_train.columns[ sampleid] + '.csv' bucket = 'granddb' # already created on S3 csv_buffer = StringIO() sampleNet.to_csv(csv_buffer) s3_resource = boto3.resource('s3') res = s3_resource.Object( bucket, 'tissues/networks/lioness/singleSample/' + fileName).put(Body=csv_buffer.getvalue()) if res['ResponseMetadata']['HTTPStatusCode'] == 200: os.system( 'aws s3api put-object-acl --bucket granddb --key tissues/networks/lioness/singleSample/' + fileName + ' --acl public-read') resURL = 'https://granddb.s3.amazonaws.com/tissues/networks/lioness/singleSample/' + fileName print(resURL) else: print('error') os.system('rm ' + pathToFile) k = k + 1