def test_logger_memory(self): output = StringIO() logger = Logger.get('memory', output, OutputStream, level=logging.INFO, memory=True, memory_capacity=2, memory_flushlevel=logging.CRITICAL) # under max capacity logger.info('1') output.seek(0) lines = len(output.readlines()) self.assertEqual(lines, 0) # max capacity logger.info('2') output.seek(0) lines = len(output.readlines()) self.assertEqual(lines, 2) # under max capacity but flush level logger.critical('3') output.seek(0) lines = len(output.readlines()) self.assertEqual(lines, 3)
def compare_configs(self, f1, f2): """ Compare two configs and output diffs :param f1: file or str of first config :param f2: file or str of second config :return: """ self.log.info("Comparing configs ...") added = "" removed = "" same = True try: stream1 = open(f1, "r") stream2 = open(f2, "r") except: from io import StringIO stream1 = StringIO(f1) stream2 = StringIO(f2) stream1 = [line.lstrip() for line in stream1.readlines()[: -1]] stream2 = [line.lstrip() for line in stream2.readlines()[: -1]] diff = difflib.ndiff(stream1, stream2) for line in diff: if line.startswith('+') and not line.startswith(('+ # Generated', '+ # Finished', '+ !!')): added = added + line[2:] same = False elif line.startswith('-') and not line.startswith(('- # Generated', '- # Finished', '- !!')): removed = removed + line[2:] same = False if same: self.log.info("Pre and Post device configs are identical!") return [same, added, removed]
def check_source(nb, filename): """ Run pyflakes on a notebook, wil catch errors such as missing passed parameters that do not have default values """ from pyflakes.api import check as pyflakes_check from pyflakes.reporter import Reporter # concatenate all cell's source code in a single string source = '\n'.join([c['source'] for c in nb.cells]) # this objects are needed to capture pyflakes output warn = StringIO() err = StringIO() reporter = Reporter(warn, err) # run pyflakes.api.check on the source code pyflakes_check(source, filename=filename, reporter=reporter) warn.seek(0) err.seek(0) # return any error messages returned by pyflakes return { 'warnings': '\n'.join(warn.readlines()), 'errors': '\n'.join(err.readlines()) }
def stringio_write(): # 将函数返回的数据在内存中写 sio = StringIO() sio.write(abc) # 可以用StringIO本身的方法查看 s = sio.getvalue() print(s) print("-----") # 如果你用file-like object的方法查看的时候,你会发现数据为空 sio = StringIO() sio.write(s) print(sio.tell()) for i in sio.readlines(): print(i.strip()) print("修改指针位置:") # 这时候我们需要修改下文件的指针位置 # 我们发现可以打印出内容了 sio = StringIO() sio.write(s) # sio是一个追加行为 sio.write("aaaaaaa") sio.seek(0, 0) print(sio.tell()) for i in sio.readlines(): print(i.strip())
def test_split(self): # Arrange data = """Test line 1. 1 19938376 Test line 2. 2 19938376 Test line 3. 2 19938376 Test line 4. 1 19938376 """ expected_lines = StringIO(data).readlines() sut = BC3ASTPreprocess() output_train = StringIO() output_test = StringIO() # Act sut.split(StringIO(data), outfile_handle_1=output_train, outfile_handle_2=output_test, split=.5) # Assert output_train.seek(0) output_test.seek(0) train_lines = output_train.readlines() test_lines = output_test.readlines() self.assertEqual(2, len(train_lines)) self.assertSequenceEqual(expected_lines, sorted(train_lines + test_lines))
def grade_string(expected, actual): """Grades a single response against the true (actual) output. Possible return values are: * MATCH_EXACT (perfect match) * MATCH_LINES (correct number of lines, non-formatting characters match) * MATCH_VALUES (non-formatting characters match) * MATCH_NONE (no match)""" # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return MATCH_EXACT format_chars = ['[', ']', ',', ' ', '\n', '"', '\''] table = dict.fromkeys(map(ord, format_chars), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate( table).lower()): partial_match = True else: partial_match = False break if exact_match: return MATCH_EXACT if not removed_blanks else MATCH_LINES elif partial_match: return MATCH_LINES # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate( table).lower(): return MATCH_VALUES return MATCH_NONE
def grade_string(expected, actual): """Grades a single response against the true (actual) output. Possible return values are: * MATCH_EXACT (perfect match) * MATCH_LINES (correct number of lines, non-formatting characters match) * MATCH_VALUES (non-formatting characters match) * MATCH_NONE (no match)""" # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return MATCH_EXACT format_chars = ['[', ']', ',', ' ', '\n', '"', '\''] table = dict.fromkeys(map(ord, format_chars), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate(table).lower()): partial_match = True else: partial_match = False break if exact_match: return MATCH_EXACT if not removed_blanks else MATCH_LINES elif partial_match: return MATCH_LINES # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate(table).lower(): return MATCH_VALUES return MATCH_NONE
def grade_string(expected, actual): # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return "exact" table = dict.fromkeys(map(ord, FORMAT_CHARS), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate( table).lower()): partial_match = True else: partial_match = False break if exact_match: return "exact" if not removed_blanks else "lines" elif partial_match: return "lines" # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate( table).lower(): return "values" return None
def clean(self, text1, text2): """remove blank space and carriage returns, we may not care about these in the test results""" text1 = StringIO(text1) text2 = StringIO(text2) text1_clean = '' for line in text1.readlines(): text1_clean += line.strip() text2_clean = '' for line in text2.readlines(): text2_clean += line.strip() return text1_clean.replace("\n", ''), text2_clean.replace("\n", '')
def grade_string(expected, actual): # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return "exact" table = dict.fromkeys(map(ord, FORMAT_CHARS), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate(table).lower()): partial_match = True else: partial_match = False break if exact_match: return "exact" if not removed_blanks else "line" elif partial_match: return "line" # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate(table).lower(): return "values" return None
def do_POST(self): """Serve a POST request.""" r, info = self.deal_post_data() print(r, info, "by: ", self.client_address) f = StringIO() f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Upload Result Page</title>\n") f.write("<body>\n<h2>Upload Result Page</h2>\n") f.write("<hr>\n") if r: f.write("<strong>Success:</strong>") else: f.write("<strong>Failed:</strong>") f.write(info) f.write("<br><a href=\"%s\">back</a>" % self.headers['referer']) f.write("<hr><small>Powered By: bones7456, Ported By:0312birdzhang ") f.write("here</a>.</small></body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: for i in f.readlines(): #self.copyfile(f, self.wfile) self.wfile.write(i.encode("utf-8")) f.close()
def read(self, markdown_file, populator): process = False # makes an empty list where we are going to store robot code robot_lines = [] # opens our file under alias 'md_file' and operates all the following # statments on it with open(markdown_file) as md_file: # creates a boolean var include_line = False # for each line of the file that we passed as arguement to this script # do the steps below for line in md_file: if not include_line: include_line = line.strip().lower() == "```robotframework" elif line.strip() == "```": include_line = False else: robot_lines.append(line) robot_data = str(''.join(robot_lines)) print(robot_data) # txtfile = BytesIO(robot_data.encode('UTF-8')) a = StringIO(robot_data) for row in a.readlines(): row = self._process_row(row) cells = [self._process_cell(cell) for cell in self.split_row(row)] if cells and cells[0].strip().startswith('*') and \ populator.start_table([c.replace('*', '') for c in cells]): process = True elif process: populator.add(cells) populator.eof()
def text_to_document(text): io = StringIO(text) d = { "abstract": "" } sec = "abstract" ignore = False extra = [ "References", "External links", "Further reading", "See also" ] for line in io.readlines(): m = re.match(RE_09, line) if m != None: mg1 = m.group(1).strip() if mg1 in extra: ignore = True else: ignore = False sec = mg1 d[sec] = "" else: if ignore: pass else: d[sec] += line.lstrip("*") for key in list(d.keys()): s = condense_text(d[key]).replace("\n", " ").strip() if len(s) == 0: del d[key] else: d[key] = s return d
def test_writing(self) -> None: written = StringIO() write_respin(written, self.respin) written.seek(0) with open("tests/test_respin_format.respin") as f: self.assertListEqual(f.readlines(), written.readlines())
def testLAMMPS(): """Test potentials.writePotentials() for LAMMPS""" pota = potentials.Potential("A", "B", lambda x: x) potb = potentials.Potential("C", "D", lambda x: 6.0 - x) expect = [ "A-B", "N 6 R 1.00000000 6.00000000", "", "1 1.00000000 1.00000000 -1.00000000", "2 2.00000000 2.00000000 -1.00000000", "3 3.00000000 3.00000000 -1.00000000", "4 4.00000000 4.00000000 -1.00000000", "5 5.00000000 5.00000000 -1.00000000", "6 6.00000000 6.00000000 -1.00000000", "", "C-D", "N 6 R 1.00000000 6.00000000", "", "1 1.00000000 5.00000000 1.00000000", "2 2.00000000 4.00000000 1.00000000", "3 3.00000000 3.00000000 1.00000000", "4 4.00000000 2.00000000 1.00000000", "5 5.00000000 1.00000000 1.00000000", "6 6.00000000 0.00000000 1.00000000" ] sbuild = StringIO() potentials.writePotentials('LAMMPS', [pota, potb], 6.0, 6, sbuild) sbuild.seek(0) actual = sbuild.readlines() msg = "%s != %s" % (expect, actual) assert len(expect) == len(actual), msg for e, a in zip(expect, actual): a = a.decode() assert os.linesep == a[-1] a = a[:-1] assert e == a
def do_clean(number=0): """ Cleans old versions from folder """ fh = StringIO() my_names = [] number = int(number) if (number == 0): number = 1 revl = "ls -ltr versions | rev | cut -d ' ' -f1 | rev" value = local(revl, capture=True) for line in value.splitlines(): my_names.append(line) my_names.pop(0) for i in range(len(my_names) - number): local("rm -rf versions/{}".format(my_names[i])) # For remote server code = "ls -ltr /data/web_static/releases | rev | cut -d ' ' -f1 | rev" fk_value = sudo(code, stdout=fh) fh.seek(0) my_fk_names = [] for line in fh.readlines(): data = line.split()[-1] if data.startswith("web_static"): my_fk_names.append(data) for i in range(len(my_fk_names) - number): sudo("rm -rf /data/web_static/releases/{}".format(my_fk_names[i]))
def report_meta(): src_folder = PurePath("/tmp") filenames = """ /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_eq1000_ineq_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_eq100_ineq100_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_eq100_ineq100_minRRF0.1_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_eq100_ineq100_minRRF0_90m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq100_ineq100_minRRF0_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq100_ineq100_minRRF0_bval0.94_2m_sf.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq100_ineq100_minRRF0_bval0.94_480m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq10_ineq1000_minRRF0_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq10_ineq1000_minRRF0_bval0.94_180m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-3_eq10_ineq1000_minRRF0_bval0.94_2m_sf.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-40_eq100_ineq100_minRRF0_10m.zip /tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_m5-5_eq100_ineq100_minRRF0_10m.zip """ paths = StringIO(filenames) for path in paths.readlines(): filepath = path.strip() report_name = filepath.replace( '/tmp/CFM_hk_slipdef0_scaling_TMG_solution_TEST_Non0_', '')[:-4] yield (report_name, filepath)
def main(): robot_lines = [] with open(markdown_file) as f: include_line = False for line in f: if not include_line: include_line = line.strip().lower() == "```robotframework" elif line.strip() == "```": include_line = False else: robot_lines.append(line) robot_data = str(''.join(robot_lines)) print(robot_data) # txtfile = BytesIO(robot_data.encode('UTF-8')) a = StringIO(robot_data) for row in a.readlines(): row = self._process_row(row) cells = [self._process_cell(cell) for cell in self.split_row(row)] if cells and cells[0].strip().startswith('*') and \ populator.start_table([c.replace('*', '') for c in cells]): process = True elif process: populator.add(cells) populator.eof()
def main(): # Generates ventilator data fake = Faker() fakeIO = StringIO() fake_ventilator_records = generate_ventilator_records(fake) fakeIO.write(str(''.join(dumps_lines(fake_ventilator_records)))) # Write the json file to S3 s3key = s3_load_prefix + filename s3object = s3.Object(s3_bucket, s3key) s3object.put(Body=(bytes(fakeIO.getvalue().encode('utf-8')))) fakeIO.close() time.sleep(5) ventilator_object= s3.Object(s3_bucket, s3key) ventilator_decoded_data = ventilator_object.get()['Body'].read().decode('utf-8') ventilator_stringio_data = StringIO(ventilator_decoded_data) # Read data line by line data = ventilator_stringio_data.readlines() # Deserialize json data ventilator_json_data = list(map(json.loads, data)) # Load DDB table load_dynamodb_table(ventilator_json_data)
def _choices_as_array(self): from io import StringIO valuebuffer = StringIO(self.list_values) choices = [[item.strip(), item.strip()] for item in valuebuffer.readlines()] valuebuffer.close() return choices
def testLogStringStream(self): """Test case - context manager - to custom string stream""" try: myLen = self.__logRecordMax dataList = [i for i in range(1, myLen)] logger.debug("dataList %d: %r", len(dataList), dataList) # slogger = logging.getLogger() slogger.propagate = False for handler in slogger.handlers: slogger.removeHandler(handler) # stream = StringIO() sh = logging.StreamHandler(stream=stream) sh.setLevel(logging.DEBUG) fmt = logging.Formatter("STRING-%(processName)s: %(message)s") sh.setFormatter(fmt) slogger.addHandler(sh) # with MultiProcLogging(logger=slogger, fmt=self.__mpFormat, level=logging.DEBUG) as wlogger: for ii in range(myLen): wlogger.error("context logging record %d", ii) # stream.seek(0) logLines = stream.readlines() logger.debug(">> dataList %d: %r", len(logLines), logLines) self.assertEqual(len(logLines), myLen) for line in logLines: self.assertIn("context logging record", line) except Exception as e: logger.exception("context logging record %s", str(e)) self.fail()
def test_simple(self): f = StringIO() c = Commit() c.committer = c.author = "Jelmer <*****@*****.**>" c.commit_time = c.author_time = 1271350201 c.commit_timezone = c.author_timezone = 0 c.message = "This is the first line\nAnd this is the second line.\n" c.tree = Tree().id write_commit_patch(f, c, "CONTENTS", (1, 1), version="custom") f.seek(0) lines = f.readlines() self.assertTrue(lines[0].startswith("From 0b0d34d1b5b596c928adc9a727a4b9e03d025298")) self.assertEqual(lines[1], "From: Jelmer <*****@*****.**>\n") self.assertTrue(lines[2].startswith("Date: ")) self.assertEqual([ "Subject: [PATCH 1/1] This is the first line\n", "And this is the second line.\n", "\n", "\n", "---\n"], lines[3:8]) self.assertEqual([ "CONTENTS-- \n", "custom\n"], lines[-2:]) if len(lines) >= 12: # diffstat may not be present self.assertEqual(lines[8], " 0 files changed\n")
def test_graph_disconnected_to_dot(self): dependencies_expected = ( ('towel-stuff', 'bacon', 'bacon (<=0.2)'), ('grammar', 'bacon', 'truffles (>=1.2)'), ('choxie', 'towel-stuff', 'towel-stuff (0.1)'), ('banana', 'strawberry', 'strawberry (>=0.5)'), ) disconnected_expected = ('cheese', 'bacon', 'strawberry') dists = [] for name in self.DISTROS_DIST + self.DISTROS_EGG: dist = get_distribution(name, use_egg_info=True) self.assertNotEqual(dist, None) dists.append(dist) graph = depgraph.generate_graph(dists) buf = StringIO() depgraph.graph_to_dot(graph, buf, skip_disconnected=False) buf.seek(0) lines = buf.readlines() dependencies_lines = [] disconnected_lines = [] # First sort output lines into dependencies and disconnected lines. # We also skip the attribute lines, and don't include the "{" and "}" # lines. disconnected_active = False for line in lines[1:-1]: # Skip first and last line if line.startswith('subgraph disconnected'): disconnected_active = True continue if line.startswith('}') and disconnected_active: disconnected_active = False continue if disconnected_active: # Skip the 'label = "Disconnected"', etc. attribute lines. if ' = ' not in line: disconnected_lines.append(line) else: dependencies_lines.append(line) dependencies_matches = [] for line in dependencies_lines: if line[-1] == '\n': line = line[:-1] match = self.EDGE.match(line.strip()) self.assertIsNot(match, None) dependencies_matches.append(match.groups()) disconnected_matches = [] for line in disconnected_lines: if line[-1] == '\n': line = line[:-1] line = line.strip('"') disconnected_matches.append(line) self.checkLists(dependencies_matches, dependencies_expected) self.checkLists(disconnected_matches, disconnected_expected)
def smi(self, subdata): if requests_version < 0x20300: subdata = subdata.content.decode("latin") else: subdata.encoding = "ISO-8859-1" subdata = subdata.text ssubdata = StringIO(subdata) timea = 0 number = 1 data = None subs = "" TAG_RE = re.compile(r'<(?!\/?i).*?>') bad_char = re.compile(r'\x96') for i in ssubdata.readlines(): i = i.rstrip() sync = re.search(r"<SYNC Start=(\d+)>", i) if sync: if int(sync.group(1)) != int(timea): if data and data != " ": subs += "%s\n%s --> %s\n" % (number, timestr(timea), timestr(sync.group(1))) text = "%s\n" % TAG_RE.sub('', data.replace("<br>", "\n")) text = decode_html_entities(text) if text[len(text) - 2] != "\n": text += "\n" subs += text number += 1 timea = sync.group(1) text = re.search("<P Class=SVCC>(.*)", i) if text: data = text.group(1) recomp = re.compile(r'\r') text = bad_char.sub('-', recomp.sub('', subs)) return text
def __init__(self, filename): self.filename = filename buf = StringIO() with open(filename) as f_in: buf.write(re.sub(r'\\\n\s*', '', f_in.read())) buf.seek(0) for line in buf.readlines(): if line[0] == '!' or line[:3] == 'END' or len(line) < 2: continue if not line[0] == ' ': parent = self c, data = line.split(' ', 1) else: assert data, 'Attempted to nest with no parent' if parent is self: parent = data c, data = line[1:].split(' ', 1) c = sanitise(c) data = to_struct(data) if hasattr(parent, c): if not isinstance(getattr(parent, c), (tuple, list)): setattr(parent, c, [getattr(parent, c)]) getattr(parent, c).append(data) else: setattr(parent, c, data) buf.close()
def inspect_email_text(text): """Extracts key information from email header in text. """ sio = StringIO(text) required_fields = ["From", "Subject", "Date"] # TODO: some emails don't # have dates! optional_fields = ["To", "Reply-To"] data = {'Subject': "", 'To': ""} while True: line = sio.readline().rstrip() if line == '': break for field in required_fields + optional_fields: start_string = field + ": " if line.startswith(start_string): data[field] = line[len(start_string):] tests = [field in data.keys() for field in required_fields] if not all(tests): print("Some required email fields were missing") import pdb; pdb.set_trace() if line == '': line = sio.readline().rstrip() else: import pdb; pdb.set_trace() data['Body'] = sio.readlines() return data
def generatefile(): """File write, extra detection of changes in existing file return True on any issues, but only if there is changes""" # get file path foutpath = getmdfile() if ARGS.verbose: print("Output path: {}".format(str(foutpath))) # write data to temp memorystream ftemp = StringIO() ret = generate(ftemp) # get old filedata, skipping header with getmdfile().open("r", encoding="utf-8") as forg: olddata = forg.readlines()[3:] # get new data, skip first empty line ftemp.seek(0) newdata = ftemp.readlines()[1:] # if new data is same as old we don't need to write anything if newdata == olddata: print("No changes, exit without write") return False # write output with foutpath.open("w", encoding="utf-8") as fout: fout.write(getmarkdownheader()) fout.write(ftemp.getvalue()) return ret
def _get_request_results(self): for chunk in iter(self.input_queue.get, 'STOP'): accessions_to_retrieve = ' '.join(chunk) response = self._make_request(accessions_to_retrieve) go_info = StringIO(response) self.output_queue.put([_.rstrip() for _ in go_info.readlines()]) self.output_queue.put('DONE')
def extract_imgur_album_urls(album_url): """ Given an imgur album URL, attempt to extract the images within that album Returns: List of qualified imgur URLs """ response = request(album_url) info = response.info() # Rudimentary check to ensure the URL actually specifies an HTML file if 'content-type' in info and not info['content-type'].startswith('text/html'): return [] filedata = response.read() # TODO: stop parsing HTML with regexes. match = re.compile(r'\"hash\":\"(.[^\"]*)\",\"title\"') items = [] memfile = StringIO(filedata.decode('utf-8')) #print("REALLY LOOKING") for line in memfile.readlines(): results = re.findall(match, line) if not results: continue items += results memfile.close() # TODO : url may contain gif image. urls = ['http://i.imgur.com/%s.jpg' % (imghash) for imghash in items] return urls
def test_graph_bad_version_to_dot(self): expected = ( ('towel-stuff', 'bacon', 'bacon (<=0.2)'), ('grammar', 'bacon', 'truffles (>=1.2)'), ('choxie', 'towel-stuff', 'towel-stuff (0.1)'), ('banana', 'strawberry', 'strawberry (>=0.5)'), ) dists = [] for name in self.DISTROS_DIST + self.DISTROS_EGG + self.BAD_EGGS: dist = get_distribution(name, use_egg_info=True) self.assertNotEqual(dist, None) dists.append(dist) graph = depgraph.generate_graph(dists) buf = StringIO() depgraph.graph_to_dot(graph, buf) buf.seek(0) matches = [] lines = buf.readlines() for line in lines[1:-1]: # skip the first and the last lines if line[-1] == '\n': line = line[:-1] match = self.EDGE.match(line.strip()) self.assertIsNot(match, None) matches.append(match.groups()) self.checkLists(matches, expected)
def test_list_files(self): out = StringIO() call_command("minio", "ls", stdout=out) out.seek(0) lines = sorted(out.readlines()) expected = sorted([f"{self.new_file}\n", f"{self.second_file}\n"]) self.assertEqual(lines, expected)
def bannerize(str, term_w, figlet_fonts, use_pyfiglet=True): # do 100 attempts to find a suitable font # criteria: # must be narrower than term window # wider than 1/4 the term window(the larger, the prettier) if not isinstance(figlet_fonts, list): figlet_fonts = [fi.strip() for fi in figlet_fonts.split(',')] out = "\n" + str + "\n" + '-' * len(str) + "\n" fi = "none" if not PYFIG or not use_pyfiglet: return (out, fi) for i in range(100): fi = rand_font(figlet_fonts) try: f = Figlet(font=fi, width=term_w) except FontNotFound: continue out = f.renderText(str) out_IO = StringIO(out) out_width = max([len(a) for a in out_IO.readlines()]) # print("outWidth: %d"%outWidth) if out_width <= term_w and out_width > (term_w / 4): # print("Font name: " + fi) return (out, fi) return (out, fi)
def smi(self, subdata): if requests_version < 0x20300: subdata = subdata.content.decode("latin") else: subdata.encoding = "ISO-8859-1" subdata = subdata.text ssubdata = StringIO(subdata) timea = 0 number = 1 data = None subs = "" TAG_RE = re.compile(r"<(?!\/?i).*?>") bad_char = re.compile(r"\x96") for i in ssubdata.readlines(): i = i.rstrip() sync = re.search(r"<SYNC Start=(\d+)>", i) if sync: if int(sync.group(1)) != int(timea): if data and data != " ": subs += "{}\n{} --> {}\n".format( number, timestr(timea), timestr(sync.group(1))) text = "%s\n" % TAG_RE.sub("", data.replace("<br>", "\n")) text = decode_html_entities(text) if text[len(text) - 2] != "\n": text += "\n" subs += text number += 1 timea = sync.group(1) text = re.search("<P Class=SVCC>(.*)", i) if text: data = text.group(1) recomp = re.compile(r"\r") text = bad_char.sub("-", recomp.sub("", subs)) return text
def test_then_it_should_pass_all_logs(self): stream = StringIO() subject = MultiProcessingHandler('mp-handler', logging.StreamHandler(stream=stream)) logger = logging.Logger('root') logger.addHandler(subject) def worker(wid, logger): for _ in range(10): logger.info("Worker %d log.", wid) logger.info("Starting workers...") procs = [ mp.Process(target=worker, args=(wid, logger)) for wid in range(2) ] for proc in procs: proc.start() logger.info("Workers started.") for proc in procs: proc.join() logger.info("Workers done.") subject.close() stream.seek(0) lines = stream.readlines() self.assertIn("Starting workers...\n", lines) self.assertIn("Workers started.\n", lines) self.assertIn("Workers done.\n", lines) self.assertEqual(10 * 2 + 3, len(lines))
def _GetReleaseForCommit(self, commit_hash: str, n: int = 100): if n > 2000: logging.error( "We couldn't find the release branch that we correspond to") return "0.0.0.0" git_dir = os.path.dirname(self.sp) cmd_args = ["log", '--format="%h %D"', "-n " + str(n)] return_buffer = StringIO() RunCmd("git", " ".join(cmd_args), workingdir=git_dir, outstream=return_buffer) return_buffer.seek(0) results = return_buffer.readlines() return_buffer.close() log_re = re.compile(r'(release|dev)/(\d{6})') for log_item in results: commit = log_item[:11] branch = log_item[11:].strip() if len(branch) == 0: continue match = log_re.search(branch) if match: logging.info("Basing our release commit off of commit " + commit) return match.group(2) return self._GetReleaseForCommit(commit_hash, n * 2)
def gen(self, fname, flags=None, dlls=[], debug=False): """Take a file input and generate the code. """ flags = flags or [] dlls = dlls or [] ofi = StringIO() gen = self._gen(ofi, fname, flags=flags, dlls=dlls) # load code namespace = {} # DEBUG # print ofi.getvalue() # DEBUG ofi.seek(0) ignore_coding = ofi.readline() # exec ofi.getvalue() in namespace output = ''.join(ofi.readlines()) self.text_output = output try: # PY3 change exec(output, namespace) except Exception: print(output) raise # except NameError: # print(output) self.namespace = ADict(namespace) if debug: print(output) return
def ascii_art_text(str, font, term_w): """ Given a str, and a font(or rand for choose a font at random) and a terminal width. Returns figletized text and the font that was used. """ # default is no figlets out = "\n" + str + "\n" + '-' * len(str) + "\n" if font != "rand": f = pyfiglet.Figlet(font=font) out = f.renderText(str) return(out, font) else: # do 100 attempts to find a suitable font # criteria: # must be narrower than term window # wider than 1/4 the term window(the larger, the prettier) fi = "none" for i in range(100): fi = rand_font() f = pyfiglet.Figlet(font=fi) # , width=term_w ) out = f.renderText(str) from io import StringIO out_IO = StringIO(out) out_width = max([len(a) for a in out_IO.readlines()]) # print("outWidth: %d"%outWidth) if out_width <= term_w and out_width > (term_w / 4): # print("Font name: " + fi) return(out, fi) return(out, fi)
def test_custom_ids(self): sio = StringIO() action = test_smart_add.AddCustomIDAction(to_file=sio, should_print=True) self.build_tree(['file1', 'dir1/', 'dir1/file2']) wt = self.make_branch_and_tree('.') if not wt._format.supports_setting_file_ids: self.assertRaises(workingtree.SettingFileIdUnsupported, wt.smart_add, ['.'], action=action) return wt.smart_add(['.'], action=action) # The order of adds is not strictly fixed: sio.seek(0) lines = sorted(sio.readlines()) self.assertEqual([ 'added dir1 with id directory-dir1\n', 'added dir1/file2 with id file-dir1%file2\n', 'added file1 with id file-file1\n', ], lines) wt.lock_read() self.addCleanup(wt.unlock) self.assertEqual([ ('', wt.path2id('')), ('dir1', b'directory-dir1'), ('file1', b'file-file1'), ('dir1/file2', b'file-dir1%file2'), ], [(path, ie.file_id) for path, ie in wt.iter_entries_by_dir()])
def test_csv(self): r = self.client.get("/?q=r&format=csv") self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "text/csv") result = StringIO(r.content.decode()) header, first_line, second_line = result.readlines() self.assertEqual( header, '"DOT_NUMBER","LEGAL_NAME","DBA_NAME","CARRIER_OPERATION","HM_FLAG",' '"PC_FLAG","PHY_STREET","PHY_CITY","PHY_STATE","PHY_ZIP","PHY_COUNTRY",' '"MAILING_STREET","MAILING_CITY","MAILING_STATE","MAILING_ZIP",' '"MAILING_COUNTRY","TELEPHONE","FAX","EMAIL_ADDRESS","MCS150_DATE",' '"MCS150_MILEAGE","MCS150_MILEAGE_YEAR","ADD_DATE","OIC_STATE",' '"NBR_POWER_UNIT","DRIVER_TOTAL"\r\n', ) self.assertEqual( first_line, '42,"Killer Carrier, Inc","Killer Carrier","C","N","Y",' '"0 Abyss Alley","Nowhere","NY","12345","US","0 Abyss Alley",' '"Nowhere","NY","12345","US","+123456789",' '"+198765432","*****@*****.**","05-MAR-20",18725329,2020,' '"04-FEB-19","MA",5,4\r\n', ) self.assertEqual( second_line, '43,"Transport Greatness","","","Y","N",' '"","","","","","",' '"","","","","",' '"","","","","",' '"03-JAN-19","","",""\r\n', )
def fix_date_and_remove_null(yaml_file, date, input_type='ruamel'): """ Remove the single quotes around the date key-value pair in the provided yaml_file and remove any 'null' values :param yaml_file: ruamel.yaml instance or location of YAML file :param date: string date value (e.g. 2019-01-01) :param input_type: input type can be a ruamel.yaml instance or list :return: YAML file lines in a list """ _yaml = init_yaml() if input_type == 'ruamel': # ruamel does not support output to a variable. Therefore we make use of StringIO. file = StringIO() _yaml.dump(yaml_file, file) file.seek(0) new_lines = file.readlines() elif input_type == 'list': new_lines = yaml_file elif input_type == 'file': new_lines = yaml_file.readlines() fixed_lines = [l.replace('\'' + str(date) + '\'', str(date)).replace('null', '') if REGEX_YAML_DATE.match(l) else l.replace('null', '') for l in new_lines] return fixed_lines
def remove_blank_lines(source): """ Removes blank lines from 'source' and returns the result. """ io_obj = StringIO(source) source = [a for a in io_obj.readlines() if a.strip()] return "".join(source)
def test_serialize(): out = StringIO() influxdb.serialize(out, "cpu", dict(load=10.0, alert=True), dict(host="Server A", region="us west"), datetime(2015, 2, 22, 17, 55)) out.seek(0) r = out.readlines() assert "cpu,host=Server\ A,region=us\ west alert=t,load=10.0 1424624100000000000\n" == r[0]
def send_typical_request(self, message): input = StringIO(message) output = StringIO() self.handler.rfile = input self.handler.wfile = output self.handler.handle_one_request() output.seek(0) return output.readlines()
def filter_commented_out_code(source): """Yield code with commented out code removed.""" marked_lines = list(commented_out_code_line_numbers(source)) sio = StringIO(source) previous_line = '' for line_number, line in enumerate(sio.readlines(), start=1): if (line_number not in marked_lines or previous_line.rstrip().endswith('\\')): yield line previous_line = line
def open_mock(content, **kwargs): content_out = StringIO() m = mock_open(read_data=content) with patch('__builtin__.open', m, create=True, **kwargs) as mo: stream = StringIO(content) rv = mo.return_value rv.write = lambda x: content_out.write(bytes(x, "utf-8")) rv.content_out = lambda: content_out.getvalue() rv.__iter__ = lambda x: iter(stream.readlines()) yield rv
def printable_error(error, tb=None): printable = [repr(error)] if tb is None: tb = sys.exc_info()[2] printed = StringIO() traceback.print_tb(tb, file=printed) printed.seek(0) for line in printed.readlines(): printable.append(line.rstrip('\n')) return printable
def solve_it(file_name, mode=1, threads=4): pool = ThreadPool(threads) results = [] with open(file_name, 'r') as input_data_file: input_data = input_data_file.read() buf = StringIO(input_data) first_line = buf.readline().split() node_count = int(first_line[0]) if node_count == 1000: for i in range(threads): results.append(pool.apply_async(run_prog, args=(file_name, str(mode), './ls.out'))) else: for i in range(threads): results.append(pool.apply_async(run_prog, args=(file_name, str(2), 'greedy.py'))) pool.close() pool.join() results = [r.get().decode('utf-8') for r in results] opt_num_colors = sys.maxsize output_data = 'no sol' if node_count == 1000: for i in results: buf = StringIO(i) all_lines = buf.readlines() colors_and_proof = all_lines[4].split() if int(colors_and_proof[0]) < opt_num_colors: output_data = all_lines[4] + all_lines[5] else: for i in results: buf = StringIO(i) all_lines = buf.readlines() # print(all_lines) colors_and_proof = all_lines[2].split() if int(colors_and_proof[0]) < opt_num_colors: output_data = all_lines[2] + all_lines[3] return output_data
def test_logger_unicode_level(self): output = StringIO() logger = Logger.get('unicode', output, OutputStream, level='info') logger.info('fline') output.seek(0) log_lines = output.readlines() self.assertEqual(len(log_lines), 1)
def test_info_duplicate_columns_shows_correct_dtypes(self): # GH11761 io = StringIO() frame = DataFrame([[1, 2.0]], columns=['a', 'a']) frame.info(buf=io) io.seek(0) lines = io.readlines() assert 'a 1 non-null int64\n' == lines[3] assert 'a 1 non-null float64\n' == lines[4]
def _parse_preamble(reader): lines = [] if reader[0:2] == "{%": reader.consume(2) end = reader.find("%}") content = reader.consume(end) reader.consume(2) io = StringIO(content) lines = [l[:-1] for l in io.readlines()] io.close() return lines
def test_logger_stream(self): output = StringIO() logger = Logger.get('stream', output, OutputStream) logger.info('first_line') logger.info('second_line') output.seek(0) log_lines = output.readlines() self.assertEqual(len(log_lines), 2)
def build_dict_from_csv(body): if not isinstance(body, str): body = body.decode(errors='ignore') buf = StringIO(body) raw_fields = buf.readlines()[0] fields = raw_fields.strip().split(',') buf.seek(len(raw_fields)) r = csv.DictReader(buf, fields) ret = [] for row in r: ret.append(row) return ret
def get_archives(self): """A list of archives as returned by --list-archives. Queried the first time it is accessed, and then subsequently cached. """ if self._queried_archives is None: response = StringIO(self.call('--list-archives')) self._queried_archives = [l.rstrip() for l in response.readlines()] if ['v'] in self.options: # Filter out extraneous info if tarsnap was run with # verbose flag self._queried_archives = [ l.rsplit('\t', 1)[0] for l in self._queried_archives] return self._queried_archives + self._known_archives
def format_exception(type, value, tb): """ Format just exception, just a traceback or both. Return a list of lines. """ buffer = StringIO() if tb is not None: if hasattr(tb, "tb_frame"): tb = traceback.extract_tb(tb) for line in traceback.format_list(tb): buffer.write(unicode(line)) if (type is not None) and (value is not None): for line in traceback.format_exception_only(type, value): buffer.write(unicode(line)) buffer.seek(0) return buffer.readlines()
def _parse(self, data): """ Parse the output from the 'mntr' 4letter word command """ h = StringIO(data.decode()) result = {} for line in h.readlines(): try: key, value = self._parse_line(line) result[key] = value except ValueError: pass # ignore broken lines return result
def hosts_map(device='eth0',exclude=None): output = subprocess.check_output(['arp','-n']) r = StringIO(output.decode('utf-8')) reb = re.compile('\s+') res = set() for line in r.readlines(): item = reb.split(line.strip()) if(len(item)<5): continue if(item[4] != device or item[0] in exclude): continue res.add((item[0],item[2])) r.close() return res
def fetch_results(_id): url_data = UrlSync.objects.get(id=_id) processed_line = url_data.current_line class_ = load_class(url_data.competition.processing_class) processing_class = class_(url_data.competition.id) try: resp = requests.get(url_data.url) if resp.status_code != 200: return buf = StringIO(resp.text) file_lines = tuple(buf.readlines()) lines_to_process = file_lines[processed_line:] for line in lines_to_process: print(line) if len(line.strip()) == 0: print('empty line') continue number, time_text = line.strip().split(',') if number == '0': print('skipping 0 number') continue scan, created = ChipScan.objects.get_or_create(competition=url_data.competition, nr_text=number, time_text=time_text, url_sync=url_data) scan.time = time_text try: scan.nr = Number.objects.get(competition_id__in=url_data.competition.get_ids(), number=number, group='') except: print('number not found') continue finally: scan.save() process_chip_result.delay(scan.id) # processing_class.process_chip_result(scan.id) url_data.current_line = len(file_lines) url_data.save() # TODO: This should be removed after process review processing_class.recalculate_all_standings() send_smses() except: error = traceback.format_exc() Log.objects.create(content_object=url_data, action="Error processing file", params={ 'error': error, }) raise Exception('Error processing external chip file')