def get_curlbomb(self, args, script=None, override_defaults={}):
"""Prepare curlbomb to run in a thread
Assumes args has a '{script}' formatter in it to replace a temporary path with
If no '{script}' formatter is found, stdin is mocked through settings['stdin']
Returns tuple(curlbomb_thread, client_command)
"""
if type(script) == str:
script = bytes(script, "utf-8")
stdin = "{script}" not in args and script is not None
try:
log.info("Using stdin: {}".format(stdin))
if stdin:
s = TextIOWrapper(BytesIO(script))
override_defaults['stdin'] = s
else:
s = NamedTemporaryFile()
if script is not None:
s.write(script)
s.flush()
args = args.format(script=s.name)
args = shlex.split(args)
log.warn("starting curlbomb: {}".format(args))
settings = curlbomb.get_settings(args, override_defaults)
client_cmd = settings['get_curlbomb_command'](settings)
curlbomb_thread = CurlbombThread(settings)
curlbomb_thread.start()
return (curlbomb_thread,
client_cmd)
finally:
s.close()
def _write_table(profile_dir, table_name, rows, fields,
append=False, gzip=False):
# don't gzip if empty
rows = iter(rows)
try:
first_row = next(rows)
except StopIteration:
gzip = False
else:
rows = chain([first_row], rows)
if gzip and append:
logging.warning('Appending to a gzip file may result in '
'inefficient compression.')
if not os.path.exists(profile_dir):
raise ItsdbError('Profile directory does not exist: {}'
.format(profile_dir))
tbl_filename = os.path.join(profile_dir, table_name)
mode = 'a' if append else 'w'
if gzip:
# text mode only from py3.3; until then use TextIOWrapper
#mode += 't' # text mode for gzip
f = TextIOWrapper(gzopen(tbl_filename + '.gz', mode=mode))
else:
f = open(tbl_filename, mode=mode)
for row in rows:
f.write(make_row(row, fields) + '\n')
f.close()
def _grep_a_file(bucketstr: str, key: str, regex: str,
output: io.TextIOWrapper):
'''
parse the s3 file line to see if it matches the regex
if yes, dump the line into output buffer
:param bucket:
:param key:
:param regex:
:param output: the output buffer
:return:
'''
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucketstr)
for obj in bucket.objects.filter(Prefix=key):
datadict = obj.get()
instream = boto_stream.BotoStreamBody(datadict['Body'])
instream = io.BufferedReader(instream, buffer_size=1 * 2 ^ 20)
filename, file_extension = os.path.splitext(key)
if file_extension == '.gz':
instream = gzip.GzipFile(fileobj=instream, mode='rb')
for line in io.TextIOWrapper(instream):
if re.search(regex, line) is not None:
output.write(obj.key + ":" + line)
def fake_stdin(data):
if PY2:
stdin = tempfile.TemporaryFile()
else:
stdin = TextIOWrapper(tempfile.TemporaryFile())
stdin.write(data)
stdin.flush()
stdin.seek(0)
return stdin
def test_io_wrapper(self):
content = "vive l'été\n"
with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
test_file.write(content.encode())
test_file.seek(0)
wrapper = TextIOWrapper(test_file, 'utf-8', newline='\n')
self.assertEqual(wrapper.read(), content)
wrapper.write(content)
wrapper.seek(0)
self.assertEqual(wrapper.read(), content * 2)
test_file = wrapper.detach()
test_file.seek(0)
self.assertEqual(test_file.read(), (content * 2).encode())
def test_io_wrapper(self):
content = "vive l'été\n"
with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
test_file.write(content.encode('utf-8'))
test_file.seek(0)
wrapper = TextIOWrapper(test_file, 'utf-8', newline='\n')
self.assertEqual(wrapper.read(), content)
# The following seek() call is required on Windows Python 2 when
# switching from reading to writing.
wrapper.seek(0, 2)
wrapper.write(content)
wrapper.seek(0)
self.assertEqual(wrapper.read(), content * 2)
test_file = wrapper.detach()
test_file.seek(0)
self.assertEqual(test_file.read(), (content * 2).encode('utf-8'))
def write(self, outfile: TextIOWrapper):
"""Write the connectivity of the CAtlas to file."""
# doesn't matter how we traverse the graph, so we use DFS for ease of
# implementation
stack = [self]
seen = set()
while len(stack) > 0:
# remove from the stack
curr = stack.pop()
# write node information
child_str = " ".join(str(child.idx) for child in curr.children)
outfile.write("{},{},{},{}\n".format(curr.idx,
curr.vertex,
curr.level,
child_str))
# all nodes already seen don't get re-added
seen.add(curr)
stack.extend(filter(lambda x: x not in seen, curr.children))
def fix_headerguard(filename):
supposed = get_guard_name(filename)
with open(filename, "r", encoding='utf-8', errors='ignore') as f:
inlines = f.readlines()
tmp = TextIOWrapper(BytesIO(), encoding="utf-8", errors="ignore")
tmp.seek(0)
guard_found = 0
guard_name = ""
ifstack = 0
for line in inlines:
if guard_found == 0:
if line.startswith("#ifndef"):
guard_found += 1
guard_name = line[8:].rstrip()
line = "#ifndef %s\n" % (supposed)
elif guard_found == 1:
if line.startswith("#define") and line[8:].rstrip() == guard_name:
line = "#define %s\n" % (supposed)
guard_found += 1
else:
break
elif guard_found == 2:
if line.startswith("#if"):
ifstack += 1
elif line.startswith("#endif"):
if ifstack > 0:
ifstack -= 1
else:
guard_found += 1
line = "#endif /* %s */\n" % supposed
tmp.write(line)
tmp.seek(0)
if guard_found == 3:
for line in difflib.unified_diff(inlines, tmp.readlines(),
"%s" % filename, "%s" % filename):
sys.stdout.write(line)
else:
print("%s: no / broken header guard" % filename, file=sys.stderr)
return False
def header_bytes(self, any_chunks):
if self.status_code not in (100, 204):
enc_hdr, enc_msg = self.encapsulated(any_chunks)
self.headers['Encapsulated'] = enc_hdr
else:
enc_msg = None
if any_chunks:
# http://www.measurement-factory.com/std/icap/#e1
raise ValueError("no encapsulation allowed")
bio = BytesIO()
sio = TextIOWrapper(bio, encoding='iso-8859-1')
status_line = u'{} {} {}\r\n'.format(self.protocol,
self.status_code,
self.reason)
sio.write(status_line)
for key, value in iteritems(self.headers):
if isinstance(value, list):
values = [text_type(v) for v in value]
line = u'{}: {}\r\n'.format(key, ', '.join(values))
else:
line = u'{}: {}\r\n'.format(key, value)
sio.write(line)
sio.write(u'\r\n')
sio.flush()
if enc_msg:
bio.write(enc_msg)
return bio.getvalue()
def isa(file: TextIOWrapper):
for key, val in races.items():
for r in races.values():
if r == '__': continue
file.write(f'\tdef test_isa_when_{key}_with_{r}(self):\n')
file.write(
f'\t\tself.assert{val == r}(self.{key}.isa(Race.{r}))\n')
file.write('\n\n')
def isa(file: TextIOWrapper):
for key, val in poss.items():
for pos in poss.values():
if pos == '__': continue
file.write(f'\tdef test_isa_when_{key}_with_{pos}(self):\n')
boolean = bool(eval(f'enums.Position.{val}') & eval(f'enums.Position.{pos}'))
file.write(f'\t\tself.assert{boolean}(self.{key}.isa(Position.{pos}))\n')
file.write('\n\n')
def smart_log(
sentence: list,
predicted: list,
expected: list,
log_file: TextIOWrapper,
uncertain: torch.Tensor,
csv_writer: Optional[csv.writer],
) -> None:
header = '''
<----------------------------------->
Sentence: {}
Expected: {}
Predicted: {}
Uncertain: {}
'''.format(sentence, expected, predicted, uncertain)
log_file.write(header)
for i in range(len(predicted)):
if csv_writer is not None: # pragma: no cover
csv_writer.writerow([sentence[i], predicted[i], expected[i]])
if predicted[i] != expected[i]:
log_file.write("{}, {}, {}\n".format(sentence[i], predicted[i],
expected[i]))
def part2(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
It's a completely full flight, so your seat should be the only missing
boarding pass in your list. However, there's a catch: some of the seats at
the very front and back of the plane don't exist on this aircraft, so
they'll be missing from your list as well.
Your seat wasn't at the very front or back, though; the seats with IDs +1
and -1 from yours will be in your list.
What is the ID of your seat?
"""
seats = sorted(seat_ids(parse(stdin)))
for i in range(len(seats) - 1):
stderr.write(
f"{i}: {seats[i + 1]} - {seats[i]} = {seats[i + 1] - seats[i]}\n")
if seats[i + 1] - seats[i] == 2:
return seats[i] + 1
raise Exception("No matches found.")
def write(self, writeFile: TextIOWrapper):
from PyVili import ViliParser
if self.visible:
for i in range(self.getDepth()):
writeFile.write(ViliParser.spacing * " ")
writeFile.write(self.id + ":\n")
for child in self.children:
self.children[child].write(writeFile)
writeFile.write("\n")
def isa(file: TextIOWrapper):
for key, val in attributes.items():
for attr in attributes.values():
if attr == '__': continue
file.write(f'\tdef test_isa_when_{key}_with_{attr}(self):\n')
file.write(
f'\t\tself.assert{val == attr}(self.{key}.isa(Attribute.{attr}))\n'
)
file.write('\n\n')
def part1(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
ecl:gry pid:860033327 eyr:2020 hcl:#fffffd
byr:1937 iyr:2017 cid:147 hgt:183cm
iyr:2013 ecl:amb cid:350 eyr:2023 pid:028048884
hcl:#cfa07d byr:1929
hcl:#ae17e1 iyr:2013
eyr:2024
ecl:brn pid:760753108 byr:1931
hgt:179cm
hcl:#cfa07d eyr:2025 pid:166559648
iyr:2011 ecl:brn hgt:59in
Count the number of valid passports - those that have all required fields.
Treat cid as optional. In your batch file, how many passports are valid?
"""
passports = parse(stdin)
valid_passports = 0
def validate(passport):
for key in ("byr", "iyr", "eyr", "hgt", "hcl", "ecl", "pid"):
if key not in passport:
return False
return True
for passport in passports:
stderr.write(f"{passport}\n")
if validate(passport):
valid_passports += 1
return valid_passports
def write(self, writeFile: TextIOWrapper):
from PyVili import ViliParser
if self.visible:
for i in range(self.getDepth()):
writeFile.write(ViliParser.spacing * " ")
writeFile.write((
self.id + ":" +
self.dumpData()) if self.id[0] != "#" else self.dumpData())
writeFile.write("\n")
def part2(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
What do you get if you add up the results of evaluating the homework
problems using these new rules?
"""
def evaluate(expression: list) -> int:
for i, element in enumerate(expression):
if isinstance(element, list):
expression[i] = evaluate(element)
while "+" in expression:
index = expression.index("+")
lhs, _, rhs = expression[index-1:index+2]
expression[index-1:index+2] = [lhs + rhs]
while "*" in expression:
index = expression.index("*")
lhs, _, rhs = expression[index-1:index+2]
expression[index-1:index+2] = [lhs * rhs]
if len(expression) != 1:
raise Exception(
f"Expected expression to have 1 element, got {expression}"
)
return expression[0]
total = 0
expressions = parse(stdin)
for expression in expressions:
stderr.write(f"{expression} = ")
result = evaluate(expression)
stderr.write(f"{result}\n")
total += result
return total
def part1(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
Play the small crab in a game of Combat using the two decks you just dealt.
What is the winning player's score?
"""
def play_round(hands: tuple):
cards = (hands[0].pop(0), hands[1].pop(0))
if cards[0] > cards[1]:
hands[0].append(cards[0])
hands[0].append(cards[1])
else:
hands[1].append(cards[1])
hands[1].append(cards[0])
hands = parse(stdin)
while hands[0] and hands[1]:
stderr.write(f"{hands}\n")
play_round(hands)
stderr.write(f"{hands}\n")
return score(hands[0] if hands[0] else hands[1])
def run(self,
output_stream: io.TextIOWrapper = None,
verbose: bool = False,
**kwargs) -> List[Dict]:
"""
Runs graph
:param output_stream: opened file for writing
:param verbose: parameter for logging. True sets INFO logging level, False sets ERROR logging level
:param kwargs: dict of all input arguments for graph running
:return: list of dicts, result of computing graph
"""
if verbose:
logger.setLevel(logging.INFO)
graphs_in_topological_order = self._topological_sort()
for graph_to_run in graphs_in_topological_order:
graph_to_run._internal_run(**kwargs)
if output_stream is not None:
for record in self._output:
output_stream.write(json.dumps(record) + "\n")
return []
else:
return list(self._output)
def download(output_basepath: str, manifest_dict: dict[str, str],
downloaded_dict: dict, downloaded_file: TextIOWrapper,
simulate: bool, add_to_downloaded_only: bool):
main_logger = logging.getLogger(__name__)
if not os.access(output_basepath, os.W_OK):
main_logger.error(f"can't write to directory {output_basepath}")
exit(1)
else:
ydl_opts = {
"v": "true",
"nocheckcertificate": "true",
"restrictfilenames": "true",
"logger": logging.getLogger("youtube-dl")
}
for filename in manifest_dict:
manifest = manifest_dict[filename]
if manifest not in downloaded_dict:
output_path = os.path.join(output_basepath, filename)
ydl_opts["outtmpl"] = output_path + ".%(ext)s"
main_logger.info(f"Downloading {filename}")
if not simulate:
if not add_to_downloaded_only:
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([manifest])
downloaded_dict[manifest] = filename
downloaded_file.seek(0)
downloaded_file.write(json_dumps(downloaded_dict))
downloaded_file.truncate()
else:
main_logger.info(
f"Not downloading {filename} since it'd already been downloaded"
)
downloaded_file.close()
main_logger.info("Downloaded completed")
def part1(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
Consider the validity of the nearby tickets you scanned. What is your
ticket scanning error rate?
"""
fields, tickets = parse(stdin)
groups = valid_groups(fields)
invalid_values = [
value for ticket in tickets for value in ticket.values()
if not is_valid_value(value, groups)
]
stderr.write(f"fields: {fields}\n")
stderr.write(f"concatenated groups: {groups}\n")
stderr.write(f"tickets: {tickets}\n")
stderr.write(f"invalid values: {invalid_values}\n")
return sum(invalid_values)
def make_source(base_obj: Dict[str, Any], output_file: TextIOWrapper) -> None:
if base_obj['string_array_specifiers']:
base_obj['string_array_specifiers'] += ' '
output_file.write(STRINGS_START.format(**base_obj).lstrip())
for string_def in base_obj['gui_strings']:
string = string_def['string']
# We use JSON to quote the string because JSON strings are equivalent to C strings
output_file.write(f'{TAB}{json.dumps(string)},\n')
output_file.write(CURLY_BRACE_END)
def _copy(write_lock: RLock, dst: TextIOWrapper, src: BufferedReader,
prefix: str, colorize: ColorizeFunc):
while True:
line = src.readline()
if line is None or len(line) == 0:
break
with write_lock:
dst.write(colorize(prefix))
dst.write(" ")
dst.write(line.decode('UTF-8'))
def subtitle(self, file: TextIOWrapper, subtitle: str):
"""Writes a formatted subtitle string to a file object.
Parameters
----------
file : :obj:`TextIOWrapper`
The file object to write to.
subtitle : :obj:`str`
The subtitle string.
Returns
-------
None
"""
file.write("\n")
file.write("".join("-" for i in range(len(subtitle))) + "\n")
file.write(subtitle + "\n")
file.write("".join("-" for i in range(len(subtitle))) + "\n")
def add_translatable_string(f: io.TextIOWrapper,
context: str,
text: str,
disambiguation: str = '',
comment: str = ''):
if comment:
f.write('//: ' + comment + '\n')
if disambiguation:
f.write(
'QT_TRANSLATE_NOOP3("{context}", "{text}", "{disambiguation}"),\n'.
format(context=context, text=text, disambiguation=disambiguation))
else:
f.write('QT_TRANSLATE_NOOP("{context}", "{text}"),\n'.format(
context=context, text=text))
def write_dependencies(f: TextIOWrapper, data: dict, dependency_type: str, subset_dependencies: list):
if subset_dependencies is not None:
diff = list(set(subset_dependencies) - set(data[dependency_type]))
if len(diff):
print(f'WARNING: dependencies {diff} were not found in Pipfile.lock in \'{dependency_type}\' section')
for dependency in data[dependency_type]:
if subset_dependencies is not None and dependency not in subset_dependencies:
continue
dependency_section = data[dependency_type][dependency]
if "git" in dependency_section:
git_string = f"git+{dependency_section['git']}@{dependency_section['ref']}#egg={dependency}"
f.write(f"-e {git_string}\n") if "editable" in dependency_section else f.write(git_string + "\n")
else: # not final: maybe there is more flags like 'markers'
if "markers" in dependency_section:
f.write(f"{dependency}{dependency_section['version']} ; {dependency_section['markers']}\n")
else:
f.write(f"{dependency}{dependency_section['version']}\n")
def write_index_ones(a, f: TextIOWrapper):
previous_index = 0
# print(f'i: {i} \n')
for x in a:
val = golomb_coding(x, 1000)
index = 0
previous_index = 0
while True:
index = val.find("1", index + 1, len(val))
if index == -1:
break
f.write(f'{index-previous_index+1}')
previous_index = index + 1
f.write('.')
f.write('\n')
def part2(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
What is the earliest timestamp such that all of the listed bus IDs depart
at offsets matching their positions in the list?
"""
_, buses_dict = parse(stdin)
buses = [{"number": v, "offset": k} for k, v in buses_dict.items()]
stderr.write(f"{buses}\n")
def gcd(a, b):
while b != 0:
(a, b) = (b, a % b)
return a
def lcm(a, b):
return a * b // gcd(a, b)
result = 0
increment = 1
iterations = 0
for bus in buses:
stderr.write(f"Testing bus {bus}\n")
while (result + bus["offset"]) % bus["number"] != 0:
result += increment
iterations += 1
stderr.write(
f"({result} + {bus['offset']}) % {bus['number']} == 0, "
f"now incrementing by lcm({increment}, {bus['number']}) = "
f"{lcm(increment, bus['number'])}\n")
increment = lcm(increment, bus["number"])
stderr.write(f"Complete in {iterations} iterations\n")
return result
def writeHeader(self, fp, metainfo, image):
fp.seek(0)
wrapper = TextIOWrapper(fp, encoding='utf-8')
wrapper.write('\n%s PUMA Polarisation File Header V2.0\n' %
(self.sink.commentchar * 3))
# XXX(dataapi): add a utility function to convert metainfo to old
# by-category format
bycategory = {}
for (device, key), (_, val, unit, category) in metainfo.items():
if category:
bycategory.setdefault(category, []).append(
('%s_%s' % (device, key), (val + ' ' + unit).strip()))
for category, catname in INFO_CATEGORIES:
if category not in bycategory:
continue
wrapper.write('%s %s\n' % (self.sink.commentchar * 3, catname))
for key, value in sorted(bycategory[category]):
wrapper.write('%25s : %s\n' % (key, value))
# to ease interpreting the data...
# wrapper.write('\n%r' % self._arraydesc)
wrapper.write('\n')
wrapper.detach()
fp.flush()
def printAttributes(self, out: TextIOWrapper, file: PGFile,
pyClass: PyClass):
for item in file.declarations:
if util.inferParentClass(item) == pyClass.fqname:
v_type, req = file.declarations[item]
req = '**required**' if req else '*optional*'
short = util.inferShortName(item)
if v_type in ACCEPTED_TYPES:
out.write(
f"* {short} → [`{v_type}`](./standard_types#{v_type}) {req}\n"
)
else:
out.write(
f"* {short} → [`{v_type}`](#{v_type}) {req}\n"
)
out.write('\n')
def part1(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
You have a shiny gold bag. If you wanted to carry it in at least one
other bag, how many different bag colors would be valid for the
outermost bag?
"""
containers = parse(stdin)
stderr.write(f"{containers}\n")
contents = flip(containers)
stderr.write(f"{contents}\n")
count = 0
options = {"shiny gold"}
while count < len(options):
stderr.write(f"{count}\n")
count = len(options)
options.update(
*(contents[x] for x in options)
)
stderr.write(f"{options}\n")
return count - 1
def part2(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
How many # are not part of a sea monster?
"""
tiles = parse(stdin)
stderr.write(f"tiles: {tiles}\n")
grid = compute_grid(tiles)
stderr.write(f"grid: {grid}\n")
image = assemble_image(tiles, grid)
paint_sea_monsters(image)
stderr.write("\n".join(image) + "\n")
return len([char for line in image for char in line if char == '#'])
def dump(self, file: io.TextIOWrapper):
file.writelines([
f'lang {self.lang}\n', f'type {self.node_type}\n',
f'root {self.root}\n'
])
for key in self.priorities():
rules = self.rules_by_priority[key]
r = rules[0]
file.write(f'rule {key}' + f' {r["order"]} {r["argc"]}' +
f' {r["left"]} {r["right"]} {r["next"]}')
for rule in rules:
file.write(f' {rule["key"]} {rule["function"]}')
file.write("\n")
def compile(self, src: io.TextIOWrapper, dst: io.TextIOWrapper, width: int = 0x100, height: int = 0x100) -> int:
s = "".join([c for c in src.read() if c in "><v^+-,.[]"])
while len(s):
up = s.find("^")
down = s.find("v")
if up != -1 and (down == -1 or up < down):
dst.write(s[:up] + ("<" * width))
s = s[up + 1:]
elif down != -1:
dst.write(s[:down] + (">" * width))
s = s[down + 1:]
else:
dst.write(s)
s = ""
return width * height
def part1(stdin: io.TextIOWrapper, stderr: io.TextIOWrapper) -> int:
"""
What is the ID of the earliest bus you can take to the airport multiplied
by the number of minutes you'll need to wait for that bus?
"""
time, buses = parse(stdin)
stderr.write(f"{time}\n")
stderr.write(f"{buses}\n")
departure_times = {
bus: math.ceil(time / bus) * bus - time
for bus in buses.values()
}
stderr.write(f"{departure_times}\n")
next_bus = min(departure_times, key=departure_times.get)
return next_bus * departure_times[next_bus]
def printFactory(self, out: TextIOWrapper, file: PGFile):
outString = (
"\n\nclass {}Factory(object):\n"
" @staticmethod\n"
" def deserialize(data: bytes):\n"
" data = Serializable.deserialize(data)\n"
" if len(data) > 1:\n"
" raise AttributeError('This is likely not a Protogen packet.')\n"
"\n"
" packetType = None\n"
" for item in data:\n"
" packetType = item[item.rfind('.')+1:]\n")
out.write(outString.format(file.header))
for item in file.classes:
if item.parent is None: # root-level class
out.write(f'{tab*3}if packetType == \'{item.name}\':\n'
f'{tab*4}return {item.name}(data[item])\n')
out.write(
" else:\n"
" raise AttributeError('Respective class not found.')\n"
)
def _write_properties(output_file: TextIOWrapper,
property_files: list):
output_file.write(
"\n| Property file | Keys | Keys translated | Keys not translated | % translated |\n"
)
output_file.write(
"| ------------- | ---- | --------------- | ------------------- | ------------ |\n"
)
for file in property_files:
lines = self.__read_file_as_lines(file)
keys = Keys(lines)
num_keys = len(keys.translations_as_dict())
num_keys_missing_value = len(keys.empty_keys())
num_keys_translated = num_keys - num_keys_missing_value
output_file.write(
f"| [{os.path.basename(file)}]({URL_BASE}{os.path.basename(file)}) | "
f"{num_keys} | "
f"{num_keys_translated} | "
f"{num_keys_missing_value} | "
f"{_percentage(num_keys, num_keys_translated)} |\n")
def write(self, text):
if type(text) is unicode:
TextIOWrapper.write(self, text)
else:
TextIOWrapper.write(self, unicode(text, self.encoding))
def write(self, s):
if type(s) is unicode:
TextIOWrapper.write(self, s)
else:
TextIOWrapper.write(self, unicode(s, self.encoding))
def make_repo(self, revs):
dump = BytesIO()
dump_message(dump, (("SVN-fs-dump-format-version", "2"),))
dump_message(dump, (
("UUID", "00000000-0000-0000-0000-000000000000"),))
for (i, rev) in enumerate(revs, 1):
props = {
"svn:date": "1970-01-01T00:00:00.000000Z",
"svn:log": "",
}
props.update(rev.setdefault("props", dict()))
headers = (("Revision-number", format(i)),)
dump_message(dump, headers, props=props)
for node in rev.setdefault("nodes", {}):
headers = list()
for name in (
"action", "kind", "path",
"copyfrom-path", "copyfrom-rev",
):
value = node.get(name.replace("-", "_"))
if value is not None:
headers.append(("Node-" + name, format(value)))
dump_message(dump, headers,
props=node.get("props"), content=node.get("content"))
dump.seek(0)
log = TextIOWrapper(BytesIO(), "ascii")
log.write("<log>")
for [i, rev] in enumerate(reversed(revs)):
i = format(len(revs) - i)
log.write(f"<logentry revision={saxutils.quoteattr(i)}>")
author = rev["props"].get("svn:author")
if author is not None:
log.write(f"<author>{saxutils.escape(author)}</author>")
log.write("<date>1970-01-01T00:00:00.000000Z</date><paths>")
for node in rev["nodes"]:
action = {"add": "A", "change": "M", "delete": "D"}[node['action']]
log.write(f"<path action={saxutils.quoteattr(action)}>/{saxutils.escape(node['path'])}</path>")
log.write("</paths></logentry>")
log.write("</log>")
log.seek(0)
return (dump, patch("svnex.stdin", log))
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
OptionsClass = SmtLibOptions
def __init__(self, args, environment, logic, LOGICS=None, **options):
Solver.__init__(self,
environment,
logic=logic,
**options)
self.to = self.environment.typeso
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.declared_sorts = set()
self.solver = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE,
bufsize=-1)
# Give time to the process to start-up
time.sleep(0.01)
self.parser = SmtLibParser(interactive=True)
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
# Initialize solver
self.options(self)
self.set_logic(logic)
def set_option(self, name, value):
self._send_silent_command(SmtLibCommand(smtcmd.SET_OPTION,
[name, value]))
def set_logic(self, logic):
self._send_silent_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def _debug(self, msg, *format_args):
if self.options.debug_interaction:
print(msg % format_args)
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
self._debug("Sending: %s", cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
self._debug("Read: %s", res)
return res
def _get_value_answer(self):
"""Reads and parses an assignment from the STDOUT pipe"""
lst = self.parser.get_assignment_list(self.solver_stdout)
self._debug("Read: %s", lst)
return lst
def _declare_sort(self, sort):
cmd = SmtLibCommand(smtcmd.DECLARE_SORT, [sort])
self._send_silent_command(cmd)
self.declared_sorts.add(sort)
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_silent_command(cmd)
self.declared_vars.add(symbol)
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_silent_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
# This is needed because Z3 (and possibly other solvers) incorrectly
# recognize N * M * x as a non-linear term
formula = formula.simplify()
sorts = self.to.get_types(formula, custom_only=True)
for s in sorts:
if s not in self.declared_sorts:
self._declare_sort(s)
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_silent_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def _exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.stderr.close()
self.solver.terminate()
return
class SerialData(PanBase):
"""
Main serial class
"""
def __init__(self,
port=None,
baudrate=115200,
threaded=True,
name="serial_data"):
PanBase.__init__(self)
try:
self.ser = serial.Serial()
self.ser.port = port
self.ser.baudrate = baudrate
self.is_threaded = threaded
self.ser.bytesize = serial.EIGHTBITS
self.ser.parity = serial.PARITY_NONE
self.ser.stopbits = serial.STOPBITS_ONE
self.ser.timeout = 1.0
self.ser.xonxoff = False
self.ser.rtscts = False
self.ser.dsrdtr = False
self.ser.write_timeout = False
self.ser.open()
self.name = name
self.queue = deque([], 1)
self._is_listening = False
self.loop_delay = 2.
if self.is_threaded:
self._serial_io = TextIOWrapper(BufferedRWPair(
self.ser, self.ser),
newline='\r\n',
encoding='ascii',
line_buffering=True)
self.logger.debug("Using threads (multiprocessing)")
self.process = Thread(target=self.receiving_function,
args=(self.queue, ))
self.process.daemon = True
self.process.name = "PANOPTES_{}".format(name)
self.logger.debug(
'Serial connection set up to {}, sleeping for two seconds'.
format(self.name))
time.sleep(2)
self.logger.debug('SerialData created')
except Exception as err:
self.ser = None
self.logger.critical('Could not set up serial port {} {}'.format(
port, err))
@property
def is_connected(self):
"""
Checks the serial connection on the mount to determine if connection is open
"""
connected = False
if self.ser:
connected = self.ser.isOpen()
return connected
@property
def is_listening(self):
return self._is_listening
def start(self):
""" Starts the separate process """
self.logger.debug("Starting serial process: {}".format(
self.process.name))
self._is_listening = True
self.process.start()
def stop(self):
""" Starts the separate process """
self.logger.debug("Stopping serial process: {}".format(
self.process.name))
self._is_listening = False
self.process.join()
def connect(self):
""" Actually set up the Thread and connect to serial """
self.logger.debug('Serial connect called')
if not self.ser.isOpen():
try:
self.ser.open()
except serial.serialutil.SerialException as err:
raise BadSerialConnection(msg=err)
if not self.ser.isOpen():
raise BadSerialConnection(msg="Serial connection is not open")
self.logger.debug('Serial connection established to {}'.format(
self.name))
return self.ser.isOpen()
def disconnect(self):
"""Closes the serial connection
Returns:
bool: Indicates if closed or not
"""
self.ser.close()
return not self.is_connected
def receiving_function(self, q):
self.connect()
while self.is_listening:
try:
line = self.read()
ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime())
self.queue.append((ts, line))
except IOError as err:
self.logger.warning(
"Device is not sending messages. IOError: {}".format(err))
time.sleep(2)
except UnicodeDecodeError:
self.logger.warning("Unicode problem")
time.sleep(2)
except Exception:
self.logger.warning("Unknown problem")
time.sleep(self.loop_delay)
def write(self, value):
"""
For now just pass the value along to serial object
"""
assert self.ser
assert self.ser.isOpen()
# self.logger.debug('Serial write: {}'.format(value))
if self.is_threaded:
response = self._serial_io.write(value)
else:
response = self.ser.write(value.encode())
return response
def read(self):
"""
Reads value using readline
If no response is given, delay and then try to read again. Fail after 10 attempts
"""
assert self.ser
assert self.ser.isOpen()
retry_limit = 5
delay = 0.5
while True and retry_limit:
if self.is_threaded:
response_string = self._serial_io.readline()
else:
response_string = self.ser.readline(
self.ser.inWaiting()).decode()
if response_string > '':
break
time.sleep(delay)
retry_limit -= 1
# self.logger.debug('Serial read: {}'.format(response_string))
return response_string
def get_reading(self):
""" Get reading from the queue
Returns:
str: Item in queue
"""
try:
if self.is_threaded:
info = self.queue.pop()
else:
ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime())
info = (ts, self.read())
except IndexError:
raise IndexError
else:
return info
def clear_buffer(self):
""" Clear Response Buffer """
count = 0
while self.ser.inWaiting() > 0:
count += 1
self.ser.read(1)
# self.logger.debug('Cleared {} bytes from buffer'.format(count))
def __del__(self):
if self.ser:
self.ser.close()
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
def __init__(self, args, environment, logic, user_options=None,
LOGICS=None):
Solver.__init__(self,
environment,
logic=logic,
user_options=user_options)
# Flag used to debug interaction with the solver
self.dbg = False
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.solver = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.parser = SmtLibParser(interactive=True)
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
# Initialize solver
self.set_option(":print-success", "true")
if self.options.generate_models:
self.set_option(":produce-models", "true")
# Redirect diagnostic output to stdout
self.set_option(":diagnostic-output-channel", '"stdout"')
if self.options is not None:
for o,v in iteritems(self.options):
self.set_option(o,v)
self.set_logic(logic)
def set_option(self, name, value):
self._send_silent_command(SmtLibCommand(smtcmd.SET_OPTION,
[name, value]))
def set_logic(self, logic):
self._send_silent_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
if self.dbg: print("Sending: " + cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
if self.dbg: print("Read: " + str(res))
return res
def _get_value_answer(self):
"""Reads and parses an assignment from the STDOUT pipe"""
lst = self.parser.get_assignment_list(self.solver_stdout)
if self.dbg: print("Read: " + str(lst))
return lst
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_silent_command(cmd)
self.declared_vars.add(symbol)
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_silent_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_silent_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def _exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.stderr.close()
self.solver.terminate()
return
def do_stats_general(
instance: EdenInstance, out: io.TextIOWrapper = stdoutWrapper
) -> None:
with instance.get_thrift_client() as client:
stat_info = client.getStatInfo()
private_bytes = stats_print.format_size(stat_info.privateBytes)
resident_bytes = stats_print.format_size(stat_info.vmRSSBytes)
if stat_info.blobCacheStats is not None:
blob_cache_size = stats_print.format_size(
stat_info.blobCacheStats.totalSizeInBytes
)
blob_cache_entry_count = stat_info.blobCacheStats.entryCount
else:
blob_cache_size = None
blob_cache_entry_count = None
out.write(
textwrap.dedent(
f"""\
edenfs memory usage
▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔
private bytes: {private_bytes} ({resident_bytes} resident)
"""
)
)
if blob_cache_size is not None and blob_cache_entry_count is not None:
out.write(f"blob cache: {blob_cache_size} in {blob_cache_entry_count} blobs\n")
out.write(
textwrap.dedent(
f"""\
active mounts
▔▔▔▔▔▔▔▔▔▔▔▔▔
"""
)
)
inode_info = stat_info.mountPointInfo
for key in inode_info:
info = inode_info[key]
mount_path = os.fsdecode(key)
in_memory = info.loadedInodeCount
files = info.loadedFileCount
trees = info.loadedTreeCount
out.write(
textwrap.dedent(
f"""\
{mount_path}
- Inodes in memory: {in_memory} ({trees} trees, {files} files)
- Unloaded, tracked inodes: {info.unloadedInodeCount}
- Loaded and materialized inodes: {info.materializedInodeCount}
"""
)
)
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files.
.. seealso:: :func:`gevent.os.make_nonblocking`
"""
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
return getattr(self._fobj, name)
class FileObjectPosix:
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, list):
self.io.writelines(list)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def __iter__(self):
return self.io
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
.. caution::
This object is most effective wrapping files that can be used appropriately
with :func:`select.select` such as sockets and pipes.
In general, on most platforms, operations on regular files
(e.g., ``open('/etc/hosts')``) are considered non-blocking
already, even though they can take some time to complete as
data is copied to the kernel and flushed to disk (this time
is relatively bounded compared to sockets or pipes, though).
A :func:`~os.read` or :func:`~os.write` call on such a file
will still effectively block for some small period of time.
Therefore, wrapping this class around a regular file is
unlikely to make IO gevent-friendly: reading or writing large
amounts of data could still block the event loop.
If you'll be working with regular files and doing IO in large
chunks, you may consider using
:class:`~gevent.fileobject.FileObjectThread` or
:func:`~gevent.os.tp_read` and :func:`~gevent.os.tp_write` to bypass this
concern.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and
so can itself be passed to :func:`fcntl.fcntl`, setting the
:data:`os.O_NONBLOCK` flag will have no effect; however, removing
that flag will cause this object to no longer be cooperative.
.. versionchanged:: 1.1
Now uses the :mod:`io` package internally. Under Python 2, previously
used the undocumented class :class:`socket._fileobject`. This provides
better file-like semantics (and portability to Python 3).
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:keyword fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, IO will be done on text, otherwise bytes.
:keyword int bufsize: If given, the size of the buffer to use. The default
value means to use a platform-specific default, and a value of 0 is translated
to a value of 1. Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1 and mode not in 'rw': # pragma: no cover
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled.
# NOTE: This is preventing a mode like 'rwb' for binary random access;
# that code was never tested and was explicitly marked as "not used"
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0 or bufsize == 1:
bufsize = self.default_bufsize
elif bufsize == 0:
bufsize = 1
if mode == 'r':
self.io = BufferedReader(self.fileio, bufsize)
else:
assert mode == 'w'
self.io = BufferedWriter(self.fileio, bufsize)
#else: # QQQ: not used, not reachable
#
# self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is closed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only run once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
"""
.. versionadded:: 1.1b2
"""
return self.io.readable()
def writable(self):
"""
.. versionadded:: 1.1b2
"""
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files.
.. seealso:: :func:`gevent.os.make_nonblocking`
"""
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1:
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores in. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and
so can itself be passed to :func:`fcntl.fcntl`, setting the
:data:`os.O_NONBLOCK` flag will have no effect; likewise, removing
that flag will cause this object to no longer be cooperative.
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:keyword fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, IO will be done on text, otherwise bytes.
:keyword int bufsize: If given, the size of the buffer to use. The default
value means to use a platform-specific default, and a value of 0 is translated
to a value of 1. Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1 and mode not in 'rw': # pragma: no cover
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled.
# NOTE: This is preventing a mode like 'rwb' for binary random access;
# that code was never tested and was explicitly marked as "not used"
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0 or bufsize == 1:
bufsize = self.default_bufsize
elif bufsize == 0:
bufsize = 1
if mode == 'r':
self.io = BufferedReader(self.fileio, bufsize)
else:
assert mode == 'w'
self.io = BufferedWriter(self.fileio, bufsize)
#else: # QQQ: not used, not reachable
#
# self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only run once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)