def write_log_and_progress(n_epoch, n_processed, mode: str, dict_metrics, logger: io.TextIOWrapper,
output_log: bool, output_std: bool):
func_value_to_str = lambda v: f"{v:1.7f}" if isinstance(v,float) else f"{v}"
metrics = {
"epoch":n_epoch,
"processed":n_processed,
"mode":mode
}
metrics.update(dict_metrics)
if output_log:
sep = "\t"
## output log file
if os.stat(logger.name).st_size == 0:
s_header = sep.join(metrics.keys()) + "\n"
logger.write(s_header)
s_record = sep.join( map(func_value_to_str, metrics.values()) ) + "\n"
logger.write(s_record)
logger.flush()
## output metrics
if output_std:
prefix = metrics["mode"]
s_print = ", ".join( [f"{prefix}_{k}:{func_value_to_str(v)}" for k,v in metrics.items()] )
print(s_print)
def serialize(self, bib_data):
stream = BytesIO()
unicode_stream = TextIOWrapper(stream, 'UTF-8')
self.writer.write_stream(bib_data, unicode_stream if self.writer.unicode_io else stream)
unicode_stream.flush()
stream.seek(0)
return unicode_stream
def header_bytes(self, any_chunks):
if self.status_code not in (100, 204):
enc_hdr, enc_msg = self.encapsulated(any_chunks)
self.headers['Encapsulated'] = enc_hdr
else:
enc_msg = None
if any_chunks:
# http://www.measurement-factory.com/std/icap/#e1
raise ValueError("no encapsulation allowed")
bio = BytesIO()
sio = TextIOWrapper(bio, encoding='iso-8859-1')
status_line = u'{} {} {}\r\n'.format(self.protocol,
self.status_code,
self.reason)
sio.write(status_line)
for key, value in iteritems(self.headers):
if isinstance(value, list):
values = [text_type(v) for v in value]
line = u'{}: {}\r\n'.format(key, ', '.join(values))
else:
line = u'{}: {}\r\n'.format(key, value)
sio.write(line)
sio.write(u'\r\n')
sio.flush()
if enc_msg:
bio.write(enc_msg)
return bio.getvalue()
def get_curlbomb(self, args, script=None, override_defaults={}):
"""Prepare curlbomb to run in a thread
Assumes args has a '{script}' formatter in it to replace a temporary path with
If no '{script}' formatter is found, stdin is mocked through settings['stdin']
Returns tuple(curlbomb_thread, client_command)
"""
if type(script) == str:
script = bytes(script, "utf-8")
stdin = "{script}" not in args and script is not None
try:
log.info("Using stdin: {}".format(stdin))
if stdin:
s = TextIOWrapper(BytesIO(script))
override_defaults['stdin'] = s
else:
s = NamedTemporaryFile()
if script is not None:
s.write(script)
s.flush()
args = args.format(script=s.name)
args = shlex.split(args)
log.warn("starting curlbomb: {}".format(args))
settings = curlbomb.get_settings(args, override_defaults)
client_cmd = settings['get_curlbomb_command'](settings)
curlbomb_thread = CurlbombThread(settings)
curlbomb_thread.start()
return (curlbomb_thread,
client_cmd)
finally:
s.close()
def struncate(file: TextIOWrapper, amount: int):
"""
Truncate the first n bytes from the beginning of file
:param file
:param amount: amount of bytes to remove from start
:type file: TextIOWrapper
:type amount: int
"""
#Get file size
file.seek(0, 2)
file_size = file.tell()
#Go to the beginning of file
file_offset = amount
file.seek(0, 0)
bytes_to_write = file_size - amount
bytes_written = 0
while bytes_written < bytes_to_write:
#Move to offset + bytes_written
file.seek(file_offset + bytes_written, 0)
#Get bytes to rewrite
block_size = 1024
if bytes_to_write - bytes_written < block_size:
block_size = bytes_to_write - bytes_written
#Read block
block_data = file.read(block_size)
#Move to the beginning of file + bytes_written
file.seek(bytes_written, 0)
#Write block
bytes_written += file.write(block_data)
#Then truncate
file.flush() #Flush write first
file.seek(bytes_written)
file.truncate()
def readCmd(sio: io.TextIOWrapper, cmd: str):
dt_out = sio.write(cmd + "\n")
sio.flush()
if dt_out==0:
raise Exception("Unable to write to device.")
if sio.readline().strip()!=cmd:
raise Exception("Device is not responding.")
return sio.readline().strip()
def fake_stdin(data):
if PY2:
stdin = tempfile.TemporaryFile()
else:
stdin = TextIOWrapper(tempfile.TemporaryFile())
stdin.write(data)
stdin.flush()
stdin.seek(0)
return stdin
def fake_stdin(data):
if PY2:
stdin = tempfile.TemporaryFile()
else:
stdin = TextIOWrapper(tempfile.TemporaryFile())
stdin.write(data)
stdin.flush()
stdin.seek(0)
return stdin
def _test_input(self, plugin):
parser = find_plugin('pybtex.database.input', plugin)(encoding='UTF-8')
writer = find_plugin('pybtex.database.output', plugin)(encoding='UTF-8')
stream = BytesIO()
writer_stream = TextIOWrapper(stream, 'UTF-8') if writer.unicode_io else stream
parser_stream = TextIOWrapper(stream, 'UTF-8') if parser.unicode_io else stream
writer.write_stream(self.reference_data, writer_stream)
writer_stream.flush()
stream.seek(0)
parser.parse_stream(parser_stream)
loaded_data = parser.data
self.assertEqual(loaded_data, self.reference_data)
def extract(self, writer: TextIOWrapper = sys.stdout):
"""Extract the master source BibTex matching citation references from the LaTex
file(s) and write them to ``writer``.
:param writer: the BibTex entry data sink
"""
bwriter = BibTexWriter()
for bid, entry in self.entries.items():
logger.info(f'writing entry {bid}')
writer.write(bwriter._entry_to_bibtex(entry))
logger.debug(f'extracting: {bid}: <{entry}>')
writer.flush()
class MockStdout(object):
def __init__(self, name, as_bytes):
self.bytes_io = BytesIO()
object.__setattr__(self.bytes_io, 'name', name)
self.wrapper = TextIOWrapper(cast(BufferedIOBase, self.bytes_io))
self.wrapper.mode = 'w'
self.as_bytes = as_bytes
def getvalue(self):
self.wrapper.flush()
val = self.bytes_io.getvalue()
if not self.as_bytes:
val = val.decode()
return val
def test_simple():
serializer = JsonSerializer()
out = BytesIO()
wrapper = TextIOWrapper(out)
obj = {'a': 1}
serializer.dump(obj, wrapper)
wrapper.flush()
out_bytes = out.getvalue()
assert out_bytes == b'''{
"a": 1
}'''
in_ = TextIOWrapper(BytesIO(out_bytes))
read_obj = serializer.load(in_)
assert read_obj == obj
class CSVSerializerBase(TaskBase):
def __init__(self, outstream, entities_iterable, delimiter=','):
self.outstream = outstream
try:
self.is_binary = 'b' in self.outstream.mode
except AttributeError:
self.is_binary = True
if self.is_binary:
try:
self.outstream = TextIOWrapper(outstream, 'utf8', newline="")
except AttributeError:
# must be Py2
pass
self.writer = csv.writer(self.outstream, delimiter=delimiter)
self._entities_iterable = entities_iterable
def status_update(self, message):
status_logger.info(message)
def writerows(self, iterable):
self.writer.writerows(iterable)
def get_header(self):
raise NotImplementedError()
def convert_object(self, obj):
raise NotImplementedError()
@property
def header(self):
return self.get_header()
def run(self):
if self.header:
header = self.header
self.writer.writerow(header)
gen = (self.convert_object(entity)
for entity in self._entities_iterable)
for i, row in enumerate(gen):
if i % 100 == 0 and i != 0:
self.status_update("Handled %d Entities" % i)
self.outstream.flush()
self.writer.writerow(row)
def _text_io_wrapper(stream, mode, encoding, errors, newline):
"""Wrap a binary stream to Text stream.
Args:
stream (file-like object): binary stream.
mode (str): Open mode.
encoding (str): Stream encoding.
errors (str): Decoding error handling.
newline (str): Universal newlines
"""
if "t" in mode and not hasattr(stream, "encoding"):
text_stream = TextIOWrapper(stream,
encoding=encoding,
errors=errors,
newline=newline)
yield text_stream
text_stream.flush()
else:
yield stream
def _test_input(self, plugin):
parser = find_plugin('pybtex.database.input', plugin)(encoding='UTF-8')
writer = find_plugin('pybtex.database.output',
plugin)(encoding='UTF-8')
stream = BytesIO()
writer_stream = TextIOWrapper(stream,
'UTF-8') if writer.unicode_io else stream
parser_stream = TextIOWrapper(stream,
'UTF-8') if parser.unicode_io else stream
writer.write_stream(self.reference_data, writer_stream)
writer_stream.flush()
stream.seek(0)
parser.parse_stream(parser_stream)
loaded_data = parser.data
self.assertEqual(loaded_data, self.reference_data)
self.assertEqual(pickle.loads(pickle.dumps(loaded_data, 0)),
self.reference_data)
self.assertEqual(pickle.loads(pickle.dumps(loaded_data, 1)),
self.reference_data)
self.assertEqual(pickle.loads(pickle.dumps(loaded_data, 2)),
self.reference_data)
def print_table(table, max_width, print_color=False): from sys import stdout from io import TextIOWrapper wrapped_stdout = TextIOWrapper(stdout.buffer, encoding='utf-8', line_buffering=True) def color(c, s): esc = chr(27) return esc + "[" + str(c) + "m" + str(s) + esc + "[0m" if max_width is None: max_width = [10] * len(table[0]) for line in table: for i, item in zip(range(100000), line): if print_color and item == 'T': print(color(92, str(item).rjust(max_width[i])), end=" | ", file=wrapped_stdout) elif print_color and item == 'F': print(color(91, str(item).rjust(max_width[i])), end=" | ", file=wrapped_stdout) else: print(str(item).rjust(max_width[i]), end=" | ", file=wrapped_stdout) print(file=wrapped_stdout) wrapped_stdout.flush()
def _text_io_wrapper(stream, mode, encoding, errors, newline):
"""Wrap a binary stream to Text stream.
Args:
stream (file-like object): binary stream.
mode (str): Open mode.
encoding (str): Stream encoding.
errors (str): Decoding error handling.
newline (str): Universal newlines
"""
# Text mode, if not already a text stream
# That has the "encoding" attribute
if "t" in mode and not hasattr(stream, 'encoding'):
text_stream = TextIOWrapper(stream,
encoding=encoding,
errors=errors,
newline=newline)
yield text_stream
text_stream.flush()
# Binary mode (Or already text stream)
else:
yield stream
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
OptionsClass = SmtLibOptions
def __init__(self, args, environment, logic, LOGICS=None, **options):
Solver.__init__(self,
environment,
logic=logic,
**options)
self.to = self.environment.typeso
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.declared_sorts = set()
self.solver = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE,
bufsize=-1)
# Give time to the process to start-up
time.sleep(0.01)
self.parser = SmtLibParser(interactive=True)
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
# Initialize solver
self.options(self)
self.set_logic(logic)
def set_option(self, name, value):
self._send_silent_command(SmtLibCommand(smtcmd.SET_OPTION,
[name, value]))
def set_logic(self, logic):
self._send_silent_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def _debug(self, msg, *format_args):
if self.options.debug_interaction:
print(msg % format_args)
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
self._debug("Sending: %s", cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
self._debug("Read: %s", res)
return res
def _get_value_answer(self):
"""Reads and parses an assignment from the STDOUT pipe"""
lst = self.parser.get_assignment_list(self.solver_stdout)
self._debug("Read: %s", lst)
return lst
def _declare_sort(self, sort):
cmd = SmtLibCommand(smtcmd.DECLARE_SORT, [sort])
self._send_silent_command(cmd)
self.declared_sorts.add(sort)
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_silent_command(cmd)
self.declared_vars.add(symbol)
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_silent_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
# This is needed because Z3 (and possibly other solvers) incorrectly
# recognize N * M * x as a non-linear term
formula = formula.simplify()
sorts = self.to.get_types(formula, custom_only=True)
for s in sorts:
if s not in self.declared_sorts:
self._declare_sort(s)
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_silent_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def _exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.stderr.close()
self.solver.terminate()
return
class YamlDatafileSinkHandler(AsciiScanfileSinkHandler):
"""Write the STRESS-SPEC specific yaml-formatted scan data file."""
filetype = 'MLZ.StressSpec.2.0 / proposal 0.2'
accept_file_images_only = False
max_yaml_width = 120
# def _readdev(self, devname, mapper=lambda x: x):
# try:
# return mapper(session.getDevice(devname).read())
# except NicosError:
# return None
# def _devpar(self, devname, parname, mapper=lambda x: x):
# try:
# return mapper(getattr(session.getDevice(devname), parname))
# except NicosError:
# return None
def _dict(self):
return AutoDefaultODict()
# def _flowlist(self, *args):
# return None
# # return quickyaml.flowlist(*args)
objects = [
'time', 'angle', 'clearance', 'collimator_fhwm', 'position',
'wavelength'
]
_millimeter = 'millimeter'
units = [
'second', 'degree', _millimeter, _millimeter, _millimeter, 'angstrom'
]
def _fill_format(self, formats):
formats['identifier'] = self.filetype
_units = formats['units'] = self._dict()
for obj, unit in zip(self.objects, self.units):
_units[obj] = unit
def _fill_position(self, position, value, offset, precision):
position['value'] = value
position['offset_coder'] = offset
position['precision'] = precision
def _fill_user(self, user, name, roles):
_user = re.split(r'(<)?(\w+(?:[\.-]\w+)+@\w+(?:\.\w+)+)(?(1)>)', name)
user['name'] = _user[0].strip()
user['email'] = _user[2] if len(_user) > 2 else ''
user['roles'] = roles
def _write_tolerances(self, valuelist):
sample = self._data['measurement']['sample']
setup = self._data['measurement']['setup']
for device, _key, value in valuelist:
if device in ['xt', 'yt', 'zt']:
p = sample['position'][device]
p['precision'] = value
elif device in ['tths', 'omgs', 'chis', 'phis']:
p = sample['orientation'][device]
p['precision'] = value
elif device == 'slitm':
p = setup['slit_m']['horizontal_clearance']
p['precision'] = value[0]
p = setup['slit_m']['vertical_clearance']
p['precision'] = value[1]
elif device == 'slitp':
p = setup['slit_p']['horizontal_clearance']
p['precision'] = value[0]
p = setup['slit_p']['vertical_clearance']
p['precision'] = value[1]
elif device == 'slite':
p = setup['slit_e']['clearance']
p['precision'] = value
elif device in ['omgm', 'tthm']:
p = setup['monochromator'][device]
p['precision'] = value
else:
self.log.debug('tolerance: %s.%s: %s', device, _key, value)
def _write_offsets(self, valuelist):
sample = self._data['measurement']['sample']
setup = self._data['measurement']['setup']
for device, _key, value in valuelist:
if device in ['xt', 'yt', 'zt']:
p = sample['position'][device]
p['offset_coder'] = value
elif device in ['tths', 'omgs', 'chis', 'phis']:
p = sample['orientation'][device]
p['offset_coder'] = value
elif device == 'slitm':
p = setup['slit_m']['horizontal_clearance']
p['offset_coder'] = value[0]
p = setup['slit_m']['vertical_clearance']
p['offset_coder'] = value[1]
elif device == 'slitp':
p = setup['slit_p']['horizontal_clearance']
p['offset_coder'] = value[0]
p = setup['slit_p']['vertical_clearance']
p['offset_coder'] = value[1]
elif device == 'slite':
p = setup['slit_e']['clearance']
p['offset_coder'] = value
elif device in ['omgm', 'tthm']:
p = setup['monochromator'][device]
p['offset_coder'] = value
else:
self.log.debug('offset: %s.%s: %s', device, _key, value)
def _write_sample(self, valuelist):
# self.log.info('Sample: %r', valuelist)
sample = self._data['measurement']['sample']
for _device, key, value in valuelist:
if key == 'samplename':
sample['description']['name'] = value
def _write_experiment(self, valuelist):
# self.log.info('Experiment: %r', valuelist)
experiment = self._data['experiment']
for _device, key, value in valuelist:
if key in ['proposal', 'title', 'remark']:
experiment[key] = value
elif key == 'users':
authors = experiment['authors']
author = self._dict()
self._fill_user(author, value, ['principal_investigator'])
authors.append(author)
elif key == 'localcontact':
authors = experiment['authors']
author = self._dict()
self._fill_user(author, value, ['local_contact'])
authors.append(author)
def _write_status(self, valuelist):
# self.log.info('Status: %r', valuelist)
# for device, _key, value in valuelist:
# self.log.info('%s.%s: %s', device, _key, value)
pass
def _write_instrument(self, valuelist):
# self.log.info('Instrument: %r', valuelist)
instrument = self._data['instrument']
for device, key, value in valuelist:
if device not in ['demo', 'DEMO']:
if key in ['facility', 'website']:
instrument[key] = value.encode()
elif key == 'instrument':
instrument['name'] = value
elif key == 'operators':
instrument[key] = []
for operator in value:
instrument[key].append(operator.encode())
elif key == 'doi':
instrument['references'] = []
instrument['references'].append(value.encode())
def _write_limits(self, valuelist):
# self.log.info('Limits: %r', valuelist)
# for device, _key, value in valuelist:
# self.log.info('%s.%s: %s', device, _key, value)
pass
def _write_general(self, valuelist):
sample = self._data['measurement']['sample']
setup = self._data['measurement']['setup']
for device, _key, value in valuelist:
if device in ['xt', 'yt', 'zt']:
p = sample['position'][device]
p['value'] = value
elif device in ['tths', 'omgs', 'chis', 'phis']:
p = sample['orientation'][device]
p['value'] = value
elif device == 'slitm':
p = setup['slit_m']['horizontal_clearance']
p['value'] = value[0]
p = setup['slit_m']['vertical_clearance']
p['value'] = value[1]
elif device == 'slitp':
p = setup['slit_p']['horizontal_clearance']
p['value'] = value[0]
p = setup['slit_p']['vertical_clearance']
p['value'] = value[1]
elif device == 'slite':
p = setup['slit_e']['clearance']
p['value'] = value
elif device == 'transm':
setup['monochromator']['crystal'] = value
elif device == 'wav':
setup['monochromator']['incident_wavelength'] = value
elif device in ['omgm', 'tthm']:
p = setup['monochromator'][device]
p['value'] = value
else:
self.log.debug('general: %s.%s: %s', device, _key, value)
def __init__(self, sink, dataset, detector):
AsciiScanfileSinkHandler.__init__(self, sink, dataset, detector)
self._wrote_header = False
self._file = None
self._fname = None
self._template = sink.filenametemplate
self._data = None
self._scan_type = None
def prepare(self):
self.log.debug('prepare: %r', self.dataset.settype)
if self.dataset.settype == POINT:
return
if self._data is None:
self._data = self._dict()
self._scan_type = 'SGEN1'
self._number = self.manager.assignCounter(self.dataset)
fp = self.manager.createDataFile(self.dataset, self._template[0])
self._fname = fp.shortpath
self._filepath = fp.filepath
if not quickyaml:
self._file = TextIOWrapper(fp, encoding='utf-8')
else:
fp.close()
self._file = io.FileIO(self._filepath, 'w')
self._data['instrument'] = self._dict()
self._data['format'] = self._dict()
self._data['experiment'] = self._dict()
self._data['measurement'] = self._dict()
self._fill_format(self._data['format'])
def begin(self):
if self.dataset.settype == POINT:
return
ds = self.dataset
scaninfo = ds.info.split('-')[-1].strip()
if scaninfo.startswith('contscan'):
self._scan_type = 'SGEN2'
elif scaninfo.startswith('scan'):
self._scan_type = 'SGEN1'
else:
self._scan_type = 'SGEN1'
instrument = self._data['instrument']
instrument['name'] = ''
instrument['operators'] = []
instrument['facility'] = ''
instrument['website'] = ''
instrument['references'] = []
measurement = self._data['measurement']
measurement['unique_identifier'] = self.dataset.uid.urn
experiment = self._data['experiment']
experiment['number'] = measurement['number'] = self.dataset.counter
experiment['proposal'] = ''
experiment['title'] = ''
experiment['authors'] = []
history = measurement['history'] = self._dict()
history['started'] = time.strftime(TIMEFMT)
history['stopped'] = time.strftime(TIMEFMT)
history['scan'] = self.dataset.info
sample = measurement['sample'] = self._dict()
sample['description'] = self._dict()
sample['description']['name'] = ''
sample['temperature'] = self._dict()
sample['orientation'] = self._dict()
for dev in ['tths', 'omgs', 'chis', 'phis']:
sample['orientation'][dev] = self._dict()
self._fill_position(sample['orientation'][dev], 0, 0, 0)
sample['position'] = self._dict()
for dev in ['xt', 'yt', 'zt']:
sample['position']['xt'] = self._dict()
self._fill_position(sample['position']['xt'], 0, 0, 0)
setup = measurement['setup'] = self._dict()
setup['collimator_1'] = "15'"
setup['slit_m'] = self._dict()
for x in ['horizontal_clearance', 'vertical_clearance']:
p = setup['slit_m'][x] = self._dict()
self._fill_position(p, 0, 0, 0)
p = setup['monochromator'] = self._dict()
p['crystal'] = 'Si'
for dev in ['omgm', 'tthm']:
p[dev] = self._dict()
self._fill_position(p[dev], 0, 0, 0)
p['angle'] = 0
p['incident_wavelength'] = 0
setup['slit_e'] = self._dict()
p = setup['slit_e']['clearance'] = self._dict()
self._fill_position(p, 0, 0, 0)
setup['slit_p'] = self._dict()
for x in [
'horizontal_clearance', 'horizontal_translation',
'vertical_clearance', 'vertical_translation'
]:
p = setup['slit_p'][x] = self._dict()
self._fill_position(p, 0, 0, 0)
setup['collimator_2'] = self._dict()
p = setup['collimator_2']['fhwm'] = 5
measurement['scan'] = []
self._wrote_headers = False
self._detvalues = None
def _float(self, value):
return float(eval(value))
def _integer(self, value):
return int(eval(value))
def _fill_header(self):
bycategory = {}
for (dev, key), (_v, v, _, cat) in self.dataset.metainfo.items():
if cat:
if key == 'operators': # don't use the formatted list
bycategory.setdefault(cat, []).append((dev, key, _v))
else:
bycategory.setdefault(cat, []).append((
dev,
key,
v,
))
if 'experiment' in bycategory:
self._write_experiment(bycategory['experiment'])
if 'sample' in bycategory:
self._write_sample(bycategory['sample'])
if 'instrument' in bycategory:
self._write_instrument(bycategory['instrument'])
if 'offsets' in bycategory:
self._write_offsets(bycategory['offsets'])
if 'limits' in bycategory:
self._write_limits(bycategory['limits'])
if 'precisions' in bycategory:
self._write_tolerances(bycategory['precisions'])
if 'status' in bycategory:
self._write_status(bycategory['status'])
if 'general' in bycategory:
self._write_general(bycategory['general'])
def putMetainfo(self, metainfo):
self.log.debug('ADD META INFO %r', metainfo)
def putResults(self, quality, results):
"""Called when the point dataset main results are updated.
The *quality* is one of the constants defined in the module:
* LIVE is for intermediate data that should not be written to files.
* INTERMEDIATE is for intermediate data that should be written.
* FINAL is for final data.
* INTERRUPTED is for data that has been read after the counting was
interrupted by an exception.
Argument *results* contains the new results. ``dataset.results``
contains all results so far.
"""
self.log.debug('%s', quality)
if quality != FINAL and self.dataset.settype != POINT:
return
def addSubset(self, point):
if not self._wrote_header:
self._fill_header()
self._wrote_header = True
if point.settype != POINT:
self.log.info('add subset: %s', point.settype)
return
self.log.debug('%r - %r', self.dataset.detvalueinfo,
point.detvaluelist)
# the image data are hopefully always at this place
try:
if not self.sink.detectors:
det = session.experiment.detectors[0]
else:
det = session.getDevice(self.sink.detectors[0])
self._detvalues = point.results[det.name][1][0]
except IndexError:
# create empty data set
self.log.error('Could not get the image data from %s', det.name)
self._detvalues = np.zeros((256, 256))
scanpoint = self._dict()
scanparams = self._dict()
if point.devvaluelist:
scanparams[point.devvalueinfo[0].name] = self._float(
'%.2f' % point.devvaluelist[0])
scanpoint['scan_parameters'] = scanparams
for (info, val) in zip(self.dataset.detvalueinfo, point.detvaluelist):
if info.type == 'time':
scanpoint['time'] = self._float('%.2f' % val)
elif info.type == 'counter':
scanpoint['sum'] = self._integer('%d' % val)
# elif info.type == 'filename':
# scanpoint['image'] = '%s' % val
elif info.type == 'monitor':
scanpoint['monitor'] = self._integer('%d' % val)
else:
self.log.info('%s %s', info.name, info.type)
scanpoint['image'] = [] if self._detvalues is None else self._detvalues
self._data['measurement']['scan'].append(scanpoint)
def _dump(self):
if quickyaml:
quickyaml.Dumper(width=self.max_yaml_width,
array_handling=quickyaml.ARRAY_AS_SEQ).dump(
self._data, self._file)
elif yaml:
yaml.dump(self._data,
self._file,
allow_unicode=True,
canonical=False,
default_flow_style=False,
indent=4)
def end(self):
if self.dataset.settype == POINT:
return
if self._data:
history = self._data['measurement']['history']
history['stopped'] = time.strftime(TIMEFMT)
self._dump()
if self._file:
self._file.flush()
self._file.close()
self._file = None
self._data = None
class AsciiScanfileSinkHandler(DataSinkHandler):
def __init__(self, sink, dataset, detector):
DataSinkHandler.__init__(self, sink, dataset, detector)
self._file = None
self._fname = None
self._semicolon = sink.semicolon
self._commentc = sink.commentchar
self._template = sink.filenametemplate
def prepare(self):
self.manager.assignCounter(self.dataset)
fp = self.manager.createDataFile(self.dataset, self._template,
self.sink.subdir)
self._fname = fp.shortpath
self._filepath = fp.filepath
self._file = TextIOWrapper(fp, encoding='utf-8')
def _write_section(self, section):
self._file.write('%s %s\n' % (self._commentc * 3, section))
def _write_comment(self, comment):
self._file.write('%s %s\n' % (self._commentc, comment))
def _write_header(self, ds, nfiles):
self._write_section('NICOS data file, created at %s' %
strftime(TIMEFMT))
for name, value in [('number', self.dataset.counter),
('filename', self._fname),
('filepath', self._filepath), ('info', ds.info)]:
self._write_comment('%25s : %s' % (name, value))
bycategory = {}
for (devname, key), (_, val, unit, category) in ds.metainfo.items():
if category:
bycategory.setdefault(category, []).append(
('%s_%s' % (devname, key), (val + ' ' + unit).strip()))
for category, catname in INFO_CATEGORIES:
if category not in bycategory:
continue
self._write_section(catname)
for key, value in bycategory[category]:
self._write_comment('%25s : %s' % (key, value))
self._file.flush()
# we write every data value as a column except for arrays
xnames = [v.name for v in ds.devvalueinfo + ds.envvalueinfo]
xunits = [v.unit for v in ds.devvalueinfo + ds.envvalueinfo]
ynames = [v.name for v in ds.detvalueinfo]
yunits = [v.unit for v in ds.detvalueinfo]
# to be written later (after info)
file_names = ['file%d' % i for i in range(1, nfiles + 1)]
if self._semicolon:
self._colnames = xnames + [';'] + ynames + file_names
# make sure there are no empty units
self._colunits = [
u or '-' for u in xunits + [';'] + yunits + [''] * nfiles
]
else:
self._colnames = xnames + ynames + file_names
self._colunits = [
u or '-' for u in xunits + yunits + [''] * nfiles
]
self._file.flush()
def addSubset(self, point):
if point.settype != POINT:
return
ds = self.dataset
if point.number == 1:
self._write_header(ds, len(point.filenames))
self._write_section('Scan data')
self._write_comment('\t'.join(self._colnames))
self._write_comment('\t'.join(self._colunits))
values = [safe_format(info.fmtstr, val) for (info, val) in
zip(self.dataset.devvalueinfo, point.devvaluelist)] + \
[safe_format(info.fmtstr, val) for (info, val) in
zip(self.dataset.envvalueinfo, point.envvaluelist)]
if self._semicolon:
values += [';']
values += [
safe_format(info.fmtstr, val)
for (info,
val) in zip(self.dataset.detvalueinfo, point.detvaluelist)
]
values += self.getFilenames(point)
self._file.write('\t'.join(values) + '\n')
self._file.flush()
def getFilenames(self, point):
return point.filenames
def end(self):
if self._fname:
self._write_section('End of NICOS data file %s' % self._fname)
self._file.close()
self._file = None
def render(self):
"""
Evaluate ``bucket`` information and enrich further before
executing the designated output format rendering handler.
"""
# Variable aliases
bucket = self.bucket
df = self.dataframe
# Read designated suffix from transformation data
suffix = bucket.tdata.suffix.lower()
# Update "time_begin" and "time_end" fields to be in ISO 8601 format
tdata = deepcopy(bucket.tdata)
tdata.update({
'time_begin': slugify_datettime(bucket.tdata.time_begin),
'time_end': slugify_datettime(bucket.tdata.time_end),
})
# Compute some names and titles and pluck into ``bucket``
bucket.title = Munch(
compact=u'{gateway}_{node}'.format(**dict(tdata)).replace(
'-', '_'),
short=u'{network}_{gateway}_{node}'.format(**dict(tdata)).replace(
'-', '_'),
full=u'{network}_{gateway}_{node}_{time_begin}-{time_end}'.format(
**dict(tdata)).replace('-', '_'),
human=u'Address: {network} » {gateway} » {node}'.format(
**dict(tdata)),
)
# Buffer object most output handlers write their content to.
buffer = BytesIO()
charset = None
# Dispatch to appropriate output handler.
# TODO: XML, SQL, GBQ (Google BigQuery table), MsgPack?, Thrift?
# TODO: jsonline using Odo, see http://odo.pydata.org/en/latest/json.html
# TODO: Refactor "if response: return response" cruft
# TODO: Refactor dispatching logic to improve suffix comparison redundancy with UniversalTabularExporter
if suffix in ['csv', 'txt']:
# http://pandas.pydata.org/pandas-docs/stable/io.html#io-store-in-csv
wrapper = TextIOWrapper(buffer)
df.to_csv(wrapper,
header=True,
index=False,
encoding='utf-8',
date_format='%Y-%m-%dT%H:%M:%S.%fZ')
# Make sure that TextIOWrapper writes the content to buffer.
wrapper.flush()
charset = 'utf-8'
elif suffix == 'tsv':
wrapper = TextIOWrapper(buffer)
df.to_csv(wrapper,
header=True,
index=False,
encoding='utf-8',
date_format='%Y-%m-%dT%H:%M:%S.%fZ',
sep='\t')
# Make sure that TextIOWrapper writes the content to buffer.
wrapper.flush()
charset = 'utf-8'
elif suffix == 'json':
# http://pandas.pydata.org/pandas-docs/stable/io.html#io-json-writer
wrapper = TextIOWrapper(buffer)
df.to_json(wrapper, orient='records', date_format='iso')
# Make sure that TextIOWrapper writes the content to buffer.
wrapper.flush()
charset = 'utf-8'
elif suffix == 'html':
# http://pandas.pydata.org/pandas-docs/stable/io.html#io-html
wrapper = TextIOWrapper(buffer)
buffer.write(b'<html>\n')
#buffer.write(b'<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n')
df.to_html(wrapper, index=False, justify='center')
# Make sure that TextIOWrapper writes the content to buffer.
wrapper.flush()
buffer.write(b'\n</html>')
charset = 'utf-8'
elif suffix == 'xlsx':
exporter = UniversalTabularExporter(bucket, dataframe=df)
response = exporter.render(suffix, buffer=buffer)
if response:
return response
elif suffix in ['hdf', 'hdf5', 'h5']:
exporter = UniversalTabularExporter(bucket, dataframe=df)
response = exporter.render(suffix, buffer=buffer)
if response:
return response
elif suffix in ['nc', 'cdf']:
exporter = UniversalTabularExporter(bucket, dataframe=df)
response = exporter.render(suffix, buffer=buffer)
if response:
return response
elif suffix in ['dy', 'dygraphs']:
plotter = UniversalPlotter(bucket, dataframe=df)
response = plotter.render('html', kind='dygraphs')
if response:
return response
elif suffix in ['dt', 'datatables']:
exporter = UniversalTabularExporter(bucket, dataframe=df)
response = exporter.render(suffix, buffer=buffer)
if response:
return response
elif suffix in ['bk', 'bokeh']:
plotter = UniversalPlotter(bucket, dataframe=df)
response = plotter.render('html', kind='bokeh')
if response:
return response
elif suffix == 'vega.json':
plotter = UniversalPlotter(bucket, dataframe=df)
response = plotter.render('json', kind='vega')
if response:
return response
elif suffix == 'vega':
plotter = UniversalPlotter(bucket, dataframe=df)
response = plotter.render('html', kind='vega')
if response:
return response
elif suffix in ['png']:
plotter = UniversalPlotter(bucket, dataframe=df)
response = plotter.render('png', buffer=buffer)
if response:
return response
else:
error_message = u'# Unknown data format "{suffix}"'.format(
suffix=suffix)
bucket.request.setResponseCode(http.BAD_REQUEST)
bucket.request.setHeader('Content-Type',
'text/plain; charset=utf-8')
return error_message.encode('utf-8')
# Get hold of buffer content
payload = buffer.getvalue()
# Compute filename offered to browser
filename = '{name}.{suffix}'.format(name=bucket.title.full,
suffix=suffix)
mimetype, encoding = mimetypes.guess_type(filename, strict=False)
log.info(
u'Fetching data succeeded, filename: {filename}, Format: {mimetype}',
filename=filename,
mimetype=mimetype)
# Set "Content-Type" header
if mimetype:
content_type = mimetype
if charset:
content_type += '; charset=' + charset
bucket.request.setHeader('Content-Type', content_type)
# Set "Content-Disposition" header
disposition = 'attachment'
if mimetype in [
'text/plain', 'text/csv', 'text/html', 'application/json',
'image/png'
]:
disposition = 'inline'
bucket.request.setHeader(
'Content-Disposition', '{disposition}; filename={filename}'.format(
disposition=disposition, filename=filename))
return payload
class SerialComPort:
def read_configuration(self, data_file="PORT_COM.txt"):
self.serial_parameters = DataImport(data_file, "dict")
def serial_connection(self, ):
self.ser = Serial()
self.ser.baudrate = self.parameter("baudrate", "int",
self.serial_parameters())
self.ser.port = self.parameter("port", "str", self.serial_parameters())
self.ser.bytesize = self.parameter("bytesize", "int",
self.serial_parameters())
self.ser.parity = self.parameter("parity", "str",
self.serial_parameters())
self.ser.stopbits = self.parameter("stopbits", "int",
self.serial_parameters())
self.ser.xonxoff = self.parameter("xonxoff", "bool",
self.serial_parameters())
self.ser.rtscts = self.parameter("rtscts", "bool",
self.serial_parameters())
self.ser.write_timeout = self.parameter("dsrdtr", "None",
self.serial_parameters())
self.ser.dsrdtr = self.parameter("dsrdtr", "bool",
self.serial_parameters())
self.ser.inter_byte_timeout = self.parameter("dsrdtr", "None",
self.serial_parameters())
self.ser.timeout = self.parameter("timeout", "float",
self.serial_parameters())
self.ser.exclusive = self.parameter("dsrdtr", "None",
self.serial_parameters())
self.ser.open()
self.sio = TextIOWrapper(BufferedRWPair(self.ser, self.ser))
def read_data(self):
if self.ser.in_waiting > 0:
line = self.sio.readline()
# if line != "":
# print(line, end = "\n")
return line
else:
return ""
def send_data(self, data):
if data != "":
self.sio.write(data)
self.sio.flush(
) # it is buffering. required to get the data out *now*
def is_open(self):
return self.serialPort.is_open
def close_port(self):
self.serialPort.close()
def parameter(self, name, type, serial_parameters):
if type == "bool":
if serial_parameters[name] == "False":
return False
elif serial_parameters[name] == "True":
return True
elif serial_parameters[name] == "int":
return int(serial_parameters[name])
elif type == "str":
return serial_parameters[name]
elif type == "int":
return int(serial_parameters[name])
elif type == "float":
return float(serial_parameters[name])
elif type == "None":
return None
else:
return "wrong data"
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files.
.. seealso:: :func:`gevent.os.make_nonblocking`
"""
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
def __init__(self,
args,
environment,
logic,
user_options=None,
LOGICS=None):
Solver.__init__(self,
environment,
logic=logic,
user_options=user_options)
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.solver = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.parser = SmtLibParser()
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
self.dbg = False
# Initialize solver
self._send_command(
SmtLibCommand(smtcmd.SET_OPTION, [":print-success", "false"]))
self._send_command(
SmtLibCommand(smtcmd.SET_OPTION, [":produce-models", "true"]))
if self.options is not None:
for o, v in iteritems(self.options):
self._send_command(SmtLibCommand(smtcmd.SET_OPTION, [o, v]))
self._send_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def get_default_options(self, logic=None, user_options=None):
res = {}
for o, v in iteritems(user_options):
if o not in ["generate_models", "unsat_cores_mode"]:
res[o] = v
return res
def _send_command(self, cmd):
if self.dbg: print("Sending: " + cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _get_answer(self):
res = self.solver_stdout.readline().strip()
if self.dbg: print("Read: " + str(res))
return res
def _get_value_answer(self):
lst = self.parser.get_assignment_list(self.solver_stdout)
if self.dbg: print("Read: " + str(lst))
return lst
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_command(cmd)
self.declared_vars.add(symbol)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.terminate()
return
class SetAwareSmtLibSolver:
"""
Our own subclass of PySMT's SmtLibSolver with support for sets in models.
"""
def __init__(self, solver_path, logic, debug_interaction=False):
self.solver = Popen(solver_path,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
bufsize=-1)
self.logic = logic
self.debug_interaction = debug_interaction
# Give time to the process to start-up
time.sleep(0.01)
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
self._debug("Sending: %s", cmd)
self.solver_stdin.write(cmd)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
self._debug("Read: %s", res)
return res
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def _debug(self, msg, *format_args):
if self.debug_interaction:
print(msg % format_args)
def add_smtlib_command(self, command):
self._send_silent_command(command)
def solve(self):
self._send_command("(check-sat)")
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def get_value(self, term):
command = f'(get-value ({term}))'
self._send_command(command)
res = self.solver_stdout.readline().strip()
# Let's parse the output with a quick and nice hack, since it's going to be always a boolean expression
parse = res.replace(term, "").strip('() ')
if parse == "false":
return False
elif parse == "true":
return True
else:
raise RuntimeError(f"Unknown solver response: '{res}'")
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
.. caution::
This object is most effective wrapping files that can be used appropriately
with :func:`select.select` such as sockets and pipes.
In general, on most platforms, operations on regular files
(e.g., ``open('/etc/hosts')``) are considered non-blocking
already, even though they can take some time to complete as
data is copied to the kernel and flushed to disk (this time
is relatively bounded compared to sockets or pipes, though).
A :func:`~os.read` or :func:`~os.write` call on such a file
will still effectively block for some small period of time.
Therefore, wrapping this class around a regular file is
unlikely to make IO gevent-friendly: reading or writing large
amounts of data could still block the event loop.
If you'll be working with regular files and doing IO in large
chunks, you may consider using
:class:`~gevent.fileobject.FileObjectThread` or
:func:`~gevent.os.tp_read` and :func:`~gevent.os.tp_write` to bypass this
concern.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and
so can itself be passed to :func:`fcntl.fcntl`, setting the
:data:`os.O_NONBLOCK` flag will have no effect; likewise, removing
that flag will cause this object to no longer be cooperative.
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:keyword fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, IO will be done on text, otherwise bytes.
:keyword int bufsize: If given, the size of the buffer to use. The default
value means to use a platform-specific default, and a value of 0 is translated
to a value of 1. Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1 and mode not in 'rw': # pragma: no cover
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled.
# NOTE: This is preventing a mode like 'rwb' for binary random access;
# that code was never tested and was explicitly marked as "not used"
raise ValueError('mode can only be [rb, rU, wb], not %r' %
(orig_mode, ))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0 or bufsize == 1:
bufsize = self.default_bufsize
elif bufsize == 0:
bufsize = 1
if mode == 'r':
self.io = BufferedReader(self.fileio, bufsize)
else:
assert mode == 'w'
self.io = BufferedWriter(self.fileio, bufsize)
#else: # QQQ: not used, not reachable
#
# self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is closed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only run once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class EventWriter(object):
"""``EventWriter`` writes events and error messages to Splunk from a modular input.
Its two important methods are ``writeEvent``, which takes an ``Event`` object,
and ``log``, which takes a severity and an error message.
"""
# Severities that Splunk understands for log messages from modular inputs.
# Do not change these
DEBUG = "DEBUG"
INFO = "INFO"
WARN = "WARN"
ERROR = "ERROR"
FATAL = "FATAL"
def __init__(self, output=sys.stdout, error=sys.stderr):
"""
:param output: Where to write the output; defaults to sys.stdout.
:param error: Where to write any errors; defaults to sys.stderr.
"""
if isinstance(output, TextIOBase):
self._out = output
else:
self._out = TextIOWrapper(output)
if isinstance(error, TextIOBase):
self._err = error
else:
self._err = TextIOWrapper(error)
# has the opening <stream> tag been written yet?
self.header_written = False
def write_event(self, event):
"""Writes an ``Event`` object to Splunk.
:param event: An ``Event`` object.
"""
if not self.header_written:
self._out.write(ensure_text("<stream>"))
self.header_written = True
event.write_to(self._out)
def log(self, severity, message):
"""Logs messages about the state of this modular input to Splunk.
These messages will show up in Splunk's internal logs.
:param severity: ``string``, severity of message, see severities defined as class constants.
:param message: ``string``, message to log.
"""
self._err.write(ensure_text("%s %s\n" % (severity, message)))
self._err.flush()
def write_xml_document(self, document):
"""Writes a string representation of an
``ElementTree`` object to the output stream.
:param document: An ``ElementTree`` object.
"""
data = ET.tostring(document)
self._out.write(ensure_text(data))
self._out.flush()
def close(self):
"""Write the closing </stream> tag to make this XML well formed."""
self._out.write(ensure_text("</stream>"))
self._out.flush()
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
def __init__(self, args, environment, logic, user_options=None,
LOGICS=None):
Solver.__init__(self,
environment,
logic=logic,
user_options=user_options)
# Flag used to debug interaction with the solver
self.dbg = False
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.solver = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
self.parser = SmtLibParser(interactive=True)
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
# Initialize solver
self.set_option(":print-success", "true")
if self.options.generate_models:
self.set_option(":produce-models", "true")
# Redirect diagnostic output to stdout
self.set_option(":diagnostic-output-channel", '"stdout"')
if self.options is not None:
for o,v in iteritems(self.options):
self.set_option(o,v)
self.set_logic(logic)
def set_option(self, name, value):
self._send_silent_command(SmtLibCommand(smtcmd.SET_OPTION,
[name, value]))
def set_logic(self, logic):
self._send_silent_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
if self.dbg: print("Sending: " + cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
if self.dbg: print("Read: " + str(res))
return res
def _get_value_answer(self):
"""Reads and parses an assignment from the STDOUT pipe"""
lst = self.parser.get_assignment_list(self.solver_stdout)
if self.dbg: print("Read: " + str(lst))
return lst
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_silent_command(cmd)
self.declared_vars.add(symbol)
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_silent_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_silent_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def _exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.stderr.close()
self.solver.terminate()
return
class SmtLibSolver(Solver):
"""Wrapper for using a solver via textual SMT-LIB interface.
The solver is launched in a subprocess using args as arguments of
the executable. Interaction with the solver occurs via pipe.
"""
OptionsClass = SmtLibOptions
def __init__(self, args, environment, logic, LOGICS=None, **options):
Solver.__init__(self, environment, logic=logic, **options)
if LOGICS is not None: self.LOGICS = LOGICS
self.args = args
self.declared_vars = set()
self.solver = Popen(args,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
bufsize=-1)
# Give time to the process to start-up
time.sleep(0.01)
self.parser = SmtLibParser(interactive=True)
if PY2:
self.solver_stdin = self.solver.stdin
self.solver_stdout = self.solver.stdout
else:
self.solver_stdin = TextIOWrapper(self.solver.stdin)
self.solver_stdout = TextIOWrapper(self.solver.stdout)
# Initialize solver
self.options(self)
self.set_logic(logic)
def set_option(self, name, value):
self._send_silent_command(
SmtLibCommand(smtcmd.SET_OPTION, [name, value]))
def set_logic(self, logic):
self._send_silent_command(SmtLibCommand(smtcmd.SET_LOGIC, [logic]))
def _debug(self, msg, *format_args):
if self.options.debug_interaction:
print(msg % format_args)
def _send_command(self, cmd):
"""Sends a command to the STDIN pipe."""
self._debug("Sending: %s", cmd.serialize_to_string())
cmd.serialize(self.solver_stdin, daggify=True)
self.solver_stdin.write("\n")
self.solver_stdin.flush()
def _send_silent_command(self, cmd):
"""Sends a command to the STDIN pipe and awaits for acknowledgment."""
self._send_command(cmd)
self._check_success()
def _get_answer(self):
"""Reads a line from STDOUT pipe"""
res = self.solver_stdout.readline().strip()
self._debug("Read: %s", res)
return res
def _get_value_answer(self):
"""Reads and parses an assignment from the STDOUT pipe"""
lst = self.parser.get_assignment_list(self.solver_stdout)
self._debug("Read: %s", lst)
return lst
def _declare_variable(self, symbol):
cmd = SmtLibCommand(smtcmd.DECLARE_FUN, [symbol])
self._send_silent_command(cmd)
self.declared_vars.add(symbol)
def _check_success(self):
res = self._get_answer()
if res != "success":
raise UnknownSolverAnswerError("Solver returned: '%s'" % res)
def solve(self, assumptions=None):
assert assumptions is None
self._send_command(SmtLibCommand(smtcmd.CHECK_SAT, []))
ans = self._get_answer()
if ans == "sat":
return True
elif ans == "unsat":
return False
elif ans == "unknown":
raise SolverReturnedUnknownResultError
else:
raise UnknownSolverAnswerError("Solver returned: " + ans)
def reset_assertions(self):
self._send_silent_command(SmtLibCommand(smtcmd.RESET_ASSERTIONS, []))
return
def add_assertion(self, formula, named=None):
# This is needed because Z3 (and possibly other solvers) incorrectly
# recognize N * M * x as a non-linear term
formula = formula.simplify()
deps = formula.get_free_variables()
for d in deps:
if d not in self.declared_vars:
self._declare_variable(d)
self._send_silent_command(SmtLibCommand(smtcmd.ASSERT, [formula]))
def push(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.PUSH, [levels]))
def pop(self, levels=1):
self._send_silent_command(SmtLibCommand(smtcmd.POP, [levels]))
def get_value(self, item):
self._send_command(SmtLibCommand(smtcmd.GET_VALUE, [item]))
lst = self._get_value_answer()
assert len(lst) == 1
assert len(lst[0]) == 2
return lst[0][1]
def print_model(self, name_filter=None):
if name_filter is not None:
raise NotImplementedError
for v in self.declared_vars:
print("%s = %s" % (v, self.get_value(v)))
def get_model(self):
assignment = {}
for s in self.environment.formula_manager.get_all_symbols():
if s.is_term():
v = self.get_value(s)
assignment[s] = v
return EagerModel(assignment=assignment, environment=self.environment)
def _exit(self):
self._send_command(SmtLibCommand(smtcmd.EXIT, []))
self.solver_stdin.close()
self.solver_stdout.close()
self.solver.stderr.close()
self.solver.terminate()
return
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and
so can itself be passed to :func:`fcntl.fcntl`, setting the
:data:`os.O_NONBLOCK` flag will have no effect; likewise, removing
that flag will cause this object to no longer be cooperative.
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:keyword fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, IO will be done on text, otherwise bytes.
:keyword int bufsize: If given, the size of the buffer to use. The default
value means to use a platform-specific default, and a value of 0 is translated
to a value of 1. Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1 and mode not in 'rw': # pragma: no cover
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled.
# NOTE: This is preventing a mode like 'rwb' for binary random access;
# that code was never tested and was explicitly marked as "not used"
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0 or bufsize == 1:
bufsize = self.default_bufsize
elif bufsize == 0:
bufsize = 1
if mode == 'r':
self.io = BufferedReader(self.fileio, bufsize)
else:
assert mode == 'w'
self.io = BufferedWriter(self.fileio, bufsize)
#else: # QQQ: not used, not reachable
#
# self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only run once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
.. caution::
This object is most effective wrapping files that can be used appropriately
with :func:`select.select` such as sockets and pipes.
In general, on most platforms, operations on regular files
(e.g., ``open('/etc/hosts')``) are considered non-blocking
already, even though they can take some time to complete as
data is copied to the kernel and flushed to disk (this time
is relatively bounded compared to sockets or pipes, though).
A :func:`~os.read` or :func:`~os.write` call on such a file
will still effectively block for some small period of time.
Therefore, wrapping this class around a regular file is
unlikely to make IO gevent-friendly: reading or writing large
amounts of data could still block the event loop.
If you'll be working with regular files and doing IO in large
chunks, you may consider using
:class:`~gevent.fileobject.FileObjectThread` or
:func:`~gevent.os.tp_read` and :func:`~gevent.os.tp_write` to bypass this
concern.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and
so can itself be passed to :func:`fcntl.fcntl`, setting the
:data:`os.O_NONBLOCK` flag will have no effect; however, removing
that flag will cause this object to no longer be cooperative.
.. versionchanged:: 1.1
Now uses the :mod:`io` package internally. Under Python 2, previously
used the undocumented class :class:`socket._fileobject`. This provides
better file-like semantics (and portability to Python 3).
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:keyword fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, IO will be done on text, otherwise bytes.
:keyword int bufsize: If given, the size of the buffer to use. The default
value means to use a platform-specific default, and a value of 0 is translated
to a value of 1. Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1 and mode not in 'rw': # pragma: no cover
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled.
# NOTE: This is preventing a mode like 'rwb' for binary random access;
# that code was never tested and was explicitly marked as "not used"
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0 or bufsize == 1:
bufsize = self.default_bufsize
elif bufsize == 0:
bufsize = 1
if mode == 'r':
self.io = BufferedReader(self.fileio, bufsize)
else:
assert mode == 'w'
self.io = BufferedWriter(self.fileio, bufsize)
#else: # QQQ: not used, not reachable
#
# self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is closed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only run once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
"""
.. versionadded:: 1.1b2
"""
return self.io.readable()
def writable(self):
"""
.. versionadded:: 1.1b2
"""
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files.
.. seealso:: :func:`gevent.os.make_nonblocking`
"""
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
return getattr(self._fobj, name)
class FileObjectPosix:
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, list):
self.io.writelines(list)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def __iter__(self):
return self.io
class FileObjectPosix(object):
"""
A file-like object that operates on non-blocking files.
.. seealso:: :func:`gevent.os.make_nonblocking`
"""
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1:
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores in. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled
raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, lines):
self.io.writelines(lines)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def readable(self):
return self.io.readable()
def writable(self):
return self.io.writable()
def seek(self, *args, **kwargs):
return self.io.seek(*args, **kwargs)
def seekable(self):
return self.io.seekable()
def tell(self):
return self.io.tell()
def truncate(self, size=None):
return self.io.truncate(size)
def __iter__(self):
return self.io
def __getattr__(self, name):
# XXX: Should this really be _fobj, or self.io?
# _fobj can easily be None but io never is
return getattr(self._fobj, name)