def read(rfile: io.BufferedReader) -> typing.Any:
x = rfile.readline().strip()
return json.loads(x)
def load(cls, filename, metadata_only=False):
"""
Load ring data from a file.
:param filename: Path to a file serialized by the save() method.
:param bool metadata_only: If True, only load `devs` and `part_shift`.
:returns: A RingData instance containing the loaded data.
"""
gz_file = BufferedReader(GzipFile(filename, 'rb'))
# See if the file is in the new format
magic = gz_file.read(4)
if magic == b'R1NG':
format_version, = struct.unpack('!H', gz_file.read(2))
if format_version == 1:
ring_data = cls.deserialize_v1(
gz_file, metadata_only=metadata_only)
else:
raise Exception('Unknown ring format version %d' %
format_version)
else:
# Assume old-style pickled ring
gz_file.seek(0)
ring_data = pickle.load(gz_file)
if not hasattr(ring_data, 'devs'):
ring_data = RingData(ring_data['replica2part2dev_id'],
ring_data['devs'], ring_data['part_shift'],
ring_data.get('next_part_power'))
return ring_data
def test_io_bufferedreader(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp)
self.assertEqual(br.read(), b'foo')
br.close()
self.assertEqual(resp.closed, True)
def tst_io_bufferedreader(self):
fp = self._fake_fp(b'foo')
#fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp)
self.assertEqual(br.read(), b'foo')
br.close()
self.assertEqual(resp.closed, True)
b = b'fooandahalf'
fp = self._fake_fp(b)
#fp = BytesIO(b)
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp, 5)
br.read(1) # sets up the buffer, reading 5
self.assertEqual(len(fp.read()), len(b) - 5)
# This is necessary to make sure the "no bytes left" part of `readinto`
# gets tested.
while not br.closed:
br.read(5)
def __enter__(self):
if self.__to_close is not None:
raise Exception(f"{self!r} is already a context manager")
stream = IOWrapper(self.__stream)
reader = BufferedReader(stream.__enter__())
to_close = [reader]
if reader.peek(len(GZIP_MAGIC)) == GZIP_MAGIC:
ret = GzipIOWrapper(reader)
to_close.append(ret)
ret = ret.__enter__()
else:
ret = reader
self.__to_close = (stream,) + tuple(to_close)
return ret
def __init__(self, data=None, stream=None):
if data:
self.stream = BytesIO(data)
elif stream:
self.stream = stream
else:
raise InvalidParameterError(
'Either bytes or a stream must be provided')
self.reader = BufferedReader(self.stream)
def __init__(self, data=None, stream=None):
if data:
self.stream = BytesIO(data)
elif stream:
self.stream = stream
else:
raise ValueError('Either bytes or a stream must be provided')
self.reader = BufferedReader(self.stream)
self._last = None # Should come in handy to spot -404 errors
def __init__(self, stream, boundary=None):
b = "\r\n--" + boundary + "--"
stream = _StreamWrapper(stream)
self.buf_reader = BufferedReader(stream)
self.nl = b[:2]
self.nl_dash_boundary = b[:len(b)-2]
self.dash_boundary_dash = b[2:]
self.dash_boundary = b[2:len(b)-2]
self.headers = {}
self.parts_read = 0
self.current_part = None
def _wait_for_line_in_stream(stream: io.BufferedReader, timeout: float) -> str:
"""Wait for a line to appear in a stream and return it.
This will only work on Unix.
If something does appear in the stream, but it isn't terminated with a newline, then this
function will hang. But since the program that we will use this for will write its output in
lines, I don't think that the additional robustness is needed.
"""
line_selector = select.poll()
line_selector.register(stream, select.POLLIN)
start_time = time.perf_counter()
while time.perf_counter() - start_time < timeout:
if line_selector.poll(0.01):
return stream.readline()
pytest.fail('Waiting for a stream line timed out.')
def test_io_bufferedreader(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp)
assert br.read() == b'foo'
br.close()
assert resp.closed
b = b'fooandahalf'
fp = BytesIO(b)
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp, 5)
br.read(1) # sets up the buffer, reading 5
assert len(fp.read()) == (len(b) - 5)
# This is necessary to make sure the "no bytes left" part of `readinto`
# gets tested.
while not br.closed:
br.read(5)
def read_line(self):
if not self.file:
path = self.get_path()
if not path:
raise Exception('Unable to find the location of "Plex Media Server.log"')
# Open file
self.file = ASIO.open(path, opener=False)
self.file.seek(self.file.get_size(), SEEK_ORIGIN_CURRENT)
# Create buffered reader
self.reader = BufferedReader(self.file)
self.path = self.file.get_path()
log.info('Opened file path: "%s"' % self.path)
return self.reader.readline()
def read(self, file_path):
while not self.writing:
time.sleep(1)
print "Read starting..."
f = ASIO.open(file_path, opener=False)
s = BufferedReader(f)
orig_path = f.get_path()
stale_since = None
while True:
if f is None:
print 'Opening file...'
f = ASIO.open(file_path, opener=False)
s = BufferedReader(f)
# Try read line
line = s.readline()
if line:
stale_since = None
time.sleep(0.05)
else:
if stale_since is None:
stale_since = time.time()
time.sleep(0.1)
continue
elif (time.time() - stale_since) > 2 and f.get_path() != orig_path:
s.close()
s = None
f.close()
f = None
elif not self.writing:
break
else:
time.sleep(0.1)
continue
print 'read %r' % (line,)
print 'finished'
s.close()
f.close()
def __init__(self, fileobj):
self.fp = fileobj
BufferedReader.__init__(self, _DeflateReader(fileobj))
def __init__(self):
"""
Constructor
"""
# Read the configuration parameters from tulsi.conf
self.timestr = time.strftime("%Y:%m:%d-%H:%M:%S")
logging.basicConfig(filename="/var/log/tulsi.log", level=logging.DEBUG)
try:
self.conf = ConfigParser.ConfigParser()
self.conf.read("/etc/tulsi/tulsi.conf")
self.udp_ip = self.conf.get("tulsi", "host")
self.udp_port = int(self.conf.get("tulsi", "port"))
# printing the host and port of tulsi
logging.info("%s The IP of the host: %s" % (self.timestr, self.udp_ip))
logging.info("%s The Port number of the host :%s" % (self.timestr, self.udp_port))
except:
# Error message of tulsi not working
logging.error("The tulsi configuration file is not found")
# Creating objects of MessageEncode and HostInfo
msg_encode = MessageEncode()
host_info = HostInfo()
# Initializing empty lists
self.drives = []
self.service = []
self.ip_array = []
self.ring_ip = []
self.ring_conf_ip = []
self.ring_drives = []
self.ip_set_array = []
self.my_ring_conf = dict()
# Read the ring Configuration file
self.gz_file = GzipFile("/etc/swift/container.ring.gz", "rb")
if hasattr(self.gz_file, "_checkReadable"):
self.gz_file = BufferedReader(self.gz_file)
magic = self.gz_file.read(4)
if magic == "R1NG":
version, = struct.unpack("!H", self.gz_file.read(2))
if version == 1:
self.ring_data = self.read_ring_file(self.gz_file)
else:
logging.error("%s Unknown ring format version %d" % (self.timestr, version))
raise Exception("Unknown ring format version %d" % version)
# While loop to continuously check the status of swift services and
# drives and send information to tulsi client
while True:
self.ip_array = host_info.read_ip()
self.service = host_info.read_services()
self.drives = host_info.read_drives(self.drives)
self.message = msg_encode.create_message(
self.my_ring_conf, self.ring_conf_ip, self.ip_array, self.service, self.drives
)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet # UDP
sock.sendto(self.message, (self.udp_ip, self.udp_port))
time.sleep(5)
self.ip_array = []
self.service = []
self.drives = []
def __init__(self, f: io.BufferedReader):
self.f = f
self.total_size = os.fstat(f.fileno()).st_size
self.pbar = tqdm(total=self.total_size, leave=False)
self.read = f.read
f.read = self._read
def _recv(self):
bytes_buffer = BufferedReader(BytesIO(self.socket.recv(4096)))
if len(bytes_buffer.peek()) < 4:
bytes_buffer = BufferedReader(
BytesIO(bytes_buffer.read() + self.socket.recv(4096))
)
res = self.__recv(bytes_buffer)
# while res[-1] != 0x17:
# count =1
while True:
if res[-1] == 0x17:
yield res[:-1]
if res[-1] == 0x16:
plaintext_buffer = BufferedReader(BytesIO(res[:-1]))
while plaintext_buffer.peek():
hh = HandshakeHeader.deserialize(plaintext_buffer.read(4))
hh_payload_buffer = plaintext_buffer.read(hh.size)
# print("recv", len(hh_payload_buffer), hh.size, hh)
# print(res[:-1])
hh_payload = HANDSHAKE_HEADER_TYPES[hh.message_type].deserialize(
hh_payload_buffer
)
# print(hh_payload)
if type(hh_payload) is NewSessionTicketHandshakePayload:
self.session_tickets.append(hh_payload)
if len(bytes_buffer.peek()) < 4:
bytes_buffer = BufferedReader(
BytesIO(bytes_buffer.read() + self.socket.recv(4096))
)
res = self.__recv(bytes_buffer)
class EventStorageReader:
def __init__(self, files: EventStorageFiles, settings: FileEventStorageSettings):
self.files = files
self.settings = settings
self.current_batch = None
self.buffered_reader = None
self.current_pos = self.read_state_file()
self.new_pos = self.current_pos
def read(self):
if self.current_batch is not None and self.current_batch:
log.debug("The previous batch was not discarded!")
return self.current_batch
self.current_batch = []
records_to_read = self.settings.get_max_read_records_count()
while records_to_read > 0:
try:
current_line_in_file = self.new_pos.get_line()
self.buffered_reader = self.get_or_init_buffered_reader(self.new_pos)
line = self.buffered_reader.readline()
while line != b'':
try:
self.current_batch.append(b64decode(line).decode("utf-8"))
records_to_read -= 1
except IOError as e:
log.warning("Could not parse line [%s] to uplink message! %s", line, e)
except Exception as e:
log.exception(e)
current_line_in_file += 1
self.new_pos.set_line(current_line_in_file)
self.write_info_to_state_file(self.new_pos)
break
finally:
current_line_in_file += 1
if records_to_read > 0:
line = self.buffered_reader.readline()
self.new_pos.set_line(current_line_in_file)
if records_to_read == 0:
break
if current_line_in_file >= self.settings.get_max_records_per_file():
previous_file = self.new_pos
next_file = self.get_next_file(self.files, self.new_pos)
if next_file is not None:
if self.buffered_reader is not None:
self.buffered_reader.close()
self.buffered_reader = None
self.delete_read_file(previous_file)
self.new_pos = EventStorageReaderPointer(next_file, 0)
self.write_info_to_state_file(self.new_pos)
continue
else:
# No more records to read for now
break
# continue
###################
if line == b'':
break
#######################
else:
# No more records to read for now
continue
except IOError as e:
log.warning("[{}] Failed to read file!".format(self.new_pos.get_file(), e))
break
except Exception as e:
log.exception(e)
return self.current_batch
def discard_batch(self):
try:
if self.current_pos.get_line() >= self.settings.get_max_records_per_file()-1:
if self.buffered_reader is not None and not self.buffered_reader.closed:
self.buffered_reader.close()
self.write_info_to_state_file(self.new_pos)
self.current_pos = self.new_pos
self.current_batch = None
except Exception as e:
log.exception(e)
def get_next_file(self, files: EventStorageFiles, new_pos: EventStorageReaderPointer):
found = False
data_files = files.get_data_files()
target_file = None
for file_index in range(len(data_files)):
if found:
target_file = data_files[file_index]
break
if data_files[file_index] == new_pos.get_file():
found = True
return target_file
def get_or_init_buffered_reader(self, pointer):
try:
if self.buffered_reader is None or self.buffered_reader.closed:
new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file()
self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r'))
lines_to_skip = pointer.get_line()
if lines_to_skip > 0:
while self.buffered_reader.readline() is not None:
if lines_to_skip != 0:
lines_to_skip -= 1
else:
break
return self.buffered_reader
except IOError as e:
log.error("Failed to initialize buffered reader!", e)
raise RuntimeError("Failed to initialize buffered reader!", e)
def read_state_file(self):
try:
state_data_node = {}
try:
with BufferedReader(FileIO(self.settings.get_data_folder_path() +
self.files.get_state_file(), 'r')) as br:
state_data_node = load(br)
except JSONDecodeError:
log.error("Failed to decode JSON from state file")
state_data_node = 0
except IOError as e:
log.warning("Failed to fetch info from state file!", e)
reader_file = None
reader_pos = 0
if state_data_node:
reader_pos = state_data_node['position']
for file in sorted(self.files.get_data_files()):
if file == state_data_node['file']:
reader_file = file
break
if reader_file is None:
reader_file = sorted(self.files.get_data_files())[0]
reader_pos = 0
log.info("FileStorage_reader -- Initializing from state file: [%s:%i]",
self.settings.get_data_folder_path() + reader_file,
reader_pos)
return EventStorageReaderPointer(reader_file, reader_pos)
except Exception as e:
log.exception(e)
def write_info_to_state_file(self, pointer: EventStorageReaderPointer):
try:
state_file_node = {'file': pointer.get_file(), 'position': pointer.get_line()}
with open(self.settings.get_data_folder_path() + self.files.get_state_file(), 'w') as outfile:
outfile.write(dumps(state_file_node))
except IOError as e:
log.warning("Failed to update state file!", e)
except Exception as e:
log.exception(e)
def delete_read_file(self, current_file: EventStorageReaderPointer):
data_files = self.files.get_data_files()
if exists(self.settings.get_data_folder_path() + current_file.file):
remove(self.settings.get_data_folder_path() + current_file.file)
try:
data_files = data_files[1:]
except Exception as e:
log.exception(e)
log.info("FileStorage_reader -- Cleanup old data file: %s%s!", self.settings.get_data_folder_path(), current_file.file)
def destroy(self):
if self.buffered_reader is not None:
self.buffered_reader.close()
raise IOError
def _pass(fp: io.BufferedReader, no_bytes: int):
data = fp.read(no_bytes)
return
def awslogs_handler(event, context, metadata):
# Get logs
with gzip.GzipFile(fileobj=BytesIO(
base64.b64decode(event["awslogs"]["data"]))) as decompress_stream:
# Reading line by line avoid a bug where gzip would take a very long
# time (>5min) for file around 60MB gzipped
data = "".join(BufferedReader(decompress_stream))
logs = json.loads(str(data))
# Set the source on the logs
source = logs.get("logGroup", "cloudwatch")
metadata[DD_SOURCE] = parse_event_source(event, source)
# Default service to source value
metadata[DD_SERVICE] = metadata[DD_SOURCE]
# Build aws attributes
aws_attributes = {
"aws": {
"awslogs": {
"logGroup": logs["logGroup"],
"logStream": logs["logStream"],
"owner": logs["owner"],
}
}
}
# Set host as log group where cloudwatch is source
if metadata[DD_SOURCE] == "cloudwatch":
metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"]
# When parsing rds logs, use the cloudwatch log group name to derive the
# rds instance name, and add the log name of the stream ingested
if metadata[DD_SOURCE] == "rds":
match = rds_regex.match(logs["logGroup"])
if match is not None:
metadata[DD_HOST] = match.group("host")
metadata[DD_CUSTOM_TAGS] = (metadata[DD_CUSTOM_TAGS] +
",logname:" + match.group("name"))
# We can intuit the sourcecategory in some cases
if match.group("name") == "postgresql":
metadata[DD_CUSTOM_TAGS] + ",sourcecategory:" + match.group(
"name")
# For Lambda logs we want to extract the function name,
# then rebuild the arn of the monitored lambda using that name.
# Start by splitting the log group to get the function name
if metadata[DD_SOURCE] == "lambda":
log_group_parts = logs["logGroup"].split("/lambda/")
if len(log_group_parts) > 1:
function_name = log_group_parts[1].lower()
# Split the arn of the forwarder to extract the prefix
arn_parts = context.invoked_function_arn.split("function:")
if len(arn_parts) > 0:
arn_prefix = arn_parts[0]
# Rebuild the arn by replacing the function name
arn = arn_prefix + "function:" + function_name
# Add the arn as a log attribute
arn_attributes = {"lambda": {"arn": arn}}
aws_attributes = merge_dicts(aws_attributes, arn_attributes)
# Add the function name as tag
metadata[DD_CUSTOM_TAGS] += ",functionname:" + function_name
# Set the arn as the hostname
metadata[DD_HOST] = arn
# Create and send structured logs to Datadog
for log in logs["logEvents"]:
yield merge_dicts(log, aws_attributes)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=help_msg)
parser.add_argument('filename', help='input file')
parser.add_argument('output', help='output file')
parser.add_argument('convert_to', choices=['bin', 'txt'])
args = parser.parse_args()
embeddings = []
if args.convert_to == 'txt': # then format must be bin
with FileIO(args.filename, 'rb') as f:
reader = BufferedReader(f)
vocab_size, dimension = map(int, f.readline().split())
for _ in range(vocab_size):
w = ''.join(takewhile(lambda x: x != ' ', (reader.read(1) for _ in count())))
s = reader.read(4 * dimension)
reader.read(1) # end of line character
arr = np.fromstring(s, dtype=np.float32)
embeddings.append((w, arr))
assert not reader.peek(1)
else:
with open(args.filename) as f:
vocab_size, dimension = map(int, f.readline().split())
for line in f:
w, s = line.strip().split(' ', 1)
arr = np.fromstring(s, dtype=np.float32, sep=' ')
embeddings.append((w, arr))
class Unpacker:
def __init__(self, initial_bytes=None):
self.initial_bytes = initial_bytes
self.buf = BufferedReader(raw=BytesIO(initial_bytes))
def struct_unpack(self, fmt):
return struct.unpack(fmt, self.buf.read(struct.calcsize(fmt)))
def read_byte(self):
return self.struct_unpack('B')[0]
def read_signed_byte(self):
return self.struct_unpack('b')[0]
def read_short(self, prefix=None):
if prefix is None:
prefix = self.read_byte()
assert prefix in [Prefix.INT16, Prefix.UINT16], 'read_short expects prefix %s or %s but read %s' % (
Prefix.INT16, Prefix.UINT16, prefix)
return self.struct_unpack('>H' if prefix == Prefix.UINT16 else '>h')[0]
def read_int(self, prefix=None):
if prefix is None:
prefix = self.read_byte()
assert prefix in [Prefix.INT32, Prefix.UINT32], 'read_int expects prefix %s or %s but read %s' % (
Prefix.INT32, Prefix.UINT32, prefix)
return self.struct_unpack('>I' if prefix == Prefix.UINT32 else '>i')[0]
def read_long(self, prefix=None):
if prefix is None:
prefix = self.read_byte()
assert prefix in [Prefix.INT64, Prefix.UINT64], 'read_long expects prefix %s or %s but read %s' % (
Prefix.INT64, Prefix.UINT64, prefix)
return self.struct_unpack('>Q' if prefix == Prefix.UINT64 else '>q')[0]
def read_float(self, prefix=None):
if prefix is None:
prefix = self.read_byte()
assert prefix == Prefix.FLOAT32, 'read_float expects prefix %s but read %s' % (Prefix.FLOAT32, prefix)
return self.struct_unpack('>f')[0]
def read_double(self, prefix=None):
if prefix is None:
prefix = self.read_byte()
assert prefix == Prefix.FLOAT64, 'read_double expects prefix %s but read %s' % (Prefix.FLOAT64, prefix)
return self.struct_unpack('>d')[0]
def read_payload(self, length):
return self.buf.read(length)
def peek(self):
return struct.unpack('B', self.buf.peek()[0])[0]
def unpack_nil(self):
assert self.read_byte() == Prefix.NIL, 'unpack_nil expects to read %s' % Prefix.NIL
def unpack_boolean(self):
prefix = self.read_byte()
assert prefix in [Prefix.TRUE, Prefix.FALSE], 'unpack_boolean expects prefix %s or %s but read %s' % (
Prefix.TRUE, Prefix.FALSE, prefix)
return prefix == Prefix.TRUE
def unpack_int(self):
prefix = self.peek()
if Prefix.is_fix_int(prefix):
return self.read_byte() if Prefix.is_pos_fix_int(prefix) else self.read_signed_byte()
if prefix == Prefix.INT8:
self.read_byte()
return self.read_signed_byte()
elif prefix == Prefix.UINT8:
self.read_byte()
return self.read_byte()
elif prefix in [Prefix.UINT16, Prefix.INT16]:
return self.read_short()
elif prefix in [Prefix.UINT32, Prefix.INT32]:
return self.read_int()
elif prefix in [Prefix.UINT64, Prefix.INT64]:
return self.read_long()
else:
raise ValueError('unexpected int prefix %s' % prefix)
def unpack_float(self):
return self.read_float()
def unpack_double(self):
return self.read_double()
def unpack(self):
prefix = self.peek()
if Prefix.is_fix_int(prefix) or prefix in [Prefix.INT8, Prefix.UINT8, Prefix.INT16, Prefix.UINT16, Prefix.INT32,
Prefix.UINT32, Prefix.INT64, Prefix.UINT64]:
return self.unpack_int()
elif Prefix.is_fixed_array(prefix) or prefix in [Prefix.ARRAY16, Prefix.ARRAY32]:
return self.unpack_array()
elif Prefix.is_fixed_map(prefix) or prefix in [Prefix.MAP16, Prefix.MAP32]:
return self.unpack_map()
elif Prefix.is_fix_str(prefix) or prefix in [Prefix.STR8, Prefix.STR16, Prefix.STR32]:
return self.unpack_string()
elif prefix in [Prefix.TRUE, Prefix.FALSE]:
return self.unpack_boolean()
elif prefix == Prefix.NIL:
return self.unpack_nil()
elif prefix == Prefix.FLOAT32:
return self.unpack_float()
elif prefix == Prefix.FLOAT64:
return self.unpack_double()
elif prefix in [Prefix.BIN8, Prefix.BIN16, Prefix.BIN32]:
return self.unpack_binary()
else:
raise ValueError('unknown prefix %s' % prefix)
def unpack_string(self):
return str(bytearray(self.read_payload(self.unpack_raw_string_header())))
def unpack_raw_string_header(self):
prefix = self.read_byte()
if Prefix.is_fixed_raw(prefix):
return prefix & 0x1f
elif prefix == Prefix.STR8:
return self.read_byte()
elif prefix == Prefix.STR16:
return self.read_short(Prefix.UINT16)
elif prefix == Prefix.STR32:
return self.read_int(Prefix.UINT32)
else:
raise ValueError('unexpected raw string header prefix %s' % prefix)
def unpack_array(self):
size = self.unpack_array_header()
array = []
for i in range(0, size):
array.append(self.unpack())
return array
def unpack_array_header(self):
prefix = self.read_byte()
if Prefix.is_fixed_array(prefix):
return prefix & 0x0f
elif prefix == Prefix.ARRAY16:
return self.read_short(Prefix.UINT16)
elif prefix == Prefix.ARRAY32:
return self.read_int(Prefix.UINT32)
else:
raise ValueError('unexpected array header prefix %s' % prefix)
def unpack_map(self):
size = self.unpack_map_header()
map = OrderedDict()
for i in range(0, size):
map[self.unpack()] = self.unpack()
return map
def unpack_map_header(self):
prefix = self.read_byte()
if Prefix.is_fixed_map(prefix):
return prefix & 0x1f
elif prefix == Prefix.MAP16:
return self.read_short(Prefix.UINT16)
elif prefix == Prefix.MAP32:
return self.read_int(Prefix.UINT32)
else:
raise ValueError('unexpected map header prefix %s' % prefix)
def unpack_binary(self):
return self.read_payload(self.unpack_binary_header())
def unpack_binary_header(self):
prefix = self.read_byte()
if prefix == Prefix.BIN8:
return self.read_byte()
elif prefix == Prefix.BIN16:
return self.read_short(Prefix.UINT16)
elif prefix == Prefix.BIN32:
return self.read_int(Prefix.UINT32)
else:
raise ValueError('unexpected binary header prefix %s' % prefix)
class Logging(Source):
name = 'logging'
events = [
'logging.playing',
'logging.action.played',
'logging.action.unplayed'
]
parsers = []
path = None
path_hints = PATH_HINTS
def __init__(self, activity):
super(Logging, self).__init__()
self.parsers = [p(self) for p in Logging.parsers]
self.file = None
self.reader = None
self.path = None
# Pipe events to the main activity instance
self.pipe(self.events, activity)
def run(self):
line = self.read_line_retry(ping=True, stale_sleep=0.5)
if not line:
log.info('Unable to read log file')
return
log.debug('Ready')
while True:
# Grab the next line of the log
line = self.read_line_retry(ping=True)
if line:
self.process(line)
else:
log.info('Unable to read log file')
def process(self, line):
for parser in self.parsers:
if parser.process(line):
return True
return False
def read_line(self):
if not self.file:
path = self.get_path()
if not path:
raise Exception('Unable to find the location of "Plex Media Server.log"')
# Open file
self.file = ASIO.open(path, opener=False)
self.file.seek(self.file.get_size(), SEEK_ORIGIN_CURRENT)
# Create buffered reader
self.reader = BufferedReader(self.file)
self.path = self.file.get_path()
log.info('Opened file path: "%s"' % self.path)
return self.reader.readline()
def read_line_retry(self, timeout=60, ping=False, stale_sleep=1.0):
line = None
stale_since = None
while not line:
line = self.read_line()
if line:
stale_since = None
time.sleep(0.05)
break
if stale_since is None:
stale_since = time.time()
time.sleep(stale_sleep)
continue
elif (time.time() - stale_since) > timeout:
return None
elif (time.time() - stale_since) > timeout / 2:
# Nothing returned for 5 seconds
if self.file.get_path() != self.path:
log.debug("Log file moved (probably rotated), closing")
self.close()
elif ping:
# Ping server to see if server is still active
Plex.detail()
ping = False
time.sleep(stale_sleep)
return line
def close(self):
if not self.file:
return
try:
# Close the buffered reader
self.reader.close()
except Exception as ex:
log.error('reader.close() - raised exception: %s', ex, exc_info=True)
finally:
self.reader = None
try:
# Close the file handle
self.file.close()
except OSError as ex:
if ex.errno == 9:
# Bad file descriptor, already closed?
log.warn('file.close() - ignoring raised exception: %s (already closed)', ex)
else:
log.error('file.close() - raised exception: %s', ex, exc_info=True)
except Exception as ex:
log.error('file.close() - raised exception: %s', ex, exc_info=True)
finally:
self.file = None
@classmethod
def get_path(cls):
if cls.path:
return cls.path
hints = cls.get_hints()
log.debug('hints: %r', hints)
if not hints:
log.error('Unable to find any hints for "%s", operating system not supported', platform.system())
return None
for hint in hints:
log.debug('Testing if "%s" exists', hint)
if os.path.exists(hint):
cls.path = hint
break
if cls.path:
log.debug('Using the path: %r', cls.path)
else:
log.error('Unable to find a valid path for "Plex Media Server.log"', extra={
'data': {
'hints': hints
}
})
return cls.path
@classmethod
def add_hint(cls, path, system=None):
if system not in cls.path_hints:
cls.path_hints[system] = []
cls.path_hints[system].append(path)
@classmethod
def get_hints(cls):
# Retrieve system hints
hints_system = PATH_HINTS.get(platform.system(), [])
# Retrieve global hints
hints_global = PATH_HINTS.get(None, [])
# Retrieve hint from server preferences (if available)
data_path = Plex[':/prefs'].get('LocalAppDataPath')
if data_path:
hints_global.append(os.path.join(data_path.value, "Plex Media Server", "Logs", "Plex Media Server.log"))
else:
log.info('Unable to retrieve "LocalAppDataPath" from server')
hints = []
for hint in (hints_global + hints_system):
# Resolve hint function
if inspect.isfunction(hint):
hint = hint()
# Check for duplicate
if hint in hints:
continue
hints.append(hint)
return hints
@classmethod
def test(cls):
# TODO "Logging" source testing
return True
@classmethod
def register(cls, parser):
cls.parsers.append(parser)
# set port, baudrate and timeout to suit your device configuration
if platform == "win32": # Windows
port = "COM13"
elif platform == "darwin": # MacOS
port = "/dev/tty.usbmodem14101"
else: # Linux
port = "/dev/ttyACM1"
baudrate = 9600
timeout = 0.1
RATE = 4 # set to 0 to disable NAV messages on USB and UART1 ports
with Serial(port, baudrate, timeout=timeout) as serial:
# create UBXReader instance, reading only UBX messages
ubr = UBXReader(BufferedReader(serial), protfilter=2)
print("\nStarting read thread...\n")
reading = True
serial_lock = Lock()
read_thread = start_thread(serial, serial_lock, ubr)
# set the UART1 and USB message rate for each UBX-NAV message
# via a CFG-MSG command
print("\nSending CFG-MSG message rate configuration messages...\n")
for (msgid, msgname) in UBX_MSGIDS.items():
if msgid[0] == 0x01: # NAV
msg = UBXMessage(
"CFG",
"CFG-MSG",
SET,
def test_io_bufferedreader(self):
fp = BytesIO(b"foo")
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp)
assert br.read() == b"foo"
br.close()
assert resp.closed
b = b"fooandahalf"
fp = BytesIO(b)
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp, 5)
br.read(1) # sets up the buffer, reading 5
assert len(fp.read()) == (len(b) - 5)
# This is necessary to make sure the "no bytes left" part of `readinto`
# gets tested.
while not br.closed:
br.read(5)
def convertInversions(samtools, refFasta, vcfFile, invMateDict):
isHeaderInfoAdded = False
isHeaderAltAdded = False
lineBuffer = []
bufferedChr = ""
bufferedPos = -1
if vcfFile.endswith('gz'):
gzfp = gzip.open(vcfFile, 'rb')
fpVcf = BufferedReader(gzfp)
else:
fpVcf = open(vcfFile, 'rb')
for line in fpVcf:
if line.startswith('#'):
if (not isHeaderInfoAdded) and line.startswith("##FORMAT="):
sys.stdout.write(
"##INFO=<ID=INV3,Number=0,Type=Flag,Description=\"Inversion breakends open 3' of reported location\">\n"
)
sys.stdout.write(
"##INFO=<ID=INV5,Number=0,Type=Flag,Description=\"Inversion breakends open 5' of reported location\">\n"
)
isHeaderInfoAdded = True
if (not isHeaderAltAdded) and line.startswith("##ALT="):
sys.stdout.write("##ALT=<ID=INV,Description=\"Inversion\">\n")
isHeaderAltAdded = True
sys.stdout.write(line)
continue
vcfRec = VcfRecord(line)
# skip mate record
if vcfRec.vid in invMateDict:
continue
vcfRec.checkInversion()
if vcfRec.isINV3 or vcfRec.isINV5:
if vcfRec.isINV5:
# adjust POS for INV5
vcfRec.pos -= 1
vcfRec.matePos -= 1
vcfRec.ref = getReference(samtools, refFasta, vcfRec.chr,
vcfRec.pos, vcfRec.pos)
# update manta ID
vidSuffix = vcfRec.vid.split("MantaBND")[1]
idx = vidSuffix.rfind(':')
vcfRec.vid = "MantaINV%s" % vidSuffix[:idx]
# symbolic ALT
vcfRec.alt = "<INV>"
# add END
infoEndStr = "END=%d" % vcfRec.matePos
newInfo = [infoEndStr]
for infoItem in vcfRec.info:
if infoItem.startswith("SVTYPE"):
# change SVTYPE
newInfo.append("SVTYPE=INV")
# add SVLEN
infoSvLenStr = "SVLEN=%d" % (vcfRec.matePos - vcfRec.pos)
newInfo.append(infoSvLenStr)
elif infoItem.startswith("CIPOS"):
newInfo.append(infoItem)
# set CIEND
isImprecise = "IMPRECISE" in vcfRec.infoDict
# for imprecise calls, set CIEND to the mate breakpoint's CIPOS
if isImprecise:
mateId = vcfRec.infoDict["MATEID"]
mateInfoDict = invMateDict[mateId]
infoCiEndStr = "CIEND=%s" % (mateInfoDict["CIPOS"])
newInfo.append(infoCiEndStr)
# for precise calls, set CIEND w.r.t HOMLEN
else:
if "HOMLEN" in vcfRec.infoDict:
infoCiEndStr = "CIEND=-%s,0" % vcfRec.infoDict[
"HOMLEN"]
newInfo.append(infoCiEndStr)
elif infoItem.startswith("HOMSEQ"):
# update HOMSEQ for INV5
if vcfRec.isINV5:
cipos = vcfRec.infoDict["CIPOS"].split(',')
homSeqStart = vcfRec.pos + int(cipos[0]) + 1
homSeqEnd = vcfRec.pos + int(cipos[1])
refSeq = getReference(samtools, refFasta, vcfRec.chr,
homSeqStart, homSeqEnd)
infoHomSeqStr = "HOMSEQ=%s" % refSeq
newInfo.append(infoHomSeqStr)
else:
newInfo.append(infoItem)
# skip BND-specific tags
elif (infoItem.startswith("MATEID")
or infoItem.startswith("BND_DEPTH")
or infoItem.startswith("MATE_BND_DEPTH")):
continue
# update event ID
elif infoItem.startswith("EVENT"):
eidSuffix = vcfRec.infoDict["EVENT"].split("MantaBND")[1]
idx = vidSuffix.rfind(':')
infoEventStr = "EVENT=MantaINV%s" % eidSuffix[:idx]
newInfo.append(infoEventStr)
# apply all other tags
else:
newInfo.append(infoItem)
# add INV3/INV5 tag
if vcfRec.isINV3:
newInfo.append("INV3")
elif vcfRec.isINV5:
newInfo.append("INV5")
vcfRec.info = newInfo
vcfRec.makeLine()
# make sure the vcf is sorted in genomic order
if (not vcfRec.chr == bufferedChr) or (vcfRec.pos > bufferedPos):
if lineBuffer:
writeLines(lineBuffer)
lineBuffer = [vcfRec.line]
bufferedChr = vcfRec.chr
bufferedPos = vcfRec.pos
elif vcfRec.pos < bufferedPos:
lineBuffer.insert(0, vcfRec.line)
else:
lineBuffer.append(vcfRec.line)
if lineBuffer:
writeLines(lineBuffer)
def initialize_stdio(
global_bootstrap_options: OptionValueContainer) -> Iterator[None]:
"""Mutates sys.std* and logging to route stdio for a Pants process to thread local destinations.
In this context, `sys.std*` and logging handlers will route through Rust code that uses
thread-local information to decide whether to write to a file, or to stdio file handles.
To control the stdio destination set by this method, use the `stdio_destination` context manager.
This is called in two different processes:
* PantsRunner, after it has determined that LocalPantsRunner will be running in process, and
immediately before setting a `stdio_destination` for the remainder of the run.
* PantsDaemon, immediately on startup. The process will then default to sending stdio to the log
until client connections arrive, at which point `stdio_destination` is used per-connection.
"""
global_level = global_bootstrap_options.level
log_show_rust_3rdparty = global_bootstrap_options.log_show_rust_3rdparty
show_target = global_bootstrap_options.show_log_target
log_levels_by_target = _get_log_levels_by_target(global_bootstrap_options)
print_stacktrace = global_bootstrap_options.print_stacktrace
local_cleanup = global_bootstrap_options.process_execution_local_cleanup
literal_filters = []
regex_filters = []
for filt in cast("list[str]", global_bootstrap_options.ignore_warnings):
if filt.startswith("$regex$"):
regex_filters.append(strip_prefix(filt, "$regex$"))
else:
literal_filters.append(filt)
# Set the pants log destination.
log_path = str(
pants_log_path(PurePath(global_bootstrap_options.pants_workdir)))
safe_mkdir_for(log_path)
# Initialize thread-local stdio, and replace sys.std* with proxies.
original_stdin, original_stdout, original_stderr = sys.stdin, sys.stdout, sys.stderr
try:
raw_stdin, sys.stdout, sys.stderr = native_engine.stdio_initialize(
global_level.level,
log_show_rust_3rdparty,
show_target,
{k: v.level
for k, v in log_levels_by_target.items()},
tuple(literal_filters),
tuple(regex_filters),
log_path,
)
sys.stdin = TextIOWrapper(
BufferedReader(raw_stdin),
# NB: We set the default encoding explicitly to bypass logic in the TextIOWrapper
# constructor that would poke the underlying file (which is not valid until a
# `stdio_destination` is set).
encoding=locale.getpreferredencoding(False),
)
sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr
# Install a Python logger that will route through the Rust logger.
with _python_logging_setup(global_level,
print_stacktrace=print_stacktrace,
local_cleanup=local_cleanup):
yield
finally:
sys.stdin, sys.stdout, sys.stderr = original_stdin, original_stdout, original_stderr
sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr
class Server:
"""
classdocs
"""
def __init__(self):
"""
Constructor
"""
# Read the configuration parameters from tulsi.conf
self.timestr = time.strftime("%Y:%m:%d-%H:%M:%S")
logging.basicConfig(filename="/var/log/tulsi.log", level=logging.DEBUG)
try:
self.conf = ConfigParser.ConfigParser()
self.conf.read("/etc/tulsi/tulsi.conf")
self.udp_ip = self.conf.get("tulsi", "host")
self.udp_port = int(self.conf.get("tulsi", "port"))
# printing the host and port of tulsi
logging.info("%s The IP of the host: %s" % (self.timestr, self.udp_ip))
logging.info("%s The Port number of the host :%s" % (self.timestr, self.udp_port))
except:
# Error message of tulsi not working
logging.error("The tulsi configuration file is not found")
# Creating objects of MessageEncode and HostInfo
msg_encode = MessageEncode()
host_info = HostInfo()
# Initializing empty lists
self.drives = []
self.service = []
self.ip_array = []
self.ring_ip = []
self.ring_conf_ip = []
self.ring_drives = []
self.ip_set_array = []
self.my_ring_conf = dict()
# Read the ring Configuration file
self.gz_file = GzipFile("/etc/swift/container.ring.gz", "rb")
if hasattr(self.gz_file, "_checkReadable"):
self.gz_file = BufferedReader(self.gz_file)
magic = self.gz_file.read(4)
if magic == "R1NG":
version, = struct.unpack("!H", self.gz_file.read(2))
if version == 1:
self.ring_data = self.read_ring_file(self.gz_file)
else:
logging.error("%s Unknown ring format version %d" % (self.timestr, version))
raise Exception("Unknown ring format version %d" % version)
# While loop to continuously check the status of swift services and
# drives and send information to tulsi client
while True:
self.ip_array = host_info.read_ip()
self.service = host_info.read_services()
self.drives = host_info.read_drives(self.drives)
self.message = msg_encode.create_message(
self.my_ring_conf, self.ring_conf_ip, self.ip_array, self.service, self.drives
)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet # UDP
sock.sendto(self.message, (self.udp_ip, self.udp_port))
time.sleep(5)
self.ip_array = []
self.service = []
self.drives = []
# Function to extract the ip and devices from container.ring.gz
def read_ring_file(self, sgz_file):
self.json_len, = struct.unpack("!I", self.gz_file.read(4))
self.ring_dict = json.loads(self.gz_file.read(self.json_len))
logging.info("Extracted Ring data : %s", self.ring_dict)
self.ring_dict["replica2part2dev_id"] = []
self.partition_count = 1 << (32 - self.ring_dict["part_shift"])
# logging.info('%s The Ip from ring file %s' % (self.timestr, self.ring_conf_ip[]]))
# logging.info('%s The IP of host machine %s' %(self.timestr, self.ip_array))
for x in self.ring_dict["devs"]:
self.mystring = x
if self.mystring["ip"] in self.my_ring_conf:
# append the new number to the existing array at this slot
self.my_ring_conf[self.mystring["ip"]].append(self.mystring["device"])
else:
# create a new array in this slot
self.my_ring_conf[self.mystring["ip"]] = [self.mystring["device"]]
self.ring_conf_ip.append(self.mystring["ip"])
logging.info("%s The Ip from ring file %s" % (self.timestr, self.ring_conf_ip))
logging.info("%s The IP of host machine %s" % (self.timestr, self.ip_array))
def read(rfile: io.BufferedReader) -> typing.Any:
x = rfile.readline().strip()
if not x:
return None
return json.loads(x)
def s3_handler(event, context, metadata):
# Need to use path style to access s3 via VPC Endpoints
# https://github.com/gford1000-aws/lambda_s3_access_using_vpc_endpoint#boto3-specific-notes
if DD_USE_VPC:
s3 = boto3.client(
"s3",
os.environ["AWS_REGION"],
config=botocore.config.Config(s3={"addressing_style": "path"}),
)
else:
s3 = boto3.client("s3")
# if this is a S3 event carried in a SNS message, extract it and override the event
if "Sns" in event["Records"][0]:
event = json.loads(event["Records"][0]["Sns"]["Message"])
# Get the object from the event and show its content type
bucket = event["Records"][0]["s3"]["bucket"]["name"]
key = urllib.parse.unquote_plus(event["Records"][0]["s3"]["object"]["key"])
source = parse_event_source(event, key)
metadata[DD_SOURCE] = source
##default service to source value
metadata[DD_SERVICE] = source
##Get the ARN of the service and set it as the hostname
hostname = parse_service_arn(source, key, bucket, context)
if hostname:
metadata[DD_HOST] = hostname
# Extract the S3 object
response = s3.get_object(Bucket=bucket, Key=key)
body = response["Body"]
data = body.read()
# Decompress data that has a .gz extension or magic header http://www.onicos.com/staff/iz/formats/gzip.html
if key[-3:] == ".gz" or data[:2] == b"\x1f\x8b":
with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream:
# Reading line by line avoid a bug where gzip would take a very long time (>5min) for
# file around 60MB gzipped
data = b"".join(BufferedReader(decompress_stream))
if is_cloudtrail(str(key)):
cloud_trail = json.loads(data)
for event in cloud_trail["Records"]:
# Create structured object and send it
structured_line = merge_dicts(
event, {"aws": {
"s3": {
"bucket": bucket,
"key": key
}
}})
yield structured_line
else:
# Check if using multiline log regex pattern
# and determine whether line or pattern separated logs
data = data.decode("utf-8")
if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(
data):
split_data = multiline_regex.split(data)
else:
split_data = data.splitlines()
# Send lines to Datadog
for line in split_data:
# Create structured object and send it
structured_line = {
"aws": {
"s3": {
"bucket": bucket,
"key": key
}
},
"message": line,
}
yield structured_line
class MultipartReader(object):
"""
Reads a stream formatted as multipart/form-data. Provides each part as a
readable interface, avoiding loading the entire body into memory or
caching to a file.
Usage:
reader = MultipartReader(f, boundary)
part1 = reader.next_part()
print part1.form_name()
data = part1.read(1024)
"""
def __init__(self, stream, boundary=None):
b = "\r\n--" + boundary + "--"
stream = _StreamWrapper(stream)
self.buf_reader = BufferedReader(stream)
self.nl = b[:2]
self.nl_dash_boundary = b[:len(b)-2]
self.dash_boundary_dash = b[2:]
self.dash_boundary = b[2:len(b)-2]
self.headers = {}
self.parts_read = 0
self.current_part = None
def iter_parts(self):
"""
Returns an iterator over the Parts in multipart/form-data.
Do not use if you're skipping the end of the data, as the last
iteration will seek to the end of the stream.
"""
part = self.next_part()
while part != None:
yield part
part = self.next_part()
def next_part(self):
"""
Returns the next Part in the stream. If a previous part was not read
completely, it will seek to the beginning of the next part, closing the
previous one.
"""
if self.current_part != None:
self.current_part.close()
expect_new_part = False
while True:
line = self.buf_reader.readline()
is_EOF = self.buf_reader.peek(1)
if len(is_EOF) == 0 and self.is_final_boundary(line):
return None
if self.is_boundary_delimeter_line(line):
#print "Creating new part"
self.parts_read += 1
bp = _new_part(self)
self.current_part = bp
return bp
if self.is_final_boundary(line):
return None
if expect_new_part:
raise Exception("expecting a new Part, got line %s" % line)
if self.parts_read == 0:
continue
if line == self.nl:
expect_new_part = True
continue
raise Exception("Unexpected line in next_part(): %s" % line)
def is_final_boundary(self, line):
if not line.startswith(self.dash_boundary_dash):
return False
rest = line[len(self.dash_boundary_dash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 or rest == self.nl
def is_boundary_delimeter_line(self, line):
if not line.startswith(self.dash_boundary):
return False
rest = line[len(self.dash_boundary):]
rest = skipLWSPChar(rest)
if self.parts_read == 0 and len(rest) == 1 and rest[0] == "\n":
self.nl = self.nl[1:]
self.nl_dash_boundary = self.nl_dash_boundary[1:]
return rest == self.nl
def peek_buffer_is_empty_part(self, peek):
if peek.startswith(self.dash_boundary_dash):
rest = peek[len(self.dash_boundary_dash):]
rest = skipLWSPChar(rest)
return rest.startswith(self.nl) or len(rest) == 0
if not peek.startswith(self.dash_boundary):
return False
rest = peek[len(self.dash_boundary):]
rest = skipLWSPChar(rest)
return rest.startswith(self.nl)
def test_io_bufferedreader(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp)
self.assertEqual(br.read(), b'foo')
br.close()
self.assertEqual(resp.closed, True)
b = b'fooandahalf'
fp = BytesIO(b)
resp = HTTPResponse(fp, preload_content=False)
br = BufferedReader(resp, 5)
br.read(1) # sets up the buffer, reading 5
self.assertEqual(len(fp.read()), len(b) - 5)
# This is necessary to make sure the "no bytes left" part of `readinto`
# gets tested.
while not br.closed:
br.read(5)
def awslogs_handler(event, context, metadata):
# Get logs
with gzip.GzipFile(fileobj=BytesIO(
base64.b64decode(event["awslogs"]["data"]))) as decompress_stream:
# Reading line by line avoid a bug where gzip would take a very long
# time (>5min) for file around 60MB gzipped
data = b"".join(BufferedReader(decompress_stream))
logs = json.loads(data)
# Set the source on the logs
source = logs.get("logGroup", "cloudwatch")
# Use the logStream to identify if this is a CloudTrail event
# i.e. 123456779121_CloudTrail_us-east-1
if "_CloudTrail_" in logs["logStream"]:
source = "cloudtrail"
metadata[DD_SOURCE] = parse_event_source(event, source)
# Default service to source value
metadata[DD_SERVICE] = metadata[DD_SOURCE]
# Build aws attributes
aws_attributes = {
"aws": {
"awslogs": {
"logGroup": logs["logGroup"],
"logStream": logs["logStream"],
"owner": logs["owner"],
}
}
}
# Set host as log group where cloudwatch is source
if metadata[DD_SOURCE] == "cloudwatch" or metadata.get(DD_HOST,
None) == None:
metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"]
# When parsing rds logs, use the cloudwatch log group name to derive the
# rds instance name, and add the log name of the stream ingested
if metadata[DD_SOURCE] in ["rds", "mariadb", "mysql"]:
match = rds_regex.match(logs["logGroup"])
if match is not None:
metadata[DD_HOST] = match.group("host")
metadata[DD_CUSTOM_TAGS] = (metadata[DD_CUSTOM_TAGS] +
",logname:" + match.group("name"))
# We can intuit the sourcecategory in some cases
if match.group("name") == "postgresql":
metadata[DD_CUSTOM_TAGS] + ",sourcecategory:" + match.group(
"name")
# For Lambda logs we want to extract the function name,
# then rebuild the arn of the monitored lambda using that name.
# Start by splitting the log group to get the function name
if metadata[DD_SOURCE] == "lambda":
log_group_parts = logs["logGroup"].split("/lambda/")
if len(log_group_parts) > 1:
lowercase_function_name = log_group_parts[1].lower()
# Split the arn of the forwarder to extract the prefix
arn_parts = context.invoked_function_arn.split("function:")
if len(arn_parts) > 0:
arn_prefix = arn_parts[0]
# Rebuild the arn with the lowercased function name
lowercase_arn = arn_prefix + "function:" + lowercase_function_name
# Add the lowercased arn as a log attribute
arn_attributes = {"lambda": {"arn": lowercase_arn}}
aws_attributes = merge_dicts(aws_attributes, arn_attributes)
env_tag_exists = (metadata[DD_CUSTOM_TAGS].startswith("env:")
or ",env:" in metadata[DD_CUSTOM_TAGS])
# If there is no env specified, default to env:none
if not env_tag_exists:
metadata[DD_CUSTOM_TAGS] += ",env:none"
# The EKS log group contains various sources from the K8S control plane.
# In order to have these automatically trigger the correct pipelines they
# need to send their events with the correct log source.
if metadata[DD_SOURCE] == "eks":
if logs["logStream"].startswith("kube-apiserver-audit-"):
metadata[DD_SOURCE] = "kubernetes.audit"
elif logs["logStream"].startswith("kube-scheduler-"):
metadata[DD_SOURCE] = "kube_scheduler"
# In case the conditions above don't match we maintain eks as the source
# Create and send structured logs to Datadog
for log in logs["logEvents"]:
yield merge_dicts(log, aws_attributes)
def bndl_create(args):
log = logging.getLogger(__name__)
if args.source is not None and not (os.path.isfile(args.source)
or os.path.isdir(args.source)):
log.error(
'bndl: Unable to read {}. Must be the path to a valid file or directory'
.format(args.source))
return 2
buff_in = BufferedReader(sys.stdin.buffer, IO_CHUNK_SIZE)
if args.format is None:
if args.source is None:
args.format = detect_format_stream(buff_in.peek(BNDL_PEEK_SIZE))
elif os.path.isdir(args.source):
args.format = detect_format_dir(args.source)
elif os.path.isfile(args.source):
with open(args.source, 'rb') as source_in:
args.format = detect_format_stream(
source_in.read(BNDL_PEEK_SIZE))
output = sys.stdout.buffer if args.output is None else open(
args.output, 'wb')
temp_dir = tempfile.mkdtemp()
component_name = 'oci-image'
oci_image_dir = os.path.join(temp_dir, component_name)
os.mkdir(oci_image_dir)
try:
if args.format is None:
log.error(
'bndl: Unable to detect format. Provide a -f or --format argument'
)
return 2
elif args.format == 'docker':
if args.source is None:
with tarfile.open(fileobj=buff_in, mode='r|') as tar_in:
name = docker_unpack(oci_image_dir,
tar_in,
is_dir=False,
maybe_name=args.name,
maybe_tag=args.tag)
elif os.path.isfile(args.source):
with tarfile.open(args.source, mode='r') as tar_in:
name = docker_unpack(oci_image_dir,
tar_in,
is_dir=False,
maybe_name=args.name,
maybe_tag=args.tag)
else:
name = docker_unpack(oci_image_dir,
args.source,
is_dir=True,
maybe_name=args.name,
maybe_tag=args.tag)
if name is None:
log.error('bndl: Not a Docker image')
return 3
elif args.name is None:
args.name = name
elif args.format == 'oci-image':
if args.name is None:
log.error(
'bndl: OCI Image support requires that you provide a --name argument'
)
return 2
elif args.source is None:
with tarfile.open(fileobj=buff_in, mode='r|') as tar_in:
valid_image = oci_image_unpack(oci_image_dir,
tar_in,
is_dir=False)
elif os.path.isfile(args.source):
with tarfile.open(args.source, mode='r') as tar_in:
valid_image = oci_image_unpack(oci_image_dir,
tar_in,
is_dir=False)
else:
valid_image = oci_image_unpack(oci_image_dir,
args.source,
is_dir=True)
if not valid_image:
log.error('bndl: Not an OCI Image')
return 2
has_oci_layout = os.path.isfile(
os.path.join(oci_image_dir, 'oci-layout'))
refs_dir = os.path.join(oci_image_dir, 'refs')
if args.tag is None and os.path.isdir(refs_dir):
for ref in os.listdir(refs_dir):
args.tag = ref
break
ref_exists = args.tag is not None and os.path.isfile(
os.path.join(refs_dir, args.tag))
if not ref_exists:
log.error(
'bndl: Invalid OCI Image. Cannot find requested tag "{}" in OCI Image'
.format(args.tag))
return 2
if not has_oci_layout:
log.error('bndl: Invalid OCI Image. Missing oci-layout')
return 2
oci_manifest, oci_config = oci_image_extract_manifest_config(
oci_image_dir, args.tag)
bundle_conf = oci_image_bundle_conf(args, component_name, oci_manifest,
oci_config)
bundle_conf_name = os.path.join(args.name, 'bundle.conf')
bundle_conf_data = bundle_conf.encode('UTF-8')
# bundle.conf must be written first, so we explicitly do that for zip and tar
if args.use_shazar:
with tempfile.NamedTemporaryFile() as zip_file_data:
with zipfile.ZipFile(zip_file_data, 'w') as zip_file:
zip_file.writestr(bundle_conf_name, bundle_conf_data)
dir_to_zip(temp_dir, zip_file, args.name)
zip_file_data.flush()
zip_file_data.seek(0)
write_with_digest(zip_file_data, output)
else:
with tarfile.open(fileobj=output, mode='w|') as tar:
info = tarfile.TarInfo(name=bundle_conf_name)
info.size = len(bundle_conf_data)
tar.addfile(tarinfo=info, fileobj=BytesIO(bundle_conf_data))
for (dir_path, dir_names, file_names) in os.walk(temp_dir):
for file_name in file_names:
path = os.path.join(dir_path, file_name)
name = os.path.join(
args.name, os.path.relpath(path, start=temp_dir))
tar.add(path, arcname=name)
output.flush()
return 0
finally:
shutil.rmtree(temp_dir)
def play(output_dir : str, rom : io.BufferedReader) -> None:
with open(os.path.join(output_dir, 'gameloader.js'), 'wb') as fh:
fh.write(b"onDataLoad('" + base64.b64encode(rom.read()) + b"');")
copytree(os.path.join(os.path.dirname(__file__), 'assets'),
os.path.join(output_dir))
def s3_handler(event, context, metadata):
s3 = boto3.client("s3")
# Get the object from the event and show its content type
bucket = event["Records"][0]["s3"]["bucket"]["name"]
key = urllib.unquote_plus(
event["Records"][0]["s3"]["object"]["key"]).decode("utf8")
source = parse_event_source(event, key)
metadata[DD_SOURCE] = source
##default service to source value
metadata[DD_SERVICE] = source
##Get the ARN of the service and set it as the hostname
hostname = parse_service_arn(source, key, bucket, context)
if hostname:
metadata[DD_HOST] = hostname
# Extract the S3 object
response = s3.get_object(Bucket=bucket, Key=key)
body = response["Body"]
data = body.read()
# If the name has a .gz extension, then decompress the data
if key[-3:] == ".gz":
with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream:
# Reading line by line avoid a bug where gzip would take a very long time (>5min) for
# file around 60MB gzipped
data = "".join(BufferedReader(decompress_stream))
if is_cloudtrail(str(key)):
cloud_trail = json.loads(data)
for event in cloud_trail["Records"]:
# Create structured object and send it
structured_line = merge_dicts(
event, {"aws": {
"s3": {
"bucket": bucket,
"key": key
}
}})
yield structured_line
else:
# Check if using multiline log regex pattern
# and determine whether line or pattern separated logs
if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(
data):
split_data = multiline_regex.split(data)
else:
split_data = data.splitlines()
# Send lines to Datadog
for line in split_data:
# Create structured object and send it
structured_line = {
"aws": {
"s3": {
"bucket": bucket,
"key": key
}
},
"message": line,
}
yield structured_line
def callback_query(bot, update):
cb = update.callback_query
chat_id = cb.message.chat_id
data = update.callback_query.data
logger.info(data)
data = data.split('%')
action = ''
offset = 0
disabled_attachments = set()
query = ''
confirmed = False
show_download = True
for elem in data:
name, *args = elem.split('=')
if name == 'act':
action = args[0]
elif name == 'off':
offset = int(args[0])
elif name == 'noatt':
disabled_attachments = set(int(arg) for arg in args if arg != '')
elif name == 'qry':
query = '='.join(args)
elif name == 'cnf':
confirmed = bool(int(args[0]))
elif name == 'dl':
show_download = bool(int(args[0]))
reporter = get_reporter(cb.from_user)
if action == 'old':
new_offset = offset + 1
elif action == 'new':
new_offset = offset - 1
else:
new_offset = offset
try:
believers = select(
s for s in Believer if query in s.phone_nr or query in s.account_nr
or query in s.bank_name or query in s.remark).order_by(
desc(Believer.created))[new_offset:new_offset + 1]
except TypeError:
believers = None
else:
offset = new_offset
reply = None
if action in ('old', 'new'):
if believers:
believer = believers[0]
reply = str(believer)
if not believer.attached_file:
disabled_attachments.add(offset)
confirmed = reporter in believer.reported_by if reporter else False
else:
bot.answerCallbackQuery(callback_query_id=cb.id,
text="No more results")
return
elif action == 'confirm':
if not believers:
bot.answerCallbackQuery(callback_query_id=cb.id,
text="Not found, please search again")
return
believer = believers[0]
if not confirmed:
if not reporter:
reporter = Reporter(id=cb.from_user.id,
first_name=cb.from_user.first_name,
last_name=cb.from_user.last_name,
username=cb.from_user.username)
track(update, 'new_reporter')
believer.reported_by.add(reporter)
bot.answerCallbackQuery(callback_query_id=cb.id,
text="You confirmed this report.")
else:
believer.reported_by.remove(reporter)
bot.answerCallbackQuery(callback_query_id=cb.id,
text="You removed your confirmation.")
confirmed = not confirmed
reply = str(believer)
elif action == 'att':
if not believers:
bot.answerCallbackQuery(callback_query_id=cb.id,
text="Not found, please search again")
return
kind, _, file_id = believers[0].attached_file.partition(':')
if kind == 'photo':
bot.sendPhoto(chat_id,
photo=file_id,
reply_to_message_id=cb.message.message_id)
elif kind == 'document':
bot.sendDocument(chat_id,
document=file_id,
reply_to_message_id=cb.message.message_id)
disabled_attachments.add(offset)
elif action == 'dl':
bot.sendChatAction(chat_id, action=ChatAction.UPLOAD_DOCUMENT)
with db_session:
believers = select(
s for s in Believer
if query in s.phone_nr or query in s.account_nr
or query in s.bank_name or query in s.remark).limit(100)
content = "\r\n\r\n".join(str(s) for s in believers)
file = BytesIO(content.encode())
show_download = False
bot.sendDocument(
chat_id,
document=BufferedReader(file),
filename='search.txt',
reply_to_message_id=update.callback_query.message.message_id)
kb = search_keyboard(offset=offset,
show_download=show_download,
disabled_attachments=disabled_attachments,
confirmed=confirmed,
query=query)
reply_markup = InlineKeyboardMarkup(kb)
if reply:
bot.editMessageText(chat_id=chat_id,
message_id=cb.message.message_id,
text=reply,
reply_markup=reply_markup,
parse_mode=ParseMode.HTML)
else:
bot.editMessageReplyMarkup(
chat_id=chat_id,
message_id=update.callback_query.message.message_id,
reply_markup=reply_markup)
def magic_open(filename, verbose=False, cpus=None):
"""
To read uncompressed zip gzip bzip2 or tar.xx files
:param filename: either a path to a file, or a file handler
:returns: opened file ready to be iterated
"""
textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
if isinstance(filename, basestring) or isinstance(filename, basestring):
fhandler = open(filename, 'rb')
inputpath = True
if tarfile.is_tarfile(filename):
print('tar')
thandler = tarfile.open(filename)
if len(thandler.members) != 1:
raise NotImplementedError(
'Not exactly one file in this tar archieve.')
return magic_open(thandler.extractfile(thandler.getnames()[0]))
else:
fhandler = filename
filename = fhandler.name
inputpath = False
start_of_file = ''
if filename.endswith('.dsrc'):
dsrc_binary = which('dsrc')
if not dsrc_binary:
raise Exception('\n\nERROR: DSRC binary not found, install it from:'
'\nhttps://github.com/lrog/dsrc/releases')
proc = Popen([dsrc_binary, 'd', '-t%d' % (cpus or cpu_count()),
'-s', filename], stdout=PIPE, universal_newlines=True)
return proc.stdout
if inputpath:
start_of_file = fhandler.read(1024)
fhandler.seek(0)
if is_binary_string(start_of_file):
if start_of_file.startswith(b'\x50\x4b\x03\x04'):
if verbose:
print('zip')
zhandler = TextIOWrapper(zipfile.ZipFile(fhandler))
if len(zhandler.NameToInfo) != 1:
raise NotImplementedError(
'Not exactly one file in this zip archieve.')
return TextIOWrapper(BufferedReader(zhandler.open(list(zhandler.NameToInfo.keys())[0])))
if is_binary_string(start_of_file) and start_of_file.startswith(b'\x42\x5a\x68'):
if verbose:
print('bz2')
fhandler.close()
return TextIOWrapper(BufferedReader(bz2.BZ2File(filename)))
if is_binary_string(start_of_file) and start_of_file.startswith(b'\x1f\x8b\x08'):
if verbose:
print('gz')
return TextIOWrapper(BufferedReader(gzip.GzipFile(fileobj=fhandler)))
else:
if verbose:
print('text')
fhandler.close()
fhandler = open(filename, 'r')
return fhandler
def read(cls, fp: BufferedReader):
buf = fp.read(struct.calcsize(cls.__FMT))
if buf:
return cls(*struct.unpack(cls.__FMT, buf))
return None
def recv_server_encrypted_extensions(self, bytes_buffer) -> bytes:
def parse_wrapper(bytes_buffer):
wrapper = Wrapper.deserialize(bytes_buffer)
while wrapper.record_header.size > len(wrapper.payload):
wrapper.payload += self.socket.recv(
wrapper.record_header.size - len(wrapper.payload)
)
recdata = wrapper.record_header.serialize()
authtag = wrapper.auth_tag
ciphertext = wrapper.encrypted_data
decryptor = AES.new(
self.handshake_keys.server_key,
AES.MODE_GCM,
xor_iv(self.handshake_keys.server_iv, self.handshake_recv_counter),
)
decryptor.update(recdata)
plaintext = decryptor.decrypt(bytes(ciphertext))
self.handshake_recv_counter += 1
decryptor.verify(authtag)
return plaintext[:-1]
plaintext = bytearray()
plaintext += parse_wrapper(bytes_buffer)
plaintext_buffer = BufferedReader(BytesIO(plaintext))
# TODO: change this to walrus operator
while True:
# print("difference 1", plaintext_buffer.tell() < len(plaintext))
hh = HandshakeHeader.deserialize(plaintext_buffer.read(4))
# print("hh", hh)
hh_payload_buffer = plaintext_buffer.read(hh.size)
while len(hh_payload_buffer) < hh.size:
res = parse_wrapper(bytes_buffer)
plaintext += res
plaintext_buffer = BufferedReader(
BytesIO(plaintext_buffer.peek() + res)
)
prev_len = len(hh_payload_buffer)
hh_payload_buffer = hh_payload_buffer + plaintext_buffer.read(
hh.size - prev_len
)
# # print("difference", hh.size - len(hh_payload_buffer))
# new_bytes = parse_wrapper(bytes_buffer)
# index = plaintext_buffer.tell()
# plaintext_buffer.seek(0, 2)
# plaintext_buffer.write(new_bytes)
# plaintext_buffer.seek(index)
# hh_payload_buffer = hh_payload_buffer + plaintext_buffer.read(hh.size - len(hh_payload_buffer))
# print("updated plaintext_buffer!!!")
# print(bytes(plaintext_buffer.getbuffer()))
# raise Exception("We need more data!!!")
hh_payload = HANDSHAKE_HEADER_TYPES[hh.message_type].deserialize(
hh_payload_buffer
)
# print("hh_payload", len(hh_payload.data), hh, type(hh_payload))
if type(hh_payload) is HandshakeFinishedHandshakePayload:
break
# print("done!!")
return plaintext
def handle_commands(bot, update, args):
chat_id = update.message.chat.id
command = update.message.text[1::]
bot.send_chat_action(chat_id, 'typing')
user_id = wrapper(
get_user_id, {
"schema": SCHEMA,
"privilege": "select",
"uid": update.message.from_user.id
})
if user_id < 0:
contact_keyboard = KeyboardButton(text="enviar_contacto",
request_contact=True)
reply_markup = ReplyKeyboardMarkup([[contact_keyboard]])
bot.send_message(chat_id,
"Acceso no permitido!",
reply_markup=reply_markup)
else:
header = wrapper(print_header, {
"schema": SCHEMA,
"privilege": "select",
"user_id": user_id
})
if 'reproduccion' in command:
if len(args) < 1:
bot.send_message(chat_id, 'Mantener presionado el comando y luego escribir ' + \
'fecha inicio y fecha final')
elif len(args) != 2:
bot.send_message(chat_id,
'Escribir fecha inicial y fecha final.')
else:
d1, d2 = tuple(args)
if sub('\d{4}\-\d{2}\-\d{2}', '', d1) != '':
bot.send_message(
chat_id, 'Fecha inicial mal escrita. ej. yyyy-mm-dd')
elif sub('\d{4}\-\d{2}\-\d{2}', '', d2) != '':
bot.send_message(
chat_id, 'Fecha final mal escrita. ej. yyyy-mm-dd')
else:
content = wrapper(
print_repro_resumen, {
"schema": SCHEMA,
"privilege": "select",
"d1": d1,
"d2": d2
})
pdf = BytesIO(set_pdf(header[0], content[0]))
pdf.name = 'doc.pdf'
bot.send_document(chat_id, BufferedReader(pdf))
else:
content = wrapper(
print_resumen, {
"schema": SCHEMA,
"privilege": "select",
"pdf": True,
"user_id": user_id,
"code": commands[command]['code']
})
pdf = BytesIO(set_pdf(header[0], content[0]))
pdf.name = 'doc.pdf'
bot.send_document(chat_id, BufferedReader(pdf))
class BinaryReader:
"""
Small utility class to read binary data.
Also creates a "Memory Stream" if necessary
"""
def __init__(self, data=None, stream=None):
if data:
self.stream = BytesIO(data)
elif stream:
self.stream = stream
else:
raise ValueError('Either bytes or a stream must be provided')
self.reader = BufferedReader(self.stream)
self._last = None # Should come in handy to spot -404 errors
# region Reading
# "All numbers are written as little endian."
# https://core.telegram.org/mtproto
def read_byte(self):
"""Reads a single byte value."""
return self.read(1)[0]
def read_int(self, signed=True):
"""Reads an integer (4 bytes) value."""
return int.from_bytes(self.read(4), byteorder='little', signed=signed)
def read_long(self, signed=True):
"""Reads a long integer (8 bytes) value."""
return int.from_bytes(self.read(8), byteorder='little', signed=signed)
def read_float(self):
"""Reads a real floating point (4 bytes) value."""
return unpack('<f', self.read(4))[0]
def read_double(self):
"""Reads a real floating point (8 bytes) value."""
return unpack('<d', self.read(8))[0]
def read_large_int(self, bits, signed=True):
"""Reads a n-bits long integer value."""
return int.from_bytes(
self.read(bits // 8), byteorder='little', signed=signed)
def read(self, length=None):
"""Read the given amount of bytes."""
if length is None:
return self.reader.read()
result = self.reader.read(length)
if len(result) != length:
raise BufferError(
'No more data left to read (need {}, got {}: {}); last read {}'
.format(length, len(result), repr(result), repr(self._last))
)
self._last = result
return result
def get_bytes(self):
"""Gets the byte array representing the current buffer as a whole."""
return self.stream.getvalue()
# endregion
# region Telegram custom reading
def tgread_bytes(self):
"""
Reads a Telegram-encoded byte array, without the need of
specifying its length.
"""
first_byte = self.read_byte()
if first_byte == 254:
length = self.read_byte() | (self.read_byte() << 8) | (
self.read_byte() << 16)
padding = length % 4
else:
length = first_byte
padding = (length + 1) % 4
data = self.read(length)
if padding > 0:
padding = 4 - padding
self.read(padding)
return data
def tgread_string(self):
"""Reads a Telegram-encoded string."""
return str(self.tgread_bytes(), encoding='utf-8', errors='replace')
def tgread_bool(self):
"""Reads a Telegram boolean value."""
value = self.read_int(signed=False)
if value == 0x997275b5: # boolTrue
return True
elif value == 0xbc799737: # boolFalse
return False
else:
raise RuntimeError('Invalid boolean code {}'.format(hex(value)))
def tgread_date(self):
"""Reads and converts Unix time (used by Telegram)
into a Python datetime object.
"""
value = self.read_int()
if value == 0:
return None
else:
return datetime.fromtimestamp(value, tz=timezone.utc)
def tgread_object(self):
"""Reads a Telegram object."""
constructor_id = self.read_int(signed=False)
clazz = tlobjects.get(constructor_id, None)
if clazz is None:
# The class was None, but there's still a
# chance of it being a manually parsed value like bool!
value = constructor_id
if value == 0x997275b5: # boolTrue
return True
elif value == 0xbc799737: # boolFalse
return False
elif value == 0x1cb5c415: # Vector
return [self.tgread_object() for _ in range(self.read_int())]
clazz = core_objects.get(constructor_id, None)
if clazz is None:
# If there was still no luck, give up
self.seek(-4) # Go back
pos = self.tell_position()
error = TypeNotFoundError(constructor_id, self.read())
self.set_position(pos)
raise error
return clazz.from_reader(self)
def tgread_vector(self):
"""Reads a vector (a list) of Telegram objects."""
if 0x1cb5c415 != self.read_int(signed=False):
raise RuntimeError('Invalid constructor code, vector was expected')
count = self.read_int()
return [self.tgread_object() for _ in range(count)]
# endregion
def close(self):
"""Closes the reader, freeing the BytesIO stream."""
self.reader.close()
# region Position related
def tell_position(self):
"""Tells the current position on the stream."""
return self.reader.tell()
def set_position(self, position):
"""Sets the current position on the stream."""
self.reader.seek(position)
def seek(self, offset):
"""
Seeks the stream position given an offset from the current position.
The offset may be negative.
"""
self.reader.seek(offset, os.SEEK_CUR)
# endregion
# region with block
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def __init__(self, initial_bytes=None):
self.initial_bytes = initial_bytes
self.buf = BufferedReader(raw=BytesIO(initial_bytes))
def body():
data = BufferedReader(BytesIO(b"1" * 100), 10)
chunk = data.read(1)
while chunk:
yield chunk
chunk = data.read(1)
def __init__(self, init_text="", encoding="utf-8"):
"""Initialise simulated buffer."""
self._buf = BytesIO(init_text.encode(encoding))
super().__init__(BufferedReader(self._buf), encoding=encoding)
def _error_thread(self, fh: BufferedReader, buffer: list
): # TODO: move away from list, use a buffered data type
buffer.append(fh.read())
fh.flush()
def deserialize(self, f: Readable) -> None:
while True:
# read the key
try:
key = deser_string(f)
except Exception:
break
# Check for separator
if len(key) == 0:
break
# First byte of key is the type
key_type = struct.unpack("b", bytearray([key[0]]))[0]
if key_type == 0:
if self.non_witness_utxo:
raise PSBTSerializationError(
"Duplicate Key, input non witness utxo already provided"
)
elif len(key) != 1:
raise PSBTSerializationError(
"non witness utxo key is more than one byte type")
self.non_witness_utxo = CTransaction()
utxo_bytes = BufferedReader(BytesIO(
deser_string(f))) # type: ignore
self.non_witness_utxo.deserialize(utxo_bytes)
self.non_witness_utxo.rehash()
elif key_type == 1:
if self.witness_utxo:
raise PSBTSerializationError(
"Duplicate Key, input witness utxo already provided")
elif len(key) != 1:
raise PSBTSerializationError(
"witness utxo key is more than one byte type")
self.witness_utxo = CTxOut()
tx_out_bytes = BufferedReader(BytesIO(
deser_string(f))) # type: ignore
self.witness_utxo.deserialize(tx_out_bytes)
elif key_type == 2:
if len(key) != 34 and len(key) != 66:
raise PSBTSerializationError(
"Size of key was not the expected size for the type partial signature pubkey"
)
pubkey = key[1:]
if pubkey in self.partial_sigs:
raise PSBTSerializationError(
"Duplicate key, input partial signature for pubkey already provided"
)
sig = deser_string(f)
self.partial_sigs[pubkey] = sig
elif key_type == 3:
if self.sighash > 0:
raise PSBTSerializationError(
"Duplicate key, input sighash type already provided")
elif len(key) != 1:
raise PSBTSerializationError(
"sighash key is more than one byte type")
sighash_bytes = deser_string(f)
self.sighash = struct.unpack("<I", sighash_bytes)[0]
elif key_type == 4:
if len(self.redeem_script) != 0:
raise PSBTSerializationError(
"Duplicate key, input redeemScript already provided")
elif len(key) != 1:
raise PSBTSerializationError(
"redeemScript key is more than one byte type")
self.redeem_script = deser_string(f)
elif key_type == 5:
if len(self.witness_script) != 0:
raise PSBTSerializationError(
"Duplicate key, input witnessScript already provided")
elif len(key) != 1:
raise PSBTSerializationError(
"witnessScript key is more than one byte type")
self.witness_script = deser_string(f)
elif key_type == 6:
DeserializeHDKeypath(f, key, self.hd_keypaths)
elif key_type == 7:
if len(self.final_script_sig) != 0:
raise PSBTSerializationError(
"Duplicate key, input final scriptSig already provided"
)
elif len(key) != 1:
raise PSBTSerializationError(
"final scriptSig key is more than one byte type")
self.final_script_sig = deser_string(f)
elif key_type == 8:
if not self.final_script_witness.is_null():
raise PSBTSerializationError(
"Duplicate key, input final scriptWitness already provided"
)
elif len(key) != 1:
raise PSBTSerializationError(
"final scriptWitness key is more than one byte type")
witness_bytes = BufferedReader(BytesIO(
deser_string(f))) # type: ignore
self.final_script_witness.deserialize(witness_bytes)
else:
if key in self.unknown:
raise PSBTSerializationError(
"Duplicate key, key for unknown value already provided"
)
unknown_bytes = deser_string(f)
self.unknown[key] = unknown_bytes
def makefile(self, mode):
return BufferedReader(_GzipReader(self.fileobj))
def deserialize(self, psbt: str) -> None:
psbt_bytes = base64.b64decode(psbt.strip())
f = BufferedReader(BytesIO(psbt_bytes)) # type: ignore
end = len(psbt_bytes)
# Read the magic bytes
magic = f.read(5)
if magic != b"psbt\xff":
raise PSBTSerializationError("invalid magic")
# Read loop
while True:
# read the key
try:
key = deser_string(f)
except Exception:
break
# Check for separator
if len(key) == 0:
break
# First byte of key is the type
key_type = struct.unpack("b", bytearray([key[0]]))[0]
# Do stuff based on type
if key_type == 0x00:
# Checks for correctness
if not self.tx.is_null:
raise PSBTSerializationError(
"Duplicate key, unsigned tx already provided")
elif len(key) > 1:
raise PSBTSerializationError(
"Global unsigned tx key is more than one byte type")
# read in value
tx_bytes = BufferedReader(BytesIO(
deser_string(f))) # type: ignore
self.tx.deserialize(tx_bytes)
# Make sure that all scriptSigs and scriptWitnesses are empty
for txin in self.tx.vin:
if len(txin.scriptSig) != 0 or not self.tx.wit.is_null():
raise PSBTSerializationError(
"Unsigned tx does not have empty scriptSigs and scriptWitnesses"
)
else:
if key in self.unknown:
raise PSBTSerializationError(
"Duplicate key, key for unknown value already provided"
)
unknown_bytes = deser_string(f)
self.unknown[key] = unknown_bytes
# make sure that we got an unsigned tx
if self.tx.is_null():
raise PSBTSerializationError("No unsigned trasaction was provided")
# Read input data
for txin in self.tx.vin:
if f.tell() == end:
break
input = PartiallySignedInput()
input.deserialize(f)
self.inputs.append(input)
if input.non_witness_utxo:
input.non_witness_utxo.rehash()
if input.non_witness_utxo.sha256 != txin.prevout.hash:
raise PSBTSerializationError(
"Non-witness UTXO does not match outpoint hash")
if (len(self.inputs) != len(self.tx.vin)):
raise PSBTSerializationError(
"Inputs provided does not match the number of inputs in transaction"
)
# Read output data
for txout in self.tx.vout:
if f.tell() == end:
break
output = PartiallySignedOutput()
output.deserialize(f)
self.outputs.append(output)
if len(self.outputs) != len(self.tx.vout):
raise PSBTSerializationError(
"Outputs provided does not match the number of outputs in transaction"
)
def enqueue_output(out: BufferedReader, queue: Queue):
for line in iter(out.readline, b''):
queue.put(line.decode('utf-8'))
out.close()
def __init__(self, path, chunk_size=10000):
BufferedReader.__init__(self, FileIO(path))
self.chunk_size = chunk_size
self._callbacks = dict()
class BinaryReader:
"""
Small utility class to read binary data.
Also creates a "Memory Stream" if necessary
"""
def __init__(self, data=None, stream=None):
if data:
self.stream = BytesIO(data)
elif stream:
self.stream = stream
else:
raise InvalidParameterError(
'Either bytes or a stream must be provided')
self.reader = BufferedReader(self.stream)
self._last = None # Should come in handy to spot -404 errors
# region Reading
# "All numbers are written as little endian."
# https://core.telegram.org/mtproto
def read_byte(self):
"""Reads a single byte value"""
return self.read(1)[0]
def read_int(self, signed=True):
"""Reads an integer (4 bytes) value"""
return int.from_bytes(self.read(4), byteorder='little', signed=signed)
def read_long(self, signed=True):
"""Reads a long integer (8 bytes) value"""
return int.from_bytes(self.read(8), byteorder='little', signed=signed)
def read_float(self):
"""Reads a real floating point (4 bytes) value"""
return unpack('<f', self.read(4))[0]
def read_double(self):
"""Reads a real floating point (8 bytes) value"""
return unpack('<d', self.read(8))[0]
def read_large_int(self, bits, signed=True):
"""Reads a n-bits long integer value"""
return int.from_bytes(self.read(bits // 8),
byteorder='little',
signed=signed)
def read(self, length):
"""Read the given amount of bytes"""
result = self.reader.read(length)
if len(result) != length:
raise BufferError(
'No more data left to read (need {}, got {}: {}); last read {}'
.format(length, len(result), repr(result), repr(self._last)))
self._last = result
return result
def get_bytes(self):
"""Gets the byte array representing the current buffer as a whole"""
return self.stream.getvalue()
# endregion
# region Telegram custom reading
def tgread_bytes(self):
"""Reads a Telegram-encoded byte array,
without the need of specifying its length
"""
first_byte = self.read_byte()
if first_byte == 254:
length = self.read_byte() | (self.read_byte() << 8) | (
self.read_byte() << 16)
padding = length % 4
else:
length = first_byte
padding = (length + 1) % 4
data = self.read(length)
if padding > 0:
padding = 4 - padding
self.read(padding)
return data
def tgread_string(self):
"""Reads a Telegram-encoded string"""
return str(self.tgread_bytes(), encoding='utf-8', errors='replace')
def tgread_bool(self):
"""Reads a Telegram boolean value"""
value = self.read_int(signed=False)
if value == 0x997275b5: # boolTrue
return True
elif value == 0xbc799737: # boolFalse
return False
else:
raise ValueError('Invalid boolean code {}'.format(hex(value)))
def tgread_date(self):
"""Reads and converts Unix time (used by Telegram)
into a Python datetime object
"""
value = self.read_int()
return None if value == 0 else datetime.fromtimestamp(value)
def tgread_object(self):
"""Reads a Telegram object"""
constructor_id = self.read_int(signed=False)
clazz = tlobjects.get(constructor_id, None)
if clazz is None:
# The class was None, but there's still a
# chance of it being a manually parsed value like bool!
value = constructor_id
if value == 0x997275b5: # boolTrue
return True
elif value == 0xbc799737: # boolFalse
return False
# If there was still no luck, give up
raise TypeNotFoundError(constructor_id)
# Create an empty instance of the class and
# fill it with the read attributes
result = clazz.empty()
result.on_response(self)
return result
def tgread_vector(self):
"""Reads a vector (a list) of Telegram objects"""
if 0x1cb5c415 != self.read_int(signed=False):
raise ValueError('Invalid constructor code, vector was expected')
count = self.read_int()
return [self.tgread_object() for _ in range(count)]
# endregion
def close(self):
self.reader.close()
# region Position related
def tell_position(self):
"""Tells the current position on the stream"""
return self.reader.tell()
def set_position(self, position):
"""Sets the current position on the stream"""
self.reader.seek(position)
def seek(self, offset):
"""Seeks the stream position given an offset from the
current position. The offset may be negative
"""
self.reader.seek(offset, os.SEEK_CUR)
# endregion
# region with block
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def dataReceived(self, data):
"""
Parse the NATS.io protocol from chunks of data streaming from
the connected gnatsd.
The server settings will be set and connect will be sent with this
client's info upon an INFO, which should happen when the
transport connects.
Registered message callback functions will be called with MSGs
once parsed.
PONG will be called upon a ping.
An exception will be raised upon an ERR from gnatsd.
An +OK doesn't do anything.
"""
if self.remaining_bytes:
data = self.remaining_bytes + data
self.remaining_bytes = b""
data_buf = BufferedReader(BytesIO(data))
while True:
command = data_buf.read(4)
if command == b"-ERR":
raise NatsError(data_buf.read())
elif command == b"+OK\r":
val = data_buf.read(1)
if val != b"\n":
self.remaining_bytes += command
break
elif command == b"MSG ":
val = data_buf.readline()
if not val:
self.remaining_bytes += command
break
if not val.endswith(b"\r\n"):
self.remaining_bytes += command + val
break
meta_data = val.split(b" ")
n_bytes = int(meta_data[-1])
subject = meta_data[0].decode()
if len(meta_data) == 4:
reply_to = meta_data[2].decode()
elif len(meta_data) == 3:
reply_to = None
else:
self.remaining_bytes += command + val
break
sid = meta_data[1].decode()
if sid in self.sids:
on_msg = self.sids[sid]
else:
on_msg = self.on_msg
payload = data_buf.read(n_bytes)
if len(payload) != n_bytes:
self.remaining_bytes += command + val + payload
break
if on_msg:
on_msg(nats_protocol=self, sid=sid, subject=subject, reply_to=reply_to, payload=payload)
else:
stdout.write(command.decode())
stdout.write(val.decode())
stdout.write(payload.decode())
payload_post = data_buf.readline()
if payload_post != b"\r\n":
self.remaining_bytes += command + val + payload + payload_post
break
elif command == b"PING":
self.log.info("got PING")
self.pong()
val = data_buf.readline()
if val != b"\r\n":
self.remaining_bytes += command + val
break
elif command == b"PONG":
self.pout -= 1
val = data_buf.readline()
if val != b"\r\n":
self.remaining_bytes += command + val
break
elif command == b"INFO":
val = data_buf.readline()
if not val.endswith(b"\r\n"):
self.remaining_bytes += command + val
break
settings = json.loads(val.decode("utf8"))
self.server_settings = ServerInfo(**settings)
self.log.info("{server_info}", server_info=settings)
self.status = CONNECTED
self.connect()
if self.on_connect_d:
self.on_connect_d.callback(self)
self.on_connect_d = None
else:
self.log.info(b"Not handled command is: {command!r}", command=command)
val = data_buf.read()
self.remaining_bytes += command + val
if not data_buf.peek(1):
self.log.debug("emptied data")
break