def to_dict(string):
try:
if not isinstance(string, basestring):
return loads(str(string))
return loads(string)
except:
raise ValueError('Expected a string or buffer.')
def load_private_text(self,filename,pinreader=passphrase_callback):
log.info("Trying to load a private certificate.")
try:
loadsh = serializer.loads(open(filename,'r').read())
log.info("Load as plain text with no passphrase seems OK.")
except:
if pinreader != None:
try:
log.info("Load as encrypted text. Requiring passphrase.")
passphrase = pinreader(False)
if type(passphrase) != str:
raise Exception("User refused passphrase request.")
key = self.derive_savekey(passphrase)
self.private_save_key = key
decryptor = ciphers.xipher(key)
shcontent = decryptor.decrypt(open(filename,'r').read())
loadsh = serializer.loads(shcontent)
except Exception,e:
log.exception("Unable to decrypt given file: %s",e)
raise Exception("Unable to decrypt given file: %s" % e)
else:
raise Exception("Unable to load given file.")
def test_int(self):
dump = dumps(self.good_request_dict)
decoded = loads(dump)
self.assertEqual(decoded, self.good_request_dict)
with self.assertRaises(Exception):
dump = dumps(self.bad_request_dict)
decoded = loads(dump)
def read_from_bytestream(cls, buffer_stream, messageLength):
messageLength -= sizeof(MongoMsgHeader)
database = read_cstring(buffer_stream) ;messageLength -= len(database) + 1
commandName = read_cstring(buffer_stream) ;messageLength -= len(commandName) + 1
buffer = buffer_stream.read(messageLength)
metadata = bson.loads(buffer)
commandReply = bson.loads(buffer[len(bson.dumps(metadata)):])
return cls(database, commandName, metadata, commandReply)
def test_bsonrecorder(self):
# Verify bson output can be read and that it matches json.
bson_out = StringIO()
json_out = StringIO()
self.top.recorders = [BSONCaseRecorder(bson_out),
JSONCaseRecorder(json_out)]
self.top.run()
json_run = json.loads(json_out.getvalue())
inp = StringIO(bson_out.getvalue())
reclen = unpack('<L', inp.read(4))[0]
data = inp.read(reclen)
obj = bson.loads(data) # simulation_info
keys = sorted(obj.keys())
self.assertEqual(keys, sorted(json_run['simulation_info'].keys()))
for key in keys:
# graph sometimes serializes with nodes in differant order
# between json and bson. The graphs are still equivalent, but the
# assertion below will fail
if key not in ('uuid', 'graph',):
self.assertEqual(obj[key], json_run['simulation_info'][key])
driver_count = 1
case_count = 1
data = inp.read(4)
while data:
reclen = unpack('<L', data)[0]
data = inp.read(reclen)
obj = bson.loads(data) # driver_info or iteration_case
keys = sorted(obj.keys())
if '_driver_id' in obj: # iteration_case
case = 'iteration_case_%s' % case_count
self.assertEqual(keys, sorted(json_run[case].keys()))
for key in keys:
if key not in ('_driver_id', '_id', '_parent_id',
'timestamp'):
self.assertEqual(obj[key], json_run[case][key])
case_count += 1
else: # driver_info
driver = 'driver_info_%s' % driver_count
self.assertEqual(keys, sorted(json_run[driver].keys()))
for key in keys:
if key not in ('_id',):
self.assertEqual(obj[key], json_run[driver][key])
driver_count += 1
data = inp.read(4)
def getSignal(self, date=datetime.date.fromtimestamp(time.time()),
username=False, numpy=False):
username = self._checkUsername(username)
if not (type(date) == datetime.date or
type(date) == datetime.datetime):
raise Exception('date must be a date or datetime object')
return False
r = self.s.get(Beddit.APIBASE+'user/'+username +
'/%04d/%02d/%02d/signal.bson' %
(date.year, date.month, date.day))
data = bson.loads(r.content)
if not data or not numpy:
return data
import numpy as np
import numpy.lib.recfunctions as recfunctions
data['interval_start'] = datetime.datetime.strptime(
data['interval_start'], "%Y-%m-%dT%H:%M:%S")
data['interval_end'] = datetime.datetime.strptime(
data['interval_end'], "%Y-%m-%dT%H:%M:%S")
for key in data['channels']:
for key in data['channels']:
if 'sample_data' in data['channels'][key]:
dataset = data['channels'][key]['sample_data']
rate = data['channels'][key]['sample_rate']
sample_data = np.fromstring(dataset, np.dtype(
[(str(key), np.int16)]))
xaxis = np.linspace(0.0,
float(len(sample_data))/float(rate),
float(len(sample_data)))
dataset = recfunctions.append_fields(sample_data,
'time',
xaxis,
usemask=False)
data['channels'][key]['sample_data'] = dataset
return data
def test_legacy_uuid(self):
uuid = UUID('584bcd8f-6d81-485a-bac9-629c14b53847')
serialized = b' \x00\x00\x00\x05uuid\x00\x10\x00\x00\x00\x03XK\xcd\x8fm\x81HZ\xba\xc9b\x9c\x14\xb58G\x00'
obj = loads(serialized)
self.assertIsInstance(obj['uuid'], UUID)
self.assertTrue(obj['uuid'] == uuid)
def verify_sign(self,message,sign):
try:
if type(sign) == type(""):
j = serializer.loads(sign)
else:
j = sign
keyindex = 1
for key in self.keys:
signer = signature.signature(key.get_publickey())
if j.has_key(keyindex):
sig = j[keyindex]
elif j.has_key(str(keyindex)):
sig = j[str(keyindex)]
else:
return False
if not signer.verify(sig,message):
return False
keyindex += 1
except Exception,e:
log.warning("Failed verifying a sign, returning False. More details: %s",e)
print "Error: %s" % e
return False
def handle_slave_message(self, msg):
"""
Slaves send in heartbeats and test results. Handle finding the associated local slave instance and interpreting
the message.
:param msg: A bson encoded dict with message data.
action: A string representing the action being taken. Must have an associated function on the local slave class
with the @action decorator
slave_id: The id assigned to the slave. If this is not present the slave doesn't have one and should be sent one
<message specific data>
"""
# TODO: Think through the security implications of letting slaves report their own ids and uuids
data = bson.loads(msg)
if "action" in data:
# Slaves without a valid slave_id are unregistered and need to be created locally
# TODO: Move slave creation into load_slave
if "slave_id" not in data or data["slave_id"] not in self.slave_registry:
if data["action"] == "heartbeat":
# Heartbeats are typically the way slaves are discovered
slave = self.new_slave(data)
# TODO: handle slave creation failure
else:
log.error("Got non-heartbeat message from unknown slave")
return
else:
# Load the local instance
slave = self.load_slave(data)
# TODO: handle save loading failure
assert slave
# run_action looks up the @action method on the slave and runs it with data
slave.run_action(data)
def data_received(self, data: ByteString):
# Expected format: bson_encoding( {"channels": [List_of_channels], "message": message} )
# Example: bson.dumps({"channels": ["global"], "message": "Hello, world\nMultiline!"})
# Meaning: Send "Hello, world\nMultiline!" to channel "global", the sender gets identified by his socket.
# The first 4 byte specify the length of BSON document and will be used as a splitting mark.
# One attempt is made to decode the resulting object. Failure to decode discards the object.
# This sadly means if we ever loose sync to a client (can not happen incidentally with TCP/TLS)
# there is no way to decode further transmissions.
# TODO: Examine further handling in this case, maybe drop the connection since it's either an incompatible
# encoding or a malicious attempt.
logging.debug("Got raw data[{}] {!r} from {}".format(len(data), data, self.peername))
self.buffer.extend(data)
logging.debug("Buffer of {} contains {!r}".format(self.peername, self.buffer))
# TODO: maybe use memoryview to prevent useless copies on every slicing operation
if len(self.buffer) >= 4:
bson_expected_len = unpack("<i", self.buffer[:4])[0]
if len(self.buffer) >= bson_expected_len:
bson_obj = self.buffer[:bson_expected_len] # contains the (hopefully) valid BSON object
self.buffer = self.buffer[bson_expected_len:] # shifts the buffer to the start of the next object
try:
t = time.clock()
message = bson.loads(bson_obj)
delta = time.clock() - t
logging.debug("Decoded {} from {}, took {} s".format(message, self.peername, delta))
asyncio.ensure_future(self.chat.add_message(self, message))
except IndexError as e:
logging.warning("{}: BSONDecodeError: {}".format(self.peername, e))
pass
def _next(self):
""" Return next dictionary of data. """
data = self._inp.read(4)
if not data:
return None
reclen = unpack('<L', data)[0]
return bson.loads(self._inp.read(reclen))
def test_unknown_handler(self):
d = Decimal("123.45")
obj = {"decimal": d}
serialized = dumps(obj, on_unknown=float)
unserialized = loads(serialized)
self.assertEqual(float(d), unserialized["decimal"])
def __init__(self,keystr):
try:
if type(keystr) == str:
j = serializer.loads(keystr)
else:
j = keystr
if j['type'] == 'RSA_Public_Key':
self.key = _RSA()
self.is_private_key = False
elif j['type'] == 'EC_Public_Key':
self.key = _EC()
self.is_private_key = False
elif j['type'] == 'RSA_Private_Key':
self.key = _RSA()
self.is_private_key = True
elif j['type'] == 'EC_Private_Key':
self.key = _EC()
self.is_private_key = True
else:
raise Exception("Unrecognized type of public key.")
keystr = serializer.dumps(j)
if self.is_private_key:
self.key.load_privatekey(keystr)
else:
self.key.load_publickey(keystr)
except Exception,e:
raise Exception("Failed initilizing PublicKeyAlgorithm: %s" % e)
def _get_ordered_queue_owners_dict(self):
#return self._qm_redis_client.hgetall("oq_owners")
dumped_ordered_queue_owners_dict = self._qm_redis_client.get("oq_owners")
if dumped_ordered_queue_owners_dict == None:
return None
return bson.loads(dumped_ordered_queue_owners_dict)
def receivedData(self, received):
'''Receive int prefixed data until full bson doc.'''
self.recv_buffer += received
while len(self.recv_buffer) >= self.length_size:
length ,= struct.unpack(
self.length_fmt, self.recv_buffer[:self.length_size])
if length > self.max_bytes:
self.lengthLimitExceeded(length)
return # bad news bears.
if len(self.recv_buffer) < length:
break # not enough for full bson doc yet.
bsonData = self.recv_buffer[:length]
self.recv_buffer = self.recv_buffer[length:]
try:
bsonDoc = bson.loads(bsonData)
except Exception, e:
e.bsonData = bsonData
self.bsonDecodingError(e)
# Note: at this point, we may be off sync (warranting disconnect)
# but let's attempt to keep going!
else:
self.receivedBson(bsonDoc)
gevent.sleep(0) # cooperative yield. we got a document (a unit of work)
def load_signature(self,sign):
# 对于私或公用证书均可，加载一个签名信息，可能是签名或签名撤回信息
try:
if type(sign) == type(""):
j = serializer.loads(sign)
else:
j = sign
sig = j['Signature']
c = j['Content']
chash = self.signature_hash(j)
if not self.check_signature_content(c,loading=True):
raise Exception("This signature cannot be loaded. Either it is of invalid format, or it is not for this certificate.")
log.info('Signature loaded.')
for s in self.signatures:
if self.signature_hash(s) == chash:
log.warning('This signature has already been loaded.')
return
self.signatures.append(j)
except Exception,e:
log.warning('Cannot load the signature. Details: %s',e)
raise Exception("Error loading a signature: %s" % e)
def load(self,inp):
self.textbox.delete(1.0,END)
j = bson.loads(inp)
plaintext = zlib.decompress(j['t'])
stripped = self._stripText(plaintext)
def r(x,r=stripped[1],c=stripped[2]):
nr = x[0] - r
if nr == 1:
return "%d.%d" % (nr,x[1]-c)
else:
return "%d.%d" % (nr,x[1])
decorations = j['d']
self.textbox.insert(END,stripped[0])
for tagname,indexlist in decorations.items():
while indexlist:
if len(indexlist) >= 2:
cstart,cend = [r(t) for t in indexlist[0:2]]
indexlist = indexlist[2:]
else:
cstart = r(indexlist[0])
cend = END
indexlist = []
self.textbox.tag_add("tag%d" % int(tagname),cstart,cend)
def decodePacket_bson(packet):
"""解包"""
try:
packet = StringIO(packet)
fp = gzip.GzipFile(mode='rb', fileobj=packet)
packet = fp.read()
fp.close()
recv_data = bson.loads(packet)
assert isinstance(recv_data, dict), 'recv_data should be a dict'
optId = int(recv_data['opt_id'])
optKey = recv_data['opt_key']
if not isinstance(recv_data.get('para'), dict):
para = {}
else:
para = recv_data['para']
if 'stime' in recv_data:
#此处校验客户端与服务器时间
# + todo
para['ClientTime'] = int(recv_data['stime'])
else:
para['ClientTime'] = int(time.time())
return optId, optKey, para
except Exception, e:
print 'decodePacket 异常!无法解压数据 > ', e, packet
return False
def test_random_tree(self):
for i in xrange(0, 16):
p = {}
populate(p, 256, 4)
sp = dumps(p)
p2 = loads(sp)
self.assertEquals(p, p2)
def test_decimal(self):
decimal = Decimal('1234.45')
obj = {"decimal": decimal}
serialized = dumps(obj)
obj2 = loads(serialized)
self.assertIsInstance(obj2['decimal'], float)
self.assertTrue(obj2['decimal'] == float(decimal))
def __call__(self, *args):
postdata = dumps({"method": self.__serviceName, "params": args, "id": "jsonrpc"})
respdata = urllib.urlopen(self.__serviceURL, postdata).read()
resp = loads(respdata)
if resp["error"] != None:
raise JSONRPCException(resp["error"])
else:
return resp["result"]
def read_from_bytestream(cls, buffer_stream, messageLength):
messageLength -= sizeof(MongoMsgHeader)
flags = c_int.from_buffer_copy(buffer_stream.read(sizeof(c_int))).value ;messageLength -= sizeof(c_int)
fullCollectionName = read_cstring(buffer_stream) ;messageLength -= len(fullCollectionName) + 1
numberToSkip = c_int.from_buffer_copy(buffer_stream.read(sizeof(c_int))).value ;messageLength -= sizeof(c_int)
numberToReturn = c_int.from_buffer_copy(buffer_stream.read(sizeof(c_int))).value ;messageLength -= sizeof(c_int)
query = bson.loads(buffer_stream.read(messageLength))
return cls(flags, fullCollectionName, numberToSkip, numberToReturn, query)
def loadBsonObject(_hash): fpath = join(objectDir, _hash[:2], _hash[2:]) bsonBytes = open(fpath, "rb").read() if _hash != sha1(bsonBytes).hexdigest(): raise IOError( "sha1 diggest does not match for object file \"%s\"" % fpath ) return bson.loads(bsonBytes)
def test_uuid(self):
uuid = UUID('584bcd8f-6d81-485a-bac9-629c14b53847')
obj = {"uuid": uuid}
serialized = dumps(obj)
obj2 = loads(serialized)
self.assertIsInstance(obj2['uuid'], UUID)
self.assertTrue(obj2['uuid'] == uuid)
def test_false_value(self):
data = {"key": False}
serialized = bson.dumps(data)
deserialized = bson.loads(serialized)
self.assertIsInstance(deserialized["key"], bool)
self.assertFalse(deserialized["key"])
self.assertTrue(serialized == b'\x0b\x00\x00\x00\x08key\x00\x00\x00')
def create_from_received(raw_message, qm_dealer_id_tag):
"""
Returns a new message of this type from the raw message data.
"""
try:
return PeerOnlineHandshakeRequestMessage(bson.loads(raw_message[1]), raw_message[2], qm_dealer_id_tag)
except:
raise ExceptionFormatter.get_full_exception()
def GetSaveData(ID):
compressedsave = open("saves/{0}.cps".format(ID), "rb")
compressedsave.seek(12)
save = bz2.decompress(compressedsave.read())
try:
data = bson.loads(save)
except ValueError:
return None
return data
def create_from_received(raw_message, sender_id_string):
"""
Returns a new message of this type from the raw message data.
"""
try:
return PeerForwardedCommandMessage(None, bson.loads(raw_message[1]), sender_id_string)
except:
raise ExceptionFormatter.get_full_exception()
def test_datetime(self):
now = datetime.now(pytz.utc)
obj = {"now" : now}
serialized = dumps(obj)
obj2 = loads(serialized)
td = obj2["now"] - now
seconds_delta = (td.microseconds + (td.seconds + td.days * 24 * 3600) *
1e6) / 1e6
self.assertTrue(abs(seconds_delta) < 0.001)
def load(dumped_string):
"""
Returns an instance object of this class built from string which was created in the "dump" method.
"""
dumped_string = cast_bytes(dumped_string)
loaded_dictionary = bson.loads(dumped_string)
exchange_wrapper = ExchangeWrapper("", "")
exchange_wrapper.__dict__.update(loaded_dictionary)
return exchange_wrapper
def decode_bson_val(bts):
d = bson.loads(bts)
revision, item_val = d['e']
return revision, item_val
def bson_loads(s):
return bson.loads(s)
def message_to_pose(message):
pose_message = bson.loads(message.payload)
return [pose_message['timestamp']] + list(map(lambda k: pose_message['data']['position'][k], ['x', 'y', 'z'])) +\
list(map(lambda k: pose_message['data']['rotation'][k], ['w', 'x', 'y', 'z']))
def test_long_array(self):
serialized = dumps(self.doc)
doc2 = loads(serialized)
self.assertEquals(self.doc, doc2)
def read_bson(self, filename):
f = open(filename, "rb")
bson_data = f.read()
message_dict = bson.loads(bson_data)
print(message_dict)
def read_users():
chatusers = open('foundry/chatusers.bson', 'rb').read()
return bson.loads(chatusers.read())
def test_int_as_key(self):
dump = dumps(self.test_dict)
decoded = loads(dump)
dumpByJson = json.dumps(self.test_dict)
decodedByJson = json.loads(dumpByJson)
self.assertEqual(decoded, decodedByJson)
# import simplejson as serializer
import bson as serializer
import zmq
import sys
context = zmq.Context()
receiver = context.socket(zmq.SUB)
receiver.connect("tcp://127.0.0.1:6015")
receiver.setsockopt(zmq.SUBSCRIBE, '')
while True:
s = serializer.loads(receiver.recv())
sys.stdout.write(str(s) + '\n')
sys.stdout.flush()
def deserialize(self, stream):
return bson.loads(stream.read())
#encoding: utf-8
import bson
origin = {'a': 1}
print origin
data = bson.dumps(origin)
print data
data = bson.loads(data)
print data['a']
def _load_superblocks(self):
raw_data = self.disk.read(1024 * 1024)
return OrderedDict(bson.loads(raw_data)['superblocks'])
import sys import bson b = bson.loads(open(sys.argv[1], 'rb').read()) print(b)
def decode_bson(bts: bytes):
d = bson.loads(bts)
remap_d = remap_keys_decode(d)
return remap_d
def read_prefernces():
preferences = open('foundry/preferencesfile.bson', 'rb').read()
return bson.loads(preferences.read())
def process_server_msg(self, ws, raw_data):
global _print_event_tracker
try:
# raw_data can be both json or bson
# no py2 compat way to properly detect type here
# (w/o patching ws lib)
try:
msg = json.loads(raw_data)
_logger.debug('Received: ' + raw_data)
except ValueError:
msg = bson.loads(raw_data)
_logger.debug(
'received binary message ({} bytes)'.format(len(raw_data)))
for command in msg.get('commands', []):
if command["cmd"] == "pause":
self.commander.prepare_to_pause(
self._printer, **command.get('args'))
self._printer.pause_print()
if command["cmd"] == 'cancel':
self._printer.cancel_print()
if command["cmd"] == 'resume':
self._printer.resume_print()
if command["cmd"] == 'print':
self.start_print(**command.get('args'))
passsthru = msg.get('passthru')
if passsthru:
target = getattr(self, passsthru.get('target'))
func = getattr(target, passsthru['func'], None)
if not func:
return
ack_ref = passsthru.get('ref')
ret = func(*(passsthru.get("args", [])))
if ack_ref:
self.send_ws_msg_to_server({'passthru': {'ref': ack_ref, 'ret': ret}})
time.sleep(0.2) # chnages, such as setting temp will take a bit of time to be reflected in the status. wait for it
self.post_printer_status(_print_event_tracker.octoprint_data(self))
if msg.get('janus') and self.webcam_streamer:
self.webcam_streamer.pass_to_janus(msg.get('janus'))
if msg.get('remote_status'):
self.remote_status.update(msg.get('remote_status'))
if self.remote_status['viewing']:
self.jpeg_poster.post_jpeg_if_needed(force=True)
if msg.get('http.tunnel') and self.local_tunnel:
self.local_tunnel.send_http_to_local(**msg.get('http.tunnel'))
if msg.get('ws.tunnel') and self.local_tunnel:
kwargs = msg.get('ws.tunnel')
kwargs['type_'] = kwargs.pop('type')
self.local_tunnel.send_ws_to_local(**kwargs)
except:
self.sentry.captureException(tags=get_tags())
PAYLOADS = (
# OP_QUERY: {'flags': 0, 'fullCollectionName': b'admin.$cmd', 'numberToSkip': 0, 'numberToReturn': 1, 'query': {'isMaster': 1, 'client': {'application': {'name': 'MongoDB Shell'}, 'driver': {'name': 'MongoDB Internal Client', 'version': '3.4.4'}, 'os': {'type': 'Windows'$ 'name': 'Microsoft Windows 8', 'architecture': 'x86_64', 'version': '6.2 (build 9200)'}}}}
b'#\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd4\x07\x00\x00\x00\x00\x00\x00admin.$cmd\x00\x00\x00\x00\x00\x01\x00\x00\x00\xfc\x00\x00\x00\x10isMaster\x00\x01\x00\x00\x00\x03client\x00\xe1\x00\x00\x00\x03application\x00\x1d\x00\x00\x00\x02name\x00\x0e\x00\x00\x00MongoDB Shell\x00\x00\x03driver\x00:\x00\x00\x00\x02name\x00\x18\x00\x00\x00MongoDB Internal Client\x00\x02version\x00\x06\x00\x00\x003.4.4\x00\x00\x03os\x00l\x00\x00\x00\x02type\x00\x08\x00\x00\x00Windows\x00\x02name\x00\x14\x00\x00\x00Microsoft Windows 8\x00\x02architecture\x00\x07\x00\x00\x00x86_64\x00\x02version\x00\x11\x00\x00\x006.2 (build 9200)\x00\x00\x00\x00',
# OP_COMMAND: {'database': b'admin', 'commandName': b'whatsmyuri', 'metadata': {'whatsmyuri': 1}, 'commandReply': {}}
b';\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xda\x07\x00\x00admin\x00whatsmyuri\x00\x15\x00\x00\x00\x10whatsmyuri\x00\x01\x00\x00\x00\x00\x05\x00\x00\x00\x00',
# OP_COMMAND: {'database': b'admin', 'commandName': b'buildinfo', 'metadata': {'buildinfo': 1.0}, 'commandReply': {}}
b'=\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\xda\x07\x00\x00admin\x00buildinfo\x00\x18\x00\x00\x00\x01buildinfo\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x05\x00\x00\x00\x00',
# OP_COMMAND: {'database': b'test', 'commandName': b'isMaster', 'metadata': {'isMaster': 1.0, 'forShell': 1.0}, 'commandReply': {}}
b'L\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\xda\x07\x00\x00test\x00isMaster\x00)\x00\x00\x00\x01isMaster\x00\x00\x00\x00\x00\x00\x00\xf0?\x01forShell\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x05\x00\x00\x00\x00',
# OP_COMMAND: {'database': b'admin', 'commandName': b'replSetGetStatus', 'metadata': {'replSetGetStatus': 1.0, 'forShell': 1.0}, 'commandReply': {}}
b']\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\xda\x07\x00\x00admin\x00replSetGetStatus\x001\x00\x00\x00\x01replSetGetStatus\x00\x00\x00\x00\x00\x00\x00\xf0?\x01forShell\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x05\x00\x00\x00\x00',
)
tcp_connection = socket.create_connection((sys.argv[1], int(sys.argv[2])))
payload_index = int(sys.argv[3]) if len(sys.argv) > 3 else 1
tcp_connection.sendall(PAYLOADS[payload_index])
print('Payload sent')
msg_header = tcp_connection.recv(
16
) # reading MsgHeader, cf. https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#standard-message-header
if not msg_header:
raise RuntimeError('No response from server')
messageLength = c_int.from_buffer_copy(msg_header[:4]).value
resp = tcp_connection.recv(messageLength - 16)
print('SUCCESS!')
try:
import bson
print(bson.loads(resp[20:] if payload_index == 0 else resp))
except ImportError:
print(resp)
def body(self):
return bson.loads(self._packet_body)
def recvMessage(self, raw=False):
if raw:
message = super(BsonMessages, self).recvMessage()
else:
message = bson.loads(super(BsonMessages, self).recvMessage())
return message
import bson
def printTree(node, level):
if node is not None:
format = level * " "
format += "---[ "
level += 1
printTree(node["left"], level)
print(format + str(node["value"]))
printTree(node["right"], level)
pass
with open("tree.bson", "rb") as fin:
content = fin.read()
binary_tree = bson.loads(content)
printTree(binary_tree["root"], 0)
def ParseFromString(self, str_value):
self.ParseFromDict(bson.loads(str_value))
# 操作速度
# bson>json。比如，遍历查找：json需要扫字符串，而bson可以直接定位
# 修改：
# json也要大动大移，bson就不需要。
# gswewf@gswewf-PC:~$ sudo pip3 install bson
import bson
with open(
'/home/gswewf/docker/mongo/data/mongodb_192_168_3_130/page/保险条款.bson',
'rb') as f:
b_data = f.read()
data = bson.loads(b_data)
type(data)
# Out[5]: dict
list(data.keys())[:3]
# Out[6]: ['_id', 'content', '保险小类']
type(data.get('保险小类'))
# Out[7]: str
def main():
pass
if __name__ == '__main__':
main()
def get_encoded_image(self, path):
with open(path, 'rb') as f:
return image.fromDict(bson.loads(f.read()))
def run(self):
running = True
clock = pygame.time.Clock()
tickspeed = 60
last_direction = None
self.toMove = False # Flag for when player moves - reduces network stress
self.cast = False # Flag for when player casts spell.
self.status_time = 0
me = self.players.me
inputHandler = InputHandler(
) #Handles the inputs. They can get stage fright sometimes.
self.menu_setup()
self.state = "menu"
self.menu_current = self.menu_main
if me.mute == "False":
LevelMusic.play_music_repeat()
try:
while running:
self.screen.fill(colours["white"])
clock.tick(tickspeed)
if self.state == "menu":
#This means we're in the menus.
action = None
for event in pygame.event.get():
if event.type == pygame.QUIT or event.type == pygame.locals.QUIT:
running = False
break
if type(self.menu_current) is MenuInput:
action = self.menu_current.update(
inputHandler, event)
if type(self.menu_current) is MenuInput:
self.menu_current.render()
else:
action = self.menu_current.update(inputHandler)
if type(action) is Menu or type(action) is MenuInput:
self.menu_current = action
else:
if action == "help":
webbrowser.open_new_tab(
"https://github.com/neontribe/untangled-2017/wiki"
)
elif action == "mute":
if me.mute == "False":
me.set_mute("True", True)
LevelMusic.stop_music()
elif me.mute == "True":
me.set_mute("False", True)
LevelMusic.play_music_repeat()
elif action == "quit":
running = False
break
elif action == "play_ctf":
self.state = "game_ctf"
elif action == "rejoy":
inputHandler.reloadJoysticks()
elif action == "input_name":
self.players.me.set_name(
self.menu_current.char_name, True)
if self.menu_current.parent:
self.menu_current = self.menu_current.parent
elif self.state == "game_ctf": #There may only be one game at the moment, but this makes it easier to add different game types in the future.
#This means we have to actually run the game.
if last_direction == None:
last_direction = Movement.DOWN
for event in pygame.event.get():
if event.type == pygame.QUIT or event.type == pygame.locals.QUIT:
running = False
break
if not inputHandler.inputsTimeout["change"]:
inputHandler.setTimeout("change", 0.5)
inputHandler.setTimeout("enter", 0.5)
inputHandler.setTimeout("special", 10)
inputHandler.setTimeout("move", move_delay)
if inputHandler.checkPress("start"):
self.state = "menu" #Temporary workaround until a proper pause menu is added (or possibly not if this works fine).
elif inputHandler.checkHold("up"):
me.move(Movement.UP)
last_direction = Movement.UP
self.toMove = True
elif inputHandler.checkHold("down"):
me.move(Movement.DOWN)
last_direction = Movement.DOWN
self.toMove = True
elif inputHandler.checkHold("left"):
me.move(Movement.LEFT)
last_direction = Movement.LEFT
self.toMove = True
elif inputHandler.checkHold("right"):
me.move(Movement.RIGHT)
last_direction = Movement.RIGHT
self.toMove = True
elif inputHandler.checkPress("change"):
me.change_spell()
elif inputHandler.checkHold("enter"):
self.cast = me.attack(last_direction)
elif inputHandler.checkHold("special"):
me.step = 2
me.steptime = time.time()
if me.map.level.get_tile(me.x, me.y).has_attribute(
TileAttribute.SWIM):
inputHandler.setTimeout("move", 0.4)
elif me.map.level.get_tile(me.x, me.y).has_attribute(
TileAttribute.SLOW):
inputHandler.setTimeout("move", 0.2)
else:
inputHandler.setTimeout("move", move_delay)
if time.time() - me.steptime > 3:
me.step = 1
self.map.render()
for flag in self.flags.values():
flag.render()
me.render(True)
for spell in me.cast_spells:
spell.render()
self.players.set(self.network.node.peers())
# check network
events = self.network.get_events()
if events:
try:
for event in self.network.get_events():
if event.type == 'ENTER':
# Force update on first join.
self.toMove = True
auth_status = event.headers.get(
'AUTHORITY')
if auth_status == 'TRUE':
self.players.authority_uuid = str(
event.peer_uuid)
self.network.authority_uuid = str(
event.peer_uuid)
self.players.remove(event.peer_uuid)
elif event.type == "SHOUT":
if event.group == "player:name":
new_name = bson.loads(event.msg[0])
player = self.players.get(
event.peer_uuid)
if new_name['name']:
player.set_name(new_name['name'])
elif event.group == "world:position":
new_position = bson.loads(event.msg[0])
network_player = self.players.get(
event.peer_uuid)
network_player.set_position(
Position(**new_position))
elif event.group == "world:combat":
new_spell_properties = bson.loads(
event.msg[0])
network_spell_caster = self.players.get(
event.peer_uuid)
network_spell_caster.current_spell = new_spell_properties.get(
'current_spell')
network_spell_caster.cast_spells.append(
Spell(
network_spell_caster, (0, 0),
projectile_images[
network_spell_caster.
current_spell]))
network_spell_caster.cast_spells[
-1].set_properties(
SpellProperties(
**new_spell_properties))
elif event.group == "ctf:teams":
team_defs = bson.loads(event.msg[0])
self.players.set_teams(team_defs)
elif event.group == "ctf:gotflag":
flag_info = bson.loads(event.msg[0])
team = flag_info["team"]
uuid = flag_info["uuid"]
flag = self.flags[team]
if uuid == str(
self.network.node.uuid()):
flag.set_player(self.players.me)
else:
network_player = self.players.get(
uuid)
flag.set_player(network_player)
elif event.group == 'ctf:dropflag':
flag_info = bson.loads(event.msg[0])
flag = self.flags[flag_info['team']]
flag.set_player(None)
flag.set_position(
(flag_info['x'], flag_info['y']))
elif event.group == "players:whois":
self.network.node.shout(
"player:name",
bson.dumps(
{"name":
self.players.me.name}))
elif event.group == "ctf:status":
msg = bson.loads(event.msg[0])
status = msg['status']
self.status_message = status
self.status_time = time.time()
elif event.group == "ctf:scores":
scores = bson.loads(event.msg[0])
self.scores = scores
elif event.type == 'WHISPER':
msg = bson.loads(event.msg[0])
if self.players.authority_uuid == str(
event.peer_uuid):
if msg['type'] == 'teleport':
me.set_position(
(msg['x'], msg['y']))
self.toMove = True
elif msg['type'] == 'die':
me.die()
except Exception as e:
import traceback
# traceback.print_exc()
pass
# if there are other peers we can start sending to groups.
if self.toMove == True:
self.network.node.shout(
"world:position",
bson.dumps(me.get_position()._asdict()))
self.toMove = False
if self.cast == True:
try:
self.network.node.shout(
"world:combat",
bson.dumps(me.cast_spells[-1].get_properties().
_asdict()))
except:
print(error_message + ": me.cast_spells is empty")
self.cast = False
for playerUUID, player in self.players.others.items():
try:
player.render()
for spell in player.cast_spells:
spell.render()
hit_me = spell.hit_target_player(me)
if hit_me:
player.cast_spells.remove(spell)
me.deplete_health(spell.damage)
except PlayerException as e:
# PlayerException due to no initial position being set for that player
pass
score_shift = 220
for team, score in self.scores.items():
colour = colours[team]
display_rect = Rect((score_shift, 0), (200, 75))
typeface = fonts['large']
score_text = typeface.render(str(score), False,
colours["white"])
text_bounds = score_text.get_rect()
text_bounds.center = display_rect.center
pygame.draw.rect(self.screen, colour, display_rect)
self.screen.blit(score_text, text_bounds.topleft)
score_shift += 200
if time.time() - self.status_time < 5:
typeface = fonts['large']
status_text = typeface.render(self.status_message,
False, colours["white"])
text_bounds = status_text.get_rect()
text_bounds.center = self.screen.get_rect().center
pygame.draw.rect(self.screen, colours["black"],
text_bounds)
self.screen.blit(status_text, text_bounds.topleft)
self.players.minimap_render(self.screen)
pygame.display.update()
finally:
self.network.stop()
def test_non_utf8_binary(self):
dump = dumps(self.doc)
decoded = loads(dump)
self.assertEqual(decoded, self.doc)
def input_evaluation(matches):
data = []
for match in matches:
data.append((
{
"blue": match["blue"],
"red": match["red"]
},
tf.constant(match["winner"])
))
return data
with open("matches.bson", "rb") as f:
raw_matches = bson.loads(f.read())["matches"]
matches = input_evaluation(raw_matches[:10])
feature_columns = [
tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_hash_bucket(key="blue", hash_bucket_size=9000)),
tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_hash_bucket(key="red", hash_bucket_size=9000))
]
hidden_layers = 10 * [len(matches)]
classifier = tf.estimator.DNNClassifier(feature_columns=feature_columns, hidden_units=hidden_layers, n_classes=2)
def train_data():
dataset = tf.data.Dataset.from_tensor_slices(matches)
dataset = dataset.shuffle(1000).repeat().batch(100)
return dataset.make_one_shot_iterator().get_next()
from pathlib import Path
import bson
from prodict import Prodict
from main.updater import Updater
model_json = Path("model.json").read_bytes()
model = Prodict.from_dict(bson.loads(model_json))
Updater(model).update_all()
def echo(socket, address):
printl('New connection from %s:%s' % address)
(ip, port) = address
data = None
try:
data = myreceive(socket)
if data is None:
return
client_param = bson.loads(data)
except:
save_bad_request(ip, port, data, dump_dir)
socket.close
return
#{u'imei': 1, u'ts': 23}
#add new clients to clients list
incoming_imei = "%s", client_param['imei']
if not clients.has_key(client_param['imei']):
printl("adding new client ", client_param['imei'])
clients[client_param['imei']] = {'lts': 0, 'ltf': 0, 'ts_trial': 0}
else:
printl("client already present", client_param['imei'])
printl("read: %s" % client_param)
if not client_param:
printl("client disconnected")
return
# [ ts:long,frag:int,type:int,payload:b]
#check if there are more fragment to sent
#if (clients[client_param['imei']]['lts'] == client_param['ts'] and clients[client_param['imei']]['ltf'] != 0) and next_news:
next_news = echo.next_news
if echo.fragment > 0:
if client_param['lts_status'] != "0":
printl("Failed to Send fragment %d for file %s trials %s/%s" %
(echo.fragment, next_news['filepath'],
clients[client_param['imei']]['ts_trial'],
next_news['trials']))
if next_news['trials'].isdigit() and int(
next_news['trials']) >= int(
clients[client_param['imei']]['ts_trial']):
printl("try again")
echo.fragment += 1
else:
printl("no mor trial")
clients[client_param['imei']]['lts'] = int(
clients[client_param['imei']]['lts']) + 1
echo.fragment = 0
if echo.fragment == 0:
echo.next_news = get_next_news(client_param,
clients[client_param['imei']])
next_news = echo.next_news
if next_news and client_param['lts_status'] != '-3':
echo.file_list = None
printl("ready to send another news")
clients[client_param['imei']]['lts'] = next_news['date']
if os.path.isabs(next_news['filepath']):
file = dumpImage(next_news['filepath'])
else:
file = dumpImage(
os.path.dirname(os.path.realpath(__file__)) + "/" +
next_news['filepath'])
if file:
printl("image is valid")
echo.file_list = chunkstring(file, echo.fragmentSize)
echo.fragment = len(echo.file_list)
clients[client_param['imei']]['ltf'] = echo.fragment
if echo.file_list is not None and next_news and echo.fragment > 0:
sha1 = hashlib.sha1(echo.file_list[echo.fragment - 1]).hexdigest()
repl = bson.dumps({
"ts": long(next_news['date']),
"frag": echo.fragment - 1,
"type": int(next_news['type']),
"headline": next_news['headline'],
"payload": echo.file_list[echo.fragment - 1],
"content": next_news['content'],
"title": next_news['title'],
"sha1": sha1
})
echo.fragment -= 1
if echo.fragment > 0 and int(
clients[client_param['imei']]['ts_trial']) > 0:
printl("reset trials for file %s" % next_news['filepath'])
clients[client_param['imei']]['ts_trial'] = int(
clients[client_param['imei']]['ts_trial']) + 1
clients[client_param['imei']]['ltf'] = echo.fragment
mysend(socket, repl)
if echo.file_list is not None:
printl("frag=[%d/%d] frag=%d" %
(int(len(echo.file_list) - (echo.fragment)),
int(len(echo.file_list)), echo.fragment))
#else:
# repl = bson.dumps({"ts":-1L,"frag":0,"type":1,"payload":b"null image"})
printl("closing socket")
socket.close()
def protocol_parameters(self):
"""Get decoded protocol parameters if they do exist in the header"""
pp = self.header().get('content', {}).get('protocol_parameters')
if pp:
return bson.loads(bytes.fromhex(pp)[4:])
import struct
import datetime
try:
import bson
HAVE_BSON = True
except ImportError:
HAVE_BSON = False
else:
# The BSON module provided by pymongo works through its "BSON" class.
if hasattr(bson, "BSON"):
bson_decode = lambda d: bson.BSON(d).decode()
# The BSON module provided by "pip install bson" works through the
# "loads" function (just like pickle etc.)
elif hasattr(bson, "loads"):
bson_decode = lambda d: bson.loads(d)
else:
HAVE_BSON = False
from lib.cuckoo.common.utils import get_filename_from_path
from lib.cuckoo.common.exceptions import CuckooResultError
log = logging.getLogger(__name__)
###############################################################################
# Generic BSON based protocol - by rep
# Allows all kinds of languages / sources to generate input for Cuckoo,
# thus we can reuse report generation / signatures for other API trace sources.
###############################################################################
TYPECONVERTERS = {
def values(self):
result = []
for v in BDB.values(self):
result.append(bson.loads(v))
return result
