async def reader_loop(
ws: aiohttp.ClientWebSocketResponse,
market_data_callback: MarketDataCallback,
order_book_callback: OrderBookCallback,
trades_callback: TradesCallback,
trades_state_changed_callback: TradesStateChangedCallback) -> None:
msg = await receive_msg(ws, timeout=3)
xdr = xdrlib.Unpacker(msg)
version = xdr.unpack_uint()
logger.info(f'broadcast connection version {version} established')
while True:
msg = await receive_msg(ws)
try:
xdr = xdrlib.Unpacker(msg)
message_type: common.ServerMessageType = common.ServerMessageType.by_value(
xdr.unpack_enum())
if message_type != common.ServerMessageType.BROADCAST_MESSAGE:
raise exceptions.UnsupportedMessageType()
payload = json.loads(xdr.unpack_string().decode())
if market_data_callback is not None:
await market_data_callback(payload)
if payload['@type'] == 'OrderBookAgg':
if order_book_callback is not None:
asyncio.ensure_future(
order_book_callback(payload['current_order_id'],
payload['trade_pair'],
payload.get('buy_levels', dict),
payload.get('sell_levels', dict)))
elif payload['@type'] == 'AnonymousTrade':
if trades_callback is not None:
asyncio.ensure_future(
trades_callback(
datetime.utcfromtimestamp(payload['time'][0]),
payload['current_order_id'], payload['trade_pair'],
Decimal(payload['amount']),
Decimal(payload['price'])))
elif payload['@type'] == 'TradesDisabledOnPairs':
if trades_state_changed_callback:
await trades_state_changed_callback(
payload['trade_pairs'], False)
else:
logger.warning('Trades disabled for pairs {},'
'but trades_state_changed_callback'
' is not seted'.format(' '.join(
payload['trade_pairs'])))
elif payload['@type'] == 'TradesEnabledOnPairs':
if trades_state_changed_callback:
await trades_state_changed_callback(
payload['trade_pairs'], True)
else:
logger.warning('Trades enabled for pairs {},'
'but trades_state_changed_callback'
' is not seted'.format(' '.join(
payload['trade_pairs'])))
else:
raise exceptions.UnsupportedMessageType()
except (KeyError, ValueError, exceptions.UnsupportedMessageType):
logger.exception('failed to decode data')
raise exceptions.CryptologyError('failed to decode data')
def read_offline_data(self):
"""This method is responsible for reading all offline data that are
necessary for calculating the output of interest without the output
error bounds.
Args:
path_offline_data = path to the offline data folder
q_a = number of stiffness matrices (A)
q_f = number of load vectors (f)
q_l = number of attached theta objects to each output vector
n_outputs = number of output vectors (l)
Returns:
n_bfs = number of basis functions
RB_Aq = reduced stiffness matrices
RB_Fq = reduced load vectors
RB_Oq = reduced load vectors
"""
# number of basis functions
with open(self.path_offline_data + '/n_bfs.xdr', 'rb') as f_reader:
f = f_reader.read()
n_bfs = xdrlib.Unpacker(f).unpack_int()
# RB_Aq
RB_Aq = np.empty([n_bfs, n_bfs, self.q_a])
for i in range(self.q_a):
f = open(
'{}/RB_A_{}.xdr'.format(self.path_offline_data,
str(i).zfill(3)), 'rb').read()
u = xdrlib.Unpacker(f)
orig_array = u.unpack_farray(n_bfs * n_bfs, u.unpack_double)
temp = np.reshape(orig_array, [n_bfs, n_bfs])
RB_Aq[:, :, i] = temp
# RB_Fq
RB_Fq = np.empty([n_bfs, self.q_f])
for i in range(self.q_f):
f = open(
'{}/RB_F_{}.xdr'.format(self.path_offline_data,
str(i).zfill(3)), 'rb').read()
u = xdrlib.Unpacker(f)
RB_Fq[:, i] = u.unpack_farray(n_bfs, u.unpack_double)
# RB_Oq
RB_Oq = np.empty([n_bfs, self.n_outputs, self.q_l])
for i in range(self.n_outputs):
for j in range(self.q_l):
f = open(
"{}/output_{}_{}.xdr".format(self.path_offline_data,
str(i).zfill(3),
str(j).zfill(3)),
'rb').read()
u = xdrlib.Unpacker(f)
RB_Oq[:, i, j] = u.unpack_farray(n_bfs, u.unpack_double)
return (n_bfs, RB_Aq, RB_Fq, RB_Oq)
def readCheckpointFilePreambleEtc(checkpointFile, firstOffset, lastOffset):
# Read the preamble of the checkpoint file, i.e.
# read two magic numbers, a version number, the voxel size,
# the origin of the grid and the number of sites.
preambleBytes = 3 * sizeOfUint32 + 4 * sizeOfDouble + sizeOfUint64
preamble = checkpointFile.read(preambleBytes)
up1 = xdrlib.Unpacker(preamble)
firstMagicNumber = up1.unpack_uint()
assert firstMagicNumber == HemeLbMagicNumber, "firstMagicNumber (%r) is not equal to the HemeLbMagicNumber" % firstMagicNumber
secondMagicNumber = up1.unpack_uint()
assert secondMagicNumber == extractionMagicNumber, "secondMagicNumber (%r) is not equal to the extractionMagicNumber (%r)" % (
secondMagicNumber, extractionMagicNumber)
versionNumber = up1.unpack_uint()
assert versionNumber == extractionVersionNumber, "versionNumber (%r) is not the correct version number (%r)" % (
versionNumber, extractionVersionNumber)
voxelSize = up1.unpack_double()
x0 = up1.unpack_double()
y0 = up1.unpack_double()
z0 = up1.unpack_double()
origin = (x0, y0, z0)
numSites = up1.unpack_uhyper()
# Read the header of the checkpoint file, i.e. the number of fields
# and length of the field header
infoBytes = 2 * sizeOfUint32
info = checkpointFile.read(infoBytes)
up2 = xdrlib.Unpacker(info)
numberOfFields = up2.unpack_uint()
assert numberOfFields == 1, "numberOfFields (%r) is not equal to 1" % numberOfFields
lengthOfFieldHeader = up2.unpack_uint()
assert lengthOfFieldHeader == 32, "lengthOfFieldHeader (%r) is not equal to 32" % lengthOfFieldHeader
# Read the field header of the checkpoint file.
fieldHeader = checkpointFile.read(lengthOfFieldHeader)
up3 = xdrlib.Unpacker(fieldHeader)
fieldName = up3.unpack_string()
assert fieldName == "distributions", "fieldName (%r) is not equal to 'distributions'" % fieldName
numberOfComponents = up3.unpack_uint()
fieldOffset = up3.unpack_double()
assert fieldOffset == 0.0, "fieldOffset (%r) is not equal to 0.0" % fieldOffset
# Read the time step.
timestepBytes = checkpointFile.read(sizeOfUint64)
up3 = xdrlib.Unpacker(timestepBytes)
timestep = up3.unpack_uhyper()
filePosition = checkpointFile.tell()
assert firstOffset == filePosition, "filePosition (%r) is not equal to firstOffset (%r)" % (
filePosition, firstOffset)
return FileInfo(voxelSize, origin, numSites, numberOfComponents, timestep)
def _get_AH_version(filename):
"""
Returns version of AH waveform data.
:type filename: str
:param filename: AH v1 file to be checked.
:rtype: str or False
:return: version string of AH waveform data or ``False`` if unknown.
"""
with open(filename, "rb") as fh:
# read first 8 bytes with XDR library
try:
data = xdrlib.Unpacker(fh.read(8))
# check for magic version number
magic = data.unpack_int()
except:
return False
if magic == 1100:
try:
# get record length
length = data.unpack_uint()
# read first record
fh.read(length)
except:
return False
# seems to be AH v2
return '2.0'
elif magic == 6:
# AH1 has no magic variable :/
# so we have to use some fixed values as indicators
try:
fh.seek(12)
if xdrlib.Unpacker(fh.read(4)).unpack_int() != 6:
return False
fh.seek(24)
if xdrlib.Unpacker(fh.read(4)).unpack_int() != 8:
return False
fh.seek(700)
if xdrlib.Unpacker(fh.read(4)).unpack_int() != 80:
return False
fh.seek(784)
if xdrlib.Unpacker(fh.read(4)).unpack_int() != 202:
return False
except:
return False
return '1.0'
else:
return False
def _get_histogram_info(self):
""" get a histogram start and end from SensorCloud"""
response = self.url("/streams/histogram/")\
.param("version", "1")\
.accept("application/xdr")\
.get()
#if the channel doesn't have timeseries data then we'll get a 404-003 error
if response.status_code == httplib.NOT_FOUND:
error = json.loads(response.text)
if error.get("errorcode") == "404-010":
return None
#if we don't get a 200 ok then we had an error
if response.status_code != httplib.OK:
raise error(response, "get histogram info")
#packer for the units xdr format
unpacker = xdrlib.Unpacker(response.raw)
datastructure_version = unpacker.unpack_int()
assert datastructure_version == 1, "structure version should always be 1"
start_nano = unpacker.unpack_uhyper()
end_nano = unpacker.unpack_uhyper()
s = HistogramStreamInfo(start_time=start_nano, end_time=end_nano)
return s
def authenticateAlternate(device_id, username, password):
'''
authenticate with sensorcloud and get the server and auth_key for all subsequent api requests
'''
conn = http.client.HTTPSConnection(AUTH_SERVER)
headers = {'Accept': 'application/xdr'}
url = '/SensorCloud/devices/%s/authenticate/?version=1&username=%s&password=%s' % (
device_id, username, password)
logger.debug('authenticating...')
conn.request('GET', url=url, headers=headers)
response = conn.getresponse()
logger.debug('%s:%s', response.status, response.reason)
# if response is 200 ok then we can parse the response to get the auth token and server
if response.status is 200:
logger.debug('Credential are correct')
# read the body of the response
data = response.read()
# response will be in xdr format. Create an XDR unpacker and extract the token and server as strings
unpacker = xdrlib.Unpacker(data)
auth_token = unpacker.unpack_string().decode('utf-8')
server = unpacker.unpack_string().decode('utf-8')
logger.debug('unpacked xdr. server:%s token:%s' %
(server, auth_token))
return server, auth_token
def decode(data):
"""Decode a bytes object as an sFlow datagram"""
unpacker = xdrlib.Unpacker(data)
datagram = sflow.datagram.Datagram.decode(unpacker)
return datagram
def process_frames(self): # This is VERY IMPORTANT
f = self.fd.read()
self.up = xdrlib.Unpacker(f)
if self.decode_mode == "float": # This is prototype
self.decJob = self.up.unpack_float
else:
self.decJob = self.up.unpack_double
self.times = [0]
self.dts = []
self.frames = []
self._unpack_MyVersion()
# self._unpack_start()
# fr = self._unpack_frame()
##print('fr is ',fr)
##self.frames.append(fr)
# while True:
# fr = self._unpack_frame()
##try:
# fr = self._unpack_frame()
# except EOFError:
# print('crap')
# pass
# self.frames.append(fr)
return self.frames
async def handshake(
self, last_seen_order: int) -> Tuple[int, crypto.Cipher, int]:
packer = xdrlib.Packer()
packer.pack_bytes(self.client_id.encode('ascii'))
packer.pack_hyper(last_seen_order)
packer.pack_bytes(self.symmetric_key)
packer.pack_uint(self.VERSION)
await self.send_bytes(self.server_keys.encrypt(packer.get_buffer()))
logger.debug('sent handshake')
response = await receive_msg(self, timeout=3)
logger.debug('received handshake')
unpacker = xdrlib.Unpacker(self.client_keys.decrypt(response))
data_to_sign = unpacker.unpack_bytes()
last_seen_sequence = unpacker.unpack_hyper()
server_aes_key = unpacker.unpack_bytes()
try:
server_version = unpacker.unpack_uint()
except EOFError:
server_version = 1
await self.send_bytes(self.client_keys.sign(data_to_sign))
logger.debug('sent client key')
return last_seen_sequence, crypto.Cipher(
server_aes_key), server_version
def rpc_call(sock, to_call, **kwargs):
xid, data = make_call(to_call, **kwargs)
resp_data = rpc_request(sock, data)
unpacker = xdrlib.Unpacker(resp_data)
resp_xid = unpacker.unpack_uint()
if resp_xid != xid:
raise XIDError('resp xid: %d is not: %d' % (resp_xid, xid))
msg_type = unpacker.unpack_enum()
if msg_type != REPLY:
raise MsgTypeError('resp msg type: %s is not REPLY' % msg_type)
reply_stat = unpacker.unpack_enum()
if reply_stat != MSG_ACCEPTED:
raise DeniedError('Denied')
verf = (unpacker.unpack_enum(), unpacker.unpack_opaque())
accept_stat = unpacker.unpack_enum()
if accept_stat != SUCCESS:
raise NotSuccessError('accept_stat: %s, is not SUCCESS' % accept_stat)
if kwargs.get('result_unpack_func') is not None:
result = kwargs['result_unpack_func'](unpacker)
else:
result = None
return result
def decode(self, data):
print("DEBUG: GT raw data: %s" % binascii.hexlify(data))
unpacker = xdrlib.Unpacker(data)
gti = unpacker.unpack_uint() & 0xff
info = unpacker.unpack_uint()
translationType = (info >> 16) & 0xff
numberingPlan = (info >> 8) & 0xff
natureOfAddress = info & 0xff
nbDigits = (info >> 24) & 0xff
paddedDigitLen = (nbDigits / 2 + 1) & ~1
print("DEBUG: GT: tt %s, np %s, ton %s, nb digits %s, digit bytes %s" %
(translationType, numberingPlan, natureOfAddress, nbDigits,
paddedDigitLen))
# read padded number of semi-bytes
pos = unpacker.get_position()
tbcdDigits = unpacker.get_buffer()[pos:pos + paddedDigitLen]
digits = tbcd2string(tbcdDigits, nbDigits)
return dict(gti=gti,
translationType=translationType,
numberingPlan=numberingPlan,
natureOfAddress=natureOfAddress,
digits=digits)
def unpack(self, data):
self.len = len(data)
pdata = xdrlib.Unpacker(data)
self.src_vlan = pdata.unpack_uint()
self.src_priority = pdata.unpack_uint()
self.dst_vlan = pdata.unpack_uint()
self.dst_priority = pdata.unpack_uint()
def authenticate_key(device_id, key):
"""
authenticate with sensorcloud and get the server and auth_key for all subsequent api requests
"""
conn = httplib.HTTPSConnection(AUTH_SERVER)
headers = {"Accept": "application/xdr"}
url = "/SensorCloud/devices/%s/authenticate/?version=1&key=%s" % (
device_id, key)
print "Sensorcloud: authenticating..."
conn.request('GET', url=url, headers=headers)
response = conn.getresponse()
print response.status, response.reason
#if response is 200 ok then we can parse the response to get the auth token and server
if response.status is httplib.OK:
print "Sensorcloud: Credential are correct"
#read the body of the response
data = response.read()
#response will be in xdr format. Create an XDR unpacker and extract the token and server as strings
unpacker = xdrlib.Unpacker(data)
global authenticated
authenticated = True
global auth_token
auth_token = unpacker.unpack_string()
global server
server = unpacker.unpack_string()
def _readHeader(cls, filename):
"""Read the header lines, according to description in Code/io/formats/snapshot.h
"""
reader = xdrlib.Unpacker(file(filename).read(cls._headerLengthBytes))
header = {}
assert reader.unpack_uint() == HemeLbMagicNumber
assert reader.unpack_uint() == SnapshotMagicNumber
assert reader.unpack_uint() == cls.VersionNumber
bodyStart = reader.unpack_uint()
assert bodyStart == cls._headerLengthBytes
header['stable'] = reader.unpack_int()
header['voxel_size'] = reader.unpack_double()
header['origin'] = np.array(
(reader.unpack_double(), reader.unpack_double(),
reader.unpack_double()))
header['bb_min'] = np.array(
(reader.unpack_int(), reader.unpack_int(), reader.unpack_int()))
header['bb_max'] = np.array(
(reader.unpack_int(), reader.unpack_int(), reader.unpack_int()))
header['bb_len'] = header['bb_max'] - header['bb_min'] + 1
header['voxel_count'] = reader.unpack_int()
return header
def unpack(self, data):
self.len = len(data)
pdata = xdrlib.Unpacker(data)
self.mac_length = pdata.unpack_uint()
self.src_mac = pdata.unpack_fopaque(6)
self.dst_mac = pdata.unpack_fopaque(6)
self.eth_type = pdata.unpack_uint()
def HemeLbSnapshot(filename):
"""Guess which file format we were given and use the correct class
to open it.
We have to handle a number of cases:
- the original text format;
- the XDR copy thereof, and
- the updated (August 2011) version with format magic and version
numbers and more metadata.
"""
start = file(filename).read(8)
reader = xdrlib.Unpacker(start)
firstInt = reader.unpack_uint()
if firstInt == HemeLbMagicNumber:
assert reader.unpack_uint() == SnapshotMagicNumber
cls = VersionedXdrSnapshot
elif firstInt == 0 or firstInt == 1 or firstInt == 2:
# It is the basic Xdr format that starts with the stablity flag
cls = XdrSnapshotVersionOne
# Maybe text? If so, the first character should be a '0', '1' or '2', followed by a newline
elif (start[0] == '0' or start[0] == '1'
or start == '2') and start[1] == '\n':
cls = TextSnapshot
else:
raise ValueError('Cannot determine version of snapshot file "%s"' %
filename)
return cls(filename)
async def _crypto_handshake(
self) -> Tuple[str, int, int, crypto.Keys, crypto.Cipher]:
data = await self.receive_bytes(timeout=3)
unpacker = xdrlib.Unpacker(self.server_keys.decrypt(data))
client_id = unpacker.unpack_bytes().decode('ascii')
last_seen_order = unpacker.unpack_hyper()
client_aes_key = unpacker.unpack_bytes()
try:
self.client_version = unpacker.unpack_uint()
except EOFError:
pass
client_keys = CLIENT_TEST_KEYS
if client_keys is None:
raise exceptions.ClientNotFound()
sequence_id = 1
data_to_sign = os.urandom(32)
packer = xdrlib.Packer()
packer.pack_bytes(data_to_sign)
packer.pack_hyper(sequence_id)
packer.pack_bytes(self.symmetric_key)
await self.send_bytes(client_keys.encrypt(packer.get_buffer()))
signature = await self.receive_bytes(timeout=3)
client_keys.verify(signature, data_to_sign)
return client_id, sequence_id, last_seen_order, client_keys, crypto.Cipher(
client_aes_key)
def run_schedule(sc):
print('--- Simulating controlled behaviour in [block_start, block_end]')
p_sim = PBar(len(sc.devices) * (sc.i_block_end - sc.i_block_start)).start()
basedir = os.path.dirname(sc.loaded_from)
schedules = np.load(os.path.join(basedir, sc.sched_file))
sim_data = {}
sc.state_files_ctrl = {}
for d in sc.devices:
aid = str(d.typename) + str(d.id)
# Load state
statefile = sc.state_files[aid]
with open(statefile, 'rb') as data:
unpacker = xdrlib.Unpacker(data.read())
d.load_state(unpacker)
os.remove(statefile)
sched = schedules[aid]
if d.typename == 'heatpump':
# This is a consumer, so negate P_el
sched = sched * (-1.0)
# Set schedule
d.components.scheduler.schedule = sched.tolist()
# Simulate
sim_data[aid] = simulate(d, sc.i_block_start, sc.i_block_end, p_sim)
# Save state
packer = xdrlib.Packer()
d.save_state(packer)
tmpf = NamedTemporaryFile(mode='wb', dir='/tmp', delete=False)
tmpf.write(packer.get_buffer())
tmpf.close()
sc.state_files_ctrl[aid] = tmpf.name
print()
return sim_data
def _retrieve_timeseries_partitions(self):
def unpackPartition(unpacker):
partition = {}
partition['start_time'] = unpacker.unpack_uhyper()
partition['end_time'] = unpacker.unpack_uhyper()
unpacker.unpack_int()
unpacker.unpack_int()
sampleRateType = unpacker.unpack_uint()
partition['sample_rate'] = SampleRate.hertz(unpacker.unpack_uint()) if sampleRateType == 1 else \
SampleRate.seconds(unpacker.unpack_uint())
for _ in range(unpacker.unpack_uint()):
popSize = unpacker.unpack_uint() * 12
unpacker.unpack_fopaque(popSize)
return timeseries.descriptor(partition['sample_rate']), partition
response = self.url("/streams/timeseries/partitions/") \
.param("version", "1") \
.accept("application/xdr") \
.get()
if response.status_code != httplib.OK:
raise error(response, "get timeseries partitions")
partitions = []
unpacker = xdrlib.Unpacker(response.raw)
unpacker.unpack_int()
return dict(
[unpackPartition(unpacker) for _ in range(unpacker.unpack_uint())])
def downloadData(server, auth_token, device_id, sensor_name, channel_name, startTime, endTime):
"""
download the 10 minutes of data uploaded by uploadSinWave.
Returns an array of tuples, where each tuple is a timestamp and a value
"""
conn = http.client.HTTPSConnection(server)
url = "/SensorCloud/devices/%s/sensors/%s/channels/%s/streams/timeseries/data/?version=1&auth_token=%s&starttime=%s&endtime=%s" % (device_id, sensor_name, channel_name, auth_token, startTime, endTime)
headers = {"Accept":"application/xdr"}
print("Downloading data...")
conn.request("GET", url=url, headers=headers)
response = conn.getresponse()
data = []
if response.status == http.client.OK:
print("Data retrieved")
unpacker = xdrlib.Unpacker(response.read())
while True:
try:
timestamp = unpacker.unpack_uhyper()
value = unpacker.unpack_float()
data.append((timestamp, value))
except Exception as e:
print(e)
break
return data
else:
print("Status: %s" % response.status)
print("Reason: %s" % response.reason)
return data
def _retrieve_histogram_partitions(self):
def unpackPartition(unpacker):
partition = {}
partition['start_time'] = unpacker.unpack_uhyper()
partition['end_time'] = unpacker.unpack_uhyper()
unpacker.unpack_int()
unpacker.unpack_int()
sampleRateType = unpacker.unpack_uint()
partition['sample_rate'] = SampleRate.hertz(unpacker.unpack_uint()) if sampleRateType == 1 else \
SampleRate.seconds(unpacker.unpack_uint())
partition['num_bins'] = unpacker.unpack_uint()
partition['bin_start'] = unpacker.unpack_float()
partition['bin_size'] = unpacker.unpack_float()
return histogram.descriptor(partition['sample_rate'],
partition['bin_start'],
partition['bin_size'],
partition['num_bins']), partition
response = self.url("/streams/histogram/partitions/") \
.param("version", "1") \
.accept("application/xdr") \
.get()
if response.status_code != httplib.OK:
raise error(response, "get histogram partitions")
unpacker = xdrlib.Unpacker(response.raw)
unpacker.unpack_int() # version
return dict(
[unpackPartition(unpacker) for _ in range(unpacker.unpack_uint())])
def authenticate(self, os_version=None, local_ip=None):
from sensorcloud import UserAgent
#determine protocol from the auth server
if self._authServer.startswith("http://"):
PROTOCOL = "http://"
else:
PROTOCOL = "https://"
url = self._authServer + "/SensorCloud/devices/" + self._deviceId + "/authenticate/"
request = self._requests.url(url)
if os_version: request.param("os_version", os_version)
if local_ip: request.param("local_ip", local_ip)
request = request.param("version", "1")\
.param("key", self._deviceKey)\
.accept("application/xdr")\
.header("User-Agent", UserAgent)\
.get()
#check the response code for success
if request.status_code != httplib.OK:
response = SensorCloudRequests.Request(request)
raise error(response, "authenticating")
#Extract the authentication token and server from the response
unpacker = xdrlib.Unpacker(request.raw)
self._authToken = unpacker.unpack_string()
self._apiServer = PROTOCOL + unpacker.unpack_string()
if self._cache:
self._cache.token = self._authToken
self._cache.server = self._apiServer
def _ReadMainHeader(self):
"""Read data from the main header and store it in attributes.
"""
# Ensure we're at the start
self._file.seek(0)
# Read the correct number of bytes
mainHeader = self._file.read(MainHeaderLength)
assert len(mainHeader) == MainHeaderLength, \
"Did not read the correct length of the main header in extraction file '{}'".format(self.filename)
decoder = xdrlib.Unpacker(mainHeader)
assert decoder.unpack_uint() == HemeLbMagicNumber, "Incorrect HemeLB magic number"
assert decoder.unpack_uint() == ExtractionMagicNumber, "Incorrect extraction magic number"
version = decoder.unpack_uint()
assert version in self.HandledVersions, "Incorrect extraction format version number"
self.voxelSizeMetres = decoder.unpack_double()
self.originMetres = np.array([decoder.unpack_double() for i in xrange(3)])
self.siteCount = decoder.unpack_uhyper()
self.fieldCount = decoder.unpack_uint()
self._fieldHeaderLength = decoder.unpack_uint()
if version == 3:
self.parser = ExtractedPropertyV3Parser(self.fieldCount, self.siteCount)
elif version == 4:
self.parser = ExtractedPropertyV4Parser(self.fieldCount, self.siteCount)
return
def _parse_bucket(self, index, data):
"""Parse one hash bucket and store it in the bucket cache."""
u = xdrlib.Unpacker(data)
slot_no = u.unpack_int()
weight = u.unpack_int()
ip_address = u.unpack_opaque()
udp_query_port = u.unpack_int()
tcp_query_port = u.unpack_int()
admin_port = u.unpack_int()
secondary_slot_no = u.unpack_int()
ipaddr = string.joinfields(map(repr, map(ord, ip_address)), '.')
if self.debug:
print "Hash bucket index:", index
print "slot_no: ", slot_no
print "weight: ", weight
print "ip_address: ", hexstr(ip_address)
print "decoded IP addr: ", ipaddr
print "udp_query_port: ", udp_query_port
print "tcp_query_port: ", tcp_query_port
print "admin_port: ", admin_port
print "secondary_slot_no:", secondary_slot_no
print "=" * 20
result = (slot_no, weight, ipaddr, udp_query_port, tcp_query_port,
admin_port, secondary_slot_no)
self.bucket_cache[index] = result
def __init__(self,
parentAddr=None,
parentDirectory=None,
load=1,
autoShutdown=1,
readonly=False):
import atexit
import time
import xdrlib
self.logFile = None
self.setupLogFile()
self.target = ['default']
self.parent = None
self.saveTimer = None
self.shutdownTimer = None
self.lastAccess = time.time()
self.saveFilename = 'RDict.db'
self.addrFilename = 'RDict.loc'
self.parentAddr = parentAddr
self.isServer = 0
self.readonly = readonly
self.parentDirectory = parentDirectory
self.packer = xdrlib.Packer()
self.unpacker = xdrlib.Unpacker('')
self.stopCmd = pickle.dumps(('stop', ))
self.writeLogLine('Greetings')
self.connectParent(self.parentAddr, self.parentDirectory)
if load: self.load()
if autoShutdown and useThreads:
atexit.register(self.shutdown)
self.writeLogLine('SERVER: Last access ' + str(self.lastAccess))
return
def downloadData(server, auth_token, device_id, sensor_name, channel_name,
startTime, endTime):
conn = http.client.HTTPSConnection(server)
url = '/SensorCloud/devices/%s/sensors/%s/channels/%s/streams/timeseries/data/?version=1&auth_token=%s&starttime' \
'=%s&endtime=%s' % (device_id, sensor_name, channel_name, auth_token, startTime, endTime)
headers = {'Accept': 'application/xdr'}
logger.debug('Downloading data...')
conn.request('GET', url=url, headers=headers)
response = conn.getresponse()
data = []
if response.status is 200:
logger.debug('Data retrieved')
unpacker = xdrlib.Unpacker(response.read())
while True:
try:
timestamp = unpacker.unpack_uhyper()
value = unpacker.unpack_float()
data.append((timestamp, value))
except Exception as e:
logger.error(e)
break
return data
else:
logger.error('Status: %s' % response.status)
logger.error('Reason: %s' % response.reason)
return data
def authenticate_alternate(device_id, username, password):
"""
authenticate with sensorcloud and get the server and auth_key for all subsequent api requests
"""
conn = http.client.HTTPSConnection(AUTH_SERVER)
headers = {"Accept": "application/xdr"}
url = "/SensorCloud/devices/%s/authenticate/?version=1&username=%s&password=%s"%(device_id, username, password)
print("authenticating...")
conn.request('GET', url=url, headers=headers)
response =conn.getresponse()
print(response.status, response.reason)
#if response is 200 ok then we can parse the response to get the auth token and server
if response.status == http.client.OK:
print("Credential are correct")
#read the body of the response
data = response.read()
#response will be in xdr format. Create an XDR unpacker and extract the token and server as strings
unpacker = xdrlib.Unpacker(data)
auth_token = unpacker.unpack_string().decode('utf-8')
server = unpacker.unpack_string().decode('utf-8')
print("unpacked xdr. server:%s token:%s"%(server, auth_token))
return server, auth_token
def _unpack_info(self, info):
self.target_info = {}
info = xdrlib.Unpacker(info)
self.target_info["Target_Type"] = info.unpack_string() # 'VxWorks\x00'
self.target_info["Vx_Version"] = info.unpack_string() # '6.6\x00'
self.target_info["Unknow1"] = info.unpack_uint() # 80
self.target_info["Unknow2"] = info.unpack_uint() # 86
self.target_info["CPU_Type"] = info.unpack_string() # '86\x00'
self.target_info["compiler"] = info.unpack_string() # '86\x00'
self.target_info["Unknow3"] = info.unpack_uint() # 86
self.target_info["Unknow4"] = info.unpack_uint() # 86
self.target_info["Unknow5"] = info.unpack_uint() # 86
self.target_info["Unknow6"] = info.unpack_uint() # 86
self.target_info["Unknow7"] = info.unpack_uint() # 86
self.target_info["CPU_Model"] = info.unpack_string() # '86\x00'
self.target_info["Unknow8"] = info.unpack_string() # '86\x00'
self.target_info["Unknow9"] = info.unpack_uint() # 86
self.target_info["Memory_Size"] = info.unpack_uint() # 86
self.target_info["Unknow10"] = info.unpack_uint()
self.target_info["Unknow11"] = info.unpack_uint()
self.target_info["Unknow12"] = info.unpack_uint()
self.target_info["Unknow13"] = info.unpack_uint()
self.target_info["Unknow14"] = info.unpack_uint()
self.target_info["Unknow15"] = info.unpack_uint()
return self.target_info
def init_pf_data(self, ifile, to_EV=True):
#
# Reads Kurucz's partition functions and ionization potentials from a file
# Taken from RH (Uitenbroek 2001)
#
ff = open(ifile, 'rb')
f = ff.read()
ff.close()
data = xdrlib.Unpacker(f)
npf = data.unpack_uint()
nelem = 99
self.tpf = np.float64(data.unpack_farray(npf, data.unpack_double))
self.el = [None] * nelem
for ii in range(nelem):
pti = data.unpack_uint()
nstage = data.unpack_uint()
self.el[ii] = dum()
self.el[ii].pf = np.float64(
data.unpack_farray(npf * nstage, data.unpack_double)).reshape(
(nstage, npf))
self.el[ii].eion = np.float64(
data.unpack_farray(nstage,
data.unpack_double)) * self.HH * self.CC
self.el[ii].nstage = nstage
self.el[ii].npf = pti
if (to_EV): self.el[ii].eion /= self.EV
def read(self, inputs, geometry, atmos):
f = open(self.filename, 'rb')
up = xdrlib.Unpacker(f.read())
f.close()
self.Nspect = up.unpack_int()
self.waves = read_farray(self.Nspect, up, "double")
if geometry.type == "ONE_D_PLANE" or\
geometry.type == 'SPHERICAL_SYMMETRIC':
dim = [geometry.Nrays, self.Nspect]
elif geometry.type == 'TWO_D_PLANE':
dim = [geometry.Nx, geometry.Nrays, self.Nspect]
elif geometry.type == 'THREE_D_PLANE':
dim = [geometry.Nx, geometry.Ny, geometry.Nrays, self.Nspect]
self.I = read_farray(dim, up, "double")
self.vacuum_to_air = up.unpack_int()
self.air_limit = float(up.unpack_double())
if atmos.stokes or inputs.backgr_pol:
self.Q = read_farray(dim, up, "double")
self.U = read_farray(dim, up, "double")
self.V = read_farray(dim, up, "double")
up.done()
