def listAllProperties(self):
"""
This is strictly informational, calling it will print out all the
available properties of the camera, of which there are a lot..
"""
for i, node_map in enumerate(self.node_maps):
# Get the number of nodes in the node map.
p_value = ctypes.c_size_t(0)
checkErrorCode(spindll.spinNodeMapGetNumNodes(node_map, ctypes.byref(p_value)),
"spinNodeMapGetNumNodes")
p_value = p_value.value
print("Node " + str(i) + " has " + str(p_value) + " properties.")
# List all of the node names.
for j in range(p_value):
c_index = ctypes.c_size_t(j)
h_node = ctypes.c_void_p(0)
error_code = spindll.spinNodeMapGetNodeByIndex(node_map, c_index, ctypes.byref(h_node))
if (error_code == SPINNAKER_ERR_SUCCESS):
spin_node = SpinNode(h_node)
print(" " + spin_node.name + " / " + spin_node.description)
print("")
def get_multi_raw(self, keys):
keys = [key for key in keys if self.check_key(key)]
n = len(keys)
ckeys = (ctypes.c_char_p * n)(*keys)
lens = (ctypes.c_size_t * n)(*[len(k) for k in keys])
c.memcached_mget(self.mc, ckeys, lens, n)
result = {}
chunks_record = []
while True:
key = ctypes.create_string_buffer(200)
klen, vlen, flag, rc = ctypes.c_size_t(0), ctypes.c_size_t(0), ctypes.c_uint32(0), ctypes.c_int(0)
val = c.memcached_fetch(self.mc, key, ctypes.byref(klen),
ctypes.byref(vlen), ctypes.byref(flag), ctypes.byref(rc))
if not val:
break
key = key.value
val = ctypes.string_at(val, vlen.value)
flag = flag.value
if flag & _FLAG_CHUNKED:
chunks_record.append((key, int(val), flag))
else:
result[key] = (val, flag)
for key, count, flag in chunks_record:
val, flag = self._get_raw_split(key, count, flag)
if val:
result[key] = (val, flag)
return result
def cube(x, unpack=True):
"""cube a number
parameters:
`x`: number to be cubed.
"""
tdump = TextDumper()
tdump.dump(x, "x", ['I', '4'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.cube_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.cube_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def update(self, set_items, delete_items=None, virtual=True):
"""
Args:
set_items (dict): dict key, values where keys are addresses
delete_items (list): list of addresses
virtual (boolean): True if not committing to disk. I.e.,
speculative root hash
Returns:
the state root after the operations
"""
c_set_items = (ctypes.POINTER(_Entry) * len(set_items))()
for (i, (key, value)) in enumerate(set_items.items()):
c_set_items[i] = ctypes.pointer(_Entry.new(key, _encode(value)))
if delete_items is None:
delete_items = []
c_delete_items = (ctypes.c_char_p * len(delete_items))()
for (i, address) in enumerate(delete_items):
c_delete_items[i] = ctypes.c_char_p(address.encode())
(c_merkle_root, c_merkle_root_len) = ffi.prepare_byte_result()
_libexec('merkle_db_update', self.pointer,
c_set_items, ctypes.c_size_t(len(set_items)),
c_delete_items, ctypes.c_size_t(len(delete_items)),
virtual,
ctypes.byref(c_merkle_root),
ctypes.byref(c_merkle_root_len))
return ffi.from_c_bytes(
c_merkle_root, c_merkle_root_len).decode()
def move_point(p, x, unpack=True):
"""Adds a number to each of the coordinates of a point.
parameters:
`p`: the point to be moved.
`x`: the amount the point should be moved.
"""
tdump = TextDumper()
tdump.dump(p, "p", ['S', 'Point'])
tdump.dump(x, "x", ['R', '8'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.move_point_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.move_point_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def print_msg(msg, unpack=True):
"""Print a message to the standard error stream.
parameters:
`msg`: the message to be printed.
"""
tdump = TextDumper()
tdump.dump(msg, "msg", ['W'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.print_msg_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.print_msg_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def scale_array(v, s, unpack=True):
"""scale a one dimentional array.
parameters:
`v`: the array to be scaled.
`s`: the scalar factor.
"""
tdump = TextDumper()
tdump.dump(v, "v", ['T', ['R', '8']])
tdump.dump(s, "s", ['R', '8'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.scale_array_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.scale_array_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def make_line(p1, p2, unpack=True):
"""Given two points, it constructs a line.
parameters:
`p1`: the first point.
`p2`: the second point.
"""
tdump = TextDumper()
tdump.dump(p1, "p1", ['S', 'Point'])
tdump.dump(p2, "p2", ['S', 'Point'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.make_line_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.make_line_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def scale(v, alpha, unpack=True):
"""Scalar multiplication.
parameters:
`v`: the vector to multiply.
`alpha`: the scalar.
"""
tdump = TextDumper()
tdump.dump(v, "v", ['T', ['R', '8']])
tdump.dump(alpha, "alpha", ['R', '8'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.scale_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.scale_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def aead_decrypt(self, data):
"""
Decrypt data and authenticate tag
:param data: cipher text with tag
:return: plain text
"""
global buf_size, buf
cipher_out_len = c_size_t(0)
plen = len(data) - self._tlen
if buf_size < plen:
buf_size = plen * 2
buf = create_string_buffer(buf_size)
tag = data[plen:]
r = libmbedtls.mbedtls_cipher_auth_decrypt(
byref(self._ctx),
c_char_p(self._nonce.raw), c_size_t(self._nlen),
None, c_size_t(0),
c_char_p(data), c_size_t(plen),
byref(buf), byref(cipher_out_len),
c_char_p(tag), c_size_t(self._tlen)
)
if r:
raise Exception('AEAD encrypt failed {0:#x}'.format(r))
self.cipher_ctx_init()
return buf.raw[:cipher_out_len.value]
def get_memory_info(self):
"""Returns (free, total) memory in bytes in the context.
"""
free = c_size_t()
total = c_size_t()
driver.cuMemGetInfo(byref(free), byref(total))
return free.value, total.value
def _fincore(fileobj):
st_size = os.fstat(fileobj.fileno()).st_size
if not st_size:
return CacheStats(0, 0, ())
pa = mmap(0, st_size, PROT_NONE, MAP_SHARED, fileobj.fileno(), 0)
if pa == MAP_FAILED:
raise ctypes_os_error("mmap('%s')" % path)
# round the filesize to the nearest page size
# note the use of integer division here
total_pages = (st_size+PAGESIZE-1)//PAGESIZE
try:
vec = (ctypes.c_uint8*total_pages)()
ret = mincore(ctypes.c_void_p(pa), ctypes.c_size_t(st_size), vec)
if ret != 0:
raise ctypes_os_error("mincore")
finally:
ret = munmap(ctypes.c_void_p(pa), ctypes.c_size_t(st_size))
if ret != 0:
raise ctypes_os_error("munmap")
cached_count = 0
cached_count = sum(1 for page in vec if page & 1)
pages = ()
if enumerate_pages:
pages = tuple(offset
for offset, page in enumerate(vec)
if page & 1)
return CacheStats(total_pages, cached_count, pages)
def aead_encrypt(self, data):
"""
Encrypt data with authenticate tag
:param data: plain text
:return: cipher text with tag
"""
global buf_size, buf
plen = len(data)
if buf_size < plen + self._tlen:
buf_size = (plen + self._tlen) * 2
buf = create_string_buffer(buf_size)
cipher_out_len = c_size_t(0)
tag_buf = create_string_buffer(self._tlen)
r = libmbedtls.mbedtls_cipher_auth_encrypt(
byref(self._ctx),
c_char_p(self._nonce.raw), c_size_t(self._nlen),
None, c_size_t(0),
c_char_p(data), c_size_t(plen),
byref(buf), byref(cipher_out_len),
byref(tag_buf), c_size_t(self._tlen)
)
assert cipher_out_len.value == plen
if r:
raise Exception('AEAD encrypt failed {0:#x}'.format(r))
self.cipher_ctx_init()
return buf.raw[:cipher_out_len.value] + tag_buf.raw[:self._tlen]
def _commoncrypto_pbkdf2(data, salt, iterations, digest, keylen):
"""Common Crypto compatibile wrapper
"""
c_hashfunc = ctypes.c_uint32(_commoncrypto_hashlib_to_crypto_map_get(digest))
c_pass = ctypes.c_char_p(data)
c_passlen = ctypes.c_size_t(len(data))
c_salt = ctypes.c_char_p(salt)
c_saltlen = ctypes.c_size_t(len(salt))
c_iter = ctypes.c_uint(iterations)
c_keylen = ctypes.c_size_t(keylen)
c_buff = ctypes.create_string_buffer(keylen)
crypto.CCKeyDerivationPBKDF.restype = ctypes.c_int
crypto.CCKeyDerivationPBKDF.argtypes = [ctypes.c_uint32,
ctypes.c_char_p,
ctypes.c_size_t,
ctypes.c_char_p,
ctypes.c_size_t,
ctypes.c_uint32,
ctypes.c_uint,
ctypes.c_char_p,
ctypes.c_size_t]
ret = crypto.CCKeyDerivationPBKDF(2, # hardcoded 2-> PBKDF2
c_pass, c_passlen,
c_salt, c_saltlen,
c_hashfunc,
c_iter,
c_buff,
c_keylen)
return (1 - ret, c_buff)
def list_children(self):
"""
List children of the currently set snapshot (set via set_snap()).
:returns: list - a list of (pool name, image name) tuples
"""
pools_size = c_size_t(512)
images_size = c_size_t(512)
while True:
c_pools = create_string_buffer(pools_size.value)
c_images = create_string_buffer(images_size.value)
ret = self.librbd.rbd_list_children(self.image,
byref(c_pools),
byref(pools_size),
byref(c_images),
byref(images_size))
if ret >= 0:
break
elif ret != -errno.ERANGE:
raise make_ex(ret, 'error listing images')
if ret == 0:
return []
pools = c_pools.raw[:pools_size.value - 1].split('\0')
images = c_images.raw[:images_size.value - 1].split('\0')
return zip(pools, images)
def parent_info(self):
"""
Get information about a cloned image's parent (if any)
:returns: tuple - ``(pool name, image name, snapshot name)`` components
of the parent image
:raises: :class:`ImageNotFound` if the image doesn't have a parent
"""
ret = -errno.ERANGE
size = 8
while ret == -errno.ERANGE and size <= 4096:
pool = create_string_buffer(size)
name = create_string_buffer(size)
snapname = create_string_buffer(size)
ret = self.librbd.rbd_get_parent_info(self.image, byref(pool),
c_size_t(size),
byref(name),
c_size_t(size),
byref(snapname),
c_size_t(size))
if ret == -errno.ERANGE:
size *= 2
if ret != 0:
raise make_ex(ret, 'error getting parent info for image %s' % (self.name,))
return (pool.value, name.value, snapname.value)
def running_avg(timeseries, radius=16384, module_dir="/home/rkube/source/running_ma"):
"""
Compute the moving average of a time series using the openmp module
Input:
======
timeeseries: ndarray(float64): input time series
radius: int, Radius for moving average filter
Output:
======
mavg: ndarray, moving average
"""
assert(timeseries.dtype == np.core.numerictypes.float64)
ma_lib = ctypes.cdll.LoadLibrary("%s/module_moving_stat.so" % module_dir)
numel = timeseries.shape[0]
moving_avg = np.zeros(numel, dtype='float64')
# convert to ctypes
c_numel = ctypes.c_size_t(numel)
c_radius = ctypes.c_size_t(radius)
c_ma_in_ptr = ctypes.c_void_p(timeseries.ctypes.data)
c_ma_out_ptr = ctypes.c_void_p(moving_avg.ctypes.data)
ma_lib.moving_average_omp(c_ma_in_ptr, c_ma_out_ptr, c_radius, c_numel)
return moving_avg
def _execute(job, f, o=None):
"""
Executes a librsync "job" by reading bytes from `f` and writing results to
`o` if provided. If `o` is omitted, the output is ignored.
"""
# Re-use the same buffer for output, we will read from it after each
# iteration.
out = ctypes.create_string_buffer(RS_JOB_BLOCKSIZE)
while True:
block = f.read(RS_JOB_BLOCKSIZE)
buff = Buffer()
# provide the data block via input buffer.
buff.next_in = ctypes.c_char_p(block)
buff.avail_in = ctypes.c_size_t(len(block))
buff.eof_in = ctypes.c_int(not block)
# Set up our buffer for output.
buff.next_out = ctypes.cast(out, ctypes.c_char_p)
buff.avail_out = ctypes.c_size_t(RS_JOB_BLOCKSIZE)
result = _librsync.rs_job_iter(job, ctypes.byref(buff))
if o:
o.write(out.raw[:RS_JOB_BLOCKSIZE - buff.avail_out])
if result == RS_DONE:
break
elif result != RS_BLOCKED:
raise LibrsyncError(result)
if buff.avail_in > 0:
# There is data left in the input buffer, librsync did not consume
# all of it. Rewind the file a bit so we include that data in our
# next read. It would be better to simply tack data to the end of
# this buffer, but that is very difficult in Python.
f.seek(f.tell() - buff.avail_in)
if o and callable(getattr(o, 'seek', None)):
# As a matter of convenience, rewind the output file.
o.seek(0)
return o
def pass_decrypt(self, data):
"""
Decrypts the given data with the given passphrase.
:return: output array
"""
out = create_string_buffer(len(data) - TOX_PASS_ENCRYPTION_EXTRA_LENGTH)
tox_err_decryption = c_int()
self.libtoxencryptsave.tox_pass_decrypt(c_char_p(bytes(data)),
c_size_t(len(data)),
c_char_p(bytes(self._passphrase, 'utf-8')),
c_size_t(len(self._passphrase)),
out,
byref(tox_err_decryption))
tox_err_decryption = tox_err_decryption.value
if tox_err_decryption == TOX_ERR_DECRYPTION['OK']:
return out[:]
elif tox_err_decryption == TOX_ERR_DECRYPTION['NULL']:
raise ArgumentError('Some input data, or maybe the output pointer, was null.')
elif tox_err_decryption == TOX_ERR_DECRYPTION['INVALID_LENGTH']:
raise ArgumentError('The input data was shorter than TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes')
elif tox_err_decryption == TOX_ERR_DECRYPTION['BAD_FORMAT']:
raise ArgumentError('The input data is missing the magic number (i.e. wasn\'t created by this module, or is'
' corrupted)')
elif tox_err_decryption == TOX_ERR_DECRYPTION['KEY_DERIVATION_FAILED']:
raise RuntimeError('The crypto lib was unable to derive a key from the given passphrase, which is usually a'
' lack of memory issue. The functions accepting keys do not produce this error.')
elif tox_err_decryption == TOX_ERR_DECRYPTION['FAILED']:
raise RuntimeError('The encrypted byte array could not be decrypted. Either the data was corrupt or the '
'password/key was incorrect.')
def test_basic2(self):
# Test (sai)
arg = MsgArg.MsgArg()
l = [-6, -66, 666, 6666]
num = C.c_size_t(len(l))
ArrayType = C.c_int32 * len(l)
array = ArrayType()
array[:] = l
resultArray = C.POINTER(C.c_int32)() # defines the type AND create an instance of it
returnNum = C.c_size_t()
try:
arg.Set("ai", [C.c_size_t, ArrayType], [num, array])
except QStatusException as ex:
print str(ex)
assert False
try:
arg.Get("ai", [C.POINTER(C.c_size_t), C.POINTER(C.POINTER(C.c_int32))],
[C.byref(returnNum), C.byref(resultArray)])
self.assertEqual(returnNum.value, 4, 'wrong result')
l2 = [resultArray[i] for i in range(4)]
self.assertItemsEqual(l, l2, 'wrong result')
except QStatusException as ex:
print str(ex)
assert False
def sgemv(transA, M, N, alpha, bufA, offA, lda, bufX, offX, incx, beta, bufY,
offY, incy, wait_for=None):
cblas_row_major = ct.c_int(0)
transA = ct.c_int(1 if transA else 0)
lda = ct.c_size_t(int(lda))
incx = ct.c_size_t(int(incx))
incy = ct.c_size_t(int(incy))
M = ct.c_size_t(int(M))
N = ct.c_size_t(int(N))
alpha = ct.c_float(alpha)
beta = ct.c_float(beta)
if wait_for is None:
num_wait = 0
else:
num_wait = 1
wait_for = ct.byref(wait_for)
done_evt = cl.cl_event()
err = _clblaslib.clblasSgemv(cblas_row_major, transA, M, N,
alpha, bufA.ocl_buf, ct.c_size_t(offA), lda,
bufX.ocl_buf, ct.c_size_t(offX), incx, beta,
bufY.ocl_buf, ct.c_size_t(offY), incy,
ct.c_size_t(1), ct.byref(queues[0]),
ct.c_size_t(num_wait), wait_for,
ct.byref(done_evt))
if err:
raise Exception("clBLAS sgemv returned error code {}".format(err))
return done_evt
def getFrames(self):
frames = []
# Get all the images that are currently available.
while True:
#
# Create an image, possibly wasteful as if we don't get an
# image we'll immediately throw this object away.
#
image_size = self.aoi_width * self.aoi_height
s_cam_data = SCamData(size = image_size)
image = ShimImage(ctypes.c_int(self.pixel_format),
ctypes.c_size_t(self.aoi_height),
ctypes.c_size_t(image_size),
ctypes.c_size_t(self.aoi_width),
ctypes.c_void_p(s_cam_data.getDataPtr()))
# Get the image.
err_code = spinshimdll.getNextImage(self.im_event, ctypes.byref(image))
# Check if we actually got an image.
if (err_code != SPINSHIM_ERR_SUCCESS):
break
# Add to the list of images if we did.
frames.append(s_cam_data)
if (err_code != SPINSHIM_ERR_NO_NEW_IMAGES):
raise SpinnakerException("Image acquisition error!", err_code)
return [frames, [self.aoi_width, self.aoi_height]]
def test_array(self):
# Test (sai)
arg = MsgArg.MsgArg()
stringSet = C.c_char_p("Hello")
stringResult = C.c_char_p()
l = [-8, -88, 888, 8888]
num = C.c_size_t(len(l))
ArrayType = C.c_int32 * len(l)
array = ArrayType()
array[:] = l
resultArray = C.POINTER(C.c_int32)() # defines the type AND create an instance of it
returnNum = C.c_size_t()
try:
arg.Set("(sai)", [C.c_char_p, C.c_size_t, ArrayType], [stringSet, num, array])
except QStatusException as ex:
print str(ex)
assert False
try:
arg.Get("(sai)", [C.POINTER(C.c_char_p), C.POINTER(C.c_size_t), C.POINTER(C.POINTER(C.c_int32))],
[C.byref(stringResult), C.byref(returnNum), C.byref(resultArray)])
self.assertEqual(stringResult.value, "Hello", 'wrong result')
self.assertEqual(returnNum.value, 4, 'wrong result')
l2 = [resultArray[i] for i in range(4)]
self.assertItemsEqual(l, l2, 'wrong result')
except QStatusException as ex:
print str(ex)
assert False
def read(self, offset, length, fadvise_flags=0):
"""
Read data from the image. Raises :class:`InvalidArgument` if
part of the range specified is outside the image.
:param offset: the offset to start reading at
:type offset: int
:param length: how many bytes to read
:type length: int
:param fadvise_flags: fadvise flags for this read
:type fadvise_flags: int
:returns: str - the data read
:raises: :class:`InvalidArgument`, :class:`IOError`
"""
ret_buf = create_string_buffer(length)
if fadvise_flags == 0:
ret = self.librbd.rbd_read(self.image, c_uint64(offset),
c_size_t(length), byref(ret_buf))
else:
ret = self.librbd.rbd_read2(self.image, c_uint64(offset),
c_size_t(length), byref(ret_buf),
c_int(fadvise_flags))
if ret < 0:
raise make_ex(ret, 'error reading %s %ld~%ld' % (self.image, offset, length))
return ctypes.string_at(ret_buf, ret)
def msghdr_for_fd(fd, fd_type=None):
"""Create an SCM_RIGHTS message header containing `fd' in its control
message and optionally `data' (an instance of some ctypes
class) inside its io vector.
"""
if fd_type is None:
fd_type_buf = FileDescriptorType.create_buffer()
else:
fd_type_buf = fd_type.packed
iov = iovec(iov_base=ctypes.addressof(fd_type_buf),
iov_len=ctypes.c_size_t(ctypes.sizeof(fd_type_buf)))
cfd = ctypes.c_int(fd)
cmhp = cmsghdr.with_data(cmsg_len=CMSG_LEN(ctypes.sizeof(cfd)),
cmsg_level=socket.SOL_SOCKET,
cmsg_type=SCM_RIGHTS,
cmsg_data=cfd)
mh = msghdr(msg_name=None,
msg_namelen=0,
msg_iov=iovec_ptr(iov),
msg_iovlen=1,
msg_control=ctypes.addressof(cmhp),
msg_controllen=ctypes.c_size_t(ctypes.sizeof(cmhp)))
# save references to these so they don't get deleted!
mh.cmsg = cmhp
mh.iovec = iov
mh.fd_type_buf = fd_type_buf
return mh
def togglePhySimIter(lowLevelToggleSeq):
libMbus = ctypes.cdll.LoadLibrary(libSoName)
mbusPhy = MbusPhyStruct()
byteMemSize = 64
rxByteMem = (ctypes.c_byte * byteMemSize)()
txByteMem = (ctypes.c_byte * byteMemSize)()
libMbus.mbus_phy_init(ctypes.byref(mbusPhy),
ctypes.byref(rxByteMem),
ctypes.c_size_t(byteMemSize),
ctypes.byref(txByteMem),
ctypes.c_size_t(byteMemSize),
)
libMbus.mbus_phy_rx_enable(ctypes.byref(mbusPhy))
rxNibList = nibbles.MbusRawNibbleList(mbusTimeFormat="blank")
driveMbusPinLo = ctypes.c_bool()
for lowLevelToggle in lowLevelToggleSeq:
libMbus.mbus_phy_update(ctypes.byref(mbusPhy),
ctypes.c_ulong(lowLevelToggle[1]),
ctypes.c_bool(lowLevelToggle[0]),
ctypes.byref(driveMbusPinLo),
)
if not libMbus.mbus_phy_rx_is_empty(ctypes.byref(mbusPhy)):
while not libMbus.mbus_phy_rx_is_empty(ctypes.byref(mbusPhy)):
newNib = libMbus.mbus_phy_rx_pop(ctypes.byref(mbusPhy))
if newNib == nibbles.MBUS_END_MSG_CODE:
# print("{}".format("".join(rxNibList)))
# print("{}".format(rxNibList))
yield rxNibList
rxNibList = nibbles.MbusRawNibbleList(mbusTimeFormat="blank")
else:
rxNibList.append(chr((libMbus.mbus_phy_rxnibble2ascii(newNib))))
def _run_asm(self, *byte_code):
import ctypes
# Convert the byte code into a function that returns an int
restype = ctypes.c_uint32
argtypes = ()
func, address = self._asm_func(restype, argtypes, byte_code)
# Call the byte code like a function
retval = func()
byte_code = bytes.join(b'', byte_code)
size = ctypes.c_size_t(len(byte_code))
# Free the function memory segment
if is_windows:
MEM_RELEASE = ctypes.c_ulong(0x8000)
ctypes.windll.kernel32.VirtualFree(ctypes.c_void_p(address), ctypes.c_size_t(0), MEM_RELEASE)
else:
# Remove the executable tag on the memory
READ_WRITE = 0x1 | 0x2
if ctypes.pythonapi.mprotect(ctypes.c_void_p(address), size, READ_WRITE) < 0:
raise Exception("Failed to mprotect")
ctypes.pythonapi.free(ctypes.c_void_p(address))
return retval
def setsockopt(self, option, value, **kwargs):
self._verify_open()
# TODO Need to implement unicode here. The way it works in pyzmq is odd
# to me, so I need to dig into it.
encoding = getattr(kwargs, 'encoding', None)
if encoding:
raise TypeError('unicode not currently supported.')
if option in sockopts_bytes:
if not isinstance(value, bytes):
raise TypeError('expected str, got: %r' % value)
rc = libzmq.C.zmq_setsockopt(self.handle, option, byref(value),
c_size_t(sizeof(value)))
elif option in sockopts_int64:
if not isinstance(value, int):
raise TypeError('expected str, got: %r' % value)
value_int64 = c_int64(value)
rc = libzmq.C.zmq_setsockopt(self.handle, option, byref(value_int64),
c_size_t(sizeof(value)))
elif option in sockopts_int32:
if not isinstance(value, int):
raise TypeError('expected str, got: %r' % value)
rc = libzmq.C.zmq_setsockopt(self.handle, option, byref(value),
c_size_t(sizeof(value)))
else:
raise ZMQError()
if rc != 0:
raise ZMQError()
def timestwo(x, unpack=True):
"""Take a scalar and double it.
parameters:
`x`: scalar to be doubled.
"""
tdump = TextDumper()
tdump.dump(x, "x", ['R', '8'])
in_str = tdump.close()
len_in = len(in_str)
out_str = ctypes.POINTER(c_char)()
len_out = c_size_t(0)
LIB.timestwo_py(c_size_t(len_in),
c_char_p(in_str),
ctypes.byref(len_out),
ctypes.byref(out_str))
if out_str[:1] == 'E':
xc_error_msg = out_str[1:len_out.value]
raise RuntimeError(xc_error_msg)
val = TextParser(out_str[:len_out.value]).parse()
LIB.timestwo_py_clear()
if unpack:
if val:
return val.values()[0] if len(val) == 1 else val.values()
return None
else:
return val
def hscryptkdf(password, dklen, maxmem, maxmemfrac, maxtime, params = None, saltsz = 32, nocheck = False):
dk = create_string_buffer(dklen)
# get lib's param size
psz = _hgetparamsize()
# check length of params
if params is not None and len(params) < (psz + saltsz):
raise Exception('For this build of the scrypt lib the params are %s/%s bytes! The salt size is %s.' % (len(params), psz + saltsz, saltsz))
if params is None:
print('creating param bytes')
params = create_string_buffer(psz + saltsz)
recover = 0
else:
print('using param bytes')
params = c_char_p(params)
recover = 1
if nocheck:
nocheck = 1
else:
nocheck = 0
rcode = _hscryptkdf(
c_char_p(password), c_size_t(len(password)), dk, c_size_t(dklen),
c_size_t(saltsz),
c_double(maxmem), c_double(maxmemfrac), c_double(maxtime), params,
c_uint8(recover), c_uint8(nocheck)
)
if recover == 0:
# convert from string buffer into bytes object
params = bytes(params)
return (rcode, bytes(dk), params)
def kmedoids(sample, medoids, tolerance, itermax, metric_pointer, data_type):
pointer_data = package_builder(sample, c_double).create()
medoids_package = package_builder(medoids, c_size_t).create()
c_data_type = convert_data_type(data_type)
ccore = ccore_library.get()
ccore.kmedoids_algorithm.restype = POINTER(pyclustering_package)
package = ccore.kmedoids_algorithm(pointer_data, medoids_package, c_double(tolerance), c_size_t(itermax), metric_pointer, c_data_type)
result = package_extractor(package).extract()
ccore.free_pyclustering_package(package)
return result[0], result[1]
def set_nonce(self, nonce):
"""Reset the nonce associated with this object"""
_set_nonce(self._ctx, ctypes.c_size_t(len(nonce)), nonce)
def key_length(self):
kmin = c_size_t(0)
kmax = c_size_t(0)
botan.botan_cipher_query_keylen(self.cipher, byref(kmin), byref(kmax))
return kmin.value, kmax.value
def update(self, msg):
"""Add the data in msg to the hash"""
_update(self._ctx, ctypes.c_size_t(len(msg)), msg)
def parse_bitcode(self, bitcode):
buf = create_string_buffer(bitcode, len(bitcode))
mod = self.hlc.ROC_ParseBitcode(buf, c_size_t(len(bitcode)))
if not mod:
raise Error("Failed to parse bitcode")
return mod
def compile(self, **options):
"""Perform Compilation
The valid compiler options are
* - -g (enable generation of debugging information)
* - -opt=
* - 0 (disable optimizations)
* - 3 (default, enable optimizations)
* - -arch=
* - compute_20 (default)
* - compute_30
* - compute_35
* - -ftz=
* - 0 (default, preserve denormal values, when performing
* single-precision floating-point operations)
* - 1 (flush denormal values to zero, when performing
* single-precision floating-point operations)
* - -prec-sqrt=
* - 0 (use a faster approximation for single-precision
* floating-point square root)
* - 1 (default, use IEEE round-to-nearest mode for
* single-precision floating-point square root)
* - -prec-div=
* - 0 (use a faster approximation for single-precision
* floating-point division and reciprocals)
* - 1 (default, use IEEE round-to-nearest mode for
* single-precision floating-point division and reciprocals)
* - -fma=
* - 0 (disable FMA contraction)
* - 1 (default, enable FMA contraction)
*
"""
# stringify options
opts = []
if 'debug' in options:
if options.pop('debug'):
opts.append('-g')
if 'opt' in options:
opts.append('-opt=%d' % options.pop('opt'))
if options.get('arch'):
opts.append('-arch=%s' % options.pop('arch'))
other_options = (
'ftz',
'prec_sqrt',
'prec_div',
'fma',
)
for k in other_options:
if k in options:
v = int(bool(options.pop(k)))
opts.append('-%s=%d' % (k.replace('_', '-'), v))
# If there are any option left
if options:
optstr = ', '.join(map(repr, options.keys()))
raise NvvmError("unsupported option {0}".format(optstr))
# compile
c_opts = (c_char_p *
len(opts))(*[c_char_p(x.encode('utf8')) for x in opts])
err = self.driver.nvvmCompileProgram(self._handle, len(opts), c_opts)
self._try_error(err, 'Failed to compile\n')
# get result
reslen = c_size_t()
err = self.driver.nvvmGetCompiledResultSize(self._handle,
byref(reslen))
self._try_error(err, 'Failed to get size of compiled result.')
ptxbuf = (c_char * reslen.value)()
err = self.driver.nvvmGetCompiledResult(self._handle, ptxbuf)
self._try_error(err, 'Failed to get compiled result.')
# get log
self.log = self.get_log()
return ptxbuf[:]
def CMSG_LEN(length):
sizeof_cmshdr = ctypes.sizeof(cmsghdr)
return ctypes.c_size_t(CMSG_ALIGN(sizeof_cmshdr).value + length)
def deserialize_wkb(data):
geom = lgeos.GEOSGeomFromWKB_buf(c_char_p(data), c_size_t(len(data)))
if not geom:
raise WKBReadingError(
"Could not create geometry because of errors while reading input.")
return geom
def CMSG_ALIGN(length):
sizeof_size_t = ctypes.sizeof(ctypes.c_size_t)
return ctypes.c_size_t((length + sizeof_size_t - 1)
& ~(sizeof_size_t - 1))
def new_zero_native():
return c_size_t(0)
def magic_setparam(cookie, param, val):
if not _has_param:
raise NotImplementedError("magic_setparam not implemented")
v = c_size_t(val)
return _magic_setparam(cookie, param, byref(v))
def CMSG_SPACE(length):
length_align = CMSG_ALIGN(length).value
sizeof_cmsghdr = ctypes.sizeof(cmsghdr)
cmsghdr_align = CMSG_ALIGN(sizeof_cmsghdr).value
return ctypes.c_size_t(length_align + cmsghdr_align)
def get_header(handle, files, filename, type, name): size = min(c_max(ctypes.c_size_t), 2**16) # Don't create a buffer larger than 64k buf = ctypes.create_string_buffer(size) if libewf.libewf_get_header_value(handle, type, buf, ctypes.c_size_t(size)) and buf.value: files[filename] = files[filename] + "# " + name + ": " + buf.value + "\n"
def magic_getparam(cookie, param):
if not _has_param:
raise NotImplementedError("magic_getparam not implemented")
val = c_size_t()
_magic_getparam(cookie, param, byref(val))
return val.value
def _heap_alloc(heap, size):
table_mem = kernel32.HeapAlloc(heap, 0, ctypes.c_size_t(size.value))
if not table_mem:
raise exception.CloudbaseInitException(
'Unable to allocate memory for the IP forward table')
return table_mem
def main():
libewf.libewf_get_version.restype = ctypes.c_char_p
if not libewf.libewf_get_version() == libewf_version:
sys.stderr.write("Using libewf-" + libewf.libewf_get_version() + ". Tested with libewf-" + libewf_version + ".\n")
usage = """
%prog [options] <filename(s)> <mountpoint>
Note: This utility allows EWF files to be mounted as a filesystem containing a flat disk image. <filename> can be any segment of the EWF file. To be identified, all files need to be in the same directory, have the same root file name, and have the same first character of file extension. Alternatively, multiple filenames can be specified in different locations in the order to be reassembled.
"""
server = ewfFS(version="%prog " + mount_ewf_version + " FUSE " + fuse.__version__, usage=usage, dash_s_do='undef')
server.parser.add_option(mountopt="disktype", dest="disktype", metavar="DISKTYPE_BINARY", help="disktype program")
server.parser.add_option(mountopt="rw", dest="rw", metavar=" ", help="read-write support")
server.parser.add_option(mountopt="delta", dest="delta", metavar=" ", help="use previously generated delta file")
server.parser.add_option(mountopt="ewf_debug", dest="ewf_debug", metavar=" ", help="print ewf debug messages")
server.parse(values=server, errex=1)
if server.disktype == None:
server.disktype = "disktype"
if server.rw == None:
server.rw = True
libewf.libewf_get_flags_read_write.restype = ctypes.c_uint8
flag = libewf.libewf_get_flags_read_write()
else:
server.rw = False
libewf.libewf_get_flags_read.restype = ctypes.c_uint8
flag = libewf.libewf_get_flags_read()
if server.delta == None:
server.delta = True
elif type(server.delta) == 'str':
if not (os.path.isfile(server.delta) and libewf.libewf_check_file_signature(server.delta) == 1):
server.delta = False
if server.ewf_debug == None:
server.ewf_debug = True
libewf.libewf_set_notify_values(ctypes.pythonapi.PyFile_AsFile(ctypes.py_object(sys.stdout)), ctypes.c_uint8(1))
elif server.ewf_debug == "mount":
server.ewf_debug = True
libewf.libewf_set_notify_values(ctypes.pythonapi.PyFile_AsFile(ctypes.py_object(sys.stdout)), ctypes.c_uint8(0))
elif server.ewf_debug == "lib":
server.ewf_debug = False
libewf.libewf_set_notify_values(ctypes.pythonapi.PyFile_AsFile(ctypes.py_object(sys.stdout)), ctypes.c_uint8(1))
else:
server.ewf_debug = False
libewf.libewf_set_notify_values(ctypes.pythonapi.PyFile_AsFile(ctypes.py_object(sys.stdout)), ctypes.c_uint8(0))
server.mountpoint = os.path.abspath(sys.argv[-1])
filenames = []
for arg in sys.argv[1:-1]:
if os.path.isfile(arg):
if libewf.libewf_check_file_signature(arg) == 1:
filenames.append(arg)
if len(filenames) == 1:
filenames = find_image_segments(os.path.abspath(filenames[0]), server.delta)
if len(filenames) == 0:
server.parser.print_usage()
sys.stderr.write("ewf segment filename(s) required.\n")
sys.exit(1)
if server.ewf_debug:
print "Filenames:", filenames
handle = libewf.libewf_open(c_array(ctypes.c_char_p, filenames), ctypes.c_ulong(len(filenames)), ctypes.c_uint8(flag))
if handle == 0:
server.parser.print_usage()
sys.stderr.write("Couldn't open EWF file.\n")
sys.exit(1)
libewf.libewf_parse_header_values(handle, 3)
size = min(c_max(ctypes.c_size_t), 2**16) # Don't create a buffer larger than 64k
buf = ctypes.create_string_buffer(size)
prefix = "/" + os.path.splitext(os.path.basename(filenames[0]))[0]
if os.uname()[0] == "Darwin":
partition_suffix = ".img"
drive_suffix = ".dmg"
else:
partition_suffix = ""
drive_suffix = ""
info_name = prefix + ".txt"
server.files[info_name] = ""
get_header(handle, server.files, info_name, "description", "Description")
get_header(handle, server.files, info_name, "case_number", "Case number")
get_header(handle, server.files, info_name, "examiner_name", "Examiner name")
get_header(handle, server.files, info_name, "evidence_number", "Evidence number")
get_header(handle, server.files, info_name, "notes", "Notes")
get_header(handle, server.files, info_name, "acquiry_date", "Acquiry date")
get_header(handle, server.files, info_name, "system_date", "System date")
get_header(handle, server.files, info_name, "acquiry_operating_system", "Operating system used")
get_header(handle, server.files, info_name, "acquiry_software_version", "Software version used")
get_header(handle, server.files, info_name, "password", "Password")
get_header(handle, server.files, info_name, "compression_type", "Compression type")
get_header(handle, server.files, info_name, "model", "Model")
get_header(handle, server.files, info_name, "serial_number", "Serial number")
media_size = ctypes.c_uint64()
if libewf.libewf_get_media_size(handle, ctypes.byref(media_size)) != 1:
sys.stderr.write("Unable to get media size.\n")
sys.exit(4)
if media_size.value == 0:
sys.stderr.write("Invalid media size.\n")
sys.exit(4)
server.partitions[prefix + drive_suffix] = {'handle': handle, 'offset': 0, 'size': media_size.value}
if server.disktype:
p = subprocess.Popen([server.disktype] + filenames, stdout=subprocess.PIPE)
server.files[prefix + ".disktype.txt"] = p.stdout.read()
p.wait()
partitions = find_partitions(server.files[prefix + ".disktype.txt"])
for partition in partitions:
server.partitions[prefix + "p" + partition + partition_suffix] = partitions[partition]
server.partitions[prefix + "p" + partition + partition_suffix]['handle'] = handle
size = 16
md5 = (ctypes.c_uint8 * size)()
libewf.libewf_get_md5_hash(handle, ctypes.byref(md5), ctypes.c_size_t(size))
md5_printable = binascii.hexlify(md5)
full_path = os.path.join(server.mountpoint, prefix[1:] + drive_suffix)
if os.uname()[0] == "Linux":
server.files[info_name] = server.files[info_name] + md5_printable + " *" + full_path + "\n"
else:
server.files[info_name] = server.files[info_name] + "MD5 (" + full_path + ") = " + md5_printable + "\n"
server.main()
def get_physical_disks(self):
physical_disks = []
disk_guid = GUID_DEVINTERFACE_DISK
handle_disks = setupapi.SetupDiGetClassDevsW(
ctypes.byref(disk_guid), None, None,
self.DIGCF_PRESENT | self.DIGCF_DEVICEINTERFACE)
if handle_disks == self.INVALID_HANDLE_VALUE:
raise exception.CloudbaseInitException(
"SetupDiGetClassDevs failed")
try:
did = Win32_SP_DEVICE_INTERFACE_DATA()
did.cbSize = ctypes.sizeof(Win32_SP_DEVICE_INTERFACE_DATA)
index = 0
while setupapi.SetupDiEnumDeviceInterfaces(handle_disks, None,
ctypes.byref(disk_guid),
index,
ctypes.byref(did)):
index += 1
handle_disk = self.INVALID_HANDLE_VALUE
required_size = wintypes.DWORD()
if not setupapi.SetupDiGetDeviceInterfaceDetailW(
handle_disks, ctypes.byref(did), None, 0,
ctypes.byref(required_size), None):
if (kernel32.GetLastError() !=
self.ERROR_INSUFFICIENT_BUFFER):
raise exception.WindowsCloudbaseInitException(
"SetupDiGetDeviceInterfaceDetailW failed: %r")
pdidd = ctypes.cast(
msvcrt.malloc(ctypes.c_size_t(required_size.value)),
ctypes.POINTER(Win32_SP_DEVICE_INTERFACE_DETAIL_DATA_W))
try:
pdidd.contents.cbSize = ctypes.sizeof(
Win32_SP_DEVICE_INTERFACE_DETAIL_DATA_W)
if not self._is_64bit_arch():
# NOTE(cpoieana): For some reason, on x86 platforms
# the alignment or content of the struct
# is not taken into consideration.
pdidd.contents.cbSize = 6
if not setupapi.SetupDiGetDeviceInterfaceDetailW(
handle_disks, ctypes.byref(did), pdidd,
required_size, None, None):
raise exception.WindowsCloudbaseInitException(
"SetupDiGetDeviceInterfaceDetailW failed: %r")
device_path = ctypes.cast(pdidd.contents.DevicePath,
wintypes.LPWSTR).value
handle_disk = kernel32.CreateFileW(device_path, 0,
self.FILE_SHARE_READ,
None,
self.OPEN_EXISTING, 0,
0)
if handle_disk == self.INVALID_HANDLE_VALUE:
raise exception.CloudbaseInitException(
'CreateFileW failed')
sdn = Win32_STORAGE_DEVICE_NUMBER()
b = wintypes.DWORD()
if not kernel32.DeviceIoControl(
handle_disk, self.IOCTL_STORAGE_GET_DEVICE_NUMBER,
None, 0, ctypes.byref(sdn), ctypes.sizeof(sdn),
ctypes.byref(b), None):
raise exception.WindowsCloudbaseInitException(
'DeviceIoControl failed: %r')
physical_disks.append(r"\\.\PHYSICALDRIVE%d" %
sdn.DeviceNumber)
finally:
msvcrt.free(pdidd)
if handle_disk != self.INVALID_HANDLE_VALUE:
kernel32.CloseHandle(handle_disk)
finally:
setupapi.SetupDiDestroyDeviceInfoList(handle_disks)
return physical_disks
def write(self, path, buf, offset): if self.ewf_debug: print "WRITE TO:", path, "OFFSET:", offset, "SIZE:", len(buf) if self.rw: size = len(buf) for name in self.partitions: if path == name: if offset > self.partitions[name]['size']: size = 0 if offset+size > self.partitions[name]['size']: size = self.partitions[name]['size'] - offset if size <= 0: if self.ewf_debug: print "writing past end of file, returning too big" return -errno.EFBIG # fuse is expecting an int, not the long which is returned because a c_size_t is unsigned #libewf.libewf_write_random.restype = ctypes.c_size_t libewf.libewf_write_random.restype = ctypes.c_int with self.read_lock: result = libewf.libewf_write_random(self.partitions[name]['handle'], ctypes.create_string_buffer(buf[:size]), ctypes.c_size_t(size), c_off_t(offset+self.partitions[name]['offset'])) #if result == ctypes.c_size_t(-1).value: if result == -1: if self.ewf_debug: print "ERROR on write, returning IO error" return -errno.EIO if self.ewf_debug: #print "RETURNING", int(result) print "RETURNING", result #return int(result) return result return -errno.ENOENT
