def __init__(self, name, device=None, logger=None):
super(PsocI2CDevice, self).__init__(name)
self._device = device or MockDevice('mock device')
self._logger = logger or Logger()
self._map_id_to_offset = {
MSG_ID_CONTROL : ('H', self._CONTROL_OFFSET),
MSG_ID_DEBUG : ('H', self._DEBUG_OFFSET),
MSG_ID_SPEED : ('f', self._LEFT_RIGHT_SPEED_OFFSET)
}
# Slices that define the grouping of data for each message from the raw data fields
STATUS = slice(0, 2)
SPEED = slice(2, 4)
POSITION = slice(4, 6)
HEADING = 6
HEARTBEAT = 7
self._msg_data_slices = [STATUS, SPEED, POSITION, HEADING, HEARTBEAT]
self._msgs = [
lambda data: Message(type=MSG_ID_STATUS, data=StatusData(*data)),
lambda data: Message(type=MSG_ID_SPEED, data=SpeedData(*data)),
lambda data: Message(type=MSG_ID_POSITION, data=PositionData(*data)),
lambda data: Message(type=MSG_ID_HEADING, data=HeadingData(data)),
lambda data: Message(type=MSG_ID_HEARTBEAT, data=HeartbeatData(data))
]
self._msg_gen = None
def __init__(self,
address=0x28,
i2c=None,
rst=None,
gpio=None,
logger=None):
"""
Initialize the BNO055 class
:param address: the I2C address
:param i2c: the i2c device
:param rst: a reset pin
:param gpio: a gpio package
:return: None
"""
# If reset pin is provided save it and a reference to provided GPIO
# bus (or the default system GPIO bus if none is provided).
self._rst = rst
if self._rst is not None:
if gpio is None:
import Adafruit_GPIO as GPIO
gpio = GPIO.get_platform_gpio()
self._gpio = gpio
# Setup the reset pin as an output at a high level.
self._gpio.setup(self._rst, GPIO.OUT)
self._gpio.set_high(self._rst)
# Wait a 650 milliseconds in case setting the reset high reset the chip.
time.sleep(0.65)
self._address = address
self._i2c_device = i2c
self._mode = OPERATION_MODE_NDOF
self._logger = logger or Logger()
def create_i2c(cls, driver_cb, logger=None):
if not logger:
logger = Logger()
try:
i2c_device = I2CDevice(I2CDevice.DEV_I2C_1, logger=logger)
except I2CDeviceError as err:
raise ImuHwFactoryError("Unable to instantiate Imu hardware", err)
try:
imu_device = ImuDevice(name="imu device",
device=i2c_device,
logger=logger)
except ImuDeviceError as err:
raise ImuHwFactoryError("Unable to instantiate Imu hardware", err)
try:
driver = ImuDriver('imu driver', device=imu_device, logger=logger)
except ImuDriverError as err:
raise ImuHwFactoryError("Unable to instantiate Imu hardware", err)
try:
return ImuHw(driver=driver, callback=driver_cb, logger=logger)
except ImuHwError as err:
raise ImuHwFactoryError("Unable to instantiate Imu hardware", err)
def create_can(cls,
driver_cb,
channel=_DEFAULT_CHANNEL,
bustype=_DEFAULT_BUSTYPE,
logger=None):
if not logger:
logger = Logger()
try:
can_device = can.interface.Bus(channel=channel, bustype=bustype)
except can.CanError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
device = PsocCANDevice(name='psoc device',
device=can_device,
logger=logger)
except PsocDeviceError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
driver = PsocDriver(name='psoc driver',
device=device,
logger=logger)
except PsocDriverError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
return PsocHw(driver_if=driver, driver_cb=driver_cb, logger=logger)
except PsocHwError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
def create_mock(cls, driver_cb, logger=None):
if not logger:
logger = Logger()
try:
mock_device = MockDevice('mock device', logger=logger)
except MockDeviceError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
device = PsocMockDevice('psoc device',
device=mock_device,
logger=logger)
except PsocDeviceError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
driver = PsocDriver('module i2c driver',
device=device,
logger=logger)
except PsocDriverError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
try:
return PsocHw(driver_if=driver, driver_cb=driver_cb, logger=logger)
except PsocHwError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
def __init__(self, name, device, callback, logger=None):
super(DeviceDriver, self).__init__(name, logger=logger)
self._name = name
self._msgs = self._gen_msg()
self._callback = callback
self._device = device
self._logger = logger or Logger()
def __init__(self, apkObj, theLogger=Logger()):
self.apkObj = apkObj
self.activities = []
self.staticReceivers = []
self.services = []
self.log = theLogger
def run_multi(func, args, name, i, repeat):
log_name = os.path.join(log_dir, '%d.%s.log' % (i, name))
log = Logger(strm=open(log_name, 'w'))
for j in range(repeat):
t = func(*args)
t.start()
t.join()
log.log(t.get())
del log
def __init__(self, driver=None, callback=None, logger=None):
# Instantiate the IMUDriver and start it
self._driver = driver # or MockImuDriver()
# Capture the callback for local access
self._callback = callback
self._logger = logger or Logger()
def vae_train():
logger = Logger("{}/vae_train_{}.log".format(cfg.logger_save_dir, cfg.timestr))
logger.log(cfg.info)
logger.log("Loading Dataset")
data_list = glob.glob(cfg.seq_save_dir +'/*.npz')
datas = Parallel(n_jobs=cfg.num_cpus, verbose=1)(delayed(load_npz)(f) for f in data_list)
datasets = [NumpyData(x) for x in datas]
total_data = ConcatDataset(datasets)
train_data_loader = DataLoader(total_data, batch_size=cfg.vae_batch_size, shuffle=True, num_workers=10, pin_memory=False)
print('Total frames: {}'.format(len(total_data)))
model = torch.nn.DataParallel(VAE()).cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=cfg.vae_lr)
for epoch in range(cfg.vae_num_epoch):
current_loss = 0
now = time.time()
for idx, imgs in enumerate(train_data_loader):
data_duration = time.time() - now
now = time.time()
imgs = imgs.float().cuda() / 255.0
mu, logvar, imgs_rc, z = model(imgs)
r_loss = (imgs_rc - imgs).pow(2).view(imgs.size(0), -1).sum(dim=1).mean()
kl_loss = - 0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp(), dim=1)
min_kl = torch.zeros_like(kl_loss) + cfg.vae_kl_tolerance * cfg.vae_z_size
kl_loss = torch.max(kl_loss, min_kl).mean()
loss = r_loss + kl_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
current_loss += loss.item() * imgs.size(0)
model_duration = time.time() - now
total_duration = data_duration + model_duration
if idx % 10 == 0:
info = "Epoch {:2d}\t Step [{:5d}/{:5d}]\t Loss {:6.3f}\t R_Loss {:6.3f}\t \
KL_Loss {:6.3f}\t Maxvar {:6.3f}\t Speed {:6.3f}\t Time: [{:.5f}/{:.5f}]\t".format(
epoch, idx, len(train_data_loader), loss.item(), r_loss.item(),
kl_loss.item(), logvar.max().item(), imgs.size(0) / total_duration, data_duration, total_duration)
logger.log(info)
now = time.time()
to_save_data = {'model': model.module.state_dict(), 'loss': current_loss}
to_save_path = '{}/vae_{}_e{:03d}.pth'.format(cfg.model_save_dir, cfg.timestr, epoch)
torch.save(to_save_data, to_save_path)
def __init__(self, name, device=None, logger=None):
self._name = name
self._device = device
self._hw_callback = None
self._logger = logger or Logger()
super(SensorDriver, self).__init__(name,
self._device,
self._process,
logger=logger)
def __init__(self, driver_if=None, driver_cb=None, debug_if=None, debug_cb=None, cal_if=None, cal_cb=None, logger=None):
self._driver_if = driver_if #or MockPsocDriver()
self._debug_if = debug_if
self._cal_if = cal_if
self._driver_cb = driver_cb
self._debug_cb = debug_cb
self._cal_cb = cal_cb
self._logger = logger or Logger()
def module_test(selection):
def callback(msg):
logger.debug(msg)
logger = Logger()
if selection == 'can':
pass
elif selection == 'i2c':
PsocHwFactory.create_i2c(callback, logger)
elif selection == 'mock':
pass
def __init__(self, theLogger=Logger()):
self.log = theLogger
self.jsonFactory = JsonFactory()
self.logLines = []
self.logEntryList = []
self.json2pyFailedList = []
self.json2pyFailedErrorList = []
self.numControlChars = 0
def run_multi(func, name, i):
log_name = os.path.join(log_dir, '%d.%s.log' % (i, name))
log = Logger(strm=open(log_name, 'w'))
li = []
for j in range(i):
li.append(func(j))
for j in range(i):
li[j].start()
for j in range(i):
li[j].join()
log.log(li[j].get())
del log
def module_test():
from imuhwfactory import ImuHwFactory
def callback(msg):
logger.debug(str(msg))
logger = Logger()
imuhw = ImuHwFactory.create_i2c(driver_cb=callback, logger=logger)
imuhw.start()
time.sleep(0.05)
imuhw.shutdown()
def module_test():
import time
from imuhwfactory import ImuHwFactory
def callback(msg):
logger.info("delivered callback msg: {}".format(msg))
logger = Logger()
print("ImuHwFactory.create_i2c ...")
imuhw = ImuHwFactory.create_i2c(callback, logger=logger)
print("ImuHwFactory.create_i2c complete")
time.sleep(1)
imuhw.shutdown()
def main():
# Parse options
parser = OptionParser(usage='usage: %prog [options] logcatFile',
version='%prog 0.1')
parser.add_option('-v',
'--verbose',
action='store_true',
dest='verbose',
default=True)
parser.add_option('-q', '--quiet', action='store_false', dest='verbose')
(options, args) = parser.parse_args()
# Run
if options.verbose:
logger = Logger(LogLevel.DEBUG)
else:
logger = Logger()
logAnalyzer = TaintLogAnalyzer(theLogger=logger)
logAnalyzer.setLogFile(args[0])
logAnalyzer.extractLogEntries()
logAnalyzer.postProcessLogObjects()
logAnalyzer.printOverview()
def module_test():
import logging
from i2cdevice import I2CDevice
logger = Logger()
logger.debug("Start Module Test")
i2c_device = I2CDevice(I2CDevice.DEV_I2C_1, logger=logger)
imu = ImuDevice(name="imu device",
device=i2c_device,
timeout=5,
logger=logger)
logger.debug("End Module Test")
def main():
# Parse options
parser = OptionParser(usage='usage: %prog [options] apk', version='%prog 0.1')
parser.add_option('', '--sdkPath', metavar='<path>', help='Set path to Android SDK')
parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=True)
parser.add_option('-q', '--quiet', action='store_false', dest='verbose')
(options, args) = parser.parse_args()
# Run
if options.verbose:
logger = Logger(LogLevel.DEBUG)
else:
logger = Logger()
apk = APKWrapper(args[0], theSdkPath=options.sdkPath, theLogger=logger)
print apk.getManifest()
print apk.getActivityNameList()
print apk.getMainActivityName()
print apk.getServiceNameList()
print 'MD5 %s' % apk.getMd5Hash()
print 'SHA256 %s' % apk.getSha256Hash()
import codecs
report = codecs.open('xxx', "w", "utf-8")
report.write('md5: %s' % apk.getMd5Hash())
def master(comm):
logger = Logger("{}/es_train_{}.log".format(cfg.logger_save_dir, cfg.timestr))
logger.log(cfg.info)
controller = Controller()
es = cma.CMAEvolutionStrategy(flatten_controller(controller), cfg.es_sigma, {'popsize': cfg.population_size})
for step in range(cfg.es_steps):
solutions = es.ask()
for idx, solution in enumerate(solutions):
comm.send(solution, dest=idx+1, tag=1)
check = np.ones(cfg.num_workers)
rewards = []
for idx in range(cfg.num_workers):
reward = comm.recv(source=idx+1, tag=2)
rewards.append(reward)
check[idx] = 0
assert check.sum() == 0
assert len(rewards) == cfg.num_workers
r_cost = - np.array(rewards)
reg_cost = l2_reg(solutions)
cost = reg_cost + r_cost
es.tell(solutions, cost.tolist())
sigma = es.result[6]
rms_var = np.mean(sigma * sigma)
info = "Step {:d}\t Max_R {:4f}\t Mean_R {:4f}\t Min_R {:4f}\t RMS_Var {:4f}\t Reg_Cost {:4f}\t R_Cost {:4f}".format(
step, max(rewards), np.mean(rewards), min(rewards), rms_var, r_cost.mean(), reg_cost.mean())
logger.log(info)
if step % 25 == 0:
current_param = es.result[5]
current_controller = deflatten_controller(current_param)
save_path = "{}/controller_curr_{}_step_{:05d}.pth".format(cfg.model_save_dir, cfg.timestr, step)
torch.save({'model': current_controller.state_dict()}, save_path)
best_param = es.result[0]
best_controller = deflatten_controller(best_param)
save_path = "{}/controller_best_{}_step_{:05d}.pth".format(cfg.model_save_dir, cfg.timestr, step)
torch.save({'model': best_controller.state_dict()}, save_path)
def __init__(self, theApkFile, theSdkPath='', theLogger=Logger()):
self.apkFile = theApkFile
self.sdkPath = Utils.addSlashToPath(theSdkPath)
self.log = theLogger
self.sampleId = random.randint(1, 1000000)
self.apkFileName = Utils.splitFileIntoDirAndName(theApkFile)[1]
self.apkPath = Utils.splitFileIntoDirAndName(theApkFile)[0]
self.package = ''
self.manifest = {'activityList':[],
'activityAliasList':[],
'serviceList':[],
'receiverList':[],
'providerList':[],
'uses-permission':[]}
self.__extractFromPermissions()
self.__extractApplication()
def module_test():
from i2cdevice import I2CDevice
import time
logger = Logger()
logger.debug("Performing module test")
try:
i2c_device = I2CDevice(device=I2CDevice.DEV_I2C_1, logger=logger)
except I2CDeviceError as err:
logger.fatal("Unable to instantiate Psoc hardware", err)
psoc_device = PsocI2CDevice(name='psoc device', device=i2c_device)
for i in range(10):
msg = psoc_device.recv()
logger.debug(msg)
time.sleep(1)
def module_test():
import time
from ..devices.i2cdevice import I2CDevice
from ..devices.psocdevice import PsocI2CDevice
def callback(msg):
logger.debug("{} - {}".format(time.time(), msg))
logger = Logger()
logger.debug("{} {}".format(__name__, " Module Test"))
i2c_device = I2CDevice(device=I2CDevice.DEV_I2C_1)
device = PsocI2CDevice(name='psoc device', device=i2c_device)
driver = PsocDriver(name='psochw i2c driver', device=device)
driver.start(callback)
time.sleep(5)
driver.stop()
def get_structured_events():
currency_markup = MarkupCurrency()
timestamp = datetime.datetime.now()
today = datetime.datetime.now()
blob_name = "%s_%s_%s" % (today.year, today.month, today.day)
logger = Logger()
scrappers = {
"relax": RelaxScrapper(logger),
"citydog": CityDogScrapper(logger),
"tutby": TutByScrapper(logger)
}
for source, scrapper in scrappers.items():
print(source)
events = scrapper.list_events()
write_events_to_blob(blob_name + "\\%s.json" % source, events, currency_markup, timestamp, logger)
return blob_name
def vae_extract():
logger = Logger("{}/vae_extract_{}.log".format(cfg.logger_save_dir,
cfg.timestr))
logger.log(cfg.info)
print("Loading Dataset")
data_list = glob.glob(cfg.seq_save_dir + '/*.npz')
data_list.sort()
N = len(data_list) // 4
procs = []
for idx in range(4):
p = Process(target=extract,
args=(data_list[idx * N:(idx + 1) * N], idx, N))
procs.append(p)
p.start()
time.sleep(1)
for p in procs:
p.join()
def __init__(self,
theSdkPath='',
thePort=5554,
theImageDirPath='',
theRunHeadlessFlag=False,
theAvdName=None,
theLogger=Logger()):
self.sdkPath = Utils.addSlashToPath(theSdkPath)
self.port = thePort
self.imageDirPath = Utils.addSlashToPath(theImageDirPath)
self.runHeadless = theRunHeadlessFlag
self.log = theLogger
self.avdName = theAvdName
self.emulator = None
self.logcatRedirectFile = ''
self.logcatRedirectProcess = None
self.adbProcess = None
def module_test():
from psochwfactory import PsocHwFactory
def callback(msg):
logger.debug("delivered callback msg: {}".format(msg))
logger = Logger()
logger.debug("{} {}".format(__name__, "Module Test"))
#psochw = PsocHwFactory.create_can(callback)
#time.sleep(1)
#psochw.shutdown()
psochw = PsocHwFactory.create_i2c(callback, logger=logger)
logger.debug("Starting psochw ...")
psochw.start()
logger.debug("Started")
time.sleep(0.01)
logger.debug("Shutting down psochw ...")
psochw.shutdown()
logger.debug("Shutdown")
def create_i2c(cls, driver_cb, logger=None):
if not logger:
logger = Logger()
logger.debug("({}) - Creating I2CDevice ...".format(__name__))
try:
i2c_device = I2CDevice(device=I2CDevice.DEV_I2C_1, logger=logger)
except I2CDeviceError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
logger.debug("({}) - Complete".format(__name__))
logger.debug("({}) - Creating PsocI2CDevice ...".format(__name__))
try:
device = PsocI2CDevice(name='psoc device',
device=i2c_device,
logger=logger)
except PsocDeviceError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
logger.debug("({}) - Complete".format(__name__))
logger.debug("({}) - Creating PsocDriver ...".format(__name__))
try:
driver = PsocDriver(name='psochw i2c driver',
device=device,
logger=logger)
except PsocDriverError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
logger.debug("({}) - Complete".format(__name__))
logger.debug("({}) - Creating PsocHw ...".format(__name__))
try:
return PsocHw(driver_if=driver, driver_cb=driver_cb, logger=logger)
except PsocHwError as err:
raise PsocHwFactoryError("Unable to instantiate Psoc hardware",
err)
logger.debug("({}) - Complete".format(__name__))
def __init__(self, name, device=None, timeout=5, logger=None):
"""
Initialize the ImuDevice
:param timeout: the amount of time to wait for calibration status to settle
:return: None
"""
super(ImuDevice, self).__init__(name)
self._logger = logger or Logger()
self._device = device or MockDevice('mock device')
self._msg_gen = None
try:
self._bno = BNO055(i2c=self._device)
except I2CDeviceError as err:
raise ImuDeviceError("Unable to instantiate BNO055", err)
self._start_hardware(timeout)
# Mapping between message ids and functions in the BNO055 instance
# The lambda is need to ensure the Message is dynamically created and data is reacquired each cycle.
self._msgs = [
lambda: Message(type=MSG_ID_EULER,
data=EulerData(*self._bno.read_euler())),
lambda: Message(type=MSG_ID_MAGNETIC,
data=MagneticData(*self._bno.read_magnetometer())),
lambda: Message(type=MSG_ID_LINEAR_ACCEL,
data=LinearAccelData(*self._bno.
read_linear_acceleration())),
lambda: Message(type=MSG_ID_ANGULAR_VELOCITY,
data=AngularVelocityData(*self._bno.read_gyroscope(
))), lambda:
Message(type=MSG_ID_ORIENTATION,
data=OrientationData(*self._bno.read_quaternion())),
lambda: Message(type=MSG_ID_TEMPERATURE,
data=TemperatureData(*self._read_temp_wrapper()))
]
self._msg_cycle = None
