def loads(self):
result = Utils.array2d(self.cols, self.rows)
blocks = Utils.array2d(self.cols, self.rows, list)
for i in range(self.cols):
for j in range(self.rows):
sx, ex = i * self.block_size, (i + 1) * self.block_size
sy, ey = j * self.block_size, (j + 1) * self.block_size
for x in range(sx, ex):
for y in range(sy, ey):
blocks[i][j].append(self.pixels[x, y])
result[i][j] = getattr(Methods, Const.PA_method)(blocks[i][j])
if self.save is not None or self.show:
om = Image.new(
"RGB",
[self.cols * self.block_size, self.rows * self.block_size],
(255, 255, 255))
draw = ImageDraw.Draw(om)
for i in range(self.cols):
for j in range(self.rows):
sx, ex = i * self.block_size, (i + 1) * self.block_size
sy, ey = j * self.block_size, (j + 1) * self.block_size
pos = [sx, sy, ex - 1, ey - 1]
cur_color = Const.PA_colors[result[i][j]]
outline = (0, 0, 0) if self.outline else None
draw.rectangle(pos, fill=cur_color, outline=outline)
if self.save is not None:
om.save(self.save)
if self.show:
om.show()
return {"cols": self.cols, "rows": self.rows, "data": result}
def __call__(self, beginDate, endDate):
"""
"""
assert beginDate <= endDate
logger.info("----- 开始处理：from %s to %s -------" % (beginDate, endDate))
# today = datetime.date.today()
# if beginDate >= today:
# return
# 提取前一天的数据，以作一些额外的运算
try:
self.prevData = PVDaily.objects.get(date=beginDate - one_day)
except PVDaily.DoesNotExist:
self.prevData = None
while (beginDate <= endDate):
logger.info("===== 正在处理 %s =======" % beginDate)
logger.info("+++++ 开始从LogVisit中更新用户数据到UserData中... +++++++\n")
tt = Utils.TimeTrack()
# 先把新用户更新到历史访问列表中
self.uudflv = UpdateUserDataFromLogVisit()
self.uudflv(beginDate)
logger.info("+++++ 数据更新完毕，耗时：%s 秒 +++++++\n" % (tt.stop()))
t = Utils.TimeTrack()
self.process_one_day(beginDate)
logger.info("===== 处理 %s 完毕，耗时: %s 秒。 =======\n" %
(beginDate, t.stop()))
beginDate = beginDate + one_day
logger.info("----- 处理结束 -------\n\n\n")
def repackageFrw(frwDir, bootClassFolder):
""" Repackage the classes.dex into jar of frwDir.
"""
if OPTIONS.formatFrw == False : return
# Keep the old directory, we will change back after some operations.
oldDir = os.path.abspath(os.curdir)
# Some dexFiles are parted, such as framework-classes2.dex
regex = re.compile("(.*)-(classes\d?).dex")
Log.i(TAG, "Repackage JARs of %s" %(frwDir))
os.chdir(frwDir)
for dexFile in os.listdir(bootClassFolder):
if dexFile.endswith(".dex"):
jarFile = dexFile[0:-4] + ".jar"
dexName = "classes.dex"
if not os.path.exists(jarFile):
# Match out the jar file with regex
matcher = regex.match(dexFile)
if matcher != None:
jarFile = matcher.group(1) + ".jar"
dexName = matcher.group(2) + ".dex"
Log.d(TAG, "Repackage %s" %(jarFile))
# Put the dex and framework's jar in the same folder, and jar into the jarFile
shutil.move(os.path.join(bootClassFolder, dexFile), os.path.join(frwDir, dexName))
Utils.runWithOutput(["jar", "uf", jarFile, dexName])
if os.path.exists(dexName):
os.remove(dexName)
os.chdir(oldDir)
def deoatAppWithArch(appsDir, frwDir, arch, arch2):
""" De-oat app
"""
if OPTIONS.formatApp == False: return
Log.i(TAG, "De-oat files of oat-format in %s" % (appsDir))
bootClassFolderArch = os.path.join(frwDir, arch, "odex")
bootClassFolderArch2 = os.path.join(frwDir, arch2, "odex")
#for app in $( ls $appdir ); do
for app in os.listdir(appsDir):
appPath = os.path.join(appsDir, app)
apkFile = os.path.join(appPath, app + ".apk")
archPath = os.path.join(appPath, "oat", arch)
#if [[ -d "$appdir/$app/$arch" ]];
if os.path.exists(archPath):
odexFile = os.path.join(archPath, app + ".odex")
#java -Xmx512m -jar $oat2dex $appdir/$app/$arch/$app.odex $framedir/$arch/odex
Utils.runWithOutput(
[OatZip.OAT2DEX, odexFile, bootClassFolderArch])
else:
# if exists arch2
if arch2.strip():
arch2Path = os.path.join(appPath, "oat", arch2)
if os.path.exists(arch2Path):
odexFile2 = os.path.join(arch2Path, app + ".odex")
Utils.runWithOutput(
[OatZip.OAT2DEX, odexFile2, bootClassFolderArch2])
def deoatApp(oatApkDir, bootClassFolder):
""" De-oat app
"""
if OPTIONS.formatApp == False: return
Log.i(TAG, "De-oat files of oat-format in %s, with BOOTCLASSFOLDER=%s" %(oatApkDir, bootClassFolder))
for (dirpath, dirnames, filenames) in os.walk(oatApkDir):
dirnames = dirnames # no use, to avoid warning
for filename in filenames:
if filename.endswith(".odex"):
# no need to de-oat if original apk does not exist
apkFile = filename[0:-5] + ".apk"
apkPath = os.path.dirname(dirpath)
if not os.path.exists(os.path.join(apkPath, apkFile)):
continue
oatApk = os.path.join(dirpath, filename)
deoatApk = oatApk[0:-5] + ".dex"
if os.path.exists(deoatApk):
Log.d(TAG, "Delete the already exists %s" % deoatApk)
os.remove(deoatApk)
Utils.runWithOutput([OatZip.OAT2DEX, oatApk, bootClassFolder])
def DeleteVolume(self, request, context):
Utils.validate_param_exists(request, 'volume_id')
volume_id = request.volume_id
nvmesh_vol_name = volume_id
#secrets = request.secrets
err, out = VolumeAPI().delete([NVMeshVolume(_id=nvmesh_vol_name)])
if err:
self.logger.error(err)
raise DriverError(StatusCode.INTERNAL, err)
self.logger.debug(out)
if not out[0]['success']:
err = out[0]['error']
if err == "Couldn't find the specified volume":
# Idempotency - Trying to remove a Volume that doesn't exists, perhaps already deleted
# should return success
pass
else:
raise DriverError(StatusCode.FAILED_PRECONDITION, err)
return DeleteVolumeResponse()
def ui_loop(self):
self.render_init()
while True:
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
sys.exit()
elif event.type == pygame.KEYDOWN:
if event.key == K_DELETE:
self.render_init()
elif event.key == K_TAB:
self.disp_shadow = not self.disp_shadow
elif event.type == pygame.MOUSEBUTTONDOWN:
if event.button == 4:
self.mag = Utils.interval(
1, Const.UI_max_magnification, self.mag + 1)
self.update_anchor()
elif event.button == 5:
self.mag = Utils.interval(
1, Const.UI_max_magnification, self.mag - 1)
self.update_anchor()
elif event.type == pygame.MOUSEMOTION:
if any(event.buttons):
self.update_anchor(*event.rel)
self.render()
pygame.display.update()
time.sleep(Const.UI_flush_interval)
def DeleteVolume(self, request, context):
Utils.validate_param_exists(request, 'volume_id')
volume_id = request.volume_id
log = self.logger.getChild("DeleteVolume-%s" % volume_id)
log.debug('delete called')
zone, nvmesh_vol_name = Utils.zone_and_vol_name_from_co_id(volume_id)
#secrets = request.secrets
volume_api = VolumeAPIPool.get_volume_api_for_zone(zone, log)
err, out = volume_api.delete([NVMeshVolume(_id=nvmesh_vol_name)])
if err:
log.error(err)
raise DriverError(StatusCode.INTERNAL, err)
log.debug(out)
if not out[0]['success']:
err = out[0]['error']
if err == "Couldn't find the specified volume" or err.startswith(
"Failed to find marked volume"):
# Idempotency - Trying to remove a Volume that doesn't exists, perhaps already deleted
# should return success
log.debug("Volume already deleted")
pass
else:
raise DriverError(StatusCode.FAILED_PRECONDITION, err)
else:
log.debug("Volume deleted successfully from zone %s" % zone)
self.volume_to_zone_mapping.remove(nvmesh_vol_name)
return DeleteVolumeResponse()
def get(self, request):
query_params = request.query_params
context = {'request': request}
id = query_params.get('id', None)
page_no = query_params.get('page_no', None)
page_no = Utils.to_int(page_no)
page_size = query_params.get('page_size', None)
page_size = Utils.to_int(page_size)
if id:
try:
order = Order.objects.get(id=id, user=request.user)
except Order.DoesNotExist as e:
return Response({'message': 'Please provide valid id'},
status=status.HTTP_400_BAD_REQUEST)
else:
serializer = GetOrderSerializer(instance=order,
context=context)
return Response(serializer.data)
else:
if not (page_no and page_size):
return Response(
{'message': 'Please provide valid page_no and page_size'},
status=status.HTTP_400_BAD_REQUEST)
orders = Order.objects.filter(
user=request.user).order_by('-modified_on')
p = Paginator(orders, page_size)
serializer = GetOrderSerializer(instance=p.page(page_no),
context=context,
many=True)
return Response({
'payload': serializer.data,
'total_count': p.count
})
def NodeUnstageVolume(self, request, context):
Utils.validate_params_exists(request,
['volume_id', 'staging_target_path'])
reqJson = MessageToJson(request)
self.logger.debug(
'NodeUnstageVolume called with request: {}'.format(reqJson))
volume_id = request.volume_id
staging_target_path = request.staging_target_path
nvmesh_volume_name = volume_id
if os.path.exists(staging_target_path):
FileSystemManager.umount(target=staging_target_path)
if os.path.isfile(staging_target_path):
self.logger.debug(
'NodeUnstageVolume removing stage bind file: {}'.format(
staging_target_path))
os.remove(staging_target_path)
elif os.path.isdir(staging_target_path):
self.logger.debug(
'NodeUnstageVolume removing stage dir: {}'.format(
staging_target_path))
FileSystemManager.remove_dir(staging_target_path)
else:
self.logger.warning(
'NodeUnstageVolume - mount path {} not found.'.format(
staging_target_path))
Utils.nvmesh_detach_volume(nvmesh_volume_name)
return NodeUnstageVolumeResponse()
def partition(data, lo, hi):
if len(data) <= 0:
return -1
if lo > hi:
return -1
pivot = lo
left, right = lo + 1, hi
while True:
while data[left] < data[pivot]:
left += 1
if left == hi:
break
while data[right] > data[pivot]:
right -= 1
if right == lo: # 当right == lo时，切分元素刚好是最小的元素，不用换位置。
break
if left < right:
Utils.exchange(data, left, right)
else:
break
Utils.exchange(data, pivot, right) # exchange data[pivot] with data[right]
return right
def deoatApp(oatApkDir, bootClassFolder):
""" De-oat app
"""
if OPTIONS.formatApp == False: return
Log.i(
TAG, "De-oat files of oat-format in %s, with BOOTCLASSFOLDER=%s" %
(oatApkDir, bootClassFolder))
for (dirpath, dirnames, filenames) in os.walk(oatApkDir):
dirnames = dirnames # no use, to avoid warning
for filename in filenames:
if filename.endswith(".odex"):
# no need to de-oat if original apk does not exist
apkFile = filename[0:-5] + ".apk"
apkPath = os.path.dirname(dirpath)
if not os.path.exists(os.path.join(apkPath, apkFile)):
continue
oatApk = os.path.join(dirpath, filename)
deoatApk = oatApk[0:-5] + ".dex"
if os.path.exists(deoatApk):
Log.d(TAG, "Delete the already exists %s" % deoatApk)
os.remove(deoatApk)
Utils.runWithOutput(
[OatZip.OAT2DEX, oatApk, bootClassFolder])
def CreateVolume(self, request, context):
request_uuid = str(uuid.uuid4())[:8]
Utils.validate_param_exists(request, 'name')
request_name = request.name
nvmesh_vol_name = Utils.volume_id_to_nvmesh_name(request_name)
log = self.logger.getChild("CreateVolume:%s(request:%s)" %
(request_name, request_uuid))
volume_cache = self.volume_to_zone_mapping.get_or_create_new(
nvmesh_vol_name)
if volume_cache.lock.locked():
log.debug(
"volume already has a request in a progress, waiting for lock to be released"
)
with volume_cache.lock:
log.debug("processing request")
if volume_cache.csi_volume:
if volume_cache.csi_volume.capacity_bytes != self._parse_required_capacity(
request.capacity_range):
raise DriverError(
StatusCode.FAILED_PRECONDITION,
'Volume already exists with different capacity')
log.info('Returning volume from cache')
return CreateVolumeResponse(volume=volume_cache.csi_volume)
csiVolume = self.do_create_volume(log, nvmesh_vol_name, request)
volume_cache.csi_volume = csiVolume
return CreateVolumeResponse(volume=csiVolume)
def completed(self):
self.wslogger.flush()
self.sess.task = None
self.logger.debug("%s task completed", self.sess.log_str())
if self.run_log:
Utils.stash_last_run_log("", self.sess.user_id(), self.run_log)
self.logger.debug("%s stashed run log", self.sess.log_str())
def run_verify(qlogger, msg_type, roles):
qlogger.srvr_log(logging.INFO, "beginning run_verify")
outdir = None
cwd = os.getcwd()
strio = StringIO.StringIO()
try:
root_path = os.path.dirname(cs_pkg_file)
outdir = Utils.mkdtemp(prefix="verify_")
if os.path.exists(root_path):
root_path = os.path.split(root_path)[0]
os.chdir(root_path) # otherwise we are running inside a packaged folder and resources are availale at cwd
from circuitscape.verify import cs_verifyall
testResult = cs_verifyall(out_path=outdir, ext_logger=qlogger, stream=strio)
testsPassed = testResult.wasSuccessful()
except Exception as e:
qlogger.clnt_log("Unexpected error during verify.")
qlogger.srvr_log(logging.WARNING, "Exception during verify: " + str(e))
testsPassed = False
finally:
os.chdir(cwd)
Utils.rmdir(outdir)
qlogger.clnt_log(strio.getvalue())
strio.close()
qlogger.srvr_log(logging.INFO, "end run_verify")
qlogger.send_result_msg(msg_type, {'complete': True, 'success': testsPassed})
def main(args):
total_trials, num_episodes = 1, 30
lr, epsilon, gamma = 0.01, 0.01, 1.0
train_option = int(args[0])
if train_option == 1:
num_jobs, alarm_consistency = int(args[1]), bool(int(args[2]))
print "Running Sarsa with %s alarm consistency, and %d jobs" % (
str(alarm_consistency), num_jobs)
trial_ranges = Utils.get_trial_splits(total_trials,
int(total_trials / num_jobs))
sarsa_results = Parallel(n_jobs=num_jobs)(
delayed(Trainer.train_sarsa)(num_trials=trial_range,
num_episodes=num_episodes,
lr=lr,
gamma=gamma,
total_trials=total_trials,
alarm_consistency=alarm_consistency,
epsilon=epsilon)
for trial_range in trial_ranges)
np.save(
Constants.project_root + "/results/sarsa_rewards_" +
str(alarm_consistency), np.sum(sarsa_results, axis=0))
elif train_option == 2:
num_jobs, alarm_consistency = int(args[1]), bool(int(args[2]))
print "Running Q-Learning with %s alarm consistency, and %d jobs" % (
str(alarm_consistency), num_jobs)
trial_ranges = Utils.get_trial_splits(total_trials,
int(total_trials / num_jobs))
q_learning_results = Parallel(n_jobs=num_jobs)(
delayed(Trainer.train_sarsa)(num_trials=trial_range,
num_episodes=num_episodes,
lr=lr,
gamma=gamma,
total_trials=total_trials,
alarm_consistency=alarm_consistency,
epsilon=epsilon)
for trial_range in trial_ranges)
np.save(
Constants.project_root + "/results/q_learning_rewards_" +
str(alarm_consistency), np.sum(q_learning_results, axis=0))
elif train_option == 3:
result_true = np.load(Constants.project_root +
"/results/q_learning_rewards_True.npy")
result_false = np.load(Constants.project_root +
"/results/q_learning_rewards_False.npy")
avg_true = np.average(result_true, axis=0)
avg_false = np.average(result_false, axis=0)
stddev_true = np.std(result_true, axis=0)
stddev_false = np.std(result_false, axis=0)
Trainer.plot_rewards_and_episodes(avg_true, stddev_true,
"Q-Learning with Consistent Alarm",
"o", "b")
Trainer.plot_rewards_and_episodes(
avg_false, stddev_false, "Q-Learning with Inconsistent Alarm", "^",
"r")
def check_loop(self):
while True:
try:
self.check()
Utils.log("[Board check] ok")
except:
Utils.log("[Board check] failed")
time.sleep(Const.BOARD_check_interval)
def publish(self, merged_metrics):
if merged_metrics:
for (metric_group, stats) in merged_metrics.items():
for stat in stats:
if stat.enabled:
Utils.publish_gmetric(
"/etc/ganglia/hdp/HDPSlaves/conf.d/gmond.slave.conf",
stat.name, stat.value, stat.type_str)
def on_open(ws):
Utils.log("### open ###")
message = {
"type": "join_channel",
"channel": "paintboard",
"channel_param": ""
}
ws.send(json.dumps(message))
def ControllerUnpublishVolume(self, request, context):
Utils.validate_params_exists(request, ['node_id', 'volume_id'])
nvmesh_vol_name = request.volume_id
self._validate_volume_exists(nvmesh_vol_name)
self._validate_node_exists(request.node_id)
return ControllerUnpublishVolumeResponse()
def getLog(self):
"""
Returns the (full) logcat output
"""
args = ['shell', 'logcat', '-d', '-v', 'thread', '&&',
'%sadb' % Utils.getAdbPath(self.sdkPath), '-s', 'emulator-%s' % str(self.port), 'shell', 'logcat', '-b', 'events', '-d', '-v', 'thread', '&&',
'%sadb' % Utils.getAdbPath(self.sdkPath), '-s', 'emulator-%s' % str(self.port), 'shell', 'logcat', '-b', 'radio', '-d', '-v', 'thread']
logcat = self.runAdbCommand(args)[0]
return logcat
def publish(self, merged_metrics):
if merged_metrics:
for (metric_group, stats) in merged_metrics.items():
for stat in stats:
if stat.enabled:
Utils.publish_gmetric("/etc/ganglia/hdp/HDPSlaves/conf.d/gmond.slave.conf",
stat.name,
stat.value,
stat.type_str)
def dealWithAdditionalApks(systemDir, frwDir, arch, arch2, allAppDirs):
''' deal with additional apks '''
if OPTIONS.formatApp == False: return
# Keep the old directory, we will change back after some operations.
oldDir = os.path.abspath(os.curdir)
bootClassFolderArch = os.path.join(frwDir, arch, "odex")
bootClassFolderArch2 = os.path.join(frwDir, arch2, "odex")
for (dirpath, dirnames, filenames) in os.walk(systemDir):
# Exclude scanned directories
if dirpath in allAppDirs:
continue
dirnames = dirnames # no use, to avoid warning
for filename in filenames:
if filename.endswith(".apk") or filename.endswith(".jar"):
apkFile = os.path.join(dirpath, filename)
if not OatZip.check_validate(apkFile, arch, arch2):
continue
archDir = os.path.join(dirpath, "oat", arch)
#app name
app = filename[0:-4]
if os.path.exists(archDir):
if not OatZip.isDeodexed(apkFile):
odexFile = os.path.join(archDir, app + ".odex")
if os.path.exists(odexFile):
Utils.runWithOutput([
OatZip.OAT2DEX, odexFile,
bootClassFolderArch
])
OatZip.packageDexToAppWithArch(apkFile, arch)
#rm -rf $appdir/$app/$arch
shutil.rmtree(archDir)
arch2Dir = os.path.join(dirpath, "oat", arch2)
if os.path.exists(arch2Dir):
if not OatZip.isDeodexed(apkFile):
odexFile = os.path.join(arch2Dir, app + ".odex")
if os.path.exists(odexFile):
Utils.runWithOutput([
OatZip.OAT2DEX, odexFile,
bootClassFolderArch2
])
OatZip.packageDexToAppWithArch(apkFile, arch2)
#rm -rf $appdir/$app/$arch
shutil.rmtree(arch2Dir)
if os.path.exists(os.path.join(dirpath, "oat")):
shutil.rmtree(os.path.join(dirpath, "oat"))
def draw(self, x, y, color):
data = {"x": x, "y": y, "color": color}
r = self.ss.post(Const.URL_paint, data)
if r.status_code == 200:
self.last_upd_time = datetime.now().timestamp()
Utils.log("[OK]", self.uid, data, r.text, self.last_upd_time)
return True
Utils.log("[Fail]", self.uid, data, r.status_code, r.text)
return False
def ValidateVolumeCapabilities(self, request, context):
Utils.validate_params_exists(request,
['volume_id', 'volume_capabilities'])
nvmesh_vol_name = request.volume_id
#UNUSED - capabilities = request.volume_capabilities
# always return True
confirmed = ValidateVolumeCapabilitiesResponse.Confirmed(
volume_capabilities=request.volume_capabilities)
return ValidateVolumeCapabilitiesResponse(confirmed=confirmed)
def _on_auth(self, uid, creds, store):
if not creds:
raise tornado.web.HTTPError(500, "Storage auth failed")
sess_id = Session.extract_session_id(self)
sess = Session.get_session(sess_id)
logger.debug("%s storage authenticated for with credentials %s", sess.log_str(), creds.to_json())
sess.storage_auth_valid(self, creds, store)
if uid != None:
logger.debug("%s stashed credentials for %s", sess.log_str(), str(uid))
Utils.stash_storage_creds(StorageHandlerBase.SEC_SALT, uid, creds)
def NodePublishVolume(self, request, context):
# NodePublishVolume: This method is called to mount the volume from staging to target path.
Utils.validate_params_exists(request, ['volume_id', 'target_path'])
volume_id = request.volume_id
nvmesh_volume_name = volume_id
staging_target_path = request.staging_target_path
publish_path = request.target_path
volume_capability = request.volume_capability
access_mode = volume_capability.access_mode.mode
readonly = request.readonly
access_type = self._get_block_or_mount_volume(request)
block_device_path = Utils.get_nvmesh_block_device_path(
nvmesh_volume_name)
reqJson = MessageToJson(request)
self.logger.debug(
'NodePublishVolume called with request: {}'.format(reqJson))
if not Utils.is_nvmesh_volume_attached(nvmesh_volume_name):
raise DriverError(
StatusCode.NOT_FOUND,
'nvmesh volume {} was not found under /dev/nvmesh/'.format(
nvmesh_volume_name))
flags = []
# K8s Bug Workaround: readonly flag is not sent to CSI, so we try to also infer from the AccessMode
if readonly or access_mode == Consts.AccessMode.MULTI_NODE_READER_ONLY:
flags.append('-o ro')
if access_type == Consts.VolumeAccessType.BLOCK:
# create an empty file for bind mount
with open(publish_path, 'w'):
pass
# bind directly from block device to publish_path
self.logger.debug(
'NodePublishVolume trying to bind mount as block device {} to {}'
.format(block_device_path, publish_path))
FileSystemManager.bind_mount(source=block_device_path,
target=publish_path,
flags=flags)
else:
self.logger.debug(
'NodePublishVolume trying to bind mount {} to {}'.format(
staging_target_path, publish_path))
FileSystemManager.bind_mount(source=staging_target_path,
target=publish_path,
flags=flags)
return NodePublishVolumeResponse()
def deoatFrw(oatJarDir):
""" De-oat framework
"""
if not OPTIONS.formatFrw: return
Log.i(TAG, "De-oat files of oat-format in %s" % oatJarDir)
for item in os.listdir(oatJarDir):
if item.endswith(".odex"):
# COMMANDS: oat2dex boot <jar-of-oat-format>
oatJar = os.path.join(oatJarDir, item)
Utils.runWithOutput([OatZip.OAT2DEX, "boot", oatJar])
def deoatFrw(oatJarDir):
""" De-oat framework
"""
if not OPTIONS.formatFrw: return
Log.i(TAG, "De-oat files of oat-format in %s" % oatJarDir)
for item in os.listdir(oatJarDir):
if item.endswith(".odex"):
# COMMANDS: oat2dex boot <jar-of-oat-format>
oatJar = os.path.join(oatJarDir, item)
Utils.runWithOutput([OatZip.OAT2DEX, "boot", oatJar])
def register_user(cls, email, password):
if not Utils.email_is_valid(email):
raise error.InvalidEmailError('Email format error')
try:
cls.find_by_email(email)
raise error.UserAlreadyRegisteredError(
"User already exits try forgot password if you''ve lost password."
)
except error.UserNotFoundError:
User(email, Utils.hash_password(password)).save_to_mongo()
return True
def register_user(cls, first_name: str, last_name: str, email: str, password: str):
# check if email already exists
try:
cls.find_by_email(email)
# email already exists
raise CustomErrors.UserAlreadyRegisteredError('A user with that email has already been registered.')
except CustomErrors.UserNotFoundError:
# email not taken
if Utils.validate_email(email):
new_user = cls(first_name, last_name, email, Utils.hash_password(password))
Database.insert(cls.COLLECTION, new_user.json())
else:
raise CustomErrors.InvalidEmailError('The email you have entered is invalid.')
def init_session_with_single_management(logger):
connected = False
api = None
# try until able to connect to NVMesh Management
print("Looking for a NVMesh Management server using {} from servers {}".format(Config.MANAGEMENT_PROTOCOL, Config.MANAGEMENT_SERVERS))
while not connected:
try:
api, connected = NVMeshSDKHelper._try_to_connect_to_single_management(logger)
except ManagementTimeout as ex:
NVMeshSDKHelper.logger.info("Waiting for NVMesh Management servers on ({}) {}".format(Config.MANAGEMENT_PROTOCOL, Config.MANAGEMENT_SERVERS))
Utils.interruptable_sleep(10)
print("Connected to NVMesh Management server on ({}) {}".format(Config.MANAGEMENT_PROTOCOL, Config.MANAGEMENT_SERVERS))
return api
def NodeUnpublishVolume(self, request, context):
Utils.validate_params_exists(request, ['volume_id', 'target_path'])
target_path = request.target_path
reqJson = MessageToJson(request)
self.logger.debug(
'NodeUnpublishVolume called with request: {}'.format(reqJson))
if not os.path.exists(target_path):
raise DriverError(StatusCode.NOT_FOUND,
'mount path {} not found'.format(target_path))
if not FileSystemManager.is_mounted(mount_path=target_path):
self.logger.debug(
'NodeUnpublishVolume: {} is already not mounted'.format(
target_path))
else:
FileSystemManager.umount(target=target_path)
block_device_publish_path = target_path + '/mount'
if os.path.isfile(block_device_publish_path):
self.logger.debug(
'NodeUnpublishVolume removing publish bind file: {}'.format(
block_device_publish_path))
os.remove(block_device_publish_path)
if os.path.isfile(block_device_publish_path):
raise DriverError(StatusCode.INTERNAL,
'node-driver unable to delete publish path')
if os.path.isdir(target_path):
self.logger.debug(
'NodeUnpublishVolume removing publish dir: {}'.format(
target_path))
FileSystemManager.remove_dir(target_path)
if os.path.isdir(target_path):
raise DriverError(
StatusCode.INTERNAL,
'node-driver unable to delete publish directory')
elif os.path.isfile(target_path):
self.logger.debug(
'NodeUnpublishVolume removing publish file: {}'.format(
target_path))
os.remove(target_path)
self.logger.debug(
'NodeUnpublishVolume finished successfully for request: {}'.format(
reqJson))
return NodeUnpublishVolumeResponse()
def deoatFrwWithArch(frwDir, arch):
""" De-oat framework
"""
if not OPTIONS.formatFrw: return
Log.i(TAG, "De-oat files of oat-format in %s" % frwDir)
archDir = os.path.join(frwDir, arch)
odexDir = os.path.join(archDir, "odex")
for item in os.listdir(archDir):
if item.endswith(".odex"):
jarFile = os.path.join(frwDir, item[0:-5] + ".jar")
if not OatZip.isDeodexed(jarFile):
odexFile = os.path.join(archDir, item)
Utils.runWithOutput([OatZip.OAT2DEX, odexFile, odexDir])
def getLog(self):
"""
Returns the (full) logcat output
"""
args = [
'shell', 'logcat', '-d', '-v', 'thread', '&&',
'%sadb' % Utils.getAdbPath(self.sdkPath), '-s',
'emulator-%s' % str(self.port), 'shell', 'logcat', '-b', 'events',
'-d', '-v', 'thread', '&&',
'%sadb' % Utils.getAdbPath(self.sdkPath), '-s',
'emulator-%s' % str(self.port), 'shell', 'logcat', '-b', 'radio',
'-d', '-v', 'thread'
]
logcat = self.runAdbCommand(args)[0]
return logcat
def startLogcatRedirect(self, theFile='/mnt/sdcard/logcat.log', theMaxSize=4096):
"""
Start logcat redirection.
"""
self.log.debug('Start logcat redirect, file: %s, size: %dkBytes' % (theFile, theMaxSize))
if not self.logcatRedirectProcess is None:
self.endLogcatRedirect()
if not self.logcatRedirectProcess is None:
raise EmulatorClientError('Logcat redirect is already running', EmulatorClientError.LOGCAT_REDIRECT_RUNNING)
try:
args = ['%sadb' % Utils.getAdbPath(self.sdkPath), '-s', 'emulator-%s' % str(self.port),
'shell', 'logcat', '-v', 'thread', '-f', theFile, '-r', str(theMaxSize)]
self.logcatRedirectProcess = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
#if self.verbose:
# print self.logcatRedirectProcess.communicate()
except OSError, osErr:
print osErr
raise EmulatorClientError('Failed to run adb command \'%s\': %s' % (args, osErr.strerror),
theCode=EmulatorClientError.ADB_RUN_ERROR,
theBaseError=osErr)
def expand_file_system(block_device_path, fs_type):
fs_type = fs_type.strip()
if fs_type == 'devtmpfs':
raise DriverError(
StatusCode.INVALID_ARGUMENT,
'Device not formatted with FileSystem found fs type {}'.format(
fs_type))
elif fs_type.startswith('ext'):
cmd = 'resize2fs {}'.format(block_device_path)
elif fs_type == 'xfs':
cmd = 'xfs_growfs {}'.format(block_device_path)
else:
raise DriverError(StatusCode.INVALID_ARGUMENT,
'unknown fs_type {}'.format(fs_type))
exit_code, stdout, stderr = Utils.run_command(cmd)
logger.debug("resize file-system finished {} {} {}".format(
exit_code, stdout, stderr))
if exit_code != 0:
raise DriverError(
StatusCode.INTERNAL,
'Error expanding File System {} on block device {}'.format(
fs_type, block_device_path))
return exit_code, stdout, stderr
def start(self):
"""
Starts the emulator with TaintDroid images
"""
self.log.info('Start emulator')
try:
args = ['%semulator' % Utils.getEmulatorPath(self.sdkPath)]
if self.avdName is not None:
args.extend(['-avd', self.avdName])
args.extend(['-kernel', '%szImage' % self.imageDirPath])
args.extend(['-system', '%ssystem.img' % self.imageDirPath])
args.extend(['-ramdisk', '%sramdisk.img' % self.imageDirPath])
args.extend(['-sdcard', '%ssdcard.img' % self.imageDirPath])
args.extend(['-data', '%suserdata.img' % self.imageDirPath])
args.extend(['-port', str(self.port)])
args.extend(['-no-boot-anim'])
if self.runHeadless:
args.extend(['-no-window'])
self.log.debug('- args: %s' % args)
self.emulator = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=_overwriteSignalsForProcess)
except OSError, osErr:
raise EmulatorClientError('Failed to start emulator \'%s\': %s' % (args, osErr.strerror),
theCode=EmulatorClientError.START_EMULATOR_ERROR,
theBaseError=osErr)
def main():
# Get directory
parser = OptionParser(usage='usage: %prog [options] sourceDir targetDir')
parser.add_option('',
'--sdkPath',
metavar='<path>',
default='',
help='Set path to Android SDK')
(options, args) = parser.parse_args()
targetAppDir = None
if len(args) < 1:
raise ValueError('Provide a directory')
if len(args) == 2:
targetAppDir = args[1]
if not os.path.exists(targetAppDir):
os.mkdir(targetAppDir)
sourceAppDir = args[0]
# Get app names
appNameList = Utils._getAppListInDirectory(sourceAppDir)
print 'Get APK Wrapper of %d apps' % len(appNameList)
initialAppList = []
errorAppList = []
for appName in appNameList:
try:
initialAppList.append(
APKWrapper(appName, theSdkPath=options.sdkPath))
except APKWrapperError, apkwErr:
errorAppList.append((appName, apkwErr))
def start(self):
"""
Starts the emulator with TaintDroid images
"""
self.log.info('Start emulator')
try:
args = ['%semulator' % Utils.getEmulatorPath(self.sdkPath)]
if self.avdName is not None:
args.extend(['-avd', self.avdName])
args.extend(['-kernel', '%szImage' % self.imageDirPath])
args.extend(['-system', '%ssystem.img' % self.imageDirPath])
args.extend(['-ramdisk', '%sramdisk.img' % self.imageDirPath])
args.extend(['-sdcard', '%ssdcard.img' % self.imageDirPath])
args.extend(['-data', '%suserdata.img' % self.imageDirPath])
args.extend(['-port', str(self.port)])
args.extend(['-no-boot-anim'])
if self.runHeadless:
args.extend(['-no-window'])
self.log.debug('- args: %s' % args)
self.emulator = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=_overwriteSignalsForProcess)
except OSError, osErr:
raise EmulatorClientError(
'Failed to start emulator \'%s\': %s' % (args, osErr.strerror),
theCode=EmulatorClientError.START_EMULATOR_ERROR,
theBaseError=osErr)
def main():
# Get directory
parser = OptionParser(usage="usage: %prog [options] dir targetDir")
parser.add_option("-m", "--mode", metavar="#", default=0)
parser.add_option("", "--sdkPath", metavar="<path>", default="", help="Set path to Android SDK")
(options, args) = parser.parse_args()
if len(args) < 1:
raise ValueError("Provide a directory")
aDir = args[0]
targetDir = None
if len(args) > 1:
targetDir = args[1]
# Get app names
if int(options.mode) == 0:
appNameList = Utils._getAppListInDirectory(aDir)
print "Get APK Wrapper of %d apps" % len(appNameList)
for appName in appNameList:
renameApp(appName, options.sdkPath)
# Goodware
if int(options.mode) == 1:
if targetDir is None:
raise ValueError("Provide a target directory")
analyzer = helper_analyzer.Analyzer([aDir], theSdkPath=options.sdkPath)
analyzer.baseAppDir = "/home/daniel/Documents/MarketApps/apps"
appList = analyzer.generateList()
for app in appList:
oldName = os.path.join("/home/daniel/Documents/MarketApps/apps", app[1].getApkFileName())
newName = os.path.join(targetDir, app[0])
print "- Copy app form %s to %s" % (oldName, newName)
shutil.copy2(oldName, newName)
def main():
# Get directory
parser = OptionParser(usage='usage: %prog [options] sourceDir targetDir')
parser.add_option('', '--sdkPath', metavar='<path>', default='', help='Set path to Android SDK')
(options, args) = parser.parse_args()
targetAppDir = None
if len(args) < 1:
raise ValueError('Provide a directory')
if len(args) == 2:
targetAppDir = args[1]
if not os.path.exists(targetAppDir):
os.mkdir(targetAppDir)
sourceAppDir = args[0]
# Get app names
appNameList = Utils._getAppListInDirectory(sourceAppDir)
print 'Get APK Wrapper of %d apps' % len(appNameList)
initialAppList = []
errorAppList = []
for appName in appNameList:
try:
initialAppList.append(APKWrapper(appName, theSdkPath=options.sdkPath))
except APKWrapperError, apkwErr:
errorAppList.append((appName, apkwErr))
def auth_valid(self, req):
logger.info("%s authenticated %s", self.log_str(), str(self.user_id()))
self.local_work_dir = None
if not self.cfg().is_user_allowed(self.user_id()):
raise RuntimeError("user not authorized")
logger.debug("%s user %s allowed in role(s) %s", self.log_str(), str(self.user_id()), str(self.user_role()))
# reattach older session if available
old_sessions = self.remove_older_sessions()
self.task = None
self.detach = False
for sess in old_sessions:
if sess.task == None:
continue
self.task = sess.task
self.detach = True
self.local_work_dir = sess.local_work_dir
self.reattach_older_session(sess)
break
if None == self.local_work_dir:
self.local_work_dir = Utils.mkdtemp(prefix="sess_")
logger.debug("%s created temporary folder %s", self.log_str(), str(self.local_work_dir))
else:
logger.debug("%s reattached to old temporary folder %s", self.log_str(), str(self.local_work_dir))
req.set_secure_cookie(Session.cookie_name, self.sess_id, 1)
req.redirect('/auth/storage?uid=' + self.user_id())
def __init__(self, theApkFile, theSdkPath='', theLogger=Logger()):
self.apkFile = theApkFile
self.sdkPath = Utils.addSlashToPath(theSdkPath)
self.log = theLogger
self.sampleId = random.randint(1, 1000000)
self.apkFileName = Utils.splitFileIntoDirAndName(theApkFile)[1]
self.apkPath = Utils.splitFileIntoDirAndName(theApkFile)[0]
self.package = ''
self.manifest = {'activityList':[],
'activityAliasList':[],
'serviceList':[],
'receiverList':[],
'providerList':[],
'uses-permission':[]}
self.__extractFromPermissions()
self.__extractApplication()
def _get_stashed_creds(self):
uid = self.get_argument("uid", None)
credentials = None
if uid != None:
# try to retrieve storage authorization from database
credentials = Utils.retrieve_storage_creds(StorageHandlerBase.SEC_SALT, uid)
if None != credentials:
logger.debug("[%s] retrieved stashed credentials", str(uid))
return (uid, credentials)
def __init__(self, theSdkPath='',
thePort=5554,
theImageDirPath='',
theRunHeadlessFlag=False,
theAvdName=None,
theLogger=Logger()):
self.sdkPath = Utils.addSlashToPath(theSdkPath)
self.port = thePort
self.imageDirPath = Utils.addSlashToPath(theImageDirPath)
self.runHeadless = theRunHeadlessFlag
self.log = theLogger
self.avdName = theAvdName
self.emulator = None
self.logcatRedirectFile = ''
self.logcatRedirectProcess = None
self.adbProcess = None
def repackageApp(appDir):
""" Repackage the classes.dex into apk of appDir
"""
if OPTIONS.formatApp == False: return
# Keep the old directory, we will change back after some operations.
oldDir = os.path.abspath(os.curdir)
Log.i(TAG, "Repackage APKs of %s" %(appDir))
for (dirpath, dirnames, filenames) in os.walk(appDir):
dirnames = dirnames # no use, to avoid warning
for dexFile in filenames:
if dexFile.endswith(".dex"):
apkFile = dexFile[0:-4] + ".apk"
apkPath = os.path.dirname(dirpath)
if not os.path.exists(os.path.join(apkPath, apkFile)):
Log.d(TAG, "No apk matched with %s, Ignore" %dexFile)
continue
dexName = "classes.dex"
Log.d(TAG, "Repackage %s" %(apkPath))
# Put the dex and apk in the same folder, and jar into the apk
shutil.move(os.path.join(dirpath, dexFile), os.path.join(apkPath, dexName))
os.chdir(apkPath)
Utils.runWithOutput(["jar", "uf", apkFile, dexName])
if os.path.exists(dexName):
os.remove(dexName)
shutil.rmtree(dirpath)
os.chdir(oldDir)
def upload_results(qlogger, output_folder, output_zip_name, work_dir, cloud_folder_id, store, extract_folder_id=False):
qlogger.flush()
qlogger.clnt_log("Compressing results for upload...")
# copy the tee log to output folder
run_log_file = os.path.join(work_dir, "run.log")
shutil.copyfile(run_log_file, os.path.join(output_folder, "run.log"))
Utils.compress_folder(output_folder, output_zip_name)
for attempt in range(1,11):
qlogger.clnt_log("Uploading results to cloud store...")
qlogger.flush()
if None == store.copy_to_remote(cloud_folder_id, output_zip_name, mime_type='application/zip', extract_folder_id=extract_folder_id):
qlogger.clnt_log("Error uploading output.zip. attempt " + str(attempt) + " of 10")
qlogger.srvr_log(logging.WARNING, "Error uploading output.zip. attempt " + str(attempt) + " of 10")
time.sleep(5*attempt)
else:
break
qlogger.clnt_log("Uploaded results to cloud store")
Utils.rmdir(output_folder, True)
os.remove(output_zip_name)
def __extractFromPermissions(self):
"""
Extract package and permissions and store them in self.manifest
"""
try:
args = ['%saapt' % Utils.getAaptPath(self.sdkPath),
'd', 'permissions',
self.apkFile]
aapt = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError, osErr:
raise APKWrapperError('Failed to start aapt \'%s\': %s' % (args, osErr.strerror))
def runAdbCommand(self, theArgs):
"""
Runs a simple adb command
"""
args = ['%sadb' % Utils.getAdbPath(self.sdkPath), '-s', 'emulator-%s' % str(self.port)]
args.extend(theArgs)
self.log.debug('Exec adb command: %s' % args)
try:
self.adbProcess = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError, osErr:
raise EmulatorClientError('Failed to run adb command \'%s\': %s' % (args, osErr.strerror),
theCode=EmulatorClientError.ADB_RUN_ERROR,
theBaseError=osErr)
def handle_last_run_log(self, wsmsg):
run_log = Utils.open_last_run_log("", self.sess.user_id())
if None == run_log:
return ({'success': False, 'msg': "No run logs found."}, False)
resp_nv = {
'msg_type': BaseMsg.SHOW_LOG,
'data': ''
}
success = False
try:
for line in run_log.readlines():
resp_nv['data'] = line.rstrip()
self.send(resp_nv, False)
success = True
finally:
run_log.close()
return ({'success': success}, False)
def generateMainReport(theFileName, theReportData):
report = open(theFileName, "w")
report.write('<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>TaintDroid Runner Report</title></head><body><p>')
report.write('<h1>TaintDroid Runner Report</h1>')
report.write('<h2>Parameters</h2>')
report.write('<li><b>Working Directory</b>: %s' % theReportData['workingDir'])
report.write('<li><b>Time</b>: %s-%s - %s-%s' % (Utils.getDateAsString(theReportData['startTime']), Utils.getTimeAsString(theReportData['startTime']), Utils.getDateAsString(theReportData['endTime']), Utils.getTimeAsString(theReportData['endTime'])))
report.write('<li><b>numThreads</b>: %d' % theReportData['numThreads'])
report.write('<li><b>emulatorStartPort</b>: %d' % theReportData['emulatorStartPort'])
report.write('<li><b>cleanImageDir</b>: %s' % theReportData['cleanImageDir'])
report.write('<h2>Apps</h2>')
report.write('<table><tr><th>ID</th><th>Package</th><th>APK Path</th><th>Call</th><th>Cipher</th><th>FS</th><th>Net</th><th>SSL</th><th>SMS</th><th>Errors</th></tr>')
for app in theReportData['appList']:
report.write('<tr><td><li><a href="%s">%06d</a></li></td><td>%s</td><td>%s</td><td>%d</td><td>%d</td><td>%d</td><td>%d</td><td>%d</td><td>%d</td><td>%d</td>' % (app['reportName'], app['id'], app['appPackage'], app['appPath'], app['numCallAction'], app['numCipherUsage'], app['numFileSystem'], app['numNetwork'], app['numSSL'], app['numSMS'], app['numErrors']))
report.write('</table>')
report.write('<h2>Logs</h2>')
report.write('<li><a href="%s">Main Log</li>' % (theReportData['mainLogFile']))
for logFile in theReportData['threadLogFileList']:
report.write('<li><a href="%s">%s</li>' % (logFile, logFile))
report.write('</p></body></html>')
def __extractApplication(self):
"""
Extract application information and store them in self.manifest
"""
# Modes
level = self.Level.START
mainLevel = self.Level.START
subLevel = self.Level.START
obj = {}
mainObj = {}
subObj = {}
# Run aapt
try:
args = ['%saapt' % Utils.getAaptPath(self.sdkPath),
'd', 'xmltree',
self.apkFile,
'AndroidManifest.xml']
aapt = subprocess.Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError, osErr:
raise APKWrapperError('Failed to start aapt \'%s\': %s' % (args, osErr.strerror))
def deoatBootOAT(bootOAT):
""" De-oat boot.oat
"""
Log.i(TAG, "De-oat %s" % bootOAT)
Utils.runWithOutput([OatZip.OAT2DEX, "boot", bootOAT])
def run_job(qlogger, msg_type, roles, msg_data, work_dir, storage_creds, store_in_cloud, log_str):
qlogger.srvr_log(logging.INFO, "beginning run_job")
solver_failed = True
output_cloud_folder = None
output_folder = None
cfg = CSConfig()
store = GoogleDriveStore(storage_creds, log_str) if (None != storage_creds) else None
all_options_valid = True
for key in msg_data.keys():
val = msg_data[key]
if store_in_cloud and (key in CSConfig.FILE_PATH_PROPS) and (val != None):
# if val is gdrive location, translate it to local drive
if isinstance(val, str) or isinstance(val, unicode):
val = val.strip()
if val.startswith("gdrive://"):
if key == 'output_file':
# store the output gdrive folder
qlogger.clnt_log("Preparing cloud store output folder: " + val)
output_cloud_folder = val
output_folder = os.path.join(work_dir, 'output')
if not os.path.exists(output_folder):
os.mkdir(output_folder)
else:
Utils.rmdir(output_folder, True)
val = os.path.join(output_folder, 'results.out')
else:
# copy the file locally
qlogger.clnt_log("Reading from cloud store: " + val)
val = store.copy_to_local(val, work_dir)
elif (len(val) > 0):
message = "Configuration parameter %s must have a gdrive URL as value. Current value: '%s'"%(key, val)
all_options_valid = False
break
elif key == 'output_file':
message = "Output location not specified."
all_options_valid = False
break
cfg.__setattr__(key, val)
if all_options_valid:
qlogger.clnt_log("Verifying configuration...")
(all_options_valid, message) = cfg.check()
if all_options_valid:
qlogger.clnt_log("Verifying profile limits...")
(all_options_valid, message) = CircuitscapeRunner.check_role_limits(roles, cfg, qlogger)
if not all_options_valid:
qlogger.send_error_msg(message)
else:
# In cloud mode, this would be a temporary directory
outdir, _out_file = os.path.split(cfg.output_file)
try:
qlogger.clnt_log("Storing final configuration...")
configFile = os.path.join(outdir, 'circuitscape.ini')
cfg.write(configFile)
cs = Compute(configFile, qlogger)
result, solver_failed = cs.compute()
qlogger.clnt_log("Result: \n" + str(result))
except Exception as e:
message = str(e)
qlogger.send_error_msg(message)
success = not solver_failed
if success and store_in_cloud:
output_folder_zip = os.path.join(work_dir, 'output.zip')
CircuitscapeRunner.upload_results(qlogger, output_folder, output_folder_zip, work_dir, output_cloud_folder, store, extract_folder_id=False)
qlogger.srvr_log(logging.INFO, "end run_job")
qlogger.send_result_msg(msg_type, {'complete': True, 'success': success})
def run_batch(qlogger, msg_type, roles, msg_data, work_dir, storage_creds, store_in_cloud, log_str):
qlogger.srvr_log(logging.INFO, "beginning run_batch")
cwd = os.getcwd()
store = GoogleDriveStore(storage_creds, log_str) if (None != storage_creds) else None
if store_in_cloud:
batch_zip = msg_data
batch_zip_name = os.path.splitext(store.to_file_name(batch_zip))[0] + '_output.zip'
qlogger.clnt_log("Reading from cloud store: " + batch_zip)
batch_folder = Utils.mkdtemp(prefix="batch_")
batch_zip = store.copy_to_local(batch_zip, work_dir)
Utils.uncompress_folder(batch_folder, batch_zip)
os.remove(batch_zip)
output_folder_zip = os.path.join(work_dir, batch_zip_name)
else:
batch_folder = msg_data
qlogger.clnt_log("Running batch with all configurations (.ini files) under folder: " + batch_folder)
output_root_folder = Utils.mkdtemp_if_exists(prefix="output", dir=batch_folder)
if store_in_cloud:
qlogger.clnt_log("Outputs would be uploaded to your cloud store as " + batch_zip_name)
else:
qlogger.clnt_log("Outputs would be stored under folder: " + output_root_folder)
num_success = 0
num_failed = 0
# collect all configuration names
config_files = []
for root, _dirs, files in os.walk(batch_folder):
for file_name in files:
if not file_name.endswith(".ini"):
continue
config_files.append(os.path.join(root, file_name))
num_configs = len(config_files)
qlogger.clnt_log("Found " + str(num_configs) + " configuration files.")
parallelize = True if (num_configs > CircuitscapeRunner.MAX_PARALLEL) else False
batch_success = False
if parallelize:
pool = []
results = []
try:
for config_file in config_files:
root, file_name = os.path.split(config_file)
qlogger.clnt_log("Loading configuration: " + config_file)
cfg = CSConfig(config_file)
os.chdir(root)
qlogger.clnt_log("Verifying configuration...")
if parallelize: # switch off parallization of each task if we are parallelizing batch
cfg.parallelize = False
(all_options_valid, message) = cfg.check()
#qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid))
if all_options_valid:
#qlogger.clnt_log("Verifying all paths are relative...")
all_options_valid = cfg.are_all_paths_relative();
if not all_options_valid:
message = "All file paths in configuration must be relative to location of configuration file."
#qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid))
if all_options_valid:
qlogger.clnt_log("Verifying profile limits...")
(all_options_valid, message) = CircuitscapeRunner.check_role_limits(roles, cfg, qlogger)
if not all_options_valid:
qlogger.send_error_msg(message)
num_failed += 1
else:
solver_failed = True
cfgname = os.path.splitext(file_name)[0]
output_folder = os.path.join(output_root_folder, cfgname)
os.mkdir(output_folder)
cfg.output_file = os.path.join(output_folder, os.path.basename(cfg.output_file))
outdir, _out_file = os.path.split(cfg.output_file)
try:
qlogger.clnt_log("Storing final configuration...")
configFile = os.path.join(outdir, 'circuitscape.ini')
cfg.write(configFile)
if parallelize:
if len(pool) >= CircuitscapeRunner.MAX_PARALLEL:
p = pool.pop()
p.join()
result = results.pop()
if result.value == 0:
num_success += 1
else:
num_failed += 1
result = Value('i', -1)
p = Process(target=CircuitscapeRunner._run_compute, args=(configFile, qlogger, cfgname + ' => ', result))
pool.append(p)
results.append(result)
p.start()
else:
cs = Compute(configFile, qlogger)
result, solver_failed = cs.compute()
qlogger.clnt_log("Result: \n" + str(result))
except Exception as e:
message = str(e)
qlogger.send_error_msg(message)
if not parallelize:
if solver_failed:
num_failed += 1
else:
num_success += 1
if parallelize:
qlogger.srvr_log(logging.DEBUG, "waiting for processes len=" + str(len(pool)))
for idx in range(0, len(pool)):
pool[idx].join()
if results[idx].value == 0:
num_success += 1
else:
num_failed += 1
qlogger.srvr_log(logging.DEBUG, "Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed))
qlogger.clnt_log("Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed))
if num_success > 0:
if store_in_cloud:
CircuitscapeRunner.upload_results(qlogger, output_root_folder, output_folder_zip, work_dir, msg_data, store, extract_folder_id=True)
else:
qlogger.clnt_log("Outputs under folder: " + output_root_folder)
batch_success = True
except Exception as e:
qlogger.clnt_log("Unexpected error during batch run.")
qlogger.srvr_log(logging.WARNING, "Exception during batch run: " + str(e))
finally:
os.chdir(cwd)
if store_in_cloud:
try:
Utils.rmdir(batch_folder)
if os.path.exists(output_folder_zip):
os.remove(output_folder_zip)
except:
qlogger.srvr_log(logging.ERROR, "Could not clear temporary files.")
qlogger.srvr_log(logging.INFO, "end run_batch")
qlogger.send_result_msg(msg_type, {'complete': True, 'success': batch_success})
def generateAppReport(theFileName, theResultEntry):
# Inits
app = theResultEntry['app']
log = None
if theResultEntry.has_key('log'):
log = theResultEntry['log']
# Write report
report = codecs.open(theFileName, "w", "utf-8")
report.write('<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>TaintDroid Runner Report for %s</title></head><body><p>' % app.getPackage())
report.write('<h1>TaintDroid Runner Report for %s</h1>' % (app.getPackage()))
report.write('<h2>Parameters</h2>')
report.write('<li><b>Package</b>: %s' % app.getPackage())
report.write('<li><b>APK Name</b>: %s' % app.getApkFileName())
report.write('<li><b>Path</b>: %s' % app.getApk())
report.write('<li><b>Id</b>: %06d' % app.getId())
report.write('<li><b>steps</b>: %s' % SimulationSteps.getStepsAsString(theResultEntry['steps']))
report.write('<li><b>numMonkeyEvents</b>: %d' % theResultEntry['numMonkeyEvents'])
report.write('<li><b>sleepTime</b>: %d' % theResultEntry['sleepTime'])
report.write('<li><b>startTime</b>: %s-%s' % (Utils.getDateAsString(theResultEntry['startTime']), Utils.getTimeAsString(theResultEntry['startTime'])))
report.write('<li><b>endTime</b>: %s-%s' % (Utils.getDateAsString(theResultEntry['endTime']), Utils.getTimeAsString(theResultEntry['endTime'])))
report.write('<li><b>cleanImageDir</b>: %s' % theResultEntry['cleanImageDir'])
report.write('<li><b>MD5 (hex)</b>: %s' % app.getMd5Hash())
report.write('<li><b>Sha256 (hex)</b>: %s' % app.getSha256Hash())
report.write('<li><b>maxLogcatSize</b>: %d' % theResultEntry['maxLogcatSize'])
report.write('<h2>Log</h2>')
if log is None:
report.write('<h3>CallUsage</h3>')
report.write('<h3>CipherUsage</h3>')
report.write('<h3>FileSystem</h3>')
report.write('<h3>Network</h3>')
report.write('<h3>SSL</h3>')
report.write('<h3>SMS</h3>')
else:
ReportGenerator.__generateReportLogTable(report,
'Call',
['Tag', 'DialString', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=CallActionLogEntry)
)
ReportGenerator.__generateReportLogTable(report,
'CipherUsage',
['Tag', 'Mode', 'PlainText', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=CipherUsageLogEntry)
)
ReportGenerator.__generateReportLogTable(report,
'FileSystem',
['Tag', 'Action', 'File', 'Id', 'Data', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=FileSystemLogEntry)
)
ReportGenerator.__generateReportLogTable(report,
'Network',
['Tag', 'Action', 'Destination', 'Id', 'Data', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=NetworkSendLogEntry)
)
ReportGenerator.__generateReportLogTable(report,
'SSL',
['Tag', 'Action', 'Destination', 'Data', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=SSLLogEntry)
)
ReportGenerator.__generateReportLogTable(report,
'SMS',
['Tag', 'Action', 'Source', 'Destination', 'Dest Tag', 'Text', 'Timestamp', 'StackTrace'],
log.getLogEntryList(theType=SendSmsLogEntry)
)
report.write('<h2>Errors</h2>')
if theResultEntry.has_key('badCancelationFlag'):
report.write('<li>Thread could not be finished</li>')
for error in theResultEntry['errorList']:
try:
report.write('<li>%s</li>' % str(error))
except UnicodeDecodeError, udErr:
report.write('<li>Decode error: %s</li>' % str(udErr))
report.write('<li>')
report.write(error)
report.write('</li>')
def remove_temporary_files(self):
outdir = self.local_work_dir
if None != outdir:
Utils.rmdir(outdir)
logger.debug("%s removed temporary folder %s", self.log_str(), str(outdir))