class GameQueue:
def __init__(self, min_size, max_size):
self.log = Logger(log_level="DEBUG").get_logger()
self.players = OrderedDict()
self.game_dict = {}
self.max_game_size = max_size
self.min_game_size = min_size
def add_player(self, player):
self.players[player] = None
def build_game(self, game_id):
# add validation somewhere here to ensure players are still active
current_size = len(self.players)
if current_size >= self.min_game_size:
for i in range(0, min(self.max_game_size, current_size)):
player = self.players.popitem(last=False)[0]
self.game_dict[player] = game_id
self.log.debug("Adding {} to game.".format(player))
def queue_status(self, player):
# TODO: more fine grained checks
if player in self.game_dict.keys():
return self.game_dict[player]
else:
return None
def crawl():
#Use logger to debug the code
Logger.debug('Hello google')
#Get the html knowledge by parsing url
soup = get_b_soup('http://www.google.com')
#Send the data to output pipe line
Queues.send_to_output(soup.head.title.text)
Queues.send_to_output('http://www.google.com' + soup.head.meta['content'])
def collect(url_queue):
#Use logger to debug the code
Logger.debug('Hello amazon')
#Get the html knowledge by parsing url
soup = get_b_soup(base_path)
#Travel with html knowledge
for all_url in soup.findAll('a', href=re.compile('/gp/product/')):
#REQUIRED
#Send all url via url_queue
url_queue.put(base_path + all_url['href'])
def crawl_thread(url, module):
Logger.debug('Thread start')
encoding = getattr(module, 'ENCODING', None)
try:
soup = get_b_soup(url, encoding=encoding)
module.crawl(soup)
except:
Logger.error('Crawl error url: ' + url)
Logger.error(traceback.format_exc())
return
Logger.debug('Thread done')
def execute(module, name):
#Execute the nested crawl method
Logger.info('Start Execute')
if not hasattr(module, 'collect'):
raise NotImplementedError('You must implement collect()')
#Create the url queue and send to collect and crawl process
url_queue = Queue()
collect_process = Process(target=collect, args=(module, url_queue),
name=name + '_CollectProcess')
collect_process.start()
crawl_process = Process(target=crawl, args=(module, url_queue),
name=name + '_CrawlProcess')
crawl_process.start()
#Both collect and crawl are safe
collect_process.join()
crawl_process.join()
Logger.info('Execute done')
def execute_output():
Logger()
Logger.debug('Start Output Process')
while True:
try:
result = Queues.get_output()
Logger.info(result)
#Your output logic go there
except:
break
Logger.debug('End Output Process')
def run(num_iter=1, serialno=None):
num_iter = int(num_iter)
Adb.init()
Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT, prefix="quick-start")
check_props = {
"Device name": "ro.product.model",
"Project": "ro.build.product",
"ROM": "ro.product.build.fingerprint",
}
passed = True
for tag, prop in check_props.items():
out, err = Adb.execute(["shell", "getprop {}".format(prop)],
serialno=serialno)
passed &= len(err) == 0
out = out.strip()
Logger.log(LOGGER_TAG, "{}: {}".format(tag, out))
Logger.log(LOGGER_TAG,
"result: {}".format("passed" if passed else "failed"))
Logger.finalize()
def crawl(module, url_queue):
#Execute the crawl process
Logger.info('Start Crawler')
while True:
try:
url = url_queue.get(timeout=config.NESTED_CRAWL_TIMEOUT)
except:
#If all threads are done then break the loop, Otherwise continue.
#Why 2 ? because its need to deduct by the main thread and queue thread,
# You can comment out the enumerate() line to see what is going on
#Logger.debug(str(enumerate()))
if activeCount() <= 2:
Logger.info('Break crawl')
break
else:
Logger.debug('There are ' + str(activeCount() - 2) + ' threads left')
continue
#Spawn a new threads immediate after get the url
thread = Thread(target=crawl_thread, args=(url, module), name='CrawlThread')
thread.start()
Logger.info('Crawl done')
import libs.glo_browsertype as glo
# 导入友工程首页操作类
from page_objects import PMCloudIndexActions
# 导入友工程登录页操作类
from page_objects import PMCloudLoginActions
# 导入企业帐号选择页操作类
from page_objects import ApptenantActions
# 导入友工程后台外框架操作类
from page_objects import WorkbenchActions
# 导入友工程联系类型操作类
from page_objects import ContactTypeActions
__author__ = "sunxr"
__version__ = "V1.0"
logger = Logger("TestContactTypeEdit").getLog()
class TestContactTypeEdit(unittest.TestCase):
"""联系类型编辑相关测试"""
@classmethod
def setUpClass(cls):
logger.info("测试前准备.")
cls.__driver = browser(glo.GLO_BROWSER_TYPE)
PMCloudIndexActions(cls.__driver).login()
PMCloudLoginActions(cls.__driver).pmcloudLogin()
ApptenantActions(cls.__driver).apptenantLogin()
WorkbenchActions(cls.__driver).clickContactType()
# 提前新建好联系类型
def __init__(self):
self.db = sqlite3.connect('queue.db')
self.log = Logger(log_level="DEBUG").get_logger()
def __init__(self, min_size, max_size):
self.log = Logger(log_level="DEBUG").get_logger()
self.players = OrderedDict()
self.game_dict = {}
self.max_game_size = max_size
self.min_game_size = min_size
def run(num_iter, serialno):
Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT)
# Logger.init(Logger.Mode.STDOUT)
Adb.init()
Adb.execute(["root"], serialno=serialno)
Adb.execute(["shell", "'echo \'related\' > msm_subsys'"],
serialno=serialno)
Adb.execute(["shell", "svc", "power", "stayon", "true"], serialno=serialno)
out, err = Adb.execute(["shell", "getprop", "ro.vendor.build.fingerprint"],
serialno=serialno)
out = out.strip()
log("build number: '{}'".format(out))
latest_dmesg = fetch_dmesg(serialno=serialno)
latest_dmesg = latest_dmesg[0] if len(latest_dmesg) > 0 else None
pass_trial_cnt = 0
try:
for i in range(num_iter):
log("-------------------- CS call test #{} --------------------".
format(i + 1))
events = fetch_setmode_events(serialno=serialno)
latest_event = events[-1] if len(events) > 0 else None
log("The latest setMode event: '{}'".format(
latest_event.raw if latest_event else "None"))
log("Make call to 0988102544")
start_cs_call(tel="0988102544", serialno=serialno)
log("Waiting to the mode 'MODE_IN_CALL'")
if not wait_for_phone_state(state=2, timeout=10,
serialno=serialno):
log("The phone state never turns in 'MODE_IN_CALL', ignore this trial"
)
continue
log("To check if ADSP crashes during the CS call")
is_passed = True
pass_trial_cnt += 1
retry = 15
while retry > 0:
new_dmesgs = fetch_dmesg(latest_dmesg, serialno=serialno)
if len(new_dmesgs) > 0:
latest_dmesg = new_dmesgs[-1]
adsp_ssr_demsgs = filter(
lambda x: "Restart sequence requested for adsp" in x.
raw, new_dmesgs)
if len(adsp_ssr_demsgs) > 0:
for dmesg in adsp_ssr_demsgs:
log("SSR log detected: '{}'".format(dmesg.raw))
is_passed = False
phone_state = get_phone_state(serialno=serialno)
if phone_state == None:
log("the phone state is unobtainable, something wrong")
is_passed = False
if phone_state == 0:
log("the phone state is in idle, the call might be dropped"
)
is_passed = False
if not is_passed:
out, err = Adb.execute(["bugreport"])
log("bugreport to '{}'".format(out.strip()))
pass_trial_cnt -= 1
break
if retry % 5 == 0:
log("switch path to '{}'".format("speaker" if retry / 5 %
2 == 1 else "receiver"))
Adb.execute(["shell", "input", "tap", "1000", "1000"],
tolog=False,
serialno=serialno)
retry -= 1
time.sleep(1)
log("result: {} ({}/{})".format("pass" if retry == 0 else "fail",
pass_trial_cnt, i + 1))
end_cs_call(serialno=serialno)
time.sleep(10)
except:
pass
Adb.execute(["shell", "svc", "power", "stayon", "false"],
serialno=serialno)
Logger.finalize()
from libs.HTMLTestRunner import HTMLTestRunner
# 导入time模块,用于时间处理
import time
# 导入日志模块
from libs.logger import Logger
# 导入Configuration模块,用于操作配置文件
from libs.configuration import Configuration
# 导入setBrowserType方法,用于多线程启动不同浏览器的兼容性测试
from libs.glo_browsertype import setBrowserType
# 导入os模块，获取文件路径
import os
__author__ = "sunxr"
__version__ = "V1.2"
logger = Logger("TestCaseSuite").getLog()
class TestCaseSuite:
"""
定义测试用例路径,并执行测试用例
"""
def __init__(self):
# 创建配置文件实例
config = Configuration()
# 获取框架主路径信息
self.__home_path = os.path.dirname(os.path.dirname(__file__))
# 读取配置文件中的测试用例路径和测试用例执行规则
from flask import Flask, request
from flask_restful import Api
from resources.respondent import Respondent
from libs.logger import Logger
from config.config import general as config
app = Flask(__name__)
api = Api(app)
log = Logger()
# Routes
api.add_resource(Respondent, '/respondent', '/respondent/<name>')
#-------
if __name__ == '__main__':
log.info('Server listen in %s:%s' % (config['host'], config['port']))
app.run(debug=True, host=config['host'], port=config['port'])
# coding: utf-8
import unittest
from libs.browser import browser
from libs.logger import Logger
import libs.glo_browsertype as glo
from page_objects import PMCloudIndexActions
from page_objects import PMCloudLoginActions
from page_objects import ApptenantActions
logger = Logger("TestApptenant").getLog()
class TestApptenant(unittest.TestCase):
"""测试友工程的企业帐号"""
def setUp(self):
logger.info("测试前准备.")
self.driver = browser(glo.GLO_BROWSER_TYPE)
PMCloudIndexActions(self.driver).openPMCloudIndex()
PMCloudIndexActions(self.driver).clickLoginButton()
PMCloudLoginActions(self.driver).pmcloudLogin(remember_selected=True)
def tearDown(self):
logger.info("测试后退出.")
self.driver.quit()
def test_apptenant_login_success(self):
"""企业帐号验证通过"""
class CXBot(Client, Setup, Bot, IO, Server, Channel, User, Role, Features,
Base):
def __init__(self):
super(CXBot, self).__init__()
self.c = Config()
self.max_messages = 20000
self.ready = False
self.logger = Logger('logs/%s.log' % datetime.now().date().isoformat())
self.busy = False
@Base.excep
async def on_ready(self):
if not self.ready:
self.ready = True
print('Logged on as %s!' % self.user)
if len(self.servers) == 0:
print(
'You have to connect me to a server. Open https://discordapp.com/oauth2/authorize?client_id=%s\
&scope=bot&permissions=0' % self.c.clientID)
await self.logout()
await self.close()
elif len(self.servers) > 1:
print('Connected to the following servers:\n\t%s' %
'\n\t'.join(self._server_names()))
print(
'You connected me to multiple servers. Note that you may want to use multiple instances of this \
bot because otherwise all settings will be the same for all connected servers and some functions \
may could interfere in this state of development.')
else:
print('Connected to the server %s' % self._server_names()[0])
if self.c.firstStart:
self.busy = True
await self.setup()
self.busy = False
async def on_resumed(self):
pass
async def on_error(self, event, *args, **kwargs):
print(event)
@Base.excep
async def on_message(self, message):
if self.c.logMessages:
self.logger.log_message(message)
if self.busy:
return
if not message.author == self.user:
print(message.content)
e = await self.create_embed(
"Github CeroProgramming", "https://github.com/CeroProgramming",
"GitHub Profile URL", Color.dark_purple(), "CeroProgramming",
"https://github.com/CeroProgramming/",
"https://avatars3.githubusercontent.com/u/22818389?s=460&v=4",
"https://proxy.duckduckgo.com/iu/?u=https%3A%2F%2Fimage.freepik.com%2Ffree-icon%2Fgithub-logo_318-53553.jpg&f=1"
)
await self.io_embed(message.channel, e)
if message.content == "shutdown":
await self.shutdown()
async def on_message_delete(self, message):
pass # Increase Clients max_messages for more cached messages
async def on_message_edit(self, before, after):
if self.c.logMessages:
if before.content != after.content or before.embeds != after.embeds or before.attachments != after.attachments:
self.logger.log_message_change(before, after)
async def on_reaction_add(self, reaction, user):
pass
async def on_reaction_remove(self, reaction, user):
pass
async def on_reaction_clear(self, message, reactions):
pass
async def on_channel_create(self, channel):
pass
async def on_channel_delete(self, channel):
pass
async def on_channel_update(self, before, after):
pass
async def on_member_join(self, member):
pass
async def on_member_remove(self, member):
pass
async def on_member_update(self, before, after):
pass
async def on_server_join(self, server):
pass
async def on_server_remove(self, server):
pass # Includes banned, kicked, left, really removed
async def on_server_update(self, before, after):
pass
async def on_server_role_create(self, role):
pass
async def on_server_role_delete(self, role):
pass
async def on_server_role_update(self, before, after):
pass
async def on_server_emojis_update(self, before, after):
pass
async def on_server_available(self, server):
pass
async def on_server_unavailable(self, server):
pass
async def on_voice_state_update(self, before, after):
pass # leave join mute deaf
async def on_member_ban(self, member):
pass
async def on_member_unban(self, server, user):
pass
async def on_typing(self, channel, user, when):
pass
async def on_group_join(self, channel, user):
pass
async def on_group_remove(self, channel, user):
pass
async def shutdown(self): # TODO Test
await self.logout()
await self.close()
def evaluate_imagenet_c(
model,
transform,
dataset_dir: str,
log_dir: str,
corruptions: List[str],
batch_size: int,
device: str,
**kwargs,
) -> None:
"""
Evaluate corruption accuracy on ImageNet-C.
"""
if (device != "cpu") and (torch.cuda.device_count() > 1):
model = torch.nn.DataParallel(model)
log_path = os.path.join(log_dir, os.path.join("imagenet_c_result.csv"))
logger = Logger(path=log_path, mode="test")
with tqdm.tqdm(total=len(corruptions), ncols=80) as pbar:
for i, corruption_type in enumerate(corruptions):
accuracies = list()
for j in range(1, 6): # imagenet-c dataset is separated to 5 small sets.
datasetpath = os.path.join(dataset_dir, corruption_type, str(j))
dataset = torchvision.datasets.ImageFolder(datasetpath, transform)
loader = torch.utils.data.DataLoader(
dataset,
batch_size=batch_size,
shuffle=False,
num_workers=8,
pin_memory=True,
)
for x, y in loader:
x, y = x.to(device), y.to(device)
with torch.autograd.no_grad():
y_predict_std = model(x)
stdacc1, stdacc5 = accuracy(y_predict_std, y, topk=(1, 5))
accuracies.append(stdacc1.item())
log_dict = collections.OrderedDict()
log_dict["corruption_type"] = corruption_type
log_dict["accuracy"] = sum(accuracies) / float(len(accuracies))
logger.log(log_dict)
pbar.set_postfix(
collections.OrderedDict(
corruption_type="{}".format(corruption_type),
acc="{}".format(log_dict["accuracy"]),
)
)
pbar.update()
df = pd.read_csv(log_path)
result_dict = dict(zip(df["corruption_type"], df["accuracy"]))
mean_corruption_acc = sum(result_dict.values()) / float(len(result_dict))
create_barplot(
result_dict,
title="mean corruption acc: {0:0.1f}".format(mean_corruption_acc),
savepath=os.path.join(log_dir, "plot_result.png"),
)
# coding: utf-8
# 导入同目录下的logger日志模块
from libs.logger import Logger
# 导入Configuration模块,用于操作配置文件
from libs.configuration import Configuration
# 导入MySQL操作pymysql模块
import pymysql
__author__ = "sunxr"
__version__ = "V1.2"
logger = Logger("MySQLOperation").getLog()
class MySQLOperation:
"""
将pymysql中的部分方法进行封装,处理对MySQL数据库的简单操作.
"""
def __init__(self):
# 创建配置文件实例
self.__config = Configuration()
# 从配置文件获取数据库相关信息
self.__host = self.__config.getConfigValue("mysqlInfo", "host")
self.__user = self.__config.getConfigValue("mysqlInfo", "user")
self.__password = self.__config.getConfigValue("mysqlInfo", "password")
self.__database = self.__config.getConfigValue("mysqlInfo", "database")
# 连接数据库,获取操作游标
self.__db = self.__dbConnect()
def evaluate_corruption_accuracy(
model,
dataset_builder,
log_dir: str,
num_samples: int,
corruptions: list,
batch_size: int,
device: str,
**kwargs,
):
"""
"""
if (device != "cpu") and (torch.cuda.device_count() > 1):
model = torch.nn.DataParallel(model)
log_path = os.path.join(log_dir, os.path.join("corruption_result.csv"))
logger = Logger(path=log_path, mode="test")
with tqdm.tqdm(total=len(corruptions), ncols=80) as pbar:
for i, corruption_type in enumerate(corruptions):
dataset = dataset_builder(
train=False,
normalize=True,
num_samples=num_samples,
corruption_type=corruption_type,
)
loader = torch.utils.data.DataLoader(
dataset=dataset, batch_size=batch_size, shuffle=False
)
accuracies = list()
for x, y in loader:
x, y = x.to(device), y.to(device)
with torch.autograd.no_grad():
y_predict_std = model(x)
stdacc1, stdacc5 = accuracy(y_predict_std, y, topk=(1, 5))
accuracies.append(stdacc1.item())
log_dict = collections.OrderedDict()
log_dict["corruption_type"] = corruption_type
log_dict["accuracy"] = sum(accuracies) / float(len(accuracies))
logger.log(log_dict)
pbar.set_postfix(
collections.OrderedDict(
corruption_type="{}".format(corruption_type),
acc="{}".format(log_dict["accuracy"]),
)
)
pbar.update()
df = pd.read_csv(log_path)
result_dict = dict(zip(df["corruption_type"], df["accuracy"]))
mean_corruption_acc = sum(result_dict.values()) / float(len(result_dict))
create_barplot(
result_dict,
title="mean corruption acc: {0:0.1f}".format(mean_corruption_acc),
savepath=os.path.join(log_dir, "plot_result.png"),
)
class CLFNet():
def __init__(self, config):
self.config = config
model_name = 'inpaint'
self.debug = False
self.model_name = model_name
self.inpaint_model = InpaintingModel(config).to(config.DEVICE)
# summary(InpaintingModel, (3, 256, 256), 6)
# print(InpaintingModel)
self.psnr = PSNR(255.0).to(config.DEVICE)
self.ssim = SSIM(window_size=11)
val_sample = int(float((self.config.EVAL_INTERVAL)))
# test mode
if self.config.MODE == 2 or self.config.MODE == 4 or self.config.MODE == 5:
self.test_dataset = Dataset(config,
config.TEST_FLIST,
config.TEST_MASK_FLIST,
augment=False,
training=False)
else:
self.train_dataset = Dataset(config,
config.TRAIN_FLIST,
config.TRAIN_MASK_FLIST,
augment=True,
training=True)
self.val_dataset = Dataset(config,
config.VAL_FLIST,
config.VAL_MASK_FLIST,
augment=False,
training=True,
sample_interval=val_sample)
self.sample_iterator = self.val_dataset.create_iterator(
config.SAMPLE_SIZE)
self.samples_path = os.path.join(config.PATH, 'samples')
self.results_path = os.path.join(config.PATH, 'results')
if config.RESULTS is not None:
self.results_path = os.path.join(config.RESULTS)
if config.DEBUG is not None and config.DEBUG != 0:
self.debug = True
# self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat')
log_path = os.path.join(config.PATH, 'logs_' + model_name)
create_dir(log_path)
self.logger = Logger(log_path)
def load(self):
self.inpaint_model.load()
def save(self, epoch):
self.inpaint_model.save(epoch)
def train(self):
train_loader = DataLoader(dataset=self.train_dataset,
batch_size=self.config.BATCH_SIZE,
num_workers=4,
drop_last=True,
shuffle=True)
keep_training = True
model = self.config.MODEL
# max_iteration = int(float((self.config.MAX_ITERS)))
step_per_epoch = int(float((self.config.MAX_STEPS)))
max_epoches = int(float((self.config.MAX_EPOCHES)))
total = int(len(self.train_dataset))
if total == 0:
print(
'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.'
)
return
print('\nThe number of Training data is %d' % total)
epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1
print('\nTraining epoch: %d' % epoch)
progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step'])
logs_ave = {}
train_times = 0
while (keep_training):
for items in train_loader:
self.inpaint_model.train()
images, images_gray, masks = self.cuda(*items)
# edge model
# inpaint model
# train
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * (masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
logs['psnr'] = psnr.item()
# logs['mae'] = mae.item()
# backward
self.inpaint_model.backward(gen_loss, dis_loss)
if self.inpaint_model.iteration > step_per_epoch:
self.inpaint_model.iteration = 0
iteration = 0
iteration = self.inpaint_model.iteration
if iteration == 1: # first step in this epoch
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
if iteration == 0: # mean to jump to new epoch
self.sample(epoch)
self.eval(epoch)
self.save(epoch)
# log current epoch in tensorboard
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / step_per_epoch,
epoch)
# if reach max epoch
if epoch >= max_epoches:
keep_training = False
break
epoch += 1
# new epoch
print('\n\nTraining epoch: %d' % epoch)
for tag, value in logs.items():
logs_ave[tag] = value
progbar = Progbar(step_per_epoch,
width=30,
stateful_metrics=['step'])
self.inpaint_model.iteration += 1 # jump to new epoch and set the iteration to 1
iteration += 1
logs['step'] = iteration
progbar.add(
1,
values=logs.items() if self.config.VERBOSE else
[x for x in logs.items() if not x[0].startswith('l_')])
train_times += 1
print("The whole data hase been trained %d times" % train_times)
print('\nEnd training....\n')
def eval(self, epoch):
self.val_loader = DataLoader(dataset=self.val_dataset,
batch_size=self.config.BATCH_SIZE,
drop_last=True,
shuffle=False,
num_workers=4)
model = self.config.MODEL
total = int(len(self.val_dataset))
self.inpaint_model.eval()
progbar = Progbar(int(total / self.config.BATCH_SIZE),
width=30,
stateful_metrics=['step'])
iteration = 0
logs_ave = {}
with torch.no_grad():
for items in self.val_loader:
iteration += 1
images, images_gray, masks = self.cuda(*items)
# inpaint model
# eval
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * (masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs['val_psnr'] = psnr.item()
logs['val_mae'] = mae.item()
# joint model
if iteration == 1: # first iteration
logs_ave = {}
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
logs["step"] = iteration
progbar.add(1, values=logs.items())
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / iteration, epoch)
self.inpaint_model.iteration = 0
def test(self):
self.inpaint_model.eval()
damaged_dir = os.path.join(self.results_path, "damaged")
create_dir(damaged_dir)
mask_dir = os.path.join(self.results_path, "mask")
create_dir(mask_dir)
inpainted_dir = os.path.join(self.results_path, "inpainted")
create_dir(inpainted_dir)
raw_dir = os.path.join(self.results_path, "raw")
create_dir(raw_dir)
create_dir(self.results_path)
sample_interval = self.config.TEST_INTERVAL
batch_size = 1
test_loader = DataLoader(dataset=self.test_dataset,
batch_size=batch_size,
num_workers=1,
shuffle=False)
total = int(len(self.test_dataset) / batch_size / sample_interval)
progbar = Progbar(total, width=30, stateful_metrics=['step'])
index = 0
proc_start_time = time.time()
total_time = 0.
with torch.no_grad():
for items in test_loader:
name = self.test_dataset.load_name(index)
images, images_gray, masks = self.cuda(*items)
index += 1
if index >= total:
break
# Save raw
# if self.config.SAVEIMG == 1:
# path = os.path.join(damaged_dir, name)
# damaged_img = self.postprocess(images * masks + (1 - masks))[0]
# imsave(damaged_img, path)
# # Save masks
# path = os.path.join(mask_dir, name)
# imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png')
# # Save Ground Truth
# path = os.path.join(raw_dir, name)
# img = self.postprocess(images)[0]
# imsave(img, path)
# # print(index, name)
logs = {}
# run model
outputs = self.inpaint_model(images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
output = self.postprocess(outputs_merged)[0]
if self.config.SAVEIMG == 1:
path = os.path.join(inpainted_dir, name)
# print(index, name)
imsave(output, path)
cnt_time = time.time() - proc_start_time
total_time = total_time + cnt_time
# print('Image {} done, time {}, totaltime{}, {} sec/Image'.format(index, cnt_time,total_time,
# float(total_time) / index))
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
# mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float() = = L1 distance
l1 = torch.nn.L1Loss()(images, outputs_merged)
one_ssim = self.ssim(images, outputs_merged)
logs["psnr"] = psnr.item()
# logs["mae"] = mae.item()
logs["L1"] = l1.item()
logs["ssim"] = one_ssim.item()
logs["step"] = index
progbar.add(1, values=logs.items())
if self.debug:
pass
proc_start_time = time.time()
print('\nEnd test....')
print('Image {} done, time {}, average {} sec/Image'.format(
total, total_time,
float(total_time) / total))
def progressive_test(self):
self.inpaint_model.eval()
damaged_dir = os.path.join(self.results_path, "damaged")
create_dir(damaged_dir)
mask_dir = os.path.join(self.results_path, "mask")
create_dir(mask_dir)
inpainted_dir = os.path.join(self.results_path, "inpainted")
create_dir(inpainted_dir)
raw_dir = os.path.join(self.results_path, "raw")
create_dir(raw_dir)
create_dir(self.results_path)
sample_interval = self.config.TEST_INTERVAL
batch_size = 1
test_loader = DataLoader(dataset=self.test_dataset,
batch_size=batch_size,
num_workers=1,
shuffle=True)
total = int(len(self.test_dataset) / batch_size / sample_interval)
progbar = Progbar(total, width=30, stateful_metrics=['step'])
index = 0
total_time = 0.
proc_start_time = time.time()
with torch.no_grad():
for items in test_loader:
logs = {}
name = self.test_dataset.load_name(index)
images, images_gray, masks = self.cuda(*items)
index += 1
if index > total:
break
# Save damaged image
if self.config.SAVEIMG == 1:
path = os.path.join(damaged_dir, str(index) + '.jpg')
damaged_img = self.postprocess(images * masks +
(1 - masks))[0]
imsave(damaged_img, path)
# Save masks
path = os.path.join(mask_dir, str(index))
imsave(self.postprocess(masks),
os.path.splitext(path)[0] + '.png')
# Save Ground Truth
path = os.path.join(raw_dir, str(index) + '.jpg')
img = self.postprocess(images)[0]
imsave(img, path)
# print(index, name)
# run model
outputs = self.inpaint_model(images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
# # Test again:
# masks=masks.cpu().numpy().astype(np.uint8) # [0-255]
# # masks = (masks > 128).astype(np.uint8) # [0-1]
# struct = ndimage.generate_binary_structure(4, 5)
# np_masks=ndimage.binary_dilation(masks,structure=struct).astype(masks.dtype) # [0-1]
# masks=torch.from_numpy(np_masks)
np_masks = masks.cpu().numpy()
NoHoleNum = np.count_nonzero(np_masks)
ratio = (np_masks.size - NoHoleNum) / np_masks.size
i = 1
while ratio > 0.2 and i <= 10:
# Save masks again
# Erosion
masks = masks.cpu().numpy().astype(np.uint8) # [0-255]
# np_masks = ndimage.grey_dilation(masks, size=(1, 1, 9, 9))
np_masks = ndimage.grey_dilation(masks,
size=(1, 1, 15, 15))
masks = torch.from_numpy(np_masks).float().to(
self.config.DEVICE)
if self.config.SAVEIMG == 1:
path = os.path.join(mask_dir, name)
imsave(self.postprocess(masks),
os.path.splitext(path)[0] + '_%d.png' % i)
outputs = self.inpaint_model(outputs_merged, masks)
outputs_merged = (outputs *
(1 - masks)) + (outputs_merged * masks)
i += 1
# count no hole number and ratio
NoHoleNum = np.count_nonzero(np_masks)
ratio = (np_masks.size - NoHoleNum) / np_masks.size
if self.config.SAVEIMG == 1:
path = os.path.join(inpainted_dir, str(index) + '.jpg')
# print(index, name)
output = self.postprocess(outputs_merged)[0]
imsave(output, path)
cnt_time = time.time() - proc_start_time
total_time = total_time + cnt_time
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
# mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float()
l1 = torch.nn.L1Loss()(images, outputs_merged)
one_ssim = self.ssim(images, outputs_merged)
logs["psnr"] = psnr.item()
# logs["mae"] = mae.item()
logs["L1"] = l1.item()
logs["ssim"] = one_ssim.item()
logs["step"] = index
progbar.add(1, values=logs.items())
if self.debug:
pass
proc_start_time = time.time()
print('\nEnd test....')
print('Image {} done, time {}, average {} sec/Image'.format(
total, total_time,
float(total_time) / total))
def feature_visualize(self, module, input):
xs = input[0]
index = 0
for x in xs:
x = x[0] # remove batch dimension
min_num = np.minimum(16, x.size()[0])
for i in range(min_num):
plt.subplot(4, 4, i + 1)
plt.axis('off')
plt.imshow(x[i].cpu())
name = 'stage_' + str(module.stage) + 'branch_' + str(index)
for i in x.shape:
name = name + '_' + str(i)
plt.savefig(name + '.png', bbox_inches='tight')
plt.show()
index += 1
def visualization_test(self):
self.inpaint_model.eval()
for m in self.inpaint_model.generator.modules():
if isinstance(m, TwoBranchModule):
m.register_forward_pre_hook(self.feature_visualize)
# if isinstance(m, torch.nn.Conv2d):
# m.register_forward_pre_hook(self.feature_visualize)
damaged_dir = os.path.join(self.results_path, "damaged")
create_dir(damaged_dir)
mask_dir = os.path.join(self.results_path, "mask")
create_dir(mask_dir)
inpainted_dir = os.path.join(self.results_path, "inpainted")
create_dir(inpainted_dir)
raw_dir = os.path.join(self.results_path, "raw")
create_dir(raw_dir)
create_dir(self.results_path)
sample_interval = self.config.TEST_INTERVAL
batch_size = 1
test_loader = DataLoader(dataset=self.test_dataset,
batch_size=batch_size,
num_workers=1,
shuffle=False)
total = int(len(self.test_dataset))
progbar = Progbar(int(total / batch_size / sample_interval),
width=30,
stateful_metrics=['step'])
index = 0
with torch.no_grad():
for items in test_loader:
name = self.test_dataset.load_name(index)
images, images_gray, masks = self.cuda(*items)
# Save damaged image
if self.config.SAVEIMG == 1:
path = os.path.join(damaged_dir, name)
damaged_img = self.postprocess(images * masks +
(1 - masks))[0]
imsave(damaged_img, path)
# Save masks
path = os.path.join(mask_dir, name)
imsave(self.postprocess(masks),
os.path.splitext(path)[0] + '.png')
# Save Ground Truth
path = os.path.join(raw_dir, name)
img = self.postprocess(images)[0]
imsave(img, path)
# print(index, name)
index += 1
if index > total / batch_size / sample_interval:
break
logs = {}
# run model
outputs = self.inpaint_model(images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
output = self.postprocess(outputs_merged)[0]
if self.config.SAVEIMG == 1:
path = os.path.join(inpainted_dir, name)
# print(index, name)
imsave(output, path)
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
# mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float()
l1 = torch.nn.L1Loss()(images, outputs_merged)
one_ssim = self.ssim(images, outputs_merged)
logs["psnr"] = psnr.item()
# logs["mae"] = mae.item()
logs["L1"] = l1.item()
logs["ssim"] = one_ssim.item()
logs["step"] = index
progbar.add(1, values=logs.items())
if self.debug:
pass
break
print('\nEnd test....')
def sample(self, it=None):
# do not sample when validation set is empty
if len(self.val_dataset) == 0:
return
self.inpaint_model.eval()
model = self.config.MODEL
with torch.no_grad():
items = next(self.sample_iterator)
images, images_gray, masks = self.cuda(*items)
# (batch,channels,weighth,length)
iteration = self.inpaint_model.iteration
inputs = (images * masks) + (1 - masks)
outputs = self.inpaint_model(images, masks).detach()
if it is not None:
iteration = it
image_per_row = 2
if self.config.SAMPLE_SIZE <= 6:
image_per_row = 1
images = stitch_images(
self.postprocess(images),
# self.postprocess(edges),
self.postprocess(inputs),
self.postprocess(outputs),
# self.postprocess(outputs_merged),
img_per_row=image_per_row)
path = os.path.join(self.samples_path, self.model_name)
name = os.path.join(path, str(iteration).zfill(3) + ".png")
create_dir(path)
print('\nsaving sample ' + name)
images.save(name)
def log(self, logs):
with open(self.log_file, 'a') as f:
f.write('%s\n' % ' '.join([str(item[1]) for item in logs]))
def cuda(self, *args):
return (item.to(self.config.DEVICE) for item in args)
def postprocess(self, img):
# [0, 1] => [0, 255]
img = img * 255.0
img = img.permute(0, 2, 3, 1)
return img.int()
def preprocess(self, img):
# [0, 255] => [0, 1]
img = np.expand_dims(img, axis=0)
img = np.expand_dims(img, axis=0)
# img = img.astype(float) / 255.0
return img
def color_the_edge(self, img, edges, masks):
img = img.expand(-1, 3, -1, -1)
yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255.
]).reshape(1, 3, 1,
1)).to(self.config.DEVICE)
yellow = img * (1 - masks) * yellow_v
img = yellow + (edges * masks)
return img
class CLFNet():
def __init__(self, config):
self.config = config
model_name = 'inpaint'
self.debug = False
self.model_name = model_name
self.inpaint_model = InpaintingModel(config).to(config.DEVICE)
# summary(InpaintingModel, (3, 256, 256), 6)
# print(InpaintingModel)
self.psnr = PSNR(255.0).to(config.DEVICE)
self.ssim = SSIM(window_size=11)
val_sample = int(float((self.config.EVAL_INTERVAL)))
# test mode
if self.config.MODE == 2:
self.test_dataset = Dataset(config,
config.TEST_FLIST,
config.TEST_MASK_FLIST,
augment=False,
training=False)
else:
self.train_dataset = Dataset(config,
config.TRAIN_FLIST,
config.TRAIN_MASK_FLIST,
augment=True,
training=True)
self.val_dataset = Dataset(config,
config.VAL_FLIST,
config.VAL_MASK_FLIST,
augment=False,
training=True,
sample_interval=val_sample)
self.sample_iterator = self.val_dataset.create_iterator(
config.SAMPLE_SIZE)
self.samples_path = os.path.join(config.PATH, 'samples')
self.results_path = os.path.join(config.PATH, 'results')
if config.RESULTS is not None:
self.results_path = os.path.join(config.RESULTS)
if config.DEBUG is not None and config.DEBUG != 0:
self.debug = True
# self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat')
log_path = os.path.join(config.PATH, 'logs_' + model_name)
create_dir(log_path)
self.logger = Logger(log_path)
def load(self):
self.inpaint_model.load()
def save(self, epoch):
self.inpaint_model.save(epoch)
def train(self):
train_loader = DataLoader(dataset=self.train_dataset,
batch_size=self.config.BATCH_SIZE,
num_workers=4,
drop_last=True,
shuffle=True)
keep_training = True
model = self.config.MODEL
# max_iteration = int(float((self.config.MAX_ITERS)))
step_per_epoch = int(float((self.config.MAX_STEPS)))
max_epoches = int(float((self.config.MAX_EPOCHES)))
total = int(len(self.train_dataset))
if total == 0:
print(
'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.'
)
return
print('\nThe number of Training data is %d' % total)
epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1
print('\nTraining epoch: %d' % epoch)
progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step'])
logs_ave = {}
train_times = 0
while (keep_training):
for items in train_loader:
self.inpaint_model.train()
images, images_gray, masks = self.cuda(*items)
# edge model
# inpaint model
# train
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * (masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
logs['psnr'] = psnr.item()
# logs['mae'] = mae.item()
# backward
self.inpaint_model.backward(gen_loss, dis_loss)
if self.inpaint_model.iteration > step_per_epoch:
self.inpaint_model.iteration = 0
iteration = 0
iteration = self.inpaint_model.iteration
if iteration == 1: # first step in this epoch
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
if iteration == 0: # mean to jump to new epoch
self.sample(epoch)
self.eval(epoch)
self.save(epoch)
# log current epoch in tensorboard
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / step_per_epoch,
epoch)
# if reach max epoch
if epoch >= max_epoches:
keep_training = False
break
epoch += 1
# new epoch
print('\n\nTraining epoch: %d' % epoch)
for tag, value in logs.items():
logs_ave[tag] = value
progbar = Progbar(step_per_epoch,
width=30,
stateful_metrics=['step'])
self.inpaint_model.iteration += 1 # jump to new epoch and set the iteration to 1
iteration += 1
logs['step'] = iteration
progbar.add(
1,
values=logs.items() if self.config.VERBOSE else
[x for x in logs.items() if not x[0].startswith('l_')])
train_times += 1
print("The whole data hase been trained %d times" % train_times)
print('\nEnd training....\n')
def eval(self, epoch):
self.val_loader = DataLoader(dataset=self.val_dataset,
batch_size=self.config.BATCH_SIZE,
drop_last=True,
shuffle=False,
num_workers=4)
model = self.config.MODEL
total = int(len(self.val_dataset))
self.inpaint_model.eval()
progbar = Progbar(int(total / self.config.BATCH_SIZE),
width=30,
stateful_metrics=['step'])
iteration = 0
logs_ave = {}
with torch.no_grad():
for items in self.val_loader:
iteration += 1
images, images_gray, masks = self.cuda(*items)
# inpaint model
# eval
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * (masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs['val_psnr'] = psnr.item()
logs['val_mae'] = mae.item()
# joint model
if iteration == 1: # first iteration
logs_ave = {}
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
logs["step"] = iteration
progbar.add(1, values=logs.items())
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / iteration, epoch)
self.inpaint_model.iteration = 0
def test(self):
self.inpaint_model.eval()
damaged_dir = os.path.join(self.results_path, "damaged")
create_dir(damaged_dir)
mask_dir = os.path.join(self.results_path, "mask")
create_dir(mask_dir)
inpainted_dir = os.path.join(self.results_path, "inpainted")
create_dir(inpainted_dir)
raw_dir = os.path.join(self.results_path, "raw")
create_dir(raw_dir)
create_dir(self.results_path)
sample_interval = 1
batch_size = 1
test_loader = DataLoader(dataset=self.test_dataset,
batch_size=batch_size,
num_workers=1,
shuffle=False)
total = int(len(self.test_dataset))
progbar = Progbar(int(total / batch_size / sample_interval),
width=30,
stateful_metrics=['step'])
index = 0
with torch.no_grad():
for items in test_loader:
name = self.test_dataset.load_name(index)
images, images_gray, masks = self.cuda(*items)
# Save damaged image
if self.config.MODE == 2:
path = os.path.join(damaged_dir, name)
damaged_img = self.postprocess(images * masks +
(1 - masks))[0]
imsave(damaged_img, path)
# Save masks
path = os.path.join(mask_dir, name)
imsave(self.postprocess(masks),
os.path.splitext(path)[0] + '.png')
# Save Ground Truth
path = os.path.join(raw_dir, name)
img = self.postprocess(images)[0]
imsave(img, path)
# print(index, name)
index += 1
if index > total / batch_size / sample_interval:
break
logs = {}
# run model
outputs = self.inpaint_model(images, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
output = self.postprocess(outputs_merged)[0]
if self.config.MODE == 2:
path = os.path.join(inpainted_dir, name)
# print(index, name)
imsave(output, path)
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
# mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float()
# mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float()
l1 = torch.nn.L1Loss()(images, outputs_merged)
one_ssim = self.ssim(images, outputs_merged)
logs["psnr"] = psnr.item()
# logs["mae"] = mae.item()
logs["L1"] = l1.item()
logs["ssim"] = one_ssim.item()
logs["step"] = index
progbar.add(1, values=logs.items())
if self.debug:
pass
print('\nEnd test....')
def sample(self, it=None):
# do not sample when validation set is empty
if len(self.val_dataset) == 0:
return
self.inpaint_model.eval()
model = self.config.MODEL
with torch.no_grad():
items = next(self.sample_iterator)
images, images_gray, masks = self.cuda(*items)
# (batch,channels,weighth,length)
iteration = self.inpaint_model.iteration
inputs = (images * masks) + (1 - masks)
outputs = self.inpaint_model(images, masks).detach()
if it is not None:
iteration = it
image_per_row = 2
if self.config.SAMPLE_SIZE <= 6:
image_per_row = 1
images = stitch_images(
self.postprocess(images),
# self.postprocess(edges),
self.postprocess(inputs),
self.postprocess(outputs),
# self.postprocess(outputs_merged),
img_per_row=image_per_row)
path = os.path.join(self.samples_path, self.model_name)
name = os.path.join(path, str(iteration).zfill(3) + ".png")
create_dir(path)
print('\nsaving sample ' + name)
images.save(name)
def log(self, logs):
with open(self.log_file, 'a') as f:
f.write('%s\n' % ' '.join([str(item[1]) for item in logs]))
def cuda(self, *args):
return (item.to(self.config.DEVICE) for item in args)
def postprocess(self, img):
# [0, 1] => [0, 255]
img = img * 255.0
img = img.permute(0, 2, 3, 1)
return img.int()
def color_the_edge(self, img, edges, masks):
img = img.expand(-1, 3, -1, -1)
yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255.
]).reshape(1, 3, 1,
1)).to(self.config.DEVICE)
yellow = img * (1 - masks) * yellow_v
img = yellow + (edges * masks)
return img
def walk_through(self):
if not self.to_top():
Logger.log("GoogleMusicApp", "walk_through failed: unable to go to top activity")
self.cache_init = False
return False
# Get the playcard titles
vc = ViewClient(self.device, self.serialno)
self.cache_init = True
container_key = GoogleMusicApp.CONTAINER_KEY
container = [v for v in vc.getViewsById().values() if v.getId() == container_key]
container = container[0] if len(container) > 0 else None
if container:
self.cache["screen-info"] = container.getBounds()[1]
self.push_dump("screen-info: {}".format(self.cache["screen-info"]))
so = sio.StringIO()
vc.traverse(stream=so)
lines = so.getvalue().splitlines()
play_card_key = GoogleMusicApp.PLAY_CARD_KEY
playcards_idices = [idx for idx, line in enumerate(lines) if play_card_key in line]
playcards_idices.append(len(lines))
playcards_titles = []
last_idx = playcards_idices[0]
li_title_key = GoogleMusicApp.LI_TITLE_KEY
for idx in playcards_idices[1:]:
li_title_texts = [line for line in lines[last_idx:idx] if li_title_key in line]
last_idx = idx
if len(li_title_texts) != 1:
self.push_dump("li_title_texts has length {}".format(len(li_title_texts)))
playcards_titles.append(utf8str(li_title_texts[0].split(li_title_key)[-1].strip()))
self.push_dump("playcards_titles.append('{}')".format(playcards_titles[-1]))
# Get the track list of each playcard
views = [v for v in vc.getViewsById().values() if v.getId() == li_title_key and utf8str(v.getText()) in playcards_titles]
self.cache["playcard"] = dict( \
map(lambda v: (utf8str(v.getText()), { "position": v.getCenter() }), views)
)
map(lambda v: self.push_dump("view: {}".format(utf8str(v))), views)
map(lambda title: self.push_dump("playcard title: '{}'".format(title)), self.cache["playcard"].keys())
if len(views) == 0:
self.cache_init = False
return False
self.cache["shuffle_key"] = playcards_titles[0]
self.push_dump("get the shuffle keyword '{}'".format(self.cache["shuffle_key"]))
self.touch_playcard(self.cache["shuffle_key"])
time.sleep(1)
retry_count = 3
while retry_count > 0:
vc.dump()
play_pause_header_key = GoogleMusicApp.PLAY_PAUSE_HEADER_KEY
play_pause_btn_view = [v for v in vc.getViewsById().values() if v.getId() == play_pause_header_key]
play_pause_btn_view = play_pause_btn_view[0] if len(play_pause_btn_view) > 0 else None
if play_pause_btn_view:
play_desc = utf8str(play_pause_btn_view.getContentDescription())
self.check_play_status = lambda desc: desc == play_desc
self.cache["play_pause_btn"] = { "position": play_pause_btn_view.getCenter(), "desc_feat": play_desc }
art_pager_key = GoogleMusicApp.ART_PAGER_KEY
art_pager_view = [v for v in vc.getViewsById().values() if v.getId() == art_pager_key]
art_pager_view = art_pager_view[0] if len(art_pager_view) > 0 else None
if not art_pager_view:
retry_count -= 1
continue
self.cache["art_pager_view"] = { "position": art_pager_view.getCenter() }
play_pause_btn_view.touch()
break
else:
self.push_dump("cannot find the play/pause button, retry: {}".format(retry_count))
retry_count -= 1
if retry_count == 0:
return False
for li_title in self.cache["playcard"].keys():
if li_title == self.cache["shuffle_key"]:
continue
self.push_dump("now fetching information in the playcard '{}'".format(li_title))
if self.touch_playcard(li_title=li_title):
time.sleep(1)
self.cache["playcard"][li_title]["songs"] = self._fetch_songs()
self.to_top()
# Get the information of the control panel
retry_count = 3
while self.get_state() != GoogleMusicApp.State.CONTROL_PANEL and retry_count > 0:
self.device.touch(*self.cache["art_pager_view"]["position"])
retry_count -= 1
if retry_count == 0 and self.get_state() != GoogleMusicApp.State.CONTROL_PANEL:
self.to_top()
time.sleep(5)
self.touch_playcard(self.cache["shuffle_key"])
time.sleep(2)
self.device.touch(*self.cache["play_pause_btn"]["position"])
time.sleep(2)
self.device.touch(*self.cache["art_pager_view"]["position"])
time.sleep(2)
if self.get_state() != GoogleMusicApp.State.CONTROL_PANEL:
self.push_dump("cannot get the information of the control panel")
self.cache_init = False
return False
def find_view_position(vc, res_id):
v = [v for v in vc.getViewsById().values() if v.getId() == res_id]
if len(v) == 0:
return ((-1, -1), (-1, -1)), (-1, -1)
return v[0].getBounds(), v[0].getCenter()
vc.dump()
progress_bounds, progress_pos = find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PROGRESS_KEY)
self.cache["control_panel"] = {
"progress": { "position": progress_pos, "xbounds": [progress_bounds[0][0], progress_bounds[1][0]] },
"prev": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PREV_KEY)[1] },
"next": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_NEXT_KEY)[1] },
"play_pause": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PLAY_PAUSE_KEY)[1] }
}
self.control_panel = GMControlPanel(self)
self.push_dump("successfully walked through, now back to top")
self.to_top()
self.cache_init = True
return True
#!/usr/bin/env python3
import error_handling_library.test_hooks as ut
from libs.logger import Logger
LOGGER = Logger()
ut.init( logger = LOGGER )
import time, os, argparse, sys, requests, gevent, grequests
from gevent import socket, Timeout
from libs.dataGiver import DataGiver
from libs.outputDataIntoDict import Outputter
from libs.resolver import Resolver
parser = argparse.ArgumentParser(description = """Ping some ips and hosts ;)""")
parser.add_argument("-c", "--configure", help="Give the ip json file", type = str, default = False)
parser.add_argument("-s", "--stats", help="Give some statistics on the screen!", action = "store_true", default = False)
parser.add_argument("-d", "--directory", help="Put the output in written directory. WIll be created if not there.", type = str, default = False)
args = parser.parse_args()
STATS = args.stats
versionNumber = "3.0"
applicationName = "Momo's Pinger"
defaultSearchFolder = "/etc/tbmon/defaultHTMLJSON"
defaultOutputFolder = "output_data"
searchFolder = args.configure or defaultSearchFolder
def run(self):
shared_vars = {"start_time": None, "last_event": None, "last_freq": -1}
self.extra = {}
self.extra["adb-read-prop-max-elapsed"] = -1
self.extra["freq-cb-max-elapsed"] = -1
self.extra["dump"] = []
self.extra["dump-lock"] = threading.Lock()
def freq_cb(msg):
line = msg.splitlines()[0]
strs = line.split()
freq, amp_db = list(map(float, strs[-1].split(",")))
the_date, the_time = strs[:2]
time_str = the_date + " " + the_time
if shared_vars["last_freq"] != freq:
self.push_to_dump( \
"the detected freq has been changed from {} to {} Hz".format(shared_vars["last_freq"], freq))
shared_vars["last_freq"] = freq
thresh = 10 if self.target_freq else 1
if super(AATAppToneDetectorThread, self).target_detected(freq):
self.event_counter += 1
if self.event_counter == 1:
shared_vars["start_time"] = time_str
if self.event_counter == thresh:
if not shared_vars["last_event"] or shared_vars[
"last_event"] != ToneDetector.Event.TONE_DETECTED:
Logger.log(
self.get_tag(),
"send_cb({}, TONE_DETECTED)".format(
shared_vars["start_time"]))
self.cb((shared_vars["start_time"],
ToneDetector.Event.TONE_DETECTED))
shared_vars[
"last_event"] = ToneDetector.Event.TONE_DETECTED
else:
if self.event_counter > thresh:
shared_vars["start_time"] = None
self.push_to_dump(
"the tone is not detected and the event_counter is over the threshold"
)
self.push_to_dump("last_event: \"{}\"".format(
shared_vars["last_event"]))
if not shared_vars["last_event"] or shared_vars[
"last_event"] != ToneDetector.Event.TONE_MISSING:
Logger.log(self.get_tag(),
"send_cb({}, TONE_MISSING)".format(time_str))
self.cb((time_str, ToneDetector.Event.TONE_MISSING))
shared_vars["last_event"] = ToneDetector.Event.TONE_MISSING
self.event_counter = 0
if self.event_counter <= thresh:
self.push_to_dump("event_counter: {}".format(
self.event_counter))
# Adb.execute(cmd= \
# ["shell", "am", "broadcast", "-a", "audio.htc.com.intent.print.properties.enable", "--ez", "v", "1"], \
# serialno=self.serialno)
from libs.timeutils import TicToc, TimeUtils
freq_cb_tictoc = TicToc()
adb_tictoc = TicToc()
tcount = 0
freq_cb_tictoc.tic()
while not self.stoprequest.isSet():
adb_tictoc.tic()
msg, _ = Adb.execute(cmd=["shell", "cat", "sdcard/AudioFunctionsDemo-record-prop.txt"], \
serialno=self.serialno, tolog=False)
elapsed = adb_tictoc.toc()
if tcount == 0:
Adb.execute(cmd=["shell", "rm", "-f", "sdcard/AudioFunctionsDemo-record-prop.txt"], \
serialno=self.serialno, tolog=False)
if not "," in msg:
msg = "0,-30"
if elapsed > self.extra["adb-read-prop-max-elapsed"]:
self.extra["adb-read-prop-max-elapsed"] = elapsed
if "," in msg:
msg = msg.replace("\n", "")
import datetime
msg = "{} {}".format(TimeUtils.now_str(), msg)
try:
self.push_to_dump("{} (adb-shell elapsed: {} ms)".format(
msg, elapsed))
freq_cb(msg)
except Exception as e:
Logger.log(self.get_tag(),
"crashed in freq_cb('{}')".format(msg))
print(e)
elapsed = freq_cb_tictoc.toc()
if elapsed > self.extra["freq-cb-max-elapsed"]:
self.extra["freq-cb-max-elapsed"] = elapsed
time.sleep(0.01)
tcount += 1
tcount %= 10
img_title = purifyName(tit[0])
win32api.SetConsoleCtrlHandler(onClose, True)
downnum, failnum = download(title=img_title,
links=links[index][0],
names=names[index][0],
downnum=DOWN_NUM,
failnum=FAIL_NUM,
delta=delta,
done=done)
index += 1
DOWN_NUM += downnum
FAIL_NUM += failnum
except (BaseException, Exception) as e:
os.chdir(MAIN_PATH)
lastTitle = MAIN_PATH + '\\Download\\' + purifyName(END_NAME)
rmFile(lastTitle)
with open('last.txt', 'w') as f:
f.write(END_NAME)
err_log = Logger(log_name='Error.log', logger_name='err').get_logger()
err_log.error(e)
finally:
if os.getcwd() != MAIN_PATH:
os.chdir(MAIN_PATH)
output_log = Logger(log_name='Output.log',
logger_name='output').get_logger()
output_log.info('下载完成%d张' % DOWN_NUM)
output_log.info('下载失败%d张' % FAIL_NUM)
print('>>>The log file in %s.<<<' % MAIN_PATH)
print('Done!')
input('\n\nPress any key to quit.')
def main():
Logger.init(Logger.Mode.STDOUT)
Adb.init()
trial_num = 1
pass_trial_num = 0
try:
while True:
Adb.execute(["shell", "input", "keyevent", "POWER"])
Adb.execute([
"shell", AATApp.INTENT_PREFIX,
AATApp.HTC_INTENT_PREFIX + "record.start"
])
time.sleep(2)
log("play 220Hz_44100Hz_15secs_2ch.wav")
os.system(
"adb shell tinyplay /data/220Hz_44100Hz_15secs_2ch.wav > /dev/null &"
)
retry = 5
detected = False
while retry > 0:
out, err = Adb.execute([
"shell", "cat",
"/storage/emulated/0/AudioFunctionsDemo-record-prop.txt"
],
tolog=False)
try:
out = float(out.split(",")[0])
except:
out = 0.0
if out > 200.0 and out < 240.0:
log("detected tones.")
detected = True
break
time.sleep(0.1)
time.sleep(1)
log("turn off the screen.")
Adb.execute(["shell", "input", "keyevent", "POWER"])
time.sleep(2)
retry = 5
detected = False
detected_cnt = 0
while retry > 0:
out, err = Adb.execute([
"shell", "cat",
"/storage/emulated/0/AudioFunctionsDemo-record-prop.txt"
],
tolog=False)
try:
out = float(out.split(",")[0])
except:
out = 0.0
if out > 200.0 and out < 240.0:
log("detected tones.")
detected_cnt += 1
if detected_cnt == 10:
log("detected for a sufficient times.")
detected = True
break
time.sleep(0.1)
if detected:
pass_trial_num += 1
log("trial #{}: passed. ({}/{})".format(
trial_num, pass_trial_num, trial_num))
else:
log("trial #{}: failed. ({}/{})".format(
trial_num, pass_trial_num, trial_num))
trial_num += 1
time.sleep(15)
except:
pass
Logger.finalize()
class EdgeConnect():
def __init__(self, config):
self.config = config
if config.MODEL == 1:
model_name = 'edge'
elif config.MODEL == 2:
model_name = 'inpaint'
elif config.MODEL == 3:
model_name = 'edge_inpaint'
elif config.MODEL == 4:
model_name = 'joint'
self.debug = False
self.model_name = model_name
self.edge_model = EdgeModel(config).to(config.DEVICE)
self.inpaint_model = InpaintingModel(config).to(config.DEVICE)
self.psnr = PSNR(255.0).to(config.DEVICE)
self.ssim = SSIM(window_size=11)
self.edgeacc = EdgeAccuracy(config.EDGE_THRESHOLD).to(config.DEVICE)
val_sample = int(float((self.config.EVAL_INTERVAL)))
# test mode
if self.config.MODE == 2:
self.test_dataset = Dataset(config,
config.TEST_FLIST,
config.TEST_EDGE_FLIST,
config.TEST_MASK_FLIST,
augment=False,
training=False)
else:
self.train_dataset = Dataset(config,
config.TRAIN_FLIST,
config.TRAIN_EDGE_FLIST,
config.TRAIN_MASK_FLIST,
augment=True,
training=True)
self.val_dataset = Dataset(config,
config.VAL_FLIST,
config.VAL_EDGE_FLIST,
config.VAL_MASK_FLIST,
augment=False,
training=True,
sample_interval=val_sample)
self.sample_iterator = self.val_dataset.create_iterator(
config.SAMPLE_SIZE)
self.samples_path = os.path.join(config.PATH, 'samples')
self.results_path = os.path.join(config.PATH, 'results')
if config.RESULTS is not None:
self.results_path = os.path.join(config.RESULTS)
if config.DEBUG is not None and config.DEBUG != 0:
self.debug = True
# self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat')
log_path = os.path.join(config.PATH, 'logs_' + model_name)
create_dir(log_path)
self.logger = Logger(log_path)
def load(self):
if self.config.MODEL == 1:
self.edge_model.load()
elif self.config.MODEL == 2:
self.inpaint_model.load()
else:
self.edge_model.load()
self.inpaint_model.load()
def save(self, epoch):
if self.config.MODEL == 1:
self.edge_model.save(epoch)
elif self.config.MODEL == 2 or self.config.MODEL == 3:
self.inpaint_model.save(epoch)
else:
self.edge_model.save(epoch)
self.inpaint_model.save(epoch)
def train(self):
train_loader = DataLoader(dataset=self.train_dataset,
batch_size=self.config.BATCH_SIZE,
num_workers=4,
drop_last=True,
shuffle=True)
keep_training = True
model = self.config.MODEL
# max_iteration = int(float((self.config.MAX_ITERS)))
step_per_epoch = int(float((self.config.MAX_STEPS)))
max_epoches = int(float((self.config.MAX_EPOCHES)))
total = int(len(self.train_dataset))
if total == 0:
print(
'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.'
)
return
print('\nThe number of Training data is %d' % total)
if model == 1:
epoch = self.edge_model.epoch + 1 if self.edge_model.epoch != None else 1
else:
epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1
print('\nTraining epoch: %d' % epoch)
progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step'])
while (keep_training):
logs_ave = {}
for items in train_loader:
self.edge_model.train() # set the model to train mode
self.inpaint_model.train()
images, images_gray, edges, masks = self.cuda(*items)
# edge model
if model == 1:
# train
outputs, gen_loss, dis_loss, logs = self.edge_model.process(
images_gray, edges, masks)
# metrics
precision, recall = self.edgeacc(edges * (1 - masks),
outputs * (1 - masks))
logs['precision'] = precision.item()
logs['recall'] = recall.item()
# backward
self.edge_model.backward(gen_loss, dis_loss)
if self.edge_model.iteration > step_per_epoch:
self.edge_model.iteration = 0
iteration = self.edge_model.iteration
# inpaint model
elif model == 2:
# train
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, edges, masks)
outputs_merged = (outputs * (1 - masks)) + (images *
(masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs['psnr'] = psnr.item()
logs['mae'] = mae.item()
# backward
self.inpaint_model.backward(gen_loss, dis_loss)
if self.inpaint_model.iteration > step_per_epoch:
self.inpaint_model.iteration = 0
iteration = self.inpaint_model.iteration
# inpaint with edge model
elif model == 3:
# train
if True or np.random.binomial(1, 0.5) > 0:
edge_outputs = self.edge_model(images_gray, edges,
masks).detach()
edge_outputs = (edge_outputs * (1 - masks)) + (edges *
(masks))
else:
edge_outputs = edges
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, edge_outputs.detach(), masks)
outputs_merged = (outputs * (1 - masks)) + (images *
(masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs['psnr'] = psnr.item()
logs['mae'] = mae.item()
# backward
self.inpaint_model.backward(gen_loss, dis_loss)
if self.inpaint_model.iteration > step_per_epoch:
self.inpaint_model.iteration = 0
iteration = self.inpaint_model.iteration
# joint model
else:
# train
e_outputs, e_gen_loss, e_dis_loss, e_logs = self.edge_model.process(
images_gray, edges, masks)
e_outputs = e_outputs * (1 - masks) + edges * (masks)
i_outputs, i_gen_loss, i_dis_loss, i_logs = self.inpaint_model.process(
images, e_outputs, masks)
outputs_merged = (i_outputs * (1 - masks)) + (images *
(masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
precision, recall = self.edgeacc(edges * (1 - masks),
e_outputs * (1 - masks))
i_logs['psnr'] = psnr.item()
i_logs['mae'] = mae.item()
i_logs['psnr'] = psnr.item()
i_logs['mae'] = mae.item()
logs = {**e_logs, **i_logs}
# backward
self.inpaint_model.backward(i_gen_loss, i_dis_loss)
self.edge_model.backward(e_gen_loss, e_dis_loss)
if self.inpaint_model.iteration > step_per_epoch:
self.inpaint_model.iteration = 0
iteration = self.inpaint_model.iteration
if iteration == 1: # first time to train
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
if iteration == 0: # mean to jump to new epoch
self.sample(epoch)
self.eval(epoch)
self.save(epoch)
# log current epoch in tensorboard
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / step_per_epoch,
epoch)
# max epoch
if epoch >= max_epoches:
keep_training = False
break
epoch += 1
# new epoch
print('\n\nTraining epoch: %d' % epoch)
for tag, value in logs.items():
logs_ave[tag] = value
progbar = Progbar(step_per_epoch,
width=30,
stateful_metrics=['step'])
self.inpaint_model.iteration += 1
self.edge_model.iteration += 1
iteration += 1
logs['step'] = iteration
progbar.add(
1,
values=logs.items() if self.config.VERBOSE else
[x for x in logs.items() if not x[0].startswith('l_')])
print('\nEnd training....\n')
def eval(self, epoch):
self.val_loader = DataLoader(dataset=self.val_dataset,
batch_size=self.config.BATCH_SIZE,
drop_last=True,
shuffle=False,
num_workers=4)
model = self.config.MODEL
total = int(len(self.val_dataset))
self.edge_model.eval()
self.inpaint_model.eval()
progbar = Progbar(int(total / self.config.BATCH_SIZE),
width=30,
stateful_metrics=['step'])
iteration = 0
with torch.no_grad():
for items in self.val_loader:
iteration += 1
images, images_gray, edges, masks = self.cuda(*items)
# edge model
if model == 1:
# eval
outputs, gen_loss, dis_loss, _ = self.edge_model.process(
images_gray, edges, masks)
logs = {}
logs['l_val_d1'] = dis_loss.item()
logs['l_val_g1'] = gen_loss.item()
# metrics
precision, recall = self.edgeacc(edges * (1 - masks),
outputs * (1 - masks))
logs['val_precision'] = precision.item()
logs['val_recall'] = recall.item()
# inpaint model
elif model == 2:
# eval
outputs, gen_loss, dis_loss, logs = self.inpaint_model.process(
images, edges, masks)
outputs_merged = (outputs * (1 - masks)) + (images *
(masks))
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs['val_psnr'] = psnr.item()
logs['val_mae'] = mae.item()
# inpaint with edge model
elif model == 3:
# eval
edge_outputs = self.edge_model(images_gray, edges, masks)
edge_outputs = edge_outputs * (1 - masks) + edges * (masks)
outputs, gen_loss, dis_loss, log_origin = self.inpaint_model.process(
images, edge_outputs.detach(), masks)
outputs_merged = (outputs * (1 - masks)) + (images *
(masks))
# metrics
# "l_d2": dis_loss.item(),
# "l_g2": gen_gan_loss.item(),
# "l_l1": gen_l1_loss.item(),
# "l_per": gen_content_loss.item(),
# "l_sty": gen_style_loss.item()
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
logs = {}
# logs['l_val_l1'] = log_origin['l_l1']
logs['l_val_g2'] = log_origin['l_g2']
logs['l_val_d2'] = log_origin['l_d2']
# logs['l_val_per'] = log_origin['l_per']
# logs['l_val_sty'] = log_origin['l_sty']
logs['val_psnr'] = psnr.item()
logs['val_mae'] = mae.item()
# joint model
else:
# eval
e_outputs, e_gen_loss, e_dis_loss, e_logs = self.edge_model.process(
images_gray, edges, masks)
e_outputs = e_outputs * (1 - masks) + edges * masks
i_outputs, i_gen_loss, i_dis_loss, i_logs = self.inpaint_model.process(
images, e_outputs, masks)
outputs_merged = (i_outputs *
(1 - masks)) + (images * masks)
# metrics
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
precision, recall = self.edgeacc(edges * (1 - masks),
e_outputs * (1 - masks))
i_logs['val_precision'] = precision.item()
i_logs['val_recall'] = recall.item()
i_logs['val_psnr'] = psnr.item()
i_logs['val_mae'] = mae.item()
logs = {**e_logs, **i_logs}
if iteration == 1: # first iteration
logs_ave = {}
for tag, value in logs.items():
logs_ave[tag] = value
else:
for tag, value in logs.items():
logs_ave[tag] += value
logs["step"] = iteration
progbar.add(1, values=logs.items())
for tag, value in logs_ave.items():
self.logger.scalar_summary(tag, value / iteration, epoch)
self.edge_model.iteration = 0
self.inpaint_model.iteration = 0
def test(self):
self.edge_model.eval()
self.inpaint_model.eval()
damaged_dir = os.path.join(self.results_path, "damaged")
create_dir(damaged_dir)
mask_dir = os.path.join(self.results_path, "mask")
create_dir(mask_dir)
inpainted_dir = os.path.join(self.results_path, "inpainted")
create_dir(inpainted_dir)
damaged_edge_dir = os.path.join(self.results_path, "damaged_edge")
create_dir(damaged_edge_dir)
raw_dir = os.path.join(self.results_path, "raw")
create_dir(raw_dir)
model = self.config.MODEL
create_dir(self.results_path)
sample_interval = 1000
batch_size = 1
test_loader = DataLoader(dataset=self.test_dataset,
batch_size=batch_size,
num_workers=12,
shuffle=False)
total = int(len(self.test_dataset))
progbar = Progbar(int(total / batch_size / sample_interval),
width=30,
stateful_metrics=['step'])
index = 0
with torch.no_grad():
for items in test_loader:
name = self.test_dataset.load_name(index)
images, images_gray, edges, masks = self.cuda(*items)
path = os.path.join(damaged_dir, name)
damaged_img = self.postprocess(images * masks + (1 - masks))[0]
imsave(damaged_img, path)
path = os.path.join(damaged_edge_dir, name)
damaged_img = self.postprocess(edges * masks + (1 - masks))[0]
imsave(damaged_img, path)
path = os.path.join(mask_dir, name)
imsave(self.postprocess(masks),
os.path.splitext(path)[0] + '.png')
path = os.path.join(raw_dir, name)
img = self.postprocess(images)[0]
imsave(img, path)
# print(index, name)
index += 1
if index > total / batch_size / sample_interval:
break
logs = {}
# edge model
if model == 1:
outputs = self.edge_model(images_gray, edges, masks)
outputs_merged = (outputs * (1 - masks)) + (edges * masks)
# inpaint model
elif model == 2:
outputs = self.inpaint_model(images, edges, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
# edge-inpaint model
elif model == 3:
outputs = self.edge_model(images_gray, edges,
masks).detach()
outputs_merged = (outputs * (1 - masks)) + (edges * masks)
edge_save = self.color_the_edge(outputs, edges, masks)
edge_save = self.postprocess(edge_save)[0]
path = os.path.join(self.results_path + "/edge_inpainted",
name)
# print(index, name)
imsave(edge_save, path)
outputs = self.inpaint_model(images, outputs_merged, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
# joint model
else:
edges = self.edge_model(images_gray, edges, masks).detach()
outputs = self.inpaint_model(images, edges, masks)
outputs_merged = (outputs * (1 - masks)) + (images * masks)
output = self.postprocess(outputs_merged)[0]
path = os.path.join(inpainted_dir, name)
# print(index, name)
imsave(output, path)
psnr = self.psnr(self.postprocess(images),
self.postprocess(outputs_merged))
mae = (torch.sum(torch.abs(images - outputs_merged)) /
torch.sum(images)).float()
one_ssim = self.ssim(images, outputs_merged)
logs["psnr"] = psnr.item()
logs["mae"] = mae.item()
logs["ssim"] = one_ssim.item()
logs["step"] = index
progbar.add(1, values=logs.items())
if self.debug:
edges = self.postprocess(1 - edges)[0]
masked = self.postprocess(images * (masks) +
(1 - masks))[0]
fname, fext = name.split('.')
imsave(
edges,
os.path.join(self.results_path,
fname + '_edge.' + fext))
imsave(
masked,
os.path.join(self.results_path,
fname + '_masked.' + fext))
print('\nEnd test....')
def sample(self, it=None):
# do not sample when validation set is empty
if len(self.val_dataset) == 0:
return
self.edge_model.eval()
self.inpaint_model.eval()
model = self.config.MODEL
with torch.no_grad():
items = next(self.sample_iterator)
images, images_gray, edges, masks = self.cuda(*items)
# (batch,channels,weighth,length)
# edge model
if model == 1:
iteration = self.edge_model.iteration
inputs = (edges * masks) + (1 - masks)
outputs = self.edge_model(images_gray, edges, masks).detach()
outputs = self.color_the_edge(outputs, edges, masks)
# outputs_merged = (outputs * (1 - masks)) + (edges * masks)
# inpaint model
elif model == 2:
iteration = self.inpaint_model.iteration
inputs = (images * masks) + (1 - masks)
outputs = self.inpaint_model(images, edges, masks).detach()
# outputs_merged = (outputs * (1 - masks)) + (images * (masks))
# inpaint with edge model / joint model
else:
iteration = self.inpaint_model.iteration
inputs = (images * (masks)) + (1 - masks)
outputs = self.edge_model(images_gray, edges, masks).detach()
edge_merged = (outputs * (1 - masks) + edges *
(masks)).detach()
edges = self.color_the_edge(outputs, edges, masks).detach()
outputs = self.inpaint_model(images, edge_merged, masks)
# outputs_merged = (outputs * (1 - masks)) + (images * (masks))
if it is not None:
iteration = it
image_per_row = 2
if self.config.SAMPLE_SIZE <= 6:
image_per_row = 1
images = stitch_images(
self.postprocess(images),
self.postprocess(edges),
self.postprocess(inputs),
self.postprocess(outputs),
# self.postprocess(outputs_merged),
img_per_row=image_per_row)
path = os.path.join(self.samples_path, self.model_name)
name = os.path.join(path, str(iteration).zfill(3) + ".png")
create_dir(path)
print('\nsaving sample ' + name)
images.save(name)
def log(self, logs):
with open(self.log_file, 'a') as f:
f.write('%s\n' % ' '.join([str(item[1]) for item in logs]))
def cuda(self, *args):
return (item.to(self.config.DEVICE) for item in args)
def postprocess(self, img):
# [0, 1] => [0, 255]
img = img * 255.0
img = img.permute(0, 2, 3, 1)
return img.int()
def color_the_edge(self, img, edges, masks):
img = img.expand(-1, 3, -1, -1)
yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255.
]).reshape(1, 3, 1,
1)).to(self.config.DEVICE)
yellow = img * (1 - masks) * yellow_v
img = yellow + (edges * masks)
return img
def run(num_iter, serialno1, phoneno1, serialno2, phoneno2):
Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT)
# Logger.init(Logger.Mode.STDOUT)
Adb.init()
for serialno in [serialno1, serialno2]:
Adb.execute(["root"], serialno=serialno)
Adb.execute(["shell", "'echo \'related\' > msm_subsys'"], serialno=serialno)
Adb.execute(["shell", "svc", "power", "stayon", "true"], serialno=serialno)
out, err = Adb.execute(["shell", "getprop", "ro.vendor.build.fingerprint"], serialno=serialno)
out = out.strip()
log("build number: '{}'".format(out))
latest_dmesg1 = fetch_dmesg(serialno=serialno1)
latest_dmesg2 = fetch_dmesg(serialno=serialno2)
latest_dmesg1 = latest_dmesg1[0] if len(latest_dmesg1) > 0 else None
latest_dmesg2 = latest_dmesg2[0] if len(latest_dmesg2) > 0 else None
phone_dict = {
serialno1: phoneno1,
serialno2: phoneno2
}
last_dmesg_dict = {
serialno1: latest_dmesg1,
serialno2: latest_dmesg2
}
mt_serialno = serialno1
mo_serialno = serialno2
pass_trial_cnt = 0
total_trial_cnt = 0
invalid_trial_cnt = 0
failed_trial_cnt = 0
try:
for i in range(int(num_iter)):
log("-------------------- Dual-CS-call-test #{} --------------------".format(i+1))
log("{} makes call to {} ({})".format(mo_serialno, mt_serialno, phone_dict[mt_serialno]))
start_cs_call(tel=phone_dict[mt_serialno], serialno=mo_serialno)
if not wait_for_phone_state(state=1, timeout=30, serialno=mt_serialno):
log("There is no incoming call to {}, next trial".format(mt_serialno))
end_cs_call(serialno=mo_serialno)
end_cs_call(serialno=mt_serialno)
invalid_trial_cnt += 1
continue
log("{} picks up the call".format(mt_serialno))
pick_cs_call(serialno=mt_serialno)
if not wait_for_phone_state(state=2, timeout=10, serialno=mo_serialno):
log("{} is not in phone state 'MODE_INCALL', next trial".format(mo_serialno))
end_cs_call(serialno=mo_serialno)
end_cs_call(serialno=mt_serialno)
invalid_trial_cnt += 1
continue
if not wait_for_phone_state(state=2, timeout=10, serialno=mt_serialno):
log("{} is not in phone state 'MODE_INCALL', next trial".format(mt_serialno))
end_cs_call(serialno=mo_serialno)
end_cs_call(serialno=mt_serialno)
invalid_trial_cnt += 1
continue
log("To check if ADSP crashes during the CS call")
is_passed = True
pass_trial_cnt += 1
total_trial_cnt += 1
retry = 15
while retry > 0:
for serialno in [mt_serialno, mo_serialno]:
latest_dmesg = last_dmesg_dict[serialno]
new_dmesgs = fetch_dmesg(latest_dmesg, serialno=serialno)
if len(new_dmesgs) > 0:
last_dmesg_dict[serialno] = new_dmesgs[-1]
adsp_ssr_demsgs = filter(lambda x: "Restart sequence requested for adsp" in x.raw, new_dmesgs)
if len(adsp_ssr_demsgs) > 0:
for dmesg in adsp_ssr_demsgs:
log("SSR log detected in {}: '{}'".format(serialno, dmesg.raw))
is_passed = False
break
phone_state = get_phone_state(serialno=serialno)
if phone_state == None:
log("the phone state of {} is unobtainable, something wrong".format(serialno))
is_passed = False
break
if phone_state == 0:
log("the phone state of {} is in idle, the call might be dropped".format(serialno))
is_passed = False
break
if not is_passed:
for serialno in [mt_serialno, mo_serialno]:
out, err = Adb.execute(["bugreport"], serialno)
log("bugreport to '{}'".format(out.strip()))
pass_trial_cnt -= 1
failed_trial_cnt += 1
for serialno in [mt_serialno, mo_serialno]:
end_cs_call(serialno=serialno)
break
if retry % 5 == 0:
log("{} switches path to '{}'".format(mo_serialno, "speaker" if retry/5 % 2 == 1 else "receiver"))
Adb.execute(["shell", "input", "tap", "1000", "1000"], tolog=False, serialno=mo_serialno)
retry -= 1
time.sleep(1)
log("result: {} ({}/{})".format("pass" if retry == 0 else "fail", pass_trial_cnt, total_trial_cnt))
log("pass: {}% ({}/{}), fail: {}% ({}/{}), invalid: {}% ({}/{})".format(
pass_trial_cnt*100.0/total_trial_cnt, pass_trial_cnt, total_trial_cnt,
failed_trial_cnt*100.0/total_trial_cnt, failed_trial_cnt, total_trial_cnt,
invalid_trial_cnt*100.0/(i+1), invalid_trial_cnt, i+1))
for serialno in [mo_serialno, mt_serialno]:
end_cs_call(serialno=serialno)
wait_for_phone_state(state=0, timeout=10, serialno=serialno)
time.sleep(5)
except:
pass
for serialno in [serialno1, serialno2]:
end_cs_call(serialno=serialno)
Adb.execute(["shell", "svc", "power", "stayon", "false"], serialno=serialno)
Logger.finalize()
def collect(module, url_queue):
#Excute the collect process
Logger.info('Start Collector')
#Send the url queue to bot via args
module.collect(url_queue)
Logger.info('Collect done')
def run(num_iter=1):
AudioFunction.init()
Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT)
Adb.init()
os.system("mkdir -p {}{}{} > {}".format(ROOT_DIR, SEP, REPORT_DIR, STDNUL))
t = datetime.datetime.now()
filename = "report_{}{:02d}{:02d}_{:02d}{:02d}{:02d}.json".format(
t.year, t.month, t.day, t.hour, t.minute, t.second)
device, serialno = ViewClient.connectToDeviceOrExit(serialno=None)
device.press("HOME")
time.sleep(1)
gmhandler = GoogleMusicApp(device, serialno)
log("gmhandler.to_top()")
gmhandler.to_top()
clear_and_update_music_files(serialno)
time.sleep(10)
trials = []
batch_count = 1
while num_iter > 0:
log("-------- batch_run #{} --------".format(batch_count))
trials_batch = []
trials_batch += playback_task_run(num_iter=min([num_iter, BATCH_SIZE]),
num_seek_test=5,
gmhandler=gmhandler)
trials_batch += record_task_run(device,
serialno,
num_iter=min([num_iter, BATCH_SIZE]),
num_freqs=5)
map(lambda trial: trial.put_extra(name="batch_id", value=batch_count),
trials_batch)
trials += trials_batch
with open(
"{}{}{}{}{}".format(ROOT_DIR, SEP, REPORT_DIR, SEP, filename),
"w") as f:
f.write(TrialHelper.to_json(trials))
for taskname, tasktrials in TrialHelper.categorize_in(
trials, lambda t: t.ds["task"]).items():
valid_trials = zip(
tasktrials,
TrialHelper.pass_fail_list(
tasktrials, lambda t: t.ds["status"] == "valid"))
valid_trials = [
trial for trial, isvalid in valid_trials if isvalid
]
num_valid = len(valid_trials)
num_pass = len(
filter(lambda x: x, TrialHelper.pass_fail_list(valid_trials)))
log("task[{}] valid trials: {}/{}, pass trials: {}/{}".format(
taskname, num_valid, len(tasktrials), num_pass, num_valid))
num_iter -= BATCH_SIZE
batch_count += 1
AudioFunction.finalize()
Logger.finalize()
def tone_detected_event_cb(self, event):
Logger.log("DetectionStateChangeListenerThread",
"tone_detected_event_cb: {}".format(event))
self._handle_event(event)
def log(msg):
Logger.log(GLOBAL["tag"], msg)
# coding: utf-8
# 导入libs下的logger日志模块
from libs.logger import Logger
# 导入selenium中的webdriver
from selenium import webdriver
# 导入os模块，获取文件路径
import os
__author__ = "sunxr"
__version__ = "V1.2"
logger = Logger("Browser").getLog()
def browser(browser_type):
"""
打开浏览器.
:param browser_type: 浏览器类型
:return: driver
"""
# 获取框架主路径信息
home_path = os.path.dirname(os.path.dirname(__file__))
# 定义浏览器驱动的路径
chrome_driver_path = home_path + '/tools/chromedriver'
firefox_driver_path = home_path + '/tools/geckodriver'
safari_driver_path = home_path + '/tools/safaridriver'
try:
if browser_type == "firefox":
def log(msg):
Logger.log(TAG, msg)
def __init__(self):
self.log = Logger()
self.db = DB().client
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'valf')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
data_aug_scale = (0.08, 1.0) if args.modelsize == 'large' else (0.2, 1.0)
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomResizedCrop(224, scale = data_aug_scale),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=True,
num_workers=args.workers, pin_memory=True)
# create model
# if args.pretrained:
# print("=> using pre-trained model '{}'".format(args.arch))
# model = models.__dict__[args.arch](pretrained=True)
# elif 'resnext' in args.arch:
# model = models.__dict__[args.arch](
# baseWidth=args.base_width,
# cardinality=args.cardinality,
# )
# else:
# print("=> creating model '{}'".format(args.arch))
# model = models.__dict__[args.arch]()
model = se_resnet101().cuda()
flops, params = get_model_complexity_info(model, (224, 224), as_strings=False, print_per_layer_stat=False)
print('Flops: %.3f' % (flops / 1e9))
print('Params: %.2fM' % (params / 1e6))
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..', args.resume)
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
# model may have more keys
t = model.state_dict()
c = checkpoint['state_dict']
flag = True
for k in t:
if k not in c:
print('not in loading dict! fill it', k, t[k])
c[k] = t[k]
flag = False
model.load_state_dict(c)
if flag:
print('optimizer load old state')
optimizer.load_state_dict(checkpoint['optimizer'])
else:
print('new optimizer !')
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
print('Best acc:')
print(best_acc)
def tone_detected_event_cb(self, event):
Logger.log(self.get_tag(), "tone_detected_event_cb: {}".format(event))
self._handle_event(event)
class QueueSystem:
def __init__(self):
self.db = sqlite3.connect('queue.db')
self.log = Logger(log_level="DEBUG").get_logger()
def build_game(self, min_players, max_players, game_id):
cursor = self.db.cursor()
cursor.execute("select player_id, queue_time, claimed, in_game, game_id from queue where claimed is 0 "
"order by queue_time")
results = cursor.fetchall()
queue_count = len(results)
player_count = min(queue_count, max_players)
if queue_count >= min_players:
for x in range(0, player_count):
player_id = results[x][0]
cursor.execute("update queue set claimed=?, game_id=?, player_count=? where player_id=?",
(1, game_id, player_count, player_id))
self.log.debug("{} is being added to game_id {}".format(player_id, game_id))
self.db.commit()
def build_db(self):
cursor = self.db.cursor()
cursor.execute("create table queue(player_id text, queue_time int, claimed boolean, in_game boolean, "
"game_id int, player_count int)")
self.db.commit()
def dequeue_player(self, player_id):
cursor = self.db.cursor()
cursor.execute("delete from queue where player_id=?", (player_id,))
self.log.debug("Removing player_id {} from the queue.".format(player_id))
self.db.commit()
def queue_player(self, player_id):
current_time = time.time()
cursor = self.db.cursor()
cursor.execute("select player_id from queue where player_id=?", (player_id,))
result = cursor.fetchone()
if result is None:
cursor.execute("insert into queue values(?, ?, 0, 0, NULL, NULL)", (player_id, current_time))
self.db.commit()
self.log.debug("Queueing player_id {} at {}".format(player_id, current_time))
def start_game(self, game_id):
cursor = self.db.cursor()
cursor.execute("SELECT * from queue where game_id=?", (game_id,))
game_ready = True
player_list = cursor.fetchall()
player_count = len(player_list)
player_id_list = []
for player in player_list:
if player[3] == 0:
game_ready = False
self.log.debug("Game not ready because player_id {} has not confirmed the game".format(player[0]))
elif player_count < player[5]:
game_ready = False
self.log.debug("Game not ready because {} does not match the required count of {}"
.format(player_count, player[5]))
else:
player_id_list.append(player[0])
if game_ready:
self.log.debug("game_id {} is ready to be started. players in game: {}"
.format(game_id, str(player_id_list)))
else:
cursor.execute("SELECT * from queue where game_id=?", (game_id,))
player_list = cursor.fetchall()
for player in player_list:
cursor.execute("update queue set claimed=0, game_id=NULL, player_count=NULL where player_id=?",
(player[0],))
self.db.commit()
self.log.debug("player_id {} is removed from game_id {} and placed back in the queue"
.format(player[0], player[4]))
import libs.glo_browsertype as glo
# 导入友工程首页操作类
from page_objects import PMCloudIndexActions
# 导入友工程登录页操作类
from page_objects import PMCloudLoginActions
# 导入企业帐号选择页操作类
from page_objects import ApptenantActions
# 导入友工程后台外框架操作类
from page_objects import WorkbenchActions
# 导入友工程联系类型操作类
from page_objects import ContactTypeActions
__author__ = "sunxr"
__version__ = "V1.0"
logger = Logger("TestContactTypeAdd").getLog()
@unittest.skip
class TestContactTypeAdd(unittest.TestCase):
"""联系类型新增相关测试"""
def setUp(self):
logger.info("测试前准备.")
self.__driver = browser(glo.GLO_BROWSER_TYPE)
PMCloudIndexActions(self.__driver).login()
PMCloudLoginActions(self.__driver).pmcloudLogin()
ApptenantActions(self.__driver).apptenantLogin()
WorkbenchActions(self.__driver).clickContactType()
self.contacttype_page = ContactTypeActions(self.__driver)
help=
"""Indicate the full path to scripts (e.g., /Applications/ResearchSoftware/rscript-wrapper"""
)
parser.add_argument('--version', action='version', version='%(prog)s v1.0')
args = parser.parse_args()
if len(sys.argv) is None: #print help if no flags declared
parser.print_help()
elif len(sys.argv) < 3: #print help if not all flags declared
print('Need more arguements')
parser.print_help()
else:
starttime = time.time()
workingdirectory = os.getcwd()
if not os.path.exists(args.outputdir):
os.makedirs(args.outputdir)
sys.stdout = Logger(os.path.join(args.outputdir + "/"))
print('''
***************************************************************************
* fasta_header_swap start *
***************************************************************************
''')
print("Parameters used to run fasta_headers_swap:")
print("")
print(" shortnamefasta: " + str(args.shortnamefasta))
print(" keeptaxonomylookup: " + str(args.keeptaxonomylookup))
print(" newlongnamefastafile: " + str(args.newlongnamefastafile))
print(" output directory: " + str(args.outputdir))
print(" filename prefix: " + str(args.filenameprefix))
print(" path to scripts: " + str(args.pathtoscripts))
print("")
print("Software and versions detected:")
def log(self, text):
Logger.log("GoogleMusicApp", text)
