class GameQueue: def __init__(self, min_size, max_size): self.log = Logger(log_level="DEBUG").get_logger() self.players = OrderedDict() self.game_dict = {} self.max_game_size = max_size self.min_game_size = min_size def add_player(self, player): self.players[player] = None def build_game(self, game_id): # add validation somewhere here to ensure players are still active current_size = len(self.players) if current_size >= self.min_game_size: for i in range(0, min(self.max_game_size, current_size)): player = self.players.popitem(last=False)[0] self.game_dict[player] = game_id self.log.debug("Adding {} to game.".format(player)) def queue_status(self, player): # TODO: more fine grained checks if player in self.game_dict.keys(): return self.game_dict[player] else: return None
def crawl(): #Use logger to debug the code Logger.debug('Hello google') #Get the html knowledge by parsing url soup = get_b_soup('http://www.google.com') #Send the data to output pipe line Queues.send_to_output(soup.head.title.text) Queues.send_to_output('http://www.google.com' + soup.head.meta['content'])
def collect(url_queue): #Use logger to debug the code Logger.debug('Hello amazon') #Get the html knowledge by parsing url soup = get_b_soup(base_path) #Travel with html knowledge for all_url in soup.findAll('a', href=re.compile('/gp/product/')): #REQUIRED #Send all url via url_queue url_queue.put(base_path + all_url['href'])
def crawl_thread(url, module): Logger.debug('Thread start') encoding = getattr(module, 'ENCODING', None) try: soup = get_b_soup(url, encoding=encoding) module.crawl(soup) except: Logger.error('Crawl error url: ' + url) Logger.error(traceback.format_exc()) return Logger.debug('Thread done')
def execute(module, name): #Execute the nested crawl method Logger.info('Start Execute') if not hasattr(module, 'collect'): raise NotImplementedError('You must implement collect()') #Create the url queue and send to collect and crawl process url_queue = Queue() collect_process = Process(target=collect, args=(module, url_queue), name=name + '_CollectProcess') collect_process.start() crawl_process = Process(target=crawl, args=(module, url_queue), name=name + '_CrawlProcess') crawl_process.start() #Both collect and crawl are safe collect_process.join() crawl_process.join() Logger.info('Execute done')
def execute_output(): Logger() Logger.debug('Start Output Process') while True: try: result = Queues.get_output() Logger.info(result) #Your output logic go there except: break Logger.debug('End Output Process')
def run(num_iter=1, serialno=None): num_iter = int(num_iter) Adb.init() Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT, prefix="quick-start") check_props = { "Device name": "ro.product.model", "Project": "ro.build.product", "ROM": "ro.product.build.fingerprint", } passed = True for tag, prop in check_props.items(): out, err = Adb.execute(["shell", "getprop {}".format(prop)], serialno=serialno) passed &= len(err) == 0 out = out.strip() Logger.log(LOGGER_TAG, "{}: {}".format(tag, out)) Logger.log(LOGGER_TAG, "result: {}".format("passed" if passed else "failed")) Logger.finalize()
def crawl(module, url_queue): #Execute the crawl process Logger.info('Start Crawler') while True: try: url = url_queue.get(timeout=config.NESTED_CRAWL_TIMEOUT) except: #If all threads are done then break the loop, Otherwise continue. #Why 2 ? because its need to deduct by the main thread and queue thread, # You can comment out the enumerate() line to see what is going on #Logger.debug(str(enumerate())) if activeCount() <= 2: Logger.info('Break crawl') break else: Logger.debug('There are ' + str(activeCount() - 2) + ' threads left') continue #Spawn a new threads immediate after get the url thread = Thread(target=crawl_thread, args=(url, module), name='CrawlThread') thread.start() Logger.info('Crawl done')
import libs.glo_browsertype as glo # 导入友工程首页操作类 from page_objects import PMCloudIndexActions # 导入友工程登录页操作类 from page_objects import PMCloudLoginActions # 导入企业帐号选择页操作类 from page_objects import ApptenantActions # 导入友工程后台外框架操作类 from page_objects import WorkbenchActions # 导入友工程联系类型操作类 from page_objects import ContactTypeActions __author__ = "sunxr" __version__ = "V1.0" logger = Logger("TestContactTypeEdit").getLog() class TestContactTypeEdit(unittest.TestCase): """联系类型编辑相关测试""" @classmethod def setUpClass(cls): logger.info("测试前准备.") cls.__driver = browser(glo.GLO_BROWSER_TYPE) PMCloudIndexActions(cls.__driver).login() PMCloudLoginActions(cls.__driver).pmcloudLogin() ApptenantActions(cls.__driver).apptenantLogin() WorkbenchActions(cls.__driver).clickContactType() # 提前新建好联系类型
def __init__(self): self.db = sqlite3.connect('queue.db') self.log = Logger(log_level="DEBUG").get_logger()
def __init__(self, min_size, max_size): self.log = Logger(log_level="DEBUG").get_logger() self.players = OrderedDict() self.game_dict = {} self.max_game_size = max_size self.min_game_size = min_size
def run(num_iter, serialno): Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT) # Logger.init(Logger.Mode.STDOUT) Adb.init() Adb.execute(["root"], serialno=serialno) Adb.execute(["shell", "'echo \'related\' > msm_subsys'"], serialno=serialno) Adb.execute(["shell", "svc", "power", "stayon", "true"], serialno=serialno) out, err = Adb.execute(["shell", "getprop", "ro.vendor.build.fingerprint"], serialno=serialno) out = out.strip() log("build number: '{}'".format(out)) latest_dmesg = fetch_dmesg(serialno=serialno) latest_dmesg = latest_dmesg[0] if len(latest_dmesg) > 0 else None pass_trial_cnt = 0 try: for i in range(num_iter): log("-------------------- CS call test #{} --------------------". format(i + 1)) events = fetch_setmode_events(serialno=serialno) latest_event = events[-1] if len(events) > 0 else None log("The latest setMode event: '{}'".format( latest_event.raw if latest_event else "None")) log("Make call to 0988102544") start_cs_call(tel="0988102544", serialno=serialno) log("Waiting to the mode 'MODE_IN_CALL'") if not wait_for_phone_state(state=2, timeout=10, serialno=serialno): log("The phone state never turns in 'MODE_IN_CALL', ignore this trial" ) continue log("To check if ADSP crashes during the CS call") is_passed = True pass_trial_cnt += 1 retry = 15 while retry > 0: new_dmesgs = fetch_dmesg(latest_dmesg, serialno=serialno) if len(new_dmesgs) > 0: latest_dmesg = new_dmesgs[-1] adsp_ssr_demsgs = filter( lambda x: "Restart sequence requested for adsp" in x. raw, new_dmesgs) if len(adsp_ssr_demsgs) > 0: for dmesg in adsp_ssr_demsgs: log("SSR log detected: '{}'".format(dmesg.raw)) is_passed = False phone_state = get_phone_state(serialno=serialno) if phone_state == None: log("the phone state is unobtainable, something wrong") is_passed = False if phone_state == 0: log("the phone state is in idle, the call might be dropped" ) is_passed = False if not is_passed: out, err = Adb.execute(["bugreport"]) log("bugreport to '{}'".format(out.strip())) pass_trial_cnt -= 1 break if retry % 5 == 0: log("switch path to '{}'".format("speaker" if retry / 5 % 2 == 1 else "receiver")) Adb.execute(["shell", "input", "tap", "1000", "1000"], tolog=False, serialno=serialno) retry -= 1 time.sleep(1) log("result: {} ({}/{})".format("pass" if retry == 0 else "fail", pass_trial_cnt, i + 1)) end_cs_call(serialno=serialno) time.sleep(10) except: pass Adb.execute(["shell", "svc", "power", "stayon", "false"], serialno=serialno) Logger.finalize()
from libs.HTMLTestRunner import HTMLTestRunner # 导入time模块,用于时间处理 import time # 导入日志模块 from libs.logger import Logger # 导入Configuration模块,用于操作配置文件 from libs.configuration import Configuration # 导入setBrowserType方法,用于多线程启动不同浏览器的兼容性测试 from libs.glo_browsertype import setBrowserType # 导入os模块,获取文件路径 import os __author__ = "sunxr" __version__ = "V1.2" logger = Logger("TestCaseSuite").getLog() class TestCaseSuite: """ 定义测试用例路径,并执行测试用例 """ def __init__(self): # 创建配置文件实例 config = Configuration() # 获取框架主路径信息 self.__home_path = os.path.dirname(os.path.dirname(__file__)) # 读取配置文件中的测试用例路径和测试用例执行规则
from flask import Flask, request from flask_restful import Api from resources.respondent import Respondent from libs.logger import Logger from config.config import general as config app = Flask(__name__) api = Api(app) log = Logger() # Routes api.add_resource(Respondent, '/respondent', '/respondent/<name>') #------- if __name__ == '__main__': log.info('Server listen in %s:%s' % (config['host'], config['port'])) app.run(debug=True, host=config['host'], port=config['port'])
# coding: utf-8 import unittest from libs.browser import browser from libs.logger import Logger import libs.glo_browsertype as glo from page_objects import PMCloudIndexActions from page_objects import PMCloudLoginActions from page_objects import ApptenantActions logger = Logger("TestApptenant").getLog() class TestApptenant(unittest.TestCase): """测试友工程的企业帐号""" def setUp(self): logger.info("测试前准备.") self.driver = browser(glo.GLO_BROWSER_TYPE) PMCloudIndexActions(self.driver).openPMCloudIndex() PMCloudIndexActions(self.driver).clickLoginButton() PMCloudLoginActions(self.driver).pmcloudLogin(remember_selected=True) def tearDown(self): logger.info("测试后退出.") self.driver.quit() def test_apptenant_login_success(self): """企业帐号验证通过"""
class CXBot(Client, Setup, Bot, IO, Server, Channel, User, Role, Features, Base): def __init__(self): super(CXBot, self).__init__() self.c = Config() self.max_messages = 20000 self.ready = False self.logger = Logger('logs/%s.log' % datetime.now().date().isoformat()) self.busy = False @Base.excep async def on_ready(self): if not self.ready: self.ready = True print('Logged on as %s!' % self.user) if len(self.servers) == 0: print( 'You have to connect me to a server. Open https://discordapp.com/oauth2/authorize?client_id=%s\ &scope=bot&permissions=0' % self.c.clientID) await self.logout() await self.close() elif len(self.servers) > 1: print('Connected to the following servers:\n\t%s' % '\n\t'.join(self._server_names())) print( 'You connected me to multiple servers. Note that you may want to use multiple instances of this \ bot because otherwise all settings will be the same for all connected servers and some functions \ may could interfere in this state of development.') else: print('Connected to the server %s' % self._server_names()[0]) if self.c.firstStart: self.busy = True await self.setup() self.busy = False async def on_resumed(self): pass async def on_error(self, event, *args, **kwargs): print(event) @Base.excep async def on_message(self, message): if self.c.logMessages: self.logger.log_message(message) if self.busy: return if not message.author == self.user: print(message.content) e = await self.create_embed( "Github CeroProgramming", "https://github.com/CeroProgramming", "GitHub Profile URL", Color.dark_purple(), "CeroProgramming", "https://github.com/CeroProgramming/", "https://avatars3.githubusercontent.com/u/22818389?s=460&v=4", "https://proxy.duckduckgo.com/iu/?u=https%3A%2F%2Fimage.freepik.com%2Ffree-icon%2Fgithub-logo_318-53553.jpg&f=1" ) await self.io_embed(message.channel, e) if message.content == "shutdown": await self.shutdown() async def on_message_delete(self, message): pass # Increase Clients max_messages for more cached messages async def on_message_edit(self, before, after): if self.c.logMessages: if before.content != after.content or before.embeds != after.embeds or before.attachments != after.attachments: self.logger.log_message_change(before, after) async def on_reaction_add(self, reaction, user): pass async def on_reaction_remove(self, reaction, user): pass async def on_reaction_clear(self, message, reactions): pass async def on_channel_create(self, channel): pass async def on_channel_delete(self, channel): pass async def on_channel_update(self, before, after): pass async def on_member_join(self, member): pass async def on_member_remove(self, member): pass async def on_member_update(self, before, after): pass async def on_server_join(self, server): pass async def on_server_remove(self, server): pass # Includes banned, kicked, left, really removed async def on_server_update(self, before, after): pass async def on_server_role_create(self, role): pass async def on_server_role_delete(self, role): pass async def on_server_role_update(self, before, after): pass async def on_server_emojis_update(self, before, after): pass async def on_server_available(self, server): pass async def on_server_unavailable(self, server): pass async def on_voice_state_update(self, before, after): pass # leave join mute deaf async def on_member_ban(self, member): pass async def on_member_unban(self, server, user): pass async def on_typing(self, channel, user, when): pass async def on_group_join(self, channel, user): pass async def on_group_remove(self, channel, user): pass async def shutdown(self): # TODO Test await self.logout() await self.close()
def evaluate_imagenet_c( model, transform, dataset_dir: str, log_dir: str, corruptions: List[str], batch_size: int, device: str, **kwargs, ) -> None: """ Evaluate corruption accuracy on ImageNet-C. """ if (device != "cpu") and (torch.cuda.device_count() > 1): model = torch.nn.DataParallel(model) log_path = os.path.join(log_dir, os.path.join("imagenet_c_result.csv")) logger = Logger(path=log_path, mode="test") with tqdm.tqdm(total=len(corruptions), ncols=80) as pbar: for i, corruption_type in enumerate(corruptions): accuracies = list() for j in range(1, 6): # imagenet-c dataset is separated to 5 small sets. datasetpath = os.path.join(dataset_dir, corruption_type, str(j)) dataset = torchvision.datasets.ImageFolder(datasetpath, transform) loader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=8, pin_memory=True, ) for x, y in loader: x, y = x.to(device), y.to(device) with torch.autograd.no_grad(): y_predict_std = model(x) stdacc1, stdacc5 = accuracy(y_predict_std, y, topk=(1, 5)) accuracies.append(stdacc1.item()) log_dict = collections.OrderedDict() log_dict["corruption_type"] = corruption_type log_dict["accuracy"] = sum(accuracies) / float(len(accuracies)) logger.log(log_dict) pbar.set_postfix( collections.OrderedDict( corruption_type="{}".format(corruption_type), acc="{}".format(log_dict["accuracy"]), ) ) pbar.update() df = pd.read_csv(log_path) result_dict = dict(zip(df["corruption_type"], df["accuracy"])) mean_corruption_acc = sum(result_dict.values()) / float(len(result_dict)) create_barplot( result_dict, title="mean corruption acc: {0:0.1f}".format(mean_corruption_acc), savepath=os.path.join(log_dir, "plot_result.png"), )
# coding: utf-8 # 导入同目录下的logger日志模块 from libs.logger import Logger # 导入Configuration模块,用于操作配置文件 from libs.configuration import Configuration # 导入MySQL操作pymysql模块 import pymysql __author__ = "sunxr" __version__ = "V1.2" logger = Logger("MySQLOperation").getLog() class MySQLOperation: """ 将pymysql中的部分方法进行封装,处理对MySQL数据库的简单操作. """ def __init__(self): # 创建配置文件实例 self.__config = Configuration() # 从配置文件获取数据库相关信息 self.__host = self.__config.getConfigValue("mysqlInfo", "host") self.__user = self.__config.getConfigValue("mysqlInfo", "user") self.__password = self.__config.getConfigValue("mysqlInfo", "password") self.__database = self.__config.getConfigValue("mysqlInfo", "database") # 连接数据库,获取操作游标 self.__db = self.__dbConnect()
def evaluate_corruption_accuracy( model, dataset_builder, log_dir: str, num_samples: int, corruptions: list, batch_size: int, device: str, **kwargs, ): """ """ if (device != "cpu") and (torch.cuda.device_count() > 1): model = torch.nn.DataParallel(model) log_path = os.path.join(log_dir, os.path.join("corruption_result.csv")) logger = Logger(path=log_path, mode="test") with tqdm.tqdm(total=len(corruptions), ncols=80) as pbar: for i, corruption_type in enumerate(corruptions): dataset = dataset_builder( train=False, normalize=True, num_samples=num_samples, corruption_type=corruption_type, ) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=batch_size, shuffle=False ) accuracies = list() for x, y in loader: x, y = x.to(device), y.to(device) with torch.autograd.no_grad(): y_predict_std = model(x) stdacc1, stdacc5 = accuracy(y_predict_std, y, topk=(1, 5)) accuracies.append(stdacc1.item()) log_dict = collections.OrderedDict() log_dict["corruption_type"] = corruption_type log_dict["accuracy"] = sum(accuracies) / float(len(accuracies)) logger.log(log_dict) pbar.set_postfix( collections.OrderedDict( corruption_type="{}".format(corruption_type), acc="{}".format(log_dict["accuracy"]), ) ) pbar.update() df = pd.read_csv(log_path) result_dict = dict(zip(df["corruption_type"], df["accuracy"])) mean_corruption_acc = sum(result_dict.values()) / float(len(result_dict)) create_barplot( result_dict, title="mean corruption acc: {0:0.1f}".format(mean_corruption_acc), savepath=os.path.join(log_dir, "plot_result.png"), )
class CLFNet(): def __init__(self, config): self.config = config model_name = 'inpaint' self.debug = False self.model_name = model_name self.inpaint_model = InpaintingModel(config).to(config.DEVICE) # summary(InpaintingModel, (3, 256, 256), 6) # print(InpaintingModel) self.psnr = PSNR(255.0).to(config.DEVICE) self.ssim = SSIM(window_size=11) val_sample = int(float((self.config.EVAL_INTERVAL))) # test mode if self.config.MODE == 2 or self.config.MODE == 4 or self.config.MODE == 5: self.test_dataset = Dataset(config, config.TEST_FLIST, config.TEST_MASK_FLIST, augment=False, training=False) else: self.train_dataset = Dataset(config, config.TRAIN_FLIST, config.TRAIN_MASK_FLIST, augment=True, training=True) self.val_dataset = Dataset(config, config.VAL_FLIST, config.VAL_MASK_FLIST, augment=False, training=True, sample_interval=val_sample) self.sample_iterator = self.val_dataset.create_iterator( config.SAMPLE_SIZE) self.samples_path = os.path.join(config.PATH, 'samples') self.results_path = os.path.join(config.PATH, 'results') if config.RESULTS is not None: self.results_path = os.path.join(config.RESULTS) if config.DEBUG is not None and config.DEBUG != 0: self.debug = True # self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat') log_path = os.path.join(config.PATH, 'logs_' + model_name) create_dir(log_path) self.logger = Logger(log_path) def load(self): self.inpaint_model.load() def save(self, epoch): self.inpaint_model.save(epoch) def train(self): train_loader = DataLoader(dataset=self.train_dataset, batch_size=self.config.BATCH_SIZE, num_workers=4, drop_last=True, shuffle=True) keep_training = True model = self.config.MODEL # max_iteration = int(float((self.config.MAX_ITERS))) step_per_epoch = int(float((self.config.MAX_STEPS))) max_epoches = int(float((self.config.MAX_EPOCHES))) total = int(len(self.train_dataset)) if total == 0: print( 'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.' ) return print('\nThe number of Training data is %d' % total) epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1 print('\nTraining epoch: %d' % epoch) progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) logs_ave = {} train_times = 0 while (keep_training): for items in train_loader: self.inpaint_model.train() images, images_gray, masks = self.cuda(*items) # edge model # inpaint model # train outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['psnr'] = psnr.item() # logs['mae'] = mae.item() # backward self.inpaint_model.backward(gen_loss, dis_loss) if self.inpaint_model.iteration > step_per_epoch: self.inpaint_model.iteration = 0 iteration = 0 iteration = self.inpaint_model.iteration if iteration == 1: # first step in this epoch for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value if iteration == 0: # mean to jump to new epoch self.sample(epoch) self.eval(epoch) self.save(epoch) # log current epoch in tensorboard for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / step_per_epoch, epoch) # if reach max epoch if epoch >= max_epoches: keep_training = False break epoch += 1 # new epoch print('\n\nTraining epoch: %d' % epoch) for tag, value in logs.items(): logs_ave[tag] = value progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) self.inpaint_model.iteration += 1 # jump to new epoch and set the iteration to 1 iteration += 1 logs['step'] = iteration progbar.add( 1, values=logs.items() if self.config.VERBOSE else [x for x in logs.items() if not x[0].startswith('l_')]) train_times += 1 print("The whole data hase been trained %d times" % train_times) print('\nEnd training....\n') def eval(self, epoch): self.val_loader = DataLoader(dataset=self.val_dataset, batch_size=self.config.BATCH_SIZE, drop_last=True, shuffle=False, num_workers=4) model = self.config.MODEL total = int(len(self.val_dataset)) self.inpaint_model.eval() progbar = Progbar(int(total / self.config.BATCH_SIZE), width=30, stateful_metrics=['step']) iteration = 0 logs_ave = {} with torch.no_grad(): for items in self.val_loader: iteration += 1 images, images_gray, masks = self.cuda(*items) # inpaint model # eval outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['val_psnr'] = psnr.item() logs['val_mae'] = mae.item() # joint model if iteration == 1: # first iteration logs_ave = {} for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value logs["step"] = iteration progbar.add(1, values=logs.items()) for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / iteration, epoch) self.inpaint_model.iteration = 0 def test(self): self.inpaint_model.eval() damaged_dir = os.path.join(self.results_path, "damaged") create_dir(damaged_dir) mask_dir = os.path.join(self.results_path, "mask") create_dir(mask_dir) inpainted_dir = os.path.join(self.results_path, "inpainted") create_dir(inpainted_dir) raw_dir = os.path.join(self.results_path, "raw") create_dir(raw_dir) create_dir(self.results_path) sample_interval = self.config.TEST_INTERVAL batch_size = 1 test_loader = DataLoader(dataset=self.test_dataset, batch_size=batch_size, num_workers=1, shuffle=False) total = int(len(self.test_dataset) / batch_size / sample_interval) progbar = Progbar(total, width=30, stateful_metrics=['step']) index = 0 proc_start_time = time.time() total_time = 0. with torch.no_grad(): for items in test_loader: name = self.test_dataset.load_name(index) images, images_gray, masks = self.cuda(*items) index += 1 if index >= total: break # Save raw # if self.config.SAVEIMG == 1: # path = os.path.join(damaged_dir, name) # damaged_img = self.postprocess(images * masks + (1 - masks))[0] # imsave(damaged_img, path) # # Save masks # path = os.path.join(mask_dir, name) # imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png') # # Save Ground Truth # path = os.path.join(raw_dir, name) # img = self.postprocess(images)[0] # imsave(img, path) # # print(index, name) logs = {} # run model outputs = self.inpaint_model(images, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) output = self.postprocess(outputs_merged)[0] if self.config.SAVEIMG == 1: path = os.path.join(inpainted_dir, name) # print(index, name) imsave(output, path) cnt_time = time.time() - proc_start_time total_time = total_time + cnt_time # print('Image {} done, time {}, totaltime{}, {} sec/Image'.format(index, cnt_time,total_time, # float(total_time) / index)) psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() # mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float() = = L1 distance l1 = torch.nn.L1Loss()(images, outputs_merged) one_ssim = self.ssim(images, outputs_merged) logs["psnr"] = psnr.item() # logs["mae"] = mae.item() logs["L1"] = l1.item() logs["ssim"] = one_ssim.item() logs["step"] = index progbar.add(1, values=logs.items()) if self.debug: pass proc_start_time = time.time() print('\nEnd test....') print('Image {} done, time {}, average {} sec/Image'.format( total, total_time, float(total_time) / total)) def progressive_test(self): self.inpaint_model.eval() damaged_dir = os.path.join(self.results_path, "damaged") create_dir(damaged_dir) mask_dir = os.path.join(self.results_path, "mask") create_dir(mask_dir) inpainted_dir = os.path.join(self.results_path, "inpainted") create_dir(inpainted_dir) raw_dir = os.path.join(self.results_path, "raw") create_dir(raw_dir) create_dir(self.results_path) sample_interval = self.config.TEST_INTERVAL batch_size = 1 test_loader = DataLoader(dataset=self.test_dataset, batch_size=batch_size, num_workers=1, shuffle=True) total = int(len(self.test_dataset) / batch_size / sample_interval) progbar = Progbar(total, width=30, stateful_metrics=['step']) index = 0 total_time = 0. proc_start_time = time.time() with torch.no_grad(): for items in test_loader: logs = {} name = self.test_dataset.load_name(index) images, images_gray, masks = self.cuda(*items) index += 1 if index > total: break # Save damaged image if self.config.SAVEIMG == 1: path = os.path.join(damaged_dir, str(index) + '.jpg') damaged_img = self.postprocess(images * masks + (1 - masks))[0] imsave(damaged_img, path) # Save masks path = os.path.join(mask_dir, str(index)) imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png') # Save Ground Truth path = os.path.join(raw_dir, str(index) + '.jpg') img = self.postprocess(images)[0] imsave(img, path) # print(index, name) # run model outputs = self.inpaint_model(images, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) # # Test again: # masks=masks.cpu().numpy().astype(np.uint8) # [0-255] # # masks = (masks > 128).astype(np.uint8) # [0-1] # struct = ndimage.generate_binary_structure(4, 5) # np_masks=ndimage.binary_dilation(masks,structure=struct).astype(masks.dtype) # [0-1] # masks=torch.from_numpy(np_masks) np_masks = masks.cpu().numpy() NoHoleNum = np.count_nonzero(np_masks) ratio = (np_masks.size - NoHoleNum) / np_masks.size i = 1 while ratio > 0.2 and i <= 10: # Save masks again # Erosion masks = masks.cpu().numpy().astype(np.uint8) # [0-255] # np_masks = ndimage.grey_dilation(masks, size=(1, 1, 9, 9)) np_masks = ndimage.grey_dilation(masks, size=(1, 1, 15, 15)) masks = torch.from_numpy(np_masks).float().to( self.config.DEVICE) if self.config.SAVEIMG == 1: path = os.path.join(mask_dir, name) imsave(self.postprocess(masks), os.path.splitext(path)[0] + '_%d.png' % i) outputs = self.inpaint_model(outputs_merged, masks) outputs_merged = (outputs * (1 - masks)) + (outputs_merged * masks) i += 1 # count no hole number and ratio NoHoleNum = np.count_nonzero(np_masks) ratio = (np_masks.size - NoHoleNum) / np_masks.size if self.config.SAVEIMG == 1: path = os.path.join(inpainted_dir, str(index) + '.jpg') # print(index, name) output = self.postprocess(outputs_merged)[0] imsave(output, path) cnt_time = time.time() - proc_start_time total_time = total_time + cnt_time psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() # mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float() l1 = torch.nn.L1Loss()(images, outputs_merged) one_ssim = self.ssim(images, outputs_merged) logs["psnr"] = psnr.item() # logs["mae"] = mae.item() logs["L1"] = l1.item() logs["ssim"] = one_ssim.item() logs["step"] = index progbar.add(1, values=logs.items()) if self.debug: pass proc_start_time = time.time() print('\nEnd test....') print('Image {} done, time {}, average {} sec/Image'.format( total, total_time, float(total_time) / total)) def feature_visualize(self, module, input): xs = input[0] index = 0 for x in xs: x = x[0] # remove batch dimension min_num = np.minimum(16, x.size()[0]) for i in range(min_num): plt.subplot(4, 4, i + 1) plt.axis('off') plt.imshow(x[i].cpu()) name = 'stage_' + str(module.stage) + 'branch_' + str(index) for i in x.shape: name = name + '_' + str(i) plt.savefig(name + '.png', bbox_inches='tight') plt.show() index += 1 def visualization_test(self): self.inpaint_model.eval() for m in self.inpaint_model.generator.modules(): if isinstance(m, TwoBranchModule): m.register_forward_pre_hook(self.feature_visualize) # if isinstance(m, torch.nn.Conv2d): # m.register_forward_pre_hook(self.feature_visualize) damaged_dir = os.path.join(self.results_path, "damaged") create_dir(damaged_dir) mask_dir = os.path.join(self.results_path, "mask") create_dir(mask_dir) inpainted_dir = os.path.join(self.results_path, "inpainted") create_dir(inpainted_dir) raw_dir = os.path.join(self.results_path, "raw") create_dir(raw_dir) create_dir(self.results_path) sample_interval = self.config.TEST_INTERVAL batch_size = 1 test_loader = DataLoader(dataset=self.test_dataset, batch_size=batch_size, num_workers=1, shuffle=False) total = int(len(self.test_dataset)) progbar = Progbar(int(total / batch_size / sample_interval), width=30, stateful_metrics=['step']) index = 0 with torch.no_grad(): for items in test_loader: name = self.test_dataset.load_name(index) images, images_gray, masks = self.cuda(*items) # Save damaged image if self.config.SAVEIMG == 1: path = os.path.join(damaged_dir, name) damaged_img = self.postprocess(images * masks + (1 - masks))[0] imsave(damaged_img, path) # Save masks path = os.path.join(mask_dir, name) imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png') # Save Ground Truth path = os.path.join(raw_dir, name) img = self.postprocess(images)[0] imsave(img, path) # print(index, name) index += 1 if index > total / batch_size / sample_interval: break logs = {} # run model outputs = self.inpaint_model(images, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) output = self.postprocess(outputs_merged)[0] if self.config.SAVEIMG == 1: path = os.path.join(inpainted_dir, name) # print(index, name) imsave(output, path) psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() # mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float() l1 = torch.nn.L1Loss()(images, outputs_merged) one_ssim = self.ssim(images, outputs_merged) logs["psnr"] = psnr.item() # logs["mae"] = mae.item() logs["L1"] = l1.item() logs["ssim"] = one_ssim.item() logs["step"] = index progbar.add(1, values=logs.items()) if self.debug: pass break print('\nEnd test....') def sample(self, it=None): # do not sample when validation set is empty if len(self.val_dataset) == 0: return self.inpaint_model.eval() model = self.config.MODEL with torch.no_grad(): items = next(self.sample_iterator) images, images_gray, masks = self.cuda(*items) # (batch,channels,weighth,length) iteration = self.inpaint_model.iteration inputs = (images * masks) + (1 - masks) outputs = self.inpaint_model(images, masks).detach() if it is not None: iteration = it image_per_row = 2 if self.config.SAMPLE_SIZE <= 6: image_per_row = 1 images = stitch_images( self.postprocess(images), # self.postprocess(edges), self.postprocess(inputs), self.postprocess(outputs), # self.postprocess(outputs_merged), img_per_row=image_per_row) path = os.path.join(self.samples_path, self.model_name) name = os.path.join(path, str(iteration).zfill(3) + ".png") create_dir(path) print('\nsaving sample ' + name) images.save(name) def log(self, logs): with open(self.log_file, 'a') as f: f.write('%s\n' % ' '.join([str(item[1]) for item in logs])) def cuda(self, *args): return (item.to(self.config.DEVICE) for item in args) def postprocess(self, img): # [0, 1] => [0, 255] img = img * 255.0 img = img.permute(0, 2, 3, 1) return img.int() def preprocess(self, img): # [0, 255] => [0, 1] img = np.expand_dims(img, axis=0) img = np.expand_dims(img, axis=0) # img = img.astype(float) / 255.0 return img def color_the_edge(self, img, edges, masks): img = img.expand(-1, 3, -1, -1) yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255. ]).reshape(1, 3, 1, 1)).to(self.config.DEVICE) yellow = img * (1 - masks) * yellow_v img = yellow + (edges * masks) return img
class CLFNet(): def __init__(self, config): self.config = config model_name = 'inpaint' self.debug = False self.model_name = model_name self.inpaint_model = InpaintingModel(config).to(config.DEVICE) # summary(InpaintingModel, (3, 256, 256), 6) # print(InpaintingModel) self.psnr = PSNR(255.0).to(config.DEVICE) self.ssim = SSIM(window_size=11) val_sample = int(float((self.config.EVAL_INTERVAL))) # test mode if self.config.MODE == 2: self.test_dataset = Dataset(config, config.TEST_FLIST, config.TEST_MASK_FLIST, augment=False, training=False) else: self.train_dataset = Dataset(config, config.TRAIN_FLIST, config.TRAIN_MASK_FLIST, augment=True, training=True) self.val_dataset = Dataset(config, config.VAL_FLIST, config.VAL_MASK_FLIST, augment=False, training=True, sample_interval=val_sample) self.sample_iterator = self.val_dataset.create_iterator( config.SAMPLE_SIZE) self.samples_path = os.path.join(config.PATH, 'samples') self.results_path = os.path.join(config.PATH, 'results') if config.RESULTS is not None: self.results_path = os.path.join(config.RESULTS) if config.DEBUG is not None and config.DEBUG != 0: self.debug = True # self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat') log_path = os.path.join(config.PATH, 'logs_' + model_name) create_dir(log_path) self.logger = Logger(log_path) def load(self): self.inpaint_model.load() def save(self, epoch): self.inpaint_model.save(epoch) def train(self): train_loader = DataLoader(dataset=self.train_dataset, batch_size=self.config.BATCH_SIZE, num_workers=4, drop_last=True, shuffle=True) keep_training = True model = self.config.MODEL # max_iteration = int(float((self.config.MAX_ITERS))) step_per_epoch = int(float((self.config.MAX_STEPS))) max_epoches = int(float((self.config.MAX_EPOCHES))) total = int(len(self.train_dataset)) if total == 0: print( 'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.' ) return print('\nThe number of Training data is %d' % total) epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1 print('\nTraining epoch: %d' % epoch) progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) logs_ave = {} train_times = 0 while (keep_training): for items in train_loader: self.inpaint_model.train() images, images_gray, masks = self.cuda(*items) # edge model # inpaint model # train outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['psnr'] = psnr.item() # logs['mae'] = mae.item() # backward self.inpaint_model.backward(gen_loss, dis_loss) if self.inpaint_model.iteration > step_per_epoch: self.inpaint_model.iteration = 0 iteration = 0 iteration = self.inpaint_model.iteration if iteration == 1: # first step in this epoch for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value if iteration == 0: # mean to jump to new epoch self.sample(epoch) self.eval(epoch) self.save(epoch) # log current epoch in tensorboard for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / step_per_epoch, epoch) # if reach max epoch if epoch >= max_epoches: keep_training = False break epoch += 1 # new epoch print('\n\nTraining epoch: %d' % epoch) for tag, value in logs.items(): logs_ave[tag] = value progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) self.inpaint_model.iteration += 1 # jump to new epoch and set the iteration to 1 iteration += 1 logs['step'] = iteration progbar.add( 1, values=logs.items() if self.config.VERBOSE else [x for x in logs.items() if not x[0].startswith('l_')]) train_times += 1 print("The whole data hase been trained %d times" % train_times) print('\nEnd training....\n') def eval(self, epoch): self.val_loader = DataLoader(dataset=self.val_dataset, batch_size=self.config.BATCH_SIZE, drop_last=True, shuffle=False, num_workers=4) model = self.config.MODEL total = int(len(self.val_dataset)) self.inpaint_model.eval() progbar = Progbar(int(total / self.config.BATCH_SIZE), width=30, stateful_metrics=['step']) iteration = 0 logs_ave = {} with torch.no_grad(): for items in self.val_loader: iteration += 1 images, images_gray, masks = self.cuda(*items) # inpaint model # eval outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['val_psnr'] = psnr.item() logs['val_mae'] = mae.item() # joint model if iteration == 1: # first iteration logs_ave = {} for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value logs["step"] = iteration progbar.add(1, values=logs.items()) for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / iteration, epoch) self.inpaint_model.iteration = 0 def test(self): self.inpaint_model.eval() damaged_dir = os.path.join(self.results_path, "damaged") create_dir(damaged_dir) mask_dir = os.path.join(self.results_path, "mask") create_dir(mask_dir) inpainted_dir = os.path.join(self.results_path, "inpainted") create_dir(inpainted_dir) raw_dir = os.path.join(self.results_path, "raw") create_dir(raw_dir) create_dir(self.results_path) sample_interval = 1 batch_size = 1 test_loader = DataLoader(dataset=self.test_dataset, batch_size=batch_size, num_workers=1, shuffle=False) total = int(len(self.test_dataset)) progbar = Progbar(int(total / batch_size / sample_interval), width=30, stateful_metrics=['step']) index = 0 with torch.no_grad(): for items in test_loader: name = self.test_dataset.load_name(index) images, images_gray, masks = self.cuda(*items) # Save damaged image if self.config.MODE == 2: path = os.path.join(damaged_dir, name) damaged_img = self.postprocess(images * masks + (1 - masks))[0] imsave(damaged_img, path) # Save masks path = os.path.join(mask_dir, name) imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png') # Save Ground Truth path = os.path.join(raw_dir, name) img = self.postprocess(images)[0] imsave(img, path) # print(index, name) index += 1 if index > total / batch_size / sample_interval: break logs = {} # run model outputs = self.inpaint_model(images, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) output = self.postprocess(outputs_merged)[0] if self.config.MODE == 2: path = os.path.join(inpainted_dir, name) # print(index, name) imsave(output, path) psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) # mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() # mae = (torch.sum(torch.abs(images - outputs_merged)) / images.numel()).float() l1 = torch.nn.L1Loss()(images, outputs_merged) one_ssim = self.ssim(images, outputs_merged) logs["psnr"] = psnr.item() # logs["mae"] = mae.item() logs["L1"] = l1.item() logs["ssim"] = one_ssim.item() logs["step"] = index progbar.add(1, values=logs.items()) if self.debug: pass print('\nEnd test....') def sample(self, it=None): # do not sample when validation set is empty if len(self.val_dataset) == 0: return self.inpaint_model.eval() model = self.config.MODEL with torch.no_grad(): items = next(self.sample_iterator) images, images_gray, masks = self.cuda(*items) # (batch,channels,weighth,length) iteration = self.inpaint_model.iteration inputs = (images * masks) + (1 - masks) outputs = self.inpaint_model(images, masks).detach() if it is not None: iteration = it image_per_row = 2 if self.config.SAMPLE_SIZE <= 6: image_per_row = 1 images = stitch_images( self.postprocess(images), # self.postprocess(edges), self.postprocess(inputs), self.postprocess(outputs), # self.postprocess(outputs_merged), img_per_row=image_per_row) path = os.path.join(self.samples_path, self.model_name) name = os.path.join(path, str(iteration).zfill(3) + ".png") create_dir(path) print('\nsaving sample ' + name) images.save(name) def log(self, logs): with open(self.log_file, 'a') as f: f.write('%s\n' % ' '.join([str(item[1]) for item in logs])) def cuda(self, *args): return (item.to(self.config.DEVICE) for item in args) def postprocess(self, img): # [0, 1] => [0, 255] img = img * 255.0 img = img.permute(0, 2, 3, 1) return img.int() def color_the_edge(self, img, edges, masks): img = img.expand(-1, 3, -1, -1) yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255. ]).reshape(1, 3, 1, 1)).to(self.config.DEVICE) yellow = img * (1 - masks) * yellow_v img = yellow + (edges * masks) return img
def walk_through(self): if not self.to_top(): Logger.log("GoogleMusicApp", "walk_through failed: unable to go to top activity") self.cache_init = False return False # Get the playcard titles vc = ViewClient(self.device, self.serialno) self.cache_init = True container_key = GoogleMusicApp.CONTAINER_KEY container = [v for v in vc.getViewsById().values() if v.getId() == container_key] container = container[0] if len(container) > 0 else None if container: self.cache["screen-info"] = container.getBounds()[1] self.push_dump("screen-info: {}".format(self.cache["screen-info"])) so = sio.StringIO() vc.traverse(stream=so) lines = so.getvalue().splitlines() play_card_key = GoogleMusicApp.PLAY_CARD_KEY playcards_idices = [idx for idx, line in enumerate(lines) if play_card_key in line] playcards_idices.append(len(lines)) playcards_titles = [] last_idx = playcards_idices[0] li_title_key = GoogleMusicApp.LI_TITLE_KEY for idx in playcards_idices[1:]: li_title_texts = [line for line in lines[last_idx:idx] if li_title_key in line] last_idx = idx if len(li_title_texts) != 1: self.push_dump("li_title_texts has length {}".format(len(li_title_texts))) playcards_titles.append(utf8str(li_title_texts[0].split(li_title_key)[-1].strip())) self.push_dump("playcards_titles.append('{}')".format(playcards_titles[-1])) # Get the track list of each playcard views = [v for v in vc.getViewsById().values() if v.getId() == li_title_key and utf8str(v.getText()) in playcards_titles] self.cache["playcard"] = dict( \ map(lambda v: (utf8str(v.getText()), { "position": v.getCenter() }), views) ) map(lambda v: self.push_dump("view: {}".format(utf8str(v))), views) map(lambda title: self.push_dump("playcard title: '{}'".format(title)), self.cache["playcard"].keys()) if len(views) == 0: self.cache_init = False return False self.cache["shuffle_key"] = playcards_titles[0] self.push_dump("get the shuffle keyword '{}'".format(self.cache["shuffle_key"])) self.touch_playcard(self.cache["shuffle_key"]) time.sleep(1) retry_count = 3 while retry_count > 0: vc.dump() play_pause_header_key = GoogleMusicApp.PLAY_PAUSE_HEADER_KEY play_pause_btn_view = [v for v in vc.getViewsById().values() if v.getId() == play_pause_header_key] play_pause_btn_view = play_pause_btn_view[0] if len(play_pause_btn_view) > 0 else None if play_pause_btn_view: play_desc = utf8str(play_pause_btn_view.getContentDescription()) self.check_play_status = lambda desc: desc == play_desc self.cache["play_pause_btn"] = { "position": play_pause_btn_view.getCenter(), "desc_feat": play_desc } art_pager_key = GoogleMusicApp.ART_PAGER_KEY art_pager_view = [v for v in vc.getViewsById().values() if v.getId() == art_pager_key] art_pager_view = art_pager_view[0] if len(art_pager_view) > 0 else None if not art_pager_view: retry_count -= 1 continue self.cache["art_pager_view"] = { "position": art_pager_view.getCenter() } play_pause_btn_view.touch() break else: self.push_dump("cannot find the play/pause button, retry: {}".format(retry_count)) retry_count -= 1 if retry_count == 0: return False for li_title in self.cache["playcard"].keys(): if li_title == self.cache["shuffle_key"]: continue self.push_dump("now fetching information in the playcard '{}'".format(li_title)) if self.touch_playcard(li_title=li_title): time.sleep(1) self.cache["playcard"][li_title]["songs"] = self._fetch_songs() self.to_top() # Get the information of the control panel retry_count = 3 while self.get_state() != GoogleMusicApp.State.CONTROL_PANEL and retry_count > 0: self.device.touch(*self.cache["art_pager_view"]["position"]) retry_count -= 1 if retry_count == 0 and self.get_state() != GoogleMusicApp.State.CONTROL_PANEL: self.to_top() time.sleep(5) self.touch_playcard(self.cache["shuffle_key"]) time.sleep(2) self.device.touch(*self.cache["play_pause_btn"]["position"]) time.sleep(2) self.device.touch(*self.cache["art_pager_view"]["position"]) time.sleep(2) if self.get_state() != GoogleMusicApp.State.CONTROL_PANEL: self.push_dump("cannot get the information of the control panel") self.cache_init = False return False def find_view_position(vc, res_id): v = [v for v in vc.getViewsById().values() if v.getId() == res_id] if len(v) == 0: return ((-1, -1), (-1, -1)), (-1, -1) return v[0].getBounds(), v[0].getCenter() vc.dump() progress_bounds, progress_pos = find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PROGRESS_KEY) self.cache["control_panel"] = { "progress": { "position": progress_pos, "xbounds": [progress_bounds[0][0], progress_bounds[1][0]] }, "prev": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PREV_KEY)[1] }, "next": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_NEXT_KEY)[1] }, "play_pause": { "position": find_view_position(vc, GoogleMusicApp.CONTROL_PANEL_PLAY_PAUSE_KEY)[1] } } self.control_panel = GMControlPanel(self) self.push_dump("successfully walked through, now back to top") self.to_top() self.cache_init = True return True
#!/usr/bin/env python3 import error_handling_library.test_hooks as ut from libs.logger import Logger LOGGER = Logger() ut.init( logger = LOGGER ) import time, os, argparse, sys, requests, gevent, grequests from gevent import socket, Timeout from libs.dataGiver import DataGiver from libs.outputDataIntoDict import Outputter from libs.resolver import Resolver parser = argparse.ArgumentParser(description = """Ping some ips and hosts ;)""") parser.add_argument("-c", "--configure", help="Give the ip json file", type = str, default = False) parser.add_argument("-s", "--stats", help="Give some statistics on the screen!", action = "store_true", default = False) parser.add_argument("-d", "--directory", help="Put the output in written directory. WIll be created if not there.", type = str, default = False) args = parser.parse_args() STATS = args.stats versionNumber = "3.0" applicationName = "Momo's Pinger" defaultSearchFolder = "/etc/tbmon/defaultHTMLJSON" defaultOutputFolder = "output_data" searchFolder = args.configure or defaultSearchFolder
def run(self): shared_vars = {"start_time": None, "last_event": None, "last_freq": -1} self.extra = {} self.extra["adb-read-prop-max-elapsed"] = -1 self.extra["freq-cb-max-elapsed"] = -1 self.extra["dump"] = [] self.extra["dump-lock"] = threading.Lock() def freq_cb(msg): line = msg.splitlines()[0] strs = line.split() freq, amp_db = list(map(float, strs[-1].split(","))) the_date, the_time = strs[:2] time_str = the_date + " " + the_time if shared_vars["last_freq"] != freq: self.push_to_dump( \ "the detected freq has been changed from {} to {} Hz".format(shared_vars["last_freq"], freq)) shared_vars["last_freq"] = freq thresh = 10 if self.target_freq else 1 if super(AATAppToneDetectorThread, self).target_detected(freq): self.event_counter += 1 if self.event_counter == 1: shared_vars["start_time"] = time_str if self.event_counter == thresh: if not shared_vars["last_event"] or shared_vars[ "last_event"] != ToneDetector.Event.TONE_DETECTED: Logger.log( self.get_tag(), "send_cb({}, TONE_DETECTED)".format( shared_vars["start_time"])) self.cb((shared_vars["start_time"], ToneDetector.Event.TONE_DETECTED)) shared_vars[ "last_event"] = ToneDetector.Event.TONE_DETECTED else: if self.event_counter > thresh: shared_vars["start_time"] = None self.push_to_dump( "the tone is not detected and the event_counter is over the threshold" ) self.push_to_dump("last_event: \"{}\"".format( shared_vars["last_event"])) if not shared_vars["last_event"] or shared_vars[ "last_event"] != ToneDetector.Event.TONE_MISSING: Logger.log(self.get_tag(), "send_cb({}, TONE_MISSING)".format(time_str)) self.cb((time_str, ToneDetector.Event.TONE_MISSING)) shared_vars["last_event"] = ToneDetector.Event.TONE_MISSING self.event_counter = 0 if self.event_counter <= thresh: self.push_to_dump("event_counter: {}".format( self.event_counter)) # Adb.execute(cmd= \ # ["shell", "am", "broadcast", "-a", "audio.htc.com.intent.print.properties.enable", "--ez", "v", "1"], \ # serialno=self.serialno) from libs.timeutils import TicToc, TimeUtils freq_cb_tictoc = TicToc() adb_tictoc = TicToc() tcount = 0 freq_cb_tictoc.tic() while not self.stoprequest.isSet(): adb_tictoc.tic() msg, _ = Adb.execute(cmd=["shell", "cat", "sdcard/AudioFunctionsDemo-record-prop.txt"], \ serialno=self.serialno, tolog=False) elapsed = adb_tictoc.toc() if tcount == 0: Adb.execute(cmd=["shell", "rm", "-f", "sdcard/AudioFunctionsDemo-record-prop.txt"], \ serialno=self.serialno, tolog=False) if not "," in msg: msg = "0,-30" if elapsed > self.extra["adb-read-prop-max-elapsed"]: self.extra["adb-read-prop-max-elapsed"] = elapsed if "," in msg: msg = msg.replace("\n", "") import datetime msg = "{} {}".format(TimeUtils.now_str(), msg) try: self.push_to_dump("{} (adb-shell elapsed: {} ms)".format( msg, elapsed)) freq_cb(msg) except Exception as e: Logger.log(self.get_tag(), "crashed in freq_cb('{}')".format(msg)) print(e) elapsed = freq_cb_tictoc.toc() if elapsed > self.extra["freq-cb-max-elapsed"]: self.extra["freq-cb-max-elapsed"] = elapsed time.sleep(0.01) tcount += 1 tcount %= 10
img_title = purifyName(tit[0]) win32api.SetConsoleCtrlHandler(onClose, True) downnum, failnum = download(title=img_title, links=links[index][0], names=names[index][0], downnum=DOWN_NUM, failnum=FAIL_NUM, delta=delta, done=done) index += 1 DOWN_NUM += downnum FAIL_NUM += failnum except (BaseException, Exception) as e: os.chdir(MAIN_PATH) lastTitle = MAIN_PATH + '\\Download\\' + purifyName(END_NAME) rmFile(lastTitle) with open('last.txt', 'w') as f: f.write(END_NAME) err_log = Logger(log_name='Error.log', logger_name='err').get_logger() err_log.error(e) finally: if os.getcwd() != MAIN_PATH: os.chdir(MAIN_PATH) output_log = Logger(log_name='Output.log', logger_name='output').get_logger() output_log.info('下载完成%d张' % DOWN_NUM) output_log.info('下载失败%d张' % FAIL_NUM) print('>>>The log file in %s.<<<' % MAIN_PATH) print('Done!') input('\n\nPress any key to quit.')
def main(): Logger.init(Logger.Mode.STDOUT) Adb.init() trial_num = 1 pass_trial_num = 0 try: while True: Adb.execute(["shell", "input", "keyevent", "POWER"]) Adb.execute([ "shell", AATApp.INTENT_PREFIX, AATApp.HTC_INTENT_PREFIX + "record.start" ]) time.sleep(2) log("play 220Hz_44100Hz_15secs_2ch.wav") os.system( "adb shell tinyplay /data/220Hz_44100Hz_15secs_2ch.wav > /dev/null &" ) retry = 5 detected = False while retry > 0: out, err = Adb.execute([ "shell", "cat", "/storage/emulated/0/AudioFunctionsDemo-record-prop.txt" ], tolog=False) try: out = float(out.split(",")[0]) except: out = 0.0 if out > 200.0 and out < 240.0: log("detected tones.") detected = True break time.sleep(0.1) time.sleep(1) log("turn off the screen.") Adb.execute(["shell", "input", "keyevent", "POWER"]) time.sleep(2) retry = 5 detected = False detected_cnt = 0 while retry > 0: out, err = Adb.execute([ "shell", "cat", "/storage/emulated/0/AudioFunctionsDemo-record-prop.txt" ], tolog=False) try: out = float(out.split(",")[0]) except: out = 0.0 if out > 200.0 and out < 240.0: log("detected tones.") detected_cnt += 1 if detected_cnt == 10: log("detected for a sufficient times.") detected = True break time.sleep(0.1) if detected: pass_trial_num += 1 log("trial #{}: passed. ({}/{})".format( trial_num, pass_trial_num, trial_num)) else: log("trial #{}: failed. ({}/{})".format( trial_num, pass_trial_num, trial_num)) trial_num += 1 time.sleep(15) except: pass Logger.finalize()
class EdgeConnect(): def __init__(self, config): self.config = config if config.MODEL == 1: model_name = 'edge' elif config.MODEL == 2: model_name = 'inpaint' elif config.MODEL == 3: model_name = 'edge_inpaint' elif config.MODEL == 4: model_name = 'joint' self.debug = False self.model_name = model_name self.edge_model = EdgeModel(config).to(config.DEVICE) self.inpaint_model = InpaintingModel(config).to(config.DEVICE) self.psnr = PSNR(255.0).to(config.DEVICE) self.ssim = SSIM(window_size=11) self.edgeacc = EdgeAccuracy(config.EDGE_THRESHOLD).to(config.DEVICE) val_sample = int(float((self.config.EVAL_INTERVAL))) # test mode if self.config.MODE == 2: self.test_dataset = Dataset(config, config.TEST_FLIST, config.TEST_EDGE_FLIST, config.TEST_MASK_FLIST, augment=False, training=False) else: self.train_dataset = Dataset(config, config.TRAIN_FLIST, config.TRAIN_EDGE_FLIST, config.TRAIN_MASK_FLIST, augment=True, training=True) self.val_dataset = Dataset(config, config.VAL_FLIST, config.VAL_EDGE_FLIST, config.VAL_MASK_FLIST, augment=False, training=True, sample_interval=val_sample) self.sample_iterator = self.val_dataset.create_iterator( config.SAMPLE_SIZE) self.samples_path = os.path.join(config.PATH, 'samples') self.results_path = os.path.join(config.PATH, 'results') if config.RESULTS is not None: self.results_path = os.path.join(config.RESULTS) if config.DEBUG is not None and config.DEBUG != 0: self.debug = True # self.log_file = os.path.join(config.PATH, 'log_' + model_name + '.dat') log_path = os.path.join(config.PATH, 'logs_' + model_name) create_dir(log_path) self.logger = Logger(log_path) def load(self): if self.config.MODEL == 1: self.edge_model.load() elif self.config.MODEL == 2: self.inpaint_model.load() else: self.edge_model.load() self.inpaint_model.load() def save(self, epoch): if self.config.MODEL == 1: self.edge_model.save(epoch) elif self.config.MODEL == 2 or self.config.MODEL == 3: self.inpaint_model.save(epoch) else: self.edge_model.save(epoch) self.inpaint_model.save(epoch) def train(self): train_loader = DataLoader(dataset=self.train_dataset, batch_size=self.config.BATCH_SIZE, num_workers=4, drop_last=True, shuffle=True) keep_training = True model = self.config.MODEL # max_iteration = int(float((self.config.MAX_ITERS))) step_per_epoch = int(float((self.config.MAX_STEPS))) max_epoches = int(float((self.config.MAX_EPOCHES))) total = int(len(self.train_dataset)) if total == 0: print( 'No training data was provided! Check \'TRAIN_FLIST\' value in the configuration file.' ) return print('\nThe number of Training data is %d' % total) if model == 1: epoch = self.edge_model.epoch + 1 if self.edge_model.epoch != None else 1 else: epoch = self.inpaint_model.epoch + 1 if self.inpaint_model.epoch != None else 1 print('\nTraining epoch: %d' % epoch) progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) while (keep_training): logs_ave = {} for items in train_loader: self.edge_model.train() # set the model to train mode self.inpaint_model.train() images, images_gray, edges, masks = self.cuda(*items) # edge model if model == 1: # train outputs, gen_loss, dis_loss, logs = self.edge_model.process( images_gray, edges, masks) # metrics precision, recall = self.edgeacc(edges * (1 - masks), outputs * (1 - masks)) logs['precision'] = precision.item() logs['recall'] = recall.item() # backward self.edge_model.backward(gen_loss, dis_loss) if self.edge_model.iteration > step_per_epoch: self.edge_model.iteration = 0 iteration = self.edge_model.iteration # inpaint model elif model == 2: # train outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, edges, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['psnr'] = psnr.item() logs['mae'] = mae.item() # backward self.inpaint_model.backward(gen_loss, dis_loss) if self.inpaint_model.iteration > step_per_epoch: self.inpaint_model.iteration = 0 iteration = self.inpaint_model.iteration # inpaint with edge model elif model == 3: # train if True or np.random.binomial(1, 0.5) > 0: edge_outputs = self.edge_model(images_gray, edges, masks).detach() edge_outputs = (edge_outputs * (1 - masks)) + (edges * (masks)) else: edge_outputs = edges outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, edge_outputs.detach(), masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['psnr'] = psnr.item() logs['mae'] = mae.item() # backward self.inpaint_model.backward(gen_loss, dis_loss) if self.inpaint_model.iteration > step_per_epoch: self.inpaint_model.iteration = 0 iteration = self.inpaint_model.iteration # joint model else: # train e_outputs, e_gen_loss, e_dis_loss, e_logs = self.edge_model.process( images_gray, edges, masks) e_outputs = e_outputs * (1 - masks) + edges * (masks) i_outputs, i_gen_loss, i_dis_loss, i_logs = self.inpaint_model.process( images, e_outputs, masks) outputs_merged = (i_outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() precision, recall = self.edgeacc(edges * (1 - masks), e_outputs * (1 - masks)) i_logs['psnr'] = psnr.item() i_logs['mae'] = mae.item() i_logs['psnr'] = psnr.item() i_logs['mae'] = mae.item() logs = {**e_logs, **i_logs} # backward self.inpaint_model.backward(i_gen_loss, i_dis_loss) self.edge_model.backward(e_gen_loss, e_dis_loss) if self.inpaint_model.iteration > step_per_epoch: self.inpaint_model.iteration = 0 iteration = self.inpaint_model.iteration if iteration == 1: # first time to train for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value if iteration == 0: # mean to jump to new epoch self.sample(epoch) self.eval(epoch) self.save(epoch) # log current epoch in tensorboard for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / step_per_epoch, epoch) # max epoch if epoch >= max_epoches: keep_training = False break epoch += 1 # new epoch print('\n\nTraining epoch: %d' % epoch) for tag, value in logs.items(): logs_ave[tag] = value progbar = Progbar(step_per_epoch, width=30, stateful_metrics=['step']) self.inpaint_model.iteration += 1 self.edge_model.iteration += 1 iteration += 1 logs['step'] = iteration progbar.add( 1, values=logs.items() if self.config.VERBOSE else [x for x in logs.items() if not x[0].startswith('l_')]) print('\nEnd training....\n') def eval(self, epoch): self.val_loader = DataLoader(dataset=self.val_dataset, batch_size=self.config.BATCH_SIZE, drop_last=True, shuffle=False, num_workers=4) model = self.config.MODEL total = int(len(self.val_dataset)) self.edge_model.eval() self.inpaint_model.eval() progbar = Progbar(int(total / self.config.BATCH_SIZE), width=30, stateful_metrics=['step']) iteration = 0 with torch.no_grad(): for items in self.val_loader: iteration += 1 images, images_gray, edges, masks = self.cuda(*items) # edge model if model == 1: # eval outputs, gen_loss, dis_loss, _ = self.edge_model.process( images_gray, edges, masks) logs = {} logs['l_val_d1'] = dis_loss.item() logs['l_val_g1'] = gen_loss.item() # metrics precision, recall = self.edgeacc(edges * (1 - masks), outputs * (1 - masks)) logs['val_precision'] = precision.item() logs['val_recall'] = recall.item() # inpaint model elif model == 2: # eval outputs, gen_loss, dis_loss, logs = self.inpaint_model.process( images, edges, masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs['val_psnr'] = psnr.item() logs['val_mae'] = mae.item() # inpaint with edge model elif model == 3: # eval edge_outputs = self.edge_model(images_gray, edges, masks) edge_outputs = edge_outputs * (1 - masks) + edges * (masks) outputs, gen_loss, dis_loss, log_origin = self.inpaint_model.process( images, edge_outputs.detach(), masks) outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # metrics # "l_d2": dis_loss.item(), # "l_g2": gen_gan_loss.item(), # "l_l1": gen_l1_loss.item(), # "l_per": gen_content_loss.item(), # "l_sty": gen_style_loss.item() psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() logs = {} # logs['l_val_l1'] = log_origin['l_l1'] logs['l_val_g2'] = log_origin['l_g2'] logs['l_val_d2'] = log_origin['l_d2'] # logs['l_val_per'] = log_origin['l_per'] # logs['l_val_sty'] = log_origin['l_sty'] logs['val_psnr'] = psnr.item() logs['val_mae'] = mae.item() # joint model else: # eval e_outputs, e_gen_loss, e_dis_loss, e_logs = self.edge_model.process( images_gray, edges, masks) e_outputs = e_outputs * (1 - masks) + edges * masks i_outputs, i_gen_loss, i_dis_loss, i_logs = self.inpaint_model.process( images, e_outputs, masks) outputs_merged = (i_outputs * (1 - masks)) + (images * masks) # metrics psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() precision, recall = self.edgeacc(edges * (1 - masks), e_outputs * (1 - masks)) i_logs['val_precision'] = precision.item() i_logs['val_recall'] = recall.item() i_logs['val_psnr'] = psnr.item() i_logs['val_mae'] = mae.item() logs = {**e_logs, **i_logs} if iteration == 1: # first iteration logs_ave = {} for tag, value in logs.items(): logs_ave[tag] = value else: for tag, value in logs.items(): logs_ave[tag] += value logs["step"] = iteration progbar.add(1, values=logs.items()) for tag, value in logs_ave.items(): self.logger.scalar_summary(tag, value / iteration, epoch) self.edge_model.iteration = 0 self.inpaint_model.iteration = 0 def test(self): self.edge_model.eval() self.inpaint_model.eval() damaged_dir = os.path.join(self.results_path, "damaged") create_dir(damaged_dir) mask_dir = os.path.join(self.results_path, "mask") create_dir(mask_dir) inpainted_dir = os.path.join(self.results_path, "inpainted") create_dir(inpainted_dir) damaged_edge_dir = os.path.join(self.results_path, "damaged_edge") create_dir(damaged_edge_dir) raw_dir = os.path.join(self.results_path, "raw") create_dir(raw_dir) model = self.config.MODEL create_dir(self.results_path) sample_interval = 1000 batch_size = 1 test_loader = DataLoader(dataset=self.test_dataset, batch_size=batch_size, num_workers=12, shuffle=False) total = int(len(self.test_dataset)) progbar = Progbar(int(total / batch_size / sample_interval), width=30, stateful_metrics=['step']) index = 0 with torch.no_grad(): for items in test_loader: name = self.test_dataset.load_name(index) images, images_gray, edges, masks = self.cuda(*items) path = os.path.join(damaged_dir, name) damaged_img = self.postprocess(images * masks + (1 - masks))[0] imsave(damaged_img, path) path = os.path.join(damaged_edge_dir, name) damaged_img = self.postprocess(edges * masks + (1 - masks))[0] imsave(damaged_img, path) path = os.path.join(mask_dir, name) imsave(self.postprocess(masks), os.path.splitext(path)[0] + '.png') path = os.path.join(raw_dir, name) img = self.postprocess(images)[0] imsave(img, path) # print(index, name) index += 1 if index > total / batch_size / sample_interval: break logs = {} # edge model if model == 1: outputs = self.edge_model(images_gray, edges, masks) outputs_merged = (outputs * (1 - masks)) + (edges * masks) # inpaint model elif model == 2: outputs = self.inpaint_model(images, edges, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) # edge-inpaint model elif model == 3: outputs = self.edge_model(images_gray, edges, masks).detach() outputs_merged = (outputs * (1 - masks)) + (edges * masks) edge_save = self.color_the_edge(outputs, edges, masks) edge_save = self.postprocess(edge_save)[0] path = os.path.join(self.results_path + "/edge_inpainted", name) # print(index, name) imsave(edge_save, path) outputs = self.inpaint_model(images, outputs_merged, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) # joint model else: edges = self.edge_model(images_gray, edges, masks).detach() outputs = self.inpaint_model(images, edges, masks) outputs_merged = (outputs * (1 - masks)) + (images * masks) output = self.postprocess(outputs_merged)[0] path = os.path.join(inpainted_dir, name) # print(index, name) imsave(output, path) psnr = self.psnr(self.postprocess(images), self.postprocess(outputs_merged)) mae = (torch.sum(torch.abs(images - outputs_merged)) / torch.sum(images)).float() one_ssim = self.ssim(images, outputs_merged) logs["psnr"] = psnr.item() logs["mae"] = mae.item() logs["ssim"] = one_ssim.item() logs["step"] = index progbar.add(1, values=logs.items()) if self.debug: edges = self.postprocess(1 - edges)[0] masked = self.postprocess(images * (masks) + (1 - masks))[0] fname, fext = name.split('.') imsave( edges, os.path.join(self.results_path, fname + '_edge.' + fext)) imsave( masked, os.path.join(self.results_path, fname + '_masked.' + fext)) print('\nEnd test....') def sample(self, it=None): # do not sample when validation set is empty if len(self.val_dataset) == 0: return self.edge_model.eval() self.inpaint_model.eval() model = self.config.MODEL with torch.no_grad(): items = next(self.sample_iterator) images, images_gray, edges, masks = self.cuda(*items) # (batch,channels,weighth,length) # edge model if model == 1: iteration = self.edge_model.iteration inputs = (edges * masks) + (1 - masks) outputs = self.edge_model(images_gray, edges, masks).detach() outputs = self.color_the_edge(outputs, edges, masks) # outputs_merged = (outputs * (1 - masks)) + (edges * masks) # inpaint model elif model == 2: iteration = self.inpaint_model.iteration inputs = (images * masks) + (1 - masks) outputs = self.inpaint_model(images, edges, masks).detach() # outputs_merged = (outputs * (1 - masks)) + (images * (masks)) # inpaint with edge model / joint model else: iteration = self.inpaint_model.iteration inputs = (images * (masks)) + (1 - masks) outputs = self.edge_model(images_gray, edges, masks).detach() edge_merged = (outputs * (1 - masks) + edges * (masks)).detach() edges = self.color_the_edge(outputs, edges, masks).detach() outputs = self.inpaint_model(images, edge_merged, masks) # outputs_merged = (outputs * (1 - masks)) + (images * (masks)) if it is not None: iteration = it image_per_row = 2 if self.config.SAMPLE_SIZE <= 6: image_per_row = 1 images = stitch_images( self.postprocess(images), self.postprocess(edges), self.postprocess(inputs), self.postprocess(outputs), # self.postprocess(outputs_merged), img_per_row=image_per_row) path = os.path.join(self.samples_path, self.model_name) name = os.path.join(path, str(iteration).zfill(3) + ".png") create_dir(path) print('\nsaving sample ' + name) images.save(name) def log(self, logs): with open(self.log_file, 'a') as f: f.write('%s\n' % ' '.join([str(item[1]) for item in logs])) def cuda(self, *args): return (item.to(self.config.DEVICE) for item in args) def postprocess(self, img): # [0, 1] => [0, 255] img = img * 255.0 img = img.permute(0, 2, 3, 1) return img.int() def color_the_edge(self, img, edges, masks): img = img.expand(-1, 3, -1, -1) yellow_v = (torch.tensor([215. / 255., 87. / 255., 15. / 255. ]).reshape(1, 3, 1, 1)).to(self.config.DEVICE) yellow = img * (1 - masks) * yellow_v img = yellow + (edges * masks) return img
def run(num_iter, serialno1, phoneno1, serialno2, phoneno2): Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT) # Logger.init(Logger.Mode.STDOUT) Adb.init() for serialno in [serialno1, serialno2]: Adb.execute(["root"], serialno=serialno) Adb.execute(["shell", "'echo \'related\' > msm_subsys'"], serialno=serialno) Adb.execute(["shell", "svc", "power", "stayon", "true"], serialno=serialno) out, err = Adb.execute(["shell", "getprop", "ro.vendor.build.fingerprint"], serialno=serialno) out = out.strip() log("build number: '{}'".format(out)) latest_dmesg1 = fetch_dmesg(serialno=serialno1) latest_dmesg2 = fetch_dmesg(serialno=serialno2) latest_dmesg1 = latest_dmesg1[0] if len(latest_dmesg1) > 0 else None latest_dmesg2 = latest_dmesg2[0] if len(latest_dmesg2) > 0 else None phone_dict = { serialno1: phoneno1, serialno2: phoneno2 } last_dmesg_dict = { serialno1: latest_dmesg1, serialno2: latest_dmesg2 } mt_serialno = serialno1 mo_serialno = serialno2 pass_trial_cnt = 0 total_trial_cnt = 0 invalid_trial_cnt = 0 failed_trial_cnt = 0 try: for i in range(int(num_iter)): log("-------------------- Dual-CS-call-test #{} --------------------".format(i+1)) log("{} makes call to {} ({})".format(mo_serialno, mt_serialno, phone_dict[mt_serialno])) start_cs_call(tel=phone_dict[mt_serialno], serialno=mo_serialno) if not wait_for_phone_state(state=1, timeout=30, serialno=mt_serialno): log("There is no incoming call to {}, next trial".format(mt_serialno)) end_cs_call(serialno=mo_serialno) end_cs_call(serialno=mt_serialno) invalid_trial_cnt += 1 continue log("{} picks up the call".format(mt_serialno)) pick_cs_call(serialno=mt_serialno) if not wait_for_phone_state(state=2, timeout=10, serialno=mo_serialno): log("{} is not in phone state 'MODE_INCALL', next trial".format(mo_serialno)) end_cs_call(serialno=mo_serialno) end_cs_call(serialno=mt_serialno) invalid_trial_cnt += 1 continue if not wait_for_phone_state(state=2, timeout=10, serialno=mt_serialno): log("{} is not in phone state 'MODE_INCALL', next trial".format(mt_serialno)) end_cs_call(serialno=mo_serialno) end_cs_call(serialno=mt_serialno) invalid_trial_cnt += 1 continue log("To check if ADSP crashes during the CS call") is_passed = True pass_trial_cnt += 1 total_trial_cnt += 1 retry = 15 while retry > 0: for serialno in [mt_serialno, mo_serialno]: latest_dmesg = last_dmesg_dict[serialno] new_dmesgs = fetch_dmesg(latest_dmesg, serialno=serialno) if len(new_dmesgs) > 0: last_dmesg_dict[serialno] = new_dmesgs[-1] adsp_ssr_demsgs = filter(lambda x: "Restart sequence requested for adsp" in x.raw, new_dmesgs) if len(adsp_ssr_demsgs) > 0: for dmesg in adsp_ssr_demsgs: log("SSR log detected in {}: '{}'".format(serialno, dmesg.raw)) is_passed = False break phone_state = get_phone_state(serialno=serialno) if phone_state == None: log("the phone state of {} is unobtainable, something wrong".format(serialno)) is_passed = False break if phone_state == 0: log("the phone state of {} is in idle, the call might be dropped".format(serialno)) is_passed = False break if not is_passed: for serialno in [mt_serialno, mo_serialno]: out, err = Adb.execute(["bugreport"], serialno) log("bugreport to '{}'".format(out.strip())) pass_trial_cnt -= 1 failed_trial_cnt += 1 for serialno in [mt_serialno, mo_serialno]: end_cs_call(serialno=serialno) break if retry % 5 == 0: log("{} switches path to '{}'".format(mo_serialno, "speaker" if retry/5 % 2 == 1 else "receiver")) Adb.execute(["shell", "input", "tap", "1000", "1000"], tolog=False, serialno=mo_serialno) retry -= 1 time.sleep(1) log("result: {} ({}/{})".format("pass" if retry == 0 else "fail", pass_trial_cnt, total_trial_cnt)) log("pass: {}% ({}/{}), fail: {}% ({}/{}), invalid: {}% ({}/{})".format( pass_trial_cnt*100.0/total_trial_cnt, pass_trial_cnt, total_trial_cnt, failed_trial_cnt*100.0/total_trial_cnt, failed_trial_cnt, total_trial_cnt, invalid_trial_cnt*100.0/(i+1), invalid_trial_cnt, i+1)) for serialno in [mo_serialno, mt_serialno]: end_cs_call(serialno=serialno) wait_for_phone_state(state=0, timeout=10, serialno=serialno) time.sleep(5) except: pass for serialno in [serialno1, serialno2]: end_cs_call(serialno=serialno) Adb.execute(["shell", "svc", "power", "stayon", "false"], serialno=serialno) Logger.finalize()
def collect(module, url_queue): #Excute the collect process Logger.info('Start Collector') #Send the url queue to bot via args module.collect(url_queue) Logger.info('Collect done')
def run(num_iter=1): AudioFunction.init() Logger.init(Logger.Mode.BOTH_FILE_AND_STDOUT) Adb.init() os.system("mkdir -p {}{}{} > {}".format(ROOT_DIR, SEP, REPORT_DIR, STDNUL)) t = datetime.datetime.now() filename = "report_{}{:02d}{:02d}_{:02d}{:02d}{:02d}.json".format( t.year, t.month, t.day, t.hour, t.minute, t.second) device, serialno = ViewClient.connectToDeviceOrExit(serialno=None) device.press("HOME") time.sleep(1) gmhandler = GoogleMusicApp(device, serialno) log("gmhandler.to_top()") gmhandler.to_top() clear_and_update_music_files(serialno) time.sleep(10) trials = [] batch_count = 1 while num_iter > 0: log("-------- batch_run #{} --------".format(batch_count)) trials_batch = [] trials_batch += playback_task_run(num_iter=min([num_iter, BATCH_SIZE]), num_seek_test=5, gmhandler=gmhandler) trials_batch += record_task_run(device, serialno, num_iter=min([num_iter, BATCH_SIZE]), num_freqs=5) map(lambda trial: trial.put_extra(name="batch_id", value=batch_count), trials_batch) trials += trials_batch with open( "{}{}{}{}{}".format(ROOT_DIR, SEP, REPORT_DIR, SEP, filename), "w") as f: f.write(TrialHelper.to_json(trials)) for taskname, tasktrials in TrialHelper.categorize_in( trials, lambda t: t.ds["task"]).items(): valid_trials = zip( tasktrials, TrialHelper.pass_fail_list( tasktrials, lambda t: t.ds["status"] == "valid")) valid_trials = [ trial for trial, isvalid in valid_trials if isvalid ] num_valid = len(valid_trials) num_pass = len( filter(lambda x: x, TrialHelper.pass_fail_list(valid_trials))) log("task[{}] valid trials: {}/{}, pass trials: {}/{}".format( taskname, num_valid, len(tasktrials), num_pass, num_valid)) num_iter -= BATCH_SIZE batch_count += 1 AudioFunction.finalize() Logger.finalize()
def tone_detected_event_cb(self, event): Logger.log("DetectionStateChangeListenerThread", "tone_detected_event_cb: {}".format(event)) self._handle_event(event)
def log(msg): Logger.log(GLOBAL["tag"], msg)
# coding: utf-8 # 导入libs下的logger日志模块 from libs.logger import Logger # 导入selenium中的webdriver from selenium import webdriver # 导入os模块,获取文件路径 import os __author__ = "sunxr" __version__ = "V1.2" logger = Logger("Browser").getLog() def browser(browser_type): """ 打开浏览器. :param browser_type: 浏览器类型 :return: driver """ # 获取框架主路径信息 home_path = os.path.dirname(os.path.dirname(__file__)) # 定义浏览器驱动的路径 chrome_driver_path = home_path + '/tools/chromedriver' firefox_driver_path = home_path + '/tools/geckodriver' safari_driver_path = home_path + '/tools/safaridriver' try: if browser_type == "firefox":
def log(msg): Logger.log(TAG, msg)
def __init__(self): self.log = Logger() self.db = DB().client
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'valf') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) data_aug_scale = (0.08, 1.0) if args.modelsize == 'large' else (0.2, 1.0) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomResizedCrop(224, scale = data_aug_scale), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=True, num_workers=args.workers, pin_memory=True) # create model # if args.pretrained: # print("=> using pre-trained model '{}'".format(args.arch)) # model = models.__dict__[args.arch](pretrained=True) # elif 'resnext' in args.arch: # model = models.__dict__[args.arch]( # baseWidth=args.base_width, # cardinality=args.cardinality, # ) # else: # print("=> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = se_resnet101().cuda() flops, params = get_model_complexity_info(model, (224, 224), as_strings=False, print_per_layer_stat=False) print('Flops: %.3f' % (flops / 1e9)) print('Params: %.2fM' % (params / 1e6)) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..', args.resume) assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] # model may have more keys t = model.state_dict() c = checkpoint['state_dict'] flag = True for k in t: if k not in c: print('not in loading dict! fill it', k, t[k]) c[k] = t[k] flag = False model.load_state_dict(c) if flag: print('optimizer load old state') optimizer.load_state_dict(checkpoint['optimizer']) else: print('new optimizer !') logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() print('Best acc:') print(best_acc)
def tone_detected_event_cb(self, event): Logger.log(self.get_tag(), "tone_detected_event_cb: {}".format(event)) self._handle_event(event)
class QueueSystem: def __init__(self): self.db = sqlite3.connect('queue.db') self.log = Logger(log_level="DEBUG").get_logger() def build_game(self, min_players, max_players, game_id): cursor = self.db.cursor() cursor.execute("select player_id, queue_time, claimed, in_game, game_id from queue where claimed is 0 " "order by queue_time") results = cursor.fetchall() queue_count = len(results) player_count = min(queue_count, max_players) if queue_count >= min_players: for x in range(0, player_count): player_id = results[x][0] cursor.execute("update queue set claimed=?, game_id=?, player_count=? where player_id=?", (1, game_id, player_count, player_id)) self.log.debug("{} is being added to game_id {}".format(player_id, game_id)) self.db.commit() def build_db(self): cursor = self.db.cursor() cursor.execute("create table queue(player_id text, queue_time int, claimed boolean, in_game boolean, " "game_id int, player_count int)") self.db.commit() def dequeue_player(self, player_id): cursor = self.db.cursor() cursor.execute("delete from queue where player_id=?", (player_id,)) self.log.debug("Removing player_id {} from the queue.".format(player_id)) self.db.commit() def queue_player(self, player_id): current_time = time.time() cursor = self.db.cursor() cursor.execute("select player_id from queue where player_id=?", (player_id,)) result = cursor.fetchone() if result is None: cursor.execute("insert into queue values(?, ?, 0, 0, NULL, NULL)", (player_id, current_time)) self.db.commit() self.log.debug("Queueing player_id {} at {}".format(player_id, current_time)) def start_game(self, game_id): cursor = self.db.cursor() cursor.execute("SELECT * from queue where game_id=?", (game_id,)) game_ready = True player_list = cursor.fetchall() player_count = len(player_list) player_id_list = [] for player in player_list: if player[3] == 0: game_ready = False self.log.debug("Game not ready because player_id {} has not confirmed the game".format(player[0])) elif player_count < player[5]: game_ready = False self.log.debug("Game not ready because {} does not match the required count of {}" .format(player_count, player[5])) else: player_id_list.append(player[0]) if game_ready: self.log.debug("game_id {} is ready to be started. players in game: {}" .format(game_id, str(player_id_list))) else: cursor.execute("SELECT * from queue where game_id=?", (game_id,)) player_list = cursor.fetchall() for player in player_list: cursor.execute("update queue set claimed=0, game_id=NULL, player_count=NULL where player_id=?", (player[0],)) self.db.commit() self.log.debug("player_id {} is removed from game_id {} and placed back in the queue" .format(player[0], player[4]))
import libs.glo_browsertype as glo # 导入友工程首页操作类 from page_objects import PMCloudIndexActions # 导入友工程登录页操作类 from page_objects import PMCloudLoginActions # 导入企业帐号选择页操作类 from page_objects import ApptenantActions # 导入友工程后台外框架操作类 from page_objects import WorkbenchActions # 导入友工程联系类型操作类 from page_objects import ContactTypeActions __author__ = "sunxr" __version__ = "V1.0" logger = Logger("TestContactTypeAdd").getLog() @unittest.skip class TestContactTypeAdd(unittest.TestCase): """联系类型新增相关测试""" def setUp(self): logger.info("测试前准备.") self.__driver = browser(glo.GLO_BROWSER_TYPE) PMCloudIndexActions(self.__driver).login() PMCloudLoginActions(self.__driver).pmcloudLogin() ApptenantActions(self.__driver).apptenantLogin() WorkbenchActions(self.__driver).clickContactType() self.contacttype_page = ContactTypeActions(self.__driver)
help= """Indicate the full path to scripts (e.g., /Applications/ResearchSoftware/rscript-wrapper""" ) parser.add_argument('--version', action='version', version='%(prog)s v1.0') args = parser.parse_args() if len(sys.argv) is None: #print help if no flags declared parser.print_help() elif len(sys.argv) < 3: #print help if not all flags declared print('Need more arguements') parser.print_help() else: starttime = time.time() workingdirectory = os.getcwd() if not os.path.exists(args.outputdir): os.makedirs(args.outputdir) sys.stdout = Logger(os.path.join(args.outputdir + "/")) print(''' *************************************************************************** * fasta_header_swap start * *************************************************************************** ''') print("Parameters used to run fasta_headers_swap:") print("") print(" shortnamefasta: " + str(args.shortnamefasta)) print(" keeptaxonomylookup: " + str(args.keeptaxonomylookup)) print(" newlongnamefastafile: " + str(args.newlongnamefastafile)) print(" output directory: " + str(args.outputdir)) print(" filename prefix: " + str(args.filenameprefix)) print(" path to scripts: " + str(args.pathtoscripts)) print("") print("Software and versions detected:")
def log(self, text): Logger.log("GoogleMusicApp", text)