def get_http_ftp(what,url,target,overwrite): from urllib2 import Request, urlopen, URLError, HTTPError, ProxyHandler, build_opener, install_opener import shutil if os.path.exists(target): if overwrite: log.info('Removing pre-existing file %s'%target) shutil.rmtree(target) else: log.info('Pre-existing file found, not re-getting %s'%target) return target proxy = os.getenv(what+'_proxy') if proxy : proxy_support = ProxyHandler({what:proxy}) opener = build_opener(proxy_support) install_opener(opener) #print 'openning',url try: res = urlopen(url) except HTTPError, e: print e.__class__, e raise IOError,'Failed to get '+url
def list_topic(topic, start_num, count, voteup, votedown, sort_method): global db conn = db.connect() sql_clause = "select postid, category, title, filename from posts " where_clause = "where category ='%s'" % (topic) extra_where_clause = "" if voteup != -1: extra_where_clause = "%s and voteup = %d" % (extra_where_clause, voteup) if votedown != -1: extra_where_clause = "%s and votedown = %d" % (extra_where_clause, votedown) where_clause = "%s%s" % (where_clause, extra_where_clause) orderby_clause = " " if sort_method == config.SORT_METHOD_LATEST: orderby_clause = "order by voteup asc, votedown asc" if sort_method == config.SORT_METHOD_HOTEST: orderby_clause = "order by voteup desc, votedown desc" sql = "%s %s %s limit %d offset %d;" % (sql_clause, where_clause, orderby_clause, count, start_num) log.info(sql) cursor = conn.execute(sql) out = cursor.fetchall() conn.close() user_posts = POST.lists_to_posts(out) return user_posts
def _filter_methods(self, cls, methods): log.info('All Tests for {} {}'.format(cls, methods)) if not self.name_filter: return methods filtered = [method for method in methods if self.name_filter.lower() in method.lower()] log.info('Filtered Tests for {}'.format(cls, filtered)) return filtered
def write_excel(joblist, filename): """ write Excel with Workbook :param joblist: :param filename: :return: """ mkdirs_if_not_exists(EXCEL_DIR) wb = Workbook() ws = wb.active ws.title = u"职位信息" ws.cell(row=1, column=1).value = u'职位编码' ws.cell(row=1, column=2).value = u'职位名称' ws.cell(row=1, column=3).value = u'所在城市' ws.cell(row=1, column=4).value = u'发布日期' ws.cell(row=1, column=5).value = u'薪资待遇' ws.cell(row=1, column=6).value = u'公司编码' ws.cell(row=1, column=7).value = u'公司名称' ws.cell(row=1, column=8).value = u'公司全称' rownum = 2 for each_job in joblist: ws.cell(row=rownum, column=1).value = each_job.positionId ws.cell(row=rownum, column=2).value = each_job.positionName ws.cell(row=rownum, column=3).value = each_job.city ws.cell(row=rownum, column=4).value = each_job.createTime ws.cell(row=rownum, column=5).value = each_job.salary ws.cell(row=rownum, column=6).value = each_job.companyId ws.cell(row=rownum, column=7).value = each_job.companyName ws.cell(row=rownum, column=8).value = each_job.companyFullName rownum += 1 wb.save(EXCEL_DIR + filename + '.xlsx') logging.info('Excel生成成功!')
def apple_privacy(): session.permanent = True # TODO log.info("apple_privacy is called") content = 'ok' resp = make_response(content, 200) resp.headers['Content-type'] = 'application/json; charset=utf-8' return resp
def delay (args): if len(args) == 1: try: log.info('MM:00 DELAY ' + args[0]) time.sleep(float(args[0])) except Exception, e: log.error('MM:00 ERROR: DELAY: exception: ' + repr(e))
def _download_git_submodules(self): 'If gaudi is served from a git repo with a submodule per pacakge' url = self.url() if url[4] == '+': url = url[4:] log.info(self.name +' download') # Get super project self._git_clone(url,True) # Get release package self._git_checkout(self.tag(),self.rel_pkg()) self.init_project(['lcgcmt']) # Get versions import cmt pkg_dir = os.path.join(self.proj_dir()+'/'+self.rel_pkg()) uses = cmt.get_uses(pkg_dir,self.env(pkg_dir)) for use in uses: #print 'use:',use.name,use.project,use.directory,use.version if use.project == 'gaudi' and use.directory == '': if '*' in use.version: log.info('Skipping %s %s'%(use.name,use.version)) continue self._git_checkout(use.version,use.name) pass continue return
def check_returncode(self,recv,args): print 'verify code value,expect:%s,actual:%s'%(int(args.get('ExpectResult')),int(recv[1].get('CODE'))) log.info('验证code的值,期望值:%s,实际值:%s'%(int(args.get('ExpectResult')),int(recv[1].get('CODE')))) print 'The Interface return is: ' print recv print "The data I want to see is: %s" %(recv[1].get('sporderid')) self.assertEqual(int(args.get('ExpectResult')),int(recv[1].get('CODE')))
def _renameSubtitles(self): """ Rename the Subtitle files. """ includeCount = len(self.curFileNames) >= 2 if (self.subtitles): for i in range(len(self.subtitles)): subPath = self.subtitles[i] newFilePrefix = self.newFileNames[i][0:-4] # Make sure the subtitle directory exists newSubDirPath = "%s/subtitles" % (self.dirPath) if (not os.path.exists(newSubDirPath)): log.info(" >> Creating Dir: %s" % newSubDirPath) os.mkdir(newSubDirPath, 0755) # Rename SRT Files if (subPath.lower().endswith('.srt')): curSrtPath = "%s/%s" % (self.dirPath, subPath) newSrtPath = "%s/%s.srt" % (newSubDirPath, newFilePrefix) self._rename(curSrtPath, newSrtPath) # Rename IDX, SUB Files elif (subPath.lower().endswith('.idx')): curIdxPath = "%s/%s" % (self.dirPath, subPath) curSubPath = "%s.sub" % (curIdxPath[0:-4]) newIdxPath = "%s/%s.idx" % (newSubDirPath, newFilePrefix) newSubPath = "%s/%s.sub" % (newSubDirPath, newFilePrefix) self._rename(curIdxPath, newIdxPath) self._rename(curSubPath, newSubPath)
def testBootsDirectly(self): if self.metal.is_supported() is False: log.info('Metal not supported, skipping testBootsDirectly') return simulator = self.assertCreatesSimulator([self.device_type]) self.assertEventSuccesful([simulator.get_udid(), 'boot', '--direct-launch'], 'boot') self.assertEventSuccesful([simulator.get_udid(), 'shutdown'], 'shutdown')
def testRecordsVideo(self): if self.metal.is_supported() is False: log.info('Metal not supported, skipping testRecordsVideo') return (simulator, _) = self.testLaunchesSystemApplication() arguments = [ simulator.get_udid(), 'record', 'start', '--', 'listen', '--', 'record', 'stop', '--', 'shutdown', ] # Launch the process, terminate and confirm teardown is successful with self.fbsimctl.launch(arguments) as process: process.wait_for_event('listen', 'started') process.terminate() process.wait_for_event('listen', 'ended') process.wait_for_event('shutdown', 'ended') # Get the diagnostics diagnose_events = self.assertExtractAndKeyDiagnostics( self.assertEventsFromRun( [simulator.get_udid(), 'diagnose'], 'diagnostic', 'discrete', ), ) # Confirm the video exists video_path = diagnose_events['video']['location'] self.assertTrue( os.path.exists(video_path), 'Video at path {} should exist'.format(video_path), )
def download(self): '''Download missing or update pre-existing project files. As a side effect the program will be in the projects directory that contains the downloaded project''' log.info(self.name +' download') import fs, ConfigParser projdir = fs.projects() fs.goto(projdir, True) from get import get try: tag = self.tag() except ConfigParser.NoOptionError: tag = None #print 'url="%s" name="%s" tag="%s"'%(self.url(), self.name, tag) get(self.url(), self.name, True, tag=tag) tarfile = os.path.basename(self.url()) if '.tgz' in tarfile or '.tar' in tarfile: untar(tarfile) dot = tarfile.find('.') dirname = tarfile[:dot] import shutil shutil.move(dirname, self.name) pass fs.goback return
def run (args): if len(args) < 2: print 'MM:00 EXEC: ERROR usage: exec cmd cmd ...' print 'Commands are:' for c in sorted(commands): print ' ' + c + ': ' + commands[c].get('cmd', '<CMD>') return for i in range(1, len(args)): cmdname = args[i] try: c = commands[cmdname]['cmd'] except: log.error('MM:00 ERROR: EXEC FAILED unknown or poorly specified cmd: ' + cmdname) continue log.info('MM:00 EXEC: ' + cmdname + ' cmd = ' + c) ca = c.split() try: p = subprocess.Popen(ca, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() except Exception, e: out = '' err = 'Command Failed: ' + repr(e) r = out + err log.debug('MM:00 EXEC: ' + cmdname + ' output = \n' + r.strip())
def post_check(): log.info("Check post validation for all posts") postid_list = get_postid_list() status, obj = check_post_list(postid_list) if status is False: log.error("post check fail, msg: %s" % obj) return False
def _searchTrailerAddict(self, searchTitle, searchYear): """ Search TrailerAddict for a Trailer URL """ # Search TrailerAddict for the Movie log.info(" Searching TrailerAddict for: '%s' (yr: %s)" % (searchTitle, searchYear)) searchResults = traileraddict.search(searchTitle) if (not searchResults): log.fine(" TrailerAddict has no search results for: '%s' (yr: %s)" % (searchTitle, searchYear)) return None # Select the correct TrailerAddict Movie firstTitle = searchResults[0]['title'] firstYear = searchResults[0]['year'] if (firstTitle.lower() == searchTitle.lower()) and (int(firstYear) == searchYear): log.fine(" First result is exact match: %s (%s)" % (searchTitle, searchYear)) searchSelection = searchResults[0] else: log.fine(" No exact TrailerAddict match found, prompting user") choiceStr = lambda r: "%s (%s) - %s" % (r['title'], r['year'], r['url']) searchSelection = util.promptUser(searchResults, choiceStr) if (not searchSelection): log.fine(" TrailerAddict has no entry for: '%s' (yr: %s)" % (searchTitle, searchYear)) return None # Search for the correct Video (Traileraddict has many per movie) trailerUrls = traileraddict.getTrailerUrls(searchSelection['url']) trailerUrl = traileraddict.getMainTrailer(trailerUrls) if (not trailerUrl): log.info(" Main trailer not found, prompting user") choiceStr = lambda t: t trailerUrl = util.promptUser(trailerUrls, choiceStr) return trailerUrl
def _getImdbUrlFromSearch(self, foreign=False): """ Search IMDB for the specified title. """ # Search IMDB for potential matches title = self.curTitle year = self.curYear or "NA" log.info(" Searching IMDB for: '%s' (yr: %s)" % (title, year)) results = imdbpy.search_movie(title, IMDB_MAX_RESULTS) # Check first 5 Results Title and year matches exactly selection = None for result in results[0:5]: if (self._weakMatch(result['title'], title)) and (int(result['year']) == year): log.fine(" Result match: %s (%s)" % (result['title'], result['year'])) selection = result break # Ask User to Select Correct Result if (not selection): log.fine(" No exact IMDB match found, prompting user") if (not foreign): choiceStr = lambda r: "%s (%s) - %s" % (r['title'], r['year'], self.getUrl(r.movieID)) else: choiceStr = lambda r: "%s (%s-%s): %s" % (r['title'], self._getCountry(r), r['year'], self._getAka(r)) selection = util.promptUser(results, choiceStr) # If still no selection, return none if (not selection): log.fine(" IMDB has no entry for: %s (%s)" % (title, year)) return None return self.getUrl(selection.movieID)
def check_return(self,r_status,r_data,args): if r_status==200: if args['ExpectResult']: try: eresult=json.loads(args['ExpectResult']) except Exception, e: print e print u'请检查excel的ExpectResult列数据格式是不是dict' if type(eresult)==dict: for key,value in eresult.items(): if r_data.has_key(key): if type(eresult[key])==dict: for key1,value1 in eresult[key].items(): if r_data[key].has_key(key1): print u' 验证%s的值,期望值:%s,实际值:%s'%(key1,value1,r_data[key][key1]) log.info(u' 验证%s的值,期望值:%s,实际值:%s'%(key1,value1,r_data[key][key1])) self.assertEqual(value1,r_data[key][key1]) else: print '返回的接口数据无此%s'%key1 else: print u'验证%s的值,期望值:%s,实际值:%s'%(key,value,r_data[key]) log.info(u'验证%s的值,期望值:%s,实际值:%s'%(key,value,r_data[key])) self.assertEqual(value,r_data[key]) else: print '返回的接口数据无此%s'%key else: print u'请检查excel的ExpectResult列是否准备了待验证的数据'
def listener3(host, bind, port): if host not in hosts: log.error('MM:00 ERROR: LISTENER: unknown host: ' + host) return #print 'listener ' + host + ' ' + bind + ' ' + port r = generic(host, 'LISTENER', 'listener ' + bind + ' ' + port + '\n') if r is not None and len(r) > 0: log.info('MM:' + host + ' LISTENER: ' + r.strip())
def save_to_file(self, attr, path): param = attr with open(path, 'w') as f: hkl.dump(self.to_dict(), f) path = os.path.splitext(path)[0] + '.json' with open(path, 'w') as f: json.dump(param, f) log.info("Save parame to {}".format(path))
def report(self): log.info("Computing scores...") total_loss = [] for id, pred, gt in zip(self._ids, self._predictions, self._groundtruths): total_loss.append(self.compute_loss(pred, gt)) avg_loss = np.average(total_loss) log.infov("Average loss : %.4f", avg_loss)
def max_pool(input, info=False, k=2, s=2, padding='SAME', name='pool'): _ = tf.nn.max_pool(input, ksize=[1, k, k, 1], strides=[1, s, s, 1], padding=padding, name=name) if info: log.info('{} {}'.format(name, _)) return _
def try_load_checkpoint(self, model_path): ckpt_path = tf.train.latest_checkpoint(model_path) assert ckpt_path is not None, '%s is empty' % model_path log.info("Checkpoint path: %s", ckpt_path) self.saver.restore(self.session, ckpt_path) log.info( "Loaded the pretrain parameters from the provided checkpoint path")
def testScreenshot(self): if self.metal.is_supported() is False: log.info('Metal not supported, skipping testScreenshot') return simulator = self.assertCreatesSimulator(['iPhone 6']) self.assertEventSuccesful([simulator.get_udid(), 'boot'], 'boot') with self.launchWebserver() as webserver: webserver.get_binary(simulator.get_udid() + '/screenshot.png') webserver.get_binary(simulator.get_udid() + '/screenshot.jpeg')
def run_checks(client: Client) -> State: # Non-zero indicates a gating condition for check in enabled_health_checks: log.info(f"Checking: {check.__doc__}...") state = check(client) if state == State.FAIL: log.error("Failed on health check", check=check.__doc__) return state return State.OK
def blackholing (args): if len(args) < 3: log.error('MM:00 EXEC: ERROR usage: blackholing participant_id remove/insert id[,id...]') return part_id = args[0] #participant id part_action = args[1] #action insert or remove rule_ids = [] for policy_id in args[2].split(','): #rule ids rule_ids.append(int(policy_id)+2**12) #additional 4096 for cookie id client_path = '/home/vagrant/endeavour/pclnt/participant_client.py' config_file = 'participant_' + part_id + '_bh.cfg' cmd = '' for arg in args: cmd += arg + ' ' log.info('MM:00 BLACKHOLING: ' + cmd + config_file) policy_path = os.path.abspath(os.path.join(os.path.realpath(sys.argv[1]), "..", "..", "policies")) config_path = os.path.join(policy_path, config_file) part_info = config.participants[str(part_id)] part_url = 'http://' + str(part_info["EH_SOCKET"][0]) + ':' + str(part_info["EH_SOCKET"][1]) + '/bh/inbound/' content_header = {'Content-Type':'application/json'} # prepare for insert blackholing policy if part_action == 'insert': new_policy = [] # Open File and Parse with open(config_path, 'r') as f: policies=json.load(f) for policy in policies['inbound']: if int(policy['cookie']) in rule_ids: new_policy.append(policy) # insert only inbound policys data = {} data['inbound'] = new_policy data=json.dumps(data) # post to participant api r = requests.post(part_url, data=data, headers=content_header) # prepare for remove seperate blackholing policy elif part_action == 'remove': for rule_id in rule_ids: new_url = part_url + str(rule_id) # post to participant api r = requests.delete(new_url, headers=content_header) else: log.error('MM:00 EXEC: ERROR usage: error in blackholing - wrong action')
def __init__(self, config): self.config = config self.vfeat_path = config.vfeat_path self.tf_record_dir = config.tf_record_dir self.train_dir = os.path.dirname(config.checkpoint) self.vlmap_word_weight_dir = os.path.join( self.train_dir, config.vlmap_word_weight_dir.split('/')[-1]) config.vlmap_word_weight_dir = self.vlmap_word_weight_dir self.batch_size = config.batch_size with tf.name_scope('datasets'): self.target_split = tf.placeholder(tf.string) with tf.name_scope('datasets/batch'): vqa_batch = { 'train': input_ops_vqa.create( self.batch_size, self.tf_record_dir, 'train', is_train=True, scope='train_ops', shuffle=True), 'val': input_ops_vqa.create( self.batch_size, self.tf_record_dir, 'val', is_train=True, scope='val_ops', shuffle=False), 'testval': input_ops_vqa.create( self.batch_size, self.tf_record_dir, 'testval', is_train=True, scope='testval_ops', shuffle=False), 'test': input_ops_vqa.create( self.batch_size, self.tf_record_dir, 'test', is_train=True, scope='test_ops', shuffle=False) } batch_opt = { tf.equal(self.target_split, 'train'): lambda: vqa_batch['train'], tf.equal(self.target_split, 'val'): lambda: vqa_batch['val'], tf.equal(self.target_split, 'testval'): lambda: vqa_batch['testval'], tf.equal(self.target_split, 'test'): lambda: vqa_batch['test'], } self.batch = tf.case( batch_opt, default=lambda: vqa_batch['train'], exclusive=True) # Model Model = self.get_model_class(config.model_type) log.infov('using model class: {}'.format(Model)) self.model = Model(self.batch, config, is_train=True) self.checkpoint_loader = tf.train.Saver(max_to_keep=1) session_config = tf.ConfigProto( allow_soft_placement=True, gpu_options=tf.GPUOptions(allow_growth=True), device_count={'GPU': 1}) self.session = tf.Session(config=session_config) self.ckpt_path = config.checkpoint if self.ckpt_path is not None: log.info('Checkpoint path: {}'.format(self.ckpt_path)) self.checkpoint_loader.restore(self.session, self.ckpt_path) log.info('Loaded the checkpoint') log.warn('Inference initialization is done')
async def testScreenshot(self): if self.metal.is_supported() is False: log.info('Metal not supported, skipping testScreenshot') return simulator = await self.assertCreatesSimulator(['iPhone 6']) await self.assertEventSuccesful([simulator.udid, 'boot'], 'boot') async with self.launchWebserver() as webserver: webserver.get_binary(simulator.udid + '/screenshot.png') webserver.get_binary(simulator.udid + '/screenshot.jpeg')
def init(file): '''解析excel文件用例''' log.info('解析excel文件数据') cases_list=[] try: module_list=open_excel.excel_table_by_index(file) except Exception, e: print e log.error('%s文件打开失败'%file)
def symlink(config, filename): src_path = os.path.join(root_dir, config.reference_vqa_dir, filename) dst_path = os.path.join(root_dir, config.caption_split_dir, filename) if os.path.exists(dst_path): log.info("{} already exists".format(dst_path)) else: log.info("Sym link: {}->{}".format(src_path, dst_path)) os.symlink(src_path, dst_path)
def testBootsDirectly(self): if self.metal.is_supported() is False: log.info('Metal not supported, skipping testBootsDirectly') return simulator = self.assertCreatesSimulator([self.device_type]) self.assertEventSuccesful( [simulator.get_udid(), 'boot', '--direct-launch'], 'boot') self.assertEventSuccesful([simulator.get_udid(), 'shutdown'], 'shutdown')
def log_step_message(self, step_result): report_msg = ' '.join([ '{}: {}'.format(k, step_result['report'][k]) for k in sorted(step_result['report'].keys()) ]) msg = '[{}] step {}: [{}]'.format(self.split, step_result['step'], report_msg) log.info(msg) return msg
def crawl_company(havemark=0): # 定义公司信息为列表形式 COMPANY_LIST = list() # 请求的url req_url = 'https://www.lagou.com/gongsi/0-0-0.json?havemark=%d' % havemark # 请求头部 headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Host': 'www.lagou.com', 'Origin': 'https://www.lagou.com', 'Referer': 'https://www.lagou.com/gongsi/0-0-0?havemark=0', 'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 ' 'Mobile/13B143 Safari/601.1' } # 进行页数循环 for pn in range(20): params = { 'first': 'false', 'pn': str(pn), 'sortField': '0', 'havemark': str(havemark) } response = requests.post(req_url, headers=headers, params=params, cookies=m_lagou_spider.get_cookies(), timeout=10) print(response.url) if response.status_code == 200: company_list_per_page = response.json()['result'] for company in company_list_per_page: COMPANY_LIST.append([ company['companyId'], company['companyShortName'], company['city'], company['companyFeatures'], company['companyFullName'], company['financeStage'], company['industryField'], company['interviewRemarkNum'], company['positionNum'], company['processRate'] ]) log.info('page %d has been crawled down~' % (pn + 1)) elif response.status_code == 403: log.error('403 forbidden...') else: log.error(response.status_code) # 睡眠 time.sleep(config.TIME_SLEEP) return COMPANY_LIST
def start(self): self._thread = threading.Thread(None, \ self._body, \ self._name,\ (), \ {} \ ) log.info("start thread:" + str(self._name)) self._thread.start()
def crawl_jobs(positionName): JOB_DATA = list() # 创建max_page_number对象,返回最大页数 max_page_number = get_max_pageNo(positionName) log.info("%s, 共有 %s 页记录, 共约 %s 记录", positionName, max_page_number, max_page_number * 15) cookies = get_cookies() for i in range(1, max_page_number + 1): # url请求 request_url = 'https://m.lagou.com/search.json?city=%E5%85%A8%E5%9B%BD&positionName=' + parse.quote( positionName) + '&pageNo=' + str(i) + '&pageSize=15' # 请求头部 headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate, sdch', 'Host': 'm.lagou.com', 'Referer': 'https://m.lagou.com/search.html', 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, ' 'like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4', 'X-Requested-With': 'XMLHttpRequest', 'Connection': 'keep-alive' } # 发送请求 response = requests.get(request_url, headers=headers, cookies=cookies) # 返回判断 if response.status_code == 200: # 在JOB_DATA中添加positionId、positionName、createTime、salary、companyId、companyName、companyFullName for each_item in response.json( )['content']['data']['page']['result']: JOB_DATA.append([ each_item['positionId'], each_item['positionName'], each_item['city'], each_item['createTime'], each_item['salary'], each_item['companyId'], each_item['companyName'], each_item['companyFullName'] ]) # try: # crawl_job_detail(each_item['positionId'], positionName) # except: # pass print('crawling page %d done...' % i) # 休眠 time.sleep(TIME_SLEEP) elif response.status_code == 403: log.error('request is forbidden by the server...') else: log.error(response.status_code) return JOB_DATA
def init_model(): task = FLAGS.task model_params = { "feat_dim": FEAT_DIM, "word_embed": train_dataset.word_matrix, "lstm_steps": SEQUENCE_LENGTH, "architecture": FLAGS.architecture } if task == 'FrameQA': model_params["vocabulary_size"] = len(train_dataset.idx2word) model_params["answer_size"] = len(train_dataset.idx2ans) model_params.update(FLAGS.__flags) if FLAGS.checkpoint_path: checkpoint = FLAGS.checkpoint_path params_path = os.path.join( os.path.dirname(checkpoint), '%s_%s_param.hkl' % (FLAGS.task.lower(), FLAGS.name.lower())) log.info("Restored parameter set from {}".format(params_path)) model_params = hkl.load(open(params_path)) model_params["att_hidden_dim"] = FLAGS.att_hidden_dim model_params["hidden_dim"] = FLAGS.hidden_dim model = Model.from_dict(model_params) model.print_params() video = tf.placeholder(tf.float32, [FLAGS.batch_size] + list(train_dataset.get_video_feature_dimension())) video_mask = tf.placeholder(tf.float32, [FLAGS.batch_size, SEQUENCE_LENGTH]) answer = tf.placeholder(tf.int32, [FLAGS.batch_size, 1]) if (task == 'Count') or (task == 'FrameQA'): question = tf.placeholder(tf.int32, [FLAGS.batch_size, SEQUENCE_LENGTH]) question_mask = tf.placeholder(tf.int32, [FLAGS.batch_size, SEQUENCE_LENGTH]) else: question = tf.placeholder( tf.int32, [FLAGS.batch_size, Model.MULTICHOICE_COUNT, SEQUENCE_LENGTH]) question_mask = tf.placeholder( tf.float32, [FLAGS.batch_size, Model.MULTICHOICE_COUNT, SEQUENCE_LENGTH]) model.build_graph( video, video_mask, question, question_mask, answer, optimizer=tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)) return model, model_params
def Pose_Encoder(input_pose, target_pose, scope='Pose_Encoder', reuse=False): with tf.variable_scope(scope) as scope: if not reuse: log.warn(scope.name) _ = target_pose - input_pose if not reuse: log.info('{} {}'.format(_.name, _.get_shape().as_list())) return _
def downMusic(url, sid): if os.path.exists('mp3/%s.mp3' % sid): log.info('Already downloaded') return r = session.get(url, timeout=30) with open('mp3/%s.mp3' % sid, 'wb') as fp: fp.write(r.content) log.info('Down finish')
def echo (args): host = args[0] del args[0] all = '' for arg in args: all += ' ' + '"' + arg + '"' log.info('MM:' + host + ' ECHO ' + all) r = generic(host, 'ECHO', 'echo ' + all) if r is not None: log.info('MM:' + host + ' echo = ' + r.strip())
def log_final_message(self, final_report): report_msg = ' '.join([ '{}: {}'.format(k, final_report[k]) for k in sorted(final_report.keys()) ]) msg = '[{}] Final average report: [{}]'.format(self.split, report_msg) log.info(msg) with open(self.checkpoint + '.final.txt', 'r') as f: f.write(msg) return msg
def _rename(self, src, dst): """ Rename the specified file. """ if (src != dst): if (os.path.exists(dst)): log.warn(" Path already exists: %s" % dst) return None log.info(" >> Renaming: %s" % src) log.info(" to: %s" % dst) os.rename(src, dst)
def unpack(): "Unpack the previously downloaded tarfile" log.info("unpacking git source") target = srcdir() if os.path.exists(target): log.info("git appears to already be unpacked in %s" % (target)) return target fs.goto(fs.external(), True) untar(tgz()) return target
def echo(args): host = args[0] del args[0] all = '' for arg in args: all += ' ' + '"' + arg + '"' log.info('MM:' + host + ' ECHO ' + all) r = generic(host, 'ECHO', 'echo ' + all) if r is not None: log.info('MM:' + host + ' echo = ' + r.strip())
def action(self,args): log.info(u'执行用例%s'%args['TestCaseName']) print args['TestCaseName']+u'用例描述'+u':'+args['Description'] self.args=args #请求数据 r_status,r_data=request_http(args) #验证返回数据 check_return(self,r_status,r_data,args) #验证数据库的值 check_sql(self,r_data,args)
def unpack(): 'Unpack the previously downloaded tarfile' log.info('unpacking cmt source') target = srcdir() if os.path.exists(target): log.info('CMT appears to already be unpacked in %s'%(target)) return target fs.goto(fs.external(),True) untar(tgz()) return target
def decrypt(self, fpin, fpout, aes_key, salt): ''' Decrypt a fileobject using the given AES key and salt. ''' log.info("Decrypting file...") iv = '\0' * 32 cipher = M2Crypto.EVP.Cipher(alg='aes_256_cbc', key=aes_key, iv=iv, op=DECODE, salt=salt) _filter_cipher(fpin, fpout, cipher) log.info("Done")
def download(): 'Download CMT source tar file into external area.' log.info('downloading cmt tar file') target = "%s/%s"%(fs.external(),tgz()) from get import get from exception import InconsistentState target = get(url(),target) if not os.path.exists(target): raise InconsistentState,'Tar file does not exist: %s%s'%(os.getcwd(),tgz()) return target
def _filter_methods(self, cls, methods): log.info('All Tests for {} {}'.format(cls, methods)) if not self.name_filter: return methods filtered = [ method for method in methods if self.name_filter.lower() in method.lower() ] log.info('Filtered Tests for {}'.format(cls, filtered)) return filtered
def report(self): # report L2 loss log.info("Computing scores...") score = {} score = [] for id, pred, gt in zip(self._ids, self._predictions, self._groundtruths): score.append(self.compute_accuracy(pred, gt)) avg = np.average(score) log.infov("Average accuracy : %.4f", avg*100)
def __init__(self, data_dir, split, name='vlmap_memft'): self.name = name self.split = split log.warn('loading image_info ..') image_info_path = os.path.join(data_dir, '{}_image_info.pkl'.format(split)) image_info = cPickle.load(open(image_info_path, 'rb')) self._ids = image_info['image_ids'] self.image_id2idx = image_info['image_id2idx'] log.info('loading image_info done') log.warn('loading processed data ..') processed_path = os.path.join(data_dir, '{}_processed.pkl'.format(split)) self.processed = cPickle.load(open(processed_path, 'rb')) log.info('loading processed done') log.warn('loading answer_dict ..') answer_dict_path = os.path.join(data_dir, 'answer_dict.pkl') self.answer_dict = cPickle.load(open(answer_dict_path, 'rb')) self.num_answers = len(self.answer_dict['vocab']) log.info('loading answer_dict done') log.warn('loading wordset_dict ..') ws_dict_path = os.path.join(data_dir, 'wordset_dict5.pkl') self.ws_dict = cPickle.load(open(ws_dict_path, 'rb')) log.info('loading wordset_dict done') log.warn('loading enwiki_context_dict ..') enwiki_dict_pkl_path = os.path.join(data_dir, 'enwiki_context_dict_w3_n5.pkl') enwiki_dict_h5_path = os.path.join(data_dir, 'enwiki_context_dict_w3_n5.hdf5') self.enwiki_dict = cPickle.load(open(enwiki_dict_pkl_path, 'rb')) with h5py.File(enwiki_dict_h5_path, 'r') as f: self.enwiki_dict['np_context'] = f['np_context'].value self.enwiki_dict['np_context_len'] = f['np_context_len'].value with h5py.File(os.path.join(data_dir, '{}_vfeat.hdf5'.format(split)), 'r') as f: self.vfeat_dim = int(f['data_info']['vfeat_dim'].value) self.max_box_num = int(f['data_info']['max_box_num'].value) log.warn('loading {} image_features ..'.format(split)) self.image_features = np.array(f.get('image_features')) log.warn('loading {} normal_boxes ..'.format(split)) self.normal_boxes = np.array(f.get('normal_boxes')) log.warn('loading {} num_boxes ..'.format(split)) self.num_boxes = np.array(f.get('num_boxes')) log.warn('loading {} spatial_features ..'.format(split)) self.spatial_features = np.array(f.get('spatial_features')) log.warn('loading {} features done ..'.format(split)) log.info('dataset {} {} init done'.format(name, split))
def get(url,target,overwrite=False,tag=None): ''' Get the file or directory tree at the given URL and place it at the given target path. If overwrite is True and target is preexisting it will be overwritten (updated). The URL is expected to be in a standard form: SCHEME://HOSTNAME/PATH The following URL schemes are supported: http: - the file given in PATH via HTTP ftp: - the file given in PATH via anonymous FTP git+TRANSPORT: git-clone a repository. TRANSPORT can be http, rsync, ssh or empty to use native git protocol (the '+' can be omitted). For local repository, use "git+file:///path/to/git". See git-clone(1) for details. If overwriting a git-pull is done. svn+TRANSPORT: - perform "svn co" using the remaining URL with 'svn+' removed. If overwritting, an "svn update" is done. cvs+:TRANSPORT: - perform "cvs co" using the remaining URL with 'cvs+' removed. If overwritting, an "cvs update" is done. ''' log.info('Getting url "%s" --> "%s"'%(url,target)) urlp = uriparse(url) if urlp[0] == 'http' or urlp[0] == 'ftp': return get_http_ftp(urlp[0],url,target,overwrite) scheme = urlp[0].split('+') #print 'scheme=',scheme print urlp,scheme if urlp[0] == 'git' or scheme[0] == 'git': return get_git(scheme,url,target,overwrite,tag) if scheme[0] == 'svn': return get_svn(scheme[1]+'://'+urlp[1]+'/'+urlp[2]+'/'+tag,target,overwrite) if scheme[0] == 'cvs': # get_cvs(url,module,tag,target,overwrite): #print 'CVS: "%s", "%s", "%s"'%(urlp[0],urlp[1],urlp[2]) url = ':%s:%s:%s'%(scheme[1],urlp[1],'/'.join(urlp[2].split('/')[:-1])) module = urlp[2].split('/')[-1] #print 'url=%s, module=%s'%(url,module) print 'Note: getting from CVS, if this appears to hang, it is waiting for a password' return get_cvs(url,module,tag,target,overwrite) msg = 'Unhandled URL: "%s"'%url log.error(msg) raise ValueError, msg
def init_model(): task = FLAGS.task model_params = { "feat_dim": FEAT_DIM, "word_embed": train_dataset.word_matrix, "lstm_steps": SEQUENCE_LENGTH } if task == 'FrameQA': model_params["vocabulary_size"] = len(train_dataset.idx2word) model_params["answer_size"] = len(train_dataset.idx2ans) model_params.update(FLAGS.__flags) if FLAGS.checkpoint_path: checkpoint = FLAGS.checkpoint_path params_path = os.path.join( os.path.dirname(checkpoint), '%s_%s_param.hkl' % (FLAGS.task.lower(), FLAGS.name.lower())) log.info("Restored parameter set from {}".format(params_path)) model_params = hkl.load(open(params_path)) if FLAGS.test_phase: model_params["dropout_keep_prob_cell_input"] = 1. model_params["dropout_keep_prob_cell_output"] = 1. model_params["dropout_keep_prob_fully_connected"] = 1. model_params["dropout_keep_prob_output"] = 1. model_params["dropout_keep_prob_image_embed"] = 1. model = Model.from_dict(model_params) model.print_params() video = tf.placeholder(tf.float32, [FLAGS.batch_size] + list(train_dataset.get_video_feature_dimension())) video_mask = tf.placeholder(tf.float32, [FLAGS.batch_size, SEQUENCE_LENGTH]) answer = tf.placeholder(tf.int32, [FLAGS.batch_size, 1]) train_flag = tf.placeholder(tf.bool) if (task == 'Count') or (task == 'FrameQA'): question = tf.placeholder(tf.int32, [FLAGS.batch_size, SEQUENCE_LENGTH]) question_mask = tf.placeholder(tf.int32, [FLAGS.batch_size, SEQUENCE_LENGTH]) else: question = tf.placeholder( tf.int32, [FLAGS.batch_size, Model.MULTICHOICE_COUNT, SEQUENCE_LENGTH]) question_mask = tf.placeholder( tf.float32, [FLAGS.batch_size, Model.MULTICHOICE_COUNT, SEQUENCE_LENGTH]) model.build_graph(video, video_mask, question, question_mask, answer, train_flag) return model, model_params
def fc(input, output_shape, is_train, info=False, batch_norm=True, activation_fn=tf.nn.relu, name="fc"): _ = slim.fully_connected(input, output_shape, activation_fn=None) if info: log.info('{} {}'.format(name, _)) _ = bn_act(_, is_train, batch_norm=batch_norm, activation_fn=activation_fn) return tf.identity(_, name=name)
def symlink(src_path, dst_path): root_dir = os.path.dirname(os.path.realpath(__file__)) src_path = os.path.join(root_dir, src_path) dst_path = os.path.join(root_dir, dst_path) if os.path.exists(dst_path): log.info("{} already exists".format(dst_path)) else: log.info("Sym link: {}->{}".format(src_path, dst_path)) os.symlink(src_path, dst_path)
def create_input_ops(dataset, batch_size, num_threads=16, is_training=False, data_id=None, scope='input', shuffle=True): input_ops = {} if data_id is None: data_id = dataset.ids log.info("input_ops [%s]: Using %d IDs from dataset", scope, len(data_id)) else: log.info("input_ops [%s]: Using specified %d IDs", scope, len(data_id)) with tf.device("/cpu:0"), tf.name_scope(scope): input_ops['id'] = tf.train.string_input_producer( tf.convert_to_tensor(data_id), capacity=128).dequeue(name='input_ids.dequeue') m = dataset.get_data(data_id[0]) def load_fn(id): image = dataset.get_data(id) return (id, image.astype(np.float32)) input_ops['id'], input_ops['image'] = tf.py_func( load_fn, inp=[input_ops['id']], Tout=[tf.string, tf.float32], name='func_hp') input_ops['id'].set_shape([]) input_ops['image'].set_shape(list(m.shape)) capacity = 2 * batch_size * num_threads min_capacity = min(int(capacity * 0.75), 1024) if shuffle: batch_ops = tf.train.shuffle_batch(input_ops, batch_size, num_threads=num_threads, capacity=capacity, min_after_dequeue=min_capacity) else: batch_ops = tf.train.batch(input_ops, batch_size=batch_size, num_threads=num_threads, capacity=capacity) return input_ops, batch_ops
def on_websocket_close(self, code: int, reason: str) -> NoReturn: log.debug(f"connection {self.sock_id} closed: {code} - {reason}") if not self.new_sock_added: log.debug(f"going to reconnect to router, webswocket close code: {code}") self.ws_client_farm.remove_ws(str(self.register_uri)) self.when_consumed_action() self.new_sock_added = True if not self.req_complete and self.req_to_target is not None: if code != STATUS_UNEXPECTED_CONDITION: log.info(f"websocket closed before the target response was processed, This may be because the user" f"closed therire browser, Going to cancel request to target {self.req_to_target.url}") self.req_to_target.abort(Exception("Socket to Router closed"))
def fc(input, output_shape, is_train, info=False, norm='batch', activation_fn=lrelu, name="fc"): with tf.variable_scope(name): _ = slim.fully_connected(input, output_shape, activation_fn=None) _ = bn_act(_, is_train, norm=norm, activation_fn=activation_fn) if info: log.info('{} {}'.format(name, _)) return _
def __init__(self, config, dataset): self.config = config self.train_dir = config.train_dir log.info("self.train_dir = %s", self.train_dir) # --- input ops --- self.batch_size = config.batch_size self.dataset = dataset check_data_id(dataset, config.data_id) _, self.batch = create_input_ops(dataset, self.batch_size, data_id=config.data_id, is_training=False, shuffle=False) # --- create model --- Model = self.get_model_class(config.model) log.infov("Using Model class : %s", Model) self.model = Model(config) self.global_step = tf.contrib.framework.get_or_create_global_step(graph=None) self.step_op = tf.no_op(name='step_no_op') tf.set_random_seed(1234) session_config = tf.ConfigProto( allow_soft_placement=True, gpu_options=tf.GPUOptions(allow_growth=True), device_count={'GPU': 1}, ) self.session = tf.Session(config=session_config) # --- checkpoint and monitoring --- self.saver = tf.train.Saver(max_to_keep=100) self.checkpoint_path = config.checkpoint_path if self.checkpoint_path is None and self.train_dir: self.checkpoint_path = tf.train.latest_checkpoint(self.train_dir) if self.checkpoint_path is None: log.warn("No checkpoint is given. Just random initialization :-)") self.session.run(tf.global_variables_initializer()) else: log.info("Checkpoint path : %s", self.checkpoint_path) mean_std = np.load('../DatasetCreation/VG/mean_std.npz') self.img_mean = mean_std['img_mean'] self.img_std = mean_std['img_std'] self.coords_mean = mean_std['coords_mean'] self.coords_std = mean_std['coords_std']
def get_out_dir(dataset, checkpoint_path): # out_dir try: checkpoint_step = os.path.basename(checkpoint_path).split('-')[-1] except: checkpoint_step = 'latest' out_dir = os.path.abspath( os.path.join(checkpoint_path, '../../generated-{}-{}'.format(dataset, checkpoint_step))) mkdir_p(out_dir) log.info("Out directory : %s", out_dir) return out_dir
def report(self): # report L2 loss log.info("Computing scores...") score = {} score['l2_loss'] = [] for id, pred, gt in zip(self._ids, self._predictions, self._groundtruths): score['l2_loss'].extend(self.compute_l2error(pred, gt)) avg_l2loss = np.average(score['l2_loss']) log.infov("Average L2 loss : %.5f", avg_l2loss)