コード例 #1
0
 def crawl(self, args):
     file_name = args['file_name']
     target_folder = args['target_folder']
     apk_info = self.prepare_apk_info(args)
     util.makeDirs(self.lock, target_folder)
     if not util.verify_apk(file_name):
         util.write_log_warn(self.lock, self.logger, "FAILED :  %s jarsigner faild."%(file_name))
         os.remove(file_name)
         return False
     md5 = util.get_file_MD5(file_name)
     apk_info['MD5'] = md5
     target_name = target_folder + 'cell-' + md5[:2] + '/' + md5 + '.apk'
     if os.path.exists(target_name):
         os.remove(file_name)
     else:
         apk_info['new'] = 'True'
         os.rename(args['file_name'], target_name)
     util.writeApkInfoFile(self.lock, target_folder,apk_info)
     util.writeRecord(self.lock, target_folder,apk_info)
     util.write_log_info(self.lock, self.logger, "SUCCESS : %s handle success"%(args['file_name']))
     return True
コード例 #2
0
 def crawl(self, args, browser):
     source = args['source']
     list_page_url = args['list_page_url']
     try:
         appstore_modual = __import__(source)
     except ImportError:
         util.write_log_error(self.lock, self.logger, "can not find " + source + ".py in configured appstores")
         return False
     appstore_class = appstore_modual.Store()
     if self.url_rd.exists(util.get_MD5(list_page_url)):
         util.write_log_warn(self.lock, self.logger, "list_page_url:" + list_page_url + "was crawled in past %d seconds"%(config.CRAWLED_URL_REDIS_TIMEOUT))
         return False
     browser.desired_capabilities["phantomjs.page.settings.userAgent"] = appstore_class.PHANTOMJS_USER_AGENT
     util.browser_get_url(browser, list_page_url)
     # appstore_class.scroll_down(browser)
     check_more_count = appstore_class.click_checkmore(browser)
     util.write_log_warn(self.lock, self.logger, "列表页: " + list_page_url + "点击checkmore" + str(check_more_count) + "次后找不到checkmore")
     detail_urls = appstore_class.get_detail_urls(browser)
     if len(detail_urls) == 0:
         util.write_log_warn(self.lock, self.logger, "列表页: " + list_page_url + " 找到0个app")
     self.push_detail_page_message(args, detail_urls)
     next_list_page = appstore_class.click_nextpage(browser)
     if next_list_page:
         self.push_list_page_message(args, next_list_page)
     util.write_log_info(self.lock, self.logger,"SUCCESS : list_page_url: %s crawled"%(list_page_url))
     self.url_rd.set(util.get_MD5(list_page_url),1)
     self.url_rd.expire(util.get_MD5(list_page_url), config.CRAWLED_URL_REDIS_TIMEOUT)
     return True
コード例 #3
0
 def crawl(self, args, browser):
     source = args['source']
     detail_page_url = args['detail_page_url']
     try:
         appstore_modual = __import__(source)
     except ImportError:
         util.write_log_error(self.lock, self.logger, "can not find " + source + ".py in configured appstores")
         return False
     appstore_class = appstore_modual.Store()
     if self.url_rd.exists(util.get_MD5(detail_page_url)):
         util.write_log_warn(self.lock, self.logger, "detail_page_url:" + detail_page_url + "was crawled in past %d seconds"%(config.CRAWLED_URL_REDIS_TIMEOUT))
         return False
     browser.desired_capabilities["phantomjs.page.settings.userAgent"] = appstore_class.PHANTOMJS_USER_AGENT
     util.browser_get_url(browser, detail_page_url)
     download_url = appstore_class.get_download_url(browser)
     if download_url is None:
         util.write_log_error(self.lock, self.logger, "详情页: " + detail_page_url + " 找不到download_url")
         return False
     out_json_message = appstore_class.make_detail2download_json_message(browser)
     out_json_message['detail_url'] = detail_page_url
     out_json_message['download_url'] = download_url
     if args['level'] = 'high':
         out_json_message['next_queue'] = config.HIGH_DOWNLOAD_QUEUE
コード例 #4
0
 def crawl(self, args):
     file_name = args['file_name']
     target_folder = args['target_folder']
     apk_info = self.prepare_apk_info(args)
     util.makeDirs(self.lock, target_folder)
     if not util.verify_apk(file_name):
         util.write_log_warn(self.lock, self.logger,
                             "FAILED :  %s jarsigner faild." % (file_name))
         os.remove(file_name)
         return False
     md5 = util.get_file_MD5(file_name)
     apk_info['MD5'] = md5
     target_name = target_folder + 'cell-' + md5[:2] + '/' + md5 + '.apk'
     if os.path.exists(target_name):
         os.remove(file_name)
     else:
         apk_info['new'] = 'True'
         os.rename(args['file_name'], target_name)
     util.writeApkInfoFile(self.lock, target_folder, apk_info)
     util.writeRecord(self.lock, target_folder, apk_info)
     util.write_log_info(
         self.lock, self.logger,
         "SUCCESS : %s handle success" % (args['file_name']))
     return True
コード例 #5
0
 def normal(self, args, browser):
     util.write_log_warn(self.lock, self.logger, '"normal" message type did nothing.')
     return True