def import_modified_ios_resource(self):
repository.load()
fileOperator = FileOperator()
# 更新词条
for f in os.listdir(self.appConfig.ios_resources_root_directory):
language = self.__get_ios_file_language(f)
if len(language) <= 0:
continue
path = os.path.join(self.appConfig.ios_resources_root_directory, f,
"Localizable.strings")
dist = fileOperator.read_ios_keywords(path)
for k, v in dist.items():
repository.try_ro_modify_keyword(k, v, language)
# 重写 repo json
repository.rewrite_repo_json()
def import_ios_resources(self):
fileOperator = FileOperator()
# 解析所有的多语言
for f in os.listdir(self.appConfig.ios_resources_root_directory):
language = self.__get_ios_file_language(f)
if len(language) <= 0:
continue
# 语言名称
self.support_languages.append(language)
# 解析多语言的词条
for f in os.listdir(self.appConfig.ios_resources_root_directory):
language = self.__get_ios_file_language(f)
if len(language) <= 0:
continue
path = os.path.join(self.appConfig.ios_resources_root_directory, f,
"Localizable.strings")
dist = fileOperator.read_ios_keywords(path)
logging.debug("Read iOS keywords : " + str(dist))
for k, v in dist.items():
if k not in self.keywords:
self.keywords.append(k)
if k not in self.translates:
self.translates[k] = {}
for support_language in self.support_languages:
if support_language != language:
self.translates[k][support_language] = ""
else:
self.translates[k][support_language] = v
# 新增多语言的情况：要为应用初始化的时候选中的多语言设置词条
for sl in self.appConfig.support_languages:
if sl not in self.support_languages:
for k, v in self.translates.items():
self.translates[k][sl] = ""
# 输出用于调试的日志
self.appConfig.add_support_languages(self.support_languages)
logging.debug("Parsed From iOS Resources : " +
str(self.support_languages))
logging.debug("Parsed Keywords : " + str(self.keywords))
logging.debug(self.translates)
def gen_ios_resources(self):
repository.load()
ios_blacklist = self.appConfig.ios_language_black_list
fileOperator = FileOperator()
for language in repository.languages:
# 过滤黑名单
if language in ios_blacklist:
continue
dist = {}
for data in repository.datas:
keyword = data["keyword"]
translates = data["translates"]
translation = translates[language]
dist[keyword] = translation
# 写入资源
language_dir = os.path.join(
self.appConfig.ios_resources_root_directory,
language + ".lproj")
if not os.path.exists(language_dir):
os.mkdir(language_dir)
fname = os.path.join(language_dir, "Localizable.strings")
fileOperator.write_ios_resources(dist, fname)
print(' enable pin: ' + str(enable_pin))
print(' sleep time: ' + str(sleep_time))
print(' ---------------------')
print(' x_mil: ' + str(x_mil))
print(' y_mil: ' + str(y_mil))
print(' z_mil: ' + str(z_mil))
inputok = input('- Looks good? Keep going? (y/n) ')
if inputok != 'y':
sys.exit('( ! ) Okay. Stopping ...')
Motor.cleanup(motor)
motor = Motor(xdir, xstep, ydir, ystep, zdir, zstep, enable_pin, sleep_time)
Motor.setup(motor)
#import code; code.interact(local=dict(globals(), **locals()))
operator = FileOperator(motor, x_mil, y_mil, z_mil)
print('')
print('( 3 ) Choosing GCODE File')
files = os.listdir('./gcodefiles/')
filnum = -1
for file in files:
filnum += 1
print(' ' + str(filnum) + ' : ' + file)
inputfile = input('- Choose file by entering (0...' + str(filnum) + '): ')
filename = files[int(inputfile)]
filepath = './gcodefiles/' + filename
print('')
print('( 4 ) Ok. Ready for Printing: ' + filename)
class Spider:
url_queue = Queue()
bf = ScalableBloomFilter(initial_capacity=100000000, error_rate=0.00001)
lock = multiprocessing.Lock()
task_num = TASK_NUM
file_operator = FileOperator(OUT_PUT_DIR, TAR_FILE_SIZE)
def __init__(self, init_url=None, task_num=TASK_NUM):
self.bf.add(init_url)
self.url_queue.put(init_url)
# read from url_list file to build the url_queue
url_list = self.file_operator.read_url()
if len(url_list) > 0:
for url in url_list:
url = url.strip('\n')
if url not in self.bf:
self.bf.add(url)
self.task_num = task_num
def scratch_links(self, res):
res.encoding = res.apparent_encoding
soup = BeautifulSoup(res.text, 'html.parser')
text = str(soup.select('a[target="_blank"]'))
# 正则获取所有url链接
all_links = re.findall(
r'(?<=<a href=\").*?(?=\")|(?<=href=\').*?(?=\')', text)
for link in all_links:
# use = re.findall(r'https?://news.sina.com.cn/.*/doc-iiznezxr.*', link)
use = re.findall(r'https?://.*sina.com.cn/.*', link)
if len(use) > 0:
if re.findall(r'https?://slide.*', use[0]):
continue
elif re.findall(r'https?://.*video.*', use[0]):
continue
elif re.findall(r'https?://career.*', use[0]):
continue
elif re.findall(r'https?://photo.*', use[0]):
continue
elif re.findall(r'https?://.*game.*', use[0]):
continue
elif re.findall(r'https?://blog.*', use[0]):
continue
elif re.findall(r'https?://search.*', use[0]):
continue
elif re.findall(r'https?://baby.*', use[0]):
continue
elif re.findall(r'https?://.*ent..*', use[0]):
continue
elif re.findall(r'https?://app.*', use[0]):
continue
elif re.findall(r'https?://db.*', use[0]):
continue
elif re.findall(r'https?://vip.*', use[0]):
continue
elif re.findall(r'https?://book.*', use[0]):
continue
elif re.findall(r'https?://comment.*', use[0]):
continue
elif re.findall(r'https?://classad.*', use[0]):
continue
elif re.findall(r'https?://aipai.*', use[0]):
continue
elif re.findall(r'https?://ka.*', use[0]):
continue
elif re.findall(r'https?://match.*', use[0]):
continue
elif re.findall(r'https?://sports.sina.com.cn/star/.*',
use[0]):
continue
elif re.findall(r'https?://help.*', use[0]):
continue
if use[0] not in self.bf:
self.bf.add(use[0])
self.url_queue.put(use[0])
def run(self):
while not self.url_queue.empty() and self.file_operator.count < 25:
url = self.url_queue.get()
try:
res = requests.get(url)
except Exception as e:
print(f"url: {url} Error : {e} happended, will try soon")
self.url_queue.put(url)
time.sleep(2)
continue
# 爬取当前url
page_data = page_spider(url, res)
# save data into file
if page_data is None:
print("this page :%s has nothing to write" % url)
else:
self.file_operator.write_data('a', page_data)
self.file_operator.write_url(url=url)
# 将当前页面的所有链接加入队列
self.scratch_links(res)
print("the reset url number is %s" % self.url_queue.qsize())