def __test():
"""
具体的测试
"""
def print_result(c):
while True:
if c.is_avaliable():
c.print_progress()
time.sleep(3)
from config import MyConfig
MyConfig.do_default()
c = Crawler(MyConfig)
threading.Thread(group = None, target=print_result, args=(c,)).start()
c.start()
def __init__(self, **kwargs):
# Create carousel - doesn't work so well in .kv
Carousel.__init__(self,
direction='right',
loop='true',
scroll_distance=80,
scroll_timeout=100,
**kwargs)
self.config = MyConfig()
self.backlight = BacklightFactory.Make(self.config)
self.overview = Overview()
self.add_widget(self.overview)
self.add_widget(Label(text='Hello World2'))
self.game = PongGame()
self.game.serve_ball()
self.add_widget(self.game)
self.idle_clock = Clock.schedule_once(
self.on_idle, float(self.config.switch_off_time))
Window.bind(on_motion=self.on_motion)
def main():
myconfig = get_config()
if myconfig == False:
# 解析参数出错，直接退出
return
if myconfig == None:
print "do not have any configuration, then switch testself truns on!"
myconfig = MyConfig()
myconfig.testself = True
if myconfig.testself:
# 如果 testself开关 trun on
# 则 其它设置默认无效
myconfig.do_default()
else:
myconfig.check_config()
myconfig.show_config()
crawler = Crawler(myconfig)
progress_info(crawler)
crawler.start()
def start(envType, sleeptime):
'''开始入口，构造cmd'''
# global var
my_config = MyConfig()
host = my_config.get(envType)['host']
username = my_config.get(envType)['username']
password = my_config.get(envType)['password']
if 0 == sleeptime:
sleeptime = my_config.get(envType)['sleeptime']
#generate cmd
# 执行 mongotop 需要账号可以访问 admin 库 权限
my_cmd = "mongotop --host %s --username %s --password %s --authenticationDatabase=admin -vvvvv --json %d" \
% (host, username, password, sleeptime)
# my_cmd = "/usr/bin/mongotop --version"
logger.info("cmd is %s", my_cmd)
#generate connection uri
db_uri = my_config.get(envType)['dbUri']
global collection_name
collection_name = my_config.get(envType)['collection']
# 数据写到 哪个库
database = my_config.get(envType)['database']
logger.info("db uri is %s", db_uri)
client = pymongo.MongoClient(db_uri)
db = client[database]
global collection
collection = db[collection_name]
getTopOutput(my_cmd, sleeptime)
pass
from multiprocessing import Pool
import multiprocessing as multi
from send2trash import send2trash
from config import MyConfig
from common import calc_hu_moments, CalcDiff
from leveldb import LevelDB
PARALLEL_COUNT_MAX = 2
DIFF_THRESHOLD = 5.0e-7
DOUBLE_FMT = 'd'
if __name__ == "__main__":
parallel = Pool(min(PARALLEL_COUNT_MAX, multi.cpu_count()))
config = MyConfig()
images_dir = sys.argv[1] if 2 <= len(
sys.argv) else config.path.original_images_dir
db = LevelDB(config.path.end_nearly_dirs_file, DOUBLE_FMT)
for p, _, fs in os.walk(images_dir):
if fs and DIFF_THRESHOLD < (db.get(p) or 1.0):
print(p)
(hu_moments, _) = calc_hu_moments([path.join(p, f) for f in fs],
parallel)
filtered_diff = parallel.map(CalcDiff(hu_moments, DIFF_THRESHOLD),
enumerate(hu_moments))
failed = [
hu.fname for hu, b in zip(hu_moments, filtered_diff) if b
]
for fname in failed:
print(" Trash: %s" % fname)