def wrapper(*args, **kwargs): cfg = Configure('./config/config.cfg') mongo_url = cfg.get_config('mongodb', 'url') mongo_port = int(cfg.get_config('mongodb', 'port')) dbname = cfg.get_config('mongodb', 'dbname') collection_name = cfg.get_config('mongodb', tablename) client = MongoClient(mongo_url, mongo_port) db=client[dbname] collection = db[collection_name] res = func(*args, c = collection, **kwargs) return res
def calculate_index(): print '%s\tstart update index' % str(datetime.now()) cfg = Configure('./config/config.cfg') article_repo = cfg.get_config('running', 'article_repo') file_names = os.listdir( os.path.join(article_repo, datetime.today().strftime('%m%d%Y'))) doc_count = 0 for filename in file_names: if not filename.endswith('_DONE'): process(os.path.join(article_repo, filename), filename) os.rename(os.path.join(article_repo, filename), os.path.join(article_repo, filename + '_DONE')) doc_count += 1 if doc_count: update_tfidf(doc_count) print '%s\tend update index' % str(datetime.now())
def start_recording(config_name, recording_num_scale, flag_show_windows, auto_save, port=None, weathers=None, vehicle_nums=None): config, config_file = Configure.get_config(config_name) if port is None: port = config['port'] cameras = config['cameras'] if vehicle_nums is None: vehicle_nums = config['vehicle_num'] if weathers is None: weathers = config['weathers'] all_conditions = [(w, v) for w in weathers for v in vehicle_nums] if config["mode"] == "parallel": for cond in all_conditions: recorder = SRecorder(host=config['host'], port=port, save_root=config['save_root'], weather_name=cond[0], vehicle_num=cond[1], flag_show_windows=flag_show_windows, auto_save=auto_save, config=config, config_file = config_file ) camera_keys = cameras.keys() for _, k in zip(trange(len(cameras)), cameras.keys()): recorder.create_rgb_camera( width=cameras[k]["width"], height=cameras[k]["height"], x=cameras[k]["x"], y=cameras[k]["y"], z=cameras[k]["z"], fov=cameras[k]["fov"], pitch=cameras[k]["pitch"], yaw=cameras[k]["yaw"], roll=cameras[k]["roll"], max_record_frame=int(cameras[k]["max_record_frame"]*recording_num_scale), camera_tag=k ) recorder.run() elif config["mode"] == "serial": for cond in all_conditions: for k in cameras.keys(): recorder = SRecorder(host=config['host'], port=port, save_root=config['save_root'], weather_name=cond[0], vehicle_num=cond[1], flag_show_windows=flag_show_windows, auto_save=auto_save, config=config, config_file = config_file ) recorder.create_rgb_camera( width=cameras[k]["width"], height=cameras[k]["height"], x=cameras[k]["x"], y=cameras[k]["y"], z=cameras[k]["z"], fov=cameras[k]["fov"], pitch=cameras[k]["pitch"], yaw=cameras[k]["yaw"], roll=cameras[k]["roll"], max_record_frame=int(cameras[k]["max_record_frame"]*recording_num_scale), camera_tag=k ) recorder.run()
import os os.chdir(os.path.dirname(os.path.realpath(__file__))) import hashlib import time import newspaper import feedparser from datetime import datetime from random import shuffle from config import Configure from util import Util CFG = Configure('./config/config.cfg') ARTICLEREPO = os.path.join(CFG.get_config('running', 'article_repo'), datetime.today().strftime('%m%d%Y')) if not os.path.isdir(ARTICLEREPO): os.makedirs(ARTICLEREPO) def crawl_web_categorypage(url): paper = newspaper.build(url, memoize_articles=False, language='en') for category in paper.category_urls(): crawl_web(category) def crawl_web(url): paper = newspaper.build(url, memoize_articles=False, language='en')