def wrapper(*args, **kwargs):
cfg = Configure('./config/config.cfg')
mongo_url = cfg.get_config('mongodb', 'url')
mongo_port = int(cfg.get_config('mongodb', 'port'))
dbname = cfg.get_config('mongodb', 'dbname')
collection_name = cfg.get_config('mongodb', tablename)
client = MongoClient(mongo_url, mongo_port)
db=client[dbname]
collection = db[collection_name]
res = func(*args, c = collection, **kwargs)
return res
def calculate_index():
print '%s\tstart update index' % str(datetime.now())
cfg = Configure('./config/config.cfg')
article_repo = cfg.get_config('running', 'article_repo')
file_names = os.listdir(
os.path.join(article_repo,
datetime.today().strftime('%m%d%Y')))
doc_count = 0
for filename in file_names:
if not filename.endswith('_DONE'):
process(os.path.join(article_repo, filename), filename)
os.rename(os.path.join(article_repo, filename),
os.path.join(article_repo, filename + '_DONE'))
doc_count += 1
if doc_count:
update_tfidf(doc_count)
print '%s\tend update index' % str(datetime.now())
def start_recording(config_name, recording_num_scale, flag_show_windows, auto_save, port=None, weathers=None, vehicle_nums=None):
config, config_file = Configure.get_config(config_name)
if port is None:
port = config['port']
cameras = config['cameras']
if vehicle_nums is None:
vehicle_nums = config['vehicle_num']
if weathers is None:
weathers = config['weathers']
all_conditions = [(w, v) for w in weathers for v in vehicle_nums]
if config["mode"] == "parallel":
for cond in all_conditions:
recorder = SRecorder(host=config['host'],
port=port,
save_root=config['save_root'],
weather_name=cond[0],
vehicle_num=cond[1],
flag_show_windows=flag_show_windows,
auto_save=auto_save,
config=config,
config_file = config_file
)
camera_keys = cameras.keys()
for _, k in zip(trange(len(cameras)), cameras.keys()):
recorder.create_rgb_camera(
width=cameras[k]["width"],
height=cameras[k]["height"],
x=cameras[k]["x"],
y=cameras[k]["y"],
z=cameras[k]["z"],
fov=cameras[k]["fov"],
pitch=cameras[k]["pitch"],
yaw=cameras[k]["yaw"],
roll=cameras[k]["roll"],
max_record_frame=int(cameras[k]["max_record_frame"]*recording_num_scale),
camera_tag=k
)
recorder.run()
elif config["mode"] == "serial":
for cond in all_conditions:
for k in cameras.keys():
recorder = SRecorder(host=config['host'],
port=port,
save_root=config['save_root'],
weather_name=cond[0],
vehicle_num=cond[1],
flag_show_windows=flag_show_windows,
auto_save=auto_save,
config=config,
config_file = config_file
)
recorder.create_rgb_camera(
width=cameras[k]["width"],
height=cameras[k]["height"],
x=cameras[k]["x"],
y=cameras[k]["y"],
z=cameras[k]["z"],
fov=cameras[k]["fov"],
pitch=cameras[k]["pitch"],
yaw=cameras[k]["yaw"],
roll=cameras[k]["roll"],
max_record_frame=int(cameras[k]["max_record_frame"]*recording_num_scale),
camera_tag=k
)
recorder.run()
import os
os.chdir(os.path.dirname(os.path.realpath(__file__)))
import hashlib
import time
import newspaper
import feedparser
from datetime import datetime
from random import shuffle
from config import Configure
from util import Util
CFG = Configure('./config/config.cfg')
ARTICLEREPO = os.path.join(CFG.get_config('running', 'article_repo'),
datetime.today().strftime('%m%d%Y'))
if not os.path.isdir(ARTICLEREPO):
os.makedirs(ARTICLEREPO)
def crawl_web_categorypage(url):
paper = newspaper.build(url, memoize_articles=False, language='en')
for category in paper.category_urls():
crawl_web(category)
def crawl_web(url):
paper = newspaper.build(url, memoize_articles=False, language='en')
