def __init__(self, market): self.market = market resource = Resource(market) rs0 = resource.get_mongo_rs0() self.col_seed_table = rs0['crawl']['seedTable']
logger.info('invalid type {} '.format(tp)) return metadata = { 'type': tp, 'handle': handle, 'category': category } self.create(url, 'youtubev2_social_seed', 'youtubev2.{}.{}'.format(tp, handle), 'schedule', **metadata) if __name__ == '__main__': logging.basicConfig(format='%(asctime)s, %(name)s %(levelname)s %(message)s', datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) market = 'en-us' resource = Resource(market) rs0 = resource.get_mongo_rs0() col = rs0['crawl']['categories'] lder = Loader(market) import json with open(sys.argv[1]) as f: for line in f: if line.startswith('#'): continue items = line.strip().split(',', 2) tp, handle, ch = items if not ch: ch = handle lder.inject_youtube(tp, handle, ch)