def __init__(self, **kwargs): super(EdgarSpider, self).__init__(**kwargs) symbols_arg = kwargs.get('symbols') start_date = kwargs.get('startdate', '') end_date = kwargs.get('enddate', '') limit_arg = kwargs.get('limit', '') utils.check_date_arg(start_date, 'startdate') utils.check_date_arg(end_date, 'enddate') start, count = utils.parse_limit_arg(limit_arg) if symbols_arg: if os.path.exists(symbols_arg): # get symbols from a text file symbols = utils.load_symbols(symbols_arg) else: # inline symbols in command symbols = symbols_arg.split(',') self.start_urls = URLGenerator(symbols, start_date, end_date, start, count) for one_url in self.start_urls: print(one_url) else: self.start_urls = []
def __init__(self, **kwargs): super(YahooSpider, self).__init__(**kwargs) symbols_arg = kwargs.get('symbols') start_date = kwargs.get('startdate', '') end_date = kwargs.get('enddate', '') utils.check_date_arg(start_date, 'startdate') utils.check_date_arg(end_date, 'enddate') if symbols_arg: if os.path.exists(symbols_arg): # get symbols from a text file symbols = utils.load_symbols(symbols_arg) else: # inline symbols in command symbols = symbols_arg.split(',') self.start_urls = generate_urls(symbols, start_date, end_date) else: self.start_urls = []
def __init__(self, **kwargs): super(EdgarSpider, self).__init__(**kwargs) symbols_arg = kwargs.get("symbols") start_date = kwargs.get("startdate", "") end_date = kwargs.get("enddate", "") limit_arg = kwargs.get("limit", "") utils.check_date_arg(start_date, "startdate") utils.check_date_arg(end_date, "enddate") start, count = utils.parse_limit_arg(limit_arg) if symbols_arg: if os.path.exists(symbols_arg): # get symbols from a text file symbols = utils.load_symbols(symbols_arg) else: # inline symbols in command symbols = symbols_arg.split(",") self.start_urls = URLGenerator(symbols, start_date, end_date, start, count) else: self.start_urls = []
def test_check_date_arg(self): utils.check_date_arg('19830305') utils.check_date_arg('19851122') utils.check_date_arg('19980720') utils.check_date_arg('20140212') # OK to pass an empty argument utils.check_date_arg('') with self.assertRaises(ValueError): utils.check_date_arg('1234') with self.assertRaises(ValueError): utils.check_date_arg('2014111') with self.assertRaises(ValueError): utils.check_date_arg('20141301') with self.assertRaises(ValueError): utils.check_date_arg('20140132')