def __init__(self, **kwargs):
super(EdgarSpider, self).__init__(**kwargs)
symbols_arg = kwargs.get('symbols')
start_date = kwargs.get('startdate', '')
end_date = kwargs.get('enddate', '')
limit_arg = kwargs.get('limit', '')
utils.check_date_arg(start_date, 'startdate')
utils.check_date_arg(end_date, 'enddate')
start, count = utils.parse_limit_arg(limit_arg)
if symbols_arg:
if os.path.exists(symbols_arg):
# get symbols from a text file
symbols = utils.load_symbols(symbols_arg)
else:
# inline symbols in command
symbols = symbols_arg.split(',')
self.start_urls = URLGenerator(symbols, start_date, end_date,
start, count)
for one_url in self.start_urls:
print(one_url)
else:
self.start_urls = []
def test_load_symbols(self):
try:
filename = os.path.join(SAMPLE_DATA_DIR, 'test_symbols.txt')
with open(filename, 'w') as f:
f.write('AAPL Apple Inc.\nGOOG\tGoogle Inc.\n# Comment\nFB\nTWTR\nAMZN\nSPY\n\nYHOO\n# The end\n')
symbols = list(utils.load_symbols(filename))
self.assertEqual(symbols, ['AAPL', 'GOOG', 'FB', 'TWTR', 'AMZN', 'SPY', 'YHOO'])
finally:
os.remove(filename)
def test_load_symbols(self):
try:
filename = os.path.join(SAMPLE_DATA_DIR, 'test_symbols.txt')
with open(filename, 'w') as f:
f.write(
'AAPL Apple Inc.\nGOOG\tGoogle Inc.\n# Comment\nFB\nTWTR\nAMZN\nSPY\n\nYHOO\n# The end\n'
)
symbols = list(utils.load_symbols(filename))
self.assertEqual(
symbols, ['AAPL', 'GOOG', 'FB', 'TWTR', 'AMZN', 'SPY', 'YHOO'])
finally:
os.remove(filename)
def __init__(self, **kwargs):
super(YahooSpider, self).__init__(**kwargs)
symbols_arg = kwargs.get('symbols')
start_date = kwargs.get('startdate', '')
end_date = kwargs.get('enddate', '')
utils.check_date_arg(start_date, 'startdate')
utils.check_date_arg(end_date, 'enddate')
if symbols_arg:
if os.path.exists(symbols_arg):
# get symbols from a text file
symbols = utils.load_symbols(symbols_arg)
else:
# inline symbols in command
symbols = symbols_arg.split(',')
self.start_urls = generate_urls(symbols, start_date, end_date)
else:
self.start_urls = []
def __init__(self, **kwargs):
super(EdgarSpider, self).__init__(**kwargs)
symbols_arg = kwargs.get("symbols")
start_date = kwargs.get("startdate", "")
end_date = kwargs.get("enddate", "")
limit_arg = kwargs.get("limit", "")
utils.check_date_arg(start_date, "startdate")
utils.check_date_arg(end_date, "enddate")
start, count = utils.parse_limit_arg(limit_arg)
if symbols_arg:
if os.path.exists(symbols_arg):
# get symbols from a text file
symbols = utils.load_symbols(symbols_arg)
else:
# inline symbols in command
symbols = symbols_arg.split(",")
self.start_urls = URLGenerator(symbols, start_date, end_date, start, count)
else:
self.start_urls = []