def exportFile(data, downloader, file_format):
exporting_function = {'xlsx': data.xlsx,
'json': data.json.encode('UTF-8'),
'yaml': data.yaml.encode('UTF-8')}
if file_format == 'csv':
with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f:
f.write(text.join(u',', data.headers) + '\n')
writer = csv.writer(f)
for i in range(0, len(data)):
row = [text(y) if not y is None else u"" for y in data[i]]
writer.writerow(row)
elif file_format == 'sqlite':
db = sqlite3.connect(f'{downloader.type}.{file_format}')
df = data.export('df')
df.to_sql('YAHOO_TICKERS', db, if_exists='replace')
db.commit()
db.close()
elif file_format in [item for item in formats if item != 'csv']:
try:
with open(f'{downloader.type}.{file_format}', 'wb') as f:
f.write(exporting_function[file_format])
except:
logger.warning(f"Could not export .{file_format} due to a internal error")
else:
logger.error('Unknown output format')
def main():
downloader = None
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--insecure",
help="use HTTP instead of HTTPS",
action="store_true")
parser.add_argument(
"-e",
"--export",
help=
"export immediately without downloading (Only useful if you already downloaded something to the .pickle file)",
action="store_true")
parser.add_argument(
'-E',
'--Exchange',
help=
'Only export ticker symbols from this exchange (the filtering is done during the export phase)'
)
parser.add_argument('type',
nargs='?',
default='tiger',
help='The type to download, this can be: ' +
" ".join(list(options.keys())))
parser.add_argument("-s",
"--sleep",
help="The time to sleep in seconds between requests",
type=float,
default=0)
parser.add_argument(
"-p",
"--pandantic",
help="Stop and warn the user if some rare assertion fails",
action="store_true")
args = parser.parse_args()
protocol = 'http' if args.insecure else 'https'
if args.insecure:
print("Using insecure connection")
if args.export:
print("Exporting pickle file")
tickerType = args.type = args.type.lower()
print("Checking if we can resume a old download session")
try:
downloader = loadDownloader(tickerType)
print("Downloader found on disk, resuming")
except:
print("No old downloader found on disk")
print("Starting a new session")
if tickerType not in options:
print("Error: " + tickerType +
" is not a valid type option. See --help")
exit(1)
else:
downloader = options[tickerType]
rp = robotparser.RobotFileParser()
rp.set_url(protocol + '://finance.yahoo.com/robots.txt')
rp.read()
try:
if not args.export:
if (not rp.can_fetch(
user_agent, protocol +
'://finance.yahoo.com/_finance_doubledown/api/resource/searchassist'
)):
print('Execution of script halted due to robots.txt')
return 1
if not downloader.isDone():
print("Downloading " + downloader.type)
print("")
downloadEverything(downloader, tickerType, args.insecure,
args.sleep, args.pandantic)
print("Saving downloader to disk...")
saveDownloader(downloader, tickerType)
print("Downloader successfully saved.")
print("")
else:
print(
"The downloader has already finished downloading everything"
)
print("")
except Exception as ex:
print(
"A exception occurred while downloading. Suspending downloader to disk"
)
saveDownloader(downloader, tickerType)
print("Successfully saved download state")
print("Try removing {type}.pickle file if this error persists")
print(
"Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues"
)
print("")
raise
except KeyboardInterrupt as ex:
print("Suspending downloader to disk as .pickle file")
saveDownloader(downloader, tickerType)
if downloader.isDone() or args.export:
print("Exporting " + downloader.type + " symbols")
data = tablib.Dataset()
data.headers = downloader.getRowHeader()
for symbol in downloader.getCollectedSymbols():
if (args.Exchange == None):
data.append(symbol.getRow())
elif (symbol.exchange == args.Exchange):
data.append(symbol.getRow())
with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f:
f.write(text.join(u',', data.headers) + '\n')
writer = csv.writer(f)
for i in range(0, len(data)):
row = [text(y) if not y is None else u"" for y in data[i]]
writer.writerow(row)
try:
with open(downloader.type + '.xlsx', 'wb') as f:
f.write(data.xlsx)
except:
print("Could not export .xlsx due to a internal error")
try:
with open(downloader.type + '.json', 'wb') as f:
f.write(data.json.encode('UTF-8'))
except:
print("Could not export .json due to a internal error")
try:
with open(downloader.type + '.yaml', 'wb') as f:
f.write(data.yaml.encode('UTF-8'))
except:
print("Could not export .yaml due to a internal error")
def main():
downloader = None
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--insecure", help="use HTTP instead of HTTPS", action="store_true")
parser.add_argument("-e", "--export", help="export immediately without downloading (Only useful if you already downloaded something to the .pickle file)", action="store_true")
parser.add_argument('-E', '--Exchange', help='Only export ticker symbols from this exchange (the filtering is done during the export phase)')
parser.add_argument('type', help='The type to download, this can be: '+" ".join(list(options.keys())))
parser.add_argument("-s", "--sleep", help="The time to sleep in seconds between requests", type=float, default=0)
parser.add_argument("-p", "--pandantic", help="Stop and warn the user if some rare assertion fails", action="store_true")
parser.add_argument("-m", "--market", help="Specify the Region of queried exchanges (us = USA+Canada, dr=Germany, fr=France, hk=Hongkong, gb=United Kingdom, default= all)", default="all")
args = parser.parse_args()
if args.insecure:
print("Using insecure connection")
if args.export:
print("Exporting pickle file")
tickerType = args.type = args.type.lower()
market = args.market = args.market.lower()
print("Checking if we can resume a old download session")
try:
downloader = loadDownloader(tickerType)
print("Downloader found on disk, resuming")
except:
print("No old downloader found on disk")
print("Starting a new session")
if tickerType not in options:
print("Error: " + tickerType + " is not a valid type option. See --help")
exit(1)
else:
downloader = options[tickerType]
try:
if not args.export:
if not downloader.isDone():
print("Downloading " + downloader.type)
print("")
downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic, market)
print ("Saving downloader to disk...")
saveDownloader(downloader, tickerType)
print ("Downloader successfully saved.")
print ("")
else:
print("The downloader has already finished downloading everything")
print("")
except Exception as ex:
print("A exception occurred while downloading. Suspending downloader to disk")
saveDownloader(downloader, tickerType)
print("Successfully saved download state")
print("Try removing {type}.pickle file if this error persists")
print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues")
print("")
raise
except KeyboardInterrupt as ex:
print("Suspending downloader to disk as .pickle file")
saveDownloader(downloader, tickerType)
if downloader.isDone() or args.export:
print("Exporting "+downloader.type+" symbols")
data = tablib.Dataset()
data.headers = downloader.getRowHeader()
for symbol in downloader.getCollectedSymbols():
if(args.Exchange == None):
data.append(symbol.getRow())
elif (symbol.exchange == args.Exchange):
data.append(symbol.getRow())
with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f:
f.write(text.join(u',', data.headers) + '\n')
writer = csv.writer(f)
for i in range(0, len(data)):
row = [text(y) if not y is None else u"" for y in data[i]]
writer.writerow(row)
with open(downloader.type + '.xlsx', 'wb') as f:
f.write(data.xlsx)
with open(downloader.type + '.json', 'wb') as f:
f.write(data.json.encode('UTF-8'))
with open(downloader.type + '.yaml', 'wb') as f:
f.write(data.yaml.encode('UTF-8'))
def main():
downloader = None
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--insecure", help="use HTTP instead of HTTPS", action="store_true")
parser.add_argument("-e", "--export", help="export immediately without downloading (Only useful if you already downloaded something to the .pickle file)", action="store_true")
parser.add_argument('-E', '--Exchange', help='Only export ticker symbols from this exchange (the filtering is done during the export phase)')
parser.add_argument('type', nargs='?', default='generic', help='The type to download, this can be: '+" ".join(list(options.keys())))
parser.add_argument("-s", "--sleep", help="The time to sleep in seconds between requests", type=float, default=0)
parser.add_argument("-p", "--pandantic", help="Stop and warn the user if some rare assertion fails", action="store_true")
args = parser.parse_args()
protocol = 'http' if args.insecure else 'https'
if args.insecure:
print("Using insecure connection")
if args.export:
print("Exporting pickle file")
tickerType = args.type = args.type.lower()
print("Checking if we can resume a old download session")
try:
downloader = loadDownloader(tickerType)
print("Downloader found on disk, resuming")
except:
print("No old downloader found on disk")
print("Starting a new session")
if tickerType not in options:
print("Error: " + tickerType + " is not a valid type option. See --help")
exit(1)
else:
downloader = options[tickerType]
rp = robotparser.RobotFileParser()
rp.set_url(protocol + '://finance.yahoo.com/robots.txt')
rp.read()
try:
if not args.export:
if(not rp.can_fetch(user_agent, protocol + '://finance.yahoo.com/_finance_doubledown/api/resource/searchassist')):
print('Execution of script halted due to robots.txt')
return 1
if not downloader.isDone():
print("Downloading " + downloader.type)
print("")
downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic)
print ("Saving downloader to disk...")
saveDownloader(downloader, tickerType)
print ("Downloader successfully saved.")
print ("")
else:
print("The downloader has already finished downloading everything")
print("")
except Exception as ex:
print("A exception occurred while downloading. Suspending downloader to disk")
saveDownloader(downloader, tickerType)
print("Successfully saved download state")
print("Try removing {type}.pickle file if this error persists")
print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues")
print("")
raise
except KeyboardInterrupt as ex:
print("Suspending downloader to disk as .pickle file")
saveDownloader(downloader, tickerType)
if downloader.isDone() or args.export:
print("Exporting "+downloader.type+" symbols")
data = tablib.Dataset()
data.headers = downloader.getRowHeader()
for symbol in downloader.getCollectedSymbols():
if(args.Exchange == None):
data.append(symbol.getRow())
elif (symbol.exchange == args.Exchange):
data.append(symbol.getRow())
with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f:
f.write(text.join(u',', data.headers) + '\n')
writer = csv.writer(f)
for i in range(0, len(data)):
row = [text(y) if not y is None else u"" for y in data[i]]
writer.writerow(row)
try:
with open(downloader.type + '.xlsx', 'wb') as f:
f.write(data.xlsx)
except:
print("Could not export .xlsx due to a internal error")
try:
with open(downloader.type + '.json', 'wb') as f:
f.write(data.json.encode('UTF-8'))
except:
print("Could not export .json due to a internal error")
try:
with open(downloader.type + '.yaml', 'wb') as f:
f.write(data.yaml.encode('UTF-8'))
except:
print("Could not export .yaml due to a internal error")