def exportFile(data, downloader, file_format): exporting_function = {'xlsx': data.xlsx, 'json': data.json.encode('UTF-8'), 'yaml': data.yaml.encode('UTF-8')} if file_format == 'csv': with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f: f.write(text.join(u',', data.headers) + '\n') writer = csv.writer(f) for i in range(0, len(data)): row = [text(y) if not y is None else u"" for y in data[i]] writer.writerow(row) elif file_format == 'sqlite': db = sqlite3.connect(f'{downloader.type}.{file_format}') df = data.export('df') df.to_sql('YAHOO_TICKERS', db, if_exists='replace') db.commit() db.close() elif file_format in [item for item in formats if item != 'csv']: try: with open(f'{downloader.type}.{file_format}', 'wb') as f: f.write(exporting_function[file_format]) except: logger.warning(f"Could not export .{file_format} due to a internal error") else: logger.error('Unknown output format')
def main(): downloader = None parser = argparse.ArgumentParser() parser.add_argument("-i", "--insecure", help="use HTTP instead of HTTPS", action="store_true") parser.add_argument( "-e", "--export", help= "export immediately without downloading (Only useful if you already downloaded something to the .pickle file)", action="store_true") parser.add_argument( '-E', '--Exchange', help= 'Only export ticker symbols from this exchange (the filtering is done during the export phase)' ) parser.add_argument('type', nargs='?', default='tiger', help='The type to download, this can be: ' + " ".join(list(options.keys()))) parser.add_argument("-s", "--sleep", help="The time to sleep in seconds between requests", type=float, default=0) parser.add_argument( "-p", "--pandantic", help="Stop and warn the user if some rare assertion fails", action="store_true") args = parser.parse_args() protocol = 'http' if args.insecure else 'https' if args.insecure: print("Using insecure connection") if args.export: print("Exporting pickle file") tickerType = args.type = args.type.lower() print("Checking if we can resume a old download session") try: downloader = loadDownloader(tickerType) print("Downloader found on disk, resuming") except: print("No old downloader found on disk") print("Starting a new session") if tickerType not in options: print("Error: " + tickerType + " is not a valid type option. See --help") exit(1) else: downloader = options[tickerType] rp = robotparser.RobotFileParser() rp.set_url(protocol + '://finance.yahoo.com/robots.txt') rp.read() try: if not args.export: if (not rp.can_fetch( user_agent, protocol + '://finance.yahoo.com/_finance_doubledown/api/resource/searchassist' )): print('Execution of script halted due to robots.txt') return 1 if not downloader.isDone(): print("Downloading " + downloader.type) print("") downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic) print("Saving downloader to disk...") saveDownloader(downloader, tickerType) print("Downloader successfully saved.") print("") else: print( "The downloader has already finished downloading everything" ) print("") except Exception as ex: print( "A exception occurred while downloading. Suspending downloader to disk" ) saveDownloader(downloader, tickerType) print("Successfully saved download state") print("Try removing {type}.pickle file if this error persists") print( "Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues" ) print("") raise except KeyboardInterrupt as ex: print("Suspending downloader to disk as .pickle file") saveDownloader(downloader, tickerType) if downloader.isDone() or args.export: print("Exporting " + downloader.type + " symbols") data = tablib.Dataset() data.headers = downloader.getRowHeader() for symbol in downloader.getCollectedSymbols(): if (args.Exchange == None): data.append(symbol.getRow()) elif (symbol.exchange == args.Exchange): data.append(symbol.getRow()) with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f: f.write(text.join(u',', data.headers) + '\n') writer = csv.writer(f) for i in range(0, len(data)): row = [text(y) if not y is None else u"" for y in data[i]] writer.writerow(row) try: with open(downloader.type + '.xlsx', 'wb') as f: f.write(data.xlsx) except: print("Could not export .xlsx due to a internal error") try: with open(downloader.type + '.json', 'wb') as f: f.write(data.json.encode('UTF-8')) except: print("Could not export .json due to a internal error") try: with open(downloader.type + '.yaml', 'wb') as f: f.write(data.yaml.encode('UTF-8')) except: print("Could not export .yaml due to a internal error")
def main(): downloader = None parser = argparse.ArgumentParser() parser.add_argument("-i", "--insecure", help="use HTTP instead of HTTPS", action="store_true") parser.add_argument("-e", "--export", help="export immediately without downloading (Only useful if you already downloaded something to the .pickle file)", action="store_true") parser.add_argument('-E', '--Exchange', help='Only export ticker symbols from this exchange (the filtering is done during the export phase)') parser.add_argument('type', help='The type to download, this can be: '+" ".join(list(options.keys()))) parser.add_argument("-s", "--sleep", help="The time to sleep in seconds between requests", type=float, default=0) parser.add_argument("-p", "--pandantic", help="Stop and warn the user if some rare assertion fails", action="store_true") parser.add_argument("-m", "--market", help="Specify the Region of queried exchanges (us = USA+Canada, dr=Germany, fr=France, hk=Hongkong, gb=United Kingdom, default= all)", default="all") args = parser.parse_args() if args.insecure: print("Using insecure connection") if args.export: print("Exporting pickle file") tickerType = args.type = args.type.lower() market = args.market = args.market.lower() print("Checking if we can resume a old download session") try: downloader = loadDownloader(tickerType) print("Downloader found on disk, resuming") except: print("No old downloader found on disk") print("Starting a new session") if tickerType not in options: print("Error: " + tickerType + " is not a valid type option. See --help") exit(1) else: downloader = options[tickerType] try: if not args.export: if not downloader.isDone(): print("Downloading " + downloader.type) print("") downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic, market) print ("Saving downloader to disk...") saveDownloader(downloader, tickerType) print ("Downloader successfully saved.") print ("") else: print("The downloader has already finished downloading everything") print("") except Exception as ex: print("A exception occurred while downloading. Suspending downloader to disk") saveDownloader(downloader, tickerType) print("Successfully saved download state") print("Try removing {type}.pickle file if this error persists") print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues") print("") raise except KeyboardInterrupt as ex: print("Suspending downloader to disk as .pickle file") saveDownloader(downloader, tickerType) if downloader.isDone() or args.export: print("Exporting "+downloader.type+" symbols") data = tablib.Dataset() data.headers = downloader.getRowHeader() for symbol in downloader.getCollectedSymbols(): if(args.Exchange == None): data.append(symbol.getRow()) elif (symbol.exchange == args.Exchange): data.append(symbol.getRow()) with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f: f.write(text.join(u',', data.headers) + '\n') writer = csv.writer(f) for i in range(0, len(data)): row = [text(y) if not y is None else u"" for y in data[i]] writer.writerow(row) with open(downloader.type + '.xlsx', 'wb') as f: f.write(data.xlsx) with open(downloader.type + '.json', 'wb') as f: f.write(data.json.encode('UTF-8')) with open(downloader.type + '.yaml', 'wb') as f: f.write(data.yaml.encode('UTF-8'))
def main(): downloader = None parser = argparse.ArgumentParser() parser.add_argument("-i", "--insecure", help="use HTTP instead of HTTPS", action="store_true") parser.add_argument("-e", "--export", help="export immediately without downloading (Only useful if you already downloaded something to the .pickle file)", action="store_true") parser.add_argument('-E', '--Exchange', help='Only export ticker symbols from this exchange (the filtering is done during the export phase)') parser.add_argument('type', nargs='?', default='generic', help='The type to download, this can be: '+" ".join(list(options.keys()))) parser.add_argument("-s", "--sleep", help="The time to sleep in seconds between requests", type=float, default=0) parser.add_argument("-p", "--pandantic", help="Stop and warn the user if some rare assertion fails", action="store_true") args = parser.parse_args() protocol = 'http' if args.insecure else 'https' if args.insecure: print("Using insecure connection") if args.export: print("Exporting pickle file") tickerType = args.type = args.type.lower() print("Checking if we can resume a old download session") try: downloader = loadDownloader(tickerType) print("Downloader found on disk, resuming") except: print("No old downloader found on disk") print("Starting a new session") if tickerType not in options: print("Error: " + tickerType + " is not a valid type option. See --help") exit(1) else: downloader = options[tickerType] rp = robotparser.RobotFileParser() rp.set_url(protocol + '://finance.yahoo.com/robots.txt') rp.read() try: if not args.export: if(not rp.can_fetch(user_agent, protocol + '://finance.yahoo.com/_finance_doubledown/api/resource/searchassist')): print('Execution of script halted due to robots.txt') return 1 if not downloader.isDone(): print("Downloading " + downloader.type) print("") downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic) print ("Saving downloader to disk...") saveDownloader(downloader, tickerType) print ("Downloader successfully saved.") print ("") else: print("The downloader has already finished downloading everything") print("") except Exception as ex: print("A exception occurred while downloading. Suspending downloader to disk") saveDownloader(downloader, tickerType) print("Successfully saved download state") print("Try removing {type}.pickle file if this error persists") print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues") print("") raise except KeyboardInterrupt as ex: print("Suspending downloader to disk as .pickle file") saveDownloader(downloader, tickerType) if downloader.isDone() or args.export: print("Exporting "+downloader.type+" symbols") data = tablib.Dataset() data.headers = downloader.getRowHeader() for symbol in downloader.getCollectedSymbols(): if(args.Exchange == None): data.append(symbol.getRow()) elif (symbol.exchange == args.Exchange): data.append(symbol.getRow()) with io.open(downloader.type + '.csv', 'w', encoding='utf-8') as f: f.write(text.join(u',', data.headers) + '\n') writer = csv.writer(f) for i in range(0, len(data)): row = [text(y) if not y is None else u"" for y in data[i]] writer.writerow(row) try: with open(downloader.type + '.xlsx', 'wb') as f: f.write(data.xlsx) except: print("Could not export .xlsx due to a internal error") try: with open(downloader.type + '.json', 'wb') as f: f.write(data.json.encode('UTF-8')) except: print("Could not export .json due to a internal error") try: with open(downloader.type + '.yaml', 'wb') as f: f.write(data.yaml.encode('UTF-8')) except: print("Could not export .yaml due to a internal error")