def get_filings(companyCode, date='20200101', cik=None, count=100): if cik is None: with open('company_list.txt', 'r') as f: for line in f: if companyCode in line: line_arr = line.rstrip().split(' ') cik = line_arr[-1] if cik is None: print("cik not provided and not found in list. please try again.") return # create object seccrawler = SecCrawler() seccrawler.filing_10K(str(companyCode), str(cik), str(date), str(count)) dest_dir = companyCode + "/" src_dir = dest_dir + cik + "/10-K/" years_downloaded = [] for old_filename in os.listdir(src_dir): parts = old_filename.split('-') old_year = parts[1] if int(old_year) > 50: new_year = '19' + old_year else: new_year = '20' + old_year years_downloaded.append(new_year) os.rename(src_dir + old_filename, dest_dir + companyCode + '_10K_' + new_year + '.txt') shutil.rmtree(dest_dir + cik + '/') # create object seccrawler = SecCrawler() seccrawler.filing_10Q(str(companyCode), str(cik), str(date), str(count)) dest_dir = companyCode + "/" src_dir = dest_dir + cik + "/10-Q/" years_downloaded = [] for old_filename in os.listdir(src_dir): parts = old_filename.split('-') old_year = parts[1] if int(old_year) > 20: new_year = '19' + old_year else: new_year = '20' + old_year years_downloaded.append(new_year) os.rename(src_dir + old_filename, dest_dir + companyCode + '_10Q_' + new_year + '.txt') shutil.rmtree(dest_dir + cik + '/') return years_downloaded
def get_filings(cik, ticker): t1 = time.time() # create object seccrawler = SecCrawler() companyCode = ticker # company code for apple cik = cik # cik code for apple date = '20170101' # date from which filings should be downloaded count = '10' # no of filings seccrawler.filing_10K(str(companyCode), str(cik), str(date), str(count)) t2 = time.time() print("Total Time taken: "), print(t2 - t1)
def get_filings(cik, ticker): t1 = time.time() # create object seccrawler = SecCrawler() companyCode = ticker cik = cik date = '20170101' count = '10' seccrawler.filing_10K(str(companyCode), str(cik), str(date), str(count)) t2 = time.time() print("Total Time taken: "), print(t2 - t1)
def getfilings(): t1 = time.time() # create object seccrawler = SecCrawler() companyCode = 'AAPL' # compnay ticker symbol for Apple cik = '0000320193' # cik code for Apple date = '20010101' # date from which filings should be downloaded (01/01/2001 in this case) count = '10' # the number of filings seccrawler.filing_10Q(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_10K(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_8K(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_13F(str(companyCode), str(cik), str(date), str(count)) t2 = time.time() print("Total time taken: ") print(t2 - t1)
def scrape_edgar(self): """This is the function that will scrape the SEC Edgar website and download all 10-Ks. We are using a third-party package called SECEdgar which does the job, but this would be improved if we wrote our own web crawler that could pre-filter any amended 10-Ks and 10-Ks that fall outside of our date range before downloading them.""" # Remove any duplicates where CUSIP, PERMNO, and CIK match ciks = self.data.drop_duplicates(subset=['CUSIP', 'PERMNO', 'cik']) # Only keep the cik and ticker column ciks = ciks[['cik', 'tic']] # Iterate over each CIK and pull the relevant 10k filings crawler = SecCrawler() end_date = str(self.end) + '1231' count = str(math.ceil((self.end - self.start) / 10) * 10) p = Pool() rows = ciks.to_dict(orient='records') results = p.starmap(crawl, zip(rows, repeat(end_date), repeat(count), repeat(crawler)))
def get_filings(): t1 = time.time() DEFAULT_DATA_PATH = os.path.join(os.path.abspath(os.getcwd()), 'output') # create object seccrawler = SecCrawler() companyCode = 'AAPL' # company code for apple cik = '0000320193' # cik code for apple date = '20010101' # date from which filings should be downloaded count = '10' # no of filings seccrawler.filing_10Q(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_10K(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_8K(str(companyCode), str(cik), str(date), str(count)) seccrawler.filing_13F(str(companyCode), str(cik), str(date), str(count)) t2 = time.time() print("Total Time taken: "), print(t2 - t1)
def test(): t1 = time.time() # file containing company name and corresponding cik codes seccrawler = SecCrawler() company_code_list = list() # company code list cik_list = list() # cik code list date_list = list() # pror date list count_list = list() try: crs = open("data.txt", "r") except: print("No input file Found") # get the company quotes and cik number from the file. for columns in (raw.strip().split() for raw in crs): company_code_list.append(columns[0]) cik_list.append(columns[1]) date_list.append(columns[2]) count_list.append(columns[3]) # call different API from the crawler #for i in range(1, len(cik_list)): # seccrawler.filing_SD(str(company_code_list[i]), str(cik_list[i]), # str(date_list[i]), str(count_list[i])) # seccrawler.filing_10K(str(company_code_list[i]), str(cik_list[i]), # str(date_list[i]), str(count_list[i])) # seccrawler.filing_8K(str(company_code_list[i]), str(cik_list[i]), # str(date_list[i]), str(count_list[i])) seccrawler.filing_10Q('AAPL', '0000320193', '20170101', '1') t2 = time.time() print("Total Time taken: "), print(t2 - t1) crs.close()
def crawler(): obj = SecCrawler() return obj
from SECEdgar.crawler import SecCrawler crawler = SecCrawler() def get(symbol): crawler.filing_10Q(symbol, '0000320193', '20010101', '10') # http://rankandfiled.com/#/data/tickers
def crawler(): _crawler = SecCrawler() yield _crawler
def crawler(): _crawler = SecCrawler() yield _crawler shutil.rmtree(_crawler.data_path)
from SECEdgar.crawler import SecCrawler filings = SecCrawler() filings.filing_10Q("MSFT", "0000789019", "20190101", 100)