def extractSectionFromExistingFilings(numThreads):
filename = '../check.txt'
completedFilename = 'extraction_log.txt'
f = open(completedFilename, 'r')
completedLines = f.readlines()
f.close()
completed = [line.strip() for line in completedLines]
with open(filename) as check:
lines = check.readlines()
readLines = [[line.split('\t')[0].strip(), [line.split('\t')[6].strip()], \
[line.split('\t')[5].strip()], [line.split('\t')[6].strip()], '10-K'] for line in lines]
sectionsToGet = []
for line in reversed(readLines):
companyCode, filingURLList, docNameList, indexURLList, filingType = line
if docNameList[0] not in completed:
sectionsToGet.append(line)
else:
print 'Already downloaded ', docNameList[0]
if numThreads > 1:
print 'THREADED'
results = calculateParallel(sectionsToGet, extractSingleSection,
numThreads)
else:
print 'NONTHREADED'
seccrawler = SecCrawler()
for lineElem in sectionsToGet:
companyCode, filingURLList, docNameList, indexURLList, filingType = lineElem
if '.txt' not in filingURLList[0]:
seccrawler.save_in_directory(companyCode, filingURLList,
docNameList, indexURLList,
filingType)
f = open(completedFilename, 'a+')
f.write(docNameList[0] + '\n')
f.close()
def extractSingleSection(inputs):
companyCode, filingURLList, docNameList, indexURLList, filingType = inputs
logString = 'Skipping text file.'
if '.txt' not in filingURLList[0]:
t1 = time.time()
seccrawler = SecCrawler()
try:
seccrawler.save_in_directory(companyCode, filingURLList,
docNameList, indexURLList, filingType)
# Put all exception text into an exception and raise that
t2 = time.time()
completedFilename = 'extraction_log.txt'
f = open(completedFilename, 'a+')
f.write(docNameList[0] + '\n')
f.close()
logString = "Total Time taken for " + companyCode + "sections: " + str(
t2 - t1)
except:
raise Exception("".join(
traceback.format_exception(*sys.exc_info())))
return logString