def addRSSentry(entry):
formTitle = entry["title"][0:3]
if formTitle == 'D -' or formTitle == 'D/A':
filingURL = entry["link"]["@href"]
r2 = requests.get(filingURL)
soup = BeautifulSoup(filingURL)
allFilings = soup.find(_class='companyName')
# if doc["edgarSubmission"]["offeringData"]["typeOfFiling"]["newOrAmendment"]["isAmendment"] == 'true':
# addAmendedDocuments(allFilings.a['href'])
form = "http://www.sec.gov" + re.search('<a href="(.*?primary_doc.xml)">primary_doc.xml', r2.text).group(1)
print form
r3 = requests.get(form)
doc = xmltodict.parse(r3.text)
if doc["edgarSubmission"]["offeringData"]["industryGroup"]["industryGroupType"] == 'Pooled Investment Fund':
i = Issuer()
i.Name = doc["edgarSubmission"]["primaryIssuer"]["entityName"]
i.City = doc["edgarSubmission"]["primaryIssuer"]["issuerAddress"]["city"]
i.Country = doc["edgarSubmission"]["primaryIssuer"]["issuerAddress"]["stateOrCountryDescription"]
i.TierID = ''
if i.Country == 'NEW YORK' or i.Country == 'JERSEY' or i.Country == 'NEW JERSEY' or i.Country == 'CONNECTICUT' or i.Country == 'CALIFORNIA' or i.Country == 'DELAWARE' or i.Country == 'MASSACHUSETTS':
i.TierID = '1'
elif i.Country == 'FLORIA' or i.Country == 'PENNSYLVANIA' or i.Country == 'ILLINOIS' or i.Country == 'WASHINGTON' or i.Country == 'TEXAS':
i.TierID = '2'
else:
i.TierID = '3'
db_session.add(i)
db_session.commit()
j = FormD()
j.url = form
j.SubmissionDate = entryUpdatedTime
j.IndustryGroupType = doc["edgarSubmission"]["offeringData"]["industryGroup"]["industryGroupType"]
j.InvestmentFundType = doc["edgarSubmission"]["offeringData"]["industryGroup"]["investmentFundInfo"]["investmentFundType"]
if doc["edgarSubmission"]["offeringData"]["typeOfFiling"]["newOrAmendment"]["isAmendment"] == 'true':
j.Amended = 1 #1 is true; 0 is false
else:
j.Amended = 0
firstSaleDict = doc["edgarSubmission"]["offeringData"]["typeOfFiling"]["dateOfFirstSale"]
if firstSaleDict.keys() == ['yetToOccur']:
j.IsDateOfFirstSaleYetToOccur = 1
else:
unformattedDate = firstSaleDict['value']
year = int(unformattedDate[0:4])
month = int(unformattedDate[5:7])
day = int(unformattedDate[8:10])
j.DateOfFirstSale = date(year, month, day)
j.MinimumInvestmentAccepted = doc["edgarSubmission"]["offeringData"]["minimumInvestmentAccepted"]
j.TotalAmountSold = doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"]["totalAmountSold"]
if doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"]["totalOfferingAmount"] == 'Indefinite':
j.TotalOfferingAmount = None
j.IsTotalOfferingAmountIndefinite = 1
else:
j.TotalOfferingAmount = doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"]["totalOfferingAmount"]
j.IsTotalOfferingAmountIndefinite = 0
if doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"]["totalRemaining"] == 'Indefinite':
j.TotalRemaining = None
j.IsTotalRemainingIndefinite = 1
else:
j.TotalRemaining = doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"]["totalRemaining"]
j.IsTotalRemainingIndefinite = 0
j.Issuer = i
db_session.add(j)
db_session.commit()
print 'parsing complete, added it to the DB'
def addRSSentry(entry):
formTitle = entry["title"][0:3]
if formTitle == 'D -' or formTitle == 'D/A':
filingURL = entry["link"]["@href"]
r2 = requests.get(filingURL)
soup = BeautifulSoup(filingURL)
allFilings = soup.find(_class='companyName')
# if doc["edgarSubmission"]["offeringData"]["typeOfFiling"]["newOrAmendment"]["isAmendment"] == 'true':
# addAmendedDocuments(allFilings.a['href'])
form = "http://www.sec.gov" + re.search(
'<a href="(.*?primary_doc.xml)">primary_doc.xml', r2.text).group(1)
print form
r3 = requests.get(form)
doc = xmltodict.parse(r3.text)
i = Issuer()
i.Name = doc["edgarSubmission"]["primaryIssuer"]["entityName"]
i.City = doc["edgarSubmission"]["primaryIssuer"]["issuerAddress"][
"city"]
i.Country = doc["edgarSubmission"]["primaryIssuer"]["issuerAddress"][
"stateOrCountryDescription"]
db_session.add(i)
db_session.commit()
j = FormD()
j.url = form
j.SubmissionDate = entryUpdatedTime
j.IndustryGroupType = doc["edgarSubmission"]["offeringData"][
"industryGroup"]["industryGroupType"]
if j.IndustryGroupType == 'Pooled Investment Fund':
j.InvestmentFundType = doc["edgarSubmission"]["offeringData"][
"industryGroup"]["investmentFundInfo"]["investmentFundType"]
else:
j.InvestmentFundType = None
if doc["edgarSubmission"]["offeringData"]["typeOfFiling"][
"newOrAmendment"]["isAmendment"] == 'true':
j.Amended = 1 #1 is true; 0 is false
else:
j.Amended = 0
firstSaleDict = doc["edgarSubmission"]["offeringData"]["typeOfFiling"][
"dateOfFirstSale"]
if firstSaleDict.keys() == ['yetToOccur']:
j.IsDateOfFirstSaleYetToOccur = 1
else:
unformattedDate = firstSaleDict['value']
year = int(unformattedDate[0:4])
month = int(unformattedDate[5:7])
day = int(unformattedDate[8:10])
j.DateOfFirstSale = date(year, month, day)
j.MinimumInvestmentAccepted = doc["edgarSubmission"]["offeringData"][
"minimumInvestmentAccepted"]
j.TotalAmountSold = doc["edgarSubmission"]["offeringData"][
"offeringSalesAmounts"]["totalAmountSold"]
if doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"][
"totalOfferingAmount"] == 'Indefinite':
j.TotalOfferingAmount = None
j.IsTotalOfferingAmountIndefinite = 1
else:
j.TotalOfferingAmount = doc["edgarSubmission"]["offeringData"][
"offeringSalesAmounts"]["totalOfferingAmount"]
j.IsTotalOfferingAmountIndefinite = 0
if doc["edgarSubmission"]["offeringData"]["offeringSalesAmounts"][
"totalRemaining"] == 'Indefinite':
j.TotalRemaining = None
j.IsTotalRemainingIndefinite = 1
else:
j.TotalRemaining = doc["edgarSubmission"]["offeringData"][
"offeringSalesAmounts"]["totalRemaining"]
j.IsTotalRemainingIndefinite = 0
j.Issuer = i
db_session.add(j)
db_session.commit()
print 'parsing complete, added it to the DB'
# "BlackRock Global SmallCap Fund" : "/Users/jeffca/documents/blackrock/flask_base/forms/BlackRock Global SmallCap Fund/BlackRock Global SmallCap Fund.csv",
# "Charles Schwab Family Of Funds" : "/Users/jeffca/documents/blackrock/flask_base/forms/Charles Schwab Family Of Funds/Charles Schwab Family Of Funds.csv",
# "Schwab Capital Trust" : "/Users/jeffca/documents/blackrock/flask_base/forms/Schwab Capital Trust/Schwab Capital Trust.csv",
# "BlackRock Capital Appreciation Fund" : "/Users/jeffca/documents/blackrock/flask_base/forms/BlackRock Capital Appreciation Fund/BlackRock Capital Appreciation Fund.csv",
# "BlackRock Focus Growth Fund" : "/Users/jeffca/documents/blackrock/flask_base/forms/BlackRock Focus Growth Fund/BlackRock Focus Growth Fund.csv"
# }
filings = {
"BlackRock Bond Index Fund" : "/Users/jeffca/documents/blackrock/flask_base/forms/BlackRock Bond Index Fund/BlackRock Bond Index Fund.csv",
"BlackRock Emerging Markets Fund" : "/Users/jeffca/documents/blackrock/flask_base/forms/BlackRock Emerging Markets Fund/BlackRock Emerging Markets Fund.csv"
}
for filing in filings.keys():
reg = Registrant()
doc = Registrant_Document()
db_session.add(reg)
db_session.add(doc)
db_session.commit()
doc.Registrant = reg
# url = "http://localhost:8080/rest/xbrl/view?file=" + filings[filing] + "&view=facts&media=csv"
# data = requests.get(url).text
# f = StringIO.StringIO(data)
f = open("forms/" + filing + "/" + filing + '.csv' , 'r')
reader = csv.reader(f, delimiter=',')
r2 = requests.get(html[filing])
print 'parsing ' + html[filing]
doc.url = html[filing]
soup = BeautifulSoup(r2.text)
for row in reader:
if row[0] == 'Label':
continue