def parse_scheduleDjson(crd):
data = []
try:
adviser = db.Adviser.get(crd=crd)
funds = adviser.privatefunds
except (IOError, db.Adviser.DoesNotExist, KeyError):
return data
for fund in funds:
row = {
'crd':
crd,
'adviser':
adviser.name,
'fund_id':
fund['fund_id'],
'fundname':
fund['name'],
'assetsundermgmt':
floating_point(fund['fundinfo'].get('assetsundermgmt', 'n/a')),
'fundtype':
fund['fundinfo'].get('fundtype', 'n/a'),
'owners':
fund['fundinfo'].get('numberofowners', 'n/a'),
}
if 'businesses' in fund:
for business in fund['businesses']:
data.append(mergedicts(business, row))
return data
def create_relationship(cls, row, businessrow):
if re_NOTBUSINESS.search(businessrow['name']):
return
info = businessrow['info']
businessrow['info'] = re_BUSINESSINFO.search(info).group(1).upper()
entry, created = cls.get_or_create(**businessrow)
return mergedicts(row, business=entry.id)
def getfundinfo(self, linktag):
_ = linktag.find_next('p').text.strip()
self.br.open(self.br.fundurl(linktag))
__ = mergedicts(self.getsubtopics(),
fund_id=re_ID.search(
self.br.fundurl(linktag)).group(1),
fundinfo=parse_regexmap(_, re_FUNDINFODICT),
name=linktag.text)
pause(17, 49)
return __
def insert_fund(crd, fund):
info = fund['fundinfo']
_ = fund['fund_id']
row = mergedicts({
k : (v if k != 'assetsundermgmt' else floating_point(v))
for k, v in info.items() if k in db.PrivateFund._meta.fields
}, type = info['fundtype'], fund_id = "{}-{}".format(_[0:3], _[3:]),
adviser_id = crd, name = fund['name'])
return db.PrivateFund.get_or_create(**row)[0]
def addpeople(cls, crd, people):
with database.atomic():
for person in people:
row = mergedicts(getname(person['name']), adviser=crd)
while True:
try:
entry = cls.get(**row)
break
except cls.DoesNotExist:
cls.insert(**row).execute()
try:
percentage = re_PERCENTAGE.sub(r'.\1', person['ownership'])
except KeyError:
percentage = '0'
is_cntrlperson = person['controlperson']
try:
Ownership.get_or_create(person=entry,
percentowned=percentage,
controlperson=is_cntrlperson)
except IntegrityError as e:
db_logger.error(e)
try:
title = ' '.join(x.capitalize()
for x in person['title'].split())
except KeyError:
title = 'n/a'
try:
title = ' '.join(x.capitalize()
for x in person['title'].split())
except KeyError:
title = 'n/a'
try:
since = pd.to_datetime(person['since'])
except (KeyError, Exception) as e:
db_logger.error(e)
since = None
(cls.update(
date=since,
title=title).where((cls.firstname == entry.firstname)
& (cls.lastname == entry.lastname)
& (cls.adviser == crd)).execute())
def getdata(cls, crd, br):
obj = cls(crd, br)
__ = {
'crd': str(crd),
'description': obj.firmdescription,
'data': obj.getsubtopics(),
'people': obj.get_controlpersons(),
'relyingadvisers': obj.relyingadvisers,
}
linktags = obj.fundlinks
if not linktags:
return __
funds = []
for linktag in linktags:
funds.append(obj.getfundinfo(linktag))
return mergedicts(__, funds=funds)
def addfund(cls, crd, fund):
info = fund['fundinfo']
_ = fund['fund_id']
aum = info['assetsundermgmt']
if not aum:
aum = 0
row = mergedicts(
{
k: (v if k != 'assetsundermgmt' else floating_point(aum))
for k, v in info.items() if k in cls._meta.fields
},
type=info['fundtype'],
fund_id="{}-{}".format(_[0:3], _[3:]),
adviser_id=crd,
name=fund['name'])
try:
entry = cls.get_or_create(**row)
except IntegrityError as e:
db_logger.error(e)
return cls.get(adviser=crd, fund_id=row['fund_id'])
return entry[0]
def insert_people(crd, data):
people = data['people']
person_list = []
with database.atomic():
for person in people:
row = mergedicts(getname(person['name']), adviser = crd)
while True:
try:
entry = db.Person.get(**row); break
except db.Person.DoesNotExist:
db.Person.insert(**row).execute()
percentage = re_PERCENTAGE.sub(r'.\1', person['ownership'])
is_controlperson = person['controlperson']
db.Ownership.get_or_create(person = entry,
percentowned = percentage,
controlperson = is_controlperson)
title = ' '.join(x.capitalize() for x in person['title'].split())
since = pd.to_datetime(person['since'])
(db.Person.update(date = since, title = title)
.where((db.Person.firstname == entry.firstname) &
(db.Person.lastname == entry.lastname) &
(db.Person.adviser == crd)).execute())
def getlocation(self, location):
_ = siblingtext(location).replace('\n', ' ')
return mergedicts(
address_search(_).groupdict(), {'business': cleantag(location)})
def getdrp(self, plaintiff):
return mergedicts(parse_regexmap(siblingtext(plaintiff), re_DRPDICT),
plaintiff=plaintiff_search(
cleantag(plaintiff)).group(1))