def drop_many():
data = json.loads(request.form['params'])
dao = Dao()
try:
dao.drop_many('voters', 'voter_id', data)
return jsonify(msg='Records dropped!')
except Exception as ex:
return jsonify(error=str(ex))
def insert(self):
sql = ("INSERT INTO elections "
"(id, date, description) "
"VALUES (%s, %s, %s);")
vals = [
self.id, self.date, self.description
]
dao = Dao()
return dao.execute(sql, vals)
def test_get_block(self):
dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True)
contact = Contact()
contact.address = Address({'address': '3000 Newcastle Rd'})
contact.zipcode = '48105'
block = Location.get_block(dao, contact)
contact.zipcode = ''
contact.city = 'ANN ARBOR'
block = Location.get_block(dao, contact)
dao.close()
def neighborhood_drop():
params = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
turf_dao.neighborhood_drop(dao, params['id'])
return jsonify(msg='Neighborhood dropped!')
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def add_many():
data = json.loads(request.form['params'])
flds = list(data[0].keys())
vals = [list(rec.values()) for rec in data]
dao = Dao()
try:
dao.add_many('voters', flds, vals)
return jsonify(msg='Records saved!')
except Exception as ex:
return jsonify(error=str(ex))
def update_many():
data = json.loads(request.form['params'])
flds = list(data[0].keys())
vals = [list(rec.values()) for rec in data]
for i in range(0, len(data)):
vals[i].append(data[i]['voter_id'])
dao = Dao()
try:
dao.update_many('voters', 'voter_id', flds, vals)
return jsonify(msg='Records saved!')
except Exception as ex:
return jsonify(error=str(ex))
def user_drop():
user_id = json.loads(request.args['id'])
dao = Dao(stateful=True)
try:
success = User.delete_user(dao, user_id)
if not success:
msg = 'Record not deleted for unknown reason. Contact admin.'
return jsonify(error=msg)
users = User.get_users()
return jsonify(users=users)
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def user_add():
values = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
user_id = User.add_user(dao, values)
if 'precinct_ids' in values:
ids = [int(pid) for pid in values['precinct_ids'].split(',') if pid]
User.add_precinct_ids(dao, user_id, ids)
users = User.get_users()
return jsonify(users=users)
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
class TestContact(unittest.TestCase):
def setUp(self):
from dao.dao import Dao
self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True)
def tearDown(self):
self.dao.close()
def test_get_email_matches(self):
email = '*****@*****.**'
matches = ContactInfo.get_email_matches(self.dao, email, 100)
pass
def name_duplicates():
if request.method == 'GET':
dao = Dao(stateful=True)
cities = turf_dao.get_city_names(dao)
dups = con_dao.get_name_dups(dao)
dao.close()
for dup in dups:
dup['name'] = str(PersonName(dup))
dup['address'] = str(Address(dup))
return render_template(
'contacts/dups.html',
title='Name Duplicates',
dups=dups,
cities=cities
)
def login():
if request.method == 'GET':
return render_template(
'login.html',
title='Bluestreets login'
)
values = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
rec = User.login(dao, values['username'], values['password'])
session['user'] = rec
return jsonify(msg='Hello %s!' % rec['first_name'])
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def save_fund_subtraction(self, target_history_pd, fund_code, dateStr, start_trade_timestamp, end_trade_timestamp):
start_pd = Dao().get_fund_history_by_fund_code_trade_timestamp(fund_code, start_trade_timestamp)
similar_ranking_subtraction = start_pd.loc[0, 'similar_ranking'] - target_history_pd.loc[0, 'similar_ranking']
sc_subtraction = start_pd.loc[0, 'sc'] - target_history_pd.loc[0, 'sc']
info_pd = pd.DataFrame([fund_code], columns=['fund_code'])
info_pd['similar_ranking_' + dateStr + '_change'] = similar_ranking_subtraction
info_pd['sc_' + dateStr + '_change'] = sc_subtraction
self.save_fund_parameter(info_pd, end_trade_timestamp)
def user_mgt():
from models.precinct import Precinct
dao = Dao(stateful=True)
jurisdictions = Precinct.get_jurisdictions(dao)
precincts = Precinct.get_all(dao)
roles = User.get_roles(dao)
users = User.get_users(dao)
precinct_admins = User.get_precinct_admins(dao)
dao.close()
return render_template(
'users.html',
title='Bluestreets users',
jurisdictions=jurisdictions,
precincts=[precinct.serialize() for precinct in precincts],
roles=roles,
users=users,
precinct_admins=precinct_admins
)
def setFileDataStatus(self, voList):
if(voList is not None):
missingObjects = self.getMissingObjects()
if(len(missingObjects) > 0):
setattr(self.fileData, self.actualStatus, Constant.STATUS_WARNING)
FileDataDao().setErrorMessage(errorMessage=str(missingObjects)[0:149], errorKey=self.errorKey, fileData=self.fileData)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
Dao().addObject(objectToAdd=self.fileData, session=self.session, doCommit=True)
def getReportDict(self, processCache, menuCategoryAllowed, session):
#Obtengo reportes statements
xmlDict= processCache[Constant.DOCUMENT_SUMMARY]
reportDict = {}
for report in xmlDict["FilingSummary"]["MyReports"]["Report"]:
menuCategory = report.get("MenuCategory", None)
if(menuCategory is None or menuCategory in menuCategoryAllowed):
try:
reportRole = report["Role"]
if(self.isReportAllowed(reportRole)):
reportShortName = report["ShortName"]
report = Dao().getReport(reportShortName, session)
if(report is None):
report = Report()
report.shortName = reportShortName[0:299]
reportDict[reportRole] = report
except Exception:
pass
logging.getLogger(Constant.LOGGER_GENERAL).debug("REPORT LIST " + str(reportDict))
return reportDict
def getOrCreateCompany(self, CIK, entityRegistrantName=None, session=None):
company = CompanyDao().getCompany2(CIK, session)
if (company is None):
company = Company()
company.CIK = CIK
company.entityRegistrantName = entityRegistrantName
company.listed = True
elif entityRegistrantName is not None and entityRegistrantName != '':
company.entityRegistrantName = entityRegistrantName
Dao().addObject(objectToAdd=company, session=session, doCommit=True)
return company
def getOrCreatePeriod3(self, instant, session):
period = GenericDao().getOneResult(Period,
and_(Period.instant == instant),
session,
raiseNoResultFound=False)
if (period is None):
period = Period()
period.instant = instant
period.type = "INST"
Dao().addObject(objectToAdd=period, session=session, doFlush=True)
return period
def contact_matches():
contact = Contact(json.loads(request.form['params']))
dao = Dao(stateful=True)
try:
matches = contact.get_matches(dao)
for match in matches:
match['name'] = str(PersonName(match))
match['address'] = str(Address(match))
return jsonify(matches=matches)
except Exception as ex:
return jsonify(error=str(ex))
def addEntityFact(self, factVOList, fileDataOID, reportDict, session,
replace):
objectAlreadyAdded = {}
entityFactValueList = []
for factVO in factVOList:
if (len(factVO.factValueList) > 0):
concept = ConceptEngine().getOrCreateConcept(
factVO.conceptName, session)
efvList = EntityFactDao().getEntityFactValueList(
fileDataOID, concept.OID, session)
for efv in efvList:
factValueKey = str(efv.fileDataOID) + "-" + str(
efv.entityFact.conceptOID)
objectAlreadyAdded[factValueKey] = ""
entityFact = None
if (reportDict[factVO.reportRole].OID is not None):
entityFact = self.getEntityFact(
concept.OID, reportDict[factVO.reportRole].OID,
factVO.order, session)
if (entityFact is None):
entityFact = EntityFact()
entityFact.conceptOID = concept.OID
entityFact.report = reportDict[factVO.reportRole]
entityFact.order_ = factVO.order
Dao().addObject(objectToAdd=entityFact,
session=session,
doFlush=True)
for factValueVO in factVO.factValueList:
factValueKey = str(fileDataOID) + "-" + str(concept.OID)
if (objectAlreadyAdded.get(factValueKey, None) is None):
entityFactValue = EntityFactValue()
entityFactValue.value = factValueVO.value
entityFactValue.period = factValueVO.period
entityFactValue.fileDataOID = fileDataOID
entityFactValue.entityFact = entityFact
entityFactValueList.append(entityFactValue)
objectAlreadyAdded[factValueKey] = ""
for efv in entityFactValueList:
Dao().addObject(objectToAdd=efv, session=session)
session.commit()
def csv_import():
if request.method == 'GET':
return render_template(
'contacts/csv_import.html',
title='Contact Import'
)
data = json.loads(request.form['params'])
dao = Dao(stateful=True)
# precincts = Precinct.get_by_jwp(dao)
groups = Group.get_all_by_code(dao)
memberships = []
next_id = dao.get_max_id('contacts', 'id')
for rec in data:
rec['precinct_id'] = None
next_id += 1
# if rec['jurisdiction'] and rec['ward'] and rec['precinct']:
# jwp = '%s:%s:%s' % (
# rec['jurisdiction'].upper(),
# rec['ward'].zfill(2),
# rec['precinct'].zfill(3)
# )
# rec['precinct_id'] = precincts[jwp]['id']
if rec['groups']:
for code in rec['groups'].split('/'):
if code in groups:
memberships.append({
'group_id': groups[code]['id'],
'contact_id': next_id,
'role': '',
'comment': ''
})
try:
con_dao.add_many(data)
GroupMember.add_many(memberships)
return jsonify(msg='Successful!')
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def getOrCreatePeriod(self, periodType, endDate, session):
period = GenericDao().getOneResult(Period,
and_(Period.endDate == endDate,
Period.startDate.is_(None)),
session,
raiseNoResultFound=False)
if (period is None):
period = Period()
period.endDate = endDate
period.type = periodType
Dao().addObject(objectToAdd=period, session=session, doFlush=True)
return period
def grid():
if request.method == 'GET':
precincts = turf_dao.get_precincts()
return render_template(
'contacts/grid.html',
precincts=precincts,
title='My Contacts'
)
blocks = json.loads(request.form['params'])
if not blocks:
contacts = con_dao.get_all()
elif len(blocks[0]) == 1 and 'precinct_id' in blocks[0]:
contacts = con_dao.get_by_precinct(blocks[0]['precinct_id'])
else:
dao = Dao(stateful=True)
contacts = []
for block in blocks:
contacts += con_dao.get_by_block(dao, block)
dao.close()
return jsonify(contacts=to_local_format(contacts))
def lookup():
dao = Dao()
matches = Voter.lookup(dao, request.args)
result = [{
'name': str(match.name),
'address': str(match.address),
'city': match.address.city,
'zipcode': match.address.zipcode,
'gender': match.gender,
'birth_year': match.birth_year,
'voter_id': match.id
} for match in matches]
return jsonify(matches=result)
def user_update():
values = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
if 'fromPrecinctAdmin' in values:
numrows = User.delete_precinct_admins_for_user(values['id'])
numrows = User.update_user(dao, values)
if numrows != 1:
msg = 'Record not updated for unknown reason. Contact admin.'
return jsonify(error=msg)
if 'precinct_ids' in values:
ids = [int(pid) for pid in values['precinct_ids'].split(',') if pid]
del_ids = list(set(values['prev_precincts']) - set(ids))
add_ids = list(set(ids) - set(values['prev_precincts']))
User.delete_precinct_admins_for_user(dao, values['id'], del_ids)
User.add_precinct_ids(dao, values['id'], add_ids)
users = User.get_users()
return jsonify(users=users)
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def csv_import():
if request.method == 'GET':
return render_template('contacts/csv_import.html',
title='Contact Import')
data = json.loads(request.form['params'])
dao = Dao(stateful=True)
# precincts = Precinct.get_by_jwp(dao)
groups = Group.get_all_by_code(dao)
memberships = []
next_id = dao.get_max_id('contacts', 'id')
for rec in data:
rec['precinct_id'] = None
next_id += 1
# if rec['jurisdiction'] and rec['ward'] and rec['precinct']:
# jwp = '%s:%s:%s' % (
# rec['jurisdiction'].upper(),
# rec['ward'].zfill(2),
# rec['precinct'].zfill(3)
# )
# rec['precinct_id'] = precincts[jwp]['id']
if rec['groups']:
for code in rec['groups'].split('/'):
if code in groups:
memberships.append({
'group_id': groups[code]['id'],
'contact_id': next_id,
'role': '',
'comment': ''
})
try:
con_dao.add_many(data)
GroupMember.add_many(memberships)
return jsonify(msg='Successful!')
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
class TestContact(unittest.TestCase):
def setUp(self):
self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True)
def tearDown(self):
self.dao.close()
def test_get_activists(self):
rex = Contact.get_activists(self.dao)
pass
def test_get_best_voter(self):
addr = Address({'address': '3000 Newcastle Rd'})
pn = PersonName({
'last_name': 'weinblatt',
'first_name': 'howard'
})
contact = Contact()
contact.name = pn
contact.address = addr
voter = Contact.get_best_voter_rec(self.dao, contact)
pass
def test_get_best_turf(self):
contact = Contact()
contact.address = Address({
'address': '3000 Newcastle Rd',
'zipcode': '48105'
})
turf = Contact.get_best_turf(self.dao, contact)
contact.address.zipcode = ''
contact.address.city = 'ANN ARBOR'
turf = Contact.get_best_turf(self.dao, contact)
pass
def test_clean_turf(self):
num, unresolved = Contact.assign_precinct(self.dao)
pass
def user_update():
values = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
if 'fromPrecinctAdmin' in values:
numrows = User.delete_precinct_admins_for_user(values['id'])
numrows = User.update_user(dao, values)
if numrows != 1:
msg = 'Record not updated for unknown reason. Contact admin.'
return jsonify(error=msg)
if 'precinct_ids' in values:
ids = [int(pid) for pid in values['precinct_ids'].split(',') if pid]
del_ids = list(set(values['prev_precincts']) - set(ids))
add_ids = list(set(ids) - set(values['prev_precincts']))
User.delete_precinct_admins_for_user(dao, values['id'], del_ids)
User.add_precinct_ids(dao, values['id'], add_ids)
users = User.get_users()
return jsonify(users=users)
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
def doImport(self):
try:
logging.info("START")
time1 = datetime.now()
if (self.skipOrProcess()):
self.addOrModifyInit()
if (self.replace):
self.deleteImportedObject()
self.logger.debug("**********START - Processing filename " +
self.filename)
voList = self.doImport2()
persistentList = self.getPersistentList(voList)
Dao().addObjectList(persistentList, self.session)
self.setFileDataStatus(voList)
if (voList is None):
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename)
else:
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename + " objects added " +
str(len(voList)))
else:
self.logger.info("Skipped " + self.filename)
except (CustomException) as e:
self.logger.error(self.filename + " " + e.status + " " + str(e))
self.addOrModifyFDError1(e)
except MemoryError as e:
self.logger.error(self.filename + " " + str(e))
FileDataDao().addOrModifyFileData(
statusKey=self.actualStatus,
statusValue=Constant.STATUS_ERROR,
filename=self.filename,
errorMessage='MemoryError',
errorKey=self.errorKey)
except ExpatError as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2('not well-formed')
except Exception as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2(str(e)[0:149])
finally:
self.session.commit()
self.session.close()
def neighborhoods():
if request.method == 'GET':
dao = Dao(stateful=True)
types = turf_dao.get_turf_types(dao)
jurisdictions = turf_dao.get_jurisdictions(dao)
juris_names = [j['jurisdiction_name'] for j in jurisdictions]
precincts = turf_dao.get_precincts(dao)
pct_names = [
'%s, %s, %s' % (p['jurisdiction_name'], p['ward'], p['precinct'])
for p in precincts
]
wards = build_wards(precincts)
state_house_districts = sorted(
list(set([p['state_house'] for p in precincts])))
state_senate_districts = sorted(
list(set([p['state_senate'] for p in precincts])))
congressional_districts = sorted(
list(set([p['congress'] for p in precincts])))
nhoods = turf_dao.get_neighborhoods(dao)
dao.close()
return render_template('neighborhoods.html',
title="Neighborhoods",
types=types,
jurisdictions=jurisdictions,
juris_names=juris_names,
wards=wards,
precincts=precincts,
pct_names=pct_names,
state_house_districts=state_house_districts,
state_senate_districts=state_senate_districts,
congressional_districts=congressional_districts,
neighborhoods=nhoods)
params = json.loads(request.form['params'])
block_vals = []
if params['blocks']:
block_vals = [
(block['street_name'], block['street_type'], block['low_addr'],
block['high_addr'], block['odd_even'], block['precinct_id'])
for block in params['blocks']
]
dao = Dao(stateful=True)
nbh_id = turf_dao.add_neighborhood(dao, params['type'], params['name'],
params['pct_ids'], block_vals)
dao.close()
return jsonify(nbh_id=nbh_id)
def voter_lookup():
from models.voter import Voter
contact = json.loads(request.form['params'])
dao = Dao(stateful=True)
try:
voters = Voter.lookup(dao, contact)
candidates = [{
'name': {
'last': voter.name.last,
'first': voter.name.first,
'middle': voter.name.middle,
'suffix': voter.name.suffix
},
'address': {
'house_number': voter.address.house_number,
'pre_direction': voter.address.pre_direction,
'street_name': voter.address.street_name,
'street_type': voter.address.street_type,
'suf_direction': voter.address.suf_direction,
'unit': voter.address.unit,
'city': voter.address.city,
'zipcode': voter.address.zipcode
},
'voter_info': {
'voter_id': voter.voter_id,
'precinct_id': voter.precinct_id,
'birth_year': voter.birth_year,
'gender': voter.gender,
'reg_date': voter.reg_date
}
} for voter in voters]
return jsonify(candidates=candidates)
except Exception as ex:
return jsonify(error=str(ex))
finally:
dao.close()
class TestContact(unittest.TestCase):
def setUp(self):
self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db',
stateful=True)
def tearDown(self):
self.dao.close()
def test_get_activists(self):
rex = Contact.get_activists(self.dao)
pass
def test_get_best_voter(self):
addr = Address({'address': '3000 Newcastle Rd'})
pn = PersonName({'last_name': 'weinblatt', 'first_name': 'howard'})
contact = Contact()
contact.name = pn
contact.address = addr
voter = Contact.get_best_voter_rec(self.dao, contact)
pass
def test_get_best_turf(self):
contact = Contact()
contact.address = Address({
'address': '3000 Newcastle Rd',
'zipcode': '48105'
})
turf = Contact.get_best_turf(self.dao, contact)
contact.address.zipcode = ''
contact.address.city = 'ANN ARBOR'
turf = Contact.get_best_turf(self.dao, contact)
pass
def test_clean_turf(self):
num, unresolved = Contact.assign_precinct(self.dao)
pass
def assign_precinct(contacts, dao=None):
if not dao:
dao = Dao(stateful=True)
updates = []
for contact in contacts:
# get voter rex by name + address
voter = Contact.get_best_voter_rec(dao, contact)
if voter:
nn = (contact.name.nickname, contact.name.nickname_meta)
contact.name = voter.name
contact.nickname = nn[0]
contact.nickname_meta = nn[1]
contact.address = voter.address
contact.birth_year = voter.birth_year
contact.gender = voter.gender
contact.voter_id = voter.voter_id
contact.reg_date = voter.reg_date
updates.append(contact)
continue
turf = Contact.get_best_turf(dao, contact)
if turf:
contact.address.pre_direction = turf['pre_direction']
contact.address.street_name = turf['street_name']
contact.address.street_type = turf['street_type']
contact.address.suf_direction = turf['suf_direction']
contact.address.precinct_id = turf['precinct_id']
contact.address.city = turf['city']
contact.address.zipcode = turf['zipcode']
updates.append(contact)
continue
# get voter rex by name only
# Contact.update_many(dao, updates)
dao.close()
return len(updates)
def single_fund_info(self, code_s, type_s):
code, ishb, basic_info_pd, funds = ReptileFund().get_pingzhongdata(code_s, type_s)
if ishb is not None:
if ishb:
fund_wide_pd, rateInSimilar_pd, grandTotal_pd, worth_pd, current_fund_manager_pd = ProcessFund().get_hb_pingzhongdata_2_df(
basic_info_pd, funds)
else:
fund_wide_pd, rateInSimilar_pd, grandTotal_pd, worth_pd, current_fund_manager_pd = ProcessFund().get_pingzhongdata_2_df(
basic_info_pd, funds)
# print(code, '用时2', end)
Dao().save_fund_info(fund_wide_pd, 'fund_code', int(code))
# print(code, '用时3', end)
if not rateInSimilar_pd.empty:
Dao().save_fund_history(rateInSimilar_pd, 'fund_code', int(code))
pass
if not grandTotal_pd.empty:
Dao().save_fund_history(grandTotal_pd, 'fund_code', int(code))
pass
if not worth_pd.empty:
Dao().save_fund_history(worth_pd, 'fund_code', int(code))
pass
if not current_fund_manager_pd.empty:
Dao().save_manager_info(current_fund_manager_pd)
def elections():
import dao.election_dao as el_dao
if request.method == 'GET':
dao = Dao()
elects = el_dao.get(dao)
return render_template(
'voters/elections.html',
title='Elections',
elections=elects
)
dao = Dao(stateful=True)
latest = el_dao.get_latest_date(dao)
data = api_client.get('vtr_api/elections_after/%s' % (latest,))
if data:
el_dao.add(dao, data)
elects = el_dao.get(dao)
result = jsonify(elections=elects)
else:
result = jsonify(msg='No new elections')
dao.close()
return result
def get_matches(self, dao=None):
if not dao:
dao = Dao(stateful=True)
if self.info and (self.info.phone1 or self.info.phone2):
matches = self.get_phone_matches(dao)
if matches:
return matches
if self.info and self.info.email:
matches = self.get_email_matches(dao)
if matches:
return matches
if self.address:
matches = self.get_name_addr_matches(dao)
if matches:
return matches
return self.get_name_only_matches(dao)
def save_fund_mean(self, target_history_pd, fund_code, dateStr, start_trade_timestamp, end_trade_timestamp):
interval_pd = Dao().get_fund_history_by_trade_timestamp_interval(fund_code, start_trade_timestamp,
end_trade_timestamp)
info_pd = pd.DataFrame([fund_code], columns=['fund_code'])
info_pd['net_worth_report_' + dateStr + '_mean'] = round(interval_pd['equity_return'].mean(0), 10)
info_pd['similar_ranking_' + dateStr + '_mean'] = round(interval_pd['similar_ranking'].mean(0), 10)
info_pd['sc_' + dateStr + '_mean'] = round(interval_pd['sc'].mean(0), 10)
info_pd['similar_ranking_per_' + dateStr + '_mean'] = round(
interval_pd['similar_ranking_per'].mean(0), 10)
info_pd['hs300_gt_earn_per_' + dateStr + '_mean'] = round(interval_pd['hs300_grand_total'].mean(0),
10)
info_pd['similar_mean_gt_earn_per_' + dateStr + '_mean'] = round(
interval_pd['similar_mean_grand_total'].mean(0), 10)
info_pd['self_gt_earn_per_' + dateStr + '_mean'] = round(interval_pd['self_grand_total'].mean(0), 10)
info_pd['fund_code'] = fund_code
self.save_fund_parameter(info_pd, end_trade_timestamp)
def street_lookup():
from models.address import str_parse
from utils.match import MatchLib
params = json.loads(request.form['params'])
addr = str_parse(params['address'])
addr['county_code'] = '81' # TODO: get from cfg
addr['meta'] = Address.get_street_meta(addr['street_name'])
if params['city']:
addr['city'] = params['city']
if params['zipcode']:
addr['zipcode'] = params['zipcode']
dao = Dao()
try:
candidates = turf_dao.street_fuzzy_lookup(dao, addr)
matches = MatchLib.get_best_matches(
addr['street_name'], [c['street_name'] for c in candidates], 80)
matches = [match[0] for match in matches]
candidates = [
candidate for candidate in candidates
if candidate['street_name'] in matches
]
candidates = [{
'address': {
'house_num_low': candidate['house_num_low'],
'house_num_high': candidate['house_num_high'],
'odd_even': candidate['odd_even'],
'pre_direction': candidate['pre_direction'],
'street_name': candidate['street_name'],
'street_type': candidate['street_type'],
'suf_direction': candidate['suf_direction'],
'unit_low': candidate['ext_low'],
'unit_high': candidate['ext_high'],
'city': candidate['city'],
'zipcode': candidate['zipcode']
},
'voter_info': {
'precinct_id': candidate['precinct_id']
}
} for candidate in candidates]
return jsonify(candidates=candidates)
except Exception as ex:
return jsonify(error=str(ex))
def addOrModifyFileData(self, statusKey=None, statusValue=None, filename=None, externalSession=None,
errorMessage=None, errorKey=None, fileData=None, extraData=None):
try:
if (externalSession is None):
session = DBConnector().getNewSession()
else:
session = externalSession
if (fileData is None):
fileData = FileDataDao.getFileData(filename, session)
setattr(fileData, statusKey, statusValue)
self.setErrorMessage(errorMessage=errorMessage, errorKey=errorKey, extraData=extraData, fileData=fileData)
Dao().addObject(objectToAdd=fileData, session=session, doCommit=True)
if (externalSession is None):
session.close()
return fileData
except Exception as e:
logging.getLogger(Constant.LOGGER_GENERAL).exception(e)
raise e
def neighborhoods():
if request.method == 'GET':
dao = Dao(stateful=True)
types = turf_dao.get_turf_types(dao)
jurisdictions = turf_dao.get_jurisdictions(dao)
juris_names = [j['jurisdiction_name'] for j in jurisdictions]
precincts = turf_dao.get_precincts(dao)
pct_names = ['%s, %s, %s' % (p['jurisdiction_name'], p['ward'], p['precinct']) for p in precincts]
wards = build_wards(precincts)
state_house_districts = sorted(list(set([p['state_house'] for p in precincts])))
state_senate_districts = sorted(list(set([p['state_senate'] for p in precincts])))
congressional_districts = sorted(list(set([p['congress'] for p in precincts])))
nhoods = turf_dao.get_neighborhoods(dao)
dao.close()
return render_template(
'neighborhoods.html',
title="Neighborhoods",
types=types,
jurisdictions=jurisdictions,
juris_names=juris_names,
wards=wards,
precincts=precincts,
pct_names=pct_names,
state_house_districts=state_house_districts,
state_senate_districts=state_senate_districts,
congressional_districts=congressional_districts,
neighborhoods=nhoods
)
params = json.loads(request.form['params'])
block_vals = []
if params['blocks']:
block_vals = [(
block['street_name'],
block['street_type'],
block['low_addr'],
block['high_addr'],
block['odd_even'],
block['precinct_id']
) for block in params['blocks']]
dao = Dao(stateful=True)
nbh_id = turf_dao.add_neighborhood(
dao,
params['type'],
params['name'],
params['pct_ids'],
block_vals
)
dao.close()
return jsonify(nbh_id=nbh_id)
def importMasterIndexFor(self,
period,
replaceMasterFile,
session=None,
threadNumber=1):
dbconnector = DBConnector()
if (session is None):
session = dbconnector.getNewSession()
localURL = Constant.CACHE_FOLDER + 'master' + str(
period.year) + "-Q" + str(period.quarter) + '.gz'
secURL = "https://www.sec.gov/Archives/edgar/full-index/" + str(
period.year) + "/QTR" + str(period.quarter) + "/master.gz"
print(localURL, secURL)
file = getBinaryFileFromCache(localURL, secURL, replaceMasterFile)
with gzip.open(BytesIO(file), 'rb') as f:
file_content = f.read()
text = file_content.decode("ISO-8859-1")
text = text[text.find("CIK", 0, len(text)):len(text)]
point1 = text.find("\n")
point2 = text.find("\n", point1 + 1)
text2 = text[0:point1] + text[point2:len(text)]
df = pandas.read_csv(StringIO(text2), sep="|")
df.set_index("CIK", inplace=True)
df.head()
print("STARTED FOR PERIOD " + str(period.year) + "-" +
str(period.quarter))
for row in df.iterrows():
CIK = row[0]
filename = row[1]["Filename"]
formType = row[1]["Form Type"]
if (formType == "10-Q" or formType == "10-K"):
fd = FileDataDao.getFileData(filename, session)
if (fd is None):
company = CompanyEngine().getOrCreateCompany(
CIK=CIK, session=session)
fd = FileData()
fd.fileName = filename
fd.company = company
Dao().addObject(objectToAdd=fd,
session=session,
doCommit=True)
print("FD Added " + filename)
print("FINISHED")
def filter_api_imports(api_voters, nbh_id):
dao = Dao()
my_voters = vtr_dao.get_for_neighborhood(dao, nbh_id)
my_voters = {v['voter_id']: v for v in my_voters}
api_voters = {v['voter_id']: v for v in api_voters}
inserts = []
conflicts = []
deletes = []
for api_id, api_voter in api_voters.items():
if api_id in my_voters:
if has_diff(api_voter, my_voters[api_id]):
conflicts.append(api_voter)
conflicts.append(my_voters[api_id])
else:
inserts.append(api_voter)
for vid in my_voters.keys():
if vid not in api_voters:
deletes.append(my_voters[vid])
return inserts, conflicts, deletes
def fillFileData(self, fileData, processCache, session):
insXMLDict = processCache[Constant.DOCUMENT_INS]
documentType = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentType'])
#logging.getLogger(Constant.LOGGER_GENERAL).debug("documentType " + documentType)
amendmentFlag = self.getValueFromElement(['#text'], insXMLDict['dei:AmendmentFlag'])
amendmentFlag = amendmentFlag.lower() in ("true")
documentPeriodEndDate = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentPeriodEndDate'])
#logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentPeriodEndDate " + documentPeriodEndDate)
documentFiscalYearFocus = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentFiscalYearFocus'])
#logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentFiscalYearFocus " + documentFiscalYearFocus)
documentFiscalPeriodFocus = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentFiscalPeriodFocus'])
#logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentFiscalPeriodFocus " + documentFiscalPeriodFocus)
#entityRegistrantName = insXMLDict['dei:EntityRegistrantName']['#text']
fileData.documentType = documentType
fileData.amendmentFlag = amendmentFlag
fileData.documentPeriodEndDate = documentPeriodEndDate
if(len(documentFiscalYearFocus) <= 4):
fileData.documentFiscalYearFocus = documentFiscalYearFocus
fileData.documentFiscalPeriodFocus = documentFiscalPeriodFocus
#fileData.entityRegistrantName = entityRegistrantName
Dao().addObject(objectToAdd = fileData, session = session, doCommit = True)
return fileData
def getFactByReport(self, reportDict, processCache, session):
factVOList = []
#Obtengo para cada reporte sus conceptos
xmlDictPre = processCache[Constant.DOCUMENT_PRE]
for item in self.getListFromElement(Constant.PRESENTATON_LINK, self.getElementFromElement(Constant.LINKBASE, xmlDictPre)):
tempFactVOList = []
reportRole = item['@xlink:role']
isReportAllowed = False
if(reportDict.get(reportRole, None) is not None):
presentationDF = pandas.DataFrame(self.getListFromElement(Constant.PRESENTATIONARC, item))
presentationDF.set_index("@xlink:to", inplace=True)
presentationDF.head()
for item2 in self.getListFromElement(Constant.LOC, item):
factVO = FactVO()
factVO.xlink_href = item2["@xlink:href"]
factVO.reportRole = reportRole
factVO.labelID = item2["@xlink:label"]
factVO = self.setXsdValue(factVO, processCache)
if factVO.abstract != "true":
try:
factVO.order = self.getValueFromElement( ["@order"], presentationDF.loc[factVO.labelID], True)
tempFactVOList.append(factVO)
except Exception as e:
self.logger.debug("Error " + str(e))
if(self.isReportAllowed2(factVO.xlink_href)):
isReportAllowed = True
if(not isReportAllowed):
try:
del reportDict[reportRole]
except KeyError as e:
pass
else:
factVOList = factVOList + tempFactVOList
for report in reportDict.values():
Dao().addObject(objectToAdd = report, session = session, doFlush = True)
return factVOList
def worksheet():
from models.address import Address
from models.election import Election
if request.method == 'GET':
dao = Dao()
jurisdictions = Turf.get_jurisdictions(dao)
return render_template('voters/worksheet.html',
title='Voter Worksheet',
jurisdictions=jurisdictions)
blocks = json.loads(request.form['params'])['blocks']
dao = Dao(stateful=True)
elections = Election.get(dao)
data = []
for block in blocks:
data += Voter.get_by_block(dao, block, elections)
voters = []
dao.close()
for voter in data:
v = [
voter['last_name'], voter['first_name'], voter['middle_name'],
voter['name_suffix'],
str(Address(voter)), voter['city'], voter['zipcode'],
voter['gender'],
str(voter['birth_year']),
voter['party'] if voter['party'] else 'N', voter['voter_id'],
voter['reg_date'], voter['permanent_absentee']
if voter['permanent_absentee'] else 'N', voter['status'],
voter['uocava']
]
for election in elections:
v.append(voter[election['date']])
voters.append(v)
return jsonify(elections=[
election['date'] + ":" + election['description']
for election in elections
],
voters=voters)
def cal_fund_parameter_2_max_trade(self):
fund_code = 400001
max_trade_timestamp = int(Dao().get_fund_history_max_trade_timestamp_by_fund_code(fund_code)['max_trade_timestamp'][0])
fund_create_timestamp = int(Dao().fund_info(fund_code)['fund_create_timestamp'][0])
date_list = ['30d', '90d', '180d', '1y', '2y', '3y', '4y', '5y']
print('max_trade_timestamp', max_trade_timestamp)
trade_day_timestamp_list = Tool().get_trade_day_list(max_trade_timestamp)
print(trade_day_timestamp_list)
max_trade_timestamp_fund_history_pd = Dao().get_fund_history_by_fund_code_trade_timestamp(
fund_code, max_trade_timestamp)
max_trade_timestamp_fund_history_pd.drop('created_time', inplace=True, axis=1)
for timestamp, date in zip(trade_day_timestamp_list, date_list):
print('timestamp', timestamp)
if timestamp < fund_create_timestamp:
timestamp = fund_create_timestamp
self.save_fund_mean(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp)
self.save_fund_dividend(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp)
self.save_fund_report(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp)
self.save_fund_subtraction(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp)
def elections():
import dao.election_dao as el_dao
if request.method == 'GET':
dao = Dao()
elects = el_dao.get(dao)
return render_template('voters/elections.html',
title='Elections',
elections=elects)
dao = Dao(stateful=True)
latest = el_dao.get_latest_date(dao)
data = api_client.get('vtr_api/elections_after/%s' % (latest, ))
if data:
el_dao.add(dao, data)
elects = el_dao.get(dao)
result = jsonify(elections=elects)
else:
result = jsonify(msg='No new elections')
dao.close()
return result
def setUp(self):
self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True)
def hello():
dao = Dao('RNANet.db')
dao.load_chains()
chain_manager = Chain_Manager("chains.csv")
result = chain_manager.list_chain
return render_template('index.html', result=result)
def add_many(data, dao=None):
members = [GroupMember(d) for d in data]
values = [member.get_values() for member in members]
if not dao:
dao = Dao()
dao.add_many('group_members', GroupMember.db_cols, values)
def getValuesForExpression(self, fileDataOID, isCurrent, session):
if (isCurrent):
rs = Dao().getValuesForCurrentExpression(fileDataOID, session)
else:
rs = Dao().getValuesForExpression(fileDataOID, session)
return self.convertRStoDict(rs)
from dao.dao import Dao from providers.coeffprovider import CoeffProvider __author__ = 'mivanov' dao = Dao() coeffProvider = CoeffProvider() #coeffProvider.process_history(dao) coeffs = coeffProvider.get_coeff_list() dao.save_coeff_list(coeffs)
