def drop_many(): data = json.loads(request.form['params']) dao = Dao() try: dao.drop_many('voters', 'voter_id', data) return jsonify(msg='Records dropped!') except Exception as ex: return jsonify(error=str(ex))
def insert(self): sql = ("INSERT INTO elections " "(id, date, description) " "VALUES (%s, %s, %s);") vals = [ self.id, self.date, self.description ] dao = Dao() return dao.execute(sql, vals)
def test_get_block(self): dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True) contact = Contact() contact.address = Address({'address': '3000 Newcastle Rd'}) contact.zipcode = '48105' block = Location.get_block(dao, contact) contact.zipcode = '' contact.city = 'ANN ARBOR' block = Location.get_block(dao, contact) dao.close()
def neighborhood_drop(): params = json.loads(request.form['params']) dao = Dao(stateful=True) try: turf_dao.neighborhood_drop(dao, params['id']) return jsonify(msg='Neighborhood dropped!') except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
def add_many(): data = json.loads(request.form['params']) flds = list(data[0].keys()) vals = [list(rec.values()) for rec in data] dao = Dao() try: dao.add_many('voters', flds, vals) return jsonify(msg='Records saved!') except Exception as ex: return jsonify(error=str(ex))
def update_many(): data = json.loads(request.form['params']) flds = list(data[0].keys()) vals = [list(rec.values()) for rec in data] for i in range(0, len(data)): vals[i].append(data[i]['voter_id']) dao = Dao() try: dao.update_many('voters', 'voter_id', flds, vals) return jsonify(msg='Records saved!') except Exception as ex: return jsonify(error=str(ex))
def user_drop(): user_id = json.loads(request.args['id']) dao = Dao(stateful=True) try: success = User.delete_user(dao, user_id) if not success: msg = 'Record not deleted for unknown reason. Contact admin.' return jsonify(error=msg) users = User.get_users() return jsonify(users=users) except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
def user_add(): values = json.loads(request.form['params']) dao = Dao(stateful=True) try: user_id = User.add_user(dao, values) if 'precinct_ids' in values: ids = [int(pid) for pid in values['precinct_ids'].split(',') if pid] User.add_precinct_ids(dao, user_id, ids) users = User.get_users() return jsonify(users=users) except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
class TestContact(unittest.TestCase): def setUp(self): from dao.dao import Dao self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True) def tearDown(self): self.dao.close() def test_get_email_matches(self): email = '*****@*****.**' matches = ContactInfo.get_email_matches(self.dao, email, 100) pass
def name_duplicates(): if request.method == 'GET': dao = Dao(stateful=True) cities = turf_dao.get_city_names(dao) dups = con_dao.get_name_dups(dao) dao.close() for dup in dups: dup['name'] = str(PersonName(dup)) dup['address'] = str(Address(dup)) return render_template( 'contacts/dups.html', title='Name Duplicates', dups=dups, cities=cities )
def login(): if request.method == 'GET': return render_template( 'login.html', title='Bluestreets login' ) values = json.loads(request.form['params']) dao = Dao(stateful=True) try: rec = User.login(dao, values['username'], values['password']) session['user'] = rec return jsonify(msg='Hello %s!' % rec['first_name']) except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
def save_fund_subtraction(self, target_history_pd, fund_code, dateStr, start_trade_timestamp, end_trade_timestamp): start_pd = Dao().get_fund_history_by_fund_code_trade_timestamp(fund_code, start_trade_timestamp) similar_ranking_subtraction = start_pd.loc[0, 'similar_ranking'] - target_history_pd.loc[0, 'similar_ranking'] sc_subtraction = start_pd.loc[0, 'sc'] - target_history_pd.loc[0, 'sc'] info_pd = pd.DataFrame([fund_code], columns=['fund_code']) info_pd['similar_ranking_' + dateStr + '_change'] = similar_ranking_subtraction info_pd['sc_' + dateStr + '_change'] = sc_subtraction self.save_fund_parameter(info_pd, end_trade_timestamp)
def user_mgt(): from models.precinct import Precinct dao = Dao(stateful=True) jurisdictions = Precinct.get_jurisdictions(dao) precincts = Precinct.get_all(dao) roles = User.get_roles(dao) users = User.get_users(dao) precinct_admins = User.get_precinct_admins(dao) dao.close() return render_template( 'users.html', title='Bluestreets users', jurisdictions=jurisdictions, precincts=[precinct.serialize() for precinct in precincts], roles=roles, users=users, precinct_admins=precinct_admins )
def setFileDataStatus(self, voList): if(voList is not None): missingObjects = self.getMissingObjects() if(len(missingObjects) > 0): setattr(self.fileData, self.actualStatus, Constant.STATUS_WARNING) FileDataDao().setErrorMessage(errorMessage=str(missingObjects)[0:149], errorKey=self.errorKey, fileData=self.fileData) else: setattr(self.fileData, self.actualStatus, Constant.STATUS_OK) else: setattr(self.fileData, self.actualStatus, Constant.STATUS_OK) Dao().addObject(objectToAdd=self.fileData, session=self.session, doCommit=True)
def getReportDict(self, processCache, menuCategoryAllowed, session): #Obtengo reportes statements xmlDict= processCache[Constant.DOCUMENT_SUMMARY] reportDict = {} for report in xmlDict["FilingSummary"]["MyReports"]["Report"]: menuCategory = report.get("MenuCategory", None) if(menuCategory is None or menuCategory in menuCategoryAllowed): try: reportRole = report["Role"] if(self.isReportAllowed(reportRole)): reportShortName = report["ShortName"] report = Dao().getReport(reportShortName, session) if(report is None): report = Report() report.shortName = reportShortName[0:299] reportDict[reportRole] = report except Exception: pass logging.getLogger(Constant.LOGGER_GENERAL).debug("REPORT LIST " + str(reportDict)) return reportDict
def getOrCreateCompany(self, CIK, entityRegistrantName=None, session=None): company = CompanyDao().getCompany2(CIK, session) if (company is None): company = Company() company.CIK = CIK company.entityRegistrantName = entityRegistrantName company.listed = True elif entityRegistrantName is not None and entityRegistrantName != '': company.entityRegistrantName = entityRegistrantName Dao().addObject(objectToAdd=company, session=session, doCommit=True) return company
def getOrCreatePeriod3(self, instant, session): period = GenericDao().getOneResult(Period, and_(Period.instant == instant), session, raiseNoResultFound=False) if (period is None): period = Period() period.instant = instant period.type = "INST" Dao().addObject(objectToAdd=period, session=session, doFlush=True) return period
def contact_matches(): contact = Contact(json.loads(request.form['params'])) dao = Dao(stateful=True) try: matches = contact.get_matches(dao) for match in matches: match['name'] = str(PersonName(match)) match['address'] = str(Address(match)) return jsonify(matches=matches) except Exception as ex: return jsonify(error=str(ex))
def addEntityFact(self, factVOList, fileDataOID, reportDict, session, replace): objectAlreadyAdded = {} entityFactValueList = [] for factVO in factVOList: if (len(factVO.factValueList) > 0): concept = ConceptEngine().getOrCreateConcept( factVO.conceptName, session) efvList = EntityFactDao().getEntityFactValueList( fileDataOID, concept.OID, session) for efv in efvList: factValueKey = str(efv.fileDataOID) + "-" + str( efv.entityFact.conceptOID) objectAlreadyAdded[factValueKey] = "" entityFact = None if (reportDict[factVO.reportRole].OID is not None): entityFact = self.getEntityFact( concept.OID, reportDict[factVO.reportRole].OID, factVO.order, session) if (entityFact is None): entityFact = EntityFact() entityFact.conceptOID = concept.OID entityFact.report = reportDict[factVO.reportRole] entityFact.order_ = factVO.order Dao().addObject(objectToAdd=entityFact, session=session, doFlush=True) for factValueVO in factVO.factValueList: factValueKey = str(fileDataOID) + "-" + str(concept.OID) if (objectAlreadyAdded.get(factValueKey, None) is None): entityFactValue = EntityFactValue() entityFactValue.value = factValueVO.value entityFactValue.period = factValueVO.period entityFactValue.fileDataOID = fileDataOID entityFactValue.entityFact = entityFact entityFactValueList.append(entityFactValue) objectAlreadyAdded[factValueKey] = "" for efv in entityFactValueList: Dao().addObject(objectToAdd=efv, session=session) session.commit()
def csv_import(): if request.method == 'GET': return render_template( 'contacts/csv_import.html', title='Contact Import' ) data = json.loads(request.form['params']) dao = Dao(stateful=True) # precincts = Precinct.get_by_jwp(dao) groups = Group.get_all_by_code(dao) memberships = [] next_id = dao.get_max_id('contacts', 'id') for rec in data: rec['precinct_id'] = None next_id += 1 # if rec['jurisdiction'] and rec['ward'] and rec['precinct']: # jwp = '%s:%s:%s' % ( # rec['jurisdiction'].upper(), # rec['ward'].zfill(2), # rec['precinct'].zfill(3) # ) # rec['precinct_id'] = precincts[jwp]['id'] if rec['groups']: for code in rec['groups'].split('/'): if code in groups: memberships.append({ 'group_id': groups[code]['id'], 'contact_id': next_id, 'role': '', 'comment': '' }) try: con_dao.add_many(data) GroupMember.add_many(memberships) return jsonify(msg='Successful!') except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
def getOrCreatePeriod(self, periodType, endDate, session): period = GenericDao().getOneResult(Period, and_(Period.endDate == endDate, Period.startDate.is_(None)), session, raiseNoResultFound=False) if (period is None): period = Period() period.endDate = endDate period.type = periodType Dao().addObject(objectToAdd=period, session=session, doFlush=True) return period
def grid(): if request.method == 'GET': precincts = turf_dao.get_precincts() return render_template( 'contacts/grid.html', precincts=precincts, title='My Contacts' ) blocks = json.loads(request.form['params']) if not blocks: contacts = con_dao.get_all() elif len(blocks[0]) == 1 and 'precinct_id' in blocks[0]: contacts = con_dao.get_by_precinct(blocks[0]['precinct_id']) else: dao = Dao(stateful=True) contacts = [] for block in blocks: contacts += con_dao.get_by_block(dao, block) dao.close() return jsonify(contacts=to_local_format(contacts))
def lookup(): dao = Dao() matches = Voter.lookup(dao, request.args) result = [{ 'name': str(match.name), 'address': str(match.address), 'city': match.address.city, 'zipcode': match.address.zipcode, 'gender': match.gender, 'birth_year': match.birth_year, 'voter_id': match.id } for match in matches] return jsonify(matches=result)
def user_update(): values = json.loads(request.form['params']) dao = Dao(stateful=True) try: if 'fromPrecinctAdmin' in values: numrows = User.delete_precinct_admins_for_user(values['id']) numrows = User.update_user(dao, values) if numrows != 1: msg = 'Record not updated for unknown reason. Contact admin.' return jsonify(error=msg) if 'precinct_ids' in values: ids = [int(pid) for pid in values['precinct_ids'].split(',') if pid] del_ids = list(set(values['prev_precincts']) - set(ids)) add_ids = list(set(ids) - set(values['prev_precincts'])) User.delete_precinct_admins_for_user(dao, values['id'], del_ids) User.add_precinct_ids(dao, values['id'], add_ids) users = User.get_users() return jsonify(users=users) except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
def csv_import(): if request.method == 'GET': return render_template('contacts/csv_import.html', title='Contact Import') data = json.loads(request.form['params']) dao = Dao(stateful=True) # precincts = Precinct.get_by_jwp(dao) groups = Group.get_all_by_code(dao) memberships = [] next_id = dao.get_max_id('contacts', 'id') for rec in data: rec['precinct_id'] = None next_id += 1 # if rec['jurisdiction'] and rec['ward'] and rec['precinct']: # jwp = '%s:%s:%s' % ( # rec['jurisdiction'].upper(), # rec['ward'].zfill(2), # rec['precinct'].zfill(3) # ) # rec['precinct_id'] = precincts[jwp]['id'] if rec['groups']: for code in rec['groups'].split('/'): if code in groups: memberships.append({ 'group_id': groups[code]['id'], 'contact_id': next_id, 'role': '', 'comment': '' }) try: con_dao.add_many(data) GroupMember.add_many(memberships) return jsonify(msg='Successful!') except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
class TestContact(unittest.TestCase): def setUp(self): self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True) def tearDown(self): self.dao.close() def test_get_activists(self): rex = Contact.get_activists(self.dao) pass def test_get_best_voter(self): addr = Address({'address': '3000 Newcastle Rd'}) pn = PersonName({ 'last_name': 'weinblatt', 'first_name': 'howard' }) contact = Contact() contact.name = pn contact.address = addr voter = Contact.get_best_voter_rec(self.dao, contact) pass def test_get_best_turf(self): contact = Contact() contact.address = Address({ 'address': '3000 Newcastle Rd', 'zipcode': '48105' }) turf = Contact.get_best_turf(self.dao, contact) contact.address.zipcode = '' contact.address.city = 'ANN ARBOR' turf = Contact.get_best_turf(self.dao, contact) pass def test_clean_turf(self): num, unresolved = Contact.assign_precinct(self.dao) pass
def doImport(self): try: logging.info("START") time1 = datetime.now() if (self.skipOrProcess()): self.addOrModifyInit() if (self.replace): self.deleteImportedObject() self.logger.debug("**********START - Processing filename " + self.filename) voList = self.doImport2() persistentList = self.getPersistentList(voList) Dao().addObjectList(persistentList, self.session) self.setFileDataStatus(voList) if (voList is None): self.logger.info("***********FINISH AT " + str(datetime.now() - time1) + " " + self.filename) else: self.logger.info("***********FINISH AT " + str(datetime.now() - time1) + " " + self.filename + " objects added " + str(len(voList))) else: self.logger.info("Skipped " + self.filename) except (CustomException) as e: self.logger.error(self.filename + " " + e.status + " " + str(e)) self.addOrModifyFDError1(e) except MemoryError as e: self.logger.error(self.filename + " " + str(e)) FileDataDao().addOrModifyFileData( statusKey=self.actualStatus, statusValue=Constant.STATUS_ERROR, filename=self.filename, errorMessage='MemoryError', errorKey=self.errorKey) except ExpatError as e: self.logger.error(self.filename + " " + str(e)) self.session.rollback() self.session.close() self.session = DBConnector().getNewSession() self.addOrModifyFDError2('not well-formed') except Exception as e: self.logger.error(self.filename + " " + str(e)) self.session.rollback() self.session.close() self.session = DBConnector().getNewSession() self.addOrModifyFDError2(str(e)[0:149]) finally: self.session.commit() self.session.close()
def neighborhoods(): if request.method == 'GET': dao = Dao(stateful=True) types = turf_dao.get_turf_types(dao) jurisdictions = turf_dao.get_jurisdictions(dao) juris_names = [j['jurisdiction_name'] for j in jurisdictions] precincts = turf_dao.get_precincts(dao) pct_names = [ '%s, %s, %s' % (p['jurisdiction_name'], p['ward'], p['precinct']) for p in precincts ] wards = build_wards(precincts) state_house_districts = sorted( list(set([p['state_house'] for p in precincts]))) state_senate_districts = sorted( list(set([p['state_senate'] for p in precincts]))) congressional_districts = sorted( list(set([p['congress'] for p in precincts]))) nhoods = turf_dao.get_neighborhoods(dao) dao.close() return render_template('neighborhoods.html', title="Neighborhoods", types=types, jurisdictions=jurisdictions, juris_names=juris_names, wards=wards, precincts=precincts, pct_names=pct_names, state_house_districts=state_house_districts, state_senate_districts=state_senate_districts, congressional_districts=congressional_districts, neighborhoods=nhoods) params = json.loads(request.form['params']) block_vals = [] if params['blocks']: block_vals = [ (block['street_name'], block['street_type'], block['low_addr'], block['high_addr'], block['odd_even'], block['precinct_id']) for block in params['blocks'] ] dao = Dao(stateful=True) nbh_id = turf_dao.add_neighborhood(dao, params['type'], params['name'], params['pct_ids'], block_vals) dao.close() return jsonify(nbh_id=nbh_id)
def voter_lookup(): from models.voter import Voter contact = json.loads(request.form['params']) dao = Dao(stateful=True) try: voters = Voter.lookup(dao, contact) candidates = [{ 'name': { 'last': voter.name.last, 'first': voter.name.first, 'middle': voter.name.middle, 'suffix': voter.name.suffix }, 'address': { 'house_number': voter.address.house_number, 'pre_direction': voter.address.pre_direction, 'street_name': voter.address.street_name, 'street_type': voter.address.street_type, 'suf_direction': voter.address.suf_direction, 'unit': voter.address.unit, 'city': voter.address.city, 'zipcode': voter.address.zipcode }, 'voter_info': { 'voter_id': voter.voter_id, 'precinct_id': voter.precinct_id, 'birth_year': voter.birth_year, 'gender': voter.gender, 'reg_date': voter.reg_date } } for voter in voters] return jsonify(candidates=candidates) except Exception as ex: return jsonify(error=str(ex)) finally: dao.close()
class TestContact(unittest.TestCase): def setUp(self): self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True) def tearDown(self): self.dao.close() def test_get_activists(self): rex = Contact.get_activists(self.dao) pass def test_get_best_voter(self): addr = Address({'address': '3000 Newcastle Rd'}) pn = PersonName({'last_name': 'weinblatt', 'first_name': 'howard'}) contact = Contact() contact.name = pn contact.address = addr voter = Contact.get_best_voter_rec(self.dao, contact) pass def test_get_best_turf(self): contact = Contact() contact.address = Address({ 'address': '3000 Newcastle Rd', 'zipcode': '48105' }) turf = Contact.get_best_turf(self.dao, contact) contact.address.zipcode = '' contact.address.city = 'ANN ARBOR' turf = Contact.get_best_turf(self.dao, contact) pass def test_clean_turf(self): num, unresolved = Contact.assign_precinct(self.dao) pass
def assign_precinct(contacts, dao=None): if not dao: dao = Dao(stateful=True) updates = [] for contact in contacts: # get voter rex by name + address voter = Contact.get_best_voter_rec(dao, contact) if voter: nn = (contact.name.nickname, contact.name.nickname_meta) contact.name = voter.name contact.nickname = nn[0] contact.nickname_meta = nn[1] contact.address = voter.address contact.birth_year = voter.birth_year contact.gender = voter.gender contact.voter_id = voter.voter_id contact.reg_date = voter.reg_date updates.append(contact) continue turf = Contact.get_best_turf(dao, contact) if turf: contact.address.pre_direction = turf['pre_direction'] contact.address.street_name = turf['street_name'] contact.address.street_type = turf['street_type'] contact.address.suf_direction = turf['suf_direction'] contact.address.precinct_id = turf['precinct_id'] contact.address.city = turf['city'] contact.address.zipcode = turf['zipcode'] updates.append(contact) continue # get voter rex by name only # Contact.update_many(dao, updates) dao.close() return len(updates)
def single_fund_info(self, code_s, type_s): code, ishb, basic_info_pd, funds = ReptileFund().get_pingzhongdata(code_s, type_s) if ishb is not None: if ishb: fund_wide_pd, rateInSimilar_pd, grandTotal_pd, worth_pd, current_fund_manager_pd = ProcessFund().get_hb_pingzhongdata_2_df( basic_info_pd, funds) else: fund_wide_pd, rateInSimilar_pd, grandTotal_pd, worth_pd, current_fund_manager_pd = ProcessFund().get_pingzhongdata_2_df( basic_info_pd, funds) # print(code, '用时2', end) Dao().save_fund_info(fund_wide_pd, 'fund_code', int(code)) # print(code, '用时3', end) if not rateInSimilar_pd.empty: Dao().save_fund_history(rateInSimilar_pd, 'fund_code', int(code)) pass if not grandTotal_pd.empty: Dao().save_fund_history(grandTotal_pd, 'fund_code', int(code)) pass if not worth_pd.empty: Dao().save_fund_history(worth_pd, 'fund_code', int(code)) pass if not current_fund_manager_pd.empty: Dao().save_manager_info(current_fund_manager_pd)
def elections(): import dao.election_dao as el_dao if request.method == 'GET': dao = Dao() elects = el_dao.get(dao) return render_template( 'voters/elections.html', title='Elections', elections=elects ) dao = Dao(stateful=True) latest = el_dao.get_latest_date(dao) data = api_client.get('vtr_api/elections_after/%s' % (latest,)) if data: el_dao.add(dao, data) elects = el_dao.get(dao) result = jsonify(elections=elects) else: result = jsonify(msg='No new elections') dao.close() return result
def get_matches(self, dao=None): if not dao: dao = Dao(stateful=True) if self.info and (self.info.phone1 or self.info.phone2): matches = self.get_phone_matches(dao) if matches: return matches if self.info and self.info.email: matches = self.get_email_matches(dao) if matches: return matches if self.address: matches = self.get_name_addr_matches(dao) if matches: return matches return self.get_name_only_matches(dao)
def save_fund_mean(self, target_history_pd, fund_code, dateStr, start_trade_timestamp, end_trade_timestamp): interval_pd = Dao().get_fund_history_by_trade_timestamp_interval(fund_code, start_trade_timestamp, end_trade_timestamp) info_pd = pd.DataFrame([fund_code], columns=['fund_code']) info_pd['net_worth_report_' + dateStr + '_mean'] = round(interval_pd['equity_return'].mean(0), 10) info_pd['similar_ranking_' + dateStr + '_mean'] = round(interval_pd['similar_ranking'].mean(0), 10) info_pd['sc_' + dateStr + '_mean'] = round(interval_pd['sc'].mean(0), 10) info_pd['similar_ranking_per_' + dateStr + '_mean'] = round( interval_pd['similar_ranking_per'].mean(0), 10) info_pd['hs300_gt_earn_per_' + dateStr + '_mean'] = round(interval_pd['hs300_grand_total'].mean(0), 10) info_pd['similar_mean_gt_earn_per_' + dateStr + '_mean'] = round( interval_pd['similar_mean_grand_total'].mean(0), 10) info_pd['self_gt_earn_per_' + dateStr + '_mean'] = round(interval_pd['self_grand_total'].mean(0), 10) info_pd['fund_code'] = fund_code self.save_fund_parameter(info_pd, end_trade_timestamp)
def street_lookup(): from models.address import str_parse from utils.match import MatchLib params = json.loads(request.form['params']) addr = str_parse(params['address']) addr['county_code'] = '81' # TODO: get from cfg addr['meta'] = Address.get_street_meta(addr['street_name']) if params['city']: addr['city'] = params['city'] if params['zipcode']: addr['zipcode'] = params['zipcode'] dao = Dao() try: candidates = turf_dao.street_fuzzy_lookup(dao, addr) matches = MatchLib.get_best_matches( addr['street_name'], [c['street_name'] for c in candidates], 80) matches = [match[0] for match in matches] candidates = [ candidate for candidate in candidates if candidate['street_name'] in matches ] candidates = [{ 'address': { 'house_num_low': candidate['house_num_low'], 'house_num_high': candidate['house_num_high'], 'odd_even': candidate['odd_even'], 'pre_direction': candidate['pre_direction'], 'street_name': candidate['street_name'], 'street_type': candidate['street_type'], 'suf_direction': candidate['suf_direction'], 'unit_low': candidate['ext_low'], 'unit_high': candidate['ext_high'], 'city': candidate['city'], 'zipcode': candidate['zipcode'] }, 'voter_info': { 'precinct_id': candidate['precinct_id'] } } for candidate in candidates] return jsonify(candidates=candidates) except Exception as ex: return jsonify(error=str(ex))
def addOrModifyFileData(self, statusKey=None, statusValue=None, filename=None, externalSession=None, errorMessage=None, errorKey=None, fileData=None, extraData=None): try: if (externalSession is None): session = DBConnector().getNewSession() else: session = externalSession if (fileData is None): fileData = FileDataDao.getFileData(filename, session) setattr(fileData, statusKey, statusValue) self.setErrorMessage(errorMessage=errorMessage, errorKey=errorKey, extraData=extraData, fileData=fileData) Dao().addObject(objectToAdd=fileData, session=session, doCommit=True) if (externalSession is None): session.close() return fileData except Exception as e: logging.getLogger(Constant.LOGGER_GENERAL).exception(e) raise e
def neighborhoods(): if request.method == 'GET': dao = Dao(stateful=True) types = turf_dao.get_turf_types(dao) jurisdictions = turf_dao.get_jurisdictions(dao) juris_names = [j['jurisdiction_name'] for j in jurisdictions] precincts = turf_dao.get_precincts(dao) pct_names = ['%s, %s, %s' % (p['jurisdiction_name'], p['ward'], p['precinct']) for p in precincts] wards = build_wards(precincts) state_house_districts = sorted(list(set([p['state_house'] for p in precincts]))) state_senate_districts = sorted(list(set([p['state_senate'] for p in precincts]))) congressional_districts = sorted(list(set([p['congress'] for p in precincts]))) nhoods = turf_dao.get_neighborhoods(dao) dao.close() return render_template( 'neighborhoods.html', title="Neighborhoods", types=types, jurisdictions=jurisdictions, juris_names=juris_names, wards=wards, precincts=precincts, pct_names=pct_names, state_house_districts=state_house_districts, state_senate_districts=state_senate_districts, congressional_districts=congressional_districts, neighborhoods=nhoods ) params = json.loads(request.form['params']) block_vals = [] if params['blocks']: block_vals = [( block['street_name'], block['street_type'], block['low_addr'], block['high_addr'], block['odd_even'], block['precinct_id'] ) for block in params['blocks']] dao = Dao(stateful=True) nbh_id = turf_dao.add_neighborhood( dao, params['type'], params['name'], params['pct_ids'], block_vals ) dao.close() return jsonify(nbh_id=nbh_id)
def importMasterIndexFor(self, period, replaceMasterFile, session=None, threadNumber=1): dbconnector = DBConnector() if (session is None): session = dbconnector.getNewSession() localURL = Constant.CACHE_FOLDER + 'master' + str( period.year) + "-Q" + str(period.quarter) + '.gz' secURL = "https://www.sec.gov/Archives/edgar/full-index/" + str( period.year) + "/QTR" + str(period.quarter) + "/master.gz" print(localURL, secURL) file = getBinaryFileFromCache(localURL, secURL, replaceMasterFile) with gzip.open(BytesIO(file), 'rb') as f: file_content = f.read() text = file_content.decode("ISO-8859-1") text = text[text.find("CIK", 0, len(text)):len(text)] point1 = text.find("\n") point2 = text.find("\n", point1 + 1) text2 = text[0:point1] + text[point2:len(text)] df = pandas.read_csv(StringIO(text2), sep="|") df.set_index("CIK", inplace=True) df.head() print("STARTED FOR PERIOD " + str(period.year) + "-" + str(period.quarter)) for row in df.iterrows(): CIK = row[0] filename = row[1]["Filename"] formType = row[1]["Form Type"] if (formType == "10-Q" or formType == "10-K"): fd = FileDataDao.getFileData(filename, session) if (fd is None): company = CompanyEngine().getOrCreateCompany( CIK=CIK, session=session) fd = FileData() fd.fileName = filename fd.company = company Dao().addObject(objectToAdd=fd, session=session, doCommit=True) print("FD Added " + filename) print("FINISHED")
def filter_api_imports(api_voters, nbh_id): dao = Dao() my_voters = vtr_dao.get_for_neighborhood(dao, nbh_id) my_voters = {v['voter_id']: v for v in my_voters} api_voters = {v['voter_id']: v for v in api_voters} inserts = [] conflicts = [] deletes = [] for api_id, api_voter in api_voters.items(): if api_id in my_voters: if has_diff(api_voter, my_voters[api_id]): conflicts.append(api_voter) conflicts.append(my_voters[api_id]) else: inserts.append(api_voter) for vid in my_voters.keys(): if vid not in api_voters: deletes.append(my_voters[vid]) return inserts, conflicts, deletes
def fillFileData(self, fileData, processCache, session): insXMLDict = processCache[Constant.DOCUMENT_INS] documentType = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentType']) #logging.getLogger(Constant.LOGGER_GENERAL).debug("documentType " + documentType) amendmentFlag = self.getValueFromElement(['#text'], insXMLDict['dei:AmendmentFlag']) amendmentFlag = amendmentFlag.lower() in ("true") documentPeriodEndDate = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentPeriodEndDate']) #logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentPeriodEndDate " + documentPeriodEndDate) documentFiscalYearFocus = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentFiscalYearFocus']) #logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentFiscalYearFocus " + documentFiscalYearFocus) documentFiscalPeriodFocus = self.getValueFromElement(['#text'], insXMLDict['dei:DocumentFiscalPeriodFocus']) #logging.getLogger(Constant.LOGGER_GENERAL).debug("DocumentFiscalPeriodFocus " + documentFiscalPeriodFocus) #entityRegistrantName = insXMLDict['dei:EntityRegistrantName']['#text'] fileData.documentType = documentType fileData.amendmentFlag = amendmentFlag fileData.documentPeriodEndDate = documentPeriodEndDate if(len(documentFiscalYearFocus) <= 4): fileData.documentFiscalYearFocus = documentFiscalYearFocus fileData.documentFiscalPeriodFocus = documentFiscalPeriodFocus #fileData.entityRegistrantName = entityRegistrantName Dao().addObject(objectToAdd = fileData, session = session, doCommit = True) return fileData
def getFactByReport(self, reportDict, processCache, session): factVOList = [] #Obtengo para cada reporte sus conceptos xmlDictPre = processCache[Constant.DOCUMENT_PRE] for item in self.getListFromElement(Constant.PRESENTATON_LINK, self.getElementFromElement(Constant.LINKBASE, xmlDictPre)): tempFactVOList = [] reportRole = item['@xlink:role'] isReportAllowed = False if(reportDict.get(reportRole, None) is not None): presentationDF = pandas.DataFrame(self.getListFromElement(Constant.PRESENTATIONARC, item)) presentationDF.set_index("@xlink:to", inplace=True) presentationDF.head() for item2 in self.getListFromElement(Constant.LOC, item): factVO = FactVO() factVO.xlink_href = item2["@xlink:href"] factVO.reportRole = reportRole factVO.labelID = item2["@xlink:label"] factVO = self.setXsdValue(factVO, processCache) if factVO.abstract != "true": try: factVO.order = self.getValueFromElement( ["@order"], presentationDF.loc[factVO.labelID], True) tempFactVOList.append(factVO) except Exception as e: self.logger.debug("Error " + str(e)) if(self.isReportAllowed2(factVO.xlink_href)): isReportAllowed = True if(not isReportAllowed): try: del reportDict[reportRole] except KeyError as e: pass else: factVOList = factVOList + tempFactVOList for report in reportDict.values(): Dao().addObject(objectToAdd = report, session = session, doFlush = True) return factVOList
def worksheet(): from models.address import Address from models.election import Election if request.method == 'GET': dao = Dao() jurisdictions = Turf.get_jurisdictions(dao) return render_template('voters/worksheet.html', title='Voter Worksheet', jurisdictions=jurisdictions) blocks = json.loads(request.form['params'])['blocks'] dao = Dao(stateful=True) elections = Election.get(dao) data = [] for block in blocks: data += Voter.get_by_block(dao, block, elections) voters = [] dao.close() for voter in data: v = [ voter['last_name'], voter['first_name'], voter['middle_name'], voter['name_suffix'], str(Address(voter)), voter['city'], voter['zipcode'], voter['gender'], str(voter['birth_year']), voter['party'] if voter['party'] else 'N', voter['voter_id'], voter['reg_date'], voter['permanent_absentee'] if voter['permanent_absentee'] else 'N', voter['status'], voter['uocava'] ] for election in elections: v.append(voter[election['date']]) voters.append(v) return jsonify(elections=[ election['date'] + ":" + election['description'] for election in elections ], voters=voters)
def cal_fund_parameter_2_max_trade(self): fund_code = 400001 max_trade_timestamp = int(Dao().get_fund_history_max_trade_timestamp_by_fund_code(fund_code)['max_trade_timestamp'][0]) fund_create_timestamp = int(Dao().fund_info(fund_code)['fund_create_timestamp'][0]) date_list = ['30d', '90d', '180d', '1y', '2y', '3y', '4y', '5y'] print('max_trade_timestamp', max_trade_timestamp) trade_day_timestamp_list = Tool().get_trade_day_list(max_trade_timestamp) print(trade_day_timestamp_list) max_trade_timestamp_fund_history_pd = Dao().get_fund_history_by_fund_code_trade_timestamp( fund_code, max_trade_timestamp) max_trade_timestamp_fund_history_pd.drop('created_time', inplace=True, axis=1) for timestamp, date in zip(trade_day_timestamp_list, date_list): print('timestamp', timestamp) if timestamp < fund_create_timestamp: timestamp = fund_create_timestamp self.save_fund_mean(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp) self.save_fund_dividend(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp) self.save_fund_report(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp) self.save_fund_subtraction(max_trade_timestamp_fund_history_pd, fund_code, date, timestamp, max_trade_timestamp)
def elections(): import dao.election_dao as el_dao if request.method == 'GET': dao = Dao() elects = el_dao.get(dao) return render_template('voters/elections.html', title='Elections', elections=elects) dao = Dao(stateful=True) latest = el_dao.get_latest_date(dao) data = api_client.get('vtr_api/elections_after/%s' % (latest, )) if data: el_dao.add(dao, data) elects = el_dao.get(dao) result = jsonify(elections=elects) else: result = jsonify(msg='No new elections') dao.close() return result
def setUp(self): self.dao = Dao(db_file='c:/bench/bluestreets/data/26161.db', stateful=True)
def hello(): dao = Dao('RNANet.db') dao.load_chains() chain_manager = Chain_Manager("chains.csv") result = chain_manager.list_chain return render_template('index.html', result=result)
def add_many(data, dao=None): members = [GroupMember(d) for d in data] values = [member.get_values() for member in members] if not dao: dao = Dao() dao.add_many('group_members', GroupMember.db_cols, values)
def getValuesForExpression(self, fileDataOID, isCurrent, session): if (isCurrent): rs = Dao().getValuesForCurrentExpression(fileDataOID, session) else: rs = Dao().getValuesForExpression(fileDataOID, session) return self.convertRStoDict(rs)
from dao.dao import Dao from providers.coeffprovider import CoeffProvider __author__ = 'mivanov' dao = Dao() coeffProvider = CoeffProvider() #coeffProvider.process_history(dao) coeffs = coeffProvider.get_coeff_list() dao.save_coeff_list(coeffs)