def analysis_transcript(video): session = create_session()() # 转换字幕成text from transcript import get_clean_transcript, simple_token clean_transcript = get_clean_transcript(video.xml_transcript) video.clean_transcript = clean_transcript # 分词 tokens = simple_token(clean_transcript) import logging logging.basicConfig(filename='example.log',level=logging.DEBUG) # 将词汇与视频建立关系,加入数据库 from models import Word, get_or_create for word in tokens: # 记录非全数字,全字母,长度超过15的词 if len(word) > 15 or not word.isalpha() or not word.isalnum(): logging.debug(word) break if word.istitle() is False: word_instance, flag = get_or_create(session, Word, text=word) else: word_instance, flag = get_or_create(session, Word, text=word.lower()) session.commit() # 添加关系 video.words.append(word_instance) session.commit() # 统计词频,总字数,加入数据库,计算语速 word_number = len(tokens) video.word_number = word_number video.speed = word_number / duration_to_min(video.duration) from transcript import statistic_frequency video.word_frequency = statistic_frequency(tokens) session.commit() session.remove()
def save_to_db(db_file, store): engine = create_engine('sqlite:///{}'.format(db_file)) Session = sessionmaker(bind=engine) # Initialize DB if needed if not os.path.isfile(db_file): models.Base.metadata.create_all(engine) # Save to database session = Session() # Select/save store db_store = get_or_create(session, models.Store, name=store.name) store_id = db_store.id # Process the gathered data for overview_page in tqdm(store.overview_pages): # Save category hierarchy category_parent_id = None for category in overview_page['product_category']: # Check if exists... res = session.query(models.Category).filter( models.Category.name == category).filter( models.Category.parent_id == category_parent_id).first() # ... else create new category if res: category_parent_id = res.id else: new_category = models.Category(name=category, parent_id=category_parent_id) session.add(new_category) session.commit() category_parent_id = new_category.id # Save products for position, product_url in enumerate(overview_page['product_urls']): if product_url in store.product_pages.keys(): product = store.product_pages[product_url] # Select/save brand db_brand = get_or_create(session, models.Brand, name=product['brand']) brand_id = db_brand.id # Save product new_product = models.Product( name=product['product_name'], type=product['product_type'], price=product['price'], page_position=position + 1, # From 0-based to 1-based page_number=overview_page['page_number'], category_id=category_parent_id, brand_id=brand_id, store_id=store_id) session.add(new_product) session.commit() session.close()
def insert_item(self, token_object, new_session, session_time=timedelta(0)): """ Create new session or update existing session object :param token_object: Token that has to be converted into session :param new_session: Boolean that denotes if this is a new session. A new session will be created if this value is True :param session_time: If an existing session is to be updated, session_time contains the new value of total session time """ if self.file_type == settings.APACHE_COMMON: url_obj = get_or_create(self.session, Uurl, url=token_object.resource_requested) elif self.file_type == settings.APACHE_COMBINED: url_obj = get_or_create(self.session, Uurl, url=token_object.resource_requested) elif self.file_type == settings.SQUID: url_obj = get_or_create(self.session, Uurl, url=token_object.url) # If this is a new session if new_session: # Create session object session_obj = Session(ip=token_object.ip_address, session_time=session_time) # Set start and end time session_obj.start_time = token_object.date_time session_obj.end_time = token_object.date_time # If new_session is False, new session may or may not be created # (depending upon the session_time) else: # Try to get session object session_obj = get_or_create(self.session, Session, ip=token_object.ip_address) # If the object is a new session if session_obj.session_time is timedelta(0): session_obj.start_time = token_object.date_time session_obj.session_time = session_time session_obj.end_time = token_object.date_time # Add url to session session_obj.session_urls.append(url_obj) self.session.add(session_obj)
def on_track_change(pl, session): global song_to_write if song_to_write is not None \ and datetime.now() - song_to_write['start_time'] > timedelta(seconds=4): artist = models.get_or_create( session, models.Artist, search_param={'name': song_to_write['xesam:artist'][0]}, name=song_to_write['xesam:artist'][0]) album = models.get_or_create( session, models.Album, search_param={'name': song_to_write['xesam:album']}, name=song_to_write['xesam:album'], img=song_to_write['mpris:artUrl'], artist=artist) song = models.get_or_create( session, models.Song, search_param={'name': song_to_write['xesam:title']}, name=song_to_write['xesam:title'], length=int(song_to_write['mpris:length'] / 1000000), album=album, spotify_id=song_to_write['mpris:trackid'].split(':')[2]) play = models.Play( duration=((datetime.now() - song_to_write['start_time']) - song_to_write['paused_time']).seconds, start_time=song_to_write['start_time'], end_time=datetime.now(), song=song) logger.info( "Writing playback - {artist} - {title} | Duration of playback - {playing_time} sec" .format(artist=artist.name, title=song.name, playing_time=play.duration)) logger.info("Current song - {song} - {title}".format(song=pl.get_artist(), title=pl.get_title())) song_to_write = dict(start_time=datetime.now(), paused_time=timedelta(seconds=0)) song_to_write.update(dict(pl.props.metadata))
def get_attempt(package): """ Always returns a brand new models.Attempt instance, bound to the expected models.ArticlePkg instance. """ with PackageAnalyzer(package) as pkg: if pkg.is_valid_package(): article = models.get_or_create(models.ArticlePkg, **pkg.meta) pkg_checksum = utils.make_digest_file(package) attempt_meta = {'package_md5': pkg_checksum, 'articlepkg_id': article.id} attempt = models.get_or_create(models.Attempt, **attempt_meta) return attempt else: sys.stderr.write("Invalid package: %s\n" % pkg.errors) utils.mark_as_failed(package)
def post(self): email_id = self.request.form.get("email")[0] credential = EmailCredential() credential.email = email_id if self.validate(credential, email_validator): room, created = get_or_create(EmailBase, email=email_id) if created: return self.json_response( {'success': "You are successfully added. We will invite on the beta launch"}) else: return self.json_response( {'success': "We will invite on the beta launch", 'error': "You are already registered."}) else: return self.json_response( {'error': "Wrong Email Address. Please Correct yourself."})
async def on_ready(self): logger.info('Logged in as') logger.info(client.user.name) logger.info(client.user.id) logger.info('------') session = Session() session.expire_on_commit = False self.service = get_or_create(session, Service, name="discord") database_user = get_or_create(session, User, id=client.user.id, service_id=self.service.id) database_user.username = client.user.display_name try: session.commit() except: logger.error("Couldn't commit bot user to database!") session.rollback() finally: session.close() self.eve = evebot.EveBot(database_user)
async def constructMetadata(self, message): try: session = Session() session.expire_on_commit = False if (message.guild): current_server = Server(id=message.guild.id, service_id=self.service.id, server_name=message.guild.name) else: current_server = None current_user = User(id=message.author.id, service_id=self.service.id, username=message.author.display_name) if (message.channel.name): current_channel = Chat(id=message.channel.id, server_id=current_server.id, chat_name=message.channel.name, nsfw=message.channel.is_nsfw()) else: current_channel = get_or_create(session, Chat, id=message.channel.id, server_id=current_server.id, chat_name=message.channel.id) except Exception as e: logger.error("Couldn't get data from message! " + str(e)) return None # Metadata for use by commands and reactions metadata = { "service": self.service, "user": current_user, "server": current_server, "chat": current_channel, "message": message, "client": self } return metadata
def post(self): email_id = self.request.form.get("email")[0] credential = EmailCredential() credential.email = email_id if self.validate(credential, email_validator): room, created = get_or_create(EmailBase, email=email_id) if created: return self.json_response({ 'success': "You are successfully added. We will invite on the beta launch" }) else: return self.json_response({ 'success': "We will invite on the beta launch", 'error': "You are already registered." }) else: return self.json_response( {'error': "Wrong Email Address. Please Correct yourself."})
def __create_bookperson(form_data): """ Create a Contributor record from the given form_data. Return the created records as a list, if any. Else return None. form_data is expected to be a JSON list of objects. Each object should have the fields `last_name` and `first_name`. Raises ConstraintError if one of the persons in form_data has blank firstname or lastname. """ from flask_login import current_user try: parse = json.loads(form_data) persons_created = [] for parson in parse: _firstname = parson.get("firstname", "").strip() _lastname = parson.get("lastname", "").strip() if _firstname and _lastname: persons_created.insert( 0, get_or_create(Contributor, will_commit=True, firstname=_firstname, lastname=_lastname, creator=current_user)) else: raise ConstraintError("firstname and lastname are present", str(parson)) return persons_created except ValueError: # For errors in pasing JSON return []
from sqlalchemy import create_engine, Column, String, Integer, MetaData, Table from sqlalchemy.orm import mapper, create_session, sessionmaker from models import db_connect, create_tables, Venue, get_or_create import csv CSV_FILE = 'build_db/venues.csv' engine = db_connect() create_tables(engine) Session = sessionmaker(bind=engine) session = Session() with open(CSV_FILE) as f: cf = csv.DictReader(f, delimiter=',') for row in cf: v, v_created = get_or_create(session, Venue, **row) session.add(v) session.commit() session.close() def get_scrape_urls(ticketing): scrape_url = [url[0] for url in session.query(Venue.scrape_url).filter(Venue.ticketing == ticketing).all()] return scrape_url def get_venue_id(response): venue_id = session.query(Venue.id).filter(Venue.scrape_url == response).first()[0] return venue_id
#!bin/python import models import time import dbm session = models.get_session() addrdb = dbm.open('addr.db', 'c') baddrdb = dbm.open('baddr.db', 'c') for addrtxt in addrdb.keys(): (lat, lon) = addrdb[addrtxt].split(":") addr = models.get_or_create(session, models.Address, address=addrtxt) geo = models.get_or_create(session, models.Geocoding, address_id=addr.id, latitude=lat, longitude=lon, error=0) for addrtxt in baddrdb.keys(): addr = models.get_or_create(session, models.Address, address=addrtxt) geo = models.get_or_create(session, models.Geocoding, address_id=addr.id, latitude=0.0, longitude=0.0, error=1)
def save(statuses, fetch_model, fetch_datetime): new_users_count = 0 new_data_count = 0 counter = 0 # dictionary with the fetched users external id as key to prevent addition of the same user more than ones users_added = {} # iterate over all the statuses and insert new data to DB for s in statuses: counter+=1 print "\t{0:d}] id: {1:d}".format(counter, s.id) user = None if users_added.has_key(s.user.id): user = users_added[s.user.id] else: user, user_is_new = get_or_create(mysession, User, external_id = s.user.id, source = fetch_model.source ) if user_is_new: new_users_count+=1 user.name = s.user.name user.screen_name = s.user.screen_name user.description = htmlutils.unescape(s.user.description) user.profile_image_url = s.user.profile_image_url user.verified = s.user.verified user.language = s.user.lang.split("-")[0] user.fetched_data = s.user.AsDict() user.fetched_at = fetch_datetime user.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(s.user.created_at,'%a %b %d %H:%M:%S +0000 %Y')) users_added[s.user.id] = user # remember added user in case there he's got more data than this datum datum, datum_is_new = get_or_create(mysession, Datum, external_id = s.id, source = fetch_model.source ) if datum_is_new: new_data_count+=1 sDict = s.AsDict() language = fetch_model.language or s.GetLang() datum.language = language datum.domain = fetch_model.domain prediction = predictor.predict(language, fetch_model.domain, fetch_model.source, [s.text])[0] datum.class_value = prediction.value datum.gold = 0 datum.text = htmlutils.unescape(s.text) datum.user = user if sDict.has_key('hashtags') and len(sDict['hashtags'])>0: datum.hashtags = sDict['hashtags'] if sDict.has_key('urls') and len(sDict['urls'])>0: datum.urls = sDict['urls'] if sDict.has_key('media'): datum.media = sDict['media'] datum.fetched_data = sDict datum.fetched_at = fetch_datetime datum.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(s.created_at,'%a %b %d %H:%M:%S +0000 %Y')) print("\tnew datum -> insert") print("\tpredicted -> class name: {0:s}, class value: {1:s}".format(prediction.name,str(prediction.value))) else: print("\talready exist, better luck next time :)") if new_users_count>0 or new_data_count>0: print("\tinsert all {0:d} new users and {1:d} new data items".format(new_users_count, new_data_count)) mysession.commit() print ("\tdone") else: print("\tno new data to save") dispose() print("connection disposed")
def add_product(self): # to skip validation if there is no change in sizes and colors if self.product and not self.selected_colors_sizes: self.selected_colors_sizes = 'no change' product_data = { 'skucode': self.productSkucodeInput.text(), 'originalPrice': self.originalPriceInput.text(), 'suggestedPrice': self.suggestedPrice.text(), 'image': self.imagePath, 'sizes': self.selected_colors_sizes } fields_translation = { 'skucode': 'الكود', 'originalPrice': 'سعر الجملة', 'suggestedPrice': 'أقل سعر', 'image': 'صورة المنتج', 'sizes': 'المقاسات' } for field_name, value in product_data.items(): if not value: field_translated = fields_translation[field_name] QMessageBox.warning( self, field_translated, f'يرجى ملىء حقل ({field_translated}) اولا!') return product_with_same_code = self.session.query(Product).filter_by( sku_code=product_data['skucode']).first() if product_with_same_code: if not self.product == product_with_same_code: QMessageBox.warning( self, 'خطأ بالكود', f'هذا الكود تم استخدامه من قبل ، يرجى اختيار كود جديد') return img_name = product_data['image'].split('/')[-1] image_destination = str(Path.joinpath(current_path, 'images', img_name)) self.same_destination = False if self.product: if self.product.image != image_destination: self.same_destination = True # we check if the colors-sizes list changed if not self.sizesListPreview.text().startswith('المتاح'): # if changed we start changing colors depending on new values # but with respect to sold units of previous colors and its sizes self.sold_colors = set(unit.color for unit in self.product.sold_units) self.sold_colors.update(c_s[0] for c_s in self.selected_colors_sizes) for color_obj in self.product.colors: # if there's sold units or will be created in the new editing # with this color, we just delete all availabel sizes # for clean addition later in the next lines if we want if color_obj.name in self.sold_colors: self.session.query(ColorSize).filter_by( color=color_obj, ).delete() # else we delete the color else: self.session.delete(color_obj) if not self.same_destination: img = Image.open(product_data['image']) img.save(image_destination) if not self.product: self.product = get_or_create( self.session, Product, sku_code=product_data['skucode'], original_price=product_data['originalPrice'], suggested_price=product_data['suggestedPrice'], image=image_destination, ) else: self.product.sku_code = product_data['skucode'] self.product.original_price = int(product_data['originalPrice']) self.product.suggested_price = int(product_data['suggestedPrice']) self.product.image = image_destination classification = self.session.query(Classification).filter( Classification.name == self.classificationsCombo.currentText()).one() self.product.classification = classification if type(self.selected_colors_sizes) == list: self.pairs_count = 0 for color, sizes_dic in self.selected_colors_sizes: color = get_or_create(self.session, ProductColor, name=color, product=self.product) for size, counts in sizes_dic.items(): size_obj = get_or_create(self.session, ProductSize, size=int(size)) color_size_obj = get_or_create(self.session, ColorSize, color=color, size=size_obj, repeating_counter=counts) self.pairs_count += int(counts) self.product.pairs_number = self.pairs_count self.session.commit() QMessageBox.information(self, 'عملية ناجحة', 'تمت الاضافة بنجاح') self.close()
def book_adder(): """ Assumes that the data has been stripped clean of leading and trailing spaces. Possible responses: 200 Accepted - Book was added to the database successfully. 302 Error - API endpoint called while not logged in. 400 Error - The request did not validate. Client _must not_ retry. 409 IntegrityError - Database error for possible duplicate records. Client _must not_ retry. 500 - Standard server error. Client may retry after some wait period. """ form = AddBooksForm(request.form) app.logger.info(str(form)) app.logger.debug(form.debug_validate()) if has_equivalent_isbn(form.isbn.data): err_str = "A book with that ISBN (in another form) is already in the database." app.logger.error(err_str) return err_str, 409 if form.validate_on_submit(): try: from flask_login import current_user # Genre first genre = get_or_create(Genre, will_commit=False, name=form.genre.data, creator=current_user) # Publishing information publisher = get_or_create(BookCompany, will_commit=False, name=form.publisher.data, creator=current_user) has_printer = form.printer.data.strip() if has_printer: printer = get_or_create(BookCompany, will_commit=False, name=form.printer.data, creator=current_user) # Book book = Book(isbn=form.isbn.data, title=form.title.data, genre=genre, creator=current_user, publisher=publisher, publish_year=int(form.year.data)) db.session.add(book) db.session.flush() # Create printer entry if has_printer: printer_record = Printer(company=printer, book=book, creator=current_user) db.session.add(printer_record) __insert_contributions(book, form, db.session) db.session.commit() return "Accepted", 200 except IntegrityError, ierr: db.session.rollback() app.logger.error(traceback.format_exc()) err_str = '"%s" has been catalogued before. Ignoring.' % ( form.title.data) return err_str, 409 except ConstraintError, cerr: db.session.rollback() app.logger.error(traceback.format_exc()) return cerr.message, 400
def edit_book(): def contribution_exists(all_contribs, role_id, person): """ Check if the given person contributed for the given role in all the contributions related to the present book being edited. Where `all_contribs` is all the BookContribution records for the book being edited. person is an instance of librarian.utils.Person. Returns the BookContribution object if it exists, else False. """ the_contribution = [ contrib for contrib in all_contribs if (contrib.role_id == role_id and contrib.contributor.firstname == person.firstname and contrib.contributor.lastname == person.lastname) ] if len(the_contribution) > 1: raise InvalidRecordState( "uniqueness of contribution role + person + book %s" % spam) if the_contribution: return the_contribution[0] else: return False def edit_contrib(book, all_contribs, role, submitted_persons): """ Adds all new contributors to the session and deletes all removed contributors to the session. Where `submitted_persons` is the data straight out of the form (hence it is a JSON string), `all_contribs` are all the active contributions in the book as recorded in the DB (pre-edit). """ app.logger.debug("considering role %s" % role) parsons = json.loads(submitted_persons) form_records = set() # Create or load all the contributions mentioned in the form. for p in parsons: ce = contribution_exists(all_contribs, role.id, Person(**p)) if ce is not False: form_records.add(ce) else: contributor_record = get_or_create(Contributor, will_commit=False, firstname=p["firstname"], lastname=p["lastname"], creator=current_user) app.logger.debug("got contributor record %s" % contributor_record) app.logger.debug("will attach role %s" % role) if not contributor_record.active: contributor_record.active = True contribution = BookContribution(book=book, contributor=contributor_record, role=role, creator=current_user) db.session.add(contribution) form_records.add(contribution) recorded_contribs = set([ contrib for contrib in all_contribs if contrib.role.id == role.id ]) app.logger.debug("recorded contribs for %s %s" % (role, recorded_contribs)) app.logger.debug("form records %s" % form_records) deletables = recorded_contribs - form_records app.logger.debug("The deletables %s" % deletables) for d in deletables: d.active = False other_contrib = (BookContribution.query.filter( BookContribution.contributor_id == d.contributor_id).filter( or_(BookContribution.book_id != book.id, BookContribution.role_id != d.role_id)).filter( BookContribution.active).first()) app.logger.debug( "Contributor %s has another contribution %s (checked from %s)" % (d.contributor_id, other_contrib, role.name)) if other_contrib is None: app.logger.debug("Deactivating %s" % d) d.contributor.active = False from flask_login import current_user form = EditBookForm(request.form) app.logger.info(str(form)) app.logger.debug(form.debug_validate()) if form.validate_on_submit(): book_id = int(form.book_id.data) try: # Update records in books table genre = get_or_create(Genre, will_commit=False, session=db.session, name=form.genre.data, creator=current_user) publisher = get_or_create(BookCompany, will_commit=False, name=form.publisher.data, creator=current_user) book = db.session.query(Book).filter(Book.id == book_id).first() book.isbn = form.isbn.data book.title = form.title.data book.publish_year = form.year.data book.genre_id = genre.id book.publisher_id = publisher.id # Get all the contributions for this book all_contribs = (BookContribution.query.filter( BookContribution.book_id == book_id).filter(BookContribution.active).filter( BookContribution.contributor_id == Contributor.id).filter( Contributor.active).all()) edit_contrib(book, all_contribs, Role.get_preset_role("Author"), form.authors.data) r_illustrator = Role.get_preset_role("Illustrator") edit_contrib(book, all_contribs, r_illustrator, form.illustrators.data) edit_contrib(book, all_contribs, Role.get_preset_role("Editor"), form.editors.data) edit_contrib(book, all_contribs, Role.get_preset_role("Translator"), form.translators.data) db.session.commit() return "Accepted", 200 except IntegrityError, ierr: db.session.rollback() app.logger.error("Integrity Error occurred") app.logger.exception(traceback.format_exc()) return "IntegrityError", 409 except Exception as ex: db.session.rollback() app.logger.error("error except. traceback follows:") import traceback traceback.print_exc() return "Unknown error", 500
def process_item(self, item, spider): ''' Save events in the database this method is called for every item pipeline component ''' session = self.Session() # create variables with each of the newly scraped items # variables named for the columns in the 'Event' model # they will be used to create/return and edit individual events from the db title = item['title'] event_ts = item['event_ts'] artists = item['artists'] artists_clean = re.sub(r'\([^)]*\)', '', artists).split(',') ticket_price = item['ticket_price'] event_url = item['event_url'] status = item['status'] venue_id = int(item['venue_id']) purchase_url = item['purchase_url'] age_restriction = item['age_restriction'] promoter = item['promoter'] #spider.log('event artists: %s' % artists_clean) if len(purchase_url) > 200: purchase_url = "" # search event_url for set of numbers which are unique to that event # then set event_num (a unique id) with the match from event_url (m) m = re.search(r'\/(\d+)-', event_url) event_num = m.group(1) now = str(utcnow().isoformat()) # get_or_create returns an event instance from the db # if the event is not found, it then creates the event # event_num is unique for an event in the db # event_title and other characteristics are likely to change over time # need to ensure that we aren't storing multiple records for an event e, e_created = get_or_create(session, Event, event_num=event_num) if e_created: e.created_ts = now # check to see if new scraped status differs from that stored in the db # in the case that the scraped event has a status of 'Sold Out' # we then check to see if the status in the db is also 'Sold Out' # if the db shows that stored status != 'Sold Out' # then we can set the soldout_ts to the current utc datetime if ('Sold' in status) and (e.status != 'Sold Out'): e.soldout_ts = now e.status = status else: e.status = status # set all event variables e.title = title e.event_ts = event_ts e.ticket_price = ticket_price e.event_url = event_url e.purchase_url = purchase_url e.age_restriction = age_restriction e.promoter = promoter e.updated_ts = now # retrieve the venue from the db v, v_created = get_or_create(session, Venue, id=venue_id) e.venue = v for artist in artists_clean: artist_name = artist.strip() a, a_created = get_or_create(session, Artist, name=artist_name) if a_created: a.created_ts = now e.artists.append(a) session.commit() session.close() return item
async def on_message(self, message): # runs only on debug channels if debug is enabled. if config.DEBUG: if message.channel.id not in config.debug_channel_ids: return # other bots are unworthy of our attention if message.author.bot == True: return # if someone trying to run a command is not authorised, return if (commandHelpers.is_command(message.content) and not hasApprovedRole(message.author)): return # Processes messages from commands and handles errors. async def sendReply(text, edit=False, append=False, **kwargs): # Attempts to send a message up to MAX_RETRY times because sometimes discord is rubbish MAX_RETRY = 3 async def attempt(count=0): if (count < MAX_RETRY): try: if (edit): return await message.edit(text) elif (append): return await message.edit(message.content + text) return await message.channel.send(text) except discord.HTTPException as e: logger.warning("Failed to send or edit message. " + str(e)) return await attempt(count + 1) except discord.Forbidden as e: logger.error( "Cannot send message - permission forbidden! " + str(e)) return None if text != "": return await attempt() return None if (self.eve): try: session = Session() session.expire_on_commit = False if (message.guild): current_server = Server(id=message.guild.id, service_id=self.service.id, server_name=message.guild.name) else: current_server = None current_user = User(id=message.author.id, service_id=self.service.id, username=message.author.display_name) if (message.channel.name): current_channel = Chat(id=message.channel.id, server_id=current_server.id, chat_name=message.channel.name, nsfw=message.channel.is_nsfw()) else: current_channel = get_or_create( session, Chat, id=message.channel.id, server_id=current_server.id, chat_name=message.channel.id) except Exception as e: logger.error("Couldn't get data from message! " + str(e)) return # Metadata for use by commands and reactions metadata = { "session": session, "service": self.service, "user": current_user, "server": current_server, "chat": current_channel } # Actually process the message try: await self.eve.read(message.content, metadata, sendReply) except Exception as e: logger.error("Error reading message: " + str(e))
def edit_contrib(book, all_contribs, role, submitted_persons): """ Adds all new contributors to the session and deletes all removed contributors to the session. Where `submitted_persons` is the data straight out of the form (hence it is a JSON string), `all_contribs` are all the active contributions in the book as recorded in the DB (pre-edit). """ app.logger.debug("considering role %s" % role) parsons = json.loads(submitted_persons) form_records = set() # Create or load all the contributions mentioned in the form. for p in parsons: ce = contribution_exists(all_contribs, role.id, Person(**p)) if ce is not False: form_records.add(ce) else: contributor_record = get_or_create(Contributor, will_commit=False, firstname=p["firstname"], lastname=p["lastname"], creator=current_user) app.logger.debug("got contributor record %s" % contributor_record) app.logger.debug("will attach role %s" % role) if not contributor_record.active: contributor_record.active = True contribution = BookContribution(book=book, contributor=contributor_record, role=role, creator=current_user) db.session.add(contribution) form_records.add(contribution) recorded_contribs = set([ contrib for contrib in all_contribs if contrib.role.id == role.id ]) app.logger.debug("recorded contribs for %s %s" % (role, recorded_contribs)) app.logger.debug("form records %s" % form_records) deletables = recorded_contribs - form_records app.logger.debug("The deletables %s" % deletables) for d in deletables: d.active = False other_contrib = (BookContribution.query.filter( BookContribution.contributor_id == d.contributor_id).filter( or_(BookContribution.book_id != book.id, BookContribution.role_id != d.role_id)).filter( BookContribution.active).first()) app.logger.debug( "Contributor %s has another contribution %s (checked from %s)" % (d.contributor_id, other_contrib, role.name)) if other_contrib is None: app.logger.debug("Deactivating %s" % d) d.contributor.active = False
def main(): parser = argparse.ArgumentParser() parser.add_argument("--geocode", "-g", action="store_true", help="geocode arrestee address", default=False) parser.add_argument("--sort", "-s", help="geocode arrestee address", default='date') parser.add_argument("--configuration", "-c", help="use valued from configuration file FILE", default="arrests.cfg") parser.add_argument("--home", help="specify origin address for arrestee residence distance") parser.add_argument("--latitude", type=float, help="specify origin lat/long for arrestee residence distance") parser.add_argument("--longitude", type=float, help="specify origin lat/long for arrestee residence distance") parser.add_argument("--limit", type=int, help="limit processing to first n arrests, useful for limitig geocode lookups", default=25000) parser.add_argument("--api_key", help="use googlemaps api key KEY") args = parser.parse_args() logger.info("loading config") config = ConfigParser.ConfigParser() config.readfp(open('arrests.cfg')) # config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')]) logger.info("fetching arrest data") url = "http://www.fairfaxcounty.gov/police/crime/arrest.txt" r = geturl_cached(url) headers = r.readline() api_key = config.get('googlemaps', 'api_key') if args.api_key: api_key = args.api_key widths = [40, 20, 40, 5, 30, 25, 50, 100] offsets = [0] for i in widths: offsets.append(offsets[-1] + i) arrests = [] count = 0 logger.debug("limiting to %d records" % args.limit) session = models.get_session() new_arrests = 0 while count < args.limit: line = r.readline() if len(line) == 0: break count += 1 f = [] arrest = {} offset = 0 for i in widths: f.append(line[offset:offset+i].strip()) offset += i (is_new, charge) = models.get_or_create(session, models.Charge, name=f[CHARGE], description=f[DESCRIP]) (is_new, address) = models.get_or_create(session, models.Address, address=f[ADDRESS]) # geo_api_error = 1 # if not models.have_geocoding(session, address): # raise Exception('whoops missed an address') # try: # (lat, lon) = geoutil.get_coord(address.address, False) # geo_api_error = 0 # except geoutil.InvalidAddress, x: # lat=0.0 # lon=0.0 # geo = models.add_geocoding(session, # address=address, # latitude=lat, # longitude=lon, # error=geo_api_error) (is_new, arrestee) = models.get_or_create(session, models.Arrestee, lname = f[LNAME], fname = f[FNAME], mname = f[MNAME], age = f[AGE], address_id = address.id) date = time.strftime("%s", time.strptime(f[DATE], '%m/%d/%Y')) (is_new, arrest) = models.get_or_create(session, models.Arrest, date=date, charge=charge, arrestee=arrestee) if is_new: new_arrests += 1 session.add(arrest) session.commit() print "Found %d new arrest records" % new_arrests
new_poll_item = PollItem(value=estimate['value']) new_poll_item.choice = xstr(estimate['first_name'])+ " " + xstr(estimate['last_name']) if new_poll_item.choice == ' ': new_poll_item.choice = estimate['choice'] new_poll_item.region = Region.query.filter_by(abv=poll['poll_info']['poll_region']).first() new_poll_item.poll_class = estimate['poll_class'] new_poll_item.poll_date = poll['poll_info']['poll_date']['poll_end_date'] new_poll_item.poll_date_str = poll['poll_info']['poll_date']['poll_end_date'].strftime('%Y-%m-%d') new_poll_item.other = estimate['other'] new_poll_item.office = poll['poll_info']['poll_office'] if 'first_name' in estimate: if estimate['first_name']: new_politician = get_or_create(db.session, Politician, first_name = estimate['first_name'], last_name = estimate['last_name'], party = estimate['party'])[0] new_politician.update() if poll['poll_info']['poll_office']: new_politician.seeking_office = poll['poll_info']['poll_office'] if poll['poll_info']['poll_office']: new_politician.seeking_office = poll['poll_info']['poll_office'] new_politician.set_dewhash() db.session.add(new_politician) new_poll_item.politician = new_politician if 'party' in estimate: new_poll_item.party = estimate['party']