def analysis_transcript(video):
session = create_session()()
# 转换字幕成text
from transcript import get_clean_transcript, simple_token
clean_transcript = get_clean_transcript(video.xml_transcript)
video.clean_transcript = clean_transcript
# 分词
tokens = simple_token(clean_transcript)
import logging
logging.basicConfig(filename='example.log',level=logging.DEBUG)
# 将词汇与视频建立关系，加入数据库
from models import Word, get_or_create
for word in tokens:
# 记录非全数字，全字母，长度超过15的词
if len(word) > 15 or not word.isalpha() or not word.isalnum():
logging.debug(word)
break
if word.istitle() is False:
word_instance, flag = get_or_create(session, Word, text=word)
else:
word_instance, flag = get_or_create(session, Word, text=word.lower())
session.commit()
# 添加关系
video.words.append(word_instance)
session.commit()
# 统计词频，总字数，加入数据库，计算语速
word_number = len(tokens)
video.word_number = word_number
video.speed = word_number / duration_to_min(video.duration)
from transcript import statistic_frequency
video.word_frequency = statistic_frequency(tokens)
session.commit()
session.remove()
def save_to_db(db_file, store):
engine = create_engine('sqlite:///{}'.format(db_file))
Session = sessionmaker(bind=engine)
# Initialize DB if needed
if not os.path.isfile(db_file):
models.Base.metadata.create_all(engine)
# Save to database
session = Session()
# Select/save store
db_store = get_or_create(session, models.Store, name=store.name)
store_id = db_store.id
# Process the gathered data
for overview_page in tqdm(store.overview_pages):
# Save category hierarchy
category_parent_id = None
for category in overview_page['product_category']:
# Check if exists...
res = session.query(models.Category).filter(
models.Category.name == category).filter(
models.Category.parent_id == category_parent_id).first()
# ... else create new category
if res:
category_parent_id = res.id
else:
new_category = models.Category(name=category,
parent_id=category_parent_id)
session.add(new_category)
session.commit()
category_parent_id = new_category.id
# Save products
for position, product_url in enumerate(overview_page['product_urls']):
if product_url in store.product_pages.keys():
product = store.product_pages[product_url]
# Select/save brand
db_brand = get_or_create(session,
models.Brand,
name=product['brand'])
brand_id = db_brand.id
# Save product
new_product = models.Product(
name=product['product_name'],
type=product['product_type'],
price=product['price'],
page_position=position + 1, # From 0-based to 1-based
page_number=overview_page['page_number'],
category_id=category_parent_id,
brand_id=brand_id,
store_id=store_id)
session.add(new_product)
session.commit()
session.close()
def insert_item(self, token_object, new_session,
session_time=timedelta(0)):
"""
Create new session or update existing session object
:param token_object: Token that has to be converted into session
:param new_session: Boolean that denotes if this is a new session.
A new session will be created if this value
is True
:param session_time: If an existing session is to be updated,
session_time contains the new value of total
session time
"""
if self.file_type == settings.APACHE_COMMON:
url_obj = get_or_create(self.session,
Uurl,
url=token_object.resource_requested)
elif self.file_type == settings.APACHE_COMBINED:
url_obj = get_or_create(self.session,
Uurl,
url=token_object.resource_requested)
elif self.file_type == settings.SQUID:
url_obj = get_or_create(self.session, Uurl, url=token_object.url)
# If this is a new session
if new_session:
# Create session object
session_obj = Session(ip=token_object.ip_address,
session_time=session_time)
# Set start and end time
session_obj.start_time = token_object.date_time
session_obj.end_time = token_object.date_time
# If new_session is False, new session may or may not be created
# (depending upon the session_time)
else:
# Try to get session object
session_obj = get_or_create(self.session,
Session,
ip=token_object.ip_address)
# If the object is a new session
if session_obj.session_time is timedelta(0):
session_obj.start_time = token_object.date_time
session_obj.session_time = session_time
session_obj.end_time = token_object.date_time
# Add url to session
session_obj.session_urls.append(url_obj)
self.session.add(session_obj)
def on_track_change(pl, session):
global song_to_write
if song_to_write is not None \
and datetime.now() - song_to_write['start_time'] > timedelta(seconds=4):
artist = models.get_or_create(
session,
models.Artist,
search_param={'name': song_to_write['xesam:artist'][0]},
name=song_to_write['xesam:artist'][0])
album = models.get_or_create(
session,
models.Album,
search_param={'name': song_to_write['xesam:album']},
name=song_to_write['xesam:album'],
img=song_to_write['mpris:artUrl'],
artist=artist)
song = models.get_or_create(
session,
models.Song,
search_param={'name': song_to_write['xesam:title']},
name=song_to_write['xesam:title'],
length=int(song_to_write['mpris:length'] / 1000000),
album=album,
spotify_id=song_to_write['mpris:trackid'].split(':')[2])
play = models.Play(
duration=((datetime.now() - song_to_write['start_time']) -
song_to_write['paused_time']).seconds,
start_time=song_to_write['start_time'],
end_time=datetime.now(),
song=song)
logger.info(
"Writing playback - {artist} - {title} | Duration of playback - {playing_time} sec"
.format(artist=artist.name,
title=song.name,
playing_time=play.duration))
logger.info("Current song - {song} - {title}".format(song=pl.get_artist(),
title=pl.get_title()))
song_to_write = dict(start_time=datetime.now(),
paused_time=timedelta(seconds=0))
song_to_write.update(dict(pl.props.metadata))
def get_attempt(package):
"""
Always returns a brand new models.Attempt instance, bound to
the expected models.ArticlePkg instance.
"""
with PackageAnalyzer(package) as pkg:
if pkg.is_valid_package():
article = models.get_or_create(models.ArticlePkg, **pkg.meta)
pkg_checksum = utils.make_digest_file(package)
attempt_meta = {'package_md5': pkg_checksum,
'articlepkg_id': article.id}
attempt = models.get_or_create(models.Attempt, **attempt_meta)
return attempt
else:
sys.stderr.write("Invalid package: %s\n" % pkg.errors)
utils.mark_as_failed(package)
def post(self):
email_id = self.request.form.get("email")[0]
credential = EmailCredential()
credential.email = email_id
if self.validate(credential, email_validator):
room, created = get_or_create(EmailBase, email=email_id)
if created:
return self.json_response(
{'success': "You are successfully added. We will invite on the beta launch"})
else:
return self.json_response(
{'success': "We will invite on the beta launch", 'error': "You are already registered."})
else:
return self.json_response(
{'error': "Wrong Email Address. Please Correct yourself."})
async def on_ready(self):
logger.info('Logged in as')
logger.info(client.user.name)
logger.info(client.user.id)
logger.info('------')
session = Session()
session.expire_on_commit = False
self.service = get_or_create(session, Service, name="discord")
database_user = get_or_create(session,
User,
id=client.user.id,
service_id=self.service.id)
database_user.username = client.user.display_name
try:
session.commit()
except:
logger.error("Couldn't commit bot user to database!")
session.rollback()
finally:
session.close()
self.eve = evebot.EveBot(database_user)
async def constructMetadata(self, message):
try:
session = Session()
session.expire_on_commit = False
if (message.guild):
current_server = Server(id=message.guild.id,
service_id=self.service.id,
server_name=message.guild.name)
else:
current_server = None
current_user = User(id=message.author.id,
service_id=self.service.id,
username=message.author.display_name)
if (message.channel.name):
current_channel = Chat(id=message.channel.id,
server_id=current_server.id,
chat_name=message.channel.name,
nsfw=message.channel.is_nsfw())
else:
current_channel = get_or_create(session,
Chat,
id=message.channel.id,
server_id=current_server.id,
chat_name=message.channel.id)
except Exception as e:
logger.error("Couldn't get data from message! " + str(e))
return None
# Metadata for use by commands and reactions
metadata = {
"service": self.service,
"user": current_user,
"server": current_server,
"chat": current_channel,
"message": message,
"client": self
}
return metadata
def post(self):
email_id = self.request.form.get("email")[0]
credential = EmailCredential()
credential.email = email_id
if self.validate(credential, email_validator):
room, created = get_or_create(EmailBase, email=email_id)
if created:
return self.json_response({
'success':
"You are successfully added. We will invite on the beta launch"
})
else:
return self.json_response({
'success':
"We will invite on the beta launch",
'error':
"You are already registered."
})
else:
return self.json_response(
{'error': "Wrong Email Address. Please Correct yourself."})
def __create_bookperson(form_data):
"""
Create a Contributor record from the given form_data. Return the created
records as a list, if any. Else return None.
form_data is expected to be a JSON list of objects. Each object should have
the fields `last_name` and `first_name`.
Raises ConstraintError if one of the persons in form_data has blank firstname
or lastname.
"""
from flask_login import current_user
try:
parse = json.loads(form_data)
persons_created = []
for parson in parse:
_firstname = parson.get("firstname", "").strip()
_lastname = parson.get("lastname", "").strip()
if _firstname and _lastname:
persons_created.insert(
0,
get_or_create(Contributor,
will_commit=True,
firstname=_firstname,
lastname=_lastname,
creator=current_user))
else:
raise ConstraintError("firstname and lastname are present",
str(parson))
return persons_created
except ValueError:
# For errors in pasing JSON
return []
from sqlalchemy import create_engine, Column, String, Integer, MetaData, Table
from sqlalchemy.orm import mapper, create_session, sessionmaker
from models import db_connect, create_tables, Venue, get_or_create
import csv
CSV_FILE = 'build_db/venues.csv'
engine = db_connect()
create_tables(engine)
Session = sessionmaker(bind=engine)
session = Session()
with open(CSV_FILE) as f:
cf = csv.DictReader(f, delimiter=',')
for row in cf:
v, v_created = get_or_create(session, Venue, **row)
session.add(v)
session.commit()
session.close()
def get_scrape_urls(ticketing):
scrape_url = [url[0] for url in session.query(Venue.scrape_url).filter(Venue.ticketing == ticketing).all()]
return scrape_url
def get_venue_id(response):
venue_id = session.query(Venue.id).filter(Venue.scrape_url == response).first()[0]
return venue_id
#!bin/python
import models
import time
import dbm
session = models.get_session()
addrdb = dbm.open('addr.db', 'c')
baddrdb = dbm.open('baddr.db', 'c')
for addrtxt in addrdb.keys():
(lat, lon) = addrdb[addrtxt].split(":")
addr = models.get_or_create(session,
models.Address,
address=addrtxt)
geo = models.get_or_create(session,
models.Geocoding,
address_id=addr.id,
latitude=lat,
longitude=lon,
error=0)
for addrtxt in baddrdb.keys():
addr = models.get_or_create(session,
models.Address,
address=addrtxt)
geo = models.get_or_create(session,
models.Geocoding,
address_id=addr.id,
latitude=0.0,
longitude=0.0,
error=1)
def save(statuses, fetch_model, fetch_datetime):
new_users_count = 0
new_data_count = 0
counter = 0
# dictionary with the fetched users external id as key to prevent addition of the same user more than ones
users_added = {}
# iterate over all the statuses and insert new data to DB
for s in statuses:
counter+=1
print "\t{0:d}] id: {1:d}".format(counter, s.id)
user = None
if users_added.has_key(s.user.id):
user = users_added[s.user.id]
else:
user, user_is_new = get_or_create(mysession, User,
external_id = s.user.id,
source = fetch_model.source
)
if user_is_new:
new_users_count+=1
user.name = s.user.name
user.screen_name = s.user.screen_name
user.description = htmlutils.unescape(s.user.description)
user.profile_image_url = s.user.profile_image_url
user.verified = s.user.verified
user.language = s.user.lang.split("-")[0]
user.fetched_data = s.user.AsDict()
user.fetched_at = fetch_datetime
user.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(s.user.created_at,'%a %b %d %H:%M:%S +0000 %Y'))
users_added[s.user.id] = user # remember added user in case there he's got more data than this datum
datum, datum_is_new = get_or_create(mysession, Datum,
external_id = s.id,
source = fetch_model.source
)
if datum_is_new:
new_data_count+=1
sDict = s.AsDict()
language = fetch_model.language or s.GetLang()
datum.language = language
datum.domain = fetch_model.domain
prediction = predictor.predict(language, fetch_model.domain, fetch_model.source, [s.text])[0]
datum.class_value = prediction.value
datum.gold = 0
datum.text = htmlutils.unescape(s.text)
datum.user = user
if sDict.has_key('hashtags') and len(sDict['hashtags'])>0: datum.hashtags = sDict['hashtags']
if sDict.has_key('urls') and len(sDict['urls'])>0: datum.urls = sDict['urls']
if sDict.has_key('media'): datum.media = sDict['media']
datum.fetched_data = sDict
datum.fetched_at = fetch_datetime
datum.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(s.created_at,'%a %b %d %H:%M:%S +0000 %Y'))
print("\tnew datum -> insert")
print("\tpredicted -> class name: {0:s}, class value: {1:s}".format(prediction.name,str(prediction.value)))
else:
print("\talready exist, better luck next time :)")
if new_users_count>0 or new_data_count>0:
print("\tinsert all {0:d} new users and {1:d} new data items".format(new_users_count, new_data_count))
mysession.commit()
print ("\tdone")
else:
print("\tno new data to save")
dispose()
print("connection disposed")
def add_product(self):
# to skip validation if there is no change in sizes and colors
if self.product and not self.selected_colors_sizes:
self.selected_colors_sizes = 'no change'
product_data = {
'skucode': self.productSkucodeInput.text(),
'originalPrice': self.originalPriceInput.text(),
'suggestedPrice': self.suggestedPrice.text(),
'image': self.imagePath,
'sizes': self.selected_colors_sizes
}
fields_translation = {
'skucode': 'الكود',
'originalPrice': 'سعر الجملة',
'suggestedPrice': 'أقل سعر',
'image': 'صورة المنتج',
'sizes': 'المقاسات'
}
for field_name, value in product_data.items():
if not value:
field_translated = fields_translation[field_name]
QMessageBox.warning(
self, field_translated,
f'يرجى ملىء حقل ({field_translated}) اولا!')
return
product_with_same_code = self.session.query(Product).filter_by(
sku_code=product_data['skucode']).first()
if product_with_same_code:
if not self.product == product_with_same_code:
QMessageBox.warning(
self, 'خطأ بالكود',
f'هذا الكود تم استخدامه من قبل ، يرجى اختيار كود جديد')
return
img_name = product_data['image'].split('/')[-1]
image_destination = str(Path.joinpath(current_path, 'images',
img_name))
self.same_destination = False
if self.product:
if self.product.image != image_destination:
self.same_destination = True
# we check if the colors-sizes list changed
if not self.sizesListPreview.text().startswith('المتاح'):
# if changed we start changing colors depending on new values
# but with respect to sold units of previous colors and its sizes
self.sold_colors = set(unit.color
for unit in self.product.sold_units)
self.sold_colors.update(c_s[0]
for c_s in self.selected_colors_sizes)
for color_obj in self.product.colors:
# if there's sold units or will be created in the new editing
# with this color, we just delete all availabel sizes
# for clean addition later in the next lines if we want
if color_obj.name in self.sold_colors:
self.session.query(ColorSize).filter_by(
color=color_obj, ).delete()
# else we delete the color
else:
self.session.delete(color_obj)
if not self.same_destination:
img = Image.open(product_data['image'])
img.save(image_destination)
if not self.product:
self.product = get_or_create(
self.session,
Product,
sku_code=product_data['skucode'],
original_price=product_data['originalPrice'],
suggested_price=product_data['suggestedPrice'],
image=image_destination,
)
else:
self.product.sku_code = product_data['skucode']
self.product.original_price = int(product_data['originalPrice'])
self.product.suggested_price = int(product_data['suggestedPrice'])
self.product.image = image_destination
classification = self.session.query(Classification).filter(
Classification.name ==
self.classificationsCombo.currentText()).one()
self.product.classification = classification
if type(self.selected_colors_sizes) == list:
self.pairs_count = 0
for color, sizes_dic in self.selected_colors_sizes:
color = get_or_create(self.session,
ProductColor,
name=color,
product=self.product)
for size, counts in sizes_dic.items():
size_obj = get_or_create(self.session,
ProductSize,
size=int(size))
color_size_obj = get_or_create(self.session,
ColorSize,
color=color,
size=size_obj,
repeating_counter=counts)
self.pairs_count += int(counts)
self.product.pairs_number = self.pairs_count
self.session.commit()
QMessageBox.information(self, 'عملية ناجحة', 'تمت الاضافة بنجاح')
self.close()
def book_adder():
"""
Assumes that the data has been stripped clean of leading and trailing spaces.
Possible responses:
200 Accepted - Book was added to the database successfully.
302 Error - API endpoint called while not logged in.
400 Error - The request did not validate. Client _must not_ retry.
409 IntegrityError - Database error for possible duplicate records.
Client _must not_ retry.
500 - Standard server error. Client may retry after some wait period.
"""
form = AddBooksForm(request.form)
app.logger.info(str(form))
app.logger.debug(form.debug_validate())
if has_equivalent_isbn(form.isbn.data):
err_str = "A book with that ISBN (in another form) is already in the database."
app.logger.error(err_str)
return err_str, 409
if form.validate_on_submit():
try:
from flask_login import current_user
# Genre first
genre = get_or_create(Genre,
will_commit=False,
name=form.genre.data,
creator=current_user)
# Publishing information
publisher = get_or_create(BookCompany,
will_commit=False,
name=form.publisher.data,
creator=current_user)
has_printer = form.printer.data.strip()
if has_printer:
printer = get_or_create(BookCompany,
will_commit=False,
name=form.printer.data,
creator=current_user)
# Book
book = Book(isbn=form.isbn.data,
title=form.title.data,
genre=genre,
creator=current_user,
publisher=publisher,
publish_year=int(form.year.data))
db.session.add(book)
db.session.flush()
# Create printer entry
if has_printer:
printer_record = Printer(company=printer,
book=book,
creator=current_user)
db.session.add(printer_record)
__insert_contributions(book, form, db.session)
db.session.commit()
return "Accepted", 200
except IntegrityError, ierr:
db.session.rollback()
app.logger.error(traceback.format_exc())
err_str = '"%s" has been catalogued before. Ignoring.' % (
form.title.data)
return err_str, 409
except ConstraintError, cerr:
db.session.rollback()
app.logger.error(traceback.format_exc())
return cerr.message, 400
def edit_book():
def contribution_exists(all_contribs, role_id, person):
"""
Check if the given person contributed for the given role in all the
contributions related to the present book being edited.
Where
`all_contribs` is all the BookContribution records for the book being
edited.
person is an instance of librarian.utils.Person.
Returns the BookContribution object if it exists, else False.
"""
the_contribution = [
contrib for contrib in all_contribs
if (contrib.role_id == role_id
and contrib.contributor.firstname == person.firstname
and contrib.contributor.lastname == person.lastname)
]
if len(the_contribution) > 1:
raise InvalidRecordState(
"uniqueness of contribution role + person + book %s" % spam)
if the_contribution:
return the_contribution[0]
else:
return False
def edit_contrib(book, all_contribs, role, submitted_persons):
"""
Adds all new contributors to the session and deletes all removed
contributors to the session.
Where `submitted_persons` is the data straight out of the form (hence it
is a JSON string), `all_contribs` are all the active contributions in
the book as recorded in the DB (pre-edit).
"""
app.logger.debug("considering role %s" % role)
parsons = json.loads(submitted_persons)
form_records = set()
# Create or load all the contributions mentioned in the form.
for p in parsons:
ce = contribution_exists(all_contribs, role.id, Person(**p))
if ce is not False:
form_records.add(ce)
else:
contributor_record = get_or_create(Contributor,
will_commit=False,
firstname=p["firstname"],
lastname=p["lastname"],
creator=current_user)
app.logger.debug("got contributor record %s" %
contributor_record)
app.logger.debug("will attach role %s" % role)
if not contributor_record.active:
contributor_record.active = True
contribution = BookContribution(book=book,
contributor=contributor_record,
role=role,
creator=current_user)
db.session.add(contribution)
form_records.add(contribution)
recorded_contribs = set([
contrib for contrib in all_contribs if contrib.role.id == role.id
])
app.logger.debug("recorded contribs for %s %s" %
(role, recorded_contribs))
app.logger.debug("form records %s" % form_records)
deletables = recorded_contribs - form_records
app.logger.debug("The deletables %s" % deletables)
for d in deletables:
d.active = False
other_contrib = (BookContribution.query.filter(
BookContribution.contributor_id == d.contributor_id).filter(
or_(BookContribution.book_id != book.id,
BookContribution.role_id != d.role_id)).filter(
BookContribution.active).first())
app.logger.debug(
"Contributor %s has another contribution %s (checked from %s)"
% (d.contributor_id, other_contrib, role.name))
if other_contrib is None:
app.logger.debug("Deactivating %s" % d)
d.contributor.active = False
from flask_login import current_user
form = EditBookForm(request.form)
app.logger.info(str(form))
app.logger.debug(form.debug_validate())
if form.validate_on_submit():
book_id = int(form.book_id.data)
try:
# Update records in books table
genre = get_or_create(Genre,
will_commit=False,
session=db.session,
name=form.genre.data,
creator=current_user)
publisher = get_or_create(BookCompany,
will_commit=False,
name=form.publisher.data,
creator=current_user)
book = db.session.query(Book).filter(Book.id == book_id).first()
book.isbn = form.isbn.data
book.title = form.title.data
book.publish_year = form.year.data
book.genre_id = genre.id
book.publisher_id = publisher.id
# Get all the contributions for this book
all_contribs = (BookContribution.query.filter(
BookContribution.book_id ==
book_id).filter(BookContribution.active).filter(
BookContribution.contributor_id == Contributor.id).filter(
Contributor.active).all())
edit_contrib(book, all_contribs, Role.get_preset_role("Author"),
form.authors.data)
r_illustrator = Role.get_preset_role("Illustrator")
edit_contrib(book, all_contribs, r_illustrator,
form.illustrators.data)
edit_contrib(book, all_contribs, Role.get_preset_role("Editor"),
form.editors.data)
edit_contrib(book, all_contribs,
Role.get_preset_role("Translator"),
form.translators.data)
db.session.commit()
return "Accepted", 200
except IntegrityError, ierr:
db.session.rollback()
app.logger.error("Integrity Error occurred")
app.logger.exception(traceback.format_exc())
return "IntegrityError", 409
except Exception as ex:
db.session.rollback()
app.logger.error("error except. traceback follows:")
import traceback
traceback.print_exc()
return "Unknown error", 500
def process_item(self, item, spider):
'''
Save events in the database
this method is called for every item pipeline component
'''
session = self.Session()
# create variables with each of the newly scraped items
# variables named for the columns in the 'Event' model
# they will be used to create/return and edit individual events from the db
title = item['title']
event_ts = item['event_ts']
artists = item['artists']
artists_clean = re.sub(r'\([^)]*\)', '', artists).split(',')
ticket_price = item['ticket_price']
event_url = item['event_url']
status = item['status']
venue_id = int(item['venue_id'])
purchase_url = item['purchase_url']
age_restriction = item['age_restriction']
promoter = item['promoter']
#spider.log('event artists: %s' % artists_clean)
if len(purchase_url) > 200:
purchase_url = ""
# search event_url for set of numbers which are unique to that event
# then set event_num (a unique id) with the match from event_url (m)
m = re.search(r'\/(\d+)-', event_url)
event_num = m.group(1)
now = str(utcnow().isoformat())
# get_or_create returns an event instance from the db
# if the event is not found, it then creates the event
# event_num is unique for an event in the db
# event_title and other characteristics are likely to change over time
# need to ensure that we aren't storing multiple records for an event
e, e_created = get_or_create(session, Event, event_num=event_num)
if e_created:
e.created_ts = now
# check to see if new scraped status differs from that stored in the db
# in the case that the scraped event has a status of 'Sold Out'
# we then check to see if the status in the db is also 'Sold Out'
# if the db shows that stored status != 'Sold Out'
# then we can set the soldout_ts to the current utc datetime
if ('Sold' in status) and (e.status != 'Sold Out'):
e.soldout_ts = now
e.status = status
else:
e.status = status
# set all event variables
e.title = title
e.event_ts = event_ts
e.ticket_price = ticket_price
e.event_url = event_url
e.purchase_url = purchase_url
e.age_restriction = age_restriction
e.promoter = promoter
e.updated_ts = now
# retrieve the venue from the db
v, v_created = get_or_create(session, Venue, id=venue_id)
e.venue = v
for artist in artists_clean:
artist_name = artist.strip()
a, a_created = get_or_create(session, Artist, name=artist_name)
if a_created:
a.created_ts = now
e.artists.append(a)
session.commit()
session.close()
return item
async def on_message(self, message):
# runs only on debug channels if debug is enabled.
if config.DEBUG:
if message.channel.id not in config.debug_channel_ids:
return
# other bots are unworthy of our attention
if message.author.bot == True:
return
# if someone trying to run a command is not authorised, return
if (commandHelpers.is_command(message.content)
and not hasApprovedRole(message.author)):
return
# Processes messages from commands and handles errors.
async def sendReply(text, edit=False, append=False, **kwargs):
# Attempts to send a message up to MAX_RETRY times because sometimes discord is rubbish
MAX_RETRY = 3
async def attempt(count=0):
if (count < MAX_RETRY):
try:
if (edit):
return await message.edit(text)
elif (append):
return await message.edit(message.content + text)
return await message.channel.send(text)
except discord.HTTPException as e:
logger.warning("Failed to send or edit message. " +
str(e))
return await attempt(count + 1)
except discord.Forbidden as e:
logger.error(
"Cannot send message - permission forbidden! " +
str(e))
return None
if text != "":
return await attempt()
return None
if (self.eve):
try:
session = Session()
session.expire_on_commit = False
if (message.guild):
current_server = Server(id=message.guild.id,
service_id=self.service.id,
server_name=message.guild.name)
else:
current_server = None
current_user = User(id=message.author.id,
service_id=self.service.id,
username=message.author.display_name)
if (message.channel.name):
current_channel = Chat(id=message.channel.id,
server_id=current_server.id,
chat_name=message.channel.name,
nsfw=message.channel.is_nsfw())
else:
current_channel = get_or_create(
session,
Chat,
id=message.channel.id,
server_id=current_server.id,
chat_name=message.channel.id)
except Exception as e:
logger.error("Couldn't get data from message! " + str(e))
return
# Metadata for use by commands and reactions
metadata = {
"session": session,
"service": self.service,
"user": current_user,
"server": current_server,
"chat": current_channel
}
# Actually process the message
try:
await self.eve.read(message.content, metadata, sendReply)
except Exception as e:
logger.error("Error reading message: " + str(e))
def edit_contrib(book, all_contribs, role, submitted_persons):
"""
Adds all new contributors to the session and deletes all removed
contributors to the session.
Where `submitted_persons` is the data straight out of the form (hence it
is a JSON string), `all_contribs` are all the active contributions in
the book as recorded in the DB (pre-edit).
"""
app.logger.debug("considering role %s" % role)
parsons = json.loads(submitted_persons)
form_records = set()
# Create or load all the contributions mentioned in the form.
for p in parsons:
ce = contribution_exists(all_contribs, role.id, Person(**p))
if ce is not False:
form_records.add(ce)
else:
contributor_record = get_or_create(Contributor,
will_commit=False,
firstname=p["firstname"],
lastname=p["lastname"],
creator=current_user)
app.logger.debug("got contributor record %s" %
contributor_record)
app.logger.debug("will attach role %s" % role)
if not contributor_record.active:
contributor_record.active = True
contribution = BookContribution(book=book,
contributor=contributor_record,
role=role,
creator=current_user)
db.session.add(contribution)
form_records.add(contribution)
recorded_contribs = set([
contrib for contrib in all_contribs if contrib.role.id == role.id
])
app.logger.debug("recorded contribs for %s %s" %
(role, recorded_contribs))
app.logger.debug("form records %s" % form_records)
deletables = recorded_contribs - form_records
app.logger.debug("The deletables %s" % deletables)
for d in deletables:
d.active = False
other_contrib = (BookContribution.query.filter(
BookContribution.contributor_id == d.contributor_id).filter(
or_(BookContribution.book_id != book.id,
BookContribution.role_id != d.role_id)).filter(
BookContribution.active).first())
app.logger.debug(
"Contributor %s has another contribution %s (checked from %s)"
% (d.contributor_id, other_contrib, role.name))
if other_contrib is None:
app.logger.debug("Deactivating %s" % d)
d.contributor.active = False
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--geocode", "-g", action="store_true", help="geocode arrestee address", default=False)
parser.add_argument("--sort", "-s", help="geocode arrestee address", default='date')
parser.add_argument("--configuration", "-c", help="use valued from configuration file FILE", default="arrests.cfg")
parser.add_argument("--home", help="specify origin address for arrestee residence distance")
parser.add_argument("--latitude", type=float, help="specify origin lat/long for arrestee residence distance")
parser.add_argument("--longitude", type=float, help="specify origin lat/long for arrestee residence distance")
parser.add_argument("--limit", type=int, help="limit processing to first n arrests, useful for limitig geocode lookups", default=25000)
parser.add_argument("--api_key", help="use googlemaps api key KEY")
args = parser.parse_args()
logger.info("loading config")
config = ConfigParser.ConfigParser()
config.readfp(open('arrests.cfg'))
# config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')])
logger.info("fetching arrest data")
url = "http://www.fairfaxcounty.gov/police/crime/arrest.txt"
r = geturl_cached(url)
headers = r.readline()
api_key = config.get('googlemaps', 'api_key')
if args.api_key:
api_key = args.api_key
widths = [40, 20, 40, 5, 30, 25, 50, 100]
offsets = [0]
for i in widths:
offsets.append(offsets[-1] + i)
arrests = []
count = 0
logger.debug("limiting to %d records" % args.limit)
session = models.get_session()
new_arrests = 0
while count < args.limit:
line = r.readline()
if len(line) == 0:
break
count += 1
f = []
arrest = {}
offset = 0
for i in widths:
f.append(line[offset:offset+i].strip())
offset += i
(is_new, charge) = models.get_or_create(session,
models.Charge,
name=f[CHARGE],
description=f[DESCRIP])
(is_new, address) = models.get_or_create(session,
models.Address,
address=f[ADDRESS])
# geo_api_error = 1
# if not models.have_geocoding(session, address):
# raise Exception('whoops missed an address')
# try:
# (lat, lon) = geoutil.get_coord(address.address, False)
# geo_api_error = 0
# except geoutil.InvalidAddress, x:
# lat=0.0
# lon=0.0
# geo = models.add_geocoding(session,
# address=address,
# latitude=lat,
# longitude=lon,
# error=geo_api_error)
(is_new, arrestee) = models.get_or_create(session,
models.Arrestee,
lname = f[LNAME],
fname = f[FNAME],
mname = f[MNAME],
age = f[AGE],
address_id = address.id)
date = time.strftime("%s", time.strptime(f[DATE], '%m/%d/%Y'))
(is_new, arrest) = models.get_or_create(session,
models.Arrest,
date=date,
charge=charge,
arrestee=arrestee)
if is_new:
new_arrests += 1
session.add(arrest)
session.commit()
print "Found %d new arrest records" % new_arrests
new_poll_item = PollItem(value=estimate['value'])
new_poll_item.choice = xstr(estimate['first_name'])+ " " + xstr(estimate['last_name'])
if new_poll_item.choice == ' ':
new_poll_item.choice = estimate['choice']
new_poll_item.region = Region.query.filter_by(abv=poll['poll_info']['poll_region']).first()
new_poll_item.poll_class = estimate['poll_class']
new_poll_item.poll_date = poll['poll_info']['poll_date']['poll_end_date']
new_poll_item.poll_date_str = poll['poll_info']['poll_date']['poll_end_date'].strftime('%Y-%m-%d')
new_poll_item.other = estimate['other']
new_poll_item.office = poll['poll_info']['poll_office']
if 'first_name' in estimate:
if estimate['first_name']:
new_politician = get_or_create(db.session, Politician, first_name = estimate['first_name'], last_name = estimate['last_name'], party = estimate['party'])[0]
new_politician.update()
if poll['poll_info']['poll_office']:
new_politician.seeking_office = poll['poll_info']['poll_office']
if poll['poll_info']['poll_office']:
new_politician.seeking_office = poll['poll_info']['poll_office']
new_politician.set_dewhash()
db.session.add(new_politician)
new_poll_item.politician = new_politician
if 'party' in estimate:
new_poll_item.party = estimate['party']
