def _wrapper(**view_kwargs):
if config.DEBUG:
with db.transaction():
return view(**view_kwargs)
try:
with db.transaction():
return view(**view_kwargs)
except Exception as e:
response = json.jsonify(dict(
error=500,
message=e.message or "Unknown Error"
))
return response
def save(self, file_object, mime_type, **kwargs):
media_root = current_app.config.get("MEDIA_ROOT")
if not media_root:
raise Exception("MEDIA_ROOT not configured.")
letter = file_object.filename[0]
destdir = os.path.join(media_root, mime_type, letter)
if not os.path.isdir(destdir):
os.makedirs(destdir)
fname = "%s-%s" % (
str(time.time()).replace(".", "-"),
file_object.filename
)
fpath = os.path.join(mime_type, letter, fname)
full_path = os.path.join(destdir, fname)
file_object.save(full_path)
file_size = os.stat(full_path).st_size
params = (
file_object.filename,
kwargs.get("title", None),
file_size,
mime_type,
media_root,
fpath,
)
q = " ".join([
"insert into storage.media",
"(original_name, title, file_size, mime_type, media_root, path)",
"values(%s) returning id, path" % ",".join(["%s"] * len(params))
])
with db.transaction("main") as tx:
rs = tx.execute(q, params)
return rs.fetchone()
def enqueue(subject, rcpt, template, **kwargs):
txt_tpl = os.path.join("mail", template) + ".txt"
html_tpl = os.path.join("mail", template) + ".html"
txt = render(txt_tpl, **kwargs)
html = render(html_tpl, **kwargs)
sender = kwargs.get("sender", current_app.config["MAIL_DEFAULT_SENDER"])
rcpt = rcpt if isinstance(rcpt, list) else [rcpt]
with db.transaction() as tx:
st = tables.messages.insert().returning(tables.messages.c.id)
st = st.values(title=subject, body_text=txt, body_html=html)
r = tx.execute(st)
mail_id = r.fetchone()[0]
st = tables.queue.insert()
r = tx.execute(st, [
dict(
message_id=mail_id,
sender=sender,
recipient_id=rcpt_id,
) for rcpt_id in rcpt
])
r = send_mail.apply_async(
(mail_id, ),
queue="mail.low-priority",
max_retries=3,
)
async def add_tweet(user, msg, tags):
"""
add user tweet with specific tags and after that add to redis to fetch in future
:param user: login user json data
:param msg: tweet message
:param tags: list of tags
:return:
"""
try:
async with db.transaction():
tweet = Tweet(msg=msg, user_id=user.get('user_id'))
await tweet.create()
for tag_item in tags:
tag = await get_or_create(Tag, *(Tag.tag_name == tag_item, ),
**({
"tag_name": tag_item
}))
tweet_tag = TagTweet(tag_id=tag.id, tweet_id=tweet.id)
await tweet_tag.create()
if tweet_tag:
await redis.append_value(tag.tag_name, tweet.as_json())
return tweet.as_json(), None
except Exception as ex:
global_logger.write_log('error', f"error: {ex}")
return None, ex
def update(self, session_id, **kwargs):
with db.transaction("main") as tx:
st = sessions.update(returning=sessions.c)
st = st.values(**kwargs)
st = st.where(sessions.c.id == session_id)
rs = tx.execute(st)
return rs.fetchone()
def create(self, data):
with db.transaction("main") as tx:
st = tables.users.insert().returning(
tables.users.c.id,
tables.users.c.ident,
tables.users.c.username,
tables.users.c.last_success,
tables.users.c.last_failure,
tables.users.c.failures,
tables.users.c.city,
tables.users.c.country,
tables.users.c.gender,
)
params = dict(
ident=data.get("ident"),
username=data.get("username"),
password=data.get("password"),
)
if data.get("social_uid"):
params["social_uid"] = data.get("social_uid")
params["social_provider"] = data.get("social_provider")
st = st.values(**params)
rs = tx.execute(st)
return rs.fetchone()
def ingest(filepath):
'''Ingest file into database'''
print "Ingesting %s" % filepath
rows_in_file = parse_fec_file(filepath)
myfile = File.get_or_create(name=filepath)
myfile_id = myfile.id
with db.transaction(): # TODO: More sane error handling
for idx in range(0, len(rows_in_file), 500):
# Ingest 500 rows at a time
print "Inserting row %d of %s" % (idx, filepath)
rows_subset = rows_in_file[idx:idx+500]
rows_to_insert = []
for row in rows_subset:
unsaved_new_contribution = Contribution(**row_to_dict(row))
import pdb; pdb.set_trace()
# If the row isn't already there, insert it
if :
pass
# If the row is there, check for modifications
elif:
# If it has not been modified, simply add a ContributionHistory object
if:
# If it has been modified, create a new object and give the new object a contribution history
else:
pass
Contribution.insert_many(rows_subset).execute()
def update(self, comment_id, user_id, content):
q = ("update storage.post_comments " + "set content = %s " +
"where id = %s and user_id = %s and deleted = false")
with db.transaction("main") as tx:
tx.execute(q, (content, comment_id, user_id))
return True
return False
def make_session(self, **kwargs):
r = self.post_json("/api/session/", follow_redirects=True)
self.session_id = self.get_response_data(r, "id")
from app.models.session.tables import sessions
if kwargs.get("verbose", False):
with app_db.transaction("main") as tx:
for s in tx.execute(sessions.select()).fetchall():
print(s)
def authenticate(self, user, password, origin, agent, session_id):
q = ("select * from " + "auth.authenticate(%s, %s, %s, %s, %s)")
result = {}
with db.transaction("main") as tx:
rs = tx.execute(q, user, password, origin, agent, session_id)
session_id = rs.fetchone()[0]
if session_id:
result["session_id"] = session_id
return result
def create(self, data):
with db.transaction("main") as tx:
rs = tx.execute(self._query_create, data)
comment = rs.fetchone()
q = self._query_fetch + " WHERE id = %s"
rs = tx.execute(q, (data.get("post_id"), comment.id))
tx.connection.commit()
comment = rs.fetchone()
return comment
def ingest(filepath):
'''Ingest file into database'''
print "Ingesting %s" % filepath
rows = parse_fec_file(filepath)
# check history table to see if this file is done
with db.transaction():
for idx, row in enumerate(rows):
print "Checking row %d of %d from %s" % (idx, len(rows), filepath)
try:
contribution_in_db = Contribution.get(cycle=row['cycle'], sub_id=row['sub_id'])
except Contribution.DoesNotExist:
contribution_in_db = None
# If the row isn't already there, insert it
if not contribution_in_db:
print t.cyan("\tInserting new row %d of %s" % (idx, filepath))
new_contribution = Contribution.create(**row)
ContributionHistory.create(contribution=new_contribution.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
# If the row is there, check for modifications
else:
# If it has not been modified, simply add a ContributionHistory object
contribution_in_db_dict = get_dictionary_from_model(contribution_in_db)
# x = {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]}
# y = {k:v for k,v in row.iteritems() if k != "date"}
if {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]} == {k:v for k,v in row.iteritems() if k != "date"}:
print t.white("\tNo changes found in row %d of %s" % (idx, filepath))
ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
# If it has been modified, create a new object and give the new object a contribution history
else:
print t.magenta("\tDetected change in row %d of %s" % (idx, filepath))
# print diff(x,y)
# import pdb; pdb.set_trace()
ContributionChanges.create(contribution=contribution_in_db.id, **{k:v for k,v in contribution_in_db_dict.iteritems() if k != "id"})
for k,v in row.iteritems():
if v != getattr(contribution_in_db, k):
setattr(contribution_in_db, k, v)
contribution_in_db.save()
ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
myfile, _ = File.get_or_create(
name = os.path.basename(filepath),
years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))),
sha1 = sha1OfFile(filepath),
updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(),
ingested = True
)
def create(self, client_origin, client_agent):
with db.transaction("main") as tx:
session_id = str(uuid.uuid4())
st = sessions.insert(returning=sessions.c).values(
id=session_id,
client_origin=str(client_origin),
client_agent=str(client_agent),
)
rs = tx.execute(st)
tx.connection.commit()
return rs.fetchone()
def read(self, session_id):
with db.transaction("main") as tx:
rs = tx.execute(
"select * from auth.read_session(%s) as " +
"(id varchar, ctime timestamp, mtime timestamp, user_id int, "
+ "permanent boolean, data json)", (session_id, ))
record = rs.fetchone()
record = dict(record or {})
if record and record.get("data"):
record["data"] = json.loads(record.get("data"))
return record
def confirm(self, user, action_type, key):
handlers_map = {"registration": self._confirm_user_registation}
handler = handlers_map[action_type.lower()]
success = False
if handler:
success = handler(user, action_type, key)
else:
q = "SELECT * from auth.verify_confirmation(%s, %s, %s)"
with db.transaction("main") as tx:
rs = tx.execute(q, (user.id, action_type, key))
success = rs.fetchone()[0]
return success
def delete(self, media_id):
q = (
"delete from storage.media where id = %s " +
"returning (media_root || path) as fs_path"
)
with db.transaction("main") as tx:
rs = tx.execute(q, (media_id,))
record = rs.fetchone()
if record.fs_path:
try:
os.unlink(record.fs_path)
except OSError as e:
print("model::Media::delete", e)
def send_mail(self, message_id):
mailq = []
with db.transaction("main") as tx:
st = tables.mail_queue.select()
st = st.where(tables.mail_queue.c.message_id == message_id)
mailq = tx.execute(st).fetchall()
if mailq:
with current_app.app.mail.connect() as mx:
for item in mailq:
msg = None
with db.transaction("main") as mtx:
msg = Message(item.title, recipients=[item.ident])
msg.body = (item.body_text or "")
if item.body_html:
msg.html = item.body_html
if not msg:
continue
try:
mx.send(msg)
except Exception as e:
with db.transaction("main") as mtx:
st = tables.queue.update().values(
delivery_failures=(
tables.queue.c.delivery_failures + 1),
delivery_time=datetime.now())
st = st.where(tables.queue.c.id == item.id)
mtx.execute(st)
mtx.connection.commit()
raise self.retry(exc=e)
with db.transaction("main") as mtx:
st = tables.queue.update().values(
sent=True, success=True, delivery_time=datetime.now())
st = st.where(tables.queue.c.id == item.id)
mtx.execute(st)
return True
def create(self, user, action_type, max_age=None):
confirmation_key = str(uuid.uuid4())
values = dict(
confirmation_key=confirmation_key,
user_id=user.id,
action_type=action_type,
)
if max_age:
values["max_age"] = max_age
st = confirmations.insert().values(**values)
with db.transaction() as tx:
tx.execute(st)
return confirmation_key
def update_data(self, session_id, data):
data = (data or {})
session_data = None
if data:
r = self.read(session_id)
session_data = (r.get("data") or {})
session_data.update(data)
with db.transaction("main") as tx:
st = sessions.update(returning=sessions.c)
st = st.values(
data=(json.dumps(session_data) if session_data else None), )
st = st.where(sessions.c.id == session_id)
rs = tx.execute(st)
return rs.fetchone()
def seed_from(filepath):
'''Ingest file into sqlite database'''
print "Ingesting %s" % filepath
rows = parse_fec_file(filepath)
with db.transaction():
myfile, _ = File.get_or_create(
name = os.path.basename(filepath),
years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))),
sha1 = sha1OfFile(filepath),
updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(),
ingested = False
)
for idx in range(0, len(rows), 500):
print "Inserting row %d of %d from %s" % (idx, len(rows), filepath)
rows_subset = rows[idx:idx+500]
Contribution.insert_many(rows_subset).execute()
myfile.update(ingested=True)
def make_auth(self):
from app.models.user.tables import users
from app.models.session.tables import sessions
self.make_session()
pid = os.getpid()
self.username = "******".format(pid)
self.ident = "test-{}@localhost".format(pid)
self.password = "******".format(pid)
st = users.insert().returning(users.c.id).values(
username=self.username,
ident=self.username,
password=self.password
)
with app_db.transaction("main") as tx:
r = tx.execute(st)
self.uid = r.fetchone()[0]
st = sessions.update().values(user_id=self.uid).where(
sessions.c.id == self.session_id
).returning(sessions.c.id)
result = tx.execute(st).fetchone()[0]
if not result:
raise Exception("make_auth failed")
def _confirm_user_registation(self, user, action_type, key):
q = "SELECT * from auth.confirm_user_registration(%s, %s, %s)"
with db.transaction("main") as tx:
rs = tx.execute(q, (user.id, action_type, key))
return rs.fetchone()[0]
def touch(self, session_id, client_agent, client_origin):
with db.transaction("main") as tx:
rs = tx.execute(
"select * from auth.touch_session(%s, %s, %s) as result",
(session_id, str(client_agent), str(client_origin)))
return rs.fetchone()[0]
def destroy(self, session_id):
with db.transaction("main") as tx:
st = sessions.delete().where(sessions.c.id == session_id)
tx.execute(st)
tx.connection.commit()
