def _wrapper(**view_kwargs): if config.DEBUG: with db.transaction(): return view(**view_kwargs) try: with db.transaction(): return view(**view_kwargs) except Exception as e: response = json.jsonify(dict( error=500, message=e.message or "Unknown Error" )) return response
def save(self, file_object, mime_type, **kwargs): media_root = current_app.config.get("MEDIA_ROOT") if not media_root: raise Exception("MEDIA_ROOT not configured.") letter = file_object.filename[0] destdir = os.path.join(media_root, mime_type, letter) if not os.path.isdir(destdir): os.makedirs(destdir) fname = "%s-%s" % ( str(time.time()).replace(".", "-"), file_object.filename ) fpath = os.path.join(mime_type, letter, fname) full_path = os.path.join(destdir, fname) file_object.save(full_path) file_size = os.stat(full_path).st_size params = ( file_object.filename, kwargs.get("title", None), file_size, mime_type, media_root, fpath, ) q = " ".join([ "insert into storage.media", "(original_name, title, file_size, mime_type, media_root, path)", "values(%s) returning id, path" % ",".join(["%s"] * len(params)) ]) with db.transaction("main") as tx: rs = tx.execute(q, params) return rs.fetchone()
def enqueue(subject, rcpt, template, **kwargs): txt_tpl = os.path.join("mail", template) + ".txt" html_tpl = os.path.join("mail", template) + ".html" txt = render(txt_tpl, **kwargs) html = render(html_tpl, **kwargs) sender = kwargs.get("sender", current_app.config["MAIL_DEFAULT_SENDER"]) rcpt = rcpt if isinstance(rcpt, list) else [rcpt] with db.transaction() as tx: st = tables.messages.insert().returning(tables.messages.c.id) st = st.values(title=subject, body_text=txt, body_html=html) r = tx.execute(st) mail_id = r.fetchone()[0] st = tables.queue.insert() r = tx.execute(st, [ dict( message_id=mail_id, sender=sender, recipient_id=rcpt_id, ) for rcpt_id in rcpt ]) r = send_mail.apply_async( (mail_id, ), queue="mail.low-priority", max_retries=3, )
async def add_tweet(user, msg, tags): """ add user tweet with specific tags and after that add to redis to fetch in future :param user: login user json data :param msg: tweet message :param tags: list of tags :return: """ try: async with db.transaction(): tweet = Tweet(msg=msg, user_id=user.get('user_id')) await tweet.create() for tag_item in tags: tag = await get_or_create(Tag, *(Tag.tag_name == tag_item, ), **({ "tag_name": tag_item })) tweet_tag = TagTweet(tag_id=tag.id, tweet_id=tweet.id) await tweet_tag.create() if tweet_tag: await redis.append_value(tag.tag_name, tweet.as_json()) return tweet.as_json(), None except Exception as ex: global_logger.write_log('error', f"error: {ex}") return None, ex
def update(self, session_id, **kwargs): with db.transaction("main") as tx: st = sessions.update(returning=sessions.c) st = st.values(**kwargs) st = st.where(sessions.c.id == session_id) rs = tx.execute(st) return rs.fetchone()
def create(self, data): with db.transaction("main") as tx: st = tables.users.insert().returning( tables.users.c.id, tables.users.c.ident, tables.users.c.username, tables.users.c.last_success, tables.users.c.last_failure, tables.users.c.failures, tables.users.c.city, tables.users.c.country, tables.users.c.gender, ) params = dict( ident=data.get("ident"), username=data.get("username"), password=data.get("password"), ) if data.get("social_uid"): params["social_uid"] = data.get("social_uid") params["social_provider"] = data.get("social_provider") st = st.values(**params) rs = tx.execute(st) return rs.fetchone()
def ingest(filepath): '''Ingest file into database''' print "Ingesting %s" % filepath rows_in_file = parse_fec_file(filepath) myfile = File.get_or_create(name=filepath) myfile_id = myfile.id with db.transaction(): # TODO: More sane error handling for idx in range(0, len(rows_in_file), 500): # Ingest 500 rows at a time print "Inserting row %d of %s" % (idx, filepath) rows_subset = rows_in_file[idx:idx+500] rows_to_insert = [] for row in rows_subset: unsaved_new_contribution = Contribution(**row_to_dict(row)) import pdb; pdb.set_trace() # If the row isn't already there, insert it if : pass # If the row is there, check for modifications elif: # If it has not been modified, simply add a ContributionHistory object if: # If it has been modified, create a new object and give the new object a contribution history else: pass Contribution.insert_many(rows_subset).execute()
def update(self, comment_id, user_id, content): q = ("update storage.post_comments " + "set content = %s " + "where id = %s and user_id = %s and deleted = false") with db.transaction("main") as tx: tx.execute(q, (content, comment_id, user_id)) return True return False
def make_session(self, **kwargs): r = self.post_json("/api/session/", follow_redirects=True) self.session_id = self.get_response_data(r, "id") from app.models.session.tables import sessions if kwargs.get("verbose", False): with app_db.transaction("main") as tx: for s in tx.execute(sessions.select()).fetchall(): print(s)
def authenticate(self, user, password, origin, agent, session_id): q = ("select * from " + "auth.authenticate(%s, %s, %s, %s, %s)") result = {} with db.transaction("main") as tx: rs = tx.execute(q, user, password, origin, agent, session_id) session_id = rs.fetchone()[0] if session_id: result["session_id"] = session_id return result
def create(self, data): with db.transaction("main") as tx: rs = tx.execute(self._query_create, data) comment = rs.fetchone() q = self._query_fetch + " WHERE id = %s" rs = tx.execute(q, (data.get("post_id"), comment.id)) tx.connection.commit() comment = rs.fetchone() return comment
def ingest(filepath): '''Ingest file into database''' print "Ingesting %s" % filepath rows = parse_fec_file(filepath) # check history table to see if this file is done with db.transaction(): for idx, row in enumerate(rows): print "Checking row %d of %d from %s" % (idx, len(rows), filepath) try: contribution_in_db = Contribution.get(cycle=row['cycle'], sub_id=row['sub_id']) except Contribution.DoesNotExist: contribution_in_db = None # If the row isn't already there, insert it if not contribution_in_db: print t.cyan("\tInserting new row %d of %s" % (idx, filepath)) new_contribution = Contribution.create(**row) ContributionHistory.create(contribution=new_contribution.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id']) # If the row is there, check for modifications else: # If it has not been modified, simply add a ContributionHistory object contribution_in_db_dict = get_dictionary_from_model(contribution_in_db) # x = {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]} # y = {k:v for k,v in row.iteritems() if k != "date"} if {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]} == {k:v for k,v in row.iteritems() if k != "date"}: print t.white("\tNo changes found in row %d of %s" % (idx, filepath)) ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id']) # If it has been modified, create a new object and give the new object a contribution history else: print t.magenta("\tDetected change in row %d of %s" % (idx, filepath)) # print diff(x,y) # import pdb; pdb.set_trace() ContributionChanges.create(contribution=contribution_in_db.id, **{k:v for k,v in contribution_in_db_dict.iteritems() if k != "id"}) for k,v in row.iteritems(): if v != getattr(contribution_in_db, k): setattr(contribution_in_db, k, v) contribution_in_db.save() ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id']) myfile, _ = File.get_or_create( name = os.path.basename(filepath), years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))), sha1 = sha1OfFile(filepath), updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(), ingested = True )
def create(self, client_origin, client_agent): with db.transaction("main") as tx: session_id = str(uuid.uuid4()) st = sessions.insert(returning=sessions.c).values( id=session_id, client_origin=str(client_origin), client_agent=str(client_agent), ) rs = tx.execute(st) tx.connection.commit() return rs.fetchone()
def read(self, session_id): with db.transaction("main") as tx: rs = tx.execute( "select * from auth.read_session(%s) as " + "(id varchar, ctime timestamp, mtime timestamp, user_id int, " + "permanent boolean, data json)", (session_id, )) record = rs.fetchone() record = dict(record or {}) if record and record.get("data"): record["data"] = json.loads(record.get("data")) return record
def confirm(self, user, action_type, key): handlers_map = {"registration": self._confirm_user_registation} handler = handlers_map[action_type.lower()] success = False if handler: success = handler(user, action_type, key) else: q = "SELECT * from auth.verify_confirmation(%s, %s, %s)" with db.transaction("main") as tx: rs = tx.execute(q, (user.id, action_type, key)) success = rs.fetchone()[0] return success
def delete(self, media_id): q = ( "delete from storage.media where id = %s " + "returning (media_root || path) as fs_path" ) with db.transaction("main") as tx: rs = tx.execute(q, (media_id,)) record = rs.fetchone() if record.fs_path: try: os.unlink(record.fs_path) except OSError as e: print("model::Media::delete", e)
def send_mail(self, message_id): mailq = [] with db.transaction("main") as tx: st = tables.mail_queue.select() st = st.where(tables.mail_queue.c.message_id == message_id) mailq = tx.execute(st).fetchall() if mailq: with current_app.app.mail.connect() as mx: for item in mailq: msg = None with db.transaction("main") as mtx: msg = Message(item.title, recipients=[item.ident]) msg.body = (item.body_text or "") if item.body_html: msg.html = item.body_html if not msg: continue try: mx.send(msg) except Exception as e: with db.transaction("main") as mtx: st = tables.queue.update().values( delivery_failures=( tables.queue.c.delivery_failures + 1), delivery_time=datetime.now()) st = st.where(tables.queue.c.id == item.id) mtx.execute(st) mtx.connection.commit() raise self.retry(exc=e) with db.transaction("main") as mtx: st = tables.queue.update().values( sent=True, success=True, delivery_time=datetime.now()) st = st.where(tables.queue.c.id == item.id) mtx.execute(st) return True
def create(self, user, action_type, max_age=None): confirmation_key = str(uuid.uuid4()) values = dict( confirmation_key=confirmation_key, user_id=user.id, action_type=action_type, ) if max_age: values["max_age"] = max_age st = confirmations.insert().values(**values) with db.transaction() as tx: tx.execute(st) return confirmation_key
def update_data(self, session_id, data): data = (data or {}) session_data = None if data: r = self.read(session_id) session_data = (r.get("data") or {}) session_data.update(data) with db.transaction("main") as tx: st = sessions.update(returning=sessions.c) st = st.values( data=(json.dumps(session_data) if session_data else None), ) st = st.where(sessions.c.id == session_id) rs = tx.execute(st) return rs.fetchone()
def seed_from(filepath): '''Ingest file into sqlite database''' print "Ingesting %s" % filepath rows = parse_fec_file(filepath) with db.transaction(): myfile, _ = File.get_or_create( name = os.path.basename(filepath), years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))), sha1 = sha1OfFile(filepath), updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(), ingested = False ) for idx in range(0, len(rows), 500): print "Inserting row %d of %d from %s" % (idx, len(rows), filepath) rows_subset = rows[idx:idx+500] Contribution.insert_many(rows_subset).execute() myfile.update(ingested=True)
def make_auth(self): from app.models.user.tables import users from app.models.session.tables import sessions self.make_session() pid = os.getpid() self.username = "******".format(pid) self.ident = "test-{}@localhost".format(pid) self.password = "******".format(pid) st = users.insert().returning(users.c.id).values( username=self.username, ident=self.username, password=self.password ) with app_db.transaction("main") as tx: r = tx.execute(st) self.uid = r.fetchone()[0] st = sessions.update().values(user_id=self.uid).where( sessions.c.id == self.session_id ).returning(sessions.c.id) result = tx.execute(st).fetchone()[0] if not result: raise Exception("make_auth failed")
def _confirm_user_registation(self, user, action_type, key): q = "SELECT * from auth.confirm_user_registration(%s, %s, %s)" with db.transaction("main") as tx: rs = tx.execute(q, (user.id, action_type, key)) return rs.fetchone()[0]
def touch(self, session_id, client_agent, client_origin): with db.transaction("main") as tx: rs = tx.execute( "select * from auth.touch_session(%s, %s, %s) as result", (session_id, str(client_agent), str(client_origin))) return rs.fetchone()[0]
def destroy(self, session_id): with db.transaction("main") as tx: st = sessions.delete().where(sessions.c.id == session_id) tx.execute(st) tx.connection.commit()