def get_payouts(): """ Used by remote procedure call to retrieve a list of transactions to be processed. Transaction information is signed for safety. """ s = TimedSerializer(current_app.config['rpc_signature']) s.loads(request.data) payouts = (Payout.query.filter_by(transaction_id=None). join(Payout.block, aliased=True).filter_by(mature=True)) bonus_payouts = BonusPayout.query.filter_by(transaction_id=None) pids = [(p.user, p.amount, p.id) for p in payouts] bids = [(p.user, p.amount, p.id) for p in bonus_payouts] return s.dumps([pids, bids])
def confirm_user(data=None): if not utils.get_config('verify_emails'): # If the CTF doesn't care about confirming email addresses then redierct to challenges return redirect(url_for('challenges.challenges_view')) logger = logging.getLogger('regs') # User is confirming email account if data and request.method == "GET": try: s = TimedSerializer(app.config['SECRET_KEY']) email = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('confirm.html', errors=[get_tip('LINK_EXPIRED')]) except (BadSignature, TypeError, base64.binascii.Error): return render_template('confirm.html', errors=[get_tip('INVIDE_RESET_TOKEN')]) team = Teams.query.filter_by(email=email).first_or_404() team.verified = True db.session.commit() logger.warn( get_tip('USER_HAVE_CM').format(date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'), email=team.email.encode('utf-8'))) db.session.close() if utils.authed(): return redirect(url_for('challenges.challenges_view')) return redirect(url_for('auth.login')) # User is trying to start or restart the confirmation flow if not utils.authed(): return redirect(url_for('auth.login')) team = Teams.query.filter_by(id=session['id']).first_or_404() if data is None: if request.method == "POST": # User wants to resend their confirmation email if team.verified: return redirect(url_for('views.profile')) else: utils.verify_email(team.email) logger.warn( get_tip('EMAIL_CF_RESENT').format( date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'), email=team.email.encode('utf-8'))) return render_template('confirm.html', team=team, infos=[get_tip('EMAIL_CF_SENT')]) elif request.method == "GET": # User has been directed to the confirm page team = Teams.query.filter_by(id=session['id']).first_or_404() if team.verified: # If user is already verified, redirect to their profile return redirect(url_for('views.profile')) return render_template('confirm.html', team=team)
class BaseConsumer(object): def __init__(self, base_url, public_key, private_key): self.base_url = base_url self.public_key = public_key self.signer = TimedSerializer(private_key) def consume(self, path, data, max_age=None): if not path.startswith('/'): raise ValueError("Paths must start with a slash") signed_data = self.signer.dumps(data) headers = { PUBLIC_KEY_HEADER: self.public_key, 'Content-Type': 'application/json', } url = self.build_url(path) body = self.send_request(url, data=signed_data, headers=headers) return self.handle_response(body, max_age) def handle_response(self, body, max_age): return self.signer.loads(body, max_age=max_age) def send_request(self, url, data, headers): raise NotImplementedError("Implement send_request on BaseConsumer subclasses") def raise_for_status(self, status_code, message): if status_code == 400: raise BadRequest(message) elif status_code >= 300: raise WebserviceError(message) def build_url(self, path): path = path.lstrip('/') return urlparse.urljoin(self.base_url, path)
def confirm_transactions(): """ Used as a response from an rpc payout system. This will either reset the sent status of a list of transactions upon failure on the remote side, or create a new CoinTransaction object and link it to the transactions to signify that the transaction has been processed. Both request and response are signed. """ s = TimedSerializer(current_app.config['rpc_signature']) data = s.loads(request.data) # basic checking of input try: assert len(data['coin_txid']) == 64 assert isinstance(data['pids'], list) assert isinstance(data['bids'], list) for id in data['pids']: assert isinstance(id, int) for id in data['bids']: assert isinstance(id, int) except AssertionError: current_app.logger.warn("Invalid data passed to confirm", exc_info=True) abort(400) coin_trans = Transaction.create(data['coin_txid']) db.session.flush() Payout.query.filter(Payout.id.in_(data['pids'])).update( {Payout.transaction_id: coin_trans.txid}, synchronize_session=False) BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])).update( {BonusPayout.transaction_id: coin_trans.txid}, synchronize_session=False) db.session.commit() return s.dumps(True)
def get_payouts(): """ Used by remote procedure call to retrieve a list of transactions to be processed. Transaction information is signed for safety. """ s = TimedSerializer(current_app.config['rpc_signature']) args = s.loads(request.data) current_app.logger.info("get_payouts being called, args of {}!".format(args)) lock = False merged = None if isinstance(args, dict) and args['lock']: lock = True if isinstance(args, dict) and args['merged']: merged = args['merged'] with Benchmark("Fetching payout information"): pids = [(p.user, p.amount, p.id) for p in Payout.query.filter_by(transaction_id=None, locked=False, merged_type=merged). join(Payout.block, aliased=True).filter_by(mature=True)] bids = [(p.user, p.amount, p.id) for p in BonusPayout.query.filter_by(transaction_id=None, locked=False, merged_type=merged). join(BonusPayout.block, aliased=True).filter_by(mature=True)] if lock: if bids: current_app.logger.info("Locking {} bonus ids at retriever request." .format(len(bids))) (BonusPayout.query.filter(BonusPayout.id.in_(p[2] for p in bids)) .update({BonusPayout.locked: True}, synchronize_session=False)) if pids: current_app.logger.info("Locking {} payout ids at retriever request." .format(len(pids))) (Payout.query.filter(Payout.id.in_(p[2] for p in pids)) .update({Payout.locked: True}, synchronize_session=False)) db.session.commit() return s.dumps([pids, bids, lock])
def reset_password(data=None): if data is not None and request.method == "GET": return render_template('reset_password.html', mode='set') if data is not None and request.method == "POST": try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(data.decode('base64'), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=['Your link has expired']) team = Teams.query.filter_by(name=name).first() team.password = bcrypt_sha256.encrypt(request.form['password'].strip()) db.session.commit() db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() if not team: return render_template('reset_password.html', errors=['Check your email']) s = TimedSerializer(app.config['SECRET_KEY']) token = s.dumps(team.name) text = """ Did you initiate a password reset? {0}/reset_password/{1} """.format(app.config['HOST'], token.encode('base64')) sendmail(email, text) return render_template('reset_password.html', errors=['Check your email']) return render_template('reset_password.html')
def reset_password(data=None): if data is not None and request.method == "GET": return render_template('reset_password.html', mode='set') if data is not None and request.method == "POST": try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(urllib.unquote_plus(data.decode('base64')), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=['Your link has expired']) except: return render_template('reset_password.html', errors=['Your link appears broken, please try again.']) team = Teams.query.filter_by(name=name).first_or_404() team.password = bcrypt_sha256.encrypt(request.form['password'].strip()) db.session.commit() db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() if not team: return render_template('reset_password.html', errors=['If that account exists you will receive an email, please check your inbox']) s = TimedSerializer(app.config['SECRET_KEY']) token = s.dumps(team.name) text = """ Did you initiate a password reset? {0}/{1} """.format(url_for('auth.reset_password', _external=True), urllib.quote_plus(token.encode('base64'))) utils.sendmail(email, text) return render_template('reset_password.html', errors=['If that account exists you will receive an email, please check your inbox']) return render_template('reset_password.html')
def loads(self, s, *args, **kwargs): if '"' not in s and "{" not in s: try: s = base64_decode(s) except: pass return TimedSerializer.loads(self, s, *args, **kwargs)
def verify_auth_token(token): s = TimedSerializer(secret_key) try: user_id = s.loads(token, max_age=600)['user_id'] except (SignatureExpired, BadSignature): return None return user_id
def reset_password(data=None): if utils.get_config('no_emails'): return redirect(url_for('auth.login')) logger = logging.getLogger('logins') if data is not None: try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=['Your link has expired']) except (BadSignature, TypeError, base64.binascii.Error): return render_template('reset_password.html', errors=['Your reset token is invalid']) if request.method == "GET": return render_template('reset_password.html', mode='set') if request.method == "POST": team = Teams.query.filter_by(name=name).first_or_404() team.password = bcrypt_sha256.encrypt( request.form['password'].strip()) db.session.commit() logger.warn( "[{date}] {ip} - successful password reset for {username}". format(date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'))) db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() errors = [] if utils.can_send_mail() is False: return render_template( 'reset_password.html', errors=[ 'Email could not be sent due to server misconfiguration' ]) if not team: return render_template( 'reset_password.html', errors=[ 'If that account exists you will receive an email, please check your inbox' ]) utils.forgot_password(email, team.name) return render_template( 'reset_password.html', errors=[ 'If that account exists you will receive an email, please check your inbox' ]) return render_template('reset_password.html')
def test_request_token_with_post_method_and_access_key_and_signdata_and_no_login( self): url = reverse_lazy('cas_app:cas-request-token') serializer = TimedSerializer(self.secret_key) data = serializer.dumps({'redirect_to': self.redirect_to}) data_extra = { 'HTTP_X_SERVICES_PUBLIC_KEY': self.access_key, } response = self.client.post(url, data, content_type='application/json', **data_extra) response_data = serializer.loads(response.content) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(self.cas_consumer.cas_tokens.count(), 1) self.assertIn('request_token', response_data) request_token = response_data['request_token'] url = reverse_lazy('cas_app:cas-user-authentication') response = self.client.get(url, data={ 'request_token': request_token, }) self.assertEqual(response.status_code, status.HTTP_302_FOUND) print response
def reset_password(data=None): if data is not None and request.method == "GET": return render_template("reset_password.html", mode="set") if data is not None and request.method == "POST": try: s = TimedSerializer(app.config["SECRET_KEY"]) name = s.loads(data.decode("base64"), max_age=1800) except BadTimeSignature: return render_template("reset_password.html", errors=["Your link has expired"]) team = Teams.query.filter_by(name=name).first() team.password = bcrypt_sha256.encrypt(request.form["password"].strip()) db.session.commit() db.session.close() return redirect(url_for("auth.login")) if request.method == "POST": email = request.form["email"].strip() team = Teams.query.filter_by(email=email).first() if not team: return render_template("reset_password.html", errors=["Check your email"]) s = TimedSerializer(app.config["SECRET_KEY"]) token = s.dumps(team.name) text = """ Did you initiate a password reset? {0}/reset_password/{1} """.format( url_for("auth.reset_password", _external=True), token.encode("base64") ) sendmail(email, text) return render_template("reset_password.html", errors=["Check your email"]) return render_template("reset_password.html")
def get_payouts(): """ Used by remote procedure call to retrieve a list of transactions to be processed. Transaction information is signed for safety. """ s = TimedSerializer(current_app.config['rpc_signature']) args = s.loads(request.data) current_app.logger.info("get_payouts being called, args of {}!".format(args)) lock = False merged = None if isinstance(args, dict) and args['lock']: lock = True if isinstance(args, dict) and args['merged']: merged = args['merged'] payouts = (Payout.query.filter_by(transaction_id=None, locked=False, merged_type=merged). join(Payout.block, aliased=True).filter_by(mature=True)).all() bonus_payouts = (BonusPayout.query.filter_by(transaction_id=None, locked=False, merged_type=merged). join(BonusPayout.block, aliased=True).filter_by(mature=True)).all() pids = [(p.user, p.amount, p.id) for p in payouts] bids = [(p.user, p.amount, p.id) for p in bonus_payouts] if lock: current_app.logger.info("Locking pids and bids at retriever request.") for payout in payouts: payout.locked = True for payout in bonus_payouts: payout.locked = True db.session.commit() return s.dumps([pids, bids, lock])
def resetVerifyCheck(key): url_serial = URLSafeSerializer( "XHhkMVx4OTgzXFxceDhmZilceDk2Ilx4MTZceDhkXHhhYlx4ZGFceDlkXHhiYUNHXHhlM1x4YTRceDFiK0RceGNhfg==" ) timed_serial = TimedSerializer( "XHhkMVx4OTgzXFxceDhmZilceDk2Ilx4MTZceDhkXHhhYlx4ZGFceDlkXHhiYUNHXHhlM1x4YTRceDFiK0RceGNhfg==" ) status = None try: key = timed_serial.loads(key, max_age=300) key = url_serial.loads(key) username = str(sanitize(key[0])) password = sha512_crypt.encrypt(sanitize(key[1])) except: status = "Oops, The Link Has Expired..!!" return status cursor, dbconn = dbConnection() query = "UPDATE users SET password = %s WHERE username = %s" try: cursor.execute(query, (password, username)) dbconn.commit() status = "Password Updated Successfully..!!" except: dbconn.rollback() status = "Oops, Something Went Wrong Please Try Again..!!" cursor.close() dbconn.close() return status
def verify_timed_token(token): s = TimedSerializer(SECRET_KEY) print "verify_timed_token", TOKEN_EXPIRES try: data = s.loads(token, max_age=TOKEN_EXPIRES) except (SignatureExpired, BadSignature): return None return data
def verify_auth_token(token, max_age=3600): s = TimedSerializer(secret_key) try: data = s.loads(token, max_age=max_age) except SignatureExpired: return None except BadSignature: return None return data
def confirm_user(data=None): if not utils.get_config('verify_emails'): # If the CTF doesn't care about confirming email addresses then redierct to challenges return redirect(url_for('challenges.challenges_view')) logger = logging.getLogger('regs') # User is confirming email account if data and request.method == "GET": try: s = TimedSerializer(app.config['SECRET_KEY']) email = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('confirm.html', errors=['Your confirmation link has expired']) except BadSignature: return render_template('confirm.html', errors=['Your confirmation link seems wrong']) team = Teams.query.filter_by(email=email).first_or_404() team.verified = True db.session.commit() logger.warn("[{date}] {ip} - {username} confirmed their account".format( date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'), email=team.email.encode('utf-8') )) db.session.close() if utils.authed(): return redirect(url_for('challenges.challenges_view')) return redirect(url_for('auth.login')) # User is trying to start or restart the confirmation flow if not utils.authed(): return redirect(url_for('auth.login')) team = Teams.query.filter_by(id=session['id']).first_or_404() if data is None: if request.method == "POST": # User wants to resend their confirmation email if team.verified: return redirect(url_for('views.profile')) else: utils.verify_email(team.email) logger.warn("[{date}] {ip} - {username} initiated a confirmation email resend".format( date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'), email=team.email.encode('utf-8') )) return render_template('confirm.html', team=team, infos=['Your confirmation email has been resent!']) elif request.method == "GET": # User has been directed to the confirm page team = Teams.query.filter_by(id=session['id']).first_or_404() if team.verified: # If user is already verified, redirect to their profile return redirect(url_for('views.profile')) return render_template('confirm.html', team=team)
def verify_auth_token(token): s = TimedSerializer(SECRET_KEY) try: data = s.loads(token, max_age=600) except SignatureExpired: return None # valid token, but expired except BadSignature: return None # invalid token user = User.query.get(data.get('user_id')) return user
def confirm(self, token): s = TimedSerializer(current_app.config['SECRET_KEY']) try: data = s.loads(token, max_age=3600, salt="please give some salt") except: return False if data.get('confirm') != self.id: return False self.confirmed = True db.session.add(self) return True
def reset_password(data=None): logger = logging.getLogger('logins') if data is not None and request.method == "GET": return render_template('reset_password.html', mode='set') if data is not None and request.method == "POST": try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=['Your link has expired']) except: return render_template( 'reset_password.html', errors=['Your link appears broken, please try again.']) team = Teams.query.filter_by(name=name).first_or_404() team.password = bcrypt_sha256.encrypt(request.form['password'].strip()) db.session.commit() logger.warn( "[{date}] {ip} - successful password reset for {username}".format( date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'))) db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() if not team: return render_template( 'reset_password.html', errors=[ 'If that account exists you will receive an email, please check your inbox' ]) s = TimedSerializer(app.config['SECRET_KEY']) token = s.dumps(team.name) text = """ Did you initiate a password reset? {0}/{1} """.format(url_for('auth.reset_password', _external=True), utils.base64encode(token, urlencode=True)) utils.sendmail(email, text) return render_template( 'reset_password.html', errors=[ 'If that account exists you will receive an email, please check your inbox' ]) return render_template('reset_password.html')
def reset_password(new_password, token): s = TimedSerializer(current_app.config['SECRET_KEY']) try: data = s.loads(token.encode('utf-8'), max_age=1800, salt="please give some salt") except: return False user = User.query.get(id=data.get('reset')) if user is None: return False user.password = new_password db.session.add(user) return 'ok'
def reset_password(data=None): logger = logging.getLogger('logins') if data is not None: try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=['您的密码重置链接已经过期']) except (BadSignature, TypeError, base64.binascii.Error): return render_template('reset_password.html', errors=['您的密码重置令牌已经失效']) if request.method == "GET": return render_template('reset_password.html', mode='set') if request.method == "POST": team = Teams.query.filter_by(name=name).first_or_404() team.password = bcrypt_sha256.encrypt( request.form['password'].strip()) db.session.commit() logger.warn( "[{date}] {ip} - successful password reset for {username}". format(date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'))) db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() errors = [] if utils.can_send_mail() is False: return render_template('reset_password.html', errors=['邮件信息配置异常,无法发送邮件,请联系管理员']) if not team: return render_template('reset_password.html', errors=['如果邮箱有效,将会收到一封密码重置邮件,请注意查收']) utils.forgot_password(email, team.name) return render_template('reset_password.html', errors=['如果邮箱有效,将会收到一封密码重置邮件,请注意查收']) return render_template('reset_password.html')
def reset_password(data=None): logger = logging.getLogger('logins') if data is not None: try: s = TimedSerializer(app.config['SECRET_KEY']) name = s.loads(utils.base64decode(data, urldecode=True), max_age=1800) except BadTimeSignature: return render_template('reset_password.html', errors=[get_tip('LINK_EXPIRED')]) except (BadSignature, TypeError, base64.binascii.Error): return render_template('reset_password.html', errors=[get_tip('INVIDE_RESET_TOKEN')]) if request.method == "GET": return render_template('reset_password.html', mode='set') if request.method == "POST": team = Teams.query.filter_by(name=name).first_or_404() team.password = bcrypt_sha256.encrypt( request.form['password'].strip()) db.session.commit() logger.warn( get_tip('PASS_HAVE_RESET').format( date=time.strftime("%m/%d/%Y %X"), ip=utils.get_ip(), username=team.name.encode('utf-8'))) db.session.close() return redirect(url_for('auth.login')) if request.method == 'POST': email = request.form['email'].strip() team = Teams.query.filter_by(email=email).first() errors = [] if utils.can_send_mail() is False: return render_template('reset_password.html', errors=[get_tip('EMAIL_NOT_CONFIG')]) if not team: return render_template('reset_password.html', errors=[get_tip('FORGOT_PASS_NOTICE')]) utils.forgot_password(email, team.name) return render_template('reset_password.html', errors=[get_tip('FORGOT_PASS_NOTICE')]) return render_template('reset_password.html')
def confirm_email_change(self, token): serializer = TimedSerializer(config['secret_key']) try: msg = serializer.loads(token) now = arrow.utcnow().timestamp if msg.get('user_uuid') == self.user_uuid.hex\ and msg.get('expired_on', now) > now: self.email = msg[ 'new_email'] if 'new_email' in msg else self.email db.session.add(self) db.session.commit() return True except BadTimeSignature: current_app.logger.error('User %s: failed to confirm email change', self.user_uuid.hex) return False
def confirm(self, token): serializer = TimedSerializer(config['secret_key']) try: msg = serializer.loads(token) now = arrow.utcnow().timestamp if msg.get('confirm_register') == self.user_uuid.hex \ and msg.get('expired_on', now) > now: self.confirmed = True self.confirmed_on = datetime.utcnow() db.session.add(self) db.session.commit() current_app.logger.info('User %s: confirmed', self.user_uuid.hex) return True except BadTimeSignature: current_app.logger.error('User %s: failed to confirm', self.user_uuid.hex) return False
def confirm_transactions(): """ Used to confirm that a transaction is now complete on the network. """ s = TimedSerializer(current_app.config['rpc_signature']) data = s.loads(request.data) # basic checking of input try: assert isinstance(data['tids'], list) except AssertionError: current_app.logger.warn("Invalid data passed to confirm_transactions", exc_info=True) abort(400) Transaction.query.filter(Transaction.txid.in_(data['tids'])).update( {Transaction.confirmed: True}, synchronize_session=False) db.session.commit() return s.dumps(True)
def update_transactions(): """ Used as a response from an rpc payout system. This will either reset the locked status of a list of transactions upon failure on the remote side, or create a new CoinTransaction object and link it to the transactions to signify that the transaction has been processed. Both request and response are signed. """ s = TimedSerializer(current_app.config['rpc_signature']) data = s.loads(request.data) # basic checking of input try: if 'coin_txid' in data: assert len(data['coin_txid']) == 64 else: assert 'reset' in data assert isinstance(data['reset'], bool) assert isinstance(data['pids'], list) assert isinstance(data['bids'], list) for id in data['pids']: assert isinstance(id, int) for id in data['bids']: assert isinstance(id, int) except AssertionError: current_app.logger.warn("Invalid data passed to confirm", exc_info=True) abort(400) if 'coin_txid' in data: merged_type = data.get('merged', None) coin_trans = Transaction.create(data['coin_txid'], merged_type=merged_type) db.session.flush() Payout.query.filter(Payout.id.in_(data['pids'])).update( {Payout.transaction_id: coin_trans.txid}, synchronize_session=False) BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])).update( {BonusPayout.transaction_id: coin_trans.txid}, synchronize_session=False) db.session.commit() elif data['reset']: Payout.query.filter(Payout.id.in_(data['pids'])).update( {Payout.locked: False}, synchronize_session=False) BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])).update( {BonusPayout.locked: False}, synchronize_session=False) db.session.commit() return s.dumps(dict(success=True, result="Successfully reset")) return s.dumps(True)
class BaseConsumer(object): def __init__(self, base_url, public_key, private_key): self.base_url = base_url self.public_key = public_key self.signer = TimedSerializer(private_key) @classmethod def from_dsn(cls, url): base_url, public_key, private_key = getdomain(url) return cls(base_url, public_key, private_key) def consume(self, path, data, max_age=None): if not path.startswith('/'): raise ValueError("Paths must start with a slash") signed_data = self.signer.dumps(data) headers = { PUBLIC_KEY_HEADER: self.public_key, 'Content-Type': 'application/json', } url = self.build_url(path) body = self.send_request(url, data=signed_data, headers=headers) return self.handle_response(body, max_age) def handle_response(self, body, max_age): return self.signer.loads(body, max_age=max_age) def send_request(self, url, data, headers): raise NotImplementedError( 'Implement send_request on BaseConsumer subclasses') def raise_for_status(self, status_code, message): if status_code == 500: raise Exception500(message) elif status_code == 404: raise Exception404(message) elif status_code == 400: raise Exception400(message) elif status_code >= 300: raise DjConnectError(message) def build_url(self, path): path = path.lstrip('/') return urljoin(self.base_url, path)
def get_payouts(): """ Used by remote procedure call to retrieve a list of transactions to be processed. Transaction information is signed for safety. """ s = TimedSerializer(current_app.config['rpc_signature']) args = s.loads(request.data) current_app.logger.info( "get_payouts being called, args of {}!".format(args)) lock = False merged = None if isinstance(args, dict) and args['lock']: lock = True if isinstance(args, dict) and args['merged']: merged = args['merged'] with Benchmark("Fetching payout information"): pids = [(p.user, p.amount, p.id) for p in Payout.query.filter_by( transaction_id=None, locked=False, merged_type=merged).join( Payout.block, aliased=True).filter_by(mature=True)] bids = [(p.user, p.amount, p.id) for p in BonusPayout.query.filter_by( transaction_id=None, locked=False, merged_type=merged).join( BonusPayout.block, aliased=True).filter_by(mature=True)] if lock: if bids: current_app.logger.info( "Locking {} bonus ids at retriever request.".format( len(bids))) (BonusPayout.query.filter( BonusPayout.id.in_(p[2] for p in bids)).update( {BonusPayout.locked: True}, synchronize_session=False)) if pids: current_app.logger.info( "Locking {} payout ids at retriever request.".format( len(pids))) (Payout.query.filter(Payout.id.in_(p[2] for p in pids)).update( {Payout.locked: True}, synchronize_session=False)) db.session.commit() return s.dumps([pids, bids, lock])
def get_response(self, method, signed_data, get_header): if method != 'POST': return (405, ['POST']) public_key = get_header(PUBLIC_KEY_HEADER, None) if not public_key: return (400, "No public key") private_key = self.get_private_key(public_key) if not private_key: return (400, "Invalid public key") signer = TimedSerializer(private_key) try: data = signer.loads(signed_data, max_age=self.max_age) except SignatureExpired: return (400, "Signature expired") except BadSignature: return (400, "Bad Signature") try: raw_response_data = self.provide(data) except: self.report_exception() return (400, "Failed to process the request") response_data = signer.dumps(raw_response_data) return (200, response_data)
class SCRPCClient(object): def _set_config(self, **kwargs): # A fast way to set defaults for the kwargs then set them as attributes base = os.path.abspath(os.path.dirname(__file__) + '/../') self.config = dict(max_age=10, logger_name="sc_rpc_client", log_level="INFO", database_path=base + '/rpc_', log_path=base + '/sc_rpc.log', min_confirms=12, minimum_tx_output=0.00000001) self.config.update(kwargs) # Kinda sloppy, but it works self.config['database_path'] += self.config['currency_code'] + '.sqlite' required_conf = ['valid_address_versions', 'currency_code', 'rpc_signature', 'rpc_url'] error = False for req in required_conf: if req not in self.config: print("{} is a required configuration variable".format(req)) error = True if error: raise SCRPCException('Errors occurred while configuring RPCClient obj') def __init__(self, config, CoinRPC, logger=None): if not config: raise SCRPCException('Invalid configuration file') self._set_config(**config) # Setup CoinRPC self.coin_rpc = CoinRPC # Setup the sqlite database mapper self.engine = sa.create_engine('sqlite:///{}'.format(self.config['database_path']), echo=self.config['log_level'] == "DEBUG") # Pulled from SQLA docs to implement strict exclusive access to the # payout state database. # See http://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#pysqlite-serializable @sa.event.listens_for(self.engine, "connect") def do_connect(dbapi_connection, connection_record): # disable pysqlite's emitting of the BEGIN statement entirely. # also stops it from emitting COMMIT before any DDL. dbapi_connection.isolation_level = None @sa.event.listens_for(self.engine, "begin") def do_begin(conn): # emit our own BEGIN conn.execute("BEGIN EXCLUSIVE") self.db = sessionmaker(bind=self.engine) self.db.session = self.db() # Hack if flask is in the env self.db.session._model_changes = {} # Create the table if it doesn't exist Payout.__table__.create(self.engine, checkfirst=True) # Setup logger for the class if logger: self.logger = logger else: logging.Formatter.converter = datetime.time.gmtime self.logger = logging.getLogger(self.config['logger_name']) self.logger.setLevel(getattr(logging, self.config['log_level'])) log_format = logging.Formatter('%(asctime)s %(levelname)s %(message)s') # stdout handler handler = logging.StreamHandler(sys.stdout) handler.setFormatter(log_format) handler.setLevel(getattr(logging, self.config['log_level'])) self.logger.addHandler(handler) # don't attach a file handler if path evals false if self.config['log_path']: handler = logging.FileHandler(self.config['log_path']) handler.setFormatter(log_format) handler.setLevel(getattr(logging, self.config['log_level'])) self.logger.addHandler(handler) self.serializer = TimedSerializer(self.config['rpc_signature']) ######################################################################## # Helper URL methods ######################################################################## def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote('/rpc/' + url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, signed=True, **kwargs): url = urljoin(self.config['rpc_url'], url) self.logger.debug("Making request to {}".format(url)) ret = getattr(requests, method)(url, timeout=270, **kwargs) if ret.status_code != 200: raise SCRPCException("Non 200 from remote: {}".format(ret.text)) try: self.logger.debug("Got {} from remote".format(ret.text.encode('utf8'))) if signed: return self.serializer.loads(ret.text, max_age or self.config['max_age']) else: return ret.json() except BadData: self.logger.error("Invalid data returned from remote!", exc_info=True) raise SCRPCException("Invalid signature") ######################################################################## # RPC Client methods ######################################################################## def pull_payouts(self, simulate=False): """ Gets all the unpaid payouts from the server """ if simulate: self.logger.info('#'*20 + ' Simulation mode ' + '#'*20) try: payouts = self.post( 'get_payouts', data={'currency': self.config['currency_code']} )['pids'] except ConnectionError: self.logger.warn('Unable to connect to SC!', exc_info=True) return if not payouts: self.logger.info("No {} payouts to process.." .format(self.config['currency_code'])) return repeat = 0 new = 0 invalid = 0 for user, address, amount, pid in payouts: # Check address is valid if not get_bcaddress_version(address) in self.config['valid_address_versions']: self.logger.warn("Ignoring payout {} due to invalid address. " "{} address did not match a valid version {}" .format((user, address, amount, pid), self.config['currency_code'], self.config['valid_address_versions'])) invalid += 1 continue # Check payout doesn't already exist if self.db.session.query(Payout).filter_by(pid=pid).first(): self.logger.debug("Ignoring payout {} because it already exists" " locally".format((user, address, amount, pid))) repeat += 1 continue # Create local payout obj p = Payout(pid=pid, user=user, address=address, amount=amount, currency_code=self.config['currency_code'], pull_time=datetime.datetime.utcnow()) new += 1 if not simulate: self.db.session.add(p) self.db.session.commit() self.logger.info("Inserted {:,} new {} payouts and skipped {:,} old " "payouts from the server. {:,} payouts with invalid addresses." .format(new, self.config['currency_code'], repeat, invalid)) return True def send_payout(self, simulate=False): """ Collects all the unpaid payout ids (for the configured currency) and pays them out """ if simulate: self.logger.info('#'*20 + ' Simulation mode ' + '#'*20) try: self.coin_rpc.poke_rpc() except CoinRPCException as e: self.logger.warn( "Error occured while trying to get info from the {} RPC. Got " "{}".format(self.config['currency_code'], e)) return False # Grab all payouts now so that we use the same list of payouts for both # database transactions (locking, and unlocking) payouts = (self.db.session.query(Payout). filter_by(txid=None, locked=False, currency_code=self.config['currency_code']) .all()) if not payouts: self.logger.info("No payouts to process, exiting") return True # track the total payouts to each address address_payout_amounts = {} pids = {} for payout in payouts: address_payout_amounts.setdefault(payout.address, 0.0) address_payout_amounts[payout.address] += float(payout.amount) pids.setdefault(payout.address, []) pids[payout.address].append(payout.pid) # We'll lock the payout before continuing in case of a failure in # between paying out and recording that payout action payout.locked = True payout.lock_time = datetime.datetime.utcnow() for address, amount in address_payout_amounts.items(): # Convert amount from STR and coerce to a payable value. # Note that we're not trying to validate the amount here, all # validation should be handled server side. amount = round(float(amount), 8) if amount < self.config['minimum_tx_output']: # We're unable to pay, so undo the changes from the last loop self.logger.warn('Removing {} with payout amount of {} (which ' 'is lower than network output min of {}) from ' 'the {} payout dictionary' .format(address, amount, self.config['minimum_tx_output'], self.config['currency_code'])) address_payout_amounts.pop(address) pids[address] = [] for payout in payouts: if payout.address == address: payout.locked = False payout.lock_time = None else: address_payout_amounts[address] = amount total_out = sum(address_payout_amounts.values()) balance = self.coin_rpc.get_balance(self.coin_rpc.coinserv['account']) self.logger.info("Account balance for {} account \'{}\': {:,}" .format(self.config['currency_code'], self.coin_rpc.coinserv['account'], balance)) self.logger.info("Total to be paid {:,}".format(total_out)) if balance < total_out: self.logger.error("Payout wallet is out of funds!") self.db.session.rollback() # XXX: Add an email call here return False if total_out == 0: self.logger.info("Paying out 0 funds! Aborting...") self.db.session.rollback() return True if not simulate: self.db.session.commit() else: self.db.session.rollback() def format_pids(pids): lst = ", ".join(pids[:9]) if len(pids) > 9: return lst + "... ({} more)".format(len(pids) - 8) return lst summary = [(str(address), amount, str(format_pids(upids))) for (address, amount), upids in zip(address_payout_amounts.iteritems(), pids.itervalues())] self.logger.info( "Address payment summary\n" + tabulate(summary, headers=["Address", "Total", "Pids"], tablefmt="grid")) try: if simulate: coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" rpc_tx_obj = None res = raw_input("Would you like the simulation to associate a " "fake txid {} with these payouts? Don't do " "this on production. [y/n] ".format(coin_txid)) if res != "y": self.logger.info("Exiting") return True else: # finally run rpc call to payout coin_txid, rpc_tx_obj = self.coin_rpc.send_many( self.coin_rpc.coinserv['account'], address_payout_amounts) except CoinRPCException as e: self.logger.warn(e) new_balance = self.coin_rpc.get_balance(self.coin_rpc.coinserv['account']) if new_balance != balance: self.logger.error( "RPC error occured and wallet balance changed! Keeping the " "payout entries locked. simplecoin_rpc dump_incomplete can " "show you the details of the locked entries. If you're SURE" "a double payout hasn't occured, use simplecoin_rpc " "reset_all_locked to reset the entries.", exc_info=True) return False else: self.logger.error("RPC error occured and wallet balance didn't " "change. Unlocking payouts.") # Reset all the payouts so we can try again later for payout in payouts: payout.locked = False self.db.session.commit() return False else: # Success! Now associate the txid and unlock to allow association # with remote to occur payout_addrs = [address for address in address_payout_amounts.iterkeys()] finalized_payouts = [] for payout in payouts: if payout.address in payout_addrs: payout.locked = False payout.txid = coin_txid payout.paid_time = datetime.datetime.utcnow() finalized_payouts.append(payout) self.db.session.commit() self.logger.info("Updated {:,} (local) Payouts with txid {}" .format(len(finalized_payouts), coin_txid)) return coin_txid, rpc_tx_obj, finalized_payouts def associate_all(self, simulate=False): """ Looks at all local Payout objects (of the currency_code) that are paid and have a transaction id, and attempts to push that transaction ids and fees to the SC Payout object """ if simulate: self.logger.info('#'*20 + ' Simulation mode ' + '#'*20) payouts = (self.db.session.query(Payout). filter_by(associated=False, currency_code=self.config['currency_code']). filter(Payout.txid != None) .all()) # Build a dict keyed by txid to track payouts. txids = {} for payout in payouts: txids.setdefault(payout.txid, []) txids[payout.txid].append(payout) # Try to grab the fee for each txid tx_fees = {} for txid in txids.iterkeys(): try: tx_fees[txid] = self.coin_rpc.get_transaction(txid).fee except CoinRPCException as e: self.logger.warn('Skipping transaction with id {}, failed ' 'looking it up from the {} wallet' .format(txid, self.config['currency_code'])) continue for txid, payouts in txids.iteritems(): if simulate: self.logger.info("Attempting remote association of {:,} ids " "with txid {}".format(len(payouts), txid)) self.associate(txid, payouts, tx_fees[txid], simulate=simulate) def associate(self, txid, payouts, tx_fee, simulate=False): """ Attempt to associate Payout objects on SC with a specific transaction ID that paid them. Also post the fee incurred by the transaction. """ pids = [p.pid for p in payouts] self.logger.info("Trying to associate {:,} payouts with txid {}" .format(len(payouts), txid)) data = {'coin_txid': txid, 'pids': pids, 'tx_fee': float(tx_fee), 'currency': self.config['currency_code']} if simulate: self.logger.info('We\'re simulating, so don\'t actually post to SC') return res = self.post('associate_payouts', data=data) if res['result']: self.logger.info("Received success response from the server.") for payout in payouts: payout.associated = True payout.assoc_time = datetime.datetime.utcnow() self.db.session.commit() return True else: self.logger.error("Failed to push association information for {} " "payouts!".format(self.config['currency_code'])) return False def local_associate_locked(self, pid, tx_id, simulate=False): """ Locally associates a payout, with which is both unpaid and locked, with a TXID. """ payout = (self.db.session.query(Payout) .filter_by(txid=None, locked=True, id=pid).all()) self.logger.info("Associating payout id {} with TX ID {}" .format(payout.id, tx_id)) if simulate: self.logger.info("Just kidding, we're simulating... Exit.") return payout.txid = tx_id self.db.session.commit() return True def local_associate_all_locked(self, tx_id, simulate=False): """ Locally associates payouts for this _currency_ which are both unpaid and locked with a TXID If you want to locally associate an individual payout ID with a TX use local_associate_locked() You'll want to use this function if a payout went out and you have the txid, but for whatever reason it didn't get saved/updated in the local DB. After you've done this you'll still need to run the functions to associate everything on the remote server after. """ payouts = (self.db.session.query(Payout) .filter_by(txid=None, locked=True, currency_code=self.config['currency_code']) .all()) self.logger.info("Associating {:,} payout ids with TX ID {}" .format(len(payouts), tx_id)) if simulate: self.logger.info("Just kidding, we're simulating... Exit.") return for payout in payouts: payout.txid = tx_id self.db.session.commit() return True def confirm_trans(self, simulate=False): """ Grabs the unconfirmed transactions objects from the remote server and checks if they're confirmed. Also grabs and pushes the fees for the transaction if remote server supports it. """ self.logger.info("Attempting to grab unconfirmed {} transactions from " "SC, poking the RPC...".format(self.config['currency_code'])) try: self.coin_rpc.poke_rpc() except CoinRPCException as e: self.logger.warn( "Error occured while trying to get info from the {} RPC. Got " "{}".format(self.config['currency_code'], e)) return False res = self.get('api/transaction?__filter_by={{"confirmed":false,"currency":"{}"}}' .format(self.config['currency_code']), signed=False) if not res['success']: self.logger.error("Failure grabbing unconfirmed transactions: {}".format(res)) return if not res['objects']: self.logger.info("No transactions were returned to confirm...exiting.") return tids = [] for sc_obj in res['objects']: self.logger.debug("Connecting to coinserv to lookup confirms for {}" .format(sc_obj['txid'])) rpc_tx_obj = self.coin_rpc.get_transaction(sc_obj['txid']) if rpc_tx_obj.confirmations > self.config['min_confirms']: tids.append(sc_obj['txid']) self.logger.info("Confirmed txid {} with {} confirms" .format(sc_obj['txid'], rpc_tx_obj.confirmations)) else: self.logger.info("TX {} not yet confirmed. {}/{} confirms" .format(sc_obj['txid'], rpc_tx_obj.confirmations, self.config['min_confirms'])) if simulate: self.logger.info('We\'re simulating, so don\'t actually post to SC') return if tids: data = {'tids': tids} res = self.post('confirm_transactions', data=data) if res['result']: self.logger.info("Sucessfully confirmed transactions") # XXX: Add number print outs # XXX: Delete/archive payout row return True self.logger.error("Failed to push confirmation information") return False def get_open_trade_requests(self): """ Grabs the open trade requests from the server and prints off info about them """ try: trs = self.post('get_trade_requests')['trs'] except ConnectionError: self.logger.warn('Unable to connect to SC!', exc_info=True) return if not trs: self.logger.info("No {} trade requests returned from SC..." .format(self.config['currency_code'])) # basic checking of input try: for tr_id, currency, quantity, type in trs: assert isinstance(tr_id, int) assert isinstance(currency, basestring) assert isinstance(quantity, float) assert isinstance(type, basestring) assert type == 'buy' or 'sell' except AssertionError: self.logger.warn("Invalid TR format returned from RPC call " "get_trade_requests.", exc_info=True) return brs = [] srs = [] for tr_id, currency, quantity, type in trs[:]: tr = [tr_id, currency, quantity, type] # remove trs not for this currency if currency != self.config['currency_code']: trs.remove(tr) if type == 'sell': srs.append(tr) if type == 'buy': brs.append(tr) self.logger.info("Got {} {} sell requests from SC" .format(len(srs), self.config['currency_code'])) self.logger.info("Got {} {} buy requests from SC" .format(len(brs), self.config['currency_code'])) # Print headers = ['tr_id', 'currency', 'quantity', 'type'] print("@@ Open {} sell requests @@".format(self.config['currency_code'])) print(tabulate(srs, headers=headers, tablefmt="grid")) print("@@ Open {} buy requests @@".format(self.config['currency_code'])) print(tabulate(brs, headers=headers, tablefmt="grid")) def close_trade_request(self, tr_id, quantity, total_fees, simulate=False): if simulate: self.logger.info('#'*20 + ' Simulation mode ' + '#'*20) completed_trs = {tr_id: {'status': 6, 'quantity': str(quantity), 'fees': str(total_fees)}} if not simulate: # Post the dictionary response = self.post( 'update_trade_requests', data={'update': True, 'trs': completed_trs} ) if 'success' in response: self.logger.info( "Successfully posted {} updated trade requests to SC!" .format(len(completed_trs))) else: self.logger.warn( "Failed posting request updates! Attempted to post the " "following dictionary: {}".format(pformat(completed_trs))) else: self.logger.info( "Simulating - but would have posted the following dictionary: " "{}".format(pformat(completed_trs))) ######################################################################## # Helpful local data management + analysis methods ######################################################################## def reset_all_locked(self, simulate=False): """ Resets all locked payouts """ payouts = self.db.session.query(Payout).filter_by(locked=True) self.logger.info("Resetting {:,} payout ids".format(payouts.count())) if simulate: self.logger.info("Just kidding, we're simulating... Exit.") return payouts.update({Payout.locked: False}) self.db.session.commit() def init_db(self, simulate=False): """ Deletes all data from DB and rebuilds tables. Use carefully... """ Payout.__table__.drop(self.engine, checkfirst=True) Payout.__table__.create(self.engine, checkfirst=True) self.db.session.commit() def _tabulate(self, title, query, headers=None, data=None): """ Displays a table of payouts given a query to fetch payouts with, a title to label the table, and an optional list of columns to display """ print("@@ {} @@".format(title)) headers = headers if headers else ["pid", "user", "address", "amount_float", "associated", "locked", "trans_id"] data = [p.tabulize(headers) for p in query] if data: print(tabulate(data, headers=headers, tablefmt="grid")) else: print("-- Nothing to display --") print("") def dump_incomplete(self, unpaid_locked=True, paid_unassoc=True, unpaid_unlocked=True): """ Prints out a nice display of all incomplete payout records. """ if unpaid_locked: self.unpaid_locked() if paid_unassoc: self.paid_unassoc() if unpaid_unlocked: self.unpaid_unlocked() def unpaid_locked(self): self._tabulate( "Unpaid locked {} payouts".format(self.config['currency_code']), self.db.session.query(Payout).filter_by(txid=None, locked=True).all()) def paid_unassoc(self): self._tabulate( "Paid un-associated {} payouts".format(self.config['currency_code']), self.db.session.query(Payout).filter_by(associated=False).filter(Payout.txid != None).all()) def unpaid_unlocked(self): self._tabulate( "{} payouts ready to payout".format(self.config['currency_code']), self.db.session.query(Payout).filter_by(txid=None, locked=False).all()) def dump_complete(self): """ Prints out a nice display of all completed payout records. """ self._tabulate( "Paid + associated {} payouts".format(self.config['currency_code']), self.db.session.query(Payout).filter_by(associated=True).filter(Payout.txid != None).all()) def call(self, command, **kwargs): try: return getattr(self, command)(**kwargs) except Exception: self.logger.error("Unhandled exception calling {} with {}" .format(command, kwargs), exc_info=True) return False
class RPCClient(object): def __init__(self, config_path='/config.yml', root_suffix='/../', max_age=10): self.root = os.path.abspath(os.path.dirname(__file__) + root_suffix) self.config = current_app.config del current_app.logger.handlers[0] current_app.logger.addHandler(ch) self.serializer = TimedSerializer(self.config['rpc_signature']) self.max_age = max_age def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote(url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, signed=True, **kwargs): url = urljoin(self.config['rpc_url'], url) logger.debug("Making request to {}".format(url)) ret = getattr(requests, method)(url, timeout=270, **kwargs) if ret.status_code != 200: raise RPCException("Non 200 from remote: {}".format(ret.text)) try: logger.debug("Got {} from remote".format(ret.text.encode('utf8'))) if signed: return self.serializer.loads(ret.text, max_age or self.max_age) else: return ret.json() except BadData: current_app.logger.error("Invalid data returned from remote!", exc_info=True) raise RPCException("Invalid signature") def poke_rpc(self, conn): try: conn.getinfo() except JSONRPCException: raise RPCException("Coinserver not awake") def confirm_trans(self, simulate=False): res = self.get('api/transaction?__filter_by={"confirmed":false}', signed=False) if not res['success']: logger.error("Failure from remote: {}".format(res)) return tids = [] for obj in res['objects']: if obj['merged_type']: conn = merge_coinserv[obj['merged_type']] confirms = current_app.config['merged_cfg'][obj['merged_type']]['trans_confirmations'] else: conn = coinserv confirms = current_app.config['trans_confirmations'] logger.debug("Connecting to {} coinserv to lookup confirms for {}" .format(obj['merged_type'] or 'main', obj['txid'])) try: res = conn.gettransaction(obj['txid']) except CoinRPCException: logger.error("Unable to fetch txid {} from {} rpc server!" .format(obj['txid'], obj['merged_type'])) except Exception: logger.error("Unable to fetch txid {} from {} rpc server!" .format(obj['txid'], obj['merged_type']), exc_info=True) else: if res['confirmations'] > confirms: tids.append(obj['txid']) logger.info("Confirmed txid {} with {} confirms" .format(obj['txid'], res['confirmations'])) data = {'tids': tids} self.post('confirm_transactions', data=data) def reset_trans_file(self, fo, simulate=False): vals = json.load(fo) self.reset_trans(simulate=simulate, **vals) def associate_trans_file(self, fo, simulate=False): vals = json.load(fo) self.associate_trans(simulate=simulate, **vals) def string_to_list(self, string): return [int(i.strip()) for i in string.split(',')] def reset_trans(self, pids, bids, simulate=False): if isinstance(pids, basestring): pids = self.string_to_list(pids) if isinstance(bids, basestring): bids = self.string_to_list(bids) data = {'pids': pids, 'bids': bids, 'reset': True} logger.info("Resetting {:,} bonus ids and {:,} payout ids." .format(len(bids), len(pids))) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) self.post('update_payouts', data=data) def associate_trans(self, pids, bids, transaction_id, merged, simulate=False): if isinstance(pids, basestring): pids = self.string_to_list(pids) if isinstance(bids, basestring): bids = self.string_to_list(bids) data = {'coin_txid': transaction_id, 'pids': pids, 'bids': bids, 'merged': merged} logger.info("Associating {:,} payout ids and {:,} bonus ids with txid {}" .format(len(pids), len(bids), transaction_id)) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) if self.post('update_payouts', data=data): logger.info("Sucessfully associated!") return True logger.info("Failed to associate!") return False def proc_trans(self, simulate=False, merged=None, datadir=None): logger.info("Running payouts for merged = {}".format(merged)) if merged: conn = merge_coinserv[merged] valid_address_versions = current_app.config['merged_cfg'][merged]['address_version'] else: conn = coinserv valid_address_versions = current_app.config['address_version'] self.poke_rpc(conn) lock = True if simulate: lock = False payouts, bonus_payouts, lock_res = self.post( 'get_payouts', data={'lock': lock, 'merged': merged} ) if lock: assert lock_res pids = [t[2] for t in payouts] bids = [t[2] for t in bonus_payouts] if not len(pids) and not len(bids): logger.info("No payouts to process..") return if not simulate: backup_fname = os.path.join(os.path.abspath(datadir), 'locked_ids.{}'.format(int(time.time()))) fo = open(backup_fname, 'w') json.dump(dict(pids=pids, bids=bids), fo) fo.close() logger.info("Locked pid information stored at {0}. Call sc_rpc " "reset_trans_file {0} to reset these transactions" .format(backup_fname)) logger.info("Recieved {:,} payouts and {:,} bonus payouts from the server" .format(len(pids), len(bids))) # builds two dictionaries, one that tracks the total payouts to a user, # and another that tracks all the payout ids (pids) giving that amount # to the user totals = {} pids = {} bids = {} for user, amount, id in payouts: if get_bcaddress_version(user) in valid_address_versions: totals.setdefault(user, 0) totals[user] += amount pids.setdefault(user, []) pids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) for user, amount, id in bonus_payouts: if get_bcaddress_version(user) in valid_address_versions: totals.setdefault(user, 0) totals[user] += amount bids.setdefault(user, []) bids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) # identify the users who meet minimum payout and format for sending # to rpc users = {user: amount / float(100000000) for user, amount in totals.iteritems() if amount > current_app.config['minimum_payout']} logger.info("Trying to payout a total of {}".format(sum(users.values()))) if len(users) == 0: logger.info("Nobody has a big enough balance to pay out...") return # now we have all the users who we're going to send money. build a list # of the pids that will be being paid in this transaction committed_pids = [] for user in users: committed_pids.extend(pids.get(user, [])) committed_bids = [] for user in users: committed_bids.extend(bids.get(user, [])) logger.info("Total user payouts") logger.info(users) logger.debug("Total bonus IDs") logger.debug(bids) logger.debug("Total payout IDs") logger.debug(pids) logger.info("List of payout ids to be committed") logger.info(committed_pids) logger.info("List of bonus payout ids to be committed") logger.info(committed_bids) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) try: # now actually pay them coin_txid = payout_many(users, merged=merged) #coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" except CoinRPCException as e: if isinstance(e.error, dict) and e.error.get('message') == 'Insufficient funds': logger.error("Insufficient funds, reseting...") self.reset_trans(pids, bids) else: logger.error("Unkown RPC error, you'll need to manually reset the payouts", exc_info=True) else: associated = False try: logger.info("Got {} as txid for payout, now pushing result to server!" .format(coin_txid)) retries = 0 while retries < 5: try: if self.associate_trans(committed_pids, committed_bids, coin_txid, merged=merged): logger.info("Recieved success response from the server.") associated = True break except Exception: logger.error("Server returned failure response, retrying " "{} more times.".format(4 - retries), exc_info=True) retries += 1 time.sleep(15) finally: if not associated: backup_fname = os.path.join(os.path.abspath(datadir), 'associated_ids.{}'.format(int(time.time()))) fo = open(backup_fname, 'w') json.dump(dict(pids=committed_pids, bids=committed_bids, transaction_id=coin_txid, merged=merged), fo) fo.close() logger.info("Failed transaction_id association data stored in {0}. Call sc_rpc " "associate_trans_file {0} to retry manually".format(backup_fname))
class RPCClient(object): def __init__(self, config_path='/config.yml', root_suffix='/../', max_age=5): self.root = os.path.abspath(os.path.dirname(__file__) + root_suffix) self.config = current_app.config del current_app.logger.handlers[0] current_app.logger.addHandler(ch) self.serializer = TimedSerializer(self.config['rpc_signature']) self.max_age = max_age def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote(url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, signed=True, **kwargs): url = urljoin(self.config['rpc_url'], url) ret = getattr(requests, method)(url, **kwargs) if ret.status_code != 200: raise RPCException("Non 200 from remote: {}".format(ret.text)) try: logger.debug("Got {} from remote".format(ret.text)) if signed: return self.serializer.loads(ret.text, max_age or self.max_age) else: return ret.json() except BadData: raise RPCException("Invalid signature: {}".format(ret.text)) def poke_rpc(self, conn): try: conn.getinfo() except JSONRPCException: raise RPCException("Coinserver not awake") def confirm_trans(self, simulate=False): proc_pids = [] res = self.get('api/transaction?__filter_by={"confirmed":false}', signed=False) if not res['success']: logger.error("Failure from remote: {}".format(res)) return tids = [] for obj in res['objects']: if obj['merged_type']: conn = merge_coinserv[obj['merged_type']] confirms = current_app.config['merged_cfg'][ obj['merged_type']]['trans_confirmations'] else: conn = coinserv confirms = current_app.config['trans_confirmations'] logger.debug( "Connecting to {} coinserv to lookup confirms for {}".format( obj['merged_type'] or 'main', obj['txid'])) res = conn.gettransaction(obj['txid']) if res['confirmations'] > confirms: tids.append(obj['txid']) logger.info("Confirmed txid {} with {} confirms".format( obj['txid'], res['confirmations'])) data = {'tids': tids} self.post('confirm_transactions', data=data) def reset_trans(self, pids, bids, simulate=False): proc_pids = [] if pids: proc_pids = [int(i) for i in pids.split(',')] proc_bids = [] if bids: proc_bids = [int(i) for i in bids.split(',')] data = {'pids': proc_pids, 'bids': proc_bids, 'reset': True} logger.info("Resetting requested bids and pids") self.post('update_payouts', data=data) def validate_address(self, conn, address): ret = conn.validateaddress(address) return ret['isvalid'] def proc_trans(self, simulate=False, merged=None): logger.info("Running payouts for merged = {}".format(merged)) if merged: merged_cfg = current_app.config['merged_cfg'][merged] conn = merge_coinserv[merged] else: conn = coinserv self.poke_rpc(conn) lock = True if simulate: lock = False payouts, bonus_payouts, lock_res = self.post('get_payouts', data={ 'lock': lock, 'merged': merged }) if lock: assert lock_res pids = [t[2] for t in payouts] bids = [t[2] for t in bonus_payouts] if not simulate: logger.warn( "Locked all recieved payout ids and bonus payout ids. In " "the event of an error, run the following command to unlock" "for a retried payout.\nsc_rpc reset_trans '{}' '{}'".format( ",".join(str(p) for p in pids), ",".join(str(b) for b in bids))) if not len(pids) and not len(bids): logger.info("No payouts to process..") return logger.info( "Recieved {} payouts and {} bonus payouts from the server".format( len(pids), len(bids))) # builds two dictionaries, one that tracks the total payouts to a user, # and another that tracks all the payout ids (pids) giving that amount # to the user totals = {} pids = {} bids = {} for user, amount, id in payouts: if self.validate_address(conn, user): totals.setdefault(user, 0) totals[user] += amount pids.setdefault(user, []) pids[user].append(id) else: logger.warn( "User {} has been excluded due to invalid address".format( user)) for user, amount, id in bonus_payouts: if self.validate_address(conn, user): totals.setdefault(user, 0) totals[user] += amount bids.setdefault(user, []) bids[user].append(id) else: logger.warn( "User {} has been excluded due to invalid address".format( user)) # identify the users who meet minimum payout and format for sending # to rpc users = { user: amount / float(100000000) for user, amount in totals.iteritems() if amount > current_app.config['minimum_payout'] } logger.info("Trying to payout a total of {}".format(sum( users.values()))) if len(users) == 0: logger.info("Nobody has a big enough balance to pay out...") return # now we have all the users who we're going to send money. build a list # of the pids that will be being paid in this transaction committed_pids = [] for user in users: committed_pids.extend(pids.get(user, [])) committed_bids = [] for user in users: committed_bids.extend(bids.get(user, [])) logger.info("Total user payouts") logger.info(pprint.pformat(users)) logger.info("Total bonus IDs") logger.info(pprint.pformat(bids)) logger.info("Total payout IDs") logger.info(pprint.pformat(pids)) logger.info("List of payout ids to be committed") logger.info(committed_pids) logger.info("List of bonus payout ids to be committed") logger.info(committed_bids) if simulate: exit(0) # now actually pay them coin_txid = payout_many(users, merged=merged) #coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" logger.info("Got {} as txid for payout!".format(coin_txid)) data = { 'coin_txid': coin_txid, 'pids': committed_pids, 'bids': committed_bids, 'merged': merged } logger.info("Sending data back to confirm_payouts: " + str(data)) while True: try: if self.post('update_payouts', data=data): logger.info("Recieved success response from the server.") break else: logger.error("Server returned failure response") except Exception: logger.error("Error recieved, press enter to retry", exc_info=True) raw_input()
class RPCClient(object): def __init__(self, config_path='/config.yml', root_suffix='/../', max_age=5): self.root = os.path.abspath(os.path.dirname(__file__) + root_suffix) self.config = current_app.config del current_app.logger.handlers[0] current_app.logger.addHandler(ch) self.serializer = TimedSerializer(self.config['rpc_signature']) self.max_age = max_age def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote(url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, signed=True, **kwargs): url = urljoin(self.config['rpc_url'], url) ret = getattr(requests, method)(url, **kwargs) if ret.status_code != 200: raise RPCException("Non 200 from remote: {}".format(ret.text)) try: logger.debug("Got {} from remote".format(ret.text)) if signed: return self.serializer.loads(ret.text, max_age or self.max_age) else: return ret.json() except BadData: raise RPCException("Invalid signature: {}".format(ret.text)) def poke_rpc(self, conn): try: conn.getinfo() except JSONRPCException: raise RPCException("Coinserver not awake") def confirm_trans(self, simulate=False): proc_pids = [] res = self.get('api/transaction?__filter_by={"confirmed":false}', signed=False) if not res['success']: logger.error("Failure from remote: {}".format(res)) return tids = [] for obj in res['objects']: if obj['merged_type']: conn = merge_coinserv[obj['merged_type']] confirms = current_app.config['merged_cfg'][obj['merged_type']]['trans_confirmations'] else: conn = coinserv confirms = current_app.config['trans_confirmations'] logger.debug("Connecting to {} coinserv to lookup confirms for {}" .format(obj['merged_type'] or 'main', obj['txid'])) res = conn.gettransaction(obj['txid']) if res['confirmations'] > confirms: tids.append(obj['txid']) logger.info("Confirmed txid {} with {} confirms" .format(obj['txid'], res['confirmations'])) data = {'tids': tids} self.post('confirm_transactions', data=data) def reset_trans(self, pids, bids, simulate=False): proc_pids = [] if pids: proc_pids = [int(i) for i in pids.split(',')] proc_bids = [] if bids: proc_bids = [int(i) for i in bids.split(',')] data = {'pids': proc_pids, 'bids': proc_bids, 'reset': True} logger.info("Resetting requested bids and pids") self.post('update_payouts', data=data) def validate_address(self, conn, address): ret = conn.validateaddress(address) return ret['isvalid'] def proc_trans(self, simulate=False, merged=None): logger.info("Running payouts for merged = {}".format(merged)) if merged: merged_cfg = current_app.config['merged_cfg'][merged] conn = merge_coinserv[merged] else: conn = coinserv self.poke_rpc(conn) lock = True if simulate: lock = False payouts, bonus_payouts, lock_res = self.post( 'get_payouts', data={'lock': lock, 'merged': merged} ) if lock: assert lock_res pids = [t[2] for t in payouts] bids = [t[2] for t in bonus_payouts] if not simulate: logger.warn("Locked all recieved payout ids and bonus payout ids. In " "the event of an error, run the following command to unlock" "for a retried payout.\nsc_rpc reset_trans '{}' '{}'" .format(",".join(str(p) for p in pids), ",".join(str(b) for b in bids))) if not len(pids) and not len(bids): logger.info("No payouts to process..") return logger.info("Recieved {} payouts and {} bonus payouts from the server" .format(len(pids), len(bids))) # builds two dictionaries, one that tracks the total payouts to a user, # and another that tracks all the payout ids (pids) giving that amount # to the user totals = {} pids = {} bids = {} for user, amount, id in payouts: if self.validate_address(conn, user): totals.setdefault(user, 0) totals[user] += amount pids.setdefault(user, []) pids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) for user, amount, id in bonus_payouts: if self.validate_address(conn, user): totals.setdefault(user, 0) totals[user] += amount bids.setdefault(user, []) bids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) # identify the users who meet minimum payout and format for sending # to rpc users = {user: amount / float(100000000) for user, amount in totals.iteritems() if amount > current_app.config['minimum_payout']} logger.info("Trying to payout a total of {}".format(sum(users.values()))) if len(users) == 0: logger.info("Nobody has a big enough balance to pay out...") return # now we have all the users who we're going to send money. build a list # of the pids that will be being paid in this transaction committed_pids = [] for user in users: committed_pids.extend(pids.get(user, [])) committed_bids = [] for user in users: committed_bids.extend(bids.get(user, [])) logger.info("Total user payouts") logger.info(pprint.pformat(users)) logger.info("Total bonus IDs") logger.info(pprint.pformat(bids)) logger.info("Total payout IDs") logger.info(pprint.pformat(pids)) logger.info("List of payout ids to be committed") logger.info(committed_pids) logger.info("List of bonus payout ids to be committed") logger.info(committed_bids) if simulate: exit(0) # now actually pay them coin_txid = payout_many(users, merged=merged) #coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" logger.info("Got {} as txid for payout!".format(coin_txid)) data = {'coin_txid': coin_txid, 'pids': committed_pids, 'bids': committed_bids, 'merged': merged} logger.info("Sending data back to confirm_payouts: " + str(data)) while True: try: if self.post('update_payouts', data=data): logger.info("Recieved success response from the server.") break else: logger.error("Server returned failure response") except Exception: logger.error("Error recieved, press enter to retry", exc_info=True) raw_input()
class RPCClient(object): def __init__(self, config_path='/config.yml', root_suffix='/../', max_age=10): self.root = os.path.abspath(os.path.dirname(__file__) + root_suffix) self.config = current_app.config del current_app.logger.handlers[0] current_app.logger.addHandler(ch) self.serializer = TimedSerializer(self.config['rpc_signature']) self.max_age = max_age def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote(url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, signed=True, **kwargs): url = urljoin(self.config['rpc_url'], url) logger.debug("Making request to {}".format(url)) ret = getattr(requests, method)(url, timeout=270, **kwargs) if ret.status_code != 200: raise RPCException("Non 200 from remote: {}".format(ret.text)) try: logger.debug("Got {} from remote".format(ret.text.encode('utf8'))) if signed: return self.serializer.loads(ret.text, max_age or self.max_age) else: return ret.json() except BadData: current_app.logger.error("Invalid data returned from remote!", exc_info=True) raise RPCException("Invalid signature") def poke_rpc(self, conn): try: conn.getinfo() except JSONRPCException: raise RPCException("Coinserver not awake") def confirm_trans(self, simulate=False): res = self.get('api/transaction?__filter_by={"confirmed":false}', signed=False) if not res['success']: logger.error("Failure from remote: {}".format(res)) return tids = [] for obj in res['objects']: if obj['merged_type']: conn = merge_coinserv[obj['merged_type']] confirms = current_app.config['merged_cfg'][ obj['merged_type']]['trans_confirmations'] else: conn = coinserv confirms = current_app.config['trans_confirmations'] logger.debug( "Connecting to {} coinserv to lookup confirms for {}".format( obj['merged_type'] or 'main', obj['txid'])) try: res = conn.gettransaction(obj['txid']) except CoinRPCException: logger.error( "Unable to fetch txid {} from {} rpc server!".format( obj['txid'], obj['merged_type'])) except Exception: logger.error( "Unable to fetch txid {} from {} rpc server!".format( obj['txid'], obj['merged_type']), exc_info=True) else: if res['confirmations'] > confirms: tids.append(obj['txid']) logger.info("Confirmed txid {} with {} confirms".format( obj['txid'], res['confirmations'])) data = {'tids': tids} self.post('confirm_transactions', data=data) def reset_trans_file(self, fo, simulate=False): vals = json.load(fo) self.reset_trans(simulate=simulate, **vals) def associate_trans_file(self, fo, simulate=False): vals = json.load(fo) self.associate_trans(simulate=simulate, **vals) def string_to_list(self, string): return [int(i.strip()) for i in string.split(',')] def reset_trans(self, pids, bids, simulate=False): if isinstance(pids, basestring): pids = self.string_to_list(pids) if isinstance(bids, basestring): bids = self.string_to_list(bids) data = {'pids': pids, 'bids': bids, 'reset': True} logger.info("Resetting {:,} bonus ids and {:,} payout ids.".format( len(bids), len(pids))) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) self.post('update_payouts', data=data) def associate_trans(self, pids, bids, transaction_id, merged, simulate=False): if isinstance(pids, basestring): pids = self.string_to_list(pids) if isinstance(bids, basestring): bids = self.string_to_list(bids) data = { 'coin_txid': transaction_id, 'pids': pids, 'bids': bids, 'merged': merged } logger.info( "Associating {:,} payout ids and {:,} bonus ids with txid {}". format(len(pids), len(bids), transaction_id)) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) if self.post('update_payouts', data=data): logger.info("Sucessfully associated!") return True logger.info("Failed to associate!") return False def proc_trans(self, simulate=False, merged=None, datadir=None): logger.info("Running payouts for merged = {}".format(merged)) if merged: conn = merge_coinserv[merged] valid_address_versions = current_app.config['merged_cfg'][merged][ 'address_version'] else: conn = coinserv valid_address_versions = current_app.config['address_version'] self.poke_rpc(conn) lock = True if simulate: lock = False payouts, bonus_payouts, lock_res = self.post('get_payouts', data={ 'lock': lock, 'merged': merged }) if lock: assert lock_res pids = [t[2] for t in payouts] bids = [t[2] for t in bonus_payouts] if not len(pids) and not len(bids): logger.info("No payouts to process..") return if not simulate: backup_fname = os.path.join( os.path.abspath(datadir), 'locked_ids.{}'.format(int(time.time()))) fo = open(backup_fname, 'w') json.dump(dict(pids=pids, bids=bids), fo) fo.close() logger.info( "Locked pid information stored at {0}. Call sc_rpc " "reset_trans_file {0} to reset these transactions".format( backup_fname)) logger.info( "Recieved {:,} payouts and {:,} bonus payouts from the server". format(len(pids), len(bids))) # builds two dictionaries, one that tracks the total payouts to a user, # and another that tracks all the payout ids (pids) giving that amount # to the user totals = {} pids = {} bids = {} for user, amount, id in payouts: if get_bcaddress_version(user) in valid_address_versions: totals.setdefault(user, 0) totals[user] += amount pids.setdefault(user, []) pids[user].append(id) else: logger.warn( "User {} has been excluded due to invalid address".format( user)) for user, amount, id in bonus_payouts: if get_bcaddress_version(user) in valid_address_versions: totals.setdefault(user, 0) totals[user] += amount bids.setdefault(user, []) bids[user].append(id) else: logger.warn( "User {} has been excluded due to invalid address".format( user)) # identify the users who meet minimum payout and format for sending # to rpc users = { user: amount / float(100000000) for user, amount in totals.iteritems() if amount > current_app.config['minimum_payout'] } logger.info("Trying to payout a total of {}".format(sum( users.values()))) if len(users) == 0: logger.info("Nobody has a big enough balance to pay out...") return # now we have all the users who we're going to send money. build a list # of the pids that will be being paid in this transaction committed_pids = [] for user in users: committed_pids.extend(pids.get(user, [])) committed_bids = [] for user in users: committed_bids.extend(bids.get(user, [])) logger.info("Total user payouts") logger.info(users) logger.debug("Total bonus IDs") logger.debug(bids) logger.debug("Total payout IDs") logger.debug(pids) logger.info("List of payout ids to be committed") logger.info(committed_pids) logger.info("List of bonus payout ids to be committed") logger.info(committed_bids) if simulate: logger.info("Just kidding, we're simulating... Exit.") exit(0) try: # now actually pay them coin_txid = payout_many(users, merged=merged) #coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" except CoinRPCException as e: if isinstance( e.error, dict) and e.error.get('message') == 'Insufficient funds': logger.error("Insufficient funds, reseting...") self.reset_trans(pids, bids) else: logger.error( "Unkown RPC error, you'll need to manually reset the payouts", exc_info=True) else: associated = False try: logger.info( "Got {} as txid for payout, now pushing result to server!". format(coin_txid)) retries = 0 while retries < 5: try: if self.associate_trans(committed_pids, committed_bids, coin_txid, merged=merged): logger.info( "Recieved success response from the server.") associated = True break except Exception: logger.error( "Server returned failure response, retrying " "{} more times.".format(4 - retries), exc_info=True) retries += 1 time.sleep(15) finally: if not associated: backup_fname = os.path.join( os.path.abspath(datadir), 'associated_ids.{}'.format(int(time.time()))) fo = open(backup_fname, 'w') json.dump( dict(pids=committed_pids, bids=committed_bids, transaction_id=coin_txid, merged=merged), fo) fo.close() logger.info( "Failed transaction_id association data stored in {0}. Call sc_rpc " "associate_trans_file {0} to retry manually".format( backup_fname))
def check_token_password(self, token): serializer = TimedSerializer(current_app.config['SECRET_KEY']) full_token = '"' + self.email + '".' + token return serializer.loads(full_token, max_age=1800)
def get_email_confirmation_token_info(token, max_age=1800): s = TimedSerializer(current_app.secret_key, "user_id_email_confirm") try: return s.loads(token, max_age=max_age) except Exception as e: return None
class Auth_webSilvia(Auth_Base): def __init__(self, config): self.signer = TimedSerializer(config['shared_secret']) super(Auth_webSilvia, self).__init__(config) # This functions returns which credentials we need to retrieve # to get all of the requested_attributes def get_credentials(self, requested_attributes): credentials_to_request = self.config['always_retrieve'] if '/' in self.config['username_mapping']: credential, _ = self.config['username_mapping'].split('/', 1) credentials_to_request.append(credential) for requested_attribute in requested_attributes: if requested_attribute in self.config['attribute_mapping'].keys(): if '/' in self.config['attribute_mapping'][ requested_attribute]: credential, _ = self.config['attribute_mapping'][ requested_attribute].split('/', 1) credentials_to_request.append(credential) credentials_request = {} for credential_to_request in credentials_to_request: if credential_to_request in self.config['known_credentials']: credentials_request[credential_to_request] = self.config[ 'known_credentials'][credential_to_request] else: # We do not know the paths for this credential... logger.error( 'Credential was mapped to, but could not be found: %s', credential_to_request) return credentials_request # Do the actual authentication # Whatever you do, make sure to pass request.transaction_id either in GET # or POST in the field 'transaction'. # The requested_attributes can be used in authentication modules that need # to know up-front which attributes are going to be requested. Any # attributes not requested here MAY raise a NotRequestedAttributeError on # attempt to retrieval, but it's also perfectly valid for the auth module # to just return the value if it can retrieve it. # Return True when authentication was successful # Return False when authentication was cancelled # Anything else will be returned to Flask as view result def authenticate(self, login_target, form_url, requested_attributes=[]): if request.method == 'POST': # We are returning from webSilvia! result = request.form['result'] result = self.signer.loads(result) user = {} needed_credentials = self.config['required_credentials'] for credential in result['verified']: if result['verified'][credential]['status'] == 'OK': if result['verified'][credential]['expiry'] >= time.time(): if credential in needed_credentials: needed_credentials.remove(credential) user[credential] = result['verified'][credential][ 'attributes'] else: # Attribute no longer valid logger.info('Credential expired: %s', result['credentials'][credential]) else: # Attribute status != OK logger.info('Credential not status=OK: %s', result['credentials'][credential]) if len(needed_credentials) > 0: return False self.save_success(user) return True else: # Build the request for webSilvia websilvia_request = { 'protocol': 'request-1', 'return_url': '%s?transaction=%s' % (form_url, request.transaction_id), 'token': request.transaction_id, 'nonce': time.time(), 'to_verify': self.get_credentials(requested_attributes), 'to_issue': {} } websilvia_request = self.signer.dumps(websilvia_request) return render_template('webSilvia.html', request=websilvia_request, requestor_id=self.config['requestor_id'], websilvia_url=self.config['websilvia_url']) def follow_mapping(self, mapping, user): if '/' not in mapping: return mapping else: credential, attribute = mapping.split('/', 1) if credential in self._user.keys(): if attribute in self._user[credential]: return self._user[credential][attribute] else: raise NotRequestedAttributeError() else: raise NotRequestedAttributeError() def get_username(self): if not self.logged_in(): raise UnauthorizedError return self.follow_mapping(self.config['username_mapping'], self._user) def get_attribute(self, attribute): if not self.logged_in(): raise UnauthorizedError attribute = attribute.__str__() if attribute in self.config['attribute_mapping']: return self.follow_mapping( self.config['attribute_mapping'][attribute], self._user) else: raise UnknownAttributeError() def get_groups(self): return [] def get_clas(self): return [] def used_multi_factor(self): return True def used_multi_factor_physical(self): return True def used_phishing_resistant(self): return False
class RPCClient(object): def __init__(self, config_path='/config.yml', root_suffix='/../', max_age=5): self.root = os.path.abspath(os.path.dirname(__file__) + root_suffix) self.config = current_app.config del current_app.logger.handlers[0] current_app.logger.addHandler(ch) self.serializer = TimedSerializer(self.config['rpc_signature']) self.max_age = max_age def post(self, url, *args, **kwargs): if 'data' not in kwargs: kwargs['data'] = '' kwargs['data'] = self.serializer.dumps(kwargs['data']) return self.remote(url, 'post', *args, **kwargs) def get(self, url, *args, **kwargs): return self.remote(url, 'get', *args, **kwargs) def remote(self, url, method, max_age=None, **kwargs): url = urljoin(self.config['rpc_url'], url) ret = getattr(requests, method)(url, **kwargs) if ret.status_code != 200: raise RPCException("Non 200 from remote") try: return self.serializer.loads(ret.text, max_age or self.max_age) except BadData: raise RPCException("Invalid signature: {}".format(ret.text)) def poke_rpc(self): try: coinserv.getinfo() except JSONRPCException: raise RPCException("Coinserver not awake") def proc_trans(self, simulate=False): self.poke_rpc() payouts, bonus_payouts = self.post('get_payouts') pids = [t[2] for t in payouts] bids = [t[2] for t in bonus_payouts] logger.info("Recieved {} payouts and {} bonus payouts from the server" .format(len(pids), len(bids))) if not len(pids) and not len(bids): logger.info("No payouts to process..") return # builds two dictionaries, one that tracks the total payouts to a user, # and another that tracks all the payout ids (pids) giving that amount # to the user totals = {} pids = {} bids = {} for user, amount, id in payouts: if user.startswith('D'): totals.setdefault(user, 0) totals[user] += amount pids.setdefault(user, []) pids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) for user, amount, id in bonus_payouts: if user.startswith('D'): totals.setdefault(user, 0) totals[user] += amount bids.setdefault(user, []) bids[user].append(id) else: logger.warn("User {} has been excluded due to invalid address" .format(user)) # identify the users who meet minimum payout and format for sending # to rpc users = {user: amount / float(100000000) for user, amount in totals.iteritems() if amount > current_app.config['minimum_payout']} logger.info("Trying to payout a total of {}".format(sum(users.values()))) if len(users) == 0: logger.info("Nobody has a big enough balance to pay out...") return # now we have all the users who we're going to send money. build a list # of the pids that will be being paid in this transaction committed_pids = [] for user in users: committed_pids.extend(pids.get(user, [])) committed_bids = [] for user in users: committed_bids.extend(bids.get(user, [])) logger.info("Total user payouts") logger.info(pprint.pformat(users)) logger.info("Total bonus IDs") logger.info(pprint.pformat(bids)) logger.info("Total payout IDs") logger.info(pprint.pformat(pids)) logger.info("List of payout ids to be committed") logger.info(committed_pids) logger.info("List of bonus payout ids to be committed") logger.info(committed_bids) if simulate: exit(0) # now actually pay them coin_txid = payout_many(users) #coin_txid = "1111111111111111111111111111111111111111111111111111111111111111" logger.info("Got {} as txid for payout!".format(coin_txid)) data = {'coin_txid': coin_txid, 'pids': committed_pids, 'bids': committed_bids} logger.info("Sending data back to confirm_payouts: " + str(data)) while True: try: if self.post('confirm_payouts', data=data): logger.info("Recieved success response from the server.") break else: logger.error("Server returned failure response") except Exception: logger.error("Error recieved, press enter to retry", exc_info=True) raw_input()
class Auth_webSilvia(Auth_Base): def __init__(self, config): self.signer = TimedSerializer(config['shared_secret']) super(Auth_webSilvia, self).__init__(config) # This functions returns which credentials we need to retrieve # to get all of the requested_attributes def get_credentials(self, requested_attributes): credentials_to_request = self.config['always_retrieve'] if '/' in self.config['username_mapping']: credential, _ = self.config['username_mapping'].split('/', 1) credentials_to_request.append(credential) for requested_attribute in requested_attributes: if requested_attribute in self.config['attribute_mapping'].keys(): if '/' in self.config['attribute_mapping'][requested_attribute]: credential, _ = self.config['attribute_mapping'][requested_attribute].split('/', 1) credentials_to_request.append(credential) credentials_request = {} for credential_to_request in credentials_to_request: if credential_to_request in self.config['known_credentials']: credentials_request[credential_to_request] = self.config['known_credentials'][credential_to_request] else: # We do not know the paths for this credential... logger.error('Credential was mapped to, but could not be found: %s', credential_to_request) return credentials_request # Do the actual authentication # Whatever you do, make sure to pass request.transaction_id either in GET # or POST in the field 'transaction'. # The requested_attributes can be used in authentication modules that need # to know up-front which attributes are going to be requested. Any # attributes not requested here MAY raise a NotRequestedAttributeError on # attempt to retrieval, but it's also perfectly valid for the auth module # to just return the value if it can retrieve it. # Return True when authentication was successful # Return False when authentication was cancelled # Anything else will be returned to Flask as view result def authenticate(self, login_target, form_url, requested_attributes=[]): if request.method == 'POST': # We are returning from webSilvia! result = request.form['result'] result = self.signer.loads(result) user = {} needed_credentials = self.config['required_credentials'] for credential in result['verified']: if result['verified'][credential]['status'] == 'OK': if result['verified'][credential]['expiry'] >= time.time(): if credential in needed_credentials: needed_credentials.remove(credential) user[credential] = result['verified'][credential]['attributes'] else: # Attribute no longer valid logger.info('Credential expired: %s', result['credentials'][credential]) else: # Attribute status != OK logger.info('Credential not status=OK: %s', result['credentials'][credential]) if len(needed_credentials) > 0: return False self.save_success(user) return True else: # Build the request for webSilvia websilvia_request = {'protocol': 'request-1', 'return_url': '%s?transaction=%s' % (form_url, request.transaction_id), 'token': request.transaction_id, 'nonce': time.time(), 'to_verify': self.get_credentials(requested_attributes), 'to_issue': {}} websilvia_request = self.signer.dumps(websilvia_request) return render_template('webSilvia.html', request=websilvia_request, requestor_id=self.config['requestor_id'], websilvia_url=self.config['websilvia_url']) def follow_mapping(self, mapping, user): if '/' not in mapping: return mapping else: credential, attribute = mapping.split('/', 1) if credential in self._user.keys(): if attribute in self._user[credential]: return self._user[credential][attribute] else: raise NotRequestedAttributeError() else: raise NotRequestedAttributeError() def get_username(self): if not self.logged_in(): raise UnauthorizedError return self.follow_mapping(self.config['username_mapping'], self._user) def get_attribute(self, attribute): if not self.logged_in(): raise UnauthorizedError attribute = attribute.__str__() if attribute in self.config['attribute_mapping']: return self.follow_mapping(self.config['attribute_mapping'][attribute], self._user) else: raise UnknownAttributeError() def get_groups(self): return [] def get_clas(self): return [] def used_multi_factor(self): return True def used_multi_factor_physical(self): return True def used_phishing_resistant(self): return False
def update_transactions(): """ Used as a response from an rpc payout system. This will either reset the locked status of a list of transactions upon failure on the remote side, or create a new CoinTransaction object and link it to the transactions to signify that the transaction has been processed. Both request and response are signed. """ s = TimedSerializer(current_app.config['rpc_signature']) data = s.loads(request.data) # basic checking of input try: if 'coin_txid' in data: assert len(data['coin_txid']) == 64 else: assert 'reset' in data assert isinstance(data['reset'], bool) assert isinstance(data['pids'], list) assert isinstance(data['bids'], list) for id in data['pids']: assert isinstance(id, int) for id in data['bids']: assert isinstance(id, int) except AssertionError: current_app.logger.warn("Invalid data passed to confirm", exc_info=True) abort(400) if 'coin_txid' in data: with Benchmark("Associating payout transaction ids"): merged_type = data.get('merged', None) coin_trans = Transaction.create(data['coin_txid'], merged_type=merged_type) db.session.flush() user_amounts = {} user_counts = {} for payout in Payout.query.filter(Payout.id.in_(data['pids'])): user_counts.setdefault(payout.user, 0) user_amounts.setdefault(payout.user, 0) user_amounts[payout.user] += payout.amount user_counts[payout.user] += 1 for payout in BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])): user_counts.setdefault(payout.user, 0) user_amounts.setdefault(payout.user, 0) user_amounts[payout.user] += payout.amount user_counts[payout.user] += 1 for user in user_counts: TransactionSummary.create( coin_trans.txid, user, user_amounts[user], user_counts[user]) if data['pids']: Payout.query.filter(Payout.id.in_(data['pids'])).update( {Payout.transaction_id: coin_trans.txid}, synchronize_session=False) if data['bids']: BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])).update( {BonusPayout.transaction_id: coin_trans.txid}, synchronize_session=False) db.session.commit() elif data['reset']: with Benchmark("Resetting {:,} payouts and {:,} bonus payouts locked status" .format(len(data['pids']), len(data['bids']))): if data['pids']: Payout.query.filter(Payout.id.in_(data['pids'])).update( {Payout.locked: False}, synchronize_session=False) if data['bids']: BonusPayout.query.filter(BonusPayout.id.in_(data['bids'])).update( {BonusPayout.locked: False}, synchronize_session=False) db.session.commit() return s.dumps(dict(success=True, result="Successfully reset")) return s.dumps(True)