def process_invoice_data(invoice_data): log.debug('process_invoice_data', invoice_data=invoice_data) with session_scope(echo=True) as db_session: invoice = db_session.query(Invoices).filter( Invoices.id == invoice_data['id']).one() invoice.status = invoice_data['status'] invoice.data = invoice_data
def payment_notification(): r = request.get_json() log.debug('invoice_notification', request=r, session=session) if 'data' in r: r = r['data'] app.payment_processor.process_invoice_data(r) return {}
def get(self, login: str) -> dict: variables = {'userLogin': login} json_object = {'query': user_graphql_query, 'variables': variables} log.debug('getting user', json_object=json_object) r = self.graphql_post(json_object=json_object) user = r.json()['data']['user'] return user
def generate_invoice(self, bounty_id: str, recipient_user_id: str): with session_scope() as db_session: bounty: Bounties = db_session.query(Bounties).filter(Bounties.id == bounty_id).one() recipient: Users = db_session.query(Users).filter(Users.id == recipient_user_id).one() if recipient.btcpay_client is None: flash(f'{recipient.best_name} does not have BTCPay configured here. Elsewhere they may have other ways ' f'of receiving payments (Patreon, static address, etc).') return redirect(url_for('bounties-payable.index_view')) try: recipient_btcpay = RecipientBTCPay(client=recipient.btcpay_client) invoice_data = recipient_btcpay.get_pull_request_invoice( amount=bounty.amount, bounty_id=bounty_id, pull_request_number=bounty.pull_request.number ) invoice_model = Invoices() invoice_model.bounty_id = bounty.id invoice_model.id = invoice_data['id'] invoice_model.status = invoice_data['status'] invoice_model.url = invoice_data['url'] invoice_model.recipient_user_id = recipient_user_id invoice_model.payer_user_id = bounty.payer_user_id db_session.add(invoice_model) return redirect(invoice_model.url) except RequestException as e: log.debug('RequestException', exception=e, request=e.request, response=e.response) try: r: Response = e.response flash(f'{r.status_code} - {r.text}', category='error') except AttributeError as e: flash('Request error') return redirect(url_for('users.index_view'))
def graphql_post(self, json_object: dict): log.debug('graphql post', api_url=self.api_url, json=json_object) r = requests.post(self.api_url + 'graphql', auth=self.auth, headers=self.dev_preview_headers, json=json_object) r.raise_for_status() return r
def get_url(): pg_url = URL(drivername='postgresql+psycopg2', username=os.environ['POSTGRES_USER'], password=os.environ['POSTGRES_PASSWORD'], host=os.environ['PGHOST'], port=os.environ['PGPORT'], database=os.environ['POSTGRES_DB']) log.debug('get PG url', pg_url=pg_url) return pg_url
def get_query(self): log.debug('get_query', request=request.args) if request.args.get('sort'): return super().get_query() query = ( self.session.query(self.model) .order_by(self.model.is_high_priority.asc().nullslast()) ) if self._get_list_extra_args().sort is None: query = query.order_by(self.model.last_commit_pushed_date.desc().nullslast()) return query
def on_model_change(self, form, model: Bounties, is_created: bool): model.id = uuid4().hex model.published_at = datetime.utcnow() model.payer_user_id = current_user.id model.recipient_user_id = model.pull_request.author_id with session_scope() as session: total_bounty_amount = (session.query( coalesce(func.sum(Bounties.amount), 0)).filter( Bounties.pull_request_id == model.pull_request.id).one() )[0] log.debug('total_satoshis', total_bounty_amount=total_bounty_amount) model.pull_request.total_bounty_amount = total_bounty_amount + model.amount
def invoices_formatter(view, context, model, name): invoices: List[Invoices] = getattr(model, name) paid_invoices = [i for i in invoices if i.status in ('paid', 'complete')] unpaid_invoices = [i for i in invoices if i.status not in ('paid', 'complete')] output = '' for paid_invoice in paid_invoices: output += '<div style="white-space: nowrap; overflow: hidden;">' \ '<span class="label label-success">{invoice_description}</span>'.format( invoice_description=f'{paid_invoice.id} {paid_invoice.status}') if unpaid_invoices: output += f'<div style="white-space: nowrap; overflow: hidden;">{len(unpaid_invoices)} unpaid invoices</div>' log.debug('invoices_formatter', invoices=invoices, output=output) return Markup(output)
def update(self): with session_scope() as session: try: record = ( session .query(PullRequests.updated_at) .order_by(PullRequests.updated_at.desc()) .limit(1) .one() ) from_date = record.updated_at except NoResultFound: from_date = datetime(2009, 1, 1) log.debug('Updating PRs starting from', from_date=from_date) self.update_all(newer_than=from_date)
def update_pull_requests(): polling_data = PollingData('github') try: if polling_data.is_polling(): raise Exception('GitHub is already being polled') polling_data.start() with TemporaryDirectory() as temporary_directory_path: pull_requests_data = PullRequestsData( 'bitcoin', 'bitcoin', temporary_directory_path) pull_requests_data.update() except Exception as e: log.error('polling exception', exc_info=e) tb = traceback.format_exc() email.notify('Polling exception\n\n' + tb) else: log.debug('Successful poll') finally: polling_data.stop()
def github_logged_in(github_blueprint, token): if not token: flash("Failed to log in.", category="error") return redirect(url_for("github.login")) user_resp = github_blueprint.session.get("/user") log.debug('user response', resp=user_resp.json()) emails_resp = github_blueprint.session.get("/user/emails") log.debug('user emails response', resp=emails_resp.json()) if not emails_resp.ok: log.error('github_logged_in error', resp=emails_resp.json(), token=token) msg = "Failed to fetch user info." flash(msg, category="error") return False info = user_resp.json() user_id = info["node_id"] email = [e for e in emails_resp.json() if e['primary']][0]['email'] with session_scope() as db_session: try: user = db_session.query(Users).filter( Users.id == user_id).one() except NoResultFound: user = Users(id=user_id) db_session.add(user) user.is_active = True user.email = email try: db_session.query(OAuth).filter_by( provider=github_blueprint.name, provider_user_id=user_id).one() except NoResultFound: oauth = OAuth(provider=github_blueprint.name, provider_user_id=user_id, user_id=user_id, token=token) db_session.add(oauth) login_user(user) flash("Successfully signed in.") return False
def get_pull_request_invoice(self, amount: int, bounty_id: str, pull_request_number: int): notification_url = url_for('payment_notification', _external=True) if 'localhost' in notification_url: notification_url = notification_url.replace('localhost', 'webapp') payload = { 'price': amount / COIN, 'currency': 'BTC', 'orderId': bounty_id, 'itemDesc': f'Payment for pull request {pull_request_number}', 'redirectURL': url_for('bounties-payable.index_view', _external=True), 'notificationURL': notification_url, 'extendedNotifications': True } log.debug('btcpay client request', payload=payload) invoice_data = self.client.create_invoice(payload=payload) log.debug('btcpay server response', invoice_data=invoice_data) return invoice_data
def get_all(self, newer_than: datetime, state: PullRequestState = None, limit: int = None): variables = {} received = 0 ends_at = newer_than while limit is None or received < limit: if limit is None: variables['prFirst'] = self.MAX_PRS else: variables['prFirst'] = min(limit - received, self.MAX_PRS) if state is not None: variables['prState'] = state.value formatted_ends_at = ends_at.replace(microsecond=0).astimezone(pytz.utc).isoformat() variables['searchQuery'] = f'type:pr updated:>={formatted_ends_at} repo:bitcoin/bitcoin sort:updated-asc' log.debug('Variables for graphql pull requests query', variables=variables) json_object = { 'query': pull_requests_graphql_query, 'variables': variables } data = self.graphql_post(json_object=json_object).json() search_data = data['data']['search'] pull_requests_graphql_data = search_data['edges'] results_count = len(search_data['edges']) log.debug( 'response from github graphql', results_count=results_count ) if not results_count: break starts_at = pull_requests_graphql_data[0]['node']['updatedAt'] previous_ends_at = ends_at ends_at = dateutil.parser.parse(pull_requests_graphql_data[-1]['node']['updatedAt']) if previous_ends_at == ends_at: break log.debug( 'Pull requests fetched', starts_at=starts_at, ends_at=ends_at ) pull_requests_graphql_data = [r['node'] for r in pull_requests_graphql_data if r['node']] for pull_request_graphql in pull_requests_graphql_data: validated_pull_request_data = pull_request_schema.load(pull_request_graphql) self.parse_into_queue(validated_pull_request_data) if limit is not None and received == limit: break received += 1 self.flush_queue_to_database()
def send_tweet(pull_request_number=None): with session_scope() as session: now = datetime.datetime.utcnow() yesterday = now - datetime.timedelta(days=1) twitter = Twython(os.environ['BTC_TWITTER_APP_KEY'], os.environ['BTC_TWITTER_APP_SECRET'], os.environ['BTC_TWITTER_OAUTH_TOKEN'], os.environ['BTC_TWITTER_OAUTH_TOKEN_SECRET']) if pull_request_number: next_pull_request = (session.query(PullRequests).filter( PullRequests.number == pull_request_number).first()) else: next_pull_request = (session.query(PullRequests).order_by( PullRequests.merged_at.asc()).filter( PullRequests.merged_at > yesterday).filter( PullRequests.tweet_id.is_(None)).filter( PullRequests.merged_at.isnot(None)).first()) if next_pull_request is None: log.debug('No pull requests found.') return commits_url = 'https://api.github.com/repos/bitcoin/bitcoin/commits' params = {'author': next_pull_request.author.login} response = requests.get(commits_url, params=params) log.debug('github response', response=response) response_json = response.json() author_name = next_pull_request.author.name or next_pull_request.author.login if len(response_json) > 1 and next_pull_request.number != 14802: status = 'Merged PR from {0}: {1} {2}' \ .format(author_name, next_pull_request.title, next_pull_request.html_url) else: status = ''' {0}'s first merged PR: {1} Congratulations! 🎉🍾🎆 '''.format(author_name, next_pull_request.html_url) log.debug('tweet status', status=status) tweet = twitter.update_status(status=status) log.debug('tweet', tweet=tweet) new_tweet = Tweets() new_tweet.id = tweet['id'] new_tweet.pull_request_id = next_pull_request.number session.add(new_tweet) next_pull_request.tweet_id = tweet['id']
log.debug('github response', response=response) response_json = response.json() author_name = next_pull_request.author.name or next_pull_request.author.login if len(response_json) > 1 and next_pull_request.number != 14802: status = 'Merged PR from {0}: {1} {2}' \ .format(author_name, next_pull_request.title, next_pull_request.html_url) else: status = ''' {0}'s first merged PR: {1} Congratulations! 🎉🍾🎆 '''.format(author_name, next_pull_request.html_url) log.debug('tweet status', status=status) tweet = twitter.update_status(status=status) log.debug('tweet', tweet=tweet) new_tweet = Tweets() new_tweet.id = tweet['id'] new_tweet.pull_request_id = next_pull_request.number session.add(new_tweet) next_pull_request.tweet_id = tweet['id'] if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Tweet merged pull request') parser.add_argument('-p', dest='pr_number', type=int, default=None) args = parser.parse_args() log.debug(f'sending tweet for {args.pr_number}') send_tweet(pull_request_number=args.pr_number)
or 'pull_request' in pr.keys()): pull_request_numbers.add(pr['number']) pr_data = PullRequestsData(repository_path=self.repo.path, repository_name=self.repo.name) for number in pull_request_numbers: pr_data.update(number=number) if __name__ == '__main__': import os repository_path = 'bitcoin' repository_name = 'bitcoin' log.debug('Running pull request events update script', path=os.path.realpath(__file__), repository_name=repository_name, repository_path=repository_path) create_or_update_database() pr_events = PullRequestEvents(repository_path=repository_path, repository_name=repository_name) pr_data = PullRequestsData(repository_path=repository_path, repository_name=repository_name) # polling_data = PollingData(repository_path=repository_path, # repository_name=repository_name) while True: pr_events.get() # polling_data.update(last_event=True) sleep_time = (datetime.utcnow() - pr_events.rate_limit_reset ).seconds / pr_events.rate_limit_remaining time.sleep(math.ceil(sleep_time) + 5)
def update_all(self, newer_than: datetime, state: PullRequestState = None, limit: int = None): log.debug('update_all', state=state, limit=limit, newer_than=newer_than) self.get_all(state=state, limit=limit, newer_than=newer_than)
def insert_pull_requests(self): with session_scope() as db_session: missing_authors = db_session.execute( """ SELECT DISTINCT epr.data ->> 'author_login' FROM etl_data epr LEFT OUTER JOIN users authors ON epr.data ->> 'author_login' = authors.login WHERE authors.id IS NULL; """ ).fetchall() if missing_authors: log.debug('missing_authors', missing_authors=missing_authors, count=len(missing_authors)) for author in missing_authors: login = author[0] if login is None: continue user_data = self.users_data.get(login) self.users_data.upsert(user_data) with session_scope() as db_session: db_session.execute(""" WITH etl_data AS ( SELECT DISTINCT epr.data ->> 'id' AS id, (epr.data ->> 'repository_id')::int AS repository_id, author.id AS author_id, (epr.data ->> 'number')::int AS "number", epr.data ->> 'state' AS "state", epr.data ->> 'title' AS title, (epr.data ->> 'createdAt')::timestamp with time zone AS created_at, (epr.data ->> 'updatedAt')::timestamp with time zone AS updated_at, (epr.data ->> 'is_high_priority')::timestamp with time zone AS is_high_priority, (epr.data ->> 'added_to_high_priority')::timestamp with time zone AS added_to_high_priority, (epr.data ->> 'removed_from_high_priority')::timestamp with time zone AS removed_from_high_priority, (epr.data ->> 'additions')::int AS additions, (epr.data ->> 'deletions')::int AS deletions, epr.data ->> 'mergeable' AS mergeable, epr.data ->> 'last_commit_state' AS last_commit_state, epr.data ->> 'last_commit_state_description' AS last_commit_state_description, epr.data ->> 'last_commit_short_hash' AS last_commit_short_hash, (epr.data ->> 'last_commit_pushed_date')::timestamp with time zone AS last_commit_pushed_date, epr.data ->> 'bodyText' AS body, (epr.data ->> 'mergedAt')::timestamp with time zone AS merged_at, (epr.data ->> 'closedAt')::timestamp with time zone AS closed_at, (epr.data ->> 'commit_count')::int AS commit_count FROM etl_data epr LEFT OUTER JOIN users author ON epr.data ->> 'author_login' = author.login ) INSERT INTO pull_requests (id, repository_id, author_id, "number", "state", title, created_at, updated_at, is_high_priority, added_to_high_priority, removed_from_high_priority, additions, deletions, mergeable, last_commit_state, last_commit_state_description, last_commit_short_hash, last_commit_pushed_date, body, merged_at, closed_at, commit_count) SELECT * FROM etl_data ON CONFLICT ON CONSTRAINT pull_requests_unique_constraint DO UPDATE SET repository_id = excluded.repository_id, author_id = excluded.author_id, "number" = excluded.number, "state" = excluded.state, title = excluded.title, created_at = excluded.created_at, updated_at = excluded.updated_at, is_high_priority = excluded.is_high_priority, added_to_high_priority = excluded.added_to_high_priority, removed_from_high_priority = excluded.removed_from_high_priority, additions = excluded.additions, deletions = excluded.deletions, mergeable = excluded.mergeable, last_commit_state = excluded.last_commit_state, last_commit_state_description = excluded.last_commit_state_description, last_commit_short_hash = excluded.last_commit_short_hash, last_commit_pushed_date = excluded.last_commit_pushed_date, body = excluded.body, merged_at = excluded.merged_at, closed_at = excluded.closed_at, commit_count = excluded.commit_count ;""")
def insert_comments_and_reviews(self): with session_scope() as db_session: missing_authors = db_session.execute( """ SELECT DISTINCT etl_data.data ->> 'author_login' FROM etl_data LEFT OUTER JOIN users ON etl_data.data ->> 'author_login' = users.login WHERE users.id IS NULL; """ ).fetchall() if missing_authors: log.debug('missing_authors', missing_authors=missing_authors, count=len(missing_authors)) for author in missing_authors: login = author[0] if login is None: continue user_data = self.users_data.get(login) self.users_data.upsert(user_data) with session_scope() as db_session: db_session.execute( """ WITH etl_data AS ( SELECT DISTINCT etl_data.data ->> 'id' AS id, etl_data.data ->> 'bodyText' AS body, (etl_data.data ->> 'publishedAt')::timestamp with time zone AS published_at, etl_data.data ->> 'url' AS url, etl_data.data ->> 'pull_request_id' AS pull_request_id, users.id AS author_id, split_part(etl_data.data ->> 'review_decision', '.', 2)::reviewdecision AS auto_detected_review_decision FROM etl_data LEFT OUTER JOIN users ON etl_data.data ->> 'author_login' = users.login ) INSERT INTO comments (id, body, published_at, url, pull_request_id, author_id, auto_detected_review_decision) SELECT * FROM etl_data ON CONFLICT (id) DO UPDATE SET id = excluded.id, body = excluded.body, published_at = excluded.published_at, url = excluded.url, pull_request_id = excluded.pull_request_id, author_id = excluded.author_id, auto_detected_review_decision = excluded.auto_detected_review_decision ; """ ) with session_scope() as db_session: db_session.execute( """ WITH etl_data AS ( SELECT DISTINCT etl_data.data ->> 'pull_request_id' AS pull_request_id FROM etl_data ) UPDATE pull_requests SET review_decisions_count = s.review_decisions_count from (SELECT count(comments.id) as review_decisions_count, etl_data.pull_request_id FROM etl_data LEFT JOIN comments on etl_data.pull_request_id = comments.pull_request_id AND comments.auto_detected_review_decision is not null and comments.auto_detected_review_decision != 'NONE'::reviewdecision GROUP BY etl_data.pull_request_id) s WHERE s.pull_request_id = pull_requests.id; """ )
session .query(PullRequests.number) .filter( and_(PullRequests.is_high_priority.isnot(None)) ) .all() ) for r in record: pull_requests_data.update(number=int(r.number)) elif args.old: with session_scope() as session: try: record = ( session .query(PullRequests.updated_at) .order_by(PullRequests.updated_at.desc()) .limit(1) .one() ) from_date = record.updated_at.date() - timedelta(days=1) except NoResultFound: from_date = date(2000, 1, 1) pull_requests_data.update_all(newer_than=from_date, limit=args.limit) else: # All log.debug('All') pull_requests_data.update_all(limit=args.limit) polling_data.stop()
command.stamp(alembic_config, 'head') else: command.upgrade(alembic_config, 'head') except OperationalError as e: print(e) time.sleep(10) create_or_update_database(echo=echo) def drop_database(echo=True): with session_scope(echo=echo) as session: Base.metadata.drop_all(session.connection()) session.execute('DROP TABLE github.public.alembic_version;') if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Change the schema') parser.add_argument('-d', dest='drop', default=False, action='store_true') args = parser.parse_args() confirm = input(f'Dropping tables on {get_url()}, are you sure? y/n') if confirm != 'y': log.debug('User did not input y why?') sys.exit(1) else: if args.drop: drop_database() create_or_update_database()