def user_repositories_scanned(username, requestor_id=None): """ Update the timestamp on the pull request object, and delete old pull request files that weren't updated. """ user = User.get(username) prev_scan_at = user.repos_last_scanned_at user.repos_last_scanned_at = datetime.now() db.session.add(user) if prev_scan_at: # delete any repos that the user owns that were not updated # since the previous scan -- the user must have deleted those # repos from Github query = ( Repository.query.filter_by(owner_id=user.id) .filter(Repository.last_replicated_at < prev_scan_at) ) query.delete(synchronize_session=False) # delete the mutex lock_name = LOCK_TEMPLATE.format(username=username) Mutex.query.filter_by(name=lock_name).delete() logger.info("Lock {name} deleted".format(name=lock_name)) db.session.commit()
def pull_requests_scanned(owner, repo, requestor_id=None): """ Update the timestamp on the repository object, and delete old pull request that weren't updated. """ repo_name = repo repo = Repository.get(owner, repo_name) prev_scan_at = repo.pull_requests_last_scanned_at repo.pull_requests_last_scanned_at = datetime.now() db.session.add(repo) if prev_scan_at: # delete any PRs that were not updated since the previous scan -- # they have been removed from Github query = ( PullRequest.query.filter_by(base_repo_id=repo.id) .filter(PullRequest.last_replicated_at < prev_scan_at) ) query.delete() # delete the mutex lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name) Mutex.query.filter_by(name=lock_name).delete() logger.info("Lock {name} deleted".format(name=lock_name)) db.session.commit()
def issues_scanned(owner, repo, requestor_id=None): """ Update the timestamp on the repository object, and delete old issues that weren't updated. """ repo_name = repo repo = Repository.get(owner, repo_name) prev_scan_at = repo.issues_last_scanned_at repo.issues_last_scanned_at = datetime.now() db.session.add(repo) if prev_scan_at: # delete any issues that were not updated since the previous scan -- # they have been removed from Github query = ( Issue.query.filter_by(repo_id=repo.id) .filter(Issue.last_replicated_at < prev_scan_at) ) query.delete() # delete the mutex lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo_name) Mutex.query.filter_by(name=lock_name).delete() logger.info("Lock {name} deleted".format(name=lock_name)) db.session.commit()
def spawn_page_tasks_for_user_repositories( username, type="all", children=False, requestor_id=None, per_page=100, ): # acquire lock or fail (we're already in a transaction) lock_name = LOCK_TEMPLATE.format(username=username) existing = Mutex.query.get(lock_name) if existing: return False lock = Mutex(name=lock_name, user_id=requestor_id) db.session.add(lock) try: db.session.commit() except IntegrityError: return False else: logger.info("Lock {name} set by {requestor_id}".format( name=lock_name, requestor_id=requestor_id, )) repo_page_url = ( "/users/{username}/repos?type={type}&per_page={per_page}" ).format( username=username, type=type, per_page=per_page, ) if requestor_id: requestor = User.query.get(int(requestor_id)) assert requestor if requestor.login == username: # we can use the API for getting your *own* repos repo_page_url = ( "/user/repos?type={type}&per_page={per_page}" ).format( type=type, per_page=per_page, ) resp = fetch_url_from_github( repo_page_url, method="HEAD", requestor_id=requestor_id, headers={"Accept": "application/vnd.github.moondragon+json"}, ) last_page_url = URLObject(resp.links.get('last', {}).get('url', "")) last_page_num = int(last_page_url.query.dict.get('page', 1)) g = group( sync_page_of_repositories_for_user.s( username=username, type=type, children=children, requestor_id=requestor_id, per_page=per_page, page=page, ) for page in xrange(1, last_page_num+1) ) finisher = user_repositories_scanned.si( username=username, requestor_id=requestor_id, ) return (g | finisher).delay()
def spawn_page_tasks_for_milestones(owner, repo, state="all", children=False, requestor_id=None, per_page=100): # acquire lock or fail (we're already in a transaction) lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo) existing = Mutex.query.get(lock_name) if existing: return False lock = Mutex(name=lock_name, user_id=requestor_id) db.session.add(lock) try: db.session.commit() except IntegrityError: return False else: logger.info("Lock {name} set by {requestor_id}".format( name=lock_name, requestor_id=requestor_id, )) milestone_list_url = ("/repos/{owner}/{repo}/pulls?" "state={state}&per_page={per_page}").format( owner=owner, repo=repo, state=state, per_page=per_page, ) resp = fetch_url_from_github( milestone_list_url, method="HEAD", requestor_id=requestor_id, ) last_page_url = URLObject(resp.links.get('last', {}).get('url', "")) last_page_num = int(last_page_url.query.dict.get('page', 1)) g = group( sync_page_of_milestones.s( owner=owner, repo=repo, state=state, requestor_id=requestor_id, per_page=per_page, page=page, ) for page in xrange(1, last_page_num + 1)) finisher = milestones_scanned.si( owner=owner, repo=repo, requestor_id=requestor_id, ) return (g | finisher).delay()
def spawn_page_tasks_for_pull_requests(owner, repo, state="all", children=False, requestor_id=None, per_page=100): # acquire lock or fail (we're already in a transaction) lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo) existing = Mutex.query.get(lock_name) if existing: return False lock = Mutex(name=lock_name, user_id=requestor_id) db.session.add(lock) try: db.session.commit() except IntegrityError: return False else: logger.info("Lock {name} set by {requestor_id}".format( name=lock_name, requestor_id=requestor_id, )) pr_list_url = ( "/repos/{owner}/{repo}/pulls?" "state={state}&per_page={per_page}" ).format( owner=owner, repo=repo, state=state, per_page=per_page, ) resp = fetch_url_from_github( pr_list_url, method="HEAD", requestor_id=requestor_id, ) last_page_url = URLObject(resp.links.get('last', {}).get('url', "")) last_page_num = int(last_page_url.query.dict.get('page', 1)) g = group( sync_page_of_pull_requests.s( owner=owner, repo=repo, state=state, children=children, requestor_id=requestor_id, per_page=per_page, page=page ) for page in xrange(1, last_page_num+1) ) finisher = pull_requests_scanned.si( owner=owner, repo=repo, requestor_id=requestor_id, ) return (g | finisher).delay()
def user_repositories_scanned(username, requestor_id=None): """ Update the timestamp on the pull request object, and delete old pull request files that weren't updated. """ user = User.get(username) prev_scan_at = user.repos_last_scanned_at user.repos_last_scanned_at = datetime.now() db.session.add(user) if prev_scan_at: # delete any repos that the user owns that were not updated # since the previous scan -- the user must have deleted those # repos from Github query = (Repository.query.filter_by(owner_id=user.id).filter( Repository.last_replicated_at < prev_scan_at)) query.delete(synchronize_session=False) # delete the mutex lock_name = LOCK_TEMPLATE.format(username=username) Mutex.query.filter_by(name=lock_name).delete() logger.info("Lock {name} deleted".format(name=lock_name)) db.session.commit()
def fetch_url_from_github(self, url, as_user=None, requestor_id=None, **kwargs): if "method" in kwargs: method = kwargs.pop("method") else: method = "GET" if method.upper() == "HEAD": kwargs.setdefault("allow_redirects", False) username = "******" if as_user: github.blueprint.config["user"] = as_user username = "******".format(login=as_user.login) elif requestor_id: github.blueprint.config["user_id"] = int(requestor_id) username = "******".format(requestor_id) logger.info("{method} {url} as {username}".format( method=method, url=url, username=username, )) try: resp = github.request(method=method, url=url, **kwargs) except RateLimited as exc: logger.info("rate limited: {url}".format(url=url)) # if this task is being executed inline, let the exception raise # so that Flask's error-handling mechanisms can catch it if self.request.is_eager: raise # otherwise, schedule this task to retry when the rate limit is reset else: logger.warn("Retrying {url} at {reset}".format(url=url, reset=exc.reset)) self.retry(exc=exc, eta=exc.reset) if resp.status_code == 404: logger.info("not found: {url}".format(url=url)) raise NotFound(url) if not resp.ok: raise RequestException(resp.text) return resp
def spawn_page_tasks_for_user_repositories( username, type="all", children=False, requestor_id=None, per_page=100, ): # acquire lock or fail (we're already in a transaction) lock_name = LOCK_TEMPLATE.format(username=username) existing = Mutex.query.get(lock_name) if existing: return False lock = Mutex(name=lock_name, user_id=requestor_id) db.session.add(lock) try: db.session.commit() except IntegrityError: return False else: logger.info("Lock {name} set by {requestor_id}".format( name=lock_name, requestor_id=requestor_id, )) repo_page_url = ( "/users/{username}/repos?type={type}&per_page={per_page}").format( username=username, type=type, per_page=per_page, ) if requestor_id: requestor = User.query.get(int(requestor_id)) assert requestor if requestor.login == username: # we can use the API for getting your *own* repos repo_page_url = ( "/user/repos?type={type}&per_page={per_page}").format( type=type, per_page=per_page, ) resp = fetch_url_from_github( repo_page_url, method="HEAD", requestor_id=requestor_id, headers={"Accept": "application/vnd.github.moondragon+json"}, ) last_page_url = URLObject(resp.links.get('last', {}).get('url', "")) last_page_num = int(last_page_url.query.dict.get('page', 1)) g = group( sync_page_of_repositories_for_user.s( username=username, type=type, children=children, requestor_id=requestor_id, per_page=per_page, page=page, ) for page in xrange(1, last_page_num + 1)) finisher = user_repositories_scanned.si( username=username, requestor_id=requestor_id, ) return (g | finisher).delay()