def second_passed(self): # initial clone/fetch stage self.process.poll() self.age += 1 if not self.parsing: # move on to the parsing stage if fetching is complete if self.process.returncode is not None: cprint("==> Parsing {0}".format(self.repo.project.title), "blue", attrs=["bold"]) self.parsing = True self.age = 0 self.process = subprocess.Popen([python, parse_script, str(self.repo.id)]) # time out cloning, clean up the failed clone dir, and end the fetcher elif self.age > CLONE_TIMEOUT: cprint("==> Killing cloning of {0}".format(self.repo.project.title), "red", attrs=["bold"]) self.process.terminate() path = os.path.dirname(os.path.abspath(__file__)) rmtree(os.path.join(path, "..", "..", "clones", self.repo.project.url_path)) return True else: # if the process finished, we're all done if self.process.returncode is not None: if not self.process.returncode: cprint("==> Successfully parsed {0}".format(self.repo.project.title), "green", attrs=["bold"]) return True # if the parser timed out if self.age > 60 * REPO_FETCH_TIMEOUT: # kill the parser cprint("==> Killing parsing of {0}".format(self.repo.project.title), "red", attrs=["bold"]) self.process.terminate() # deal with the aftermath commits = Commit.objects.filter(repository__id = self.repo.id) try: self.repo.most_recent_date = commits.order_by("-date")[0].date self.repo.save() except IndexError: pass # we're (unfortunately) done return True return False
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import os, subprocess from fetch_core import setup_environment, cprint from sys import argv setup_environment() from dashboard.models import Blog, Project from observatory.settings import BLOG_FETCH_PROCESS_COUNT blog = Blog.objects.get(id=argv[1]) if blog.from_feed: try: title = blog.project.title except: try: title = blog.user.get_full_name() except: title = "Unknown" cprint("==> Fetching the blog for {0}".format(title), "magenta", attrs=["bold"]) blog.fetch() cprint("==> Done fetching the blog for {0}".format(title), "green", attrs=["bold"])
def second_passed(self): # initial clone/fetch stage self.process.poll() self.age += 1 if not self.parsing: # time out cloning, clean up the failed clone dir, and end the fetcher if self.age > CLONE_TIMEOUT or self.process.returncode == 1: if self.process.returncode is not None: cprint("==> Fetching failed for {0}".format(self.repo.project.title), "red", attrs=["bold"]) else: cprint("==> Killing clone of {0}".format(self.repo.project.title), "red", attrs=["bold"]) self.process.terminate() path = os.path.dirname(os.path.abspath(__file__)) clone_path = os.path.join(path, "..", "..", "clones", self.repo.project.url_path) if os.path.exists(clone_path): rmtree(clone_path) return True # move on to the parsing stage if fetching is complete elif self.process.returncode is not None: cprint("==> Successfully fetched {0}".format(self.repo.project.title)) cprint("==> Parsing {0}".format(self.repo.project.title), "blue", attrs=["bold"]) self.parsing = True self.age = 0 self.process = subprocess.Popen([python, parse_script, str(self.repo.id)]) else: # if the process finished, we're all done if self.process.returncode is not None: if not self.process.returncode: cprint("==> Successfully parsed {0}".format(self.repo.project.title), "green", attrs=["bold"]) return True # if the parser timed out if self.age > 60 * REPO_FETCH_TIMEOUT: # kill the parser cprint("==> Killing parsing of {0}".format(self.repo.project.title), "red", attrs=["bold"]) self.process.terminate() # deal with the aftermath commits = Commit.objects.filter(repository__id = self.repo.id) try: self.repo.most_recent_date = commits.order_by("-date")[0].date self.repo.save() except IndexError: pass # we're (unfortunately) done return True return False
def __init__(self, repo): self.repo = repo cprint("==> Fetching {0}".format(repo.project.title), "yellow", attrs=["bold"]) self.process = subprocess.Popen([python, fetch_script, str(repo.id), "&"])
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import os, subprocess from fetch_core import setup_environment, cprint from sys import argv setup_environment() from dashboard.models import Blog, Project from observatory.settings import BLOG_FETCH_PROCESS_COUNT blog = Blog.objects.get(id = argv[1]) if blog.from_feed: try: title = blog.project.title except: try: title = blog.user.get_full_name() except: title = "Unknown" cprint("==> Fetching the blog for {0}".format(title), "magenta", attrs=["bold"]) blog.fetch() cprint("==> Done fetching the blog for {0}".format(title), "green", attrs=["bold"])
# Copyright (c) 2013, individual contributors (see AUTHORS file) # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # Blog fetching is pretty much always fast, so it does not need to be broken # up into multiple processes. This script should just be run in cron. from fetch_core import setup_environment, cprint setup_environment() from dashboard.models import Project projects = list(Project.objects.exclude(active=False).exclude(pending=True)) for project in projects: cprint("==> Calculating Warnings for {0}".format(project.title), "magenta", attrs=["bold"]) project.do_warnings() cprint("==> Done Calculating Warnings for {0}".format(project.title), "magenta", attrs=["bold"])