def second_passed(self):
# initial clone/fetch stage
self.process.poll()
self.age += 1
if not self.parsing:
# move on to the parsing stage if fetching is complete
if self.process.returncode is not None:
cprint("==> Parsing {0}".format(self.repo.project.title),
"blue", attrs=["bold"])
self.parsing = True
self.age = 0
self.process = subprocess.Popen([python, parse_script,
str(self.repo.id)])
# time out cloning, clean up the failed clone dir, and end the fetcher
elif self.age > CLONE_TIMEOUT:
cprint("==> Killing cloning of {0}".format(self.repo.project.title),
"red", attrs=["bold"])
self.process.terminate()
path = os.path.dirname(os.path.abspath(__file__))
rmtree(os.path.join(path, "..", "..", "clones",
self.repo.project.url_path))
return True
else:
# if the process finished, we're all done
if self.process.returncode is not None:
if not self.process.returncode:
cprint("==> Successfully parsed {0}".format(self.repo.project.title),
"green", attrs=["bold"])
return True
# if the parser timed out
if self.age > 60 * REPO_FETCH_TIMEOUT:
# kill the parser
cprint("==> Killing parsing of {0}".format(self.repo.project.title),
"red", attrs=["bold"])
self.process.terminate()
# deal with the aftermath
commits = Commit.objects.filter(repository__id = self.repo.id)
try:
self.repo.most_recent_date = commits.order_by("-date")[0].date
self.repo.save()
except IndexError:
pass
# we're (unfortunately) done
return True
return False
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import os, subprocess
from fetch_core import setup_environment, cprint
from sys import argv
setup_environment()
from dashboard.models import Blog, Project
from observatory.settings import BLOG_FETCH_PROCESS_COUNT
blog = Blog.objects.get(id=argv[1])
if blog.from_feed:
try:
title = blog.project.title
except:
try:
title = blog.user.get_full_name()
except:
title = "Unknown"
cprint("==> Fetching the blog for {0}".format(title),
"magenta",
attrs=["bold"])
blog.fetch()
cprint("==> Done fetching the blog for {0}".format(title),
"green",
attrs=["bold"])
def second_passed(self):
# initial clone/fetch stage
self.process.poll()
self.age += 1
if not self.parsing:
# time out cloning, clean up the failed clone dir, and end the fetcher
if self.age > CLONE_TIMEOUT or self.process.returncode == 1:
if self.process.returncode is not None:
cprint("==> Fetching failed for {0}".format(self.repo.project.title),
"red", attrs=["bold"])
else:
cprint("==> Killing clone of {0}".format(self.repo.project.title),
"red", attrs=["bold"])
self.process.terminate()
path = os.path.dirname(os.path.abspath(__file__))
clone_path = os.path.join(path, "..", "..", "clones",
self.repo.project.url_path)
if os.path.exists(clone_path):
rmtree(clone_path)
return True
# move on to the parsing stage if fetching is complete
elif self.process.returncode is not None:
cprint("==> Successfully fetched {0}".format(self.repo.project.title))
cprint("==> Parsing {0}".format(self.repo.project.title),
"blue", attrs=["bold"])
self.parsing = True
self.age = 0
self.process = subprocess.Popen([python, parse_script,
str(self.repo.id)])
else:
# if the process finished, we're all done
if self.process.returncode is not None:
if not self.process.returncode:
cprint("==> Successfully parsed {0}".format(self.repo.project.title),
"green", attrs=["bold"])
return True
# if the parser timed out
if self.age > 60 * REPO_FETCH_TIMEOUT:
# kill the parser
cprint("==> Killing parsing of {0}".format(self.repo.project.title),
"red", attrs=["bold"])
self.process.terminate()
# deal with the aftermath
commits = Commit.objects.filter(repository__id = self.repo.id)
try:
self.repo.most_recent_date = commits.order_by("-date")[0].date
self.repo.save()
except IndexError:
pass
# we're (unfortunately) done
return True
return False
def __init__(self, repo):
self.repo = repo
cprint("==> Fetching {0}".format(repo.project.title),
"yellow", attrs=["bold"])
self.process = subprocess.Popen([python, fetch_script, str(repo.id), "&"])
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import os, subprocess
from fetch_core import setup_environment, cprint
from sys import argv
setup_environment()
from dashboard.models import Blog, Project
from observatory.settings import BLOG_FETCH_PROCESS_COUNT
blog = Blog.objects.get(id = argv[1])
if blog.from_feed:
try:
title = blog.project.title
except:
try:
title = blog.user.get_full_name()
except:
title = "Unknown"
cprint("==> Fetching the blog for {0}".format(title), "magenta", attrs=["bold"])
blog.fetch()
cprint("==> Done fetching the blog for {0}".format(title), "green", attrs=["bold"])
# Copyright (c) 2013, individual contributors (see AUTHORS file)
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Blog fetching is pretty much always fast, so it does not need to be broken
# up into multiple processes. This script should just be run in cron.
from fetch_core import setup_environment, cprint
setup_environment()
from dashboard.models import Project
projects = list(Project.objects.exclude(active=False).exclude(pending=True))
for project in projects:
cprint("==> Calculating Warnings for {0}".format(project.title), "magenta", attrs=["bold"])
project.do_warnings()
cprint("==> Done Calculating Warnings for {0}".format(project.title), "magenta", attrs=["bold"])
