def fix_commit(key):
"""Fix an individual commit if possible."""
commit_key = ndb.Key(urlsafe=key)
commit = commit_key.get()
if commit is None:
return
commit_data = commit.to_dict()
# Check the timestamp to see if we should reject/delete
if commit.timestamp is None:
logging.warning("Skipping early orphan")
return
if commit.timestamp < settings.START_DATETIME:
logging.warning("Skipping early orphan")
return
if 'project_slug' in commit_data:
del commit_data['project_slug']
new_commit = Commit.create_by_email(commit.email, [commit_data], project=commit.project)
if new_commit and new_commit[0].parent():
logging.info('Deleting orphan')
commit.key.delete()
def post(self, request):
payload = self.parse_payload(request)
logging.info(payload)
if not payload:
raise http.HttpResponseBadRequest
try:
data = json.loads(payload)
except:
logging.exception("Unable to serialize POST")
raise http.HttpResponseBadRequest
commit_data = data.get('commits', [])
repo = self._parse_repo(data)
project, _ = Project.create(**repo)
commit_dict = self.parse_commits(commit_data, project)
total_commits = []
for email, commits in commit_dict.iteritems():
# TODO: run this in a task queue?
cmts = Commit.create_by_email(email, commits, project=project)
total_commits += cmts
status = 201 if len(total_commits) else 200
self._publish_commits(total_commits)
return self.respond_json({'commits': [c.hash for c in total_commits]}, status=status)
def fix_commit(key):
"""Fix an individual commit if possible."""
commit_key = ndb.Key(urlsafe=key)
commit = commit_key.get()
if commit is None:
return
commit_data = commit.to_dict()
# Check the timestamp to see if we should reject/delete
if commit.timestamp is None:
logging.warning("Skipping early orphan")
return
if commit.timestamp < settings.START_DATETIME:
logging.warning("Skipping early orphan")
return
if 'project_slug' in commit_data:
del commit_data['project_slug']
new_commit = Commit.create_by_email(commit.email, [commit_data],
project=commit.project)
if new_commit and new_commit[0].parent():
logging.info('Deleting orphan')
commit.key.delete()
def project_details(request, slug, template_name='projects/details.html'):
project_key = ndb.Key('Project', slug)
project = project_key.get()
if project is None:
raise Http404("Project Not Found.")
limit = 100
cursor = request.GET.get('cursor')
if cursor:
cursor = Cursor(urlsafe=cursor)
# TODO: pagination
user_future = User.query().filter(ndb.GenericProperty('projects') == project.url).fetch_async(100)
query = Commit.query().filter(Commit.project_slug == slug).order(-Commit.timestamp)
commit_future = query.fetch_page_async(limit, start_cursor=cursor)
commits, next_cursor, more = commit_future.get_result()
users = user_future.get_result()
if next_cursor is not None:
next_cursor = next_cursor.urlsafe()
return render_to_response(template_name,
{'project': project, 'users': users, 'commits': commits,
'next': next_cursor, 'more': more},
context_instance=RequestContext(request))
def fix_counts():
"""Fix the global totals of points. Do a bunch of count querys."""
ranges = _get_date_ranges()
for start, end in ranges:
count = Commit.query().filter(Commit.timestamp >= start, Commit.timestamp < end).count(1000)
Accumulator.add_count('global', start, count, reset=True)
def make_commit(self, auth_id="x:no", hash=None, timestamp=None, project=None, **kwargs):
if hash is None:
hash = str(uuid.uuid4())
if timestamp is None:
timestamp = timezone.now()
commit = kwargs.copy()
commit.update({"hash": hash, "timestamp": timestamp})
return Commit.create_by_auth_id(auth_id, [commit], project=project)
def fix_player_counts(auth_id):
"""Fix a single user counts."""
user = User.get_by_auth_id(auth_id)
ranges = _get_date_ranges()
for start, end in ranges:
count = Commit.query(ancestor=user.key).filter(Commit.timestamp >= start, Commit.timestamp < end).count(1000)
Accumulator.add_count('own:%s' % user.username, start, count, reset=True)
def fix_counts():
"""Fix the global totals of points. Do a bunch of count querys."""
ranges = _get_date_ranges()
for start, end in ranges:
count = Commit.query().filter(Commit.timestamp >= start,
Commit.timestamp < end).count(1000)
Accumulator.add_count('global', start, count, reset=True)
def user_profile(request, username):
user = User.get_by_auth_id('twitter:%s' % username)
if user == None:
raise Http404("User not found")
commits = Commit.query(ancestor=user.key).order(-Commit.timestamp).fetch(100)
return render_to_response('people/profile.html',
{"commits":commits, 'profile':user},
context_instance=RequestContext(request))
def user_profile(request, username):
user = User.get_by_auth_id('own:%s' % username)
if user == None:
raise Http404("User not found")
commits = Commit.query(ancestor=user.key).order(-Commit.timestamp).fetch(100)
return render_to_response('people/profile.html',
{"commits":commits, 'profile':user},
context_instance=RequestContext(request))
def run_query(self):
if self.game is None:
self.game = Game.active_or_latest()
# Commit.calender returns a list of objects for each day a user has
# commited along with the count during the day. So we can use this
# query to get the total and the number of days.
resp = Commit.calendar(self.game, user=self.user)
objects = resp['objects']
total = 0
for obj in objects:
total += obj.get('commit_count', 0)
return {'game_commits': total, 'game_days': len(objects)}
def fix_player_counts(auth_id):
"""Fix a single user counts."""
user = User.get_by_auth_id(auth_id)
ranges = _get_date_ranges()
for start, end in ranges:
count = Commit.query(ancestor=user.key).filter(
Commit.timestamp >= start, Commit.timestamp < end).count(1000)
Accumulator.add_count('own:%s' % user.username,
start,
count,
reset=True)
def get_calendar(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
filters = {}
game = Game.active_or_latest()
username = request.GET.get('username')
if username:
filters['user__username'] = username
# user = kwargs.get('user', None)
calendar = Commit.calendar(game=game, **filters)
return self.create_response(request, calendar)
def make_commit(self,
auth_id='x:no',
hash=None,
timestamp=None,
project=None,
**kwargs):
if hash is None:
hash = str(uuid.uuid4())
if timestamp is None:
timestamp = timezone.now()
commit = kwargs.copy()
commit.update({'hash': hash, 'timestamp': timestamp})
return Commit.create_by_auth_id(auth_id, [commit], project=project)
def get_calendar(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
filters = {}
game = Game.active_or_latest()
username = request.GET.get('username')
if username:
filters['user__username'] = username
# user = kwargs.get('user', None)
calendar = Commit.calendar(game=game, **filters)
return self.create_response(request, calendar)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError('Must supply a JSON file of commits.')
commit_path = args[0]
if os.path.isdir(commit_path):
files = [os.path.join(commit_path, f) for f in os.listdir(commit_path) if f.startswith('commits')]
else:
files = [commit_path]
for commits_json in files:
logging.info("Parsing File: %s", commits_json)
with open(commits_json, 'r') as commit_file:
commits = json.loads(commit_file.read())
for commit in commits['models']:
try:
project, _ = Project.create(url=commit['project'])
c = to_commit(commit)
Commit.create_by_email(c['email'], c, project)
except Exception:
logging.exception("Error: %s" % commit)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError('Must supply a JSON file of commits.')
commit_path = args[0]
if os.path.isdir(commit_path):
files = [os.path.join(commit_path, f) for f in
os.listdir(commit_path) if f.startswith('commits')]
else:
files = [commit_path]
for commits_json in files:
logging.info("Parsing File: %s", commits_json)
with open(commits_json, 'r') as commit_file:
commits = json.loads(commit_file.read())
for commit in commits['models']:
try:
project, _ = Project.create(url=commit['project'])
c = to_commit(commit)
Commit.create_by_email(c['email'], c, project)
except Exception:
logging.exception("Error: %s" % commit)
def post(self):
payload = self.parse_payload()
logging.info(payload)
if not payload:
abort(400)
try:
data = json.loads(payload)
except:
logging.exception("Unable to serialize POST")
abort(400)
repository = data.get('repository', {})
commit_data = data.get('commits', [])
repo = self._parse_repo(repository)
commit_dict = self.parse_commits(commit_data)
total_commits = []
for email, commits in commit_dict.iteritems():
# TODO: run this in a task queue?
cmts = Commit.create_by_email(email,
commits,
project=repo.get('url', ''))
total_commits += cmts
status = 200
# If we found commits try to save the project too.
if len(total_commits):
status = 201
_, project = Project.get_or_create(**repo)
project_key = project.key
@ndb.transactional
def txn():
# TODO: run this in a task queue?
total = len(total_commits)
project = project_key.get()
logging.info("adding %s points to %s", total, project)
project.total += total
project.put()
txn()
return self.respond_json(
{'commits': [c.urlsafe() for c in total_commits]},
status_code=status)
def run_query(self):
if self.game is None:
self.game = Game.active_or_latest()
# Commit.calender returns a list of objects for each day a user has
# commited along with the count during the day. So we can use this
# query to get the total and the number of days.
resp = Commit.calendar(self.game, user=self.user)
objects = resp['objects']
total = 0
for obj in objects:
total += obj.get('commit_count', 0)
return {
'game_commits': total,
'game_days': len(objects)
}
def get_calendar(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
filters = {}
days = int(request.GET.get('days', 35))
end_date = request.GET.get('end_date', None)
end_date = end_date and datetime(end_date)
username = request.GET.get('username', None)
if username:
filters['user__username'] = username
# user = kwargs.get('user', None)
calendar = Commit.calendar(days=days, end_date=end_date, **filters)
return self.create_response(request, list(calendar))
def fix_orphans(cursor=None):
if cursor:
cursor = Cursor(urlsafe=cursor)
query = Commit.query()
models, next_cursor, more = query.fetch_page(500, keys_only=True, start_cursor=cursor)
for commit in models:
if commit.parent() is None:
logging.info("Found orphan commit: %s", commit)
deferred.defer(fix_commit, commit.urlsafe())
# if we have more keep looping
if more:
deferred.defer(fix_orphans, cursor=next_cursor.urlsafe())
def post(self):
payload = self.parse_payload()
logging.info(payload)
if not payload:
abort(400)
try:
data = json.loads(payload)
except:
logging.exception("Unable to serialize POST")
abort(400)
repository = data.get('repository', {})
commit_data = data.get('commits', [])
repo = self._parse_repo(repository)
commit_dict = self.parse_commits(commit_data)
total_commits = []
for email, commits in commit_dict.iteritems():
# TODO: run this in a task queue?
cmts = Commit.create_by_email(email, commits, project=repo.get('url', ''))
total_commits += cmts
status = 200
# If we found commits try to save the project too.
if len(total_commits):
status = 201
_, project = Project.get_or_create(**repo)
project_key = project.key
@ndb.transactional
def txn():
# TODO: run this in a task queue?
total = len(total_commits)
project = project_key.get()
logging.info("adding %s points to %s", total, project)
project.total += total
project.put()
txn()
return self.respond_json({'commits': [c.urlsafe() for c in total_commits]}, status_code=status)
def post(self):
"""Create a tweet from the api.
Example::
{ "name": "Josh Marshall",
"email": "*****@*****.**",
"message": "Working on Tornado stuff!",
"url": "https://github.com/project/commitID",
"timestamp": 5430604985.0,
"hash": "6a87af2a7eb3de1e17ac1cce41e060516b38c0e9"}
"""
form = self.parse_form()
if not form.is_valid():
return self.respond_json(form.errors, status_code=400)
commits = Commit.create_by_user(self.user, form.cleaned_data)
self.respond_json({'commits': [k.urlsafe() for k in commits]}, status_code=201)
def post(self):
"""Create a tweet from the api.
Example::
{ "name": "Josh Marshall",
"email": "*****@*****.**",
"message": "Working on Tornado stuff!",
"url": "https://github.com/project/commitID",
"timestamp": 5430604985.0,
"hash": "6a87af2a7eb3de1e17ac1cce41e060516b38c0e9"}
"""
form = self.parse_form()
if not form.is_valid():
return self.respond_json(form.errors, status_code=400)
commits = Commit.create_by_user(self.user, form.cleaned_data)
self.respond_json({'commits': [k.urlsafe() for k in commits]},
status_code=201)
def fix_orphans(cursor=None, email=None):
if cursor:
cursor = Cursor(urlsafe=cursor)
query = Commit.query()
if email:
logging.info("Fixing orphans by email")
query = query.filter(Commit.email == email)
models, next_cursor, more = query.fetch_page(500,
keys_only=True,
start_cursor=cursor)
for commit in models:
if commit.parent() is None:
logging.info("Found orphan commit: %s", commit)
deferred.defer(fix_commit, commit.urlsafe())
# if we have more keep looping
if more:
deferred.defer(fix_orphans, cursor=next_cursor.urlsafe())
def post(self, request):
payload = self.parse_payload(request)
if not payload:
return http.HttpResponseBadRequest()
elif isinstance(payload, http.HttpResponse):
return payload
try:
data = json.loads(payload)
except:
logging.exception("Unable to serialize POST")
return http.HttpResponseBadRequest()
commit_data = data.get('commits', [])
repo = self._parse_repo(data)
logging.info(repo)
project = Project.create(**repo)
if project is None:
logging.error("Project Disabled")
# TODO: discover what response codes are helpful to github
# and bitbucket
return self.respond_json({'error': 'abuse'}, status=202)
commit_dict = self.parse_commits(commit_data, project)
total_commits = []
for email, commits in commit_dict.iteritems():
# TODO: run this in a task queue?
cmts = Commit.create_by_email(email, commits, project=project)
total_commits += cmts
status = 201 if len(total_commits) else 200
self._publish_commits(total_commits)
return self.respond_json(
{'commits': [c.hash for c in total_commits]},
status=status)
def post(self, request):
payload = self.parse_payload(request)
if not payload:
return http.HttpResponseBadRequest()
elif isinstance(payload, http.HttpResponse):
return payload
try:
data = json.loads(payload)
except:
logging.exception("Unable to serialize POST")
return http.HttpResponseBadRequest()
commit_data = data.get('commits', [])
repo = self._parse_repo(data)
logging.info(repo)
project = Project.create(**repo)
if project is None:
logging.error("Project Disabled")
# TODO: discover what response codes are helpful to github
# and bitbucket
return self.respond_json({'error': 'abuse'}, status=202)
commit_dict = self.parse_commits(commit_data, project)
total_commits = []
for email, commits in commit_dict.iteritems():
# TODO: run this in a task queue?
cmts = Commit.create_by_email(email, commits, project=project)
total_commits += cmts
status = 201 if len(total_commits) else 200
self._publish_commits(total_commits)
return self.respond_json(
{'commits': [c.hash for c in total_commits]},
status=status)
def make_commit(self, user, message, save=True, **kwargs):
commit = Commit(parent=user.key, message=message, **kwargs)
if save:
commit.put()
return commit
def make_orphan(self, email, message, save=True, **kwargs):
commit = Commit(email=email, message=message, **kwargs)
if save:
commit.put()
return commit
def make_orphan(self, email, message, save=True, **kwargs):
commit = Commit(email=email, message=message, **kwargs)
if save:
commit.put()
return commit
def make_commit(self, user, message, save=True, **kwargs):
commit = Commit(parent=user.key, message=message, **kwargs)
if save:
commit.put()
return commit
