def bytags(project, data):
repo = data['source']
# Loading tags for current project
tags_url = 'https://api.github.com/repos/' + repo + '/tags'
tags = json.loads(urlfetch.fetch(tags_url).content)
refresh = False
# Looping over tags
for tag in tags:
# If tags are not in the database, add them
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, tag['commit']['sha'])
if (q.count() == 0):
refresh = True
commit_url = 'https://api.github.com/repos/' + repo + '/commits/' + tag['commit']['sha']
commit = json.loads(urlfetch.fetch(commit_url).content)
version_date = iso8601date.parse_iso8601_datetime(commit['commit']['author']['date']) # Parse the date to python datetime
version_version = re.sub('^v(?=\d)', '', tag['name']) # Remove leading v from version number
t = VersionCache(project=project,
version=version_version,
commit=tag['commit']['sha'],
date=version_date)
t.put()
if refresh:
logging.info('refreshed ' + project + ' version data')
else:
logging.info('version data for ' + project + ' unchanged')
# Return the most recent released version
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def firefox(project, data):
url = data['source']
content = memcache.get('cache:' + url)
if content is None:
content = urlfetch.fetch(url).content
memcache.add('cache:' + url, content, 120)
project_data = json.loads(content)
sha = hashlib.sha1()
sha.update(content)
sha = sha.hexdigest()
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha)
if q.count() == 0:
#logging.info('refreshing version data')
version_version = project_data[data['branch']]
t = VersionCache(project=project,
version=version_version,
commit=sha,
date=datetime.datetime.now())
t.put()
#else:
#logging.info('version data unchanged')
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def firefox(project, data):
url = 'http://people.mozilla.com/~tmielczarek/branch_versions.json'
content = urlfetch.fetch(url).content
project_data = json.loads(content)
sha = hashlib.sha1()
sha.update(content)
sha = sha.hexdigest()
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha)
if q.count() == 0:
logging.info('refreshing ' + project + ' version data')
version_version = project_data[data['key']]
t = VersionCache(project=project,
version=version_version,
commit=sha,
date=datetime.datetime.now())
t.put()
else:
logging.info('version data for ' + project + ' unchanged')
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def bytags(p, data):
repo = data['repo']
# Loading tags for current project
tags_url = ghAuth('https://api.github.com/repos/' + repo + '/tags')
tags = json.loads(urlfetch.fetch(tags_url).content)
refresh = False
# Looping over tags
for tag in tags:
# If tags are not in the database, add them
query = "SELECT * FROM VersionCache WHERE project = :1 AND commit = :2"
q = db.GqlQuery(query, p, tag['commit']['sha'])
if (q.count() == 0):
refresh = True
commit_url = ghAuth(
'https://api.github.com/repos/' + repo +
'/commits/' + tag['commit']['sha']
)
commit = json.loads(urlfetch.fetch(commit_url).content)
# Parse the date to python datetime
version_date = commit['commit']['author']['date']
version_date = iso8601date.parse_iso8601_datetime(version_date)
# Remove leading v from version number if present
version_version = re.sub('^v(?=\d)', '', tag['name'])
t = VersionCache(project=p,
version=version_version,
commit=tag['commit']['sha'],
date=version_date)
t.put()
#if refresh:
#logging.info('refreshed version data')
#else:
#logging.info('version data unchanged')
# Return the most recent released version
query = (
"SELECT version FROM VersionCache WHERE project = :1 " +
"ORDER BY date DESC"
)
q = db.GqlQuery(query, p).get()
return q.version
def byjsonfile(project, data):
repo = data["repo"]
filename = data["file"]
url = ghAuth("https://api.github.com/repos/" + repo + "/contents/" + filename)
project_data = json.loads(urlfetch.fetch(url).content)
sha = project_data["sha"]
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha)
if q.count() == 0:
# logging.info('refreshing version data')
version_version = json.loads(base64.b64decode(project_data["content"]))["version"]
t = VersionCache(project=project, version=version_version, commit=sha, date=datetime.datetime.now())
t.put()
# else:
# logging.info('version data unchanged')
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def chrome(project, data):
url = data['source']
content = memcache.get('cache:' + url)
if content is None:
content = urlfetch.fetch(url).content
memcache.add('cache:' + url, content, 120)
project_data = json.loads(content)
sha = hashlib.sha1()
sha.update(content)
sha = sha.hexdigest()
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha)
if q.count() == 0:
#logging.info(project + ': refreshing version data')
for os in project_data:
if os['os'] == data['os']:
os_versions = os['versions']
for vs in os_versions:
if vs['channel'] == data['channel']:
version_version = vs['version']
version_date = datetime.datetime.strptime(vs['date'], "%m/%d/%y")
t = VersionCache(project=project,
version=version_version,
commit=sha,
date=version_date)
t.put()
#else:
#logging.info(project + ': version data for unchanged')
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def byjsonfile(project, data):
repo = data['source']
filename = data['file']
url = 'https://api.github.com/repos/' + repo + '/contents/' + filename
project_data = json.loads(urlfetch.fetch(url).content)
sha = project_data['sha']
q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha)
if q.count() == 0:
logging.info('refreshing ' + project + ' version data')
version_version = json.loads(base64.b64decode(project_data['content']))['version']
t = VersionCache(project=project,
version=version_version,
commit=sha,
date=datetime.datetime.now())
t.put()
else:
logging.info('version data for ' + project + ' unchanged')
q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get()
return q.version
def get(self, project):
logging.info('Deleting tag cache from project: ' + project)
for person in VersionCache.gql("WHERE project = :1", project):
person.delete()
logging.info('Deletion of ' + project + ' cache done.')
self.response.write('Purged cache for: ' + project)
