def bytags(project, data): repo = data['source'] # Loading tags for current project tags_url = 'https://api.github.com/repos/' + repo + '/tags' tags = json.loads(urlfetch.fetch(tags_url).content) refresh = False # Looping over tags for tag in tags: # If tags are not in the database, add them q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, tag['commit']['sha']) if (q.count() == 0): refresh = True commit_url = 'https://api.github.com/repos/' + repo + '/commits/' + tag['commit']['sha'] commit = json.loads(urlfetch.fetch(commit_url).content) version_date = iso8601date.parse_iso8601_datetime(commit['commit']['author']['date']) # Parse the date to python datetime version_version = re.sub('^v(?=\d)', '', tag['name']) # Remove leading v from version number t = VersionCache(project=project, version=version_version, commit=tag['commit']['sha'], date=version_date) t.put() if refresh: logging.info('refreshed ' + project + ' version data') else: logging.info('version data for ' + project + ' unchanged') # Return the most recent released version q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def firefox(project, data): url = data['source'] content = memcache.get('cache:' + url) if content is None: content = urlfetch.fetch(url).content memcache.add('cache:' + url, content, 120) project_data = json.loads(content) sha = hashlib.sha1() sha.update(content) sha = sha.hexdigest() q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha) if q.count() == 0: #logging.info('refreshing version data') version_version = project_data[data['branch']] t = VersionCache(project=project, version=version_version, commit=sha, date=datetime.datetime.now()) t.put() #else: #logging.info('version data unchanged') q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def firefox(project, data): url = 'http://people.mozilla.com/~tmielczarek/branch_versions.json' content = urlfetch.fetch(url).content project_data = json.loads(content) sha = hashlib.sha1() sha.update(content) sha = sha.hexdigest() q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha) if q.count() == 0: logging.info('refreshing ' + project + ' version data') version_version = project_data[data['key']] t = VersionCache(project=project, version=version_version, commit=sha, date=datetime.datetime.now()) t.put() else: logging.info('version data for ' + project + ' unchanged') q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def bytags(p, data): repo = data['repo'] # Loading tags for current project tags_url = ghAuth('https://api.github.com/repos/' + repo + '/tags') tags = json.loads(urlfetch.fetch(tags_url).content) refresh = False # Looping over tags for tag in tags: # If tags are not in the database, add them query = "SELECT * FROM VersionCache WHERE project = :1 AND commit = :2" q = db.GqlQuery(query, p, tag['commit']['sha']) if (q.count() == 0): refresh = True commit_url = ghAuth( 'https://api.github.com/repos/' + repo + '/commits/' + tag['commit']['sha'] ) commit = json.loads(urlfetch.fetch(commit_url).content) # Parse the date to python datetime version_date = commit['commit']['author']['date'] version_date = iso8601date.parse_iso8601_datetime(version_date) # Remove leading v from version number if present version_version = re.sub('^v(?=\d)', '', tag['name']) t = VersionCache(project=p, version=version_version, commit=tag['commit']['sha'], date=version_date) t.put() #if refresh: #logging.info('refreshed version data') #else: #logging.info('version data unchanged') # Return the most recent released version query = ( "SELECT version FROM VersionCache WHERE project = :1 " + "ORDER BY date DESC" ) q = db.GqlQuery(query, p).get() return q.version
def byjsonfile(project, data): repo = data["repo"] filename = data["file"] url = ghAuth("https://api.github.com/repos/" + repo + "/contents/" + filename) project_data = json.loads(urlfetch.fetch(url).content) sha = project_data["sha"] q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha) if q.count() == 0: # logging.info('refreshing version data') version_version = json.loads(base64.b64decode(project_data["content"]))["version"] t = VersionCache(project=project, version=version_version, commit=sha, date=datetime.datetime.now()) t.put() # else: # logging.info('version data unchanged') q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def chrome(project, data): url = data['source'] content = memcache.get('cache:' + url) if content is None: content = urlfetch.fetch(url).content memcache.add('cache:' + url, content, 120) project_data = json.loads(content) sha = hashlib.sha1() sha.update(content) sha = sha.hexdigest() q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha) if q.count() == 0: #logging.info(project + ': refreshing version data') for os in project_data: if os['os'] == data['os']: os_versions = os['versions'] for vs in os_versions: if vs['channel'] == data['channel']: version_version = vs['version'] version_date = datetime.datetime.strptime(vs['date'], "%m/%d/%y") t = VersionCache(project=project, version=version_version, commit=sha, date=version_date) t.put() #else: #logging.info(project + ': version data for unchanged') q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def byjsonfile(project, data): repo = data['source'] filename = data['file'] url = 'https://api.github.com/repos/' + repo + '/contents/' + filename project_data = json.loads(urlfetch.fetch(url).content) sha = project_data['sha'] q = db.GqlQuery("SELECT * FROM VersionCache WHERE project = :1 AND commit = :2", project, sha) if q.count() == 0: logging.info('refreshing ' + project + ' version data') version_version = json.loads(base64.b64decode(project_data['content']))['version'] t = VersionCache(project=project, version=version_version, commit=sha, date=datetime.datetime.now()) t.put() else: logging.info('version data for ' + project + ' unchanged') q = db.GqlQuery("SELECT version FROM VersionCache WHERE project = :1 ORDER BY date DESC", project).get() return q.version
def get(self, project): logging.info('Deleting tag cache from project: ' + project) for person in VersionCache.gql("WHERE project = :1", project): person.delete() logging.info('Deletion of ' + project + ' cache done.') self.response.write('Purged cache for: ' + project)