def make_repo_url(x, name):
# parse comments
comments = []
if x.has_comment():
for c in x.comment.split(','):
c = c.strip()
if not c.startswith('@'):
continue
c = c.split(' ')
key = c[0][1:] # without the @
if len(c) > 1:
value = c[1]
if key == 'archived':
comments.append(make_text('archived', css_class='is-size-7'))
if key == 'created':
comments.append(make_text('since {}'.format(value), css_class='is-size-7'))
if key == 'stars':
value = int(value)
if value > 200:
comments.append(make_icon('star', 'top rated'))
elif value > 30:
comments.append(make_icon('star-half-full', 'medium rated'))
else:
comments.append(make_icon('star-o', 'low rated'))
# this is the default element
url = make_url(x.value, shortcut_url(x.value, name), css_class='is-size-7')
if comments:
return make_enumeration([url, make_enumeration(comments)], '')
else:
return url
def get_top50_games(games):
top50_games = []
for game in games:
# get stars of repositories
stars = 0
for repo in game.get('Code repository', []):
if repo.has_comment():
for c in repo.comment.split(','):
c = c.strip()
if not c.startswith('@'):
continue
c = c.split(' ')
key = c[0][1:] # without the @
if len(c) > 1:
value = c[1]
if key == 'stars':
value = int(value)
if value > stars:
stars = value
top50_games.append((game, stars))
top50_games.sort(key=lambda x:x[1], reverse=True)
top50_games = top50_games[:50]
top50_games =[game for game, stars in top50_games]
return top50_games
def gitlab_import():
"""
Import various information from Gitlab repositories (like contributors) or stars for Gitlab repos
"""
private_properties = json.loads(utils.read_text(c.private_properties_file))
files = json.loads(utils.read_text(gl_entries_file))
all_developers = osg.read_developers()
print(' {} developers read'.format(len(all_developers)))
# all exceptions that happen will be eaten (but will end the execution)
try:
# loop over each entry
for index, file in enumerate(files):
print(' process {} ({})'.format(file, index))
# read entry
entry = osg.read_entry(file)
code_repositories = entry['Code repository']
repos = [x for x in code_repositories if x.startswith(prefix)]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]
repos = [x for x in repos if x not in ignored_repos]
for repo in repos:
print(' GH repo {}'.format(repo))
info = osg_gitlab.retrieve_repo_info(repo)
new_comments = []
# add created comment
new_comments.append('@created {}'.format(info['created'].year))
# add stars
new_comments.append('@stars {}'.format(info['stars']))
# add forks
new_comments.append('@forks {}'.format(info['forks']))
# search for repository
for r in code_repositories:
if r.startswith(repo):
break
# update comment
comments = r.comment
if comments:
comments = comments.split(',')
comments = [c.strip() for c in comments]
comments = [c for c in comments
if not c.startswith('@')] # delete old ones
comments += new_comments
else:
comments = new_comments
r.comment = ', '.join(comments)
# language in languages
for language, usage in info['languages'].items():
if language in c.known_languages and usage > 5 and language not in entry[
'Code language']:
entry['Code language'].append(language)
print(' added to languages: {}'.format(language))
entry['Code repository'] = code_repositories
osg.write_entry(entry)
except:
raise
finally:
# shorten file list
utils.write_text(gl_entries_file, json.dumps(files[index:], indent=1))
# osg.write_developers(all_developers)
print('developers database updated')
def github_import():
"""
:return:
"""
private_properties = json.loads(utils.read_text(c.private_properties_file))
files = json.loads(utils.read_text(gh_entries_file))
all_developers = osg.read_developers()
print(' {} developers read'.format(len(all_developers)))
# all exceptions that happen will be eaten (but will end the execution)
try:
# loop over each entry
for index, file in enumerate(files):
print(' process {}'.format(file))
# read entry
entry = osg.read_entry(file)
code_repositories = entry['Code repository']
repos = [
x.value for x in code_repositories if x.startswith(prefix)
]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]
repos = [x for x in repos if x not in ignored_repos]
for repo in repos:
print(' GH repo {}'.format(repo))
info = osg_github.retrieve_repo_info(
repo, private_properties['github-token'])
new_comments = []
# is archived
if info['archived']:
if not osg.is_inactive(entry):
print(
'warning: repo is archived but not inactive state??'
)
# add archive to repo comment
new_comments.append('@archived')
# add created comment
new_comments.append('@created {}'.format(info['created'].year))
# add stars
new_comments.append('@stars {}'.format(info['stars']))
# add forks
new_comments.append('@forks {}'.format(info['forks']))
# update comment
for r in code_repositories:
if r.value.startswith(repo):
break
comments = r.comment
if comments:
comments = comments.split(',')
comments = [c.strip() for c in comments]
comments = [c for c in comments
if not c.startswith('@')] # delete old ones
comments += new_comments
else:
comments = new_comments
r.comment = ', '.join(comments)
# language in languages
language = info['language']
language = language_aliases.get(language, language)
if language and language not in entry[
'Code language'] and language not in ignored_languages:
entry['Code language'].append(
osg_parse.ValueWithComment(language))
print(' added to languages: {}'.format(language))
# contributors
for contributor in info['contributors']:
if contributor.type != 'User':
continue
if contributor.contributions < 4:
continue
# contributor.login/name/blog
name = contributor.name
if not name:
name = contributor.login
name = name_aliases.get(name, name)
nickname = '{}@GH'.format(contributor.login)
blog = contributor.blog
if blog:
blog = blog_alias[blog] if blog in blog_alias else blog
if not blog.startswith('http'):
blog = 'https://' + blog
if blog in ignored_blogs:
blog = None
# look up author in entry developers
if name not in entry.get('Developer', []):
print(' dev "{}" added to entry {}'.format(
name, file))
entry['Developer'] = entry.get('Developer', []) + [
osg_parse.ValueWithComment(name)
]
# look up author in developers data base
if name in all_developers:
dev = all_developers[name]
if not nickname in dev.get('Contact', []):
print(
' existing dev "{}" added nickname ({}) to developer database'
.format(name, nickname))
# check that name has not already @GH contact
if any(
x.endswith('@GH')
for x in dev.get('Contact', [])):
print('warning: already GH contact')
dev['Contact'] = dev.get('Contact',
[]) + [nickname]
if blog and blog not in dev.get('Home', []):
dev['Home'] = dev.get('Home', []) + [blog]
# TODO add to games entries!
else:
print(' dev "{}" ({}) added to developer database'.
format(name, nickname))
all_developers[name] = {
'Name': name,
'Contact': [nickname],
'Games': [entry['Title']]
}
if blog:
all_developers[name]['Home'] = [blog]
entry['Code repository'] = code_repositories
osg.write_entry(entry)
except:
raise
finally:
# shorten file list
utils.write_text(gh_entries_file, json.dumps(files[index:], indent=1))
osg.write_developers(all_developers)
print('developers database updated')
def special_ops(self):
"""
For special operations that are one-time and may change.
:return:
"""
if not self.entries:
print('entries not yet loaded')
return
# which fields have lots of comments
for field in c.valid_fields:
values = [
value for entry in self.entries
for value in entry.get(field, [])
]
if isinstance(values[0], osg_parse.ValueWithComment):
comments = [value.comment for value in values if value.comment]
# split by comma
comments = [
c.strip() for comment in comments
for c in comment.split(',')
]
print('field {} has {} comments'.format(field, len(comments)))
for comment in set(comments):
print(' {} - {}'.format(comment, comments.count(comment)))
# # remove download urls that are also in home
# for entry in self.entries:
# homes = entry['Home']
# downloads = entry.get('Download', [])
# downloads = [download for download in downloads if download not in homes]
# if downloads:
# entry['Download'] = downloads
# if not downloads and 'Download' in entry:
# del entry['Download']
# # collect statistics on git repositories
# created = {}
# stars = []
# forks = []
# for entry in self.entries:
# repos = entry['Code repository']
# comments = [x.comment for x in repos if x.value.startswith('https://github.com/') and x.comment]
# for comment in comments:
# comment = comment.split(',')
# comment = [c.strip() for c in comment]
# comment = [c for c in comment if c.startswith('@')]
# if comment:
# try:
# comment = [c.split(' ') for c in comment]
# comment = [c[1] for c in comment if len(c) > 1]
# except Exception:
# print(comment)
# raise
# created[comment[0]] = created.get(comment[0], 0) + 1
# stars.append(comment[1])
# forks.append(comment[2])
#
# for key, value in sorted(created.items(), key=lambda x: x[0]):
# print("{} : {}".format(key, value))
#
# import numpy as np
# np.set_printoptions(suppress=True)
# stars = np.array(stars, dtype=np.float)
# forks = np.array(forks, dtype=np.float)
# q = np.arange(0, 1, 0.333)
# print(q)
# print(np.quantile(stars, q))
# print(np.quantile(forks, q))
# # cvs without any git
# for entry in self.entries:
# repos = entry['Code repository']
# cvs = [repo for repo in repos if 'cvs' in repo]
# git = [repo for repo in repos if 'git' in repo]
# if len(cvs) > 0 and len(git) == 0:
# print('Entry "{}" with repos: {}'.format(entry['File'], repos))
# # combine content keywords
# n = len('content ')
# for entry in self.entries:
# keywords = entry['Keyword']
# content = [keyword for keyword in keywords if keyword.startswith('content')]
# if len(content) > 1:
# # remove from keywords
# keywords = [keyword for keyword in keywords if keyword not in content]
# # remove prefix
# content = [str(keyword)[n:].strip() for keyword in content]
# # join with +
# content = 'content {}'.format(' + '.join(content))
# keywords.append(osg_parse.ValueWithComment(content))
# entry['Keyword'] = keywords
# print('fixed "{}"'.format(entry['File']))
print('special ops finished')