def experiment_single_issue(request, experiment_id, issue_nr):
"""View single GitHub issue from experiment
Todo: this view is currently not used"""
experiment = verify_and_get_experiment(request, experiment_id)
github_helper = GitHubHelper(request.user, experiment.git_repo.name)
issue = github_helper.get_single_issue(int(issue_nr))
return JsonResponse({'contents': issue.body, 'title': issue.title})
def readme_file_of_package(self):
internalpackage = InternalPackage.objects.get(id=self.kwargs['pk'])
github_helper = GitHubHelper(internalpackage.owner,
internalpackage.git_repo.name)
readme = github_helper.view_file('README.md')
md = Markdown()
content_file = md.convert(readme)
return content_file
def test_create_empty_r_package(self, mock_social_token):
"""Test if an empty R package can be initialized"""
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
task_create_package(self.internal_r_package.pk)
self.internal_r_package.refresh_from_db()
self.assertIsNotNone(self.internal_r_package)
github_helper = GitHubHelper('test', self.r_package_name)
self.assertTrue(github_helper.view_file('DESCRIPTION'))
def test_create_new_r_experiment(self, mock_social_token):
"""Test if an R experiment can be initialized"""
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
initialize_repository(self.r_experiment.pk, self.r_cookiecutter.pk)
self.r_experiment.refresh_from_db()
self.assertIsNotNone(self.r_experiment)
github_helper = GitHubHelper('test', self.r_experiment_name)
self.assertTrue(github_helper.view_file('README.md'))
self.assertTrue(github_helper.view_file('src/.gitignore'))
def test_create_new_python_experiment(self, mock_social_token):
"""Test if a Python experiment can be initialized"""
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
initialize_repository(self.python_experiment.pk,
self.python_cookiecutter.pk)
self.python_experiment.refresh_from_db()
self.assertIsNotNone(self.python_experiment)
github_helper = GitHubHelper('test', self.python_experiment_name)
self.assertTrue(github_helper.view_file('requirements.txt'))
def create_tag_for_package_version(package_version_id):
package_version = PackageVersion.objects.get(pk=package_version_id)
internalpackage = InternalPackage.objects.get(
pk=package_version.package.pk)
github = GitHubHelper(internalpackage.owner, internalpackage.git_repo.name)
github.create_release(package_version.version_nr,
package_version.version_nr,
package_version.changelog,
package_version.pre_release)
def get(self, request, experiment_id):
context = super(FileViewGitRepository, self).get(request, experiment_id)
file_name = request.GET['file_name']
context['file_name'] = file_name
experiment = verify_and_get_experiment(request, experiment_id)
github_helper = GitHubHelper(request.user, experiment.git_repo.name)
content_file = github_helper.view_file(file_name)
pylint_results = return_results_for_file(experiment, file_name)
context['content_file'] = self.add_pylint_results_to_content(pylint_results, content_file)
return render(request, 'experiments_manager/file_detail.html', context)
def experiment_issues(request, experiment_id):
"""Retrieves the GitHub issues of experiment
To do: this view is currently not used."""
experiment = verify_and_get_experiment(request, experiment_id)
github_helper = GitHubHelper(request.user, experiment.git_repo.name)
context = {'object': experiment,
'object_type': experiment.get_object_type(),
'issues_active': True,
'issues': github_helper.get_issues()}
return render(request, 'experiments_manager/experiment_detail/experiment_issues.html', context)
def test_create_r_package_from_experiment(self, mock_social_token):
"""Test to create a R package from an existing experiment"""
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
task_create_package_from_experiment(self.internal_r_package.pk,
self.r_experiment.pk,
'/src')
self.assertIsNotNone(self.internal_r_package)
github_helper = GitHubHelper('test', self.r_package_name)
self.assertTrue(github_helper.view_file('DESCRIPTION'))
self.assertIsNotNone(github_helper.view_file('R/make_dataset.R'))
def test_create_python_package_from_experiment(self, mock_social_token):
"""Test to create a Python package from an existing experiment"""
github_token = os.environ.get('GITHUB_TOKEN')
mock_social_token.return_value = github_token
task_create_package_from_experiment(self.internal_package.pk,
self.python_experiment.pk,
'/src/data/')
self.assertIsNotNone(self.internal_package)
github_helper = GitHubHelper('test', self.python_package_name)
self.assertTrue(github_helper.view_file('requirements.txt'))
self.assertTrue(github_helper.view_file('/{0}/make_dataset.py'.format(self.python_package_name)))
def update_setup_py_with_new_version(package_version_id):
package_version = PackageVersion.objects.get(pk=package_version_id)
internal_package = InternalPackage.objects.get(
pk=package_version.package.pk)
gh_helper = GitHubHelper(internal_package.owner,
internal_package.git_repo.name)
setup_py = gh_helper.view_file('setup.py')
setup_py = update_setup_py_new_version(package_version.version_nr,
setup_py)
gh_helper.update_file('setup.py',
contents=setup_py,
commit_message="Updated setup.py for new version")
def measure(self):
docs = Docs.objects.filter(experiment=self.experiment)
docs_instance = None
if docs:
docs_instance = docs[0]
if docs_instance and docs_instance.enabled:
github_helper = GitHubHelper(self.experiment.owner,
self.experiment.git_repo.name)
steps = ChosenExperimentSteps.objects.filter(
experiment=self.experiment)
sphinx_helper = SphinxHelper(self.experiment, steps, github_helper)
coverage = sphinx_helper.get_coverage_data()
if coverage:
undoced_funcs = coverage[1]
undoced_classes = coverage[2]
if undoced_classes == 0 and undoced_funcs == 0:
self.result.result = ExperimentMeasureResult.HIGH
elif undoced_funcs < 10 and undoced_classes < 10:
self.result.result = ExperimentMeasureResult.MEDIUM
else:
self.result.result = ExperimentMeasureResult.LOW
self.add_raw_values(coverage)
else:
self.result.result = ExperimentMeasureResult.LOW
def task_write_data_schema(experiment_id):
"""Task to write data schema to GitHub in schema/schema.json file
By converting the schema to dictionary and then to Json"""
experiment = Experiment.objects.get(pk=experiment_id)
data_schema = experiment.schema
data_schema_str = str(json.dumps(data_schema.to_dict()))
logger.info('writing data schema for %s with schema: %s', experiment,
data_schema_str)
github_helper = GitHubHelper(experiment.owner.user,
experiment.git_repo.name)
github_helper.update_file('schema/schema.json',
'Updated data schema by MOOC workbench',
data_schema_str)
username = experiment.owner.user.username
send_message(username, MessageStatus.SUCCESS,
'Data schema successfully updated in GitHub')
logger.info('writing data schema success for: %s', experiment,
data_schema_str)
def tearDown(self, mock_social_token):
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
try:
github_helper = GitHubHelper('test', self.python_package_name)
github_helper._delete_repository()
except Exception as e:
print(e)
try:
r_github_helper = GitHubHelper('test', self.r_package_name)
r_github_helper._delete_repository()
except Exception as e:
print(e)
def test_create_existing_package(self, mock_social_token):
"""Test if an existing package can be added"""
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
task_create_package(self.internal_package.pk)
self.internal_package.refresh_from_db()
self.internal_package.delete()
new_internal_package = InternalPackage.objects.create(name=self.python_package_name,
description='Package',
language_id=2,
category_id=1,
owner_id=1,
template_id=1)
task_create_package(new_internal_package.pk)
new_internal_package.refresh_from_db()
self.assertIsNotNone(new_internal_package)
github_helper = GitHubHelper('test', self.python_package_name)
self.assertTrue(github_helper.view_file('requirements.txt'))
def task_read_data_schema(repository_name):
"""Reads data schema from a schema/schema.json and converts it to DataSchema models in DB"""
experiment = get_exp_or_package_from_repo_name(repository_name)
if isinstance(experiment, Experiment):
logger.info('reading data schema for: %s', experiment)
github_helper = GitHubHelper(experiment.owner.user,
experiment.git_repo.name)
schema_json = json.loads(github_helper.view_file('schema/schema.json'))
data_schema_fields = parse_json_table_schema(schema_json)
data_schema = experiment.schema
for field in data_schema.fields.all():
data_schema.fields.remove(field)
data_schema.save()
field.delete()
for new_field in data_schema_fields:
new_field.save()
data_schema.fields.add(new_field)
data_schema.save()
logger.info('read data schema for: %s', experiment, data_schema_fields)
def experiment_generate_uuid_and_make_public(request, pk, slug):
"""Publishes an experiment by generating a unique ID of length 32,
setting the experiment public, creating a Git release and saving the GitHub release URL"""
experiment = verify_and_get_experiment(request, pk)
if not experiment.unique_id:
experiment.unique_id = get_random_string(32)
experiment.public = True
experiment.save()
# create github release
github_helper = GitHubHelper(experiment.owner, experiment.git_repo.name)
tag_name = 'release_{0}'.format(experiment.unique_id)
git_release = github_helper.create_release(tag_name=tag_name, name=experiment.title,
body='This is an automatically created release by the MOOC workbench',
pre_release=False)
experiment.publish_url_zip = git_release.html_url
# https://github.com/PyGithub/PyGithub/pull/522#pullrequestreview-22171287
# once merged and released, change this to zipball_url pls kthxby
experiment.save()
return redirect(to=reverse('experiment_readonly', kwargs={'unique_id': experiment.unique_id}))
def tearDown(self, mock_social_token):
mock_social_token.return_value = os.environ.get('GITHUB_TOKEN')
try:
github_helper = GitHubHelper('test', self.python_experiment_name)
github_helper._delete_repository()
except Exception:
pass
try:
github_helper = GitHubHelper('test', self.r_experiment_name)
github_helper._delete_repository()
except Exception:
pass
def toggle_docs_status(request, object_id, object_type):
exp_or_package = get_package_or_experiment(request, object_type, object_id)
docs = exp_or_package.docs
docs.enabled = not docs.enabled
docs.save()
if docs.enabled:
github_helper = GitHubHelper(request.user,
exp_or_package.git_repo.name)
git_helper = GitHelper(github_helper)
git_helper.clone_or_pull_repository()
return redirect(exp_or_package.get_absolute_url())
def get_context_data(self, **kwargs):
experiment = self.object
self.object_type = self.get_requirement_type(experiment)
github_helper = GitHubHelper(experiment.owner, experiment.git_repo.name)
context = super(ExperimentReadOnlyView, self).get_context_data(**kwargs)
context['steps'] = get_files_for_steps(experiment, github_helper)
context['object_type'] = self.object_type
context['object_id'] = experiment.pk
context['completed'] = True
context['object_list'] = self.object.requirements.all()
context['data_schema_list'] = self.object.schema.fields.all()
context['readme'] = get_readme_of_experiment(self.object)
context['experiment_id'] = self.object.id
return context
def internalpackage_publish_update(package):
"""Publishes an internalpackage, or, if it is already published, publishes a new version.
This function is for Python only and publishes packages on the PyPi server.
:param package: The package to publish or update"""
# clone/pull repository
# copy repository to packages
package_location = os.path.join(PACKAGES_DIR, package.git_repo.name)
if not os.path.exists(package_location):
github_helper = GitHubHelper(package.owner, package.git_repo.name)
git_helper = GitHelper(github_helper)
git_helper.clone_or_pull_repository()
shutil.copytree(git_helper.repo_dir, package_location)
else:
subprocess.call(['git', 'pull'], cwd=package_location)
subprocess.call(['./shell_scripts/publish_package.sh', package_location])
def measure(self):
travis_instance = TravisInstance.objects.filter(experiment=self.experiment)
if travis_instance:
travis_instance = travis_instance[0]
if travis_instance.enabled:
github_helper = GitHubHelper(self.experiment.owner, self.experiment.git_repo.name)
travis_helper = TravisCiHelper(github_helper)
last_build_passed = travis_helper.get_last_build_status()
self.add_raw_value(last_build_passed)
if self.get_last_five_build_results():
self.result.result = ExperimentMeasureResult.HIGH
else:
self.result.result = ExperimentMeasureResult.MEDIUM
else:
self.result.result = ExperimentMeasureResult.LOW
def measure(self):
travis_ci_config = TravisInstance.objects.filter(
experiment=self.experiment).first()
if travis_ci_config:
if travis_ci_config.enabled:
github_helper = GitHubHelper(self.experiment.owner,
self.experiment.git_repo.name)
travis_helper = TravisCiHelper(github_helper)
last_build_log = travis_helper.get_log_for_last_build()
complete_requirements_file = self.find_if_missing_dependency(
last_build_log)
if complete_requirements_file:
self.result.result = ExperimentMeasureResult.HIGH
else:
self.result.result = ExperimentMeasureResult.LOW
else:
self.result.result = ExperimentMeasureResult.LOW
def step_four_experiment_first_time(request, pk):
"""Displays the page for enabling CI and Code Coverage, and
config commands for getting started with the experiment"""
experiment = verify_and_get_experiment(request, pk)
context = {}
gh_helper = GitHubHelper(experiment.owner, experiment.git_repo.name)
context['object'] = experiment
context['object_type'] = ExperimentPackageTypeMixin.EXPERIMENT_TYPE
context['object_id'] = experiment.pk
context['configured'] = experiment.travis.enabled
context['github_username'] = gh_helper.owner
context['reposlug'] = experiment.git_repo.name
context['travis'] = experiment.travis
if experiment.travis.codecoverage_set:
context[
'coverage_configured'] = experiment.travis.codecoverage_set.first(
).enabled
return render(
request,
'experiments_manager/experiment_create/experiment_enable_builds.html',
context)
def test_view_file_in_repo(self, mock_github):
mock_github.return_value = GithubMock()
github_helper = GitHubHelper(self.user, 'test')
self.assertIsNotNone(github_helper)
def test_init_github_helper_create(self, mock_github):
mock_github.return_value = GithubMock()
github_helper = GitHubHelper(self.user, 'test', create=True)
self.assertIsNotNone(github_helper)
self.assertEqual(github_helper.github_repository.repo_name,
'newly_created_repo')
def test_github_owner(self, mock_github):
mock_github.return_value = GithubMock()
github_helper = GitHubHelper(self.user, 'test')
self.assertIsNotNone(github_helper)
self.assertEqual(github_helper.owner, 'test')
def test_init_github_helper(self, mock_github):
mock_github.return_value = GithubMock()
github_helper = GitHubHelper(self.user, 'test')
self.assertIsNotNone(github_helper)
self.assertEqual(github_helper.get_clone_url(), 'https://A@clone-url')
def test_github_list_folder(self, mock_github):
mock_github.return_value = GithubMock()
github_helper = GitHubHelper(self.user, 'test')
self.assertIsNotNone(github_helper)
self.assertEqual(github_helper.list_files_in_folder('test1'),
GithubMockRepo.TEST1_LIST)
def get_readme_of_experiment(experiment):
github_helper = GitHubHelper(experiment.owner.user,
experiment.git_repo.name)
content_file = github_helper.view_file('README.md')
md = Markdown()
return md.convert(content_file)