def make_content(self):
repo = self.ctx.repository
ns = repo.provider_namespace.namespace
# Name is the content name, which in the case of multi content repos
# might not be the same as the repository name
name = repo.name
original_name = repo.original_name
if self.data.name:
name = self.data.name
if self.data.original_name:
original_name = self.data.original_name
# Check name
if not re.match('^[\w-]+$', name):
raise exc.TaskError('Invalid name, only aplhanumeric characters, '
'"-" and "_" symbols are allowed.')
obj, is_created = models.Content.objects.get_or_create(
namespace=ns,
repository=repo,
content_type=models.ContentType.get(self.data.content_type),
original_name=original_name,
defaults={
'name': self.translate_content_name(name),
'is_valid': False,
})
self._log_create_content(obj.id, is_created)
return obj
def _get_social_token(import_task):
user = import_task.owner
try:
token = auth_models.SocialToken.objects.get(
account__user=user, account__provider='github')
return token.token
except Exception:
raise exc.TaskError(
u"Failed to get GitHub account for Galaxy user {0}. "
u"You must first authenticate with GitHub.".format(user.username))
def _import_repository(import_task, logger):
repository = import_task.repository
repo_full_name = (
repository.provider_namespace.name
+ "/" + repository.original_name)
logger.info(u'Starting import: task_id={}, repository={}'
.format(import_task.id, repo_full_name))
logger.info(' ')
token = _get_social_token(import_task)
gh_api = github.Github(token)
gh_repo = gh_api.get_repo(repo_full_name)
try:
repo_info = i_repo.import_repository(
repository.clone_url,
temp_dir=settings.CONTENT_DOWNLOAD_DIR,
logger=logger)
except i_exc.ImporterError as e:
raise exc.TaskError(str(e))
repository.import_branch = repo_info.branch
repository.format = repo_info.format.value
repository.travis_status_url = import_task.travis_status_url
repository.travis_build_url = import_task.travis_build_url
if repo_info.name:
old_name = repository.name
new_name = repo_info.name
if old_name != new_name:
logger.info(
u'Updating repository name "{old_name}" -> "{new_name}"'
.format(old_name=old_name, new_name=new_name))
repository.name = new_name
context = utils.Context(
repository=repository, github_token=token,
github_client=gh_api, github_repo=gh_repo)
new_content_objs = []
for content_info in repo_info.contents:
content_logger = logutils.ContentTypeAdapter(
logger, content_info.content_type, content_info.name)
importer_cls = importers.get_importer(content_info.content_type)
importer = importer_cls(context, content_info, logger=content_logger)
issue_tracker_url = ''
if (hasattr(content_info, 'role_meta')
and getattr(content_info, 'role_meta')
and content_info.role_meta.get('issue_tracker_url')):
issue_tracker_url = content_info.role_meta['issue_tracker_url']
elif gh_repo.has_issues:
issue_tracker_url = gh_repo.html_url + '/issues'
repository.issue_tracker_url = issue_tracker_url
content_obj = importer.do_import()
new_content_objs.append(content_obj.id)
for obj in repository.content_objects.exclude(id__in=new_content_objs):
logger.info(
'Deleting Content instance: content_type={0}, '
'namespace={1}, name={2}'.format(
obj.content_type, obj.namespace, obj.name))
obj.delete()
_update_readme(repository, repo_info.readme, gh_api, gh_repo)
_update_namespace(gh_repo)
_update_repo_info(repository, gh_repo, repo_info.commit,
repo_info.description)
repository.save()
_update_task_msg_content_id(import_task)
_update_quality_score(import_task)
_cleanup_old_task_msg(import_task)
# Updating versions has to go last because:
# - we don't want to update the version number if the import fails.
# - version updates send out email notifications and we don't want to
# notify people that an update happened if it failed on one of the other
# steps
_update_repository_versions(repository, gh_repo, logger)
warnings = import_task.messages.filter(
message_type=models.ImportTaskMessage.TYPE_WARNING).count()
errors = import_task.messages.filter(
message_type=models.ImportTaskMessage.TYPE_ERROR).count()
import_task.finish_success(
'Import completed with {0} warnings and {1} '
'errors'.format(warnings, errors))
if repository.is_new:
user_notifications.author_release.delay(repository.id)
repository.is_new = False
repository.save()
namespace = repository.provider_namespace.namespace.name
fields = {
'content_name': '{}.{}'.format(namespace, repository.name),
'content_id': repository.id,
'community_score': repository.community_score,
'quality_score': repository.quality_score,
}
serializers.influx_insert_internal({
'measurement': 'content_score',
'fields': fields
})
def _import_repository(import_task, logger):
repository = import_task.repository
repo_full_name = (repository.provider_namespace.name + "/" +
repository.original_name)
logger.info(u'Starting import: task_id={}, repository={}'.format(
import_task.id, repo_full_name))
if import_task.import_branch:
repository.import_branch = import_task.import_branch
token = _get_social_token(import_task)
gh_api = github.Github(token)
gh_repo = gh_api.get_repo(repo_full_name)
try:
repo_info = i_repo.import_repository(
repository.clone_url,
branch=repository.import_branch,
temp_dir=settings.CONTENT_DOWNLOAD_DIR,
logger=logger)
except i_exc.ImporterError as e:
raise exc.TaskError(str(e))
if repository.import_branch is None:
repository.import_branch = repo_info.branch
repository.format = repo_info.format.value
if repo_info.name:
old_name = repository.name
new_name = common.sanitize_content_name(repo_info.name)
if old_name != new_name:
logger.info(
u'Updating repository name "{old_name}" -> "{new_name}"'.
format(old_name=old_name, new_name=new_name))
repository.name = new_name
context = utils.Context(repository=repository,
github_token=token,
github_client=gh_api,
github_repo=gh_repo)
new_content_objs = []
for content_info in repo_info.contents:
content_logger = logutils.ContentTypeAdapter(logger,
content_info.content_type,
content_info.name)
importer_cls = importers.get_importer(content_info.content_type)
importer = importer_cls(context, content_info, logger=content_logger)
issue_tracker_url = ''
if (hasattr(content_info, 'role_meta')
and getattr(content_info, 'role_meta')
and content_info.role_meta.get('issue_tracker_url')):
issue_tracker_url = content_info.role_meta['issue_tracker_url']
elif gh_repo.has_issues:
issue_tracker_url = gh_repo.html_url + '/issues'
repository.issue_tracker_url = issue_tracker_url
content_obj = importer.do_import()
new_content_objs.append(content_obj.id)
for obj in repository.content_objects.exclude(id__in=new_content_objs):
logger.info('Deleting Content instance: content_type={0}, '
'namespace={1}, name={2}'.format(obj.content_type,
obj.namespace, obj.name))
obj.delete()
_update_readme(repository, repo_info.readme, gh_api, gh_repo)
_update_repository_versions(repository, gh_repo, logger)
_update_namespace(gh_repo)
_update_repo_info(repository, gh_repo, repo_info.commit)
repository.save()
import_task.finish_success(u'Import completed')
