def __init__(self, content_type, path, root, logger=None): """ :param content_type: Content type. :param path: Path to content file or directory relative to repository root. :param root: Repository root path. :param logger: Optional logger instance. """ self.content_type = content_type self.rel_path = path self.root = root self.name = self.make_name() self.log = logutils.ContentTypeAdapter( logger or default_logger, self.content_type, self.name)
def _import_repository(import_task, logger): repository = import_task.repository repo_full_name = ( repository.provider_namespace.name + "/" + repository.original_name) logger.info(u'Starting import: task_id={}, repository={}' .format(import_task.id, repo_full_name)) logger.info(' ') token = _get_social_token(import_task) gh_api = github.Github(token) gh_repo = gh_api.get_repo(repo_full_name) try: repo_info = i_repo.import_repository( repository.clone_url, temp_dir=settings.CONTENT_DOWNLOAD_DIR, logger=logger) except i_exc.ImporterError as e: raise exc.TaskError(str(e)) repository.import_branch = repo_info.branch repository.format = repo_info.format.value repository.travis_status_url = import_task.travis_status_url repository.travis_build_url = import_task.travis_build_url if repo_info.name: old_name = repository.name new_name = repo_info.name if old_name != new_name: logger.info( u'Updating repository name "{old_name}" -> "{new_name}"' .format(old_name=old_name, new_name=new_name)) repository.name = new_name context = utils.Context( repository=repository, github_token=token, github_client=gh_api, github_repo=gh_repo) new_content_objs = [] for content_info in repo_info.contents: content_logger = logutils.ContentTypeAdapter( logger, content_info.content_type, content_info.name) importer_cls = importers.get_importer(content_info.content_type) importer = importer_cls(context, content_info, logger=content_logger) issue_tracker_url = '' if (hasattr(content_info, 'role_meta') and getattr(content_info, 'role_meta') and content_info.role_meta.get('issue_tracker_url')): issue_tracker_url = content_info.role_meta['issue_tracker_url'] elif gh_repo.has_issues: issue_tracker_url = gh_repo.html_url + '/issues' repository.issue_tracker_url = issue_tracker_url content_obj = importer.do_import() new_content_objs.append(content_obj.id) for obj in repository.content_objects.exclude(id__in=new_content_objs): logger.info( 'Deleting Content instance: content_type={0}, ' 'namespace={1}, name={2}'.format( obj.content_type, obj.namespace, obj.name)) obj.delete() _update_readme(repository, repo_info.readme, gh_api, gh_repo) _update_namespace(gh_repo) _update_repo_info(repository, gh_repo, repo_info.commit, repo_info.description) repository.save() _update_task_msg_content_id(import_task) _update_quality_score(import_task) _cleanup_old_task_msg(import_task) # Updating versions has to go last because: # - we don't want to update the version number if the import fails. # - version updates send out email notifications and we don't want to # notify people that an update happened if it failed on one of the other # steps _update_repository_versions(repository, gh_repo, logger) warnings = import_task.messages.filter( message_type=models.ImportTaskMessage.TYPE_WARNING).count() errors = import_task.messages.filter( message_type=models.ImportTaskMessage.TYPE_ERROR).count() import_task.finish_success( 'Import completed with {0} warnings and {1} ' 'errors'.format(warnings, errors)) if repository.is_new: user_notifications.author_release.delay(repository.id) repository.is_new = False repository.save() namespace = repository.provider_namespace.namespace.name fields = { 'content_name': '{}.{}'.format(namespace, repository.name), 'content_id': repository.id, 'community_score': repository.community_score, 'quality_score': repository.quality_score, } serializers.influx_insert_internal({ 'measurement': 'content_score', 'fields': fields })
def _import_repository(import_task, logger): repository = import_task.repository repo_full_name = (repository.provider_namespace.name + "/" + repository.original_name) logger.info(u'Starting import: task_id={}, repository={}'.format( import_task.id, repo_full_name)) if import_task.import_branch: repository.import_branch = import_task.import_branch token = _get_social_token(import_task) gh_api = github.Github(token) gh_repo = gh_api.get_repo(repo_full_name) try: repo_info = i_repo.import_repository( repository.clone_url, branch=repository.import_branch, temp_dir=settings.CONTENT_DOWNLOAD_DIR, logger=logger) except i_exc.ImporterError as e: raise exc.TaskError(str(e)) if repository.import_branch is None: repository.import_branch = repo_info.branch repository.format = repo_info.format.value if repo_info.name: old_name = repository.name new_name = common.sanitize_content_name(repo_info.name) if old_name != new_name: logger.info( u'Updating repository name "{old_name}" -> "{new_name}"'. format(old_name=old_name, new_name=new_name)) repository.name = new_name context = utils.Context(repository=repository, github_token=token, github_client=gh_api, github_repo=gh_repo) new_content_objs = [] for content_info in repo_info.contents: content_logger = logutils.ContentTypeAdapter(logger, content_info.content_type, content_info.name) importer_cls = importers.get_importer(content_info.content_type) importer = importer_cls(context, content_info, logger=content_logger) issue_tracker_url = '' if (hasattr(content_info, 'role_meta') and getattr(content_info, 'role_meta') and content_info.role_meta.get('issue_tracker_url')): issue_tracker_url = content_info.role_meta['issue_tracker_url'] elif gh_repo.has_issues: issue_tracker_url = gh_repo.html_url + '/issues' repository.issue_tracker_url = issue_tracker_url content_obj = importer.do_import() new_content_objs.append(content_obj.id) for obj in repository.content_objects.exclude(id__in=new_content_objs): logger.info('Deleting Content instance: content_type={0}, ' 'namespace={1}, name={2}'.format(obj.content_type, obj.namespace, obj.name)) obj.delete() _update_readme(repository, repo_info.readme, gh_api, gh_repo) _update_repository_versions(repository, gh_repo, logger) _update_namespace(gh_repo) _update_repo_info(repository, gh_repo, repo_info.commit) repository.save() import_task.finish_success(u'Import completed')