Exemple #1
0
    def parseSourceIndex(_index, review=False, traverse_time=False):
        if not review:
            try:
                distroName, distroRelease = _index.distribution.sourceName.split('-', 1)
            except ValueError:
                distroName = _index.distribution.sourceName
                distroRelease = _index.distribution.version
            print '  Distribution: %s-%s' % (distroName, distroRelease)

            # repo name
            try:
                if _index.distribution.sourceName.find(" ") != -1:
                    binaryName = _index.distribution.sourceName.split()[1]
                else:
                    binaryName = _index.distribution.sourceName.split('-', 1)[1]
            except IndexError:
                if _index.distribution.binaryName == 'Contrib': binaryName = 'Contrib'
                else: binaryName = 'Pardus'
            if binaryName == 'Contrib': repoType = 'contrib'
            elif binaryName == 'Corporate': repoType = 'corporate'
            else: repoType = 'stable'
        else:
            distroName = "Pardus"
            distroRelease = "Playground"
            repoType = "review"

        # Add distribution to database
        try:
            distribution = Distribution.objects.get(name=distroName, release=distroRelease, type=repoType)
        except Distribution.DoesNotExist:
            distribution = Distribution(name=distroName, release=distroRelease, type=repoType)
            distribution.save()

        def importSpec(pspec, local_uri=None):
            # Add or update developer
            maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)
            review_flag = False
            # Create the source package information
            part_of = pspec.source.partOf
            if not local_uri:
                source_uri = pspec.source.sourceURI
            else:
                index = local_uri.find("review")
                source_uri = local_uri[index:]
                review_flag = True
            print '     Source URI: %s' % source_uri

            home_page = pspec.source.homepage
            print '     Homepage: %s' % home_page
            source_info = SourcePackageDetail.objects.create(part_of=part_of, source_uri=source_uri, home_page=home_page)
            for is_a in pspec.source.isA:
                print '     IsA: %s' % is_a
                source_info.isa_set.create(name=is_a)
            for license in pspec.source.license:
                print '     License: %s' % license
                source_info.license_set.create(name=license)
            summary = pspec.source.summary
            for language in summary.keys():
                print '     Summary Language: %s' % language
                print '     Summary Text: %s' % summary[language]
                source_info.summary_set.create(language=language, text=summary[language])
            description = pspec.source.description
            for language in description.keys():
                print '     Description Language: %s' % language
                print '     Description Text: %s' % description[language]
                source_info.description_set.create(language=language, text=description[language])

            # Add source or update maintainer
            try:
                source = Source.objects.get(name=pspec.source.name, distribution=distribution)
                source.maintained_by = maintained_by
                source.info = source_info
            except Source.DoesNotExist:
                source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by, info=source_info)
            print '  Source: %s' % source.name

            # Save the traverse time of the review path for the next check
            if review_flag:
                review = ReviewInfo.objects.create(check_time=traverse_time,status=True)
            else:
                review = ReviewInfo.objects.create(status=False)
            source.review = review
            source.save()

            # Update build dependencies
            for dep in BuildDependency.objects.filter(source=source):
                dep.delete()
            for dep in pspec.source.buildDependencies:
                dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                dependency.save()

            # Add or update package info
            for pack in pspec.packages:
                try:
                    package = Package.objects.get(name=pack.name, source=source)
                    package.save()
                except Package.DoesNotExist:
                    package = Package(name=pack.name, source=source)
                    package.save()
                    print '    New package: %s' % package.name

                for rep in pack.replaces:
                    replaces = Replaces(name=str(rep), package=package)
                    replaces.save()
                    for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
                        bin.resolution = 'removed'
                        bin.save()
                        print '    Marking %s-%s as removed' % (rep, bin.no)

                # Update runtime dependencies
                for dep in RuntimeDependency.objects.filter(package=package):
                    dep.delete()
                for dep in pack.runtimeDependencies():
                    try:
                        if isinstance(dep, pisi.specfile.AnyDependency):
                            # Any dependencies
                            for any_dep in dep.dependencies:
                                dependency = RuntimeDependency(package=package, name=any_dep.package, version=toString(any_dep.version), version_to=toString(any_dep.versionTo), version_from=toString(any_dep.versionFrom), release=toInt(any_dep.release), release_to=toInt(any_dep.releaseTo), release_from=toInt(any_dep.releaseFrom))
                                dependency.save()
                        else:
                            dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                            dependency.save()
                    except:
                        dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                        dependency.save()


            up_count = 0
            for up in pspec.history:
                updated_by = createUser(up.email, up.name)
                try:
                    update = Update.objects.get(no=up.release, source=source)
                    update.updated_on = up.date
                    update.updated_by = updated_by
                    #update.version_no = up.version_no
                    update.comment = up.comment
                    update.save()
                except Update.DoesNotExist:
                    update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
                    update.save()
                    up_count += 1
            if up_count > 0:
                print '    New Updates: %s' % up_count

        if not review:
            for pspec in _index.specs:
                importSpec(pspec)
        else:
            pspec = pisi.specfile.SpecFile(_index)
            importSpec(pspec, local_uri=_index)
    def parseSourceIndex(_index, review=False, traverse_time=False):
        if not review:
            try:
                distroName, distroRelease = _index.distribution.sourceName.split('-', 1)
            except ValueError:
                distroName = _index.distribution.sourceName
                distroRelease = _index.distribution.version
            print '  Distribution: %s-%s' % (distroName, distroRelease)

            # repo name
            try:
                if _index.distribution.sourceName.find(" ") != -1:
                    binaryName = _index.distribution.sourceName.split()[1]
                else:
                    binaryName = _index.distribution.sourceName.split('-', 1)[1]
            except IndexError:
                if _index.distribution.binaryName == 'Contrib': binaryName = 'Contrib'
                else: binaryName = 'Pardus'
            if binaryName == 'Contrib': repoType = 'contrib'
            elif binaryName == 'Corporate': repoType = 'corporate'
            else: repoType = 'stable'
        else:
            distroName = "Pardus"
            distroRelease = "Playground"
            repoType = "review"

        # Add distribution to database
        try:
            distribution = Distribution.objects.get(name=distroName, release=distroRelease, type=repoType)
        except Distribution.DoesNotExist:
            distribution = Distribution(name=distroName, release=distroRelease, type=repoType)
            distribution.save()

        def importSpec(pspec, local_uri=None):
            # Add or update developer
            maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)
            review_flag = False
            # Create the source package information
            part_of = pspec.source.partOf
            if not local_uri:
                source_uri = pspec.source.sourceURI
            else:
                index = local_uri.find("review")
                source_uri = local_uri[index:]
                review_flag = True
            print '     Source URI: %s' % source_uri

            home_page = pspec.source.homepage
            print '     Homepage: %s' % home_page
            source_info = SourcePackageDetail.objects.create(part_of=part_of, source_uri=source_uri, home_page=home_page)
            for is_a in pspec.source.isA:
                print '     IsA: %s' % is_a
                source_info.isa_set.create(name=is_a)
            for license in pspec.source.license:
                print '     License: %s' % license
                source_info.license_set.create(name=license)
            summary = pspec.source.summary
            for language in summary.keys():
                print '     Summary Language: %s' % language
                print '     Summary Text: %s' % summary[language]
                source_info.summary_set.create(language=language, text=summary[language])
            description = pspec.source.description
            for language in description.keys():
                print '     Description Language: %s' % language
                print '     Description Text: %s' % description[language]
                source_info.description_set.create(language=language, text=description[language])

            # Add source or update maintainer
            try:
                source = Source.objects.get(name=pspec.source.name, distribution=distribution)
                source.maintained_by = maintained_by
                source.info = source_info
            except Source.DoesNotExist:
                source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by, info=source_info)
            print '  Source: %s' % source.name

            # Save the traverse time of the review path for the next check
            if review_flag:
                review = ReviewInfo.objects.create(check_time=traverse_time,status=True)
            else:
                review = ReviewInfo.objects.create(status=False)
            source.review = review
            source.save()

            # Update build dependencies
            for dep in BuildDependency.objects.filter(source=source):
                dep.delete()
            for dep in pspec.source.buildDependencies:
                dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                dependency.save()

            # Add or update package info
            for pack in pspec.packages:
                try:
                    package = Package.objects.get(name=pack.name, source=source)
                    package.save()
                except Package.DoesNotExist:
                    package = Package(name=pack.name, source=source)
                    package.save()
                    print '    New package: %s' % package.name

                for rep in pack.replaces:
                    replaces = Replaces(name=str(rep), package=package)
                    replaces.save()
                    for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
                        bin.resolution = 'removed'
                        bin.save()
                        print '    Marking %s-%s as removed' % (rep, bin.no)

                # Update runtime dependencies
                for dep in RuntimeDependency.objects.filter(package=package):
                    dep.delete()
                for dep in pack.runtimeDependencies():
                    try:
                        if isinstance(dep, pisi.specfile.AnyDependency):
                            # Any dependencies
                            for any_dep in dep.dependencies:
                                dependency = RuntimeDependency(package=package, name=any_dep.package, version=toString(any_dep.version), version_to=toString(any_dep.versionTo), version_from=toString(any_dep.versionFrom), release=toInt(any_dep.release), release_to=toInt(any_dep.releaseTo), release_from=toInt(any_dep.releaseFrom))
                                dependency.save()
                        else:
                            dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                            dependency.save()
                    except:
                        dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                        dependency.save()


            up_count = 0
            for up in pspec.history:
                updated_by = createUser(up.email, up.name)
                try:
                    update = Update.objects.get(no=up.release, source=source)
                    update.updated_on = up.date
                    update.updated_by = updated_by
                    #update.version_no = up.version_no
                    update.comment = up.comment
                    update.save()
                except Update.DoesNotExist:
                    update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
                    update.save()
                    up_count += 1
            if up_count > 0:
                print '    New Updates: %s' % up_count

        if not review:
            for pspec in _index.specs:
                importSpec(pspec)
        else:
            pspec = pisi.specfile.SpecFile(_index)
            importSpec(pspec, local_uri=_index)
def updateDB(path_source, full_import, newRelease):
    from django.contrib.auth.models import User
    from noan.repository.models import Distribution, Source, Package, Binary, Update, BuildDependency, RuntimeDependency

    def createUser(email, name):
        user = None
        if ' ' in name:
            first_name, last_name = name.rsplit(' ', 1)
        else:
            first_name = name
            last_name = ''
        count = 1
        username = email.split('@')[0]
        while not user:
            try:
                user = User.objects.get(first_name=first_name, last_name=last_name)
            except User.DoesNotExist:
                user = User(email=email, username=username, first_name=first_name, last_name=last_name)
                user.set_password(username)
                try:
                    user.save()
                except:
                    user = None
                    username += str(count)
                    count += 1
                    continue
                print '    New developer: %s' % name
        return user

    print 'Scanning %s...' % path_source

    distroFile = os.path.join(path_source, 'distribution.xml')
    distro = pisi.component.Distribution(distroFile)
    distroName, distroRelease = distro.sourceName.rsplit('-', 1)
    distroRelease = distro.version
    if newRelease:
        distroRelease = newRelease
    print '  Distribution: %s-%s' % (distroName, distroRelease)

    # Add distribution to database
    try:
        distribution = Distribution.objects.get(name=distroName, release=distroRelease)
    except Distribution.DoesNotExist:
        distribution = Distribution(name=distroName, release=distroRelease)
        distribution.save()

    def importSpec(_spec):
        print '  Importing %s' % _spec
        pspec = pisi.specfile.SpecFile(_spec)

        # Add or update developer
        maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)

        # Add source or update maintainer
        try:
            source = Source.objects.get(name=pspec.source.name, distribution=distribution)
            source.maintained_by = maintained_by
            source.save()
        except Source.DoesNotExist:
            source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by)
            source.save()
            print '    New source: %s' % source.name

        # Update build dependencies
        for dep in BuildDependency.objects.filter(source=source):
            dep.delete()
        for dep in pspec.source.buildDependencies:
            dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
            dependency.save()

        # Add or update package info
        for pack in pspec.packages:
            try:
                package = Package.objects.get(name=pack.name, source=source)
                package.save()
            except Package.DoesNotExist:
                package = Package(name=pack.name, source=source)
                package.save()
                print '    New package: %s' % package.name

            """
            for rep in pack.replaces:
                count = 0
                for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
                    bin.resolution = 'removed'
                    bin.save()
                    count += 1
                print '    Replaces package: %s (%d binaries)' % (rep, count)
            """

            # Update runtime dependencies
            for dep in RuntimeDependency.objects.filter(package=package):
                dep.delete()
            for dep in pack.runtimeDependencies():
                dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
                dependency.save()

        up_count = 0
        for up in pspec.history:
            updated_by = createUser(up.email, up.name)
            try:
                update = Update.objects.get(no=up.release, source=source)
                update.updated_on = up.date
                update.updated_by = updated_by
                #update.version_no = up.version_no
                update.comment = up.comment
                update.save()
            except Update.DoesNotExist:
                update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
                update.save()
                up_count += 1
        if up_count > 0:
            print '    New Updates: %s' % up_count

    # Initialize SVN client
    cli = pysvn.Client()

    # Handle SSL warnings
    def cbSSL(trust_dict):
        print "ssh shit"
        # No username/password are available
        retcode = False
        # Accept SVN_AUTH_SSL_NOTYETVALID, SVN_AUTH_SSL_EXPIRED, SVN_AUTH_SSL_CNMISMATCH, SVN_AUTH_SSL_UNKNOWNCA, SVN_AUTH_SSL_OTHER
        accepted_failures = 0x8
        # Save certificate
        save = True
        return retcode, accepted_failures, save

    cli.callback_ssl_server_trust_prompt = cbSSL

    if full_import:
        cli.update(path_source)
        specDirs = findSpecs(path_source)
        for dir in specDirs:
            importSpec(os.path.join(dir, 'pspec.xml'))
    else:
        specFiles = []
        def cbNotify(_info):
            action = _info['action']
            path = _info['path']
            if path.endswith('/pspec.xml'):
                if action in [pysvn.wc_notify_action.update_add, pysvn.wc_notify_action.update_update]:
                    specFiles.append(path)
        cli.callback_notify = cbNotify
        cli.update(path_source)
        for specFile in specFiles:
            importSpec(specFile)
Exemple #4
0
def updateDB(path_source, full_import, newRelease):
    from django.contrib.auth.models import User
    from noan.repository.models import Distribution, Source, Package, Binary, Update, BuildDependency, RuntimeDependency

    def createUser(email, name):
        user = None
        if ' ' in name:
            first_name, last_name = name.rsplit(' ', 1)
        else:
            first_name = name
            last_name = ''
        count = 1
        username = email.split('@')[0]
        while not user:
            try:
                user = User.objects.get(first_name=first_name,
                                        last_name=last_name)
            except User.DoesNotExist:
                user = User(email=email,
                            username=username,
                            first_name=first_name,
                            last_name=last_name)
                user.set_password(username)
                try:
                    user.save()
                except:
                    user = None
                    username += str(count)
                    count += 1
                    continue
                print '    New developer: %s' % name
        return user

    print 'Scanning %s...' % path_source

    distroFile = os.path.join(path_source, 'distribution.xml')
    distro = pisi.component.Distribution(distroFile)
    distroName, distroRelease = distro.sourceName.rsplit('-', 1)
    distroRelease = distro.version
    if newRelease:
        distroRelease = newRelease
    print '  Distribution: %s-%s' % (distroName, distroRelease)

    # Add distribution to database
    try:
        distribution = Distribution.objects.get(name=distroName,
                                                release=distroRelease)
    except Distribution.DoesNotExist:
        distribution = Distribution(name=distroName, release=distroRelease)
        distribution.save()

    def importSpec(_spec):
        print '  Importing %s' % _spec
        pspec = pisi.specfile.SpecFile(_spec)

        # Add or update developer
        maintained_by = createUser(pspec.source.packager.email,
                                   pspec.source.packager.name)

        # Add source or update maintainer
        try:
            source = Source.objects.get(name=pspec.source.name,
                                        distribution=distribution)
            source.maintained_by = maintained_by
            source.save()
        except Source.DoesNotExist:
            source = Source(name=pspec.source.name,
                            distribution=distribution,
                            maintained_by=maintained_by)
            source.save()
            print '    New source: %s' % source.name

        # Update build dependencies
        for dep in BuildDependency.objects.filter(source=source):
            dep.delete()
        for dep in pspec.source.buildDependencies:
            dependency = BuildDependency(source=source,
                                         name=dep.package,
                                         version=toString(dep.version),
                                         version_to=toString(dep.versionTo),
                                         version_from=toString(
                                             dep.versionFrom),
                                         release=toInt(dep.release),
                                         release_to=toInt(dep.releaseTo),
                                         release_from=toInt(dep.releaseFrom))
            dependency.save()

        # Add or update package info
        for pack in pspec.packages:
            try:
                package = Package.objects.get(name=pack.name, source=source)
                package.save()
            except Package.DoesNotExist:
                package = Package(name=pack.name, source=source)
                package.save()
                print '    New package: %s' % package.name
            """
            for rep in pack.replaces:
                count = 0
                for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
                    bin.resolution = 'removed'
                    bin.save()
                    count += 1
                print '    Replaces package: %s (%d binaries)' % (rep, count)
            """

            # Update runtime dependencies
            for dep in RuntimeDependency.objects.filter(package=package):
                dep.delete()
            for dep in pack.runtimeDependencies():
                dependency = RuntimeDependency(
                    package=package,
                    name=dep.package,
                    version=toString(dep.version),
                    version_to=toString(dep.versionTo),
                    version_from=toString(dep.versionFrom),
                    release=toInt(dep.release),
                    release_to=toInt(dep.releaseTo),
                    release_from=toInt(dep.releaseFrom))
                dependency.save()

        up_count = 0
        for up in pspec.history:
            updated_by = createUser(up.email, up.name)
            try:
                update = Update.objects.get(no=up.release, source=source)
                update.updated_on = up.date
                update.updated_by = updated_by
                #update.version_no = up.version_no
                update.comment = up.comment
                update.save()
            except Update.DoesNotExist:
                update = Update(no=up.release,
                                source=source,
                                version_no=up.version,
                                updated_by=updated_by,
                                updated_on=up.date,
                                comment=up.comment)
                update.save()
                up_count += 1
        if up_count > 0:
            print '    New Updates: %s' % up_count

    # Initialize SVN client
    cli = pysvn.Client()

    # Handle SSL warnings
    def cbSSL(trust_dict):
        print "ssh shit"
        # No username/password are available
        retcode = False
        # Accept SVN_AUTH_SSL_NOTYETVALID, SVN_AUTH_SSL_EXPIRED, SVN_AUTH_SSL_CNMISMATCH, SVN_AUTH_SSL_UNKNOWNCA, SVN_AUTH_SSL_OTHER
        accepted_failures = 0x8
        # Save certificate
        save = True
        return retcode, accepted_failures, save

    cli.callback_ssl_server_trust_prompt = cbSSL

    if full_import:
        cli.update(path_source)
        specDirs = findSpecs(path_source)
        for dir in specDirs:
            importSpec(os.path.join(dir, 'pspec.xml'))
    else:
        specFiles = []

        def cbNotify(_info):
            action = _info['action']
            path = _info['path']
            if path.endswith('/pspec.xml'):
                if action in [
                        pysvn.wc_notify_action.update_add,
                        pysvn.wc_notify_action.update_update
                ]:
                    specFiles.append(path)

        cli.callback_notify = cbNotify
        cli.update(path_source)
        for specFile in specFiles:
            importSpec(specFile)