def parseSourceIndex(_index, review=False, traverse_time=False):
if not review:
try:
distroName, distroRelease = _index.distribution.sourceName.split('-', 1)
except ValueError:
distroName = _index.distribution.sourceName
distroRelease = _index.distribution.version
print ' Distribution: %s-%s' % (distroName, distroRelease)
# repo name
try:
if _index.distribution.sourceName.find(" ") != -1:
binaryName = _index.distribution.sourceName.split()[1]
else:
binaryName = _index.distribution.sourceName.split('-', 1)[1]
except IndexError:
if _index.distribution.binaryName == 'Contrib': binaryName = 'Contrib'
else: binaryName = 'Pardus'
if binaryName == 'Contrib': repoType = 'contrib'
elif binaryName == 'Corporate': repoType = 'corporate'
else: repoType = 'stable'
else:
distroName = "Pardus"
distroRelease = "Playground"
repoType = "review"
# Add distribution to database
try:
distribution = Distribution.objects.get(name=distroName, release=distroRelease, type=repoType)
except Distribution.DoesNotExist:
distribution = Distribution(name=distroName, release=distroRelease, type=repoType)
distribution.save()
def importSpec(pspec, local_uri=None):
# Add or update developer
maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)
review_flag = False
# Create the source package information
part_of = pspec.source.partOf
if not local_uri:
source_uri = pspec.source.sourceURI
else:
index = local_uri.find("review")
source_uri = local_uri[index:]
review_flag = True
print ' Source URI: %s' % source_uri
home_page = pspec.source.homepage
print ' Homepage: %s' % home_page
source_info = SourcePackageDetail.objects.create(part_of=part_of, source_uri=source_uri, home_page=home_page)
for is_a in pspec.source.isA:
print ' IsA: %s' % is_a
source_info.isa_set.create(name=is_a)
for license in pspec.source.license:
print ' License: %s' % license
source_info.license_set.create(name=license)
summary = pspec.source.summary
for language in summary.keys():
print ' Summary Language: %s' % language
print ' Summary Text: %s' % summary[language]
source_info.summary_set.create(language=language, text=summary[language])
description = pspec.source.description
for language in description.keys():
print ' Description Language: %s' % language
print ' Description Text: %s' % description[language]
source_info.description_set.create(language=language, text=description[language])
# Add source or update maintainer
try:
source = Source.objects.get(name=pspec.source.name, distribution=distribution)
source.maintained_by = maintained_by
source.info = source_info
except Source.DoesNotExist:
source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by, info=source_info)
print ' Source: %s' % source.name
# Save the traverse time of the review path for the next check
if review_flag:
review = ReviewInfo.objects.create(check_time=traverse_time,status=True)
else:
review = ReviewInfo.objects.create(status=False)
source.review = review
source.save()
# Update build dependencies
for dep in BuildDependency.objects.filter(source=source):
dep.delete()
for dep in pspec.source.buildDependencies:
dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
# Add or update package info
for pack in pspec.packages:
try:
package = Package.objects.get(name=pack.name, source=source)
package.save()
except Package.DoesNotExist:
package = Package(name=pack.name, source=source)
package.save()
print ' New package: %s' % package.name
for rep in pack.replaces:
replaces = Replaces(name=str(rep), package=package)
replaces.save()
for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
bin.resolution = 'removed'
bin.save()
print ' Marking %s-%s as removed' % (rep, bin.no)
# Update runtime dependencies
for dep in RuntimeDependency.objects.filter(package=package):
dep.delete()
for dep in pack.runtimeDependencies():
try:
if isinstance(dep, pisi.specfile.AnyDependency):
# Any dependencies
for any_dep in dep.dependencies:
dependency = RuntimeDependency(package=package, name=any_dep.package, version=toString(any_dep.version), version_to=toString(any_dep.versionTo), version_from=toString(any_dep.versionFrom), release=toInt(any_dep.release), release_to=toInt(any_dep.releaseTo), release_from=toInt(any_dep.releaseFrom))
dependency.save()
else:
dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
except:
dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
up_count = 0
for up in pspec.history:
updated_by = createUser(up.email, up.name)
try:
update = Update.objects.get(no=up.release, source=source)
update.updated_on = up.date
update.updated_by = updated_by
#update.version_no = up.version_no
update.comment = up.comment
update.save()
except Update.DoesNotExist:
update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
update.save()
up_count += 1
if up_count > 0:
print ' New Updates: %s' % up_count
if not review:
for pspec in _index.specs:
importSpec(pspec)
else:
pspec = pisi.specfile.SpecFile(_index)
importSpec(pspec, local_uri=_index)
def parseSourceIndex(_index, review=False, traverse_time=False):
if not review:
try:
distroName, distroRelease = _index.distribution.sourceName.split('-', 1)
except ValueError:
distroName = _index.distribution.sourceName
distroRelease = _index.distribution.version
print ' Distribution: %s-%s' % (distroName, distroRelease)
# repo name
try:
if _index.distribution.sourceName.find(" ") != -1:
binaryName = _index.distribution.sourceName.split()[1]
else:
binaryName = _index.distribution.sourceName.split('-', 1)[1]
except IndexError:
if _index.distribution.binaryName == 'Contrib': binaryName = 'Contrib'
else: binaryName = 'Pardus'
if binaryName == 'Contrib': repoType = 'contrib'
elif binaryName == 'Corporate': repoType = 'corporate'
else: repoType = 'stable'
else:
distroName = "Pardus"
distroRelease = "Playground"
repoType = "review"
# Add distribution to database
try:
distribution = Distribution.objects.get(name=distroName, release=distroRelease, type=repoType)
except Distribution.DoesNotExist:
distribution = Distribution(name=distroName, release=distroRelease, type=repoType)
distribution.save()
def importSpec(pspec, local_uri=None):
# Add or update developer
maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)
review_flag = False
# Create the source package information
part_of = pspec.source.partOf
if not local_uri:
source_uri = pspec.source.sourceURI
else:
index = local_uri.find("review")
source_uri = local_uri[index:]
review_flag = True
print ' Source URI: %s' % source_uri
home_page = pspec.source.homepage
print ' Homepage: %s' % home_page
source_info = SourcePackageDetail.objects.create(part_of=part_of, source_uri=source_uri, home_page=home_page)
for is_a in pspec.source.isA:
print ' IsA: %s' % is_a
source_info.isa_set.create(name=is_a)
for license in pspec.source.license:
print ' License: %s' % license
source_info.license_set.create(name=license)
summary = pspec.source.summary
for language in summary.keys():
print ' Summary Language: %s' % language
print ' Summary Text: %s' % summary[language]
source_info.summary_set.create(language=language, text=summary[language])
description = pspec.source.description
for language in description.keys():
print ' Description Language: %s' % language
print ' Description Text: %s' % description[language]
source_info.description_set.create(language=language, text=description[language])
# Add source or update maintainer
try:
source = Source.objects.get(name=pspec.source.name, distribution=distribution)
source.maintained_by = maintained_by
source.info = source_info
except Source.DoesNotExist:
source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by, info=source_info)
print ' Source: %s' % source.name
# Save the traverse time of the review path for the next check
if review_flag:
review = ReviewInfo.objects.create(check_time=traverse_time,status=True)
else:
review = ReviewInfo.objects.create(status=False)
source.review = review
source.save()
# Update build dependencies
for dep in BuildDependency.objects.filter(source=source):
dep.delete()
for dep in pspec.source.buildDependencies:
dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
# Add or update package info
for pack in pspec.packages:
try:
package = Package.objects.get(name=pack.name, source=source)
package.save()
except Package.DoesNotExist:
package = Package(name=pack.name, source=source)
package.save()
print ' New package: %s' % package.name
for rep in pack.replaces:
replaces = Replaces(name=str(rep), package=package)
replaces.save()
for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
bin.resolution = 'removed'
bin.save()
print ' Marking %s-%s as removed' % (rep, bin.no)
# Update runtime dependencies
for dep in RuntimeDependency.objects.filter(package=package):
dep.delete()
for dep in pack.runtimeDependencies():
try:
if isinstance(dep, pisi.specfile.AnyDependency):
# Any dependencies
for any_dep in dep.dependencies:
dependency = RuntimeDependency(package=package, name=any_dep.package, version=toString(any_dep.version), version_to=toString(any_dep.versionTo), version_from=toString(any_dep.versionFrom), release=toInt(any_dep.release), release_to=toInt(any_dep.releaseTo), release_from=toInt(any_dep.releaseFrom))
dependency.save()
else:
dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
except:
dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
up_count = 0
for up in pspec.history:
updated_by = createUser(up.email, up.name)
try:
update = Update.objects.get(no=up.release, source=source)
update.updated_on = up.date
update.updated_by = updated_by
#update.version_no = up.version_no
update.comment = up.comment
update.save()
except Update.DoesNotExist:
update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
update.save()
up_count += 1
if up_count > 0:
print ' New Updates: %s' % up_count
if not review:
for pspec in _index.specs:
importSpec(pspec)
else:
pspec = pisi.specfile.SpecFile(_index)
importSpec(pspec, local_uri=_index)
def updateDB(path_source, full_import, newRelease):
from django.contrib.auth.models import User
from noan.repository.models import Distribution, Source, Package, Binary, Update, BuildDependency, RuntimeDependency
def createUser(email, name):
user = None
if ' ' in name:
first_name, last_name = name.rsplit(' ', 1)
else:
first_name = name
last_name = ''
count = 1
username = email.split('@')[0]
while not user:
try:
user = User.objects.get(first_name=first_name, last_name=last_name)
except User.DoesNotExist:
user = User(email=email, username=username, first_name=first_name, last_name=last_name)
user.set_password(username)
try:
user.save()
except:
user = None
username += str(count)
count += 1
continue
print ' New developer: %s' % name
return user
print 'Scanning %s...' % path_source
distroFile = os.path.join(path_source, 'distribution.xml')
distro = pisi.component.Distribution(distroFile)
distroName, distroRelease = distro.sourceName.rsplit('-', 1)
distroRelease = distro.version
if newRelease:
distroRelease = newRelease
print ' Distribution: %s-%s' % (distroName, distroRelease)
# Add distribution to database
try:
distribution = Distribution.objects.get(name=distroName, release=distroRelease)
except Distribution.DoesNotExist:
distribution = Distribution(name=distroName, release=distroRelease)
distribution.save()
def importSpec(_spec):
print ' Importing %s' % _spec
pspec = pisi.specfile.SpecFile(_spec)
# Add or update developer
maintained_by = createUser(pspec.source.packager.email, pspec.source.packager.name)
# Add source or update maintainer
try:
source = Source.objects.get(name=pspec.source.name, distribution=distribution)
source.maintained_by = maintained_by
source.save()
except Source.DoesNotExist:
source = Source(name=pspec.source.name, distribution=distribution, maintained_by=maintained_by)
source.save()
print ' New source: %s' % source.name
# Update build dependencies
for dep in BuildDependency.objects.filter(source=source):
dep.delete()
for dep in pspec.source.buildDependencies:
dependency = BuildDependency(source=source, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
# Add or update package info
for pack in pspec.packages:
try:
package = Package.objects.get(name=pack.name, source=source)
package.save()
except Package.DoesNotExist:
package = Package(name=pack.name, source=source)
package.save()
print ' New package: %s' % package.name
"""
for rep in pack.replaces:
count = 0
for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
bin.resolution = 'removed'
bin.save()
count += 1
print ' Replaces package: %s (%d binaries)' % (rep, count)
"""
# Update runtime dependencies
for dep in RuntimeDependency.objects.filter(package=package):
dep.delete()
for dep in pack.runtimeDependencies():
dependency = RuntimeDependency(package=package, name=dep.package, version=toString(dep.version), version_to=toString(dep.versionTo), version_from=toString(dep.versionFrom), release=toInt(dep.release), release_to=toInt(dep.releaseTo), release_from=toInt(dep.releaseFrom))
dependency.save()
up_count = 0
for up in pspec.history:
updated_by = createUser(up.email, up.name)
try:
update = Update.objects.get(no=up.release, source=source)
update.updated_on = up.date
update.updated_by = updated_by
#update.version_no = up.version_no
update.comment = up.comment
update.save()
except Update.DoesNotExist:
update = Update(no=up.release, source=source, version_no=up.version, updated_by=updated_by, updated_on=up.date, comment=up.comment)
update.save()
up_count += 1
if up_count > 0:
print ' New Updates: %s' % up_count
# Initialize SVN client
cli = pysvn.Client()
# Handle SSL warnings
def cbSSL(trust_dict):
print "ssh shit"
# No username/password are available
retcode = False
# Accept SVN_AUTH_SSL_NOTYETVALID, SVN_AUTH_SSL_EXPIRED, SVN_AUTH_SSL_CNMISMATCH, SVN_AUTH_SSL_UNKNOWNCA, SVN_AUTH_SSL_OTHER
accepted_failures = 0x8
# Save certificate
save = True
return retcode, accepted_failures, save
cli.callback_ssl_server_trust_prompt = cbSSL
if full_import:
cli.update(path_source)
specDirs = findSpecs(path_source)
for dir in specDirs:
importSpec(os.path.join(dir, 'pspec.xml'))
else:
specFiles = []
def cbNotify(_info):
action = _info['action']
path = _info['path']
if path.endswith('/pspec.xml'):
if action in [pysvn.wc_notify_action.update_add, pysvn.wc_notify_action.update_update]:
specFiles.append(path)
cli.callback_notify = cbNotify
cli.update(path_source)
for specFile in specFiles:
importSpec(specFile)
def updateDB(path_source, full_import, newRelease):
from django.contrib.auth.models import User
from noan.repository.models import Distribution, Source, Package, Binary, Update, BuildDependency, RuntimeDependency
def createUser(email, name):
user = None
if ' ' in name:
first_name, last_name = name.rsplit(' ', 1)
else:
first_name = name
last_name = ''
count = 1
username = email.split('@')[0]
while not user:
try:
user = User.objects.get(first_name=first_name,
last_name=last_name)
except User.DoesNotExist:
user = User(email=email,
username=username,
first_name=first_name,
last_name=last_name)
user.set_password(username)
try:
user.save()
except:
user = None
username += str(count)
count += 1
continue
print ' New developer: %s' % name
return user
print 'Scanning %s...' % path_source
distroFile = os.path.join(path_source, 'distribution.xml')
distro = pisi.component.Distribution(distroFile)
distroName, distroRelease = distro.sourceName.rsplit('-', 1)
distroRelease = distro.version
if newRelease:
distroRelease = newRelease
print ' Distribution: %s-%s' % (distroName, distroRelease)
# Add distribution to database
try:
distribution = Distribution.objects.get(name=distroName,
release=distroRelease)
except Distribution.DoesNotExist:
distribution = Distribution(name=distroName, release=distroRelease)
distribution.save()
def importSpec(_spec):
print ' Importing %s' % _spec
pspec = pisi.specfile.SpecFile(_spec)
# Add or update developer
maintained_by = createUser(pspec.source.packager.email,
pspec.source.packager.name)
# Add source or update maintainer
try:
source = Source.objects.get(name=pspec.source.name,
distribution=distribution)
source.maintained_by = maintained_by
source.save()
except Source.DoesNotExist:
source = Source(name=pspec.source.name,
distribution=distribution,
maintained_by=maintained_by)
source.save()
print ' New source: %s' % source.name
# Update build dependencies
for dep in BuildDependency.objects.filter(source=source):
dep.delete()
for dep in pspec.source.buildDependencies:
dependency = BuildDependency(source=source,
name=dep.package,
version=toString(dep.version),
version_to=toString(dep.versionTo),
version_from=toString(
dep.versionFrom),
release=toInt(dep.release),
release_to=toInt(dep.releaseTo),
release_from=toInt(dep.releaseFrom))
dependency.save()
# Add or update package info
for pack in pspec.packages:
try:
package = Package.objects.get(name=pack.name, source=source)
package.save()
except Package.DoesNotExist:
package = Package(name=pack.name, source=source)
package.save()
print ' New package: %s' % package.name
"""
for rep in pack.replaces:
count = 0
for bin in Binary.objects.filter(package__name=str(rep), package__source__distribution=distribution):
bin.resolution = 'removed'
bin.save()
count += 1
print ' Replaces package: %s (%d binaries)' % (rep, count)
"""
# Update runtime dependencies
for dep in RuntimeDependency.objects.filter(package=package):
dep.delete()
for dep in pack.runtimeDependencies():
dependency = RuntimeDependency(
package=package,
name=dep.package,
version=toString(dep.version),
version_to=toString(dep.versionTo),
version_from=toString(dep.versionFrom),
release=toInt(dep.release),
release_to=toInt(dep.releaseTo),
release_from=toInt(dep.releaseFrom))
dependency.save()
up_count = 0
for up in pspec.history:
updated_by = createUser(up.email, up.name)
try:
update = Update.objects.get(no=up.release, source=source)
update.updated_on = up.date
update.updated_by = updated_by
#update.version_no = up.version_no
update.comment = up.comment
update.save()
except Update.DoesNotExist:
update = Update(no=up.release,
source=source,
version_no=up.version,
updated_by=updated_by,
updated_on=up.date,
comment=up.comment)
update.save()
up_count += 1
if up_count > 0:
print ' New Updates: %s' % up_count
# Initialize SVN client
cli = pysvn.Client()
# Handle SSL warnings
def cbSSL(trust_dict):
print "ssh shit"
# No username/password are available
retcode = False
# Accept SVN_AUTH_SSL_NOTYETVALID, SVN_AUTH_SSL_EXPIRED, SVN_AUTH_SSL_CNMISMATCH, SVN_AUTH_SSL_UNKNOWNCA, SVN_AUTH_SSL_OTHER
accepted_failures = 0x8
# Save certificate
save = True
return retcode, accepted_failures, save
cli.callback_ssl_server_trust_prompt = cbSSL
if full_import:
cli.update(path_source)
specDirs = findSpecs(path_source)
for dir in specDirs:
importSpec(os.path.join(dir, 'pspec.xml'))
else:
specFiles = []
def cbNotify(_info):
action = _info['action']
path = _info['path']
if path.endswith('/pspec.xml'):
if action in [
pysvn.wc_notify_action.update_add,
pysvn.wc_notify_action.update_update
]:
specFiles.append(path)
cli.callback_notify = cbNotify
cli.update(path_source)
for specFile in specFiles:
importSpec(specFile)