def test_update_paragraph(self):
from tribus.common.recorder import update_paragraph
from tribus.config.pkgrecorder import PACKAGE_FIELDS, DETAIL_FIELDS
old_paragraph = deb822.Packages(open(get_path([SMPLDIR, 'Blender'])))
new_paragraph = deb822.Packages(open(get_path([SMPLDIR,
'BlenderNew'])))
Package.objects.create_auto(old_paragraph, test_dist, 'main')
update_paragraph(new_paragraph, test_dist, 'main')
p = Package.objects.get(Name='blender')
d = Details.objects.get(package=p)
name, mail = email.Utils.parseaddr(new_paragraph.get('Maintainer'))
total_relations = 0
total_labels = 0
self.assertEqual(p.Maintainer.Name, name)
self.assertEqual(p.Maintainer.Email, mail)
self.assertEqual(d.Distribution, test_dist)
tag_list = new_paragraph['Tag'].replace('\n', '').split(', ')
clean_list = [tuple(tag.split('::')) for tag in tag_list]
for label in p.Labels.all():
total_labels += 1
self.assertIn((label.Name, label.Tags.Value), clean_list)
self.assertEqual(total_labels, len(clean_list))
for field, field_db in PACKAGE_FIELDS.items():
if new_paragraph.get(field):
self.assertEqual(str(getattr(p, field_db)),
new_paragraph[field])
for field, field_db in DETAIL_FIELDS.items():
if new_paragraph.get(field):
self.assertEqual(str(getattr(d, field_db)),
new_paragraph[field])
for _, relations in new_paragraph.relations.items():
if relations:
for relation in relations:
if len(relation) > 1:
for _ in relation:
total_relations += 1
else:
total_relations += 1
self.assertEqual(d.Relations.all().count(), total_relations)
def to822(self, component=""):
"""Create deb822.Package object from model."""
ret = deb822.Packages()
for k, v in self.TRANSLATION_DICT.items():
value = self.data.get(k)
if value is not None:
ret[v] = value
try:
artifact = self.instance._artifacts.get()
if artifact.md5:
ret["MD5sum"] = artifact.md5
if artifact.sha1:
ret["SHA1"] = artifact.sha1
ret["SHA256"] = artifact.sha256
except Artifact.DoesNotExist:
artifact = RemoteArtifact.objects.filter(
sha256=self.instance.sha256).first()
if artifact.md5:
ret["MD5sum"] = artifact.md5
if artifact.sha1:
ret["SHA1"] = artifact.sha1
ret["SHA256"] = artifact.sha256
ret["Filename"] = self.instance.filename(component)
return ret
def create_pulp3_content(self):
"""
Create a Pulp 3 detail Content unit for saving it later in a bulk operation.
Return an unsaved Pulp 3 Content.
"""
package_paragraph = deb822.Packages()
object_dict = self.__dict__
for cls_field, control_field in self.control_field_map.items():
if object_dict[cls_field]:
package_paragraph[control_field] = object_dict[cls_field]
if self.extra_control_fields:
extra_control_fields = json.loads(self.extra_control_fields)
for control_field, value in extra_control_fields.items():
package_paragraph[control_field] = value
serializer = Package822Serializer.from822(data=package_paragraph)
serializer.is_valid(raise_exception=True)
pulp3_package = pulp3_models.Package(
relative_path=self.filename,
sha256=self.checksum,
**serializer.validated_data,
)
return (pulp3_package, None)
def test_record_relations(self):
from tribus.common.recorder import record_relations
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
details, _ = Details.objects.get_or_create(Version = '2.63a-1',
Architecture = 'i386',
Distribution = test_dist)
record_relations(details, paragraph.relations.items())
total_relations = 0
for relation_type, relations in paragraph.relations.items():
if relations:
for relation in relations:
if len(relation) > 1:
for element in relation:
version = element.get('version', None)
if version:
vn, vo = version
else:
vn, vo = (None, None)
self.assertTrue(Relation.objects.get(relation_type = relation_type,
related_package__Package = element['name'],
version = vo, relation = vn))
total_relations += 1
else:
version = relation[0].get('version', None)
if version:
vn, vo = version
else:
vn, vo = (None, None)
self.assertTrue(Relation.objects.get(relation_type = relation_type,
related_package__Package = relation[0]['name'],
version = vo, relation = vn))
total_relations += 1
self.assertEqual(details.Relations.all().count(), total_relations)
def to822(self, component=""):
"""Create deb822.Package object from model."""
ret = deb822.Packages()
for k, v in self.TRANSLATION_DICT.items():
value = self.data.get(k)
if value is not None:
ret[v] = value
custom_fields = self.data.get("custom_fields")
if custom_fields:
ret.update(custom_fields)
try:
artifact = self.instance._artifacts.get()
artifact.touch() # Orphan cleanup protection until we are done!
if artifact.md5:
ret["MD5sum"] = artifact.md5
if artifact.sha1:
ret["SHA1"] = artifact.sha1
ret["SHA256"] = artifact.sha256
ret["Size"] = str(artifact.size)
except Artifact.DoesNotExist:
artifact = RemoteArtifact.objects.filter(
sha256=self.instance.sha256).first()
if artifact.md5:
ret["MD5sum"] = artifact.md5
if artifact.sha1:
ret["SHA1"] = artifact.sha1
ret["SHA256"] = artifact.sha256
ret["Size"] = str(artifact.size)
ret["Filename"] = self.instance.filename(component)
return ret
def test_update_paragraph(self):
from tribus.common.recorder import record_paragraph, update_paragraph
from tribus.config.pkgrecorder import PACKAGE_FIELDS, DETAIL_FIELDS
old_paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
new_paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "BlenderNew")))
record_paragraph(old_paragraph, test_dist)
update_paragraph(new_paragraph, test_dist)
p = Package.objects.get(Package = "blender")
d = Details.objects.get(package = p)
name, mail = email.Utils.parseaddr(new_paragraph.get('Maintainer'))
total_relations = 0
self.assertEqual(p.Maintainer.Name, name)
self.assertEqual(p.Maintainer.Email, mail)
self.assertEqual(d.Distribution, test_dist)
tag_list = new_paragraph['Tag'].replace("\n", "").split(", ")
clean_list = [tuple(tag.split("::")) for tag in tag_list]
for label in p.Labels.all():
self.assertIn((label.Name, label.Tags.Value),
clean_list)
for field in PACKAGE_FIELDS:
if new_paragraph.get(field):
self.assertEqual(str(getattr(p,
field.replace("-", "") if "-" in field else field)),
new_paragraph[field])
for field in DETAIL_FIELDS:
if new_paragraph.get(field):
self.assertEqual(str(getattr(d,
field.replace("-", "") if "-" in field else field)),
new_paragraph[field])
for _, relations in new_paragraph.relations.items():
if relations:
for relation in relations:
if len(relation) > 1:
for _ in relation:
total_relations += 1
else:
total_relations += 1
self.assertEqual(d.Relations.all().count(), total_relations)
def write_packages_file_paragraph(packages_file, component, package):
"""
Writes a single 'Packages' file paragraph (the entry for a single package).
This function assumes the object supplied by :package: is correct.
The order of entries are extracted from an official Debian mirror.
:param packages_file: the file handle that is written to
:param component: the release component to which the packages file belongs
(this is needed for the 'pool' path of each package)
:param package: a single DebPackage object
"""
# Ordered list of control file fields explicitly known to pulp_deb:
known_control_fields = [
'Package',
'Source',
'Version',
'Installed-Size',
'Maintainer',
'Original-Maintainer',
'Architecture',
'Replaces',
'Provides',
'Depends',
'Pre-Depends',
'Recommends',
'Suggests',
'Enhances',
'Conflicts',
'Breaks',
'Description',
'Multi-Arch',
'Homepage',
'Section',
'Priority',
]
packages_object = deb822.Packages()
control_fields = deepcopy(package.control_fields)
# Add all known control fields (in order):
for field in known_control_fields:
if field in control_fields:
packages_object[field] = control_fields.pop(field).encode('utf8')
# Also add remaining (unknown) control fields:
for field, value in control_fields.iteritems():
packages_object[field] = value.encode('utf8')
# Explicitly add various non control fields:
relative_pool_path = os.path.join('pool', component, package.filename,)
packages_object['Filename'] = relative_pool_path
packages_object['Size'] = str(package.size)
packages_object['MD5sum'] = package.md5sum
packages_object['SHA1'] = package.sha1
packages_object['SHA256'] = package.sha256
packages_object.dump(packages_file)
def test_select_paragraph_data_fields(self):
from tribus.common.recorder import select_paragraph_data_fields
from tribus.config.pkgrecorder import PACKAGE_FIELDS, DETAIL_FIELDS
section = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
selected_pckage_fields = select_paragraph_data_fields(section, PACKAGE_FIELDS)
selected_details_fields = select_paragraph_data_fields(section, DETAIL_FIELDS)
self.assertLessEqual(len(selected_pckage_fields), len(PACKAGE_FIELDS))
# La prueba falla si se verifican los campos en cuyo nombre hay un guion (-)
# hace falta una solucion
#for field in selected_package_fields.keys():
# self.assertIn(field, PACKAGE_FIELDS)
self.assertLessEqual(len(selected_details_fields), len(DETAIL_FIELDS))
def test_package_add_labels(self):
'''
El objetivo de este test es verificar que las etiquetas de un
paquete se registran correctamente.
'''
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
package, _ = Package.objects.get_or_create(Name='blender')
package.add_labels(paragraph)
tag_list = paragraph['Tag'].replace("\n", "").split(", ")
clean_list = [tuple(tag.split("::")) for tag in tag_list]
for label in package.Labels.all():
self.assertIn((label.Name, label.Tags.Value), clean_list)
def _get_available_nexus_versions(self):
# validate nexus settings are configured first
for suffix in ('host', 'port', 'user', 'repository'):
if not getattr(self, '_nexus_%s' % suffix):
print red('~/.mendel.conf is missing %s in [nexus] configuration section' % suffix)
sys.exit(1)
url_for_debug = (
'http://%(nexus_host)s:%(nexus_port)s'
'/nexus/content/repositories'
'/%(nexus_repository)s'
'/Packages' % dict(
nexus_host=self._nexus_host,
nexus_port=self._nexus_port,
nexus_repository=self._nexus_repository
))
print blue('Downloading packages from %s' % url_for_debug)
#TODO maybe read password from maven settings?
nexus_password = os.environ.get('MENDEL_NEXUS_PASSWORD') or \
getpass.getpass(prompt='Enter nexus password: '******'http://%(nexus_user)s:%(nexus_password)s'
'@%(nexus_host)s:%(nexus_port)s'
'/nexus/content/repositories'
'/%(nexus_repository)s'
'/Packages' % dict(
nexus_user=self._nexus_user,
nexus_password=nexus_password,
nexus_host=self._nexus_host,
nexus_port=self._nexus_port,
nexus_repository=self._nexus_repository
)
).content
package_entries = [
deb822.Packages(package_info)
for package_info in packages_file.split('\n\n')
if package_info
]
available_versions = [
p for p in package_entries
if p.get('Package') == self._service_name
]
print blue('Found %s available versions of %s' % (len(package_entries), self._service_name))
return available_versions
def test_details_create(self):
'''
El objetivo de este test es verificar que los campos de detalles
sean registrados correctamente.
'''
from tribus.config.pkgrecorder import DETAIL_FIELDS
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
package, _ = Package.objects.get_or_create(Name='blender')
details = Details.objects.create_auto(paragraph, package, test_dist,
"main")
self.assertEqual(details.Distribution, test_dist)
for field, field_db in DETAIL_FIELDS.items():
self.assertEqual(getattr(details, field_db), paragraph[field])
def test_record_package(self):
'''
El objetivo de este test es verificar que los campos basicos
de un paquete sean registrados correctamente.
'''
from tribus.common.recorder import record_package
from tribus.config.pkgrecorder import PACKAGE_FIELDS
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
package = record_package(paragraph)
for field in PACKAGE_FIELDS:
if paragraph.get(field):
self.assertEqual(getattr(package,
field.replace("-", "") if "-" in field else field),
paragraph[field])
def to822(self, component=""):
"""Create deb822.Package object from model."""
ret = deb822.Packages()
for k, v in self.TRANSLATION_DICT.items():
value = getattr(self, k, None)
if value is not None:
ret[v] = value
artifact = self._artifacts.get()
ret["MD5sum"] = artifact.md5
ret["SHA1"] = artifact.sha1
ret["SHA256"] = artifact.sha256
ret["Filename"] = self.filename(component)
return ret
def test_record_details(self):
'''
El objetivo de este test es verificar que los campos de detalles
sean registrados correctamente.
'''
from tribus.common.recorder import record_details
from tribus.config.pkgrecorder import DETAIL_FIELDS
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, "Blender")))
package, _ = Package.objects.get_or_create(Package = 'blender')
details = record_details(paragraph, package, test_dist)
self.assertEqual(details.Distribution, test_dist)
for field in DETAIL_FIELDS:
self.assertEqual(getattr(details,
field.replace("-", "") if "-" in field else field),
paragraph[field])
def process_control(control_contents):
control = deb822.Packages(control_contents)
package = control["package"].encode("ascii")
try:
source = control["source"].encode("ascii").split()[0]
except KeyError:
source = package
version = control["version"].encode("ascii")
architecture = control["architecture"].encode("ascii")
depends = set(dep[0]["name"].encode("ascii")
for dep in control.relations.get("depends", ())
if len(dep) == 1)
return dict(package=package,
source=source,
version=version,
architecture=architecture,
depends=depends)
def test_package_create_auto(self):
"""
El objetivo de este test es verificar que los campos basicos
de un paquete sean registrados correctamente.
"""
from tribus.config.pkgrecorder import PACKAGE_FIELDS, DETAIL_FIELDS
paragraph = deb822.Packages(open(os.path.join(SAMPLESDIR, 'Blender')))
p = Package.objects.create_auto(paragraph, test_dist, 'main')
d = Details.objects.get(package=p)
total_relations = 0
name, mail = parseaddr(paragraph.get('Maintainer'))
self.assertEqual(d.Distribution, test_dist)
self.assertEqual(p.Maintainer.Name, name)
self.assertEqual(p.Maintainer.Email, mail)
tag_list = paragraph['Tag'].replace('\n', '').split(', ')
clean_list = [tuple(tag.split('::')) for tag in tag_list]
for label in p.Labels.all():
self.assertIn((label.Name, label.Tags.Value), clean_list)
for field, field_db in PACKAGE_FIELDS.items():
if paragraph.get(field):
self.assertEqual(str(getattr(p, field_db)), paragraph[field])
for field, field_db in DETAIL_FIELDS.items():
if paragraph.get(field):
self.assertEqual(str(getattr(d, field_db)), paragraph[field])
for _, relations in paragraph.relations.items():
if relations:
for relation in relations:
if len(relation) > 1:
for _ in relation:
total_relations += 1
else:
total_relations += 1
self.assertEqual(d.Relations.all().count(), total_relations)
def main(path='.', control_file_path='debian/control', excludes=''):
if len(sys.argv) >= 3:
path = sys.argv[1]
control_file_path = sys.argv[2]
try:
excludes = sys.argv[3]
except IndexError:
pass
finally:
excludes = excludes.split(',')
else:
exit(1)
if not os.path.isdir(path) and os.path.exists(control_file_path):
exit(1)
fl = open(control_file_path, 'rb')
sources_control_file_obj = deb822.Sources(sequence=fl)
packages_control_file_obj = deb822.Packages(sequence=fl)
if 'Depends' not in packages_control_file_obj:
sys.stderr.write('Could not find `Depends` in control file\n')
exit(1)
dependencies = map(extract_from_file_name,
filtered(os.listdir(path), excludes))
packages_control_file_obj['Depends'] = ', '.join(
[packages_control_file_obj['Depends']] +
['{} (={})'.format(k, v) for k, v in dependencies]
)
out = open(control_file_path, 'wb')
out.write(_b(
sources_control_file_obj.dump()
+ '\n'
+ packages_control_file_obj.dump()
))
out.close()
return 0
def load_package_from_file(self, repository, filename):
filepath = utils.get_path_from_url(repository.url + filename)
_, size, checksum = next(
iter(
utils.get_size_and_checksum_for_files([filepath],
_checksum_collector)))
with closing(debfile.DebFile(filepath)) as deb:
debcontrol = deb822.Packages(
deb.control.get_content(debfile.CONTROL_FILE))
return Package(
repository=repository,
name=debcontrol["package"],
version=Version(debcontrol['version']),
filesize=int(debcontrol.get('size', size)),
filename=filename,
checksum=FileChecksum(*checksum),
mandatory=self._is_mandatory(debcontrol),
requires=self._get_relations(debcontrol, "depends", "pre-depends",
"recommends"),
provides=self._get_relations(debcontrol, "provides"),
obsoletes=[],
group=debcontrol.get('section'),
)
def _get_current_package_version(self):
with hide('stdout'):
package_info = run('dpkg-query -s %s' % self._service_name)
pkg = deb822.Packages(package_info)
return pkg.get('Version')
