def _init_test(
swh_scheduler: SchedulerInterface,
debian_sources: Dict[Suite, SourcesText],
requests_mock,
) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]:
lister = DebianLister(
scheduler=swh_scheduler,
mirror_url=_mirror_url,
suites=list(debian_sources.keys()),
components=_components,
)
suite_pkg_info: DebianSuitePkgSrcInfo = {}
for suite, sources in debian_sources.items():
suite_pkg_info[suite] = defaultdict(list)
for pkg_src in Sources.iter_paragraphs(sources):
suite_pkg_info[suite][pkg_src["Package"]].append(pkg_src)
for idx_url, compression in lister.debian_index_urls(
suite, _components[0]):
if compression:
requests_mock.get(idx_url, status_code=404)
else:
requests_mock.get(idx_url, text=sources)
return lister, suite_pkg_info
def transport_response_simplified(self, response):
"""Decompress and parse the package index fetched in `transport_request`.
For each package, we "pivot" the file list entries (Files,
Checksums-Sha1, Checksums-Sha256), to return a files dict mapping
filenames to their checksums.
"""
if self.decompressor:
data = self.decompressor(response.raw)
else:
data = response.raw
for src_pkg in Sources.iter_paragraphs(data.readlines()):
files = defaultdict(dict)
for field in src_pkg._multivalued_fields:
if field.startswith('checksums-'):
sum_name = field[len('checksums-'):]
else:
sum_name = 'md5sum'
if field in src_pkg:
for entry in src_pkg[field]:
name = entry['name']
files[name]['name'] = entry['name']
files[name]['size'] = int(entry['size'], 10)
files[name][sum_name] = entry[sum_name]
yield {
'name': src_pkg['Package'],
'version': src_pkg['Version'],
'directory': src_pkg['Directory'],
'files': files,
}
def extract_deb_packages(data, packages):
""" Extract package metadata from debian Packages file
"""
extracted = extract(data)
package_re = re.compile('^Package: ', re.M)
plen = len(package_re.findall(extracted))
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
sio = StringIO(extracted)
for i, stanza in enumerate(Sources.iter_paragraphs(sio)):
fullversion = Version(stanza['version'])
arch = stanza['architecture']
name = stanza['package']
epoch = fullversion._BaseVersion__epoch
if epoch is None:
epoch = ''
version = fullversion._BaseVersion__upstream_version
release = fullversion._BaseVersion__debian_revision
if release is None:
release = ''
progress_update_s.send(sender=None, index=i + 1)
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='D')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo\n')
async def iter_sources(url):
async with ClientSession() as session:
async with session.get(url) as resp:
if resp.status != 200:
raise Exception("URL %s returned response code %d" % (url, resp.status))
contents = await resp.read()
if url.endswith(".gz"):
contents = gzip.decompress(contents)
for source in Sources.iter_paragraphs(contents):
yield source
def find_dsc(self, source):
sources = "{root}/dists/{suite}/{component}/source/Sources.gz".format(
root=self.root,
suite=source.suite.name,
component=source.component.name
)
for entry in Sources.iter_paragraphs(GzipFile(filename=sources)):
if entry['Package'] == source.name and entry['Version'] == source.version:
dsc = None
for line in entry['Files']:
if line['name'].endswith(".dsc"):
dsc = line['name']
break
return (entry['Directory'], dsc)
raise Exception("Package not found in Sources.gz")
def spec_show(request, spec):
tasks = BuildTask.objects.filter(specification=spec).select_related()
packages = Package.objects.filter(specification=spec)
# Combined build and task logs
logs = []
logs += [(log.created, '', str(log), 'spec', log)
for log in SpecificationLog.objects.filter(spec=spec)]
logs += [(log.created, getattr(log.task.builder, 'name', None), str(log), 'task', log)
for log in BuildTaskLog.objects.filter(task__specification=spec) \
.select_related(depth=3)]
logs = reversed(sorted(logs))
task_logs = BuildTaskLog.objects.filter(task__specification=spec)
# Source files
sources = []
dsc = spec.dsc()
if dsc is not None:
dsc_path = os.path.join(settings.DOWNLOAD_TARGET, str(spec.id), dsc)
if os.path.exists(dsc_path):
dsc_size = os.stat(dsc_path).st_size
sources.append((reverse('build_spec_source',
args=[spec.id, dsc]), dsc, dsc_size))
src = Sources(open(dsc_path))
for info in src['Files']:
fname = info['name']
size = int(info['size'])
path = os.path.join(settings.DOWNLOAD_TARGET, str(spec.id),
fname)
if os.path.exists(path):
sources.append((reverse('build_spec_source',
args=[spec.id,
fname]), fname, size))
context = {
'build': spec,
'tasks': tasks,
'packages': packages,
'logs': logs,
'sources': sources
}
return render_to_response('build/spec_show.html',
context,
context_instance=RequestContext(request))
def find_dsc(self, source):
sources = "{root}/dists/{suite}/{component}/source/Sources.gz".format(
root=self.root,
suite=source.suite.name,
component=source.component.name)
for entry in Sources.iter_paragraphs(GzipFile(filename=sources)):
if entry['Package'] == source.name and entry[
'Version'] == source.version:
dsc = None
for line in entry['Files']:
if line['name'].endswith(".dsc"):
dsc = line['name']
break
return (entry['Directory'], dsc)
raise RepoPackageNotFound('{0}-{1}'.format(source.name,
source.version))
def page_request(self, suite: Suite,
component: Component) -> DebianPageType:
"""Return parsed package Sources file for a given debian suite and component."""
for url, compression in self.debian_index_urls(suite, component):
response = requests.get(url, stream=True)
logging.debug("Fetched URL: %s, status code: %s", url,
response.status_code)
if response.status_code == 200:
break
else:
raise Exception("Could not retrieve sources index for %s/%s",
suite, component)
decompressor = decompressors.get(compression)
if decompressor:
data = decompressor(response.raw)
else:
data = response.raw
return Sources.iter_paragraphs(data.readlines())
def _init_test(
swh_scheduler: SchedulerInterface,
debian_sources: Dict[Suite, SourcesText],
requests_mock,
) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]:
lister = DebianLister(
scheduler=swh_scheduler,
mirror_url=_mirror_url,
suites=list(debian_sources.keys()),
components=_components,
)
suite_pkg_info: DebianSuitePkgSrcInfo = {}
for i, (suite, sources) in enumerate(debian_sources.items()):
# ensure to generate a different date for each suite
last_modified = formatdate(timeval=datetime.now().timestamp() + i, usegmt=True)
suite_pkg_info[suite] = defaultdict(list)
for pkg_src in Sources.iter_paragraphs(sources):
suite_pkg_info[suite][pkg_src["Package"]].append(pkg_src)
# backup package last update date
global _last_modified
_last_modified[pkg_src["Package"]] = last_modified
for idx_url, compression in lister.debian_index_urls(suite, _components[0]):
if compression:
requests_mock.get(idx_url, status_code=404)
else:
requests_mock.get(
idx_url,
text=sources,
headers={"Last-Modified": last_modified},
)
for idx_url, _ in lister.debian_index_urls(suite, _components[1]):
requests_mock.get(idx_url, status_code=404)
return lister, suite_pkg_info
def extract_deb_packages(data, url):
""" Extract package metadata from debian Packages file
"""
extracted = extract(data, url)
package_re = re.compile(b'^Package: ', re.M)
plen = len(package_re.findall(extracted))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
bio = BytesIO(extracted)
for i, stanza in enumerate(Sources.iter_paragraphs(bio)):
fullversion = Version(stanza['version'])
arch = stanza['architecture']
name = stanza['package']
epoch = fullversion._BaseVersion__epoch
if epoch is None:
epoch = ''
version = fullversion._BaseVersion__upstream_version
release = fullversion._BaseVersion__debian_revision
if release is None:
release = ''
progress_update_s.send(sender=None, index=i + 1)
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='D')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def download(self, target):
dsc = os.path.basename(self.source)
dsc_path = os.path.join(target, dsc)
# Download .dsc file
self.log.debug('Downloading dsc file: %s' % self.source)
tmp, headers = urlretrieve(self.source)
shutil.move(tmp, dsc_path)
# Download required files
src = Sources(open(dsc_path))
files = [info['name'] for info in src['Files']]
for fname in files:
url = os.path.join(self.base, fname)
self.log.debug('Downloading source file: %s' % url)
tmp, headers = urlretrieve(url)
path = os.path.join(target, fname)
shutil.move(tmp, path)
self.log.debug('All files downloaded')
return dsc
def page_request(self, suite: Suite,
component: Component) -> DebianPageType:
"""Return parsed package Sources file for a given debian suite and component."""
for url, compression in self.debian_index_urls(suite, component):
response = requests.get(url, stream=True)
logging.debug("Fetched URL: %s, status code: %s", url,
response.status_code)
if response.status_code == 200:
last_modified = response.headers.get("Last-Modified")
self.last_sources_update = (
parsedate_to_datetime(last_modified)
if last_modified else None)
decompressor = decompressors.get(compression)
if decompressor:
data = decompressor(response.raw).readlines()
else:
data = response.raw.readlines()
break
else:
data = ""
logger.debug("Could not retrieve sources index for %s/%s", suite,
component)
return Sources.iter_paragraphs(data)
def download(self):
# Prepare source package builder
from irgsh.source import SourcePackageBuilder
spec = self.spec
srcpkg = SourcePackageBuilder(
spec.source, spec.source_type, spec.source_opts, spec.orig,
spec.extraorig_set.all().values_list('orig', flat=True))
orig_path = None
logger = None
try:
# Download source and build source package
build_dir = tempfile.mkdtemp('-irgsh-build')
source_dir = tempfile.mkdtemp('-irgsh-build-source')
# Prepare logger
logdir = os.path.dirname(self.spec.source_log_path())
if not os.path.exists(logdir):
os.makedirs(logdir)
logname = os.path.join(logdir, 'source.log')
logger = open(logname, 'wb')
# Build source package
spec.add_log(_('Downloading source code'))
self.log.debug('[%s] Downloading source code' % (self.spec_id, ))
self.set_status(101)
self.dsc = srcpkg.build(build_dir, logger)
spec.add_log(_('Source code downloaded'))
self.log.debug('[%s] Source code downloaded' % (self.spec_id, ))
# Extract source package and send package description
dsc_file = os.path.join(build_dir, self.dsc)
source = os.path.join(source_dir, 'source')
cmd = 'dpkg-source -x %s %s' % (dsc_file, source)
logger.write('# Extracting source package\n')
logger.write('# Command: %s\n' % cmd)
logger.flush()
p = Popen(cmd.split(), stdout=logger, stderr=STDOUT)
p.communicate()
logger.write('\n')
changelog = os.path.join(source, 'debian', 'changelog')
control = os.path.join(source, 'debian', 'control')
self.send_description(changelog, control)
# Copy source packages
logger.write('# Copying source files:\n')
src = Sources(open(dsc_file))
files = [self.dsc] + [info['name'] for info in src['Files']]
for fname in files:
logger.write('# - %s\n' % os.path.basename(fname))
target = os.path.join(self.target, fname)
if os.path.exists(target):
os.unlink(target)
shutil.move(os.path.join(build_dir, fname), target)
os.chmod(target, 0644)
return files
except StandardError, e:
import traceback
traceback.print_stack()
logger.write('# Exception happened: %s: %s' % (type(e), str(e)))
send_tweet(self.spec, 'Failed to initialize')
raise
def get_sources(self, dist, component):
request = requests.get(self._get_source_url(dist, component))
data = io.BytesIO(request.content)
stream = gzip.GzipFile(fileobj=data)
yield from (Upload(x, self) for x in Sources.iter_paragraphs(stream))
elif (
f.endswith('/sources.deb822.gz')
or f.endswith('.sources.deb822.gz')
):
source_lists.add(f)
else:
runtime_package_lists.add(f)
sources = {} # type: typing.Dict[str, typing.Dict[str, Version]]
for f in source_lists:
with GzipFile(f, 'rb') as gzip_reader:
for source_stanza in Sources.iter_paragraphs(
sequence=gzip_reader,
encoding='utf-8',
):
source = Source(
source_stanza['package'],
Version(source_stanza['version']),
stanza=source_stanza,
)
sources.setdefault(source.name, {})[source.version] = source
for f in runtime_package_lists:
test.diag('Examining runtime %s...' % f)
with GzipFile(f, 'rb') as gzip_reader:
for binary_stanza in Packages.iter_paragraphs(
sequence=gzip_reader,
encoding='utf-8',
):
def fetch_sources(self):
fp = self.fetch_indexed_file('Sources')
return Sources.iter_paragraphs(fp, use_apt_pkg=False)
def build(self, target, logger=None):
'''Build source package.
This function returns the .dsc filename
'''
try:
cwd = os.getcwd()
build_path = tempfile.mkdtemp('-irgsh-srcpkg')
# Prepare source directory
self.slog(logger, '# Preparing source code directory')
package, version = self.prepare_source(build_path, logger)
source = '%s-%s' % (package, version)
self.slog(logger)
os.chdir(build_path)
self.slog(logger, '# File listing')
cmd = 'find -ls'
p = Popen(cmd.split(), stdout=logger, stderr=STDOUT,
preexec_fn=os.setsid)
out, err = p.communicate()
self.slog(logger)
# Build
self.log.debug('Building source package: ' \
'source=%s type=%s opts=%s orig=%s extra_orig=%s' % \
(self.source, self.source_type, \
self.source_opts, self.orig, self.extra_orig))
self.slog(logger, '# Building source package')
cmd = 'dpkg-source -b %s' % source
self.slog(logger, '# Command:', cmd)
p = Popen(cmd.split(), stdout=logger, stderr=STDOUT,
preexec_fn=os.setsid)
out, err = p.communicate()
self.slog(logger, '# Return code:', p.returncode, '\n')
if p.returncode != 0:
raise SourcePackageBuildError(p.returncode, out, err)
# Move result to the given target directory,
# existing files will be replaced
dsc = '%s_%s.dsc' % (package, version)
files = [dsc]
dsc_path = os.path.join(build_path, dsc)
src = Sources(open(dsc_path))
if not src.has_key('Files'):
raise KeyError, 'Invalid source package'
files += [item['name'] for item in src['Files']]
self.log.debug('Moving source package files: %s' % ', '.join(files))
for fname in files:
target_path = os.path.join(target, fname)
if os.path.exists(target_path):
os.unlink(target_path)
shutil.move(os.path.join(build_path, fname), target_path)
os.chmod(target_path, 0644)
self.log.debug('Source package built: %s' % dsc)
return dsc
finally:
shutil.rmtree(build_path)
os.chdir(cwd)
def _queue_sources_gz_pkgs(self, path):
sources = os.path.join(path, 'Sources')
with open(sources) as sources_file:
for src in Sources.iter_paragraphs(sources_file):
package = self.package_to_enqueue(src)
super().watch(**package)
elif (
f.endswith('/sources.deb822.gz')
or f.endswith('.sources.deb822.gz')
):
source_lists.add(f)
else:
runtime_package_lists.add(f)
sources = {} # type: typing.Dict[str, typing.Dict[str, Version]]
for f in source_lists:
with GzipFile(f, 'rb') as gzip_reader:
for source_stanza in Sources.iter_paragraphs(
sequence=gzip_reader,
encoding='utf-8',
):
source = Source(
source_stanza['package'],
Version(source_stanza['version']),
stanza=source_stanza,
)
sources.setdefault(source.name, {})[source.version] = source
for f in runtime_package_lists:
test.diag('Examining runtime %s...' % f)
with GzipFile(f, 'rb') as gzip_reader:
for binary_stanza in Packages.iter_paragraphs(
sequence=gzip_reader,
encoding='utf-8',
):
