Esempio n. 1
0
def process_archives(config, incoming_dir, feed):
	"""feed is the parsed XML being processed. Any archives are in 'incoming_dir'."""

	# Pick a digest to check (maybe we should check all of them?)
	# Find required archives and check they're in 'incoming'
	archives = []
	for impl in feed.implementations.values():
		required_digest = pick_digest(impl)
		for method in impl.download_sources:
			archives += process_method(config, incoming_dir, impl, method, required_digest)

	# Copy to archives directory and upload
	config.upload_archives(archives)

	# Test uploads
	for archive in archives:
		if config.ARCHIVES_BASE_URL.startswith('http://TESTING/'): continue
		url = config.ARCHIVES_BASE_URL + archive.rel_url
		actual_size = urltest.get_size(url)
		if actual_size != archive.size:
			raise SafeException("Archive {url} has size {actual}, but expected {expected} bytes".format(
				url = url, actual = actual_size, expected = archive.size))

	for archive in archives:
		sha1 = get_sha1(archive.source_path)
		config.archive_db.add(archive.basename, config.ARCHIVES_BASE_URL + archive.rel_url, sha1)

	return archives
Esempio n. 2
0
def process_method(config, incoming_dir, impl, method, required_digest):
	archives = []

	if not isinstance(method, model.Recipe):
		# turn an individual method into a single-step Recipe
		step = method
		method = model.Recipe()
		method.steps.append(step)

	has_external_archives = False

	for step in method.steps:
		if not hasattr(step, 'url'): continue
		archive = step.url

		if '/' in archive:
			has_external_archives = True
			url = archive
			actual_size = urltest.get_size(url)
			if actual_size != step.size:
				raise SafeException("External archive {url} has size {actual}, but expected {expected} bytes".format(
					url = url, actual = actual_size, expected = step.size))
			continue		# Hosted externally

		if not valid_simple_name.match(archive):
			raise SafeException("Illegal archive name '{name}'".format(name = archive))

		archive_path = join(incoming_dir, archive)
		if not os.path.isfile(archive_path):
			raise SafeException("Referenced upload '{path}' not found".format(path = archive_path))

		existing = config.archive_db.entries.get(archive, None)
		if existing is not None:
			new_sha1 = get_sha1(archive_path)
			if new_sha1 != existing.sha1:
				raise SafeException("A different archive with basename '{name}' is "
						    "already in the repository: {archive}".format(name = archive, archive = existing))
		else:
			archive_rel_url = paths.get_archive_rel_url(config, archive, impl)

			# Copy to archives directory

			backup_dir = config.LOCAL_ARCHIVES_BACKUP_DIR	# note: may be relative; that's OK
			backup_target_dir = join(backup_dir, dirname(archive_rel_url))
			paths.ensure_dir(backup_target_dir)
			copy_path = join(backup_dir, archive_rel_url)
			shutil.copyfile(archive_path, copy_path)

			stored_archive = Archive(abspath(copy_path), archive_rel_url, step.size, archive_path)
			actual_size = os.path.getsize(stored_archive.source_path)
			if step.size != actual_size:
				raise SafeException("Archive '{archive}' has size '{actual}', but XML says size should be {expected}".format(
					archive = archive,
					actual = actual_size,
					expected = step.size))
			archives.append(stored_archive)

		step.url = os.path.abspath(archive_path)			# (just used below to test it)

	if not has_external_archives:
		# Check archives unpack to give the correct digests
		impl.feed.local_path = "/is-local-hack.xml"
		try:
			blocker = config.zconfig.fetcher.cook(required_digest, method,
						config.zconfig.stores, impl_hint = impl, dry_run = True, may_use_mirror = False)
			tasks.wait_for_blocker(blocker)
		finally:
			impl.feed.local_path = None

	return archives
Esempio n. 3
0
	def default_test(archive, url):
		actual_size = urltest.get_size(url)
		if actual_size != archive.size:
			raise SafeException("Archive {url} has size {actual}, but expected {expected} bytes".format(
				url = url, actual = actual_size, expected = archive.size))
Esempio n. 4
0
def _default_archive_test(archive, url):
    actual_size = urltest.get_size(url)
    if actual_size != archive.size:
        raise SafeException(
            "Archive {url} has size {actual}, but expected {expected} bytes".
            format(url=url, actual=actual_size, expected=archive.size))