Beispiel #1
0
	def setData(self, data):
		self.input_validated = False
		if not self._validateInput(data):
			return []

		self.package = data["package"]
		self.product = data["product"]
		self.distribution = data["distribution"]

		# TODO(jchaloup): check the date is in required format
		if 'start_date' in data:
			self.start_ts = dateToTimestamp(data["start_date"])
		else:
			self.start_ts = dateToTimestamp('1970-01-02')

		if 'end_date' in data:
			self.end_ts = dateToTimestamp(data["end_date"])
		else:
			self.end_ts = int(time.time() + 86400)

		if 'start_timestamp' in data:
			self.start_ts = data["start_timestamp"]

		if 'end_timestamp' in data:
			self.end_ts = data["end_timestamp"]

		return True
Beispiel #2
0
	def setData(self, data):
		self.input_validated = False
		if not self._validateInput(data):
			return False

		self.repository = data['repository']
		self.repository_directory = data['resource']

		self.client = RepositoryClientBuilder().buildWithLocalClient(self.repository, self.repository_directory)

		# TODO(jchaloup): check the date is in required format
		if 'start_date' in data:
			self.start_date = dateToTimestamp(data["start_date"])
		else:
			self.start_date = dateToTimestamp('1970-01-02')

		if 'end_date' in data:
			self.end_date = dateToTimestamp(data["end_date"])
		else:
			self.end_date = int(time.time() + 86400)

		if 'start_timestamp' in data:
			self.start_date = data["start_timestamp"]

		if 'end_timestamp' in data:
			self.end_date = data["end_timestamp"]

		# Check single commit only?
		self.commit = ""
		if 'commit' in data:
			self.commit = data["commit"]

		# Extract data from a single branch?
		self.branch = ""
		if 'branch' in data:
			self.branch = data["branch"]

		return True
Beispiel #3
0
	def execute(self):
		"""Impementation of concrete data processor"""

		self.itemset_info = {}
		self.items = {}

		# check storage
		# read the current list of builds stored in storage
		# if the youngest build is younger than end_date, no need to extract data
		# if so, filter out builds older than start_date
		# otherwise extract the missing builds from build system,
		# update the current list of builds and return the relevant sublist
		# with all builds inside the sublist

		# cached?
		data = {
			"artefact": ARTEFACT_CACHE_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS,
			"product": self.product,
			"distribution": self.distribution,
			"package": self.package
		}

		cache_found = False
		if self.retrieve_artefacts:
			cache_found = True
			try:
				cache = self.ff.bake(self.read_storage_plugin).call(data)
			except KeyError:
				cache_found = False

		# Is the date interval covered?
		if cache_found and (self.end_date != "" or self.end_timestamp != "") and (self.start_date != "" or self.start_timestamp != ""):
			# both ends of date interval are specified
			if self.start_date != "":
				start = dateToTimestamp(self.start_date)
			else:
				start = self.start_timestamp

			if self.end_date != "":
				end = dateToTimestamp(self.end_date)
			else:
				end = self.end_timestamp

			req_interval = (start, end)

			covered = False
			for coverage in cache["coverage"]:
				coverage_interval = (coverage["start"], coverage["end"])
				# if date interval not covered => extract
				if intervalCovered(req_interval, coverage_interval):
					covered = True
					break

			# data interval is covered, retrieve a list of all relevant commits
			if covered:
				# retrieve repository info artefact
				data = {
					"artefact": ARTEFACT_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS,
					"product": self.product,
					"distribution": self.distribution,
					"package": self.package
				}

				info_found, itemset_info = self.ff.bake(self.read_storage_plugin).call(data)
				if info_found:
					# retrieve commits (if any of them not found, continue)
					self.items = self._retrieveItemsFromCache(cache, itemset_info, start, end)
					self.itemset_info = itemset_info

					return True

		# ================================================================
		# cache not found, info not retrieved or data interval not covered
		# ================================================================

		# extract artefacts
		data = self._extractItemSetInfo()
		# store item artefacts, get a list of stored items and item set info artefact
		extracted_itemset_info = data["package_builds"]
		stored_items, not_stored_items = self._storeItems(data["builds"])

		# create the cache from extracted item
		extracted_itemset_cache = ItemSetCache().addItems(stored_items, not_stored_items)

		# return unchanged repository that was extracted
		self.itemset_info = extracted_itemset_info
		# update the coverage intervals with the list of stored items
		extracted_itemset_info["coverage"] = extracted_itemset_cache.intervals()
		# TODO(jchaloup): check for empty list of commits => we end here

		# =============================================
		# if no cache found
		# - info artefact may still exists (cache was removed)
		# - no info artefact
		#
		# if no info artefact => store the created info and cache
		# if info artefact => reconstruct cache from the info by reading all referrenced commit artefact
		# In both cases, i am retrieving info artefact
		# =============================================
		# if cache found:
		# - if info artefact found => compare the cache with a list of commits and coverage of the info.
		#   if there is inconsistency, regenerate cache based on the retrieved repository info artefact.
		# - if info artefact not found => store the extracted info artefact and replace the cache with new cache.
		# In both cases, I am retrieving info artefact
		# =============================================
		# Retranslated:
		# if no info found => replace the current cache with extracted one
		# if info found => sync the current cache with the info and merge both pairs of caches and infos

		# retrieve repository info artefact
		data = {
			"artefact": ARTEFACT_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS,
			"product": self.product,
			"distribution": self.distribution,
			"package": self.package
		}

		info_found = False
		if self.retrieve_artefacts:
			info_found = True
			try:
				itemset_info = self.ff.bake(self.read_storage_plugin).call(data)
			except KeyError:
				info_found = False

		# info not found
		if not info_found:
			updated_itemset_info = extracted_itemset_info
		# info found
		else:
			extracted_itemset_cache.addItems(
				# reconstruct points
				self._generatePointsFromItemSetInfoArtefact(itemset_info)
			)

			# merge both infos
			updated_itemset_info = self._mergeItemSetInfoArtefacts(
				itemset_info,
				extracted_itemset_info,
				extracted_itemset_cache.intervals()
			)

		# store both new info and cache
		self._storeInfo(updated_itemset_info)
		self._storeCache(
			self._generateNewCache(extracted_itemset_cache)
		)

		return True