def setData(self, data): self.input_validated = False if not self._validateInput(data): return [] self.package = data["package"] self.product = data["product"] self.distribution = data["distribution"] # TODO(jchaloup): check the date is in required format if 'start_date' in data: self.start_ts = dateToTimestamp(data["start_date"]) else: self.start_ts = dateToTimestamp('1970-01-02') if 'end_date' in data: self.end_ts = dateToTimestamp(data["end_date"]) else: self.end_ts = int(time.time() + 86400) if 'start_timestamp' in data: self.start_ts = data["start_timestamp"] if 'end_timestamp' in data: self.end_ts = data["end_timestamp"] return True
def setData(self, data): self.input_validated = False if not self._validateInput(data): return False self.repository = data['repository'] self.repository_directory = data['resource'] self.client = RepositoryClientBuilder().buildWithLocalClient(self.repository, self.repository_directory) # TODO(jchaloup): check the date is in required format if 'start_date' in data: self.start_date = dateToTimestamp(data["start_date"]) else: self.start_date = dateToTimestamp('1970-01-02') if 'end_date' in data: self.end_date = dateToTimestamp(data["end_date"]) else: self.end_date = int(time.time() + 86400) if 'start_timestamp' in data: self.start_date = data["start_timestamp"] if 'end_timestamp' in data: self.end_date = data["end_timestamp"] # Check single commit only? self.commit = "" if 'commit' in data: self.commit = data["commit"] # Extract data from a single branch? self.branch = "" if 'branch' in data: self.branch = data["branch"] return True
def execute(self): """Impementation of concrete data processor""" self.itemset_info = {} self.items = {} # check storage # read the current list of builds stored in storage # if the youngest build is younger than end_date, no need to extract data # if so, filter out builds older than start_date # otherwise extract the missing builds from build system, # update the current list of builds and return the relevant sublist # with all builds inside the sublist # cached? data = { "artefact": ARTEFACT_CACHE_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS, "product": self.product, "distribution": self.distribution, "package": self.package } cache_found = False if self.retrieve_artefacts: cache_found = True try: cache = self.ff.bake(self.read_storage_plugin).call(data) except KeyError: cache_found = False # Is the date interval covered? if cache_found and (self.end_date != "" or self.end_timestamp != "") and (self.start_date != "" or self.start_timestamp != ""): # both ends of date interval are specified if self.start_date != "": start = dateToTimestamp(self.start_date) else: start = self.start_timestamp if self.end_date != "": end = dateToTimestamp(self.end_date) else: end = self.end_timestamp req_interval = (start, end) covered = False for coverage in cache["coverage"]: coverage_interval = (coverage["start"], coverage["end"]) # if date interval not covered => extract if intervalCovered(req_interval, coverage_interval): covered = True break # data interval is covered, retrieve a list of all relevant commits if covered: # retrieve repository info artefact data = { "artefact": ARTEFACT_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS, "product": self.product, "distribution": self.distribution, "package": self.package } info_found, itemset_info = self.ff.bake(self.read_storage_plugin).call(data) if info_found: # retrieve commits (if any of them not found, continue) self.items = self._retrieveItemsFromCache(cache, itemset_info, start, end) self.itemset_info = itemset_info return True # ================================================================ # cache not found, info not retrieved or data interval not covered # ================================================================ # extract artefacts data = self._extractItemSetInfo() # store item artefacts, get a list of stored items and item set info artefact extracted_itemset_info = data["package_builds"] stored_items, not_stored_items = self._storeItems(data["builds"]) # create the cache from extracted item extracted_itemset_cache = ItemSetCache().addItems(stored_items, not_stored_items) # return unchanged repository that was extracted self.itemset_info = extracted_itemset_info # update the coverage intervals with the list of stored items extracted_itemset_info["coverage"] = extracted_itemset_cache.intervals() # TODO(jchaloup): check for empty list of commits => we end here # ============================================= # if no cache found # - info artefact may still exists (cache was removed) # - no info artefact # # if no info artefact => store the created info and cache # if info artefact => reconstruct cache from the info by reading all referrenced commit artefact # In both cases, i am retrieving info artefact # ============================================= # if cache found: # - if info artefact found => compare the cache with a list of commits and coverage of the info. # if there is inconsistency, regenerate cache based on the retrieved repository info artefact. # - if info artefact not found => store the extracted info artefact and replace the cache with new cache. # In both cases, I am retrieving info artefact # ============================================= # Retranslated: # if no info found => replace the current cache with extracted one # if info found => sync the current cache with the info and merge both pairs of caches and infos # retrieve repository info artefact data = { "artefact": ARTEFACT_GOLANG_PROJECT_DISTRIBUTION_PACKAGE_BUILDS, "product": self.product, "distribution": self.distribution, "package": self.package } info_found = False if self.retrieve_artefacts: info_found = True try: itemset_info = self.ff.bake(self.read_storage_plugin).call(data) except KeyError: info_found = False # info not found if not info_found: updated_itemset_info = extracted_itemset_info # info found else: extracted_itemset_cache.addItems( # reconstruct points self._generatePointsFromItemSetInfoArtefact(itemset_info) ) # merge both infos updated_itemset_info = self._mergeItemSetInfoArtefacts( itemset_info, extracted_itemset_info, extracted_itemset_cache.intervals() ) # store both new info and cache self._storeInfo(updated_itemset_info) self._storeCache( self._generateNewCache(extracted_itemset_cache) ) return True