def fetch(self,
statepath: str,
update: bool = True,
logger: Logger = NoopLogger()) -> bool:
if os.path.isdir(statepath) and not update:
logger.log('no update requested, skipping')
return False
persdata: Dict[str, Any] = {}
perspath = statepath + '.persdata'
try:
with open(perspath, 'rb') as rpersfile:
persdata = pickle.load(rpersfile)
except (EOFError, FileNotFoundError, pickle.UnpicklingError):
pass
with AtomicDir(statepath) as statedir:
have_changes = self._do_fetch(statedir, persdata, logger)
if persdata:
with AtomicFile(perspath, 'wb') as wpersfile:
pickle.dump(persdata, wpersfile.get_file())
wpersfile.get_file().flush()
os.fsync(wpersfile.get_file().fileno())
if not have_changes:
statedir.cancel()
return have_changes
def fetch(self,
statepath: str,
update: bool = True,
logger: Logger = NoopLogger()) -> bool:
if os.path.isdir(statepath) and not update:
logger.log('no update requested, skipping')
return False
persdata: Dict[str, Any] = {}
perspath = statepath + '.persdata'
if os.path.exists(perspath):
with open(perspath, 'rb') as rpersfile:
persdata = pickle.load(rpersfile)
with AtomicDir(statepath) as statedir:
have_changes = self._do_fetch(statedir, persdata, logger)
if persdata:
with AtomicFile(perspath, 'wb') as wpersfile:
pickle.dump(persdata, wpersfile.get_file())
if not have_changes:
statedir.cancel()
return have_changes
def fetch(self,
statepath: str,
update: bool = True,
logger: Logger = NoopLogger()) -> bool:
if not os.path.isdir(statepath):
with AtomicDir(statepath) as statedir:
return self._do_fetch(statedir.get_path(), logger)
elif update:
return self._do_update(statepath, logger)
else:
logger.log('no update requested, skipping')
return False
def _parse(self, repository: RepositoryMetadata, transformer: Optional[PackageTransformer], logger: Logger) -> None:
logger.log('parsing started')
if not os.path.isdir(self.parseddir):
os.mkdir(self.parseddir)
with AtomicDir(self._get_parsed_path(repository)) as state_dir:
serializer = ChunkedSerializer(state_dir.get_path(), MAX_PACKAGES_PER_CHUNK)
serializer.serialize(self._iter_parse_all_sources(repository, transformer, logger))
if self.safety_checks and serializer.get_num_packages() < repository['minpackages']:
raise TooLittlePackages(serializer.get_num_packages(), repository['minpackages'])
logger.log('parsing complete, {} packages'.format(serializer.get_num_packages()))