def _get_local_file(self, url_obj, asset_path, _): """ Create a symlink for a local file into the cache. :param url_obj: object from urlparse. :param asset_path: full path of the asset file. :returns: if the local file matches the hash. :rtype: bool """ if os.path.isdir(url_obj.path): path = os.path.join(url_obj.path, self.name) else: path = url_obj.path with FileLock(asset_path, 1): try: os.symlink(path, asset_path) self._create_hash_file(asset_path) return self._verify_hash(asset_path) except OSError as detail: if detail.errno == errno.EEXIST: os.remove(asset_path) os.symlink(path, asset_path) self._create_hash_file(asset_path) return self._verify_hash(asset_path)
def file_lock_action(args): path, players = args max_individual_timeout = 0.021 max_timeout = max_individual_timeout * players with FileLock(path, max_timeout): sleeptime = random.random() / 100 time.sleep(sleeptime)
def test_fetch_lockerror(self): with FileLock(os.path.join(self.cache_dir, self.assetname)): a = asset.Asset(self.url, asset_hash=self.assethash, algorithm='sha1', locations=None, cache_dirs=[self.cache_dir], expire=None) self.assertRaises(EnvironmentError, a.fetch)
def test_fetch_lockerror(self): dirname = os.path.join(self.cache_dir, 'by_name') os.makedirs(dirname) with FileLock(os.path.join(dirname, self.assetname)): a = asset.Asset(self.assetname, asset_hash=self.assethash, algorithm='sha1', locations=['file://foo1', 'file://foo2'], cache_dirs=[self.cache_dir], expire=None) self.assertRaises(OSError, a.fetch)
def _create_hash_file(self, asset_path): """ Compute the hash of the asset file and add it to the CHECKSUM file. :param asset_path: full path of the asset file. """ result = crypto.hash_file(asset_path, algorithm=self.algorithm) hash_file = self._get_hash_file(asset_path) with FileLock(hash_file, 30): with open(hash_file, 'w') as fp: fp.write('%s %s\n' % (self.algorithm, result))
def _create_hash_file(self, asset_path): """ Compute the hash of the asset file and add it to the CHECKSUM file. :param asset_path: full path of the asset file. """ result = crypto.hash_file(asset_path, algorithm=self.algorithm) hash_file = self._get_hash_file(asset_path) with FileLock(hash_file, 30): with open(hash_file, 'w', encoding='utf-8') as fp: fp.write(f'{self.algorithm} {result}\n')
def test_filelock(self): # Calculate the timeout based on t_100_iter + 2e-5*players start = time.time() for _ in range(100): with FileLock(self.tmpdir.name): pass timeout = 0.02 + (time.time() - start) players = 1000 pool = multiprocessing.Pool(players) args = [(self.tmpdir.name, players, timeout)] * players try: pool.map(file_lock_action, args) except Exception: msg = 'Failed to run FileLock with %s players:\n%s' msg %= (players, prepare_exc_info(sys.exc_info())) self.fail(msg)
def read_hash_from_file(cls, filename): """Read the CHECKSUM file and return the hash. This method raises a FileNotFoundError if file is missing and assumes that filename is the CHECKSUM filename. :rtype: list with algorithm and hash """ try: with FileLock(filename, 30): with open(filename, 'r') as hash_file: for line in hash_file: # md5 is 32 chars big and sha512 is 128 chars big. # others supported algorithms are between those. if re.match('^.* [a-f0-9]{32,128}', line): return line.split() except Exception: # pylint: disable=W0703 exc_type, exc_value = sys.exc_info()[:2] LOG.error('%s: %s', exc_type.__name__, exc_value) return [None, None]
def _download(self, url_obj, asset_path, timeout=None): """ Download the asset from an uri. :param url_obj: object from urlparse. :param asset_path: full path of the asset file. :param timeout: timeout in seconds. Default is :data:`avocado.utils.asset.DOWNLOAD_TIMEOUT`. :returns: if the downloaded file matches the hash. :rtype: bool """ timeout = timeout or DOWNLOAD_TIMEOUT try: # Temporary unique name to use while downloading temp = '%s.%s' % (asset_path, str(uuid.uuid4())) # To avoid parallel downloads of the same asset, and errors during # the write after download, let's get the lock before start the # download. with FileLock(asset_path, 120): try: self.find_asset_file(create_metadata=True) return True except OSError: LOG.debug("Asset not in cache after lock, fetching it.") url_download(url_obj.geturl(), temp, timeout=timeout) shutil.copy(temp, asset_path) self._create_hash_file(asset_path) if not self._verify_hash(asset_path): msg = "Hash mismatch. Ignoring asset from the cache" raise OSError(msg) return True finally: try: os.remove(temp) except FileNotFoundError: LOG.info("Temporary asset file unavailable due to failed" " download attempt.")
def test_locked_by_me(self): with FileLock(self.filename): self.assertRaises(AlreadyLocked, self._readfile)
def _readfile(self): with FileLock(self.filename): with open(self.filename, "r", encoding="utf-8") as f: return f.read()
def _readfile(self): with FileLock(self.filename): with open(self.filename, 'r', encoding='utf-8') as f: return f.read()
def _readfile(self): with FileLock(self.filename): with open(self.filename, 'r') as f: return f.read()