def _get_local_file(self, url_obj, asset_path, _):
"""
Create a symlink for a local file into the cache.
:param url_obj: object from urlparse.
:param asset_path: full path of the asset file.
:returns: if the local file matches the hash.
:rtype: bool
"""
if os.path.isdir(url_obj.path):
path = os.path.join(url_obj.path, self.name)
else:
path = url_obj.path
with FileLock(asset_path, 1):
try:
os.symlink(path, asset_path)
self._create_hash_file(asset_path)
return self._verify_hash(asset_path)
except OSError as detail:
if detail.errno == errno.EEXIST:
os.remove(asset_path)
os.symlink(path, asset_path)
self._create_hash_file(asset_path)
return self._verify_hash(asset_path)
def file_lock_action(args):
path, players = args
max_individual_timeout = 0.021
max_timeout = max_individual_timeout * players
with FileLock(path, max_timeout):
sleeptime = random.random() / 100
time.sleep(sleeptime)
def test_fetch_lockerror(self):
with FileLock(os.path.join(self.cache_dir, self.assetname)):
a = asset.Asset(self.url,
asset_hash=self.assethash,
algorithm='sha1',
locations=None,
cache_dirs=[self.cache_dir],
expire=None)
self.assertRaises(EnvironmentError, a.fetch)
def test_fetch_lockerror(self):
dirname = os.path.join(self.cache_dir, 'by_name')
os.makedirs(dirname)
with FileLock(os.path.join(dirname, self.assetname)):
a = asset.Asset(self.assetname,
asset_hash=self.assethash,
algorithm='sha1',
locations=['file://foo1', 'file://foo2'],
cache_dirs=[self.cache_dir],
expire=None)
self.assertRaises(OSError, a.fetch)
def _create_hash_file(self, asset_path):
"""
Compute the hash of the asset file and add it to the CHECKSUM
file.
:param asset_path: full path of the asset file.
"""
result = crypto.hash_file(asset_path, algorithm=self.algorithm)
hash_file = self._get_hash_file(asset_path)
with FileLock(hash_file, 30):
with open(hash_file, 'w') as fp:
fp.write('%s %s\n' % (self.algorithm, result))
def _create_hash_file(self, asset_path):
"""
Compute the hash of the asset file and add it to the CHECKSUM
file.
:param asset_path: full path of the asset file.
"""
result = crypto.hash_file(asset_path, algorithm=self.algorithm)
hash_file = self._get_hash_file(asset_path)
with FileLock(hash_file, 30):
with open(hash_file, 'w', encoding='utf-8') as fp:
fp.write(f'{self.algorithm} {result}\n')
def test_filelock(self):
# Calculate the timeout based on t_100_iter + 2e-5*players
start = time.time()
for _ in range(100):
with FileLock(self.tmpdir.name):
pass
timeout = 0.02 + (time.time() - start)
players = 1000
pool = multiprocessing.Pool(players)
args = [(self.tmpdir.name, players, timeout)] * players
try:
pool.map(file_lock_action, args)
except Exception:
msg = 'Failed to run FileLock with %s players:\n%s'
msg %= (players, prepare_exc_info(sys.exc_info()))
self.fail(msg)
def read_hash_from_file(cls, filename):
"""Read the CHECKSUM file and return the hash.
This method raises a FileNotFoundError if file is missing and assumes
that filename is the CHECKSUM filename.
:rtype: list with algorithm and hash
"""
try:
with FileLock(filename, 30):
with open(filename, 'r') as hash_file:
for line in hash_file:
# md5 is 32 chars big and sha512 is 128 chars big.
# others supported algorithms are between those.
if re.match('^.* [a-f0-9]{32,128}', line):
return line.split()
except Exception: # pylint: disable=W0703
exc_type, exc_value = sys.exc_info()[:2]
LOG.error('%s: %s', exc_type.__name__, exc_value)
return [None, None]
def _download(self, url_obj, asset_path, timeout=None):
"""
Download the asset from an uri.
:param url_obj: object from urlparse.
:param asset_path: full path of the asset file.
:param timeout: timeout in seconds. Default is
:data:`avocado.utils.asset.DOWNLOAD_TIMEOUT`.
:returns: if the downloaded file matches the hash.
:rtype: bool
"""
timeout = timeout or DOWNLOAD_TIMEOUT
try:
# Temporary unique name to use while downloading
temp = '%s.%s' % (asset_path, str(uuid.uuid4()))
# To avoid parallel downloads of the same asset, and errors during
# the write after download, let's get the lock before start the
# download.
with FileLock(asset_path, 120):
try:
self.find_asset_file(create_metadata=True)
return True
except OSError:
LOG.debug("Asset not in cache after lock, fetching it.")
url_download(url_obj.geturl(), temp, timeout=timeout)
shutil.copy(temp, asset_path)
self._create_hash_file(asset_path)
if not self._verify_hash(asset_path):
msg = "Hash mismatch. Ignoring asset from the cache"
raise OSError(msg)
return True
finally:
try:
os.remove(temp)
except FileNotFoundError:
LOG.info("Temporary asset file unavailable due to failed"
" download attempt.")
def test_locked_by_me(self):
with FileLock(self.filename):
self.assertRaises(AlreadyLocked, self._readfile)
def _readfile(self):
with FileLock(self.filename):
with open(self.filename, "r", encoding="utf-8") as f:
return f.read()
def _readfile(self):
with FileLock(self.filename):
with open(self.filename, 'r', encoding='utf-8') as f:
return f.read()
def _readfile(self):
with FileLock(self.filename):
with open(self.filename, 'r') as f:
return f.read()