def merge_flm_files(self, new_flm, task, output):
computer = LocalComputer(task, self.root_path, self.__verify_flm_ready,
self.__verify_flm_failure,
lambda: self.query_extra_data_for_advanced_verification(new_flm),
use_task_resources=False,
additional_resources=[self.test_flm, new_flm])
computer.run()
if computer.tt is not None:
computer.tt.join()
else:
return False
if self.verification_error:
return False
commonprefix = common_dir(computer.tt.result['data'])
flm = find_file_with_ext(commonprefix, [".flm"])
stderr = filter(lambda x: os.path.basename(x) == "stderr.log", computer.tt.result['data'])
if flm is None or len(stderr) == 0:
return False
else:
try:
with open(stderr[0]) as f:
stderr_in = f.read()
if "ERROR" in stderr_in:
return False
except (IOError, OSError):
return False
shutil.copy(flm, os.path.join(self.tmp_dir, "test_result.flm"))
return True
def add_task(
self,
files,
task_id, # pylint: disable=too-many-arguments
resource_hash=None,
async_=True,
client_options=None):
prefix = self.storage.cache.get_prefix(task_id)
resources = self.storage.get_resources(task_id)
if prefix and resources:
logger.warning("Resource manager: Task '%s' exists", task_id)
return resources[0].hash, resources[0].files
if not files:
raise ResourceError("Empty files for task {}".format(task_id))
if len(files) == 1:
prefix = os.path.dirname(next(iter(files)))
else:
prefix = common_dir(files)
self.storage.cache.set_prefix(task_id, prefix)
return self._add_files(files,
task_id,
resource_hash=resource_hash,
client_options=client_options,
async_=async_)
def _add(self, files: dict):
from golem.core.fileshelper import common_dir
resource_hash = str(uuid.uuid4())
self._resources[resource_hash] = files
self._paths[resource_hash] = common_dir(files.keys())
return resource_hash
def _prepare_file_dict(cls, disk_files) -> Dict[str, str]:
prefix = common_dir(disk_files)
return {
absolute_path: relative_path(absolute_path, prefix)
for absolute_path in disk_files
}
def __final_flm_ready(self, results, time_spent):
commonprefix = common_dir(results['data'])
flm = find_file_with_ext(commonprefix, [".flm"])
if flm is None:
self.__final_flm_failure("No flm file created")
return
shutil.copy(flm, os.path.dirname(self.output_file))
new_flm = os.path.join(os.path.dirname(self.output_file),
os.path.basename(flm))
self.__generate_final_file(new_flm)
def __final_img_ready(self, results, time_spent):
commonprefix = common_dir(results['data'])
img = find_file_with_ext(commonprefix, ["." + self.output_format])
if img is None:
# TODO Maybe we should try again?
logger.error("No final file generated...")
else:
try:
shutil.copy(img, self.output_file + "." + self.output_format)
except (IOError, OSError) as err:
logger.warning("Couldn't rename and copy img file. %s", err)
self.notify_update_task()
def _prepare_resources(self, resources):
self.test_task_res_path = self.dir_manager.get_task_test_dir("")
def onerror(func, target_path, exc_info):
# Try to set write permissions
if not os.access(target_path, os.W_OK):
os.chmod(target_path, stat.S_IWUSR)
func(target_path)
else:
raise OSError('Cannot remove {}: {}'.format(
target_path, exc_info))
if os.path.exists(self.test_task_res_path):
shutil.rmtree(self.test_task_res_path, onerror=onerror)
if resources:
if len(resources) == 1 and os.path.isdir(resources[0]):
shutil.copytree(resources[0], self.test_task_res_path)
else:
# no trailing separator
if len(resources) == 1:
base_dir = os.path.dirname(resources[0])
else:
base_dir = common_dir(resources)
base_dir = os.path.normpath(base_dir)
for resource in filter(None, resources):
norm_path = os.path.normpath(resource)
sub_path = norm_path.replace(base_dir + os.path.sep, '', 1)
sub_dir = os.path.dirname(sub_path)
dst_dir = os.path.join(self.test_task_res_path, sub_dir)
os.makedirs(dst_dir, exist_ok=True)
name = os.path.basename(resource)
shutil.copy2(resource, os.path.join(dst_dir, name))
for res in self.additional_resources:
if not os.path.exists(self.test_task_res_path):
os.makedirs(self.test_task_res_path)
shutil.copy(res, self.test_task_res_path)
return True
def _add_task(self, files, task_id,
client=None, client_options=None):
if self.storage.cache.get_prefix(task_id):
logger.warn("Resource manager: Task {} already exists"
.format(task_id))
return
if not files:
raise RuntimeError("Empty input task resources")
elif len(files) == 1:
prefix = os.path.dirname(next(iter(files)))
else:
prefix = common_dir(files)
self.storage.cache.set_prefix(task_id, prefix)
self.add_files(files, task_id,
absolute_path=True,
client=client,
client_options=client_options)
def testGetResources(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
rm = rs.resource_manager
rm.storage.clear_cache()
rm.add_files(self.target_resources, self.task_id)
common_path = common_dir(self.target_resources)
resources = rm.storage.get_resources(self.task_id)
assert len(resources) == len(self.target_resources)
assert len(rs.pending_resources) == 0
rs.download_resources(resources, self.task_id)
assert len(rs.pending_resources[self.task_id]) == len(resources)
rs_aux = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager_aux, keys_auth, client)
relative_resources = []
for resource in resources:
relative_resources.append((resource.path.replace(common_path, '', 1),
resource.hash))
task_id_2 = str(uuid.uuid4())
assert len(rs_aux.pending_resources) == 0
rs_aux.download_resources(relative_resources, task_id_2)
assert len(rs_aux.pending_resources[task_id_2]) == len(resources)
rs_aux._download_resources(async=False)
for entry in relative_resources:
assert os.path.exists(entry[0])
assert client.downloaded
def testCommonDir(self):
paths = {
'win': [
(['C:/dir', "C:/"], 'C:'),
(['C:/dir', "C:\\"], "C:"),
(['C:/dir', 'C:\\'], "C:"),
(['C:/', "C:\\"], 'C:'),
(['Ł:/dir', "Ł:\\"], "Ł:"),
(['C:\\dirę', 'C:\\dirą', ], "C:"),
([
'C:/dir/file.txt',
"C:\\dir\\subdir\\file.txt",
],
'C:/dir'),
([
'C:\\dir\\file.txt',
'C:\\dir/subdir/file.txt',
],
"C:\\dir"),
([
'C:\\dir\\file.txt',
'C:/dir/subdir\\file.txt',
],
"C:\\dir"),
([
'C:/dir\\file.txt',
'C:\\dir/subdir\file.txt',
],
"C:/dir"),
([
'C:/dir/subdir/file.txt',
'C:\\dir/subdir-d\\subdir/file.txt',
],
'C:/dir'),
([
'C:/dir/subdir-d/file.txt',
'C:\\dir/subdir\\subdir/file.txt',
],
'C:/dir'),
([
'C:/dir/subdir',
'C:\\dir/subdir\\subdir/file.txt',
],
'C:/dir/subdir'),
([
'C:/dir/Subdir',
'C:\\dir/subdir\\subdir/file.txt',
],
'C:/dir/Subdir')
],
'other': [
(['/var/log/daemon.log'], ''),
(['/', '/var'], ''),
([], ''),
([
'/var/log/daemon/daemon.log',
'/var/log/daemon.log',
],
'/var/log'),
([
'/var/log-other/daemon/daemon.log',
'/var/log/daemon.log',
],
'/var'),
([
'/var/log-other/daemon/daemon.log',
'/var/log/daemon.log',
],
'/var'),
([
'/var/log-other/daemon/daemon.log',
'/var/log/daemon.log',
],
'/var'),
([
'/var/log-other/daemon/daemon.log',
'/var/log/daemon.log',
'/var/run/daemon.sock'
],
'/var'),
([
'/vąr/log/daęmon/daemon.log',
'/vąr/log/daęmon/daęmon.log',
'/vąr/lóg/daęmon/daęmon.log'
],
'/vąr'),
([
'/vąr/log/daęmon/daemon.log',
'/vąr/log/daęmon/daęmon.log'
],
'/vąr/log/daęmon'),
([
'/vąr/log/daęmon',
'/vąr/log/daęmon/subdir/daęmon.log'
],
'/vąr/log/daęmon'),
([
'/vąr/log/daęmon',
'/vąr/log/daęmon-d/subdir/daęmon.log'
],
'/vąr/log'),
([
'/var/log/daemon/daemon.log',
'/var/log/daemon/file.log',
'/var/log/daemon/file3.log',
'/var/log/daemon/other/file.log',
],
'/var/log/daemon'),
([
'/var/log/daemon',
'/var/log/daemon',
'/var/log/daemon',
'/var/log/daemon/other',
],
'/var/log/daemon'),
([
'/var/log/daemon',
'/var/log/Daemon',
'/var/log/daemon'
],
'/var/log'),
([
'/var/log/',
'/var/log/'
],
'/var/log')
]
}
def check(key, ign_case):
for t in paths[key]:
r = common_dir(t[0], ign_case=ign_case)
if r != t[1]:
self.fail("{} -> {} != {}".format(t[0], r, t[1]))
check('win', ign_case=True)
check('other', ign_case=False)
self.assertEqual(common_dir(None), '')
self.assertEqual(common_dir(['/var/log']), '')
def check(key, ign_case):
for t in paths[key]:
r = common_dir(t[0], ign_case=ign_case)
if r != t[1]:
self.fail("{} -> {} != {}".format(t[0], r, t[1]))