def create_state_files(self, targets, lock):
"""
Create state files for all targets.
"""
for t in targets:
orig_target, processed_data_item = t
input, data_item = orig_target
output = data_item.data.relative
if processed_data_item == None:
Logger.debug(
'Skipping creating state file for failed import {}'.format(
data_item.state.relative))
continue
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
def run_command(self,
cmd_args,
data_items_from_args,
not_data_items_from_args,
stdout=None,
stderr=None,
shell=False):
Logger.debug(
'Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'
.format(' '.join(cmd_args),
', '.join([x.data.dvc for x in data_items_from_args]),
stdout, stderr, shell))
repo_change = RepositoryChange(cmd_args,
self.settings,
stdout,
stderr,
shell=shell)
if not self.no_git_actions and not self._validate_file_states(
repo_change):
self.remove_new_files(repo_change)
raise RunError('Errors occurred.')
output_set = set(self.declaration_output_data_items +
repo_change.changed_data_items)
output_files_dvc = [x.data.dvc for x in output_set]
input_set = set(data_items_from_args +
self.declaration_input_data_items) - output_set
input_files_dvc = [x.data.dvc for x in input_set]
code_dependencies_dvc = self.git.abs_paths_to_dvc(
self.code_dependencies + not_data_items_from_args)
result = []
for data_item in repo_change.changed_data_items:
Logger.debug(
'Move output file "{}" to cache dir "{}" and create a symlink'.
format(data_item.data.relative, data_item.cache.relative))
data_item.move_data_to_cache()
Logger.debug('Create state file "{}"'.format(
data_item.state.relative))
state_file = StateFile(StateFile.COMMAND_RUN,
data_item.state.relative,
self.settings,
input_files_dvc,
output_files_dvc,
code_dependencies_dvc,
argv=cmd_args,
lock=self.lock,
stdout=self._stdout_to_dvc(stdout),
stderr=self._stdout_to_dvc(stderr),
shell=shell)
state_file.save()
result.append(state_file)
return result
def import_file(self, input, output, lock=False):
if not CmdImportFile.is_url(input):
if not os.path.exists(input):
raise ImportFileError(
'Input file "{}" does not exist'.format(input))
if not os.path.isfile(input):
raise ImportFileError(
'Input file "{}" has to be a regular file'.format(input))
if os.path.isdir(output):
output = os.path.join(output, os.path.basename(input))
data_item = self.settings.path_factory.data_item(output)
if os.path.exists(data_item.data.relative):
raise ImportFileError('Output file "{}" already exists'.format(
data_item.data.relative))
if not os.path.isdir(os.path.dirname(data_item.data.relative)):
raise ImportFileError(
'Output file directory "{}" does not exists'.format(
os.path.dirname(data_item.data.relative)))
cache_dir = os.path.dirname(data_item.cache.relative)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if CmdImportFile.is_url(input):
Logger.debug('Downloading file {} ...'.format(input))
self.download_file(input, data_item.cache.relative)
Logger.debug('Input file "{}" was downloaded to cache "{}"'.format(
input, data_item.cache.relative))
else:
copyfile(input, data_item.cache.relative)
Logger.debug('Input file "{}" was copied to cache "{}"'.format(
input, data_item.cache.relative))
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
pass
def import_file(self, input, output, is_reproducible):
if not CmdDataImport.is_url(input):
if not os.path.exists(input):
raise DataImportError(
'Input file "{}" does not exist'.format(input))
if not os.path.isfile(input):
raise DataImportError(
'Input file "{}" has to be a regular file'.format(input))
if os.path.isdir(output):
output = os.path.join(output, os.path.basename(input))
data_item = self.settings.path_factory.data_item(output)
if os.path.exists(data_item.data.relative):
raise DataImportError('Output file "{}" already exists'.format(
data_item.data.relative))
if not os.path.isdir(os.path.dirname(data_item.data.relative)):
raise DataImportError(
'Output file directory "{}" does not exists'.format(
os.path.dirname(data_item.data.relative)))
cache_dir = os.path.dirname(data_item.cache.relative)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if CmdDataImport.is_url(input):
Logger.debug('Downloading file {} ...'.format(input))
self.download_file(input, data_item.cache.relative)
Logger.debug('Input file "{}" was downloaded to cache "{}"'.format(
input, data_item.cache.relative))
else:
copyfile(input, data_item.cache.relative)
Logger.debug('Input file "{}" was copied to cache "{}"'.format(
input, data_item.cache.relative))
data_item.create_symlink()
Logger.debug(
'Symlink from data file "{}" to the cache file "{}" was created'.
format(data_item.data.relative, data_item.cache.relative))
state_file = StateFile(data_item.state.relative, self.git, [],
[output], [], is_reproducible)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
pass
def create_state_files(self, targets):
"""
Create state files for all targets.
"""
for data_item in targets:
Logger.debug('Creating state file for {}'.format(
data_item.data.relative))
fname = os.path.basename(data_item.data.relative +
StateFile.STATE_FILE_SUFFIX)
out = StateFile.parse_deps_state(self.settings,
[data_item.data.relative],
currdir=os.path.curdir)
state_file = StateFile(fname=fname,
cmd=None,
out=out,
out_git=[],
deps=[],
locked=True)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
def create_state_files(self, targets, lock):
"""
Create state files for all targets.
"""
for t in targets:
input = t[0]
output = t[1]
data_item = t[2]
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))