def load_archives(session, repository):
if not repository.processed & consts.R_EXTRACTED_FILES:
if repository.zip_path.exists():
vprint(1, 'Extracting files')
result = process_repository(session, repository, skip_if_error=0)
try:
session.commit()
if result != "done":
raise Exception("Extraction failure. Fallback")
vprint(1, result)
except Exception as err:
vprint(1, 'Failed: {}'.format(err))
try:
tarzip = tarfile.open(str(repository.zip_path))
if repository.processed & consts.R_COMPRESS_ERROR:
repository.processed -= consts.R_COMPRESS_ERROR
session.add(repository)
except tarfile.ReadError:
repository.processed |= consts.R_COMPRESS_ERROR
session.add(repository)
return True, None
zip_path = to_unicode(repository.hash_dir2)
return False, (tarzip, zip_path)
elif repository.path.exists():
repo_path = to_unicode(repository.path)
return False, (None, repo_path)
else:
repository.processed |= consts.R_UNAVAILABLE_FILES
session.add(repository)
vprint(1, "Failed to load repository. Skipping")
return True, None
tarzip = {
fil.path
for fil in session.query(RepositoryFile).filter(
RepositoryFile.repository_id == repository.id)
}
zip_path = ""
if tarzip:
return False, (tarzip, zip_path)
return True, None
def load_notebook(session, cell, dispatches, repository, skip_repo,
skip_notebook, notebook_id, archives, checker):
if notebook_id != cell.notebook_id:
notebook_id = cell.notebook_id
notebook = cell.notebook_obj
if not notebook.compatible_version:
pyexec = get_pyexec(notebook.py_version, config.VERSIONS)
if sys.executable != pyexec:
dispatches.add((notebook.id, pyexec))
return skip_repo, True, cell.notebook_id, archives, None
if archives == "todo":
skip_repo, archives = load_archives(session, repository)
if skip_repo:
return skip_repo, skip_notebook, cell.notebook_id, archives, None
if archives is None:
return True, True, cell.notebook_id, archives, None
vprint(1, 'Processing notebook: {}'.format(notebook))
name = to_unicode(notebook.name)
tarzip, repo_path = archives
notebook_path = os.path.join(repo_path, name)
try:
if isinstance(tarzip, set):
checker = SetLocalChecker(tarzip, notebook_path)
elif tarzip:
checker = CompressedLocalChecker(tarzip, notebook_path)
else:
checker = PathLocalChecker(notebook_path)
if not checker.exists(notebook_path):
raise Exception(
"Repository content problem. Notebook not found")
return skip_repo, False, cell.notebook_id, archives, checker
except Exception as err:
vprint(
2,
"Failed to load notebook {} due to {}".format(notebook, err))
return skip_repo, True, cell.notebook_id, archives, checker
return skip_repo, skip_notebook, notebook_id, archives, checker
def __init__(self, tarzip, notebook_path):
path = to_unicode(notebook_path)
self.base = os.path.dirname(path)
self.tarzip = tarzip
def __init__(self, dirset, notebook_path):
path = to_unicode(notebook_path)
self.base = os.path.dirname(path)
self.dirset = dirset
def __init__(self, path):
path = to_unicode(path)
self.base = os.path.dirname(path)