class LocationTests:
def setUp(self):
self.locn = TemporaryDirectory()
self.locn.__enter__()
def tearDown(self):
self.locn.__exit__(None, None, None)
class ArchiveCopy(ar.Archive):
def __init__(self, path):
self.__dir = TemporaryDirectory()
dest = '%s/%s' % (self.__dir.name, os.path.basename(path))
shutil.copyfile(path, dest)
super().__init__(dest)
def __enter__(self):
self.__dir.__enter__()
return super().__enter__()
def __exit__(self, type, value, traceback):
super().__exit__(type, value, traceback)
self.__dir.__exit__(type, value, traceback)
def temp_dir(self) -> str:
if not self._temp_dir:
temp_dir_context = TemporaryDirectory( # pylint: disable=consider-using-with
suffix='-sb-parser')
self.exit_stack.push(temp_dir_context)
self._temp_dir = temp_dir_context.__enter__()
return self._temp_dir
class InTemporaryDirectory:
''' Create, return, and change directory to a temporary directory
Examples
--------
>>> import os
>>> my_cwd = os.getcwd()
>>> with InTemporaryDirectory() as tmpdir:
... _ = open('test.txt', 'wt').write('some text')
... assert os.path.isfile('test.txt')
... assert os.path.isfile(os.path.join(tmpdir, 'test.txt'))
>>> os.path.exists(tmpdir)
False
>>> os.getcwd() == my_cwd
True
'''
def __init__(self):
self._tmpdir = TemporaryDirectory()
@property
def name(self):
return self._tmpdir.name
def __enter__(self):
self._pwd = os.getcwd()
os.chdir(self._tmpdir.name)
return self._tmpdir.__enter__()
def __exit__(self, exc, value, tb):
os.chdir(self._pwd)
return self._tmpdir.__exit__(exc, value, tb)
def prepare_venv(packagename):
tempdir = TemporaryDirectory()
atexit.register(tempdir.cleanup)
tempdir = tempdir.__enter__()
call(['python3', '-m', 'venv', tempdir])
venv_pip = [tempdir + '/bin/python', '-m', 'pip']
call(venv_pip + ['install', packagename])
return venv_pip, tempdir
class DefaultFileDataset(AbstractContextManager):
def __enter__(self):
self.tempdir = TemporaryDirectory()
self.tempdir_path = self.tempdir.__enter__()
file_path = Path(self.tempdir_path) / "file.json"
with open(file_path, "w") as fp:
for item_id in range(10):
line = f'"item_id": {item_id}, "start": "2021-01-01 00:00:00", "target": [1.0, 2.0, 3.0, 4.0, 5.0]'
fp.write("{" + line + "}\n")
return FileDataset(self.tempdir_path, "H")
def __exit__(self, *args, **kwargs):
self.tempdir.__exit__(*args, **kwargs)
class ArFileCtx:
def __init__(self, ar_name: str) -> None:
self.ar_name = os.path.abspath(ar_name)
self._tmpdir = TemporaryDirectory()
def __enter__(self) -> str:
self._pwd = os.getcwd()
rc = self._tmpdir.__enter__()
subprocess.check_call(['ar', 'x', self.ar_name])
return rc
def __exit__(self, ex, value, tb) -> None:
os.chdir(self._pwd)
return self._tmpdir.__exit__(ex, value, tb)
class SingleFileWebServer:
def __init__(self, filename):
self._filename = filename
self.content_type = mimetypes.guess_type(filename)[0]
self._pseudoname = ("video%s" %
mimetypes.guess_extension(self.content_type))
def __enter__(self):
self._tempdir = TemporaryDirectory()
self._tempdir.__enter__()
dir_ = self._tempdir.name
symlink_target = os.path.join(dir_, self._pseudoname)
os.symlink(self._filename, symlink_target)
os.chdir(self._tempdir.name)
Handler = http.server.SimpleHTTPRequestHandler
self._httpd = socketserver.TCPServer(("", PORT), Handler)
print("serving at port", PORT)
threading._start_new_thread(self._httpd.serve_forever, ())
self.url = "http://%s:%s/%s" % (get_own_ip(), PORT, self._pseudoname)
print(self.url)
return self
def __exit__(self, *args, **kwargs):
print("Stopping web server...")
self._httpd.shutdown()
print("Removing tempdir...")
self._tempdir.__exit__(*args, **kwargs)
print("Done.")
class TestDirectory(object):
"""
A thin wrapper over TemporaryDirectory, entering it and leaving
via context manager.
"""
def __init__(self):
self.tempdir = TemporaryDirectory()
self.saved_cwd = os.getcwd()
os.chdir(self.tempdir.name)
def __enter__(self):
return self.tempdir.__enter__()
def __exit__(self, *args):
os.chdir(self.saved_cwd)
return self.tempdir.__exit__(*args)
class TestTearer(unittest.TestCase):
def setUp(self):
self.testPkgDirO = TemporaryDirectory(suffix=None,
prefix=None,
dir=None)
self.testPkgDir = Path(self.testPkgDirO.__enter__())
def tearDown(self):
self.testPkgDirO.__exit__(None, None, None)
def testTearer(self):
testFileVirtualPath = Path("/usr/share/locale/cmn")
testFilePath = nestPath(self.testPkgDir, testFileVirtualPath)
testFilePath.parent.mkdir(parents=True, exist_ok=True)
testFilePath.write_text("")
#print(list(self.testPkgDir.glob("**/*")))
res = FHSTearer(self.testPkgDir)
self.assertEqual(res['data'], [testFileVirtualPath])
class cd:
def __init__(self, subdirectory=None):
self.cm_temp = TemporaryDirectory(dir=os.getcwd(
)) if subdirectory is None else nullcontext(subdirectory)
def enter(self):
self.__enter__()
def exit(self):
self.__exit__(None, None, None)
def __enter__(self):
self.previous = os.getcwd()
self.current = self.cm_temp.__enter__()
os.chdir(self.current)
return self
def __exit__(self, type, value, tb):
# os.chdir has to be first. Otherwise we cannot remove a temporary file
# with temporary directory context maanger
os.chdir(self.previous)
self.cm_temp.__exit__(type, value, tb)
class TemporaryNiceFilenameManager(object):
def __init__(self, db, pub, sha1, max_title_len=60, ext='pdf'):
self.pub = pub
self.sha1 = sha1
self.src = libpath(self.sha1, ext)
fas = db.get_pub_fas(pub.id) or 'No-Author'
year = pub.year or 'NoYr'
title = pub.title or 'Untitled'
if len(title) > max_title_len:
words = title.split()
tlen = 0
for i, word in enumerate(words):
tlen += len(word) + 1
if tlen > max_title_len:
break
if i == 0:
# Seriously? First word of the title is too long
title = title[:max_title_len - 4].strip() + ' ...'
else:
title = ' '.join(words[:i]) + ' ...'
self.nicename = '%s %s - %s.%s' % (fas, year, title, ext)
def __enter__(self):
from tempfile import TemporaryDirectory
self.tempdir_obj = TemporaryDirectory(dir=bibpath(), prefix='nicefn')
tempdir_path = self.tempdir_obj.__enter__()
dest = os.path.join(tempdir_path, self.nicename)
os.link(self.src, dest)
return dest
def __exit__(self, etype, evalue, etb):
return self.tempdir_obj.__exit__(etype, evalue, etb)
class cd:
def __init__(self,subdirectory=None):
self.temp_dir = TemporaryDirectory(dir=os.getcwd()) if subdirectory is None else nullcontext(subdirectory)
def __enter__(self):
self.current = self.temp_dir.__enter__()
self.previous = os.getcwd()
os.chdir(self.current)
return self
def enter(self):
self.__enter__()
def __exit__(self,*args):
os.chdir(self.previous)
self.temp_dir.__exit__(None,None,None)
def exit(self):
self.__exit__(None,None,None)
class InTemporaryDirectory:
"""Create, return, and change directory to a temporary directory
Examples
--------
>>> import os
>>> my_cwd = os.getcwd()
>>> with InTemporaryDirectory() as tmpdir:
... _ = open('test.txt', 'wt').write('some text')
... assert os.path.isfile('test.txt')
... assert os.path.isfile(os.path.join(tmpdir, 'test.txt'))
>>> os.path.exists(tmpdir)
False
>>> os.getcwd() == my_cwd
True
"""
def __init__(self) -> None:
self._tmpdir = TemporaryDirectory()
@property
def name(self) -> str:
return self._tmpdir.name
def __enter__(self) -> str:
self._pwd = os.getcwd()
os.chdir(self._tmpdir.name)
return self._tmpdir.__enter__()
def __exit__(
self,
exc: Optional[Type[BaseException]],
value: Optional[BaseException],
tb: Optional[TracebackType],
) -> None:
os.chdir(self._pwd)
return self._tmpdir.__exit__(exc, value, tb)
class TestSuite:
""" Provides handy test utilities.
self.imagination is the root object of a running imagination process.
It is populated by self.start() and reset to None after self.quit().
self.temp is a temporary directory.
"""
temp: Path
def __init__(self):
self.imagination = None
self.tempdir_object = None
self.temp = None
os.environ["LC_ALL"] = "C"
def __enter__(self):
self.tempdir_object = TemporaryDirectory()
obj = self.tempdir_object.__enter__()
self.temp = Path(obj)
self.temp.resolve()
return self
def __exit__(self, *args, **kwargs):
if self.tempdir_object:
self.tempdir_object.__exit__(*args, **kwargs)
self.temp = None
self.tempdir_object = None
def add_slide(self, filename: Path):
""" Add a new slide """
self.menu("Slideshow", "Import pictures")
self.open_file(filename)
def add_empty_slide(self):
""" Add a new black slide """
n = self.n_slides()
self.menu("Slide", "Add empty slide")
dialog = self.imagination.childNamed("Create empty slide")
dialog.button("OK").click()
assert n + 1 == self.n_slides(), "created no slide!"
def _save(self):
""" Click on save """
self.imagination.child(roleName="tool bar").child(
description="Save the slideshow").button("").click()
def save(self):
""" Save, and assert that no save as file chooser pops up """
self._save()
try:
self.imagination.child(roleName="file chooser", retry=False)
except SearchError:
return
raise (RuntimeError("Saved but a file chooser popped up"))
def save_as(self, filename: Path):
""" Save as"""
d = filename.parent
d.resolve()
assert filename.suffix == ".img"
path = str(d / filename.stem)
self.menu("Slideshow", "Save As")
filechooser = self.imagination.child(roleName="file chooser")
button = filechooser.childNamed("Save")
type(str(path))
button.click()
def open_file(self, filename: Path):
""" When a file chooser is open, open the following path """
filename.resolve()
filechooser = self.imagination.child(roleName="file chooser")
whatever = filechooser.child(description="Open your personal folder")
whatever.keyCombo("<Control>L")
text = filechooser.child(roleName="text")
assert text.visible
text.click()
text.typeText(str(filename))
button = filechooser.childNamed("Open")
button.click()
if not button.dead:
# need to lose the focus first, but not always...
button.click()
assert button.dead
def exif_rotate(self, filename: Path, rotation: int,
flip: bool) -> Optional[Path]:
""" Returns the filename of the same file but with the exif tag refering to the
same rotation"""
new = self.temp / (str(nonce((filename, rotation, flip))) + ".jpg")
rotation = (rotation % 360 + 360) % 360
if rotation == 0:
tag = 2 if flip else 1
elif rotation == 90:
tag = 8
if flip:
return None
elif rotation == 180:
tag = 4 if flip else 3
elif rotation == 270:
tag = 6
if flip:
return None
else:
raise ValueError(
f"Cannot rotate by a non multiple of 90 amount {rotation}")
shutil.copy(filename, new)
subprocess.run(["exiftool", f"-Orientation={tag}", "-n", str(new)])
new.resolve()
return new
def text2img(self, text: str) -> Path:
""" Creates an image with the text in question on it. """
filename = self.temp / (str(nonce(text)) + ".jpg")
subprocess.run(
["convert", "-size", "400x600", "label:" + text,
str(filename)])
filename.resolve()
return filename
def menu(self, root: str, item: str):
""" Clicks on the specified menu and submenu """
menu = self.imagination.menu(root)
menu.click()
sleep(0.1)
menu.menuItem(item).click()
def set_slide_text(self, text: str):
"""Sets the text of the selected slide"""
pane = self.imagination.childNamed("Video")
pane = pane.childNamed("Slide Text")
entry = pane.child(roleName="text")
entry.click()
entry.typeText(text)
def quit(self):
""" Quits.
Fails if there is a "did you mean to quit without saving" dialog.
"""
self.menu("Slideshow", "Quit")
sleep(0.1)
assert self.imagination.dead
self.imagination = None
def open_slideshow(self, filename: Path):
""" Opens the specified slideshow """
self.menu("Slideshow", "Open")
self.open_file(filename)
def start(self, slideshow=None):
""" start imagination, returns its root dogtail object """
app = os.environ.get(
"IMAGINATION",
str(Path(__file__).parent.parent / "src" / "imagination"))
cmd = app
if slideshow:
cmd += " " + str(slideshow)
print("launching", cmd)
run(cmd, timeout=4)
self.imagination = root.application("imagination")
def frame_at(self, video: Path, seconds: float) -> Path:
""" Extracts one frame of the video, whose path is returned
seconds is the time of the frame. The output format is jpg.
"""
out = self.temp / (str(nonce(video, seconds)) + ".jpg")
subprocess.run([
"ffmpeg", "-i",
str(video), "-ss",
str(seconds), "-vframes", "1",
str(out)
])
out.resolve()
return out
def ocr(self, image: Path) -> str:
""" Returns the text on the image in argument.
Assumes that there is only one line of text.
"""
out = self.temp / str(nonce(image))
intermediate = self.temp / (str(nonce(image)) + ".jpg")
subprocess.run([
"convert",
str(image),
"-auto-orient",
"-fuzz",
"1%",
"-trim",
str(intermediate),
])
subprocess.run(["tesseract", "-psm", "7", str(intermediate), str(out)])
with open(f"{out}.txt", "r") as f:
txt = f.read().strip()
print(f"ocr={txt}")
return txt
def n_slides(self) -> int:
""" Returns the current number of slides """
label = self.imagination.child(description="Total number of slides")
n = int(label.name)
print("n_slides =", n)
return n
def export(self) -> Path:
""" Export the slideshow to a file whose path is returned """
out = self.temp / "export.vob"
self.menu("Slideshow", "Export")
self.imagination.child("Export Settings").child(
roleName="text").click()
type(str(out))
self.imagination.child(roleName="dialog").button("OK").click()
pause = self.imagination.child("Exporting the slideshow").child(
"Pause")
while pause.showing:
sleep(0.3)
status = (self.imagination.child("Exporting the slideshow").child(
description="Status of export").text)
self.imagination.child("Exporting the slideshow").button(
"Close").click()
if "failed" in status.lower():
raise ExportFailed(status)
out.resolve()
return out
def choose_slide(self, i: int):
""" Choose slide index """
entry = self.imagination.child(description="Current slide number")
entry.typeText(str(i) + "\n")
assert self.current_slide() == i
def current_slide(self) -> int:
""" Choose slide index """
entry = self.imagination.child(description="Current slide number")
res = int(entry.text.strip())
print("current slide = ", res)
return res
def goto_next_slide(self):
""" Click on the next slide button """
panel = self.imagination.child(
description="Go to the next slide of the slideshow")
panel.button("").click()
def goto_first_slide(self):
""" Click on the next slide button """
panel = self.imagination.child(
description="Go to the first slide of the slideshow")
panel.button("").click()
def set_transition_type(self, category: str, index: int):
""" Set the transition type of the current slide.
Index is the 0-based number of the row in the category submenu
"""
combo = self.imagination.child("Slide Settings").child(
description="Transition type")
combo.click()
menu = combo.menu(category)
menu.click()
rawinput.pressKey("Right")
rawinput.pressKey("Home")
for _ in range(index):
rawinput.pressKey("Down")
rawinput.pressKey("Return")
def assert_should_save(self):
""" Checks that a warning dialog fires when trying to quit without saving """
main = self.imagination.child(roleName="frame")
# never saved or saved once but unsaved changes
assert main.name.startswith("Imagination") or main.name.startswith("*")
self.menu("Slideshow", "Quit")
assert not self.imagination.dead
alert = self.imagination.child(roleName="alert")
alert.button("Cancel").click()
def rotate(self, angle: int):
""" Rotate the current image by $angle degrees in trigonometric direction. $angle must be a multiple of 90. """
assert angle % 90 == 0
angle = (angle % 360 + 360) % 360
angle = angle // 90
if angle == 3:
self.menu("Slide", "Rotate clockwise")
else:
for i in range(angle):
self.menu("Slide", "Rotate counter-clockwise")
def flip(self):
""" Flip the current image horizontally """
self.imagination.child(
description="Flip horizontally the selected slides").child(
roleName="push button").click()
class JobRunner:
JOB_FILE_REGEX = re.compile(r"^job_([0-9a-fA-F]+)\.json")
BUILD_FILE_REGEX = re.compile(r"^out_([0-9a-fA-F]+)\.([a-z]+)\.txt")
def __init__(self,
job_dir=None,
out_dir=None,
timeout=600.0,
verbose=False):
self._proc = None
self._job_dir = TmpDir() if job_dir is None else ntpl(name=job_dir)
self._out_dir = TmpDir() if out_dir is None else ntpl(name=out_dir)
self._timeout = timeout
self._verbose = verbose
# Make sure that the output directories exist, and that only the current
# user has access to these directories
def mk_secure_dir(dir):
if not os.path.isdir(dir):
os.makedirs(dir, exist_ok=True, mode=0o700)
os.chmod(dir, 0o700)
mk_secure_dir(self._job_dir.name)
mk_secure_dir(self._out_dir.name)
def __enter__(self):
if isinstance(self._job_dir, TmpDir):
self._job_dir.__enter__()
if isinstance(self._out_dir, TmpDir):
self._out_dir.__enter__()
return self
def __exit__(self, type, value, traceback):
if isinstance(self._job_dir, TmpDir):
self._job_dir.__exit__(type, value, traceback)
if isinstance(self._out_dir, TmpDir):
self._out_dir.__exit__(type, value, traceback)
if self.is_child_running():
logger.info("Waiting for child process to exit...")
try:
self._proc.terminate()
self._proc.wait(timeout=10.0)
except subprocess.TimeoutExpired:
logger.info("Timeout expired, killing child process...")
self._proc.kill()
self._proc = None
def list_jobs(self):
jobs = []
for job_file in os.listdir(self._job_dir.name):
path = os.path.join(self._job_dir.name, job_file)
if not os.path.isfile(path):
continue
match = self.JOB_FILE_REGEX.match(job_file)
if match:
jobs.append(ntpl(suffix=match.groups(1)[0], path=path))
return sorted(jobs)
def list_builds(self):
builds = []
for build_file in os.listdir(self._out_dir.name):
path = os.path.join(self._out_dir.name, build_file)
if not os.path.isfile(path):
continue
match = self.BUILD_FILE_REGEX.match(build_file)
if match:
builds.append(
ntpl(suffix=match.groups(1)[0],
status=match.groups(1)[1],
path=path))
return builds
def aggregate_jobs_and_builds(self):
jobs, builds, res = self.list_jobs(), self.list_builds(), {}
for job in jobs:
res[job.suffix] = ntpl(status="queued",
id=job.suffix,
time=int(job.suffix, base=16),
output=None)
for build in builds:
res[build.suffix] = ntpl(status=build.status,
id=build.suffix,
time=int(build.suffix, base=16),
output=build.path)
return res
def delete_job_file(self, job_file_path):
# Deletes the job file at the given path
try:
logger.debug("Deleting job file %s", job_file_path)
os.unlink(job_file_path)
except OSError as e:
logger.error(str(e))
def fetch_newest_job(self, jobs=None, skip_obsolete=True):
# Fetch the job files
jobs = self.list_jobs() if jobs is None else jobs
# If so desired, skip obsolete jobs and mark obsolete jobs for deletion
obsolete_jobs, data, res = [], None, None
for i in range(len(jobs)):
try:
with open(jobs[i].path, 'r') as f:
data = json.load(f)
if not res is None:
obsolete_jobs.append(res.path)
res = ntpl(data=data, path=jobs[i].path, suffix=jobs[i].suffix)
if not skip_obsolete:
break
except (json.JSONDecodeError, OSError, KeyError) as e:
logger.error(str(e))
obsolete_jobs.append(jobs[i].path)
# Delete all the obsolete jobs
for path in obsolete_jobs:
self.delete_job_file(path)
return res
def execute_job(self, job):
import select
# Function used to generate the possible output file names
mk_out_file = lambda x: os.path.join(
self._out_dir.name, "out_" + job.suffix + ".{}.txt".format(x))
proc, out_file = None, None
# Create a pipe for stdout/stderr redirection.
p_rd, p_wr = os.pipe()
try:
# Assemble the target environment. Discard all variables but "PATH".
# This will make sure that none of the secrets stored in environment
# variables are visible to the launched subprocess.
environ = {}
if "PATH" in os.environ:
environ["PATH"] = os.environ["PATH"]
# Open the target file, as well as a pipe to which we redirect
# stdout and stderr. Then open a process connected to that file.
output, out_file = [bytes()], mk_out_file("running")
logger.debug("Executing subprocess [%s], writing to \"%s\"",
", ".join(job.data["args"]), out_file)
t0, has_timeout = time.time(), False
with open(out_file, "wb") as f, \
subprocess.Popen(job.data["args"],
env=environ,
stdout=p_wr,
stderr=p_wr,
stdin=subprocess.DEVNULL) as proc:
# Helper function used to write to both the output buffer and
# the output file
def append_to_output(buf):
output[0] += buf
f.write(buf)
f.flush()
# Try to read output until both the process is dead and there is
# no more data to read
had_data_last_time = False
while (proc.poll() is None) or had_data_last_time:
had_data_last_time = False
# Time-out after ten minutes
if time.time() - t0 > self._timeout:
proc.kill()
has_timeout = True
append_to_output(b"\n\nTimeout.\n")
break
# Wait for the output pipe to become readable. Check whether the
# subprocess still exists at least every 0.1 seconds.
rds, _, _ = select.select([p_rd], [], [], 0.1)
if len(rds) == 0:
continue
# Load all available data from the output pipe
buf = os.read(p_rd, 8192)
if len(buf) > 0:
append_to_output(buf)
had_data_last_time = True
if not has_timeout:
append_to_output(
"\n\nExecution took {:0.1f}s, exit code {}\n".format(
time.time() - t0, proc.returncode).encode("utf-8"))
return proc.returncode, output[0]
except KeyError as e:
logger.error("Invalid job descriptor: " + str(e))
except OSError as e:
logger.error("Error while executing the task: " + str(e))
finally:
# Make sure that the pipe is closed
os.close(p_rd)
os.close(p_wr)
# Make sure that the output file is either deleted, or, based on
# the process output, renamed to a file name indicating either
# success or failure
if (not out_file is None) and os.path.isfile(out_file):
if proc is None:
logger.debug("Failure while executing the subprocess.")
os.unlink(out_file)
elif (has_timeout) or (proc.poll() is None):
logger.debug("Timeout while executing the subprocess.")
os.rename(out_file, mk_out_file("timeout"))
elif proc.returncode == 0:
logger.debug("Subprocess exited successfully")
os.rename(out_file, mk_out_file("success"))
else:
logger.debug("Subprocess exited with an error code")
os.rename(out_file, mk_out_file("failure"))
def queue_job(self,
args,
commit_id="",
repository_name="",
url="",
email_to=[]):
# Generate the output file
job_file_path = os.path.join(self._job_dir.name,
"job_{:020x}.json".format(time.time_ns()))
# Create a temporary file in the job directory and write the job
# descriptor to that file. Then, in a last (atomic) step rename that
# file to a valid job file that can be picked up by the job runner.
can_delete_tmp_file, f = True, None
try:
# Write to the temporary job file
with TmpFile(mode="w", dir=self._job_dir.name, delete=False) as f:
json.dump(
{
"args": args,
"commit_id": commit_id,
"repository_name": repository_name,
"url": url,
"email_to": email_to
}, f)
# Once writing has finished, expose the job file at the final
# location.
os.rename(f.name, job_file_path)
can_delete_tmp_file = False
except OSError as e:
logger.error(e)
finally:
# If something goes wrong, delete the created temporary file
if (not f is None) and can_delete_tmp_file:
os.unlink(f.name)
def is_child_running(self):
return (not self._proc is None) and (self._proc.poll() is None)
def check_queue_and_spawn(self):
if (len(self.list_jobs()) > 0) and not self.is_child_running():
args = list(
filter(bool, [
sys.executable, __file__, "child", self._job_dir.name,
self._out_dir.name, "--verbose" if self._verbose else None,
"--timeout",
str(self._timeout)
]))
logger.debug("Spawning child process: [%s]", ", ".join(args))
self._proc = subprocess.Popen(args, stdin=subprocess.DEVNULL)
