class LocationTests: def setUp(self): self.locn = TemporaryDirectory() self.locn.__enter__() def tearDown(self): self.locn.__exit__(None, None, None)
class DefaultFileDataset(AbstractContextManager): def __enter__(self): self.tempdir = TemporaryDirectory() self.tempdir_path = self.tempdir.__enter__() file_path = Path(self.tempdir_path) / "file.json" with open(file_path, "w") as fp: for item_id in range(10): line = f'"item_id": {item_id}, "start": "2021-01-01 00:00:00", "target": [1.0, 2.0, 3.0, 4.0, 5.0]' fp.write("{" + line + "}\n") return FileDataset(self.tempdir_path, "H") def __exit__(self, *args, **kwargs): self.tempdir.__exit__(*args, **kwargs)
class ArchiveCopy(ar.Archive): def __init__(self, path): self.__dir = TemporaryDirectory() dest = '%s/%s' % (self.__dir.name, os.path.basename(path)) shutil.copyfile(path, dest) super().__init__(dest) def __enter__(self): self.__dir.__enter__() return super().__enter__() def __exit__(self, type, value, traceback): super().__exit__(type, value, traceback) self.__dir.__exit__(type, value, traceback)
class InTemporaryDirectory: ''' Create, return, and change directory to a temporary directory Examples -------- >>> import os >>> my_cwd = os.getcwd() >>> with InTemporaryDirectory() as tmpdir: ... _ = open('test.txt', 'wt').write('some text') ... assert os.path.isfile('test.txt') ... assert os.path.isfile(os.path.join(tmpdir, 'test.txt')) >>> os.path.exists(tmpdir) False >>> os.getcwd() == my_cwd True ''' def __init__(self): self._tmpdir = TemporaryDirectory() @property def name(self): return self._tmpdir.name def __enter__(self): self._pwd = os.getcwd() os.chdir(self._tmpdir.name) return self._tmpdir.__enter__() def __exit__(self, exc, value, tb): os.chdir(self._pwd) return self._tmpdir.__exit__(exc, value, tb)
class TestTearer(unittest.TestCase): def setUp(self): self.testPkgDirO = TemporaryDirectory(suffix=None, prefix=None, dir=None) self.testPkgDir = Path(self.testPkgDirO.__enter__()) def tearDown(self): self.testPkgDirO.__exit__(None, None, None) def testTearer(self): testFileVirtualPath = Path("/usr/share/locale/cmn") testFilePath = nestPath(self.testPkgDir, testFileVirtualPath) testFilePath.parent.mkdir(parents=True, exist_ok=True) testFilePath.write_text("") #print(list(self.testPkgDir.glob("**/*"))) res = FHSTearer(self.testPkgDir) self.assertEqual(res['data'], [testFileVirtualPath])
class cd: def __init__(self, subdirectory=None): self.cm_temp = TemporaryDirectory(dir=os.getcwd( )) if subdirectory is None else nullcontext(subdirectory) def enter(self): self.__enter__() def exit(self): self.__exit__(None, None, None) def __enter__(self): self.previous = os.getcwd() self.current = self.cm_temp.__enter__() os.chdir(self.current) return self def __exit__(self, type, value, tb): # os.chdir has to be first. Otherwise we cannot remove a temporary file # with temporary directory context maanger os.chdir(self.previous) self.cm_temp.__exit__(type, value, tb)
class cd: def __init__(self,subdirectory=None): self.temp_dir = TemporaryDirectory(dir=os.getcwd()) if subdirectory is None else nullcontext(subdirectory) def __enter__(self): self.current = self.temp_dir.__enter__() self.previous = os.getcwd() os.chdir(self.current) return self def enter(self): self.__enter__() def __exit__(self,*args): os.chdir(self.previous) self.temp_dir.__exit__(None,None,None) def exit(self): self.__exit__(None,None,None)
class ArFileCtx: def __init__(self, ar_name: str) -> None: self.ar_name = os.path.abspath(ar_name) self._tmpdir = TemporaryDirectory() def __enter__(self) -> str: self._pwd = os.getcwd() rc = self._tmpdir.__enter__() subprocess.check_call(['ar', 'x', self.ar_name]) return rc def __exit__(self, ex, value, tb) -> None: os.chdir(self._pwd) return self._tmpdir.__exit__(ex, value, tb)
class SingleFileWebServer: def __init__(self, filename): self._filename = filename self.content_type = mimetypes.guess_type(filename)[0] self._pseudoname = ("video%s" % mimetypes.guess_extension(self.content_type)) def __enter__(self): self._tempdir = TemporaryDirectory() self._tempdir.__enter__() dir_ = self._tempdir.name symlink_target = os.path.join(dir_, self._pseudoname) os.symlink(self._filename, symlink_target) os.chdir(self._tempdir.name) Handler = http.server.SimpleHTTPRequestHandler self._httpd = socketserver.TCPServer(("", PORT), Handler) print("serving at port", PORT) threading._start_new_thread(self._httpd.serve_forever, ()) self.url = "http://%s:%s/%s" % (get_own_ip(), PORT, self._pseudoname) print(self.url) return self def __exit__(self, *args, **kwargs): print("Stopping web server...") self._httpd.shutdown() print("Removing tempdir...") self._tempdir.__exit__(*args, **kwargs) print("Done.")
class TestDirectory(object): """ A thin wrapper over TemporaryDirectory, entering it and leaving via context manager. """ def __init__(self): self.tempdir = TemporaryDirectory() self.saved_cwd = os.getcwd() os.chdir(self.tempdir.name) def __enter__(self): return self.tempdir.__enter__() def __exit__(self, *args): os.chdir(self.saved_cwd) return self.tempdir.__exit__(*args)
class TemporaryNiceFilenameManager(object): def __init__(self, db, pub, sha1, max_title_len=60, ext='pdf'): self.pub = pub self.sha1 = sha1 self.src = libpath(self.sha1, ext) fas = db.get_pub_fas(pub.id) or 'No-Author' year = pub.year or 'NoYr' title = pub.title or 'Untitled' if len(title) > max_title_len: words = title.split() tlen = 0 for i, word in enumerate(words): tlen += len(word) + 1 if tlen > max_title_len: break if i == 0: # Seriously? First word of the title is too long title = title[:max_title_len - 4].strip() + ' ...' else: title = ' '.join(words[:i]) + ' ...' self.nicename = '%s %s - %s.%s' % (fas, year, title, ext) def __enter__(self): from tempfile import TemporaryDirectory self.tempdir_obj = TemporaryDirectory(dir=bibpath(), prefix='nicefn') tempdir_path = self.tempdir_obj.__enter__() dest = os.path.join(tempdir_path, self.nicename) os.link(self.src, dest) return dest def __exit__(self, etype, evalue, etb): return self.tempdir_obj.__exit__(etype, evalue, etb)
class InTemporaryDirectory: """Create, return, and change directory to a temporary directory Examples -------- >>> import os >>> my_cwd = os.getcwd() >>> with InTemporaryDirectory() as tmpdir: ... _ = open('test.txt', 'wt').write('some text') ... assert os.path.isfile('test.txt') ... assert os.path.isfile(os.path.join(tmpdir, 'test.txt')) >>> os.path.exists(tmpdir) False >>> os.getcwd() == my_cwd True """ def __init__(self) -> None: self._tmpdir = TemporaryDirectory() @property def name(self) -> str: return self._tmpdir.name def __enter__(self) -> str: self._pwd = os.getcwd() os.chdir(self._tmpdir.name) return self._tmpdir.__enter__() def __exit__( self, exc: Optional[Type[BaseException]], value: Optional[BaseException], tb: Optional[TracebackType], ) -> None: os.chdir(self._pwd) return self._tmpdir.__exit__(exc, value, tb)
class TestSuite: """ Provides handy test utilities. self.imagination is the root object of a running imagination process. It is populated by self.start() and reset to None after self.quit(). self.temp is a temporary directory. """ temp: Path def __init__(self): self.imagination = None self.tempdir_object = None self.temp = None os.environ["LC_ALL"] = "C" def __enter__(self): self.tempdir_object = TemporaryDirectory() obj = self.tempdir_object.__enter__() self.temp = Path(obj) self.temp.resolve() return self def __exit__(self, *args, **kwargs): if self.tempdir_object: self.tempdir_object.__exit__(*args, **kwargs) self.temp = None self.tempdir_object = None def add_slide(self, filename: Path): """ Add a new slide """ self.menu("Slideshow", "Import pictures") self.open_file(filename) def add_empty_slide(self): """ Add a new black slide """ n = self.n_slides() self.menu("Slide", "Add empty slide") dialog = self.imagination.childNamed("Create empty slide") dialog.button("OK").click() assert n + 1 == self.n_slides(), "created no slide!" def _save(self): """ Click on save """ self.imagination.child(roleName="tool bar").child( description="Save the slideshow").button("").click() def save(self): """ Save, and assert that no save as file chooser pops up """ self._save() try: self.imagination.child(roleName="file chooser", retry=False) except SearchError: return raise (RuntimeError("Saved but a file chooser popped up")) def save_as(self, filename: Path): """ Save as""" d = filename.parent d.resolve() assert filename.suffix == ".img" path = str(d / filename.stem) self.menu("Slideshow", "Save As") filechooser = self.imagination.child(roleName="file chooser") button = filechooser.childNamed("Save") type(str(path)) button.click() def open_file(self, filename: Path): """ When a file chooser is open, open the following path """ filename.resolve() filechooser = self.imagination.child(roleName="file chooser") whatever = filechooser.child(description="Open your personal folder") whatever.keyCombo("<Control>L") text = filechooser.child(roleName="text") assert text.visible text.click() text.typeText(str(filename)) button = filechooser.childNamed("Open") button.click() if not button.dead: # need to lose the focus first, but not always... button.click() assert button.dead def exif_rotate(self, filename: Path, rotation: int, flip: bool) -> Optional[Path]: """ Returns the filename of the same file but with the exif tag refering to the same rotation""" new = self.temp / (str(nonce((filename, rotation, flip))) + ".jpg") rotation = (rotation % 360 + 360) % 360 if rotation == 0: tag = 2 if flip else 1 elif rotation == 90: tag = 8 if flip: return None elif rotation == 180: tag = 4 if flip else 3 elif rotation == 270: tag = 6 if flip: return None else: raise ValueError( f"Cannot rotate by a non multiple of 90 amount {rotation}") shutil.copy(filename, new) subprocess.run(["exiftool", f"-Orientation={tag}", "-n", str(new)]) new.resolve() return new def text2img(self, text: str) -> Path: """ Creates an image with the text in question on it. """ filename = self.temp / (str(nonce(text)) + ".jpg") subprocess.run( ["convert", "-size", "400x600", "label:" + text, str(filename)]) filename.resolve() return filename def menu(self, root: str, item: str): """ Clicks on the specified menu and submenu """ menu = self.imagination.menu(root) menu.click() sleep(0.1) menu.menuItem(item).click() def set_slide_text(self, text: str): """Sets the text of the selected slide""" pane = self.imagination.childNamed("Video") pane = pane.childNamed("Slide Text") entry = pane.child(roleName="text") entry.click() entry.typeText(text) def quit(self): """ Quits. Fails if there is a "did you mean to quit without saving" dialog. """ self.menu("Slideshow", "Quit") sleep(0.1) assert self.imagination.dead self.imagination = None def open_slideshow(self, filename: Path): """ Opens the specified slideshow """ self.menu("Slideshow", "Open") self.open_file(filename) def start(self, slideshow=None): """ start imagination, returns its root dogtail object """ app = os.environ.get( "IMAGINATION", str(Path(__file__).parent.parent / "src" / "imagination")) cmd = app if slideshow: cmd += " " + str(slideshow) print("launching", cmd) run(cmd, timeout=4) self.imagination = root.application("imagination") def frame_at(self, video: Path, seconds: float) -> Path: """ Extracts one frame of the video, whose path is returned seconds is the time of the frame. The output format is jpg. """ out = self.temp / (str(nonce(video, seconds)) + ".jpg") subprocess.run([ "ffmpeg", "-i", str(video), "-ss", str(seconds), "-vframes", "1", str(out) ]) out.resolve() return out def ocr(self, image: Path) -> str: """ Returns the text on the image in argument. Assumes that there is only one line of text. """ out = self.temp / str(nonce(image)) intermediate = self.temp / (str(nonce(image)) + ".jpg") subprocess.run([ "convert", str(image), "-auto-orient", "-fuzz", "1%", "-trim", str(intermediate), ]) subprocess.run(["tesseract", "-psm", "7", str(intermediate), str(out)]) with open(f"{out}.txt", "r") as f: txt = f.read().strip() print(f"ocr={txt}") return txt def n_slides(self) -> int: """ Returns the current number of slides """ label = self.imagination.child(description="Total number of slides") n = int(label.name) print("n_slides =", n) return n def export(self) -> Path: """ Export the slideshow to a file whose path is returned """ out = self.temp / "export.vob" self.menu("Slideshow", "Export") self.imagination.child("Export Settings").child( roleName="text").click() type(str(out)) self.imagination.child(roleName="dialog").button("OK").click() pause = self.imagination.child("Exporting the slideshow").child( "Pause") while pause.showing: sleep(0.3) status = (self.imagination.child("Exporting the slideshow").child( description="Status of export").text) self.imagination.child("Exporting the slideshow").button( "Close").click() if "failed" in status.lower(): raise ExportFailed(status) out.resolve() return out def choose_slide(self, i: int): """ Choose slide index """ entry = self.imagination.child(description="Current slide number") entry.typeText(str(i) + "\n") assert self.current_slide() == i def current_slide(self) -> int: """ Choose slide index """ entry = self.imagination.child(description="Current slide number") res = int(entry.text.strip()) print("current slide = ", res) return res def goto_next_slide(self): """ Click on the next slide button """ panel = self.imagination.child( description="Go to the next slide of the slideshow") panel.button("").click() def goto_first_slide(self): """ Click on the next slide button """ panel = self.imagination.child( description="Go to the first slide of the slideshow") panel.button("").click() def set_transition_type(self, category: str, index: int): """ Set the transition type of the current slide. Index is the 0-based number of the row in the category submenu """ combo = self.imagination.child("Slide Settings").child( description="Transition type") combo.click() menu = combo.menu(category) menu.click() rawinput.pressKey("Right") rawinput.pressKey("Home") for _ in range(index): rawinput.pressKey("Down") rawinput.pressKey("Return") def assert_should_save(self): """ Checks that a warning dialog fires when trying to quit without saving """ main = self.imagination.child(roleName="frame") # never saved or saved once but unsaved changes assert main.name.startswith("Imagination") or main.name.startswith("*") self.menu("Slideshow", "Quit") assert not self.imagination.dead alert = self.imagination.child(roleName="alert") alert.button("Cancel").click() def rotate(self, angle: int): """ Rotate the current image by $angle degrees in trigonometric direction. $angle must be a multiple of 90. """ assert angle % 90 == 0 angle = (angle % 360 + 360) % 360 angle = angle // 90 if angle == 3: self.menu("Slide", "Rotate clockwise") else: for i in range(angle): self.menu("Slide", "Rotate counter-clockwise") def flip(self): """ Flip the current image horizontally """ self.imagination.child( description="Flip horizontally the selected slides").child( roleName="push button").click()
class JobRunner: JOB_FILE_REGEX = re.compile(r"^job_([0-9a-fA-F]+)\.json") BUILD_FILE_REGEX = re.compile(r"^out_([0-9a-fA-F]+)\.([a-z]+)\.txt") def __init__(self, job_dir=None, out_dir=None, timeout=600.0, verbose=False): self._proc = None self._job_dir = TmpDir() if job_dir is None else ntpl(name=job_dir) self._out_dir = TmpDir() if out_dir is None else ntpl(name=out_dir) self._timeout = timeout self._verbose = verbose # Make sure that the output directories exist, and that only the current # user has access to these directories def mk_secure_dir(dir): if not os.path.isdir(dir): os.makedirs(dir, exist_ok=True, mode=0o700) os.chmod(dir, 0o700) mk_secure_dir(self._job_dir.name) mk_secure_dir(self._out_dir.name) def __enter__(self): if isinstance(self._job_dir, TmpDir): self._job_dir.__enter__() if isinstance(self._out_dir, TmpDir): self._out_dir.__enter__() return self def __exit__(self, type, value, traceback): if isinstance(self._job_dir, TmpDir): self._job_dir.__exit__(type, value, traceback) if isinstance(self._out_dir, TmpDir): self._out_dir.__exit__(type, value, traceback) if self.is_child_running(): logger.info("Waiting for child process to exit...") try: self._proc.terminate() self._proc.wait(timeout=10.0) except subprocess.TimeoutExpired: logger.info("Timeout expired, killing child process...") self._proc.kill() self._proc = None def list_jobs(self): jobs = [] for job_file in os.listdir(self._job_dir.name): path = os.path.join(self._job_dir.name, job_file) if not os.path.isfile(path): continue match = self.JOB_FILE_REGEX.match(job_file) if match: jobs.append(ntpl(suffix=match.groups(1)[0], path=path)) return sorted(jobs) def list_builds(self): builds = [] for build_file in os.listdir(self._out_dir.name): path = os.path.join(self._out_dir.name, build_file) if not os.path.isfile(path): continue match = self.BUILD_FILE_REGEX.match(build_file) if match: builds.append( ntpl(suffix=match.groups(1)[0], status=match.groups(1)[1], path=path)) return builds def aggregate_jobs_and_builds(self): jobs, builds, res = self.list_jobs(), self.list_builds(), {} for job in jobs: res[job.suffix] = ntpl(status="queued", id=job.suffix, time=int(job.suffix, base=16), output=None) for build in builds: res[build.suffix] = ntpl(status=build.status, id=build.suffix, time=int(build.suffix, base=16), output=build.path) return res def delete_job_file(self, job_file_path): # Deletes the job file at the given path try: logger.debug("Deleting job file %s", job_file_path) os.unlink(job_file_path) except OSError as e: logger.error(str(e)) def fetch_newest_job(self, jobs=None, skip_obsolete=True): # Fetch the job files jobs = self.list_jobs() if jobs is None else jobs # If so desired, skip obsolete jobs and mark obsolete jobs for deletion obsolete_jobs, data, res = [], None, None for i in range(len(jobs)): try: with open(jobs[i].path, 'r') as f: data = json.load(f) if not res is None: obsolete_jobs.append(res.path) res = ntpl(data=data, path=jobs[i].path, suffix=jobs[i].suffix) if not skip_obsolete: break except (json.JSONDecodeError, OSError, KeyError) as e: logger.error(str(e)) obsolete_jobs.append(jobs[i].path) # Delete all the obsolete jobs for path in obsolete_jobs: self.delete_job_file(path) return res def execute_job(self, job): import select # Function used to generate the possible output file names mk_out_file = lambda x: os.path.join( self._out_dir.name, "out_" + job.suffix + ".{}.txt".format(x)) proc, out_file = None, None # Create a pipe for stdout/stderr redirection. p_rd, p_wr = os.pipe() try: # Assemble the target environment. Discard all variables but "PATH". # This will make sure that none of the secrets stored in environment # variables are visible to the launched subprocess. environ = {} if "PATH" in os.environ: environ["PATH"] = os.environ["PATH"] # Open the target file, as well as a pipe to which we redirect # stdout and stderr. Then open a process connected to that file. output, out_file = [bytes()], mk_out_file("running") logger.debug("Executing subprocess [%s], writing to \"%s\"", ", ".join(job.data["args"]), out_file) t0, has_timeout = time.time(), False with open(out_file, "wb") as f, \ subprocess.Popen(job.data["args"], env=environ, stdout=p_wr, stderr=p_wr, stdin=subprocess.DEVNULL) as proc: # Helper function used to write to both the output buffer and # the output file def append_to_output(buf): output[0] += buf f.write(buf) f.flush() # Try to read output until both the process is dead and there is # no more data to read had_data_last_time = False while (proc.poll() is None) or had_data_last_time: had_data_last_time = False # Time-out after ten minutes if time.time() - t0 > self._timeout: proc.kill() has_timeout = True append_to_output(b"\n\nTimeout.\n") break # Wait for the output pipe to become readable. Check whether the # subprocess still exists at least every 0.1 seconds. rds, _, _ = select.select([p_rd], [], [], 0.1) if len(rds) == 0: continue # Load all available data from the output pipe buf = os.read(p_rd, 8192) if len(buf) > 0: append_to_output(buf) had_data_last_time = True if not has_timeout: append_to_output( "\n\nExecution took {:0.1f}s, exit code {}\n".format( time.time() - t0, proc.returncode).encode("utf-8")) return proc.returncode, output[0] except KeyError as e: logger.error("Invalid job descriptor: " + str(e)) except OSError as e: logger.error("Error while executing the task: " + str(e)) finally: # Make sure that the pipe is closed os.close(p_rd) os.close(p_wr) # Make sure that the output file is either deleted, or, based on # the process output, renamed to a file name indicating either # success or failure if (not out_file is None) and os.path.isfile(out_file): if proc is None: logger.debug("Failure while executing the subprocess.") os.unlink(out_file) elif (has_timeout) or (proc.poll() is None): logger.debug("Timeout while executing the subprocess.") os.rename(out_file, mk_out_file("timeout")) elif proc.returncode == 0: logger.debug("Subprocess exited successfully") os.rename(out_file, mk_out_file("success")) else: logger.debug("Subprocess exited with an error code") os.rename(out_file, mk_out_file("failure")) def queue_job(self, args, commit_id="", repository_name="", url="", email_to=[]): # Generate the output file job_file_path = os.path.join(self._job_dir.name, "job_{:020x}.json".format(time.time_ns())) # Create a temporary file in the job directory and write the job # descriptor to that file. Then, in a last (atomic) step rename that # file to a valid job file that can be picked up by the job runner. can_delete_tmp_file, f = True, None try: # Write to the temporary job file with TmpFile(mode="w", dir=self._job_dir.name, delete=False) as f: json.dump( { "args": args, "commit_id": commit_id, "repository_name": repository_name, "url": url, "email_to": email_to }, f) # Once writing has finished, expose the job file at the final # location. os.rename(f.name, job_file_path) can_delete_tmp_file = False except OSError as e: logger.error(e) finally: # If something goes wrong, delete the created temporary file if (not f is None) and can_delete_tmp_file: os.unlink(f.name) def is_child_running(self): return (not self._proc is None) and (self._proc.poll() is None) def check_queue_and_spawn(self): if (len(self.list_jobs()) > 0) and not self.is_child_running(): args = list( filter(bool, [ sys.executable, __file__, "child", self._job_dir.name, self._out_dir.name, "--verbose" if self._verbose else None, "--timeout", str(self._timeout) ])) logger.debug("Spawning child process: [%s]", ", ".join(args)) self._proc = subprocess.Popen(args, stdin=subprocess.DEVNULL)