def perform_needle_comparison(self, seq_a, seq_b): protDB = db_handling.ProteinDatabase() tempfile = MemoryTempfile(preferred_paths=['/mnt/tmp']) f1 = tempfile.NamedTemporaryFile(delete=False) f1.write(seq_a[1].encode('utf-8')) f1.close() f2 = tempfile.NamedTemporaryFile(delete=False) f2.write(seq_b[1].encode('utf-8')) f2.close() f3 = tempfile.NamedTemporaryFile(delete=False) subprocess.run([ 'needle --asequence ' + f1.name + ' --bsequence ' + f2.name + ' --gapopen 10.0 --gapextend 0.5 --stdout ' + f3.name + ' < enter_param > /dev/null 2>&1' ], stdout=subprocess.PIPE, shell=True) lines = [line.decode('utf-8') for line in f3.readlines()] f3.close() values = self.get_values_from_needle_align(lines) values.insert(0, seq_a[0]) values.insert(0, seq_b[0]) if values[2] == 0 and values[3] == 0 and values[4] == 0 and values[ 5] == 0 and values[6] == 0.0: os.unlink(f1.name) os.unlink(f2.name) os.unlink(f3.name) self.perform_needle_comparison(seq_a, seq_b) return protDB.insert_needle_alignment(values) os.unlink(f1.name) os.unlink(f2.name) os.unlink(f3.name)
def example1(): from memory_tempfile import MemoryTempfile tempfile = MemoryTempfile() with tempfile.TemporaryDirectory() as td: # work as usual pass
def example1(): from memory_tempfile import MemoryTempfile tempfile = MemoryTempfile() print(tempfile.fallback) with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as td: print(td.name) pass
def __init__(self, scratch_dev=None, fallback_dev="/tmp/", addl_fs_types=None): """Create a list of writable tmp devices and space allocations on them.""" self.fallback_dev = fallback_dev self.scratch_dev = scratch_dev fs_types = ["tmpfs", "shm"] if addl_fs_types is not None: fs_types += addl_fs_types self.memdev_list = [ append_slash(dev) for dev in MemoryTempfile( filesystem_types=fs_types).get_usable_mem_tempdir_paths() if is_writable(dev) ] logger.debug(f"Memory devices are {self.memdev_list}") if self.scratch_dev is not None: if not is_writable(self.scratch_dev): logger.warn( f"Scratch device {self.scratch_dev} is not writable.") self.scratch_dev = None else: logger.debug(f"Scratch device is {self.scratch_dev}") if not is_writable(self.fallback_dev): logger.error( f"fallback working device {self.fallback_dev} is not writable!" ) sys.exit(1) self.allocatable_devices = self.memdev_list.copy() if self.scratch_dev is not None: self.allocatable_devices.append(self.scratch_dev) self.allocations = {k: 0 for k in self.allocatable_devices} self.min_free = {k: free_mb(k) for k in self.allocatable_devices}
def __init(self, space_needed_mb=None): """Select the fastest device with enough memory.""" self.space_needed = space_needed_mb self.dev = fallback_dir if self.space_needed is not None: if space_needed_mb < FREE_MEM_MB: memdev_list = MemoryTempfile( filesystem_types=["tmpfs", "shm" ]).get_usable_mem_tempdir_paths() for memdev in memdev_list: self.initial_space_free = free_mb(memdev) if self.initial_space_free >= space_needed_mb: self.dev = memdev break if self.dev == fallback_dir and SCRATCH_DEV is not None: self.initial_space_free = free_mb(SCRATCH_DEV) if self.initial_space_free >= space_needed_mb: self.dev = SCRATCH_DEV if self.dev == fallback_dir: self.initial_space_free = free_mb(self.dev)
def example2(): # We now do not want to use /dev/shm or /run/shm and no ramfs paths # If /run/user/{uid} is available, we prefer it to /tmp # And we want to try /var/run as a last resort # If all fails, fallback to platform's tmp dir from memory_tempfile import MemoryTempfile # By the way, all paths with string {uid} will have it replaced with the user id tempfile = MemoryTempfile(preferred_paths=['/run/user/{uid}'], remove_paths=['/dev/shm', '/run/shm'], additional_paths=['/var/run'], filesystem_types=['tmpfs'], fallback=True) if tempfile.found_mem_tempdir(): print('We could use any of the followig paths: {}'.format(tempfile.get_usable_mem_tempdir_paths())) print('And we are using now: {}'.format(tempfile.gettempdir())) with tempfile.NamedTemporaryFile() as ntf: # use it as usual... pass
#! /usr/bin/python3 # -*- coding: utf-8 -*- import dill import gzip import os from memory_tempfile import MemoryTempfile tempfile = MemoryTempfile() PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/") + "/" f = tempfile.NamedTemporaryFile(prefix="Test", suffix=".txt") TEMP_DIR = f.name[:f.name.rfind("/")] + "/" TEMP_DIR_OBJS = TEMP_DIR + f"python_objects/" if not os.path.exists(TEMP_DIR_OBJS): os.makedirs(TEMP_DIR_OBJS) def save_dict_object(d, file_name_prefix): with gzip.open(TEMP_DIR_OBJS + file_name_prefix + '.pkl.gz', 'wb') as f: dill.dump(d, f) def load_dict_object(file_name_prefix): with gzip.open(TEMP_DIR_OBJS + file_name_prefix + '.pkl.gz', 'rb') as f: d = dill.load(f) return d
import numpy as np import pandas as pd import multiprocessing as mp from copy import deepcopy, copy from dotmap import DotMap from functools import reduce from memory_tempfile import MemoryTempfile from shutil import copyfile from pprint import pprint from typing import List, Set, Tuple, Dict, Union, Any from PIL import Image PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) HOME_DIR = os.path.expanduser("~") TEMP_DIR = MemoryTempfile().gettempdir() import importlib.util as imp_util # TODO: change the optaining of the git root folder from a config file first! spec = imp_util.spec_from_file_location( "utils", os.path.join(HOME_DIR, "git/python_programs/utils.py")) utils = imp_util.module_from_spec(spec) spec.loader.exec_module(utils) mkdirs = utils.mkdirs spec = imp_util.spec_from_file_location( "utils_multiprocessing_manager", os.path.join(HOME_DIR, "git/python_programs/utils_multiprocessing_manager.py"))
import re import string import sys import time import numpy as np import pandas as pd import scipy.stats as st # import scipy as sp from PIL import Image, ImageDraw, ImageFont import matplotlib.pyplot as plt from memory_tempfile import MemoryTempfile tempfile = MemoryTempfile() PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/") + "/" HOME_DIR = os.path.expanduser("~") TEMP_DIR = tempfile.gettempdir() + "/" def func_timer(f, args): start_time = time.time() r = f(*args) end_time = time.time() return end_time - start_time, r if __name__ == '__main__':
# -*- coding: utf-8 -*- import dill import gzip import os import sys import string import shutil from typing import List, Dict, Set, Mapping, Any, Tuple # import tempfile from memory_tempfile import MemoryTempfile tempfile: MemoryTempfile = MemoryTempfile() from collections import defaultdict from copy import deepcopy from dotmap import DotMap from operator import itemgetter from pprint import pprint from os.path import expanduser import itertools import multiprocessing as mp PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\",
import dill import gzip import os import sys # import tempfile from memory_tempfile import MemoryTempfile tempfile = MemoryTempfile() TEMP_ROOT_DIR_PATH = tempfile.gettempdir() TEMP_FOLDER_PATH = os.path.join(TEMP_ROOT_DIR_PATH, 'python_objs/') if not os.path.exists(TEMP_FOLDER_PATH): os.makedirs(TEMP_FOLDER_PATH) def save_object(obj_name, obj): FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name) with gzip.open(FILE_PATH_ABS, 'wb') as f: dill.dump(obj, f) def load_object(obj_name): FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name) with gzip.open(FILE_PATH_ABS, 'rb') as f: obj = dill.load(f) return obj def do_object_exist(obj_name): FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name) return os.path.exists(FILE_PATH_ABS)
from dotmap import DotMap from functools import reduce from pathlib import Path from memory_tempfile import MemoryTempfile from shutil import copyfile from collections import defaultdict from pprint import pprint import matplotlib.pyplot as plt sys.path.append('..') from utils import mkdirs PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))+"/" HOME_DIR = os.path.expanduser("~")+"/" TEMP_DIR = MemoryTempfile().gettempdir()+"/" OBJS_DIR_PATH = PATH_ROOT_DIR+'objs/' mkdirs(OBJS_DIR_PATH) def main(d_env: Dict[str, Any]) -> None: def load_obj_in_d_env(obj_name: str, func: Callable[[Dict[str, Any]], Any], d_env: Dict[str, Any]) -> None: file_path_obj = OBJS_DIR_PATH + obj_name + '.pkl.gz' if not os.path.exists(file_path_obj): print("Creating '{}' object.".format(obj_name)) obj = func(d_env) with gzip.open(file_path_obj, 'wb') as f: dill.dump(obj, f)
except ValueError: SPINNER_UPDATE_PERIOD = 5.0 else: SPINNER_UPDATE_PERIOD = 1.0 # Fast scratch disk (e.g., SSD or /dev/shm), if other than /tmp if "SCRATCH_DEV" in os.environ and is_writable(os.environ["SCRATCH_DEV"]): SCRATCH_DEV = os.environ["SCRATCH_DEV"] else: SCRATCH_DEV = "/tmp" # Build disk for installer, needs to allow exe bit set if "BUILD_DEV" in os.environ and is_writable(os.environ["BUILD_DEV"]): BUILD_DEV = os.environ["BUILD_DEV"] elif sys.platform == "linux": try: BUILD_DEV = MemoryTempfile( preferred_paths=["/run/user/{uid}"], fallback=True).get_usable_mem_tempdir_paths()[0] except AttributeError: BUILD_DEV = "/tmp" else: BUILD_DEV = "/tmp" def enforce_canonical_dtypes(frame): """Enforce that dtypes of columns meet expectations.""" for col in frame.columns: if col.startswith("tmp."): continue column_type = frame[col].dtype should_be_type = DEFAULT_DTYPE if col in NONDEFAULT_DTYPES:
from git import Repo from git.exc import GitCommandError from memory_tempfile import MemoryTempfile from see import Hook from oswatcher.model import GraphInode, InodeType, OSType from oswatcher.utils import get_hard_drive_path if typing.TYPE_CHECKING: from guestfs import GuestFS from hooks.security import ELFChecksec from hooks.static_analyzer import PEChecksec STATS = Counter() try: TEMPFILE = MemoryTempfile(fallback=False) except RuntimeError: logging.warning( "Memory-based filesystem not available for temporary files. Fallback to disk-based" ) TEMPFILE = MemoryTempfile(fallback=True) class Inode: def __init__(self, gfs, node): self._logger = logging.getLogger(self.__class__.__name__) self._gfs = gfs self._tmp_local_file = None # public attributes self.path = node self.name = self.path.name
# We now do not want to use /dev/shm or /run/shm and no ramfs paths # If /run/user/{uid} is available, we prefer it to /tmp # And we want to try /var/run as a last resort # If all fails, fallback to platform's tmp dir from memory_tempfile import MemoryTempfile import memory_tempfile # By the way, all paths with string {uid} will have it replaced with the user id tempfile = MemoryTempfile(fallback=False) if tempfile.found_mem_tempdir(): print('We could use any of the followig paths: {}'.format( tempfile.get_usable_mem_tempdir_paths())) print('And we are using now: {}'.format(tempfile.gettempdir())) with tempfile.NamedTemporaryFile() as ntf: # use it as usual... pass