def perform_needle_comparison(self, seq_a, seq_b):
protDB = db_handling.ProteinDatabase()
tempfile = MemoryTempfile(preferred_paths=['/mnt/tmp'])
f1 = tempfile.NamedTemporaryFile(delete=False)
f1.write(seq_a[1].encode('utf-8'))
f1.close()
f2 = tempfile.NamedTemporaryFile(delete=False)
f2.write(seq_b[1].encode('utf-8'))
f2.close()
f3 = tempfile.NamedTemporaryFile(delete=False)
subprocess.run([
'needle --asequence ' + f1.name + ' --bsequence ' + f2.name +
' --gapopen 10.0 --gapextend 0.5 --stdout ' + f3.name +
' < enter_param > /dev/null 2>&1'
],
stdout=subprocess.PIPE,
shell=True)
lines = [line.decode('utf-8') for line in f3.readlines()]
f3.close()
values = self.get_values_from_needle_align(lines)
values.insert(0, seq_a[0])
values.insert(0, seq_b[0])
if values[2] == 0 and values[3] == 0 and values[4] == 0 and values[
5] == 0 and values[6] == 0.0:
os.unlink(f1.name)
os.unlink(f2.name)
os.unlink(f3.name)
self.perform_needle_comparison(seq_a, seq_b)
return
protDB.insert_needle_alignment(values)
os.unlink(f1.name)
os.unlink(f2.name)
os.unlink(f3.name)
def example1():
from memory_tempfile import MemoryTempfile
tempfile = MemoryTempfile()
with tempfile.TemporaryDirectory() as td:
# work as usual
pass
def example1():
from memory_tempfile import MemoryTempfile
tempfile = MemoryTempfile()
print(tempfile.fallback)
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as td:
print(td.name)
pass
def __init__(self,
scratch_dev=None,
fallback_dev="/tmp/",
addl_fs_types=None):
"""Create a list of writable tmp devices and space allocations on them."""
self.fallback_dev = fallback_dev
self.scratch_dev = scratch_dev
fs_types = ["tmpfs", "shm"]
if addl_fs_types is not None:
fs_types += addl_fs_types
self.memdev_list = [
append_slash(dev) for dev in MemoryTempfile(
filesystem_types=fs_types).get_usable_mem_tempdir_paths()
if is_writable(dev)
]
logger.debug(f"Memory devices are {self.memdev_list}")
if self.scratch_dev is not None:
if not is_writable(self.scratch_dev):
logger.warn(
f"Scratch device {self.scratch_dev} is not writable.")
self.scratch_dev = None
else:
logger.debug(f"Scratch device is {self.scratch_dev}")
if not is_writable(self.fallback_dev):
logger.error(
f"fallback working device {self.fallback_dev} is not writable!"
)
sys.exit(1)
self.allocatable_devices = self.memdev_list.copy()
if self.scratch_dev is not None:
self.allocatable_devices.append(self.scratch_dev)
self.allocations = {k: 0 for k in self.allocatable_devices}
self.min_free = {k: free_mb(k) for k in self.allocatable_devices}
def __init(self, space_needed_mb=None):
"""Select the fastest device with enough memory."""
self.space_needed = space_needed_mb
self.dev = fallback_dir
if self.space_needed is not None:
if space_needed_mb < FREE_MEM_MB:
memdev_list = MemoryTempfile(
filesystem_types=["tmpfs", "shm"
]).get_usable_mem_tempdir_paths()
for memdev in memdev_list:
self.initial_space_free = free_mb(memdev)
if self.initial_space_free >= space_needed_mb:
self.dev = memdev
break
if self.dev == fallback_dir and SCRATCH_DEV is not None:
self.initial_space_free = free_mb(SCRATCH_DEV)
if self.initial_space_free >= space_needed_mb:
self.dev = SCRATCH_DEV
if self.dev == fallback_dir:
self.initial_space_free = free_mb(self.dev)
def example2():
# We now do not want to use /dev/shm or /run/shm and no ramfs paths
# If /run/user/{uid} is available, we prefer it to /tmp
# And we want to try /var/run as a last resort
# If all fails, fallback to platform's tmp dir
from memory_tempfile import MemoryTempfile
# By the way, all paths with string {uid} will have it replaced with the user id
tempfile = MemoryTempfile(preferred_paths=['/run/user/{uid}'], remove_paths=['/dev/shm', '/run/shm'],
additional_paths=['/var/run'], filesystem_types=['tmpfs'], fallback=True)
if tempfile.found_mem_tempdir():
print('We could use any of the followig paths: {}'.format(tempfile.get_usable_mem_tempdir_paths()))
print('And we are using now: {}'.format(tempfile.gettempdir()))
with tempfile.NamedTemporaryFile() as ntf:
# use it as usual...
pass
#! /usr/bin/python3
# -*- coding: utf-8 -*-
import dill
import gzip
import os
from memory_tempfile import MemoryTempfile
tempfile = MemoryTempfile()
PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\",
"/") + "/"
f = tempfile.NamedTemporaryFile(prefix="Test", suffix=".txt")
TEMP_DIR = f.name[:f.name.rfind("/")] + "/"
TEMP_DIR_OBJS = TEMP_DIR + f"python_objects/"
if not os.path.exists(TEMP_DIR_OBJS):
os.makedirs(TEMP_DIR_OBJS)
def save_dict_object(d, file_name_prefix):
with gzip.open(TEMP_DIR_OBJS + file_name_prefix + '.pkl.gz', 'wb') as f:
dill.dump(d, f)
def load_dict_object(file_name_prefix):
with gzip.open(TEMP_DIR_OBJS + file_name_prefix + '.pkl.gz', 'rb') as f:
d = dill.load(f)
return d
import numpy as np
import pandas as pd
import multiprocessing as mp
from copy import deepcopy, copy
from dotmap import DotMap
from functools import reduce
from memory_tempfile import MemoryTempfile
from shutil import copyfile
from pprint import pprint
from typing import List, Set, Tuple, Dict, Union, Any
from PIL import Image
PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
HOME_DIR = os.path.expanduser("~")
TEMP_DIR = MemoryTempfile().gettempdir()
import importlib.util as imp_util
# TODO: change the optaining of the git root folder from a config file first!
spec = imp_util.spec_from_file_location(
"utils", os.path.join(HOME_DIR, "git/python_programs/utils.py"))
utils = imp_util.module_from_spec(spec)
spec.loader.exec_module(utils)
mkdirs = utils.mkdirs
spec = imp_util.spec_from_file_location(
"utils_multiprocessing_manager",
os.path.join(HOME_DIR,
"git/python_programs/utils_multiprocessing_manager.py"))
import re
import string
import sys
import time
import numpy as np
import pandas as pd
import scipy.stats as st
# import scipy as sp
from PIL import Image, ImageDraw, ImageFont
import matplotlib.pyplot as plt
from memory_tempfile import MemoryTempfile
tempfile = MemoryTempfile()
PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\",
"/") + "/"
HOME_DIR = os.path.expanduser("~")
TEMP_DIR = tempfile.gettempdir() + "/"
def func_timer(f, args):
start_time = time.time()
r = f(*args)
end_time = time.time()
return end_time - start_time, r
if __name__ == '__main__':
# -*- coding: utf-8 -*-
import dill
import gzip
import os
import sys
import string
import shutil
from typing import List, Dict, Set, Mapping, Any, Tuple
# import tempfile
from memory_tempfile import MemoryTempfile
tempfile: MemoryTempfile = MemoryTempfile()
from collections import defaultdict
from copy import deepcopy
from dotmap import DotMap
from operator import itemgetter
from pprint import pprint
from os.path import expanduser
import itertools
import multiprocessing as mp
PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\",
import dill
import gzip
import os
import sys
# import tempfile
from memory_tempfile import MemoryTempfile
tempfile = MemoryTempfile()
TEMP_ROOT_DIR_PATH = tempfile.gettempdir()
TEMP_FOLDER_PATH = os.path.join(TEMP_ROOT_DIR_PATH, 'python_objs/')
if not os.path.exists(TEMP_FOLDER_PATH):
os.makedirs(TEMP_FOLDER_PATH)
def save_object(obj_name, obj):
FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name)
with gzip.open(FILE_PATH_ABS, 'wb') as f:
dill.dump(obj, f)
def load_object(obj_name):
FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name)
with gzip.open(FILE_PATH_ABS, 'rb') as f:
obj = dill.load(f)
return obj
def do_object_exist(obj_name):
FILE_PATH_ABS = TEMP_FOLDER_PATH + '{}.pkl.gz'.format(obj_name)
return os.path.exists(FILE_PATH_ABS)
from dotmap import DotMap
from functools import reduce
from pathlib import Path
from memory_tempfile import MemoryTempfile
from shutil import copyfile
from collections import defaultdict
from pprint import pprint
import matplotlib.pyplot as plt
sys.path.append('..')
from utils import mkdirs
PATH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))+"/"
HOME_DIR = os.path.expanduser("~")+"/"
TEMP_DIR = MemoryTempfile().gettempdir()+"/"
OBJS_DIR_PATH = PATH_ROOT_DIR+'objs/'
mkdirs(OBJS_DIR_PATH)
def main(d_env: Dict[str, Any]) -> None:
def load_obj_in_d_env(obj_name: str, func: Callable[[Dict[str, Any]], Any], d_env: Dict[str, Any]) -> None:
file_path_obj = OBJS_DIR_PATH + obj_name + '.pkl.gz'
if not os.path.exists(file_path_obj):
print("Creating '{}' object.".format(obj_name))
obj = func(d_env)
with gzip.open(file_path_obj, 'wb') as f:
dill.dump(obj, f)
except ValueError:
SPINNER_UPDATE_PERIOD = 5.0
else:
SPINNER_UPDATE_PERIOD = 1.0
# Fast scratch disk (e.g., SSD or /dev/shm), if other than /tmp
if "SCRATCH_DEV" in os.environ and is_writable(os.environ["SCRATCH_DEV"]):
SCRATCH_DEV = os.environ["SCRATCH_DEV"]
else:
SCRATCH_DEV = "/tmp"
# Build disk for installer, needs to allow exe bit set
if "BUILD_DEV" in os.environ and is_writable(os.environ["BUILD_DEV"]):
BUILD_DEV = os.environ["BUILD_DEV"]
elif sys.platform == "linux":
try:
BUILD_DEV = MemoryTempfile(
preferred_paths=["/run/user/{uid}"],
fallback=True).get_usable_mem_tempdir_paths()[0]
except AttributeError:
BUILD_DEV = "/tmp"
else:
BUILD_DEV = "/tmp"
def enforce_canonical_dtypes(frame):
"""Enforce that dtypes of columns meet expectations."""
for col in frame.columns:
if col.startswith("tmp."):
continue
column_type = frame[col].dtype
should_be_type = DEFAULT_DTYPE
if col in NONDEFAULT_DTYPES:
from git import Repo
from git.exc import GitCommandError
from memory_tempfile import MemoryTempfile
from see import Hook
from oswatcher.model import GraphInode, InodeType, OSType
from oswatcher.utils import get_hard_drive_path
if typing.TYPE_CHECKING:
from guestfs import GuestFS
from hooks.security import ELFChecksec
from hooks.static_analyzer import PEChecksec
STATS = Counter()
try:
TEMPFILE = MemoryTempfile(fallback=False)
except RuntimeError:
logging.warning(
"Memory-based filesystem not available for temporary files. Fallback to disk-based"
)
TEMPFILE = MemoryTempfile(fallback=True)
class Inode:
def __init__(self, gfs, node):
self._logger = logging.getLogger(self.__class__.__name__)
self._gfs = gfs
self._tmp_local_file = None
# public attributes
self.path = node
self.name = self.path.name
# We now do not want to use /dev/shm or /run/shm and no ramfs paths
# If /run/user/{uid} is available, we prefer it to /tmp
# And we want to try /var/run as a last resort
# If all fails, fallback to platform's tmp dir
from memory_tempfile import MemoryTempfile
import memory_tempfile
# By the way, all paths with string {uid} will have it replaced with the user id
tempfile = MemoryTempfile(fallback=False)
if tempfile.found_mem_tempdir():
print('We could use any of the followig paths: {}'.format(
tempfile.get_usable_mem_tempdir_paths()))
print('And we are using now: {}'.format(tempfile.gettempdir()))
with tempfile.NamedTemporaryFile() as ntf:
# use it as usual...
pass
