def check_max_open_files(opts):
mof_c = opts.get('max_open_files', 100000)
if sys.platform.startswith('win'):
# Check the windows api for more detail on this
# http://msdn.microsoft.com/en-us/library/xt874334(v=vs.71).aspx
# and the python binding http://timgolden.me.uk/pywin32-docs/win32file.html
mof_s = mof_h = win32file._getmaxstdio()
else:
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
accepted_keys_dir = os.path.join(opts.get('pki_dir'), 'minions')
accepted_count = len([
key for key in os.listdir(accepted_keys_dir) if
os.path.isfile(os.path.join(accepted_keys_dir, key))
])
log.debug(
'This salt-master instance has accepted {0} minion keys.'.format(
accepted_count
)
)
level = logging.INFO
if (accepted_count * 4) <= mof_s:
# We check for the soft value of max open files here because that's the
# value the user chose to raise to.
#
# The number of accepted keys multiplied by four(4) is lower than the
# soft value, everything should be OK
return
msg = (
'The number of accepted minion keys({0}) should be lower than 1/4 '
'of the max open files soft setting({1}). '.format(
accepted_count, mof_s
)
)
if accepted_count >= mof_s:
# This should never occur, it might have already crashed
msg += 'salt-master will crash pretty soon! '
level = logging.CRITICAL
elif (accepted_count * 2) >= mof_s:
# This is way too low, CRITICAL
level = logging.CRITICAL
elif (accepted_count * 3) >= mof_s:
level = logging.WARNING
# The accepted count is more than 3 time, WARN
elif (accepted_count * 4) >= mof_s:
level = logging.INFO
if mof_c < mof_h:
msg += ('According to the system\'s hard limit, there\'s still a '
'margin of {0} to raise the salt\'s max_open_files '
'setting. ').format(mof_h - mof_c)
msg += 'Please consider raising this value.'
log.log(level=level, msg=msg)
def check_max_open_files(opts):
"""
Check the number of max allowed open files and adjust if needed
"""
mof_c = opts.get("max_open_files", 100000)
if sys.platform.startswith("win"):
# Check the Windows API for more detail on this
# http://msdn.microsoft.com/en-us/library/xt874334(v=vs.71).aspx
# and the python binding http://timgolden.me.uk/pywin32-docs/win32file.html
mof_s = mof_h = win32file._getmaxstdio()
else:
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
accepted_keys_dir = os.path.join(opts.get("pki_dir"), "minions")
accepted_count = sum(1 for _ in os.listdir(accepted_keys_dir))
log.debug("This salt-master instance has accepted {0} minion keys.".format(accepted_count))
level = logging.INFO
if (accepted_count * 4) <= mof_s:
# We check for the soft value of max open files here because that's the
# value the user chose to raise to.
#
# The number of accepted keys multiplied by four(4) is lower than the
# soft value, everything should be OK
return
msg = (
"The number of accepted minion keys({0}) should be lower than 1/4 "
"of the max open files soft setting({1}). ".format(accepted_count, mof_s)
)
if accepted_count >= mof_s:
# This should never occur, it might have already crashed
msg += "salt-master will crash pretty soon! "
level = logging.CRITICAL
elif (accepted_count * 2) >= mof_s:
# This is way too low, CRITICAL
level = logging.CRITICAL
elif (accepted_count * 3) >= mof_s:
level = logging.WARNING
# The accepted count is more than 3 time, WARN
elif (accepted_count * 4) >= mof_s:
level = logging.INFO
if mof_c < mof_h:
msg += (
"According to the system's hard limit, there's still a "
"margin of {0} to raise the salt's max_open_files "
"setting. "
).format(mof_h - mof_c)
msg += "Please consider raising this value."
log.log(level=level, msg=msg)
def set_filehandle_limits(self, limits='integration'):
'''
Set soft and hard limits on open file handles at required thresholds
for integration tests or unit tests
'''
# Get current limits
if salt.utils.platform.is_windows():
import win32file
prev_hard = win32file._getmaxstdio()
prev_soft = 512
else:
prev_soft, prev_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
# Get required limits
min_soft = MAX_OPEN_FILES[limits]['soft_limit']
min_hard = MAX_OPEN_FILES[limits]['hard_limit']
# Check minimum required limits
set_limits = False
if prev_soft < min_soft:
soft = min_soft
set_limits = True
else:
soft = prev_soft
if prev_hard < min_hard:
hard = min_hard
set_limits = True
else:
hard = prev_hard
# Increase limits
if set_limits:
print(
' * Max open files settings is too low (soft: {0}, hard: {1}) '
'for running the tests'.format(prev_soft, prev_hard))
print(' * Trying to raise the limits to soft: '
'{0}, hard: {1}'.format(soft, hard))
try:
if salt.utils.platform.is_windows():
hard = 2048 if hard > 2048 else hard
win32file._setmaxstdio(hard)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
except Exception as err:
print('ERROR: Failed to raise the max open files settings -> '
'{0}'.format(err))
print('Please issue the following command on your console:')
print(' ulimit -n {0}'.format(soft))
self.exit()
finally:
print('~' * getattr(self.options, 'output_columns', PNUM))
def set_max_open_files_limits(min_soft=3072, min_hard=4096):
# Get current limits
if salt.utils.platform.is_windows():
import win32file
prev_hard = win32file._getmaxstdio()
prev_soft = 512
else:
import resource
prev_soft, prev_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
# Check minimum required limits
set_limits = False
if prev_soft < min_soft:
soft = min_soft
set_limits = True
else:
soft = prev_soft
if prev_hard < min_hard:
hard = min_hard
set_limits = True
else:
hard = prev_hard
# Increase limits
if set_limits:
log.debug(
" * Max open files settings is too low (soft: %s, hard: %s) for running the tests. "
"Trying to raise the limits to soft: %s, hard: %s",
prev_soft,
prev_hard,
soft,
hard,
)
try:
if salt.utils.platform.is_windows():
hard = 2048 if hard > 2048 else hard
win32file._setmaxstdio(hard)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
except Exception as err: # pylint: disable=broad-except
log.error(
"Failed to raise the max open files settings -> %s. Please issue the following command "
"on your console: 'ulimit -u %s'",
err,
soft,
)
exit(1)
return soft, hard
def set_filehandle_limits(self, limits="integration"):
"""
Set soft and hard limits on open file handles at required thresholds
for integration tests or unit tests
"""
# Get current limits
if salt.utils.platform.is_windows():
import win32file
prev_hard = win32file._getmaxstdio()
prev_soft = 512
else:
prev_soft, prev_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
# Get required limits
min_soft = MAX_OPEN_FILES[limits]["soft_limit"]
min_hard = MAX_OPEN_FILES[limits]["hard_limit"]
# Check minimum required limits
set_limits = False
if prev_soft < min_soft:
soft = min_soft
set_limits = True
else:
soft = prev_soft
if prev_hard < min_hard:
hard = min_hard
set_limits = True
else:
hard = prev_hard
# Increase limits
if set_limits:
print(" * Max open files settings is too low (soft: {}, hard: {}) "
"for running the tests".format(prev_soft, prev_hard))
print(" * Trying to raise the limits to soft: "
"{}, hard: {}".format(soft, hard))
try:
if salt.utils.platform.is_windows():
hard = 2048 if hard > 2048 else hard
win32file._setmaxstdio(hard)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
except Exception as err: # pylint: disable=broad-except
print("ERROR: Failed to raise the max open files settings -> "
"{}".format(err))
print("Please issue the following command on your console:")
print(" ulimit -n {}".format(soft))
self.exit()
finally:
print("~" * getattr(self.options, "output_columns", PNUM))
def setMaxfilesopened(limit):
try:
if sys.platform == "win32":
import win32file
maxstdio = win32file._getmaxstdio()
if maxstdio < limit:
logging.debug("Current maxstdio: %s, changing to %s..." % (maxstdio, limit))
win32file._setmaxstdio(limit)
return True
else:
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft < limit:
logging.debug("Current RLIMIT_NOFILE: %s (max: %s), changing to %s..." % (soft, hard, limit))
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
return True
except Exception, err:
logging.error("Failed to modify max files open limit: %s" % err)
return False
from PyPDF3 import PdfFileWriter, PdfFileReader, PdfFileMerger
import os
import tqdm
import sys
from collections import OrderedDict
import win32file
import fitz
win32file._setmaxstdio(4096)
i = 0
print(win32file._getmaxstdio())
sys.setrecursionlimit(30000)
#with open('allPDFs.txt') as f:
# pdflines = f.readlines()
pdffiles = [
os.path.join(name) for root, dirs, files in os.walk(os.getcwd())
for name in files if name.endswith((".pdf"))
]
#get page number
def getPageNr(arg1):
stro = str(arg1)
stro = stro.replace('.pdf', '')
listR = stro.split(' - ')
listR[len(listR) - 1] = listR[len(listR) - 1].replace('-', '')
listR[len(listR) - 1] = listR[len(listR) - 1].replace('Page ', '')
pgNr = int(listR[len(listR) - 1])
def test_max_open_files(self):
with TstSuiteLoggingHandler() as handler:
logmsg_dbg = (
'DEBUG:This salt-master instance has accepted {0} minion keys.'
)
logmsg_chk = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). According '
'to the system\'s hard limit, there\'s still a margin of {3} '
'to raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
logmsg_crash = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). '
'salt-master will crash pretty soon! According to the '
'system\'s hard limit, there\'s still a margin of {3} to '
'raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
if sys.platform.startswith('win'):
logmsg_crash = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). '
'salt-master will crash pretty soon! Please consider '
'raising this value.')
if sys.platform.startswith('win'):
# Check the Windows API for more detail on this
# http://msdn.microsoft.com/en-us/library/xt874334(v=vs.71).aspx
# and the python binding http://timgolden.me.uk/pywin32-docs/win32file.html
mof_s = mof_h = win32file._getmaxstdio()
else:
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
tempdir = tempfile.mkdtemp(prefix='fake-keys')
keys_dir = os.path.join(tempdir, 'minions')
os.makedirs(keys_dir)
mof_test = 256
if sys.platform.startswith('win'):
win32file._setmaxstdio(mof_test)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h))
try:
prev = 0
for newmax, level in ((24, None), (66, 'INFO'),
(127, 'WARNING'), (196, 'CRITICAL')):
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, six.text_type(n))
with salt.utils.files.fopen(kpath, 'w') as fp_:
fp_.write(
str(n)
) # future lint: disable=blacklisted-function
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
if level is None:
# No log message is triggered, only the DEBUG one which
# tells us how many minion keys were accepted.
self.assertEqual([logmsg_dbg.format(newmax)],
handler.messages)
else:
self.assertIn(logmsg_dbg.format(newmax),
handler.messages)
self.assertIn(
logmsg_chk.format(
level,
newmax,
mof_test,
mof_test - newmax
if sys.platform.startswith('win') else mof_h -
newmax,
), handler.messages)
handler.clear()
prev = newmax
newmax = mof_test
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, six.text_type(n))
with salt.utils.files.fopen(kpath, 'w') as fp_:
fp_.write(str(
n)) # future lint: disable=blacklisted-function
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_crash.format(
'CRITICAL',
newmax,
mof_test,
mof_test -
newmax if sys.platform.startswith('win') else mof_h -
newmax,
), handler.messages)
handler.clear()
except IOError as err:
if err.errno == 24:
# Too many open files
self.skipTest('We\'ve hit the max open files setting')
raise
finally:
if sys.platform.startswith('win'):
win32file._setmaxstdio(mof_h)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))
shutil.rmtree(tempdir)
def check_max_open_files(opts):
'''
Check the number of max allowed open files and adjust if needed
'''
mof_c = opts.get('max_open_files', 100000)
if sys.platform.startswith('win'):
# Check the Windows API for more detail on this
# http://msdn.microsoft.com/en-us/library/xt874334(v=vs.71).aspx
# and the python binding http://timgolden.me.uk/pywin32-docs/win32file.html
mof_s = mof_h = win32file._getmaxstdio()
else:
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
accepted_keys_dir = os.path.join(opts.get('pki_dir'), 'minions')
accepted_count = len([
key for key in os.listdir(accepted_keys_dir) if
os.path.isfile(os.path.join(accepted_keys_dir, key))
])
log.debug(
'This salt-master instance has accepted {0} minion keys.'.format(
accepted_count
)
)
level = logging.INFO
if (accepted_count * 4) <= mof_s:
# We check for the soft value of max open files here because that's the
# value the user chose to raise to.
#
# The number of accepted keys multiplied by four(4) is lower than the
# soft value, everything should be OK
return
msg = (
'The number of accepted minion keys({0}) should be lower than 1/4 '
'of the max open files soft setting({1}). '.format(
accepted_count, mof_s
)
)
if accepted_count >= mof_s:
# This should never occur, it might have already crashed
msg += 'salt-master will crash pretty soon! '
level = logging.CRITICAL
elif (accepted_count * 2) >= mof_s:
# This is way too low, CRITICAL
level = logging.CRITICAL
elif (accepted_count * 3) >= mof_s:
level = logging.WARNING
# The accepted count is more than 3 time, WARN
elif (accepted_count * 4) >= mof_s:
level = logging.INFO
if mof_c < mof_h:
msg += ('According to the system\'s hard limit, there\'s still a '
'margin of {0} to raise the salt\'s max_open_files '
'setting. ').format(mof_h - mof_c)
msg += 'Please consider raising this value.'
log.log(level=level, msg=msg)
flags_list_string += "(Barcodes In Working Directory)"
flags_count += 1
if args.keep_barcode_files:
flags_list_string += "(Keep Barcodes)"
flags_count += 1
old_workbook_path = ""
new_workbook_path = ""
program_launch_cwd = os.getcwd()
try:
if platform.system() == 'Windows':
import win32file
file_limit = win32file._getmaxstdio()
else:
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
file_limit = soft
except Exception as error:
warnings.warn("Getting open file limit failed with: " + str(error) + " setting internal file limit to 100")
file_limit = 100
if args.log:
import sys
class Logger(object):
def __init__(self):
def test_max_open_files(self):
with TstSuiteLoggingHandler() as handler:
logmsg_dbg = "DEBUG:This salt-master instance has accepted {0} minion keys."
logmsg_chk = (
"{0}:The number of accepted minion keys({1}) should be lower "
"than 1/4 of the max open files soft setting({2}). According "
"to the system's hard limit, there's still a margin of {3} "
"to raise the salt's max_open_files setting. Please consider "
"raising this value."
)
logmsg_crash = (
"{0}:The number of accepted minion keys({1}) should be lower "
"than 1/4 of the max open files soft setting({2}). "
"salt-master will crash pretty soon! According to the "
"system's hard limit, there's still a margin of {3} to "
"raise the salt's max_open_files setting. Please consider "
"raising this value."
)
if sys.platform.startswith("win"):
logmsg_crash = (
"{0}:The number of accepted minion keys({1}) should be lower "
"than 1/4 of the max open files soft setting({2}). "
"salt-master will crash pretty soon! Please consider "
"raising this value."
)
if sys.platform.startswith("win"):
# Check the Windows API for more detail on this
# http://msdn.microsoft.com/en-us/library/xt874334(v=vs.71).aspx
# and the python binding http://timgolden.me.uk/pywin32-docs/win32file.html
mof_s = mof_h = win32file._getmaxstdio()
else:
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
tempdir = tempfile.mkdtemp(prefix="fake-keys")
keys_dir = os.path.join(tempdir, "minions")
os.makedirs(keys_dir)
mof_test = 256
if sys.platform.startswith("win"):
win32file._setmaxstdio(mof_test)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h))
try:
prev = 0
for newmax, level in (
(24, None),
(66, "INFO"),
(127, "WARNING"),
(196, "CRITICAL"),
):
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.files.fopen(kpath, "w") as fp_:
fp_.write(str(n))
opts = {"max_open_files": newmax, "pki_dir": tempdir}
check_max_open_files(opts)
if level is None:
# No log message is triggered, only the DEBUG one which
# tells us how many minion keys were accepted.
self.assertEqual([logmsg_dbg.format(newmax)], handler.messages)
else:
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_chk.format(
level,
newmax,
mof_test,
mof_test - newmax
if sys.platform.startswith("win")
else mof_h - newmax,
),
handler.messages,
)
handler.clear()
prev = newmax
newmax = mof_test
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.files.fopen(kpath, "w") as fp_:
fp_.write(str(n))
opts = {"max_open_files": newmax, "pki_dir": tempdir}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_crash.format(
"CRITICAL",
newmax,
mof_test,
mof_test - newmax
if sys.platform.startswith("win")
else mof_h - newmax,
),
handler.messages,
)
handler.clear()
except OSError as err:
if err.errno == 24:
# Too many open files
self.skipTest("We've hit the max open files setting")
raise
finally:
if sys.platform.startswith("win"):
win32file._setmaxstdio(mof_h)
else:
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))
shutil.rmtree(tempdir)
import asyncio
import functools
import sys
from time import time
from wormhole.authentication import get_ident
from wormhole.authentication import verify
from wormhole.handler import process_http
from wormhole.handler import process_https
from wormhole.handler import process_request
from wormhole.logger import get_logger
MAX_RETRY = 3
if sys.platform == 'win32':
import win32file
MAX_TASKS = win32file._getmaxstdio()
else:
import resource
MAX_TASKS = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
wormhole_semaphore = None
def get_wormhole_semaphore(loop):
max_wormholes = int(0.9 * MAX_TASKS) # Use only 90% of open files limit.
global wormhole_semaphore
if wormhole_semaphore is None:
wormhole_semaphore = asyncio.Semaphore(max_wormholes, loop=loop)
return wormhole_semaphore
def debug_wormhole_semaphore(client_reader, client_writer):
def create_mosaic(in_files, out_file):
"""
Creates mosaic from in_files.
:param in_files: list of paths to input files
:param out_file: path to output mosaic
:return: path to output file
"""
# This is some hacky, dumb shit
# There is a limit on how many file descriptors we can have open at once
# So we will up that limit for a bit and then set it back
if os.name == 'posix':
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if len(in_files) >= soft:
new_limit = len(in_files) * 2
resource.setrlimit(resource.RLIMIT_NOFILE, (new_limit, hard))
logger.debug(f'Hard limit changed from: {hard} to {new_limit}')
logger.debug(f'Soft limit: {soft}')
elif os.name == 'nt':
import win32file
soft = win32file._getmaxstdio()
if len(in_files) >= soft:
new_limit = len(in_files) * 2
win32file._setmaxstdio(new_limit)
logger.debug(f'Limit changed from {soft} to {new_limit}')
file_objs = []
for file in in_files:
src = rasterio.open(file)
file_objs.append(src)
mosaic, out_trans = rasterio.merge.merge(file_objs)
out_meta = src.meta.copy()
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans
})
with rasterio.open(out_file, "w", **out_meta) as dest:
dest.write(mosaic)
logger.debug(
f'Mosaic created at {out_file} with resolution: {dest.res}')
# Reset soft limit
if os.name == 'posix':
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if len(in_files) >= soft:
# Todo: this doesn't seem correct
resource.setrlimit(resource.RLIMIT_NOFILE,
(len(in_files) * 2, hard))
elif os.name == 'nt':
import win32file
soft = win32file._getmaxstdio()
if len(in_files) >= soft:
win32file._setmaxstdio(len(in_files) * 2)
# Todo: log limit reset
return Path(out_file).resolve()