def __init__(self, path, flags, *args, **kwargs):
_Lock.__init__(self,
path,
*args,
flags=flags,
timeout=None,
**kwargs)
def lock_file(file, mode, exclusive=True):
if PORTALOCKER:
if parse_version(PORTALOCKER.version) > parse_version("0.6.1"):
return Lock(file, mode, timeout=60, flags=LOCK_EX if exclusive else LOCK_SH)
else:
return Lock(file, mode, timeout=60, flags=LOCK_EX if exclusive else LOCK_SH, truncate=None)
else:
return io.open(file, mode)
def __append_features(self):
tool.write_to_log("Appending features to %s" % self.profile['SQL_TBL'])
if self.val_sql_col('Edit_DT'):
query = '''
UPDATE {0}
SET
Edit_DT = GETDATE()
WHERE
Source_File = 'Updated Records {1}'
'''.format(self.profile['SQL_TBL'], self.batch)
sql.sql_execute(query_str=query, execute=True)
elif self.val_sql_col('Edit_Date'):
query = '''
UPDATE {0}
SET
Edit_Date = GETDATE()
WHERE
Source_File = 'Updated Records {1}'
'''.format(self.profile['SQL_TBL'], self.batch)
sql.sql_execute(query_str=query, execute=True)
path = join(sql_dir, self.profile['Acc_TBL'])
if exists(path):
for file in list(Path(path).glob('*.sql')):
with Lock(str(file), 'r') as f:
query = f.read()
sql.sql_execute(query_str=query, execute=True)
def function(*args, **kwargs):
# atomic operation
with Lock(".backup/jobs.json", access) as f:
if reads:
# read, or create an empty one
try:
jobs = json.load(f)
except: # noqa F841
jobs = []
else:
jobs = []
# pass the jobs and an arg and delegate
kwargs.update({"jobs": jobs})
ret = fn(*args, **kwargs)
# check whether to write
if writes:
# write out and truncate to new size
f.seek(0)
json.dump(jobs, f, indent=2)
f.truncate()
# update git, for backup
repo = get_repo()
repo.index.add("jobs.json")
repo.index.commit("Changed due to '{}' from {}".format(
fn.__name__, request.remote_addr))
# propagate back result
return ret
def _save_current_data(self):
"""Save examples to data file."""
assert "timestamp" not in self._current_example, "multiple _save_current_data calls on _current_example = {_coconut_format_0}".format(_coconut_format_0=(self._current_example))
with Lock(self.data_file, "rb+", timeout=lock_timeout) as df:
# we create the timestamp while we have the lock to ensure its uniqueness
self._current_example["timestamp"] = time.time()
self._add_examples([self._current_example])
self._save_to(df)
def __task_execute(self, task_profile):
try:
shell_line = None
shell_comm = task_profile['Task_SComm']
ext = splitext(task_profile['Task'])[1].lower()
if shell_comm:
shell_comm = ". '%s'" % shell_comm
elif ext == '.py':
shell_comm = 'python'
elif ext == '.ps1':
shell_comm = '.'
elif ext == '.vbs':
shell_comm = 'cscript'
if task_profile['Params']:
params = task_profile['Params']
else:
params = ''
if ext == '.sql':
with Lock(task_profile['Task'], 'r') as f:
query = f.read()
self.__sql_execute(task_profile, query)
elif ext == '.exe':
shell_line = "'{0}' {1}".format(str(task_profile['Task']),
params)
elif shell_comm:
shell_line = [
'powershell.exe',
"{0} '{1}' {2}".format(shell_comm, task_profile['Task'],
params)
]
if shell_line:
self.__sub_proc = Popen(shell_line,
startupinfo=startupinfo,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
stdout, stderr = self.__sub_proc.communicate()
self.__process_std_output(task_profile['Task_Name'], stdout)
self.__process_std_output(task_profile['Task_Name'],
stderr,
is_error=True)
if self.__sub_proc.returncode:
task_profile['Task_Error'] = [
self.__sub_proc.returncode,
'Ran into error while executing program'
]
except Exception as e:
self.write_job_log(format_exc())
task_profile['Task_Error'] = [type(e).__name__, str(e)]
pass
def wrapper(*args, **kwargs):
datadir = set_active_environment(**kwargs)
kwargs['datadir'] = datadir
try:
with Lock(os.path.join(datadir, 'LOCK'), timeout=1):
func(*args, **kwargs)
except LockException:
logger.error(f'directory {datadir} is locked, possibly used by '
'another Golem instance')
def open_with_lock(fpath, mode="rb+", timeout=lock_timeout, **kwargs):
with Lock(fpath, mode, timeout=timeout, **kwargs) as file_handle:
try:
yield file_handle
finally:
file_handle.flush()
if "w" in mode or "+" in mode or "a" in mode:
try:
os.fsync(file_handle.fileno())
except OSError:
pass
def store(self):
"""Store parameters using the filename specified in self.file_path.
"""
# Store settings in file
with Lock(self.file_path, 'w', self['file_timeout']) as fout:
json.dump(self.data, fout)
# Ensure write is fully complete even for network file systems
fout.flush()
os.fsync(fout.fileno())
return
def write(self, file_or_path, append=False, timeout=10):
"""
Write Smother results to a file.
Parameters
----------
fiile_or_path : str
Path to write report to
append : bool
If True, read an existing smother report from `outpath`
and combine it with this file before writing.
timeout : int
Time in seconds to wait to acquire a file lock, before
raising an error.
Note
----
Append mode is atomic when file_or_path is a path,
and can be safely run in a multithreaded or
multiprocess test environment.
When using `parallel_mode`, file_or_path is given a unique
suffix based on the machine name and process id.
"""
if isinstance(file_or_path, six.string_types):
if self.coverage:
file_or_path = get_smother_filename(
file_or_path, self.coverage.config.parallel)
outfile = Lock(file_or_path,
mode='a+',
timeout=timeout,
fail_when_locked=False)
else:
outfile = noclose(file_or_path)
with outfile as fh:
if append:
fh.seek(0)
try:
other = Smother.load(fh)
except ValueError: # no smother data
pass
else:
self |= other
fh.seek(0)
fh.truncate() # required to overwrite data in a+ mode
json.dump(self.data, fh)
def open_with_lock(fpath, mode="rb+", timeout=None, **kwargs):
"""Open file with lock."""
if timeout is None:
timeout = constants.lock_timeout
with Lock(fpath, mode, timeout=timeout, **kwargs) as file_handle:
try:
yield file_handle
finally:
file_handle.flush()
if "w" in mode or "+" in mode or "a" in mode:
try:
os.fsync(file_handle.fileno())
except OSError:
pass
class _RLock:
def __init__(self, outer_lock_path, inner_lock_path):
self.outer_lock = FileLock(outer_lock_path, timeout=120, flags=LOCK_SH)
self.inner_lock = FileLock(inner_lock_path, timeout=120, flags=LOCK_SH)
def acquire(self):
self.outer_lock.acquire()
self.inner_lock.acquire()
self.outer_lock.release()
def release(self):
self.inner_lock.release()
def __enter__(self):
self.acquire()
def __exit__(self, type_, value, tb):
self.release()
def append_data(identifier, data):
identifier = list(map(str, identifier))
directory = "data/" + "/".join(identifier[:-1])
filename = identifier[-1] + ".pickle"
try:
os.makedirs(directory)
except FileExistsError:
pass
path = "%s/%s" % (directory, filename)
try:
with open(path, "rb") as f:
file_content = pickle.load(f)
except (FileNotFoundError, EOFError):
file_content = []
with Lock(path, "wb") as f:
file_content.append(data)
pickle.dump(file_content, f)
def should_refresh_token(self, con=None):
"""
Method for refreshing the token when there are concurrently running instances.
"""
for _ in range(TOKEN_REFRESH_MAX_TRIES):
if self.token.is_access_expired:
try:
with Lock(self.token_path,
'r+',
fail_when_locked=True,
timeout=0):
if con.refresh_token() is False:
raise RuntimeError('Error refreshing token')
return None
except LockException:
self.fs_wait = True
time.sleep(1)
self.token = self.load_token()
else:
self.fs_wait = False
return False
raise RuntimeError('Could not access locked token file')
def write(self, file_or_path, timeout=10):
"""
Write Smother results to a file.
Parameters
----------
file_or_path : str
Path to write report to
timeout : int
Time in seconds to wait to acquire a file lock, before
raising an error.
Note
----
Append mode is atomic when file_or_path is a path,
and can be safely run in a multithreaded or
multiprocess test environment.
When using `parallel_mode`, file_or_path is given a unique
suffix based on the machine name and process id.
"""
if isinstance(file_or_path, six.string_types):
if self.coverage:
file_or_path = get_smother_filename(
file_or_path, self.coverage.config.parallel)
outfile = Lock(file_or_path,
mode='a+',
timeout=timeout,
fail_when_locked=False)
else:
outfile = noclose(file_or_path)
with outfile as fh:
fh.seek(0)
fh.truncate() # required to overwrite data in a+ mode
json.dump(self.data, fh)
def should_refresh_token(self, con=None):
"""
Method for refreshing the token when there are concurrently running
O365 instances. Determines if we need to call the MS server and refresh
the token and its file, or if another Connection instance has already
updated it and we should just load that updated token from the file.
It will always return False, None, OR raise an error if a token file
couldn't be accessed after X tries. That is because this method
completely handles token refreshing via the passed Connection object
argument. If it determines that the token should be refreshed, it locks
the token file, calls the Connection's 'refresh_token' method (which
loads the fresh token from the server into memory and the file), then
unlocks the file. Since refreshing has been taken care of, the calling
method does not need to refresh and we return None.
If we are blocked because the file is locked, that means another
instance is using it. We'll change the backend's state to waiting,
sleep for 2 seconds, reload a token into memory from the file (since
another process is using it, we can assume it's being updated), and
loop again.
If this newly loaded token is not expired, the other instance loaded
a new token to file, and we can happily move on and return False.
(since we don't need to refresh the token anymore). If the same token
was loaded into memory again and is still expired, that means it wasn't
updated by the other instance yet. Try accessing the file again for X
more times. If we don't suceed after the loop has terminated, raise a
runtime exception
"""
for _ in range(self.max_tries, 0, -1):
if self.token.is_access_expired:
try:
with Lock(self.token_path,
'r+',
fail_when_locked=True,
timeout=0):
log.debug('Locked oauth token file')
if con.refresh_token() is False:
raise RuntimeError('Token Refresh Operation not '
'working')
log.info('New oauth token fetched')
log.debug('Unlocked oauth token file')
return None
except LockException:
self.fs_wait = True
log.warning('Oauth file locked. Sleeping for 2 seconds...'
f'retrying {_ - 1} more times.')
time.sleep(2)
log.debug('Waking up and rechecking token file for update'
' from other instance...')
self.token = self.load_token()
else:
log.info('Token was refreshed by another instance...')
self.fs_wait = False
return False
# if we exit the loop, that means we were locked out of the file after
# multiple retries give up and throw an error - something isn't right
raise RuntimeError('Could not access locked token file after '
f'{self.max_tries}')
def __package_attach(self):
if self.__attachments:
from zipfile import ZipFile
from exchangelib import FileAttachment
from portalocker import Lock
file_dir = join(attach_dir, datetime.today().__format__("%Y%m%d"))
file_path = join(
file_dir, '{0}_{1}.zip'.format(
hash(datetime.now().__format__("%I:%M:%S %p")),
self.__job_profile['Job_Name']))
if not exists(file_dir):
makedirs(file_dir)
if exists(file_path):
try:
remove(file_path)
except Exception as e:
tool.write_to_log(
"Attach File Rem - ECode '{0}', {1}".format(
type(e).__name__, str(e)))
return
zip_file = ZipFile(file_path, mode='w')
try:
for attachment in self.__attachments:
try:
if isfile(attachment) and exists(
attachment) and stat(attachment).st_size > 0:
zip_file.write(attachment, basename(attachment))
except Exception as e:
self.write_job_log(format_exc())
tool.write_to_log(
"Job '{0}' failed to zip file due to ECode '{1}', {2}"
.format(self.__job_profile['Job_Name'],
type(e).__name__, str(e)))
pass
finally:
if isfile(attachment) and exists(
attachment) and stat(attachment).st_size > 0:
remove(attachment)
except Exception as e:
self.write_job_log(format_exc())
tool.write_to_log(
"Job '{0}' failed to zip file in mainloop due to ECode '{1}', {2}"
.format(self.__job_profile['Job_Name'],
type(e).__name__, str(e)))
pass
finally:
zip_file.close()
if exists(file_path) and stat(file_path).st_size > 0:
with Lock(file_path, 'rb') as f:
self.__email.attach(
FileAttachment(name=basename(file_path),
content_type='zip',
content=f.read(),
is_inline=False))
elif exists(file_path):
remove(file_path)
try:
unlink(file_dir)
except Exception as e:
tool.write_to_log("Dir Unlink - ECode '{0}', {1}".format(
type(e).__name__, str(e)))
pass
def save_data(self):
"""Forcibly saves data."""
with Lock(self.data_file, "rb+", timeout=lock_timeout) as df:
self._save_to(df)
def __init__(self, outer_lock_path, inner_lock_path):
self.outer_lock = FileLock(outer_lock_path, timeout=120, flags=LOCK_EX)
self.inner_lock = FileLock(inner_lock_path, timeout=120, flags=LOCK_EX)
def _load_data(self):
"""Load examples from data file."""
ensure_file(self.data_file)
with Lock(self.data_file, "rb", timeout=lock_timeout) as df:
self._load_from(df)
data_dir = '/home/travis/build/pysatData'
params['data_dirs'] = [data_dir]
print(''.join(
("\nHi there! pysat will nominally store data in a ",
"'pysatData' directory which needs to be assigned. ",
"Please run `pysat.params['data_dirs'] = path` where path ",
"specifies one or more existing top-level directories that ",
"may be used to store science data. `path` may either be ",
"a single string or a list of strings.")))
else:
# Load up existing parameters file
params = _params.Parameters()
# Load up version information
with Lock(version_filename, 'r', params['file_timeout']) as version_file:
__version__ = version_file.read().strip()
from pysat import utils
from pysat._constellation import Constellation
from pysat._instrument import Instrument
from pysat._meta import Meta, MetaLabels
from pysat._files import Files
from pysat._orbits import Orbits
from pysat import instruments
__all__ = ['instruments', 'utils']
# Cleanup
del here
def __init__(self, path=None, create_new=False):
sfname = 'pysat_settings.json'
self.data = {}
self.file_path = None
# Define default parameters and values
dir_format = os.path.join('{platform}', '{name}', '{tag}', '{inst_id}')
defaults = {
'clean_level': 'clean',
'directory_format': dir_format,
'ignore_empty_files': False,
'file_timeout': 10,
'update_files': True,
'user_modules': {},
'warn_empty_file_list': False
}
# Attach default parameters and values to object
self.defaults = defaults
# Define stored pysat parameters without a default setting
non_defaults = ['data_dirs']
self.non_defaults = non_defaults
# If path provided, use it. Otherwise, iterate through potential
# locations until file is found.
if path is not None:
# Confirm path is valid
if not os.path.exists(path):
estr = ''.join(('Supplied path does not exist on the local ',
'system. Please create it and try again.'))
raise OSError(estr)
# Store full file path including fixed settings file name
self.file_path = os.path.join(path, sfname)
else:
# Cycle through locations and load any pysat parameter files found.
# First, check current working directory, then pysat user directory.
# Stop looking once an existing file is found.
file_locs = [
os.path.join('.', sfname),
os.path.join(os.path.expanduser('~'), '.pysat', sfname)
]
for fileloc in file_locs:
if os.path.isfile(fileloc):
self.file_path = fileloc
break
# Ensure we have a valid file if the user isn't creating a new one.
if self.file_path is None and (not create_new):
estr = ''.join(
('pysat is unable to locate a user settings ',
'file. Please check the locations, "./" or ',
'"~/.pysat" for the file "pysat_settings.json"', '.'))
raise OSError(estr)
if create_new:
# Initialize new settings file. Method below includes a .store call
self.clear_and_restart()
# Load parameters in thread-safe manner.
# Can't use user set file_timeout since we don't know what it is yet.
# Can't use NetworkLock in pysat.utils either since this object (params)
# doesn't exist yet.
with Lock(self.file_path, 'r', timeout=10) as fout:
self.data = json.load(fout)
# In case of network file system
fout.flush()
os.fsync(fout.fileno())
return
def start( # pylint: disable=too-many-arguments, too-many-locals
monitor, concent, datadir, node_address, rpc_address, peer, mainnet,
net, geth_address, password, accept_terms, accept_concent_terms,
accept_all_terms, version, log_level, enable_talkback, m):
freeze_support()
delete_reactor()
# Crossbar
if m == 'crossbar.worker.process':
start_crossbar_worker(m)
return 0
if version:
print("EVERA version: {}".format(evera.__version__))
return 0
set_environment('mainnet' if mainnet else net, concent)
# These are done locally since they rely on evera.config.active to be set
from evera.config.active import CONCENT_VARIANT
from evera.appconfig import AppConfig
from evera.node import Node
# We should use different directories for different chains
datadir = get_local_datadir('default', root_dir=datadir)
os.makedirs(datadir, exist_ok=True)
def _start():
generate_rpc_certificate(datadir)
# Workarounds for pyinstaller executable
sys.modules['win32com.gen_py.os'] = None
sys.modules['win32com.gen_py.pywintypes'] = None
sys.modules['win32com.gen_py.pythoncom'] = None
app_config = AppConfig.load_config(datadir)
config_desc = ClientConfigDescriptor()
config_desc.init_from_app_config(app_config)
config_desc = ConfigApprover(config_desc).approve()
if rpc_address:
config_desc.rpc_address = rpc_address.address
config_desc.rpc_port = rpc_address.port
if node_address:
config_desc.node_address = node_address
# Evera headless
install_reactor()
from evera.core.common import config_logging
config_logging(
datadir=datadir,
loglevel=log_level,
config_desc=config_desc)
log_evera_version()
log_platform_info()
log_ethereum_chain()
log_concent_choice(CONCENT_VARIANT)
node = Node(
datadir=datadir,
app_config=app_config,
config_desc=config_desc,
peers=peer,
use_monitor=monitor,
use_talkback=enable_talkback,
concent_variant=CONCENT_VARIANT,
geth_address=geth_address,
password=password,
)
if accept_terms:
node.accept_terms()
if accept_concent_terms:
node.accept_concent_terms()
if accept_all_terms:
node.accept_terms()
node.accept_concent_terms()
node.start()
try:
with Lock(os.path.join(datadir, 'LOCK'), timeout=1):
_start()
except LockException:
logger.error(f'directory {datadir} is locked, possibly used by '
'another Evera instance')
return 1
return 0
