class ManagementLock(object):
def __init__(self):
self.lock = None
def acquire(self):
self.lock = InterProcessLock(LOCK_PATH)
# Attempt to obtain a lock, retry every 10 seconds. Wait at most 5 minutes.
# The retrying is necessary so we can report on stderr that we are waiting
# for a lock. Otherwise, a user trying to run the command manually might
# get confused why the command execution is delayed.
if self.lock.acquire(blocking=False):
return
print("Another management command is running, waiting for lock...", file=sys.stderr)
if self.lock.acquire(delay=10, max_delay=10, timeout=300):
return
self.lock = None
raise RuntimeError("Failed to acquire lock.")
def release(self):
if self.lock is not None:
self.lock.release()
def __enter__(self):
self.acquire()
return self
def __exit__(self, _exc_type, _exc_val, _exc_tb):
self.release()
return False
def acquire_inter_process_lock(lock_name):
# Lock preventing simultaneous crawling processes
lock_name = _compute_lock_name(lock_name)
lock = InterProcessLock('/tmp/%s' % lock_name)
acquired_lock = lock.acquire(blocking=False)
try:
yield acquired_lock
finally:
if acquired_lock:
lock.release()
def _update_db(self):
if self._prim_db:
lock = InterProcessLock("%s.lock" % self._prim_db)
acquired = lock.acquire(blocking=False)
if not acquired:
logger.debug("Waiting 60 seconds for file lock")
acquired = lock.acquire(blocking=True, timeout=60)
if acquired:
try:
with open(self._prim_db, "w", encoding="utf-8") as out:
out.write(unicode(
json.dumps(self._dbs[self._prim_db])))
return True
finally:
lock.release()
else:
logger.error("Could not update platform database: "
"Lock acquire failed after 60 seconds")
return False
else:
logger.error("Can't update platform database: "
"destination database is ambiguous")
return False
def wrapper(self, *args):
ret = None
lock = InterProcessLock(self.lock_file)
acquired = lock.acquire(blocking=False)
if not acquired:
self.debug("timed_mbedls_lock", "Waiting %d seconds for mock file lock." % timeout)
acquired = lock.acquire(blocking=True, timeout=timeout)
if acquired:
try:
ret = wrapper.original(self, *args)
except Exception as e:
lock.release()
raise e
lock.release()
else:
self.err("timed_mbedls_lock", "Failed to acquired mock file lock in %d seconds!" % timeout)
sys.exit(1)
return ret
def run(self):
"""
Run application.
"""
# output work with
# configure SQLAlchemy logging
# log_level = self.logger.getEffectiveLevel()
# logging.getLogger('sqlalchemy.engine').setLevel(log_level)
exit_code = ExitCodes.EXIT_SUCCESS
try:
pid_lock = InterProcessLock(self.args.path_pidfile)
pid_lock_gotten = pid_lock.acquire(blocking=False)
if not pid_lock_gotten:
raise AlreadyHarvesting(self.args.path_pidfile)
self.logger.debug('Aquired PID lock {0!r}'.format(
self.args.path_pidfile))
if (self.args.no_routes and self.args.no_vnetworks
and not self.args.truncate):
raise NothingToDo()
harvesting = not (self.args.no_routes and self.args.no_vnetworks)
Session = db.ScopedSession()
Session.configure(bind=self.args.db_engine)
db.configure_db(self.DB_PRAGMAS)
# TODO(damb): Implement multithreaded harvesting using a thread
# pool.
try:
if harvesting:
self.logger.info('Start harvesting.')
if not self.args.no_routes:
self._harvest_routes(Session)
else:
self.logger.warn(
'Disabled processing <route></route> information.')
if not self.args.no_vnetworks:
self._harvest_vnetworks(Session)
else:
self.logger.warn(
'Disabled processing <vnetwork></vnetwork> '
'information.')
if harvesting:
self.logger.info('Finished harvesting successfully.')
if self.args.truncate:
self.logger.warning('Removing outdated data.')
session = Session()
with db.session_guard(session) as _session:
num_removed_rows = db.clean(_session,
self.args.truncate)
self.logger.info('Number of rows removed: {}'.format(
num_removed_rows))
except OperationalError as err:
raise db.StationLiteDBEngineError(err)
# TODO(damb): signal handling
except Error as err:
self.logger.error(err)
exit_code = ExitCodes.EXIT_ERROR
except Exception as err:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.critical('Local Exception: %s' % err)
self.logger.critical('Traceback information: ' + repr(
traceback.format_exception(exc_type, exc_value, exc_traceback))
)
exit_code = ExitCodes.EXIT_ERROR
finally:
try:
if pid_lock_gotten:
pid_lock.release()
except NameError:
pass
sys.exit(exit_code)
def main(argv=None):
'''Command line options.'''
# setup argparser
parser = argparse.ArgumentParser()
parser.add_argument('--version', action='version', version='%s v%s (%s)' %
(os.path.basename(__file__), __version__, __updated__))
# Actions
action_group = parser.add_argument_group("Actions", "A single action must be selected.")
actions = action_group.add_mutually_exclusive_group(required=True)
actions.add_argument("--report", dest="report", type=str, help="Submit the given textual report", metavar="TEXT")
actions.add_argument("--report-from-file", dest="report_file", type=str,
help="Submit the given file as textual report", metavar="FILE")
actions.add_argument("--cycle", dest="cycle", type=str, help="Cycle the pool with the given ID", metavar="ID")
actions.add_argument("--disable", dest="disable", type=str, help="Disable the pool with the given ID", metavar="ID")
actions.add_argument("--enable", dest="enable", type=str, help="Enable the pool with the given ID", metavar="ID")
# Options
parser.add_argument("--keep-reporting", dest="keep_reporting", default=0, type=int,
help="Keep reporting from the specified file with specified interval", metavar="SECONDS")
parser.add_argument("--random-offset", dest="random_offset", default=0, type=int,
help="Random offset for the reporting interval (+/-)", metavar="SECONDS")
# Settings
parser.add_argument("--serverhost", dest="serverhost",
help="Server hostname for remote signature management", metavar="HOST")
parser.add_argument("--serverport", dest="serverport", type=int, help="Server port to use", metavar="PORT")
parser.add_argument("--serverproto", dest="serverproto",
help="Server protocol to use (default is https)", metavar="PROTO")
parser.add_argument("--serverauthtokenfile", dest="serverauthtokenfile",
help="File containing the server authentication token", metavar="FILE")
parser.add_argument("--clientid", dest="clientid", help="Client ID to use when submitting issues", metavar="ID")
# process options
opts = parser.parse_args(argv)
if opts.keep_reporting and not opts.report_file:
print("Error: --keep-reporting is only valid with --report-from-file", file=sys.stderr)
return 2
serverauthtoken = None
if opts.serverauthtokenfile:
with open(opts.serverauthtokenfile) as f:
serverauthtoken = f.read().rstrip()
reporter = EC2Reporter(opts.serverhost, opts.serverport, opts.serverproto, serverauthtoken, opts.clientid)
report = None
if opts.cycle:
reporter.cycle(opts.cycle)
return 0
elif opts.enable:
reporter.enable(opts.enable)
return 0
elif opts.disable:
reporter.disable(opts.disable)
return 0
elif opts.report_file:
if opts.keep_reporting:
if opts.random_offset > 0:
random.seed(reporter.clientId)
lock = InterProcessLock(opts.report_file + ".lock")
while True:
if os.path.exists(opts.report_file):
if not lock.acquire(timeout=opts.keep_reporting):
continue
try:
with open(opts.report_file) as f:
report = f.read()
try:
reporter.report(report)
except RuntimeError as e:
# Ignore errors if the server is temporarily unavailable
print("Failed to contact server: %s" % e, file=sys.stderr)
finally:
lock.release()
random_offset = 0
if opts.random_offset:
random_offset = random.randint(-opts.random_offset, opts.random_offset)
time.sleep(opts.keep_reporting + random_offset)
else:
with open(opts.report_file) as f:
report = f.read()
else:
report = opts.report
reporter.report(report)
return 0
def _cluster_settings_lock(daemon_name):
return InterProcessLock(
os.path.join(
tempfile.gettempdir(),
'cluster-{0}-{1}.lock'.format(getpass.getuser(), daemon_name)))
def dataframe(self, modify=True):
with InterProcessLock(self.storagepath / 'lockfile'):
df = self._load_dataframe()
yield df
if modify:
self._save_dataframe(df)
class Model:
'''Mod management model'''
def __init__(self, ignorelock=False):
if not ignorelock:
self.lock = InterProcessLock(self.lockfile)
if not self.lock.acquire(False):
raise IOError('could not lock ' + self.lockfile)
self.modList: Dict[str, Mod] = {}
self.reload()
def reload(self) -> None:
self.modList = {}
if path.exists(self.xmlfile):
tree = XML.parse(self.xmlfile)
root = tree.getroot()
for xmlmod in root.findall('mod'):
mod = self.populateModFromXml(Mod(), xmlmod)
self.modList[mod.name] = mod
def write(self) -> None:
root = XML.ElementTree(XML.Element('installed'))
for mod in self.all():
root = self.writeModToXml(mod, root)
indent(root.getroot())
root.write(self.xmlfile)
def get(self, modname: str) -> Mod:
return self.modList[modname]
def list(self) -> KeysView[str]:
return self.modList.keys()
def all(self) -> ValuesView[Mod]:
return self.modList.values()
def add(self, modname: str, mod: Mod):
self.modList[modname] = mod
def remove(self, modname: str):
if modname in self.modList:
del self.modList[modname]
def rename(self, modname: str, newname: str) -> bool:
if not modname in self.modList:
return False
mod = self.modList[modname]
del self.modList[modname]
mod.name = newname
self.modList[newname] = mod
return True
def explore(self, modname: str) -> None:
mod = self.modList[modname]
for file in mod.files:
moddir = data.config.mods + ('/~'
if not mod.enabled else '/') + file
openFolder(moddir)
@property
def xmlfile(self) -> str:
return data.config.configuration + '/installed.xml'
@property
def lockfile(self) -> str:
return data.config.configuration + '/installed.lock'
@staticmethod
def populateModFromXml(mod: Mod, root: XML.Element) -> Mod:
mod.date = str(root.get('date'))
enabled = str(root.get('enabled'))
if enabled == 'True':
mod.enabled = True
else:
mod.enabled = False
mod.name = str(root.get('name'))
prt = str(root.get('priority'))
if prt != 'Not Set':
mod.priority = prt
for elem in root.findall('data'):
mod.files.append(str(elem.text))
for elem in root.findall('dlc'):
mod.dlcs.append(str(elem.text))
for elem in root.findall('menu'):
mod.menus.append(str(elem.text))
for elem in root.findall('xmlkey'):
mod.xmlkeys.append(str(elem.text))
for elem in root.findall('hidden'):
mod.hidden.append(str(elem.text))
for elem in root.findall('key'):
key = Key(elem.get('context'), str(elem.text))
mod.inputsettings.append(key)
for elem in root.findall('settings'):
# legacy usersetting storage format
settings = fetchUserSettings(str(elem.text))
for setting in iter(settings):
mod.usersettings.append(setting)
for elem in root.findall('setting'):
usersetting = Usersetting(str(elem.get('context')), str(elem.text))
mod.usersettings.append(usersetting)
mod.checkPriority()
return mod
@staticmethod
def writeModToXml(mod: Mod, root: XML.ElementTree) -> XML.ElementTree:
elem = XML.SubElement(root.getroot(), 'mod')
elem.set('name', mod.name)
elem.set('enabled', str(mod.enabled))
elem.set('date', mod.date)
elem.set('priority', mod.priority)
if mod.files:
for file in mod.files:
XML.SubElement(elem, 'data').text = file
if mod.dlcs:
for dlc in mod.dlcs:
XML.SubElement(elem, 'dlc').text = dlc
if mod.menus:
for menu in mod.menus:
XML.SubElement(elem, 'menu').text = menu
if mod.xmlkeys:
for xml in mod.xmlkeys:
XML.SubElement(elem, 'xmlkey').text = xml
if mod.hidden:
for xml in mod.hidden:
XML.SubElement(elem, 'hidden').text = xml
if mod.inputsettings:
for key in mod.inputsettings:
ky = XML.SubElement(elem, 'key')
ky.text = str(key)
ky.set('context', key.context)
if mod.usersettings:
for usersetting in mod.usersettings:
us = XML.SubElement(elem, 'setting')
us.text = str(usersetting)
us.set('context', usersetting.context)
return root
def run(self):
"""
Run application.
"""
# configure SQLAlchemy logging
# log_level = self.logger.getEffectiveLevel()
# logging.getLogger('sqlalchemy.engine').setLevel(log_level)
exit_code = ExitCodes.EXIT_SUCCESS
self.logger.info(f"{self.PROG}: Version v{__version__}")
self.logger.debug(f"Configuration: {dict(self.config)!r}")
try:
path_pidfile = self.config["path_pidfile"]
pid_lock = InterProcessLock(path_pidfile)
pid_lock_gotten = pid_lock.acquire(blocking=False)
if not pid_lock_gotten:
raise AlreadyHarvesting(path_pidfile)
self.logger.debug(
f"Aquired PID lock {self.config['path_pidfile']!r}"
)
if (
self.config["no_routes"]
and self.config["no_vnetworks"]
and not self.config["truncate"]
):
raise NothingToDo()
harvesting = not (
self.config["no_routes"] and self.config["no_vnetworks"]
)
Session = db.ScopedSession()
engine = create_engine(
self.config["sqlalchemy_database_uri"], echo=False
)
Session.configure(bind=engine)
if engine.name == "sqlite":
db.configure_sqlite(self.DB_PRAGMAS)
# TODO(damb): Implement multithreaded harvesting using a thread
# pool.
try:
if harvesting:
self.logger.info("Start harvesting.")
if not self.config["no_routes"]:
self._harvest_routes(Session)
else:
self.logger.info(
"Disabled processing <route></route> information."
)
if not self.config["no_vnetworks"]:
self._harvest_vnetworks(Session)
else:
self.logger.info(
"Disabled processing <vnetwork></vnetwork> "
"information."
)
if harvesting:
self.logger.info("Finished harvesting successfully.")
if self.config["truncate"]:
self.logger.warning("Removing outdated data.")
session = Session()
with db.session_guard(session) as _session:
num_removed_rows = db.clean(
_session,
self.config["truncate"],
)
self.logger.info(
f"Number of rows removed: {num_removed_rows}"
)
except OperationalError as err:
raise db.StationLiteDBEngineError(err)
# TODO(damb): signal handling
except Error as err:
self.logger.error(err)
exit_code = ExitCodes.EXIT_ERROR
except Exception as err:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.critical("Local Exception: %s" % err)
self.logger.critical(
"Traceback information: "
+ repr(
traceback.format_exception(
exc_type, exc_value, exc_traceback
)
)
)
exit_code = ExitCodes.EXIT_ERROR
finally:
try:
if pid_lock_gotten:
pid_lock.release()
except NameError:
pass
sys.exit(exit_code)
def _list_outputs(self):
"""Execute this module.
"""
# Initiate outputs
outputs = self._base_outputs()
out_files = []
missing_files = []
# Get output dir from base ArchiveSink class (will change depending on
# whether it is per session/subject/visit/project)
out_path = self._get_output_path()
out_dir = os.path.abspath(os.path.join(*out_path))
# Make session dir
if not os.path.exists(out_dir):
os.makedirs(out_dir, stat.S_IRWXU | stat.S_IRWXG)
# Loop through datasets connected to the sink and copy them to archive
# directory
for spec in self.datasets:
assert spec.derived, (
"Should only be sinking derived datasets, not '{}'".format(
spec.name))
filename = getattr(self.inputs, spec.name + PATH_SUFFIX)
ext = spec.format.extension
if not isdefined(filename):
missing_files.append(spec.name)
continue # skip the upload for this file
if lower(split_extension(filename)[1]) != lower(ext):
raise NiAnalysisError(
"Mismatching extension '{}' for format '{}' ('{}')".format(
split_extension(filename)[1], spec.format, ext))
assert spec.frequency == self.frequency
# Copy to local system
src_path = os.path.abspath(filename)
out_fname = spec.fname()
dst_path = os.path.join(out_dir, out_fname)
out_files.append(dst_path)
if os.path.isfile(src_path):
shutil.copyfile(src_path, dst_path)
elif os.path.isdir(src_path):
shutil.copytree(src_path, dst_path)
else:
assert False
if missing_files:
# FIXME: Not sure if this should be an exception or not,
# indicates a problem but stopping now would throw
# away the datasets that were created
logger.warning("Missing input datasets '{}' in LocalSink".format(
"', '".join(missing_files)))
# Return cache file paths
outputs['out_files'] = out_files
# Loop through fields connected to the sink and save them in the
# fields JSON file
out_fields = []
fpath = self.fields_path(self.frequency)
# Open fields JSON, locking to prevent other processes
# reading or writing
if self.fields:
with InterProcessLock(fpath + LOCK, logger=logger):
try:
with open(fpath, 'rb') as f:
fields = json.load(f)
except IOError as e:
if e.errno == errno.ENOENT:
fields = {}
else:
raise
# Update fields JSON and write back to file.
for spec in self.fields:
value = getattr(self.inputs, spec.name + FIELD_SUFFIX)
qual_name = self.prefix_study_name(spec.name)
if spec.dtype is str:
assert isinstance(value, basestring)
else:
assert isinstance(value, spec.dtype)
fields[qual_name] = value
out_fields.append((qual_name, value))
with open(fpath, 'wb') as f:
json.dump(fields, f)
outputs['out_fields'] = out_fields
return outputs
def lock_if_check_fails(check,
lock_path,
operation=None,
blocking=True,
_return_acquired=False,
**kwargs):
"""A context manager to establish a lock conditionally on result of a check
It is intended to be used as a lock for a specific file and/or operation,
e.g. for `annex get`ing a file or extracting an archive, so only one process
would be performing such an operation.
If verification of the check fails, it tries to acquire the lock, but if
that fails on the first try, it will rerun check before proceeding to func
checker and lock_path_prefix could be a value, or callable, or
a tuple composing callable and its args
Unfortunately yoh did not find any way in Python 2 to have a context manager
which just skips the entire block if some condition is met (in Python3 there
is ExitStack which could potentially be used). So we would need still to
check in the block body if the context manager return value is not None.
Note also that the used type of the lock (fasteners.InterprocessLock) works
only across processes and would not lock within the same (threads) process.
Parameters
----------
check: callable or (callable, args) or value
If value (possibly after calling a callable) evaluates to True, no
lock is acquired, and no context is executed
lock_path: callable or (callable, args) or value
Provides a path for the lock file, composed from that path + '.lck'
extension
operation: str, optional
If provided, would be part of the locking extension
blocking: bool, optional
If blocking, process would be blocked until acquired and verified that it
was acquired after it gets the lock
_return_acquired: bool, optional
Return also if lock was acquired. For "private" use within DataLad (tests),
do not rely on it in 3rd party solutions.
**kwargs
Passed to `.acquire` of the fasteners.InterProcessLock
Returns
-------
result of check, lock[, acquired]
"""
check1 = _get(check)
if check1: # we are done - nothing to do
yield check1, None
return
# acquire blocking lock
lock_filename = _get(lock_path)
lock_filename += '.'
if operation:
lock_filename += operation + '-'
lock_filename += 'lck'
lock = InterProcessLock(lock_filename)
acquired = False
try:
lgr.debug("Acquiring a lock %s", lock_filename)
acquired = lock.acquire(blocking=blocking, **kwargs)
lgr.debug("Acquired? lock %s: %s", lock_filename, acquired)
if blocking:
assert acquired
check2 = _get(check)
ret_lock = None if check2 else lock
if _return_acquired:
yield check2, ret_lock, acquired
else:
yield check2, ret_lock
finally:
if acquired:
lgr.debug("Releasing lock %s", lock_filename)
lock.release()
if exists(lock_filename):
unlink(lock_filename)
class Lock:
"""A inter-process and inter-thread lock
This internally uses :class:`fasteners.InterProcessLock` but provides non-blocking
acquire. It also guarantees thread-safety when using the :meth:`singleton` class
method to create / retrieve a lock instance.
:param path: Path of the lock file to use / create.
"""
_instances: Dict[str, "Lock"] = {}
_singleton_lock = threading.Lock()
@classmethod
def singleton(cls, path: str) -> "Lock":
"""
Retrieve an existing lock object with a given 'name' or create a new one. Use
this method for thread-safe locks.
:param path: Path of the lock file to use / create.
"""
with cls._singleton_lock:
try:
instance = cls._instances[path]
except KeyError:
instance = cls(path)
cls._instances[path] = instance
return instance
def __init__(self, path: str) -> None:
self.path = path
self._internal_lock = threading.Semaphore()
self._external_lock = InterProcessLock(self.path)
self._lock = threading.RLock()
def acquire(self) -> bool:
"""
Attempts to acquire the given lock.
:returns: Whether or not the acquisition succeeded.
"""
with self._lock:
locked_internal = self._internal_lock.acquire(blocking=False)
if not locked_internal:
return False
try:
locked_external = self._external_lock.acquire(blocking=False)
except Exception:
self._internal_lock.release()
raise
else:
if locked_external:
return True
else:
self._internal_lock.release()
return False
def release(self) -> None:
"""Release the previously acquired lock."""
with self._lock:
self._external_lock.release()
self._internal_lock.release()
def locked(self) -> bool:
"""
Checks if the lock is currently held by any thread or process.
:returns: Whether the lock is acquired.
"""
with self._lock:
gotten = self.acquire()
if gotten:
self.release()
return not gotten
def locking_pid(self) -> Optional[int]:
"""
Returns the PID of the process which currently holds the lock or ``None``. This
should work on macOS, OpenBSD and Linux but may fail on some platforms. Always
use :meth:`locked` to check if the lock is held by any process.
:returns: The PID of the process which currently holds the lock or ``None``.
"""
with self._lock:
if self._external_lock.acquired:
return os.getpid()
try:
# Don't close again in case we are the locking process.
self._external_lock._do_open()
lockdata, fmt, pid_index = _get_lockdata()
lockdata = fcntl.fcntl(self._external_lock.lockfile,
fcntl.F_GETLK, lockdata)
lockdata_list = struct.unpack(fmt, lockdata)
pid = lockdata_list[pid_index]
if pid > 0:
return pid
except OSError:
pass
return None
user32.SetWinEventHook.restype = ctypes.wintypes.HANDLE
hook = user32.SetWinEventHook(
EVENT_SYSTEM_FOREGROUND,
EVENT_SYSTEM_FOREGROUND,
0,
WinEventProc,
0,
0,
WINEVENT_OUTOFCONTEXT
)
if hook == 0:
exit(1)
msg = ctypes.wintypes.MSG()
while user32.GetMessageW(ctypes.byref(msg), 0, 0, 0) != 0:
user32.TranslateMessageW(msg)
user32.DispatchMessageW(msg)
user32.UnhookWinEvent(hook)
ole32.CoUninitialize()
logger.info('Acquiring lock')
lock = InterProcessLock(gettempdir() + '/bdbc_lock_file')
gotten = lock.acquire(timeout=10)
if gotten:
logger.info('Lock acquired')
foreground_window_hook()
else:
logger.info('Lock failed')
class ProjectManager(collections.Iterable):
_basic_directories = (
"backups",
"intermediate",
"lci",
"processed",
)
_is_temp_dir = False
read_only = False
def __init__(self):
self._base_data_dir, self._base_logs_dir = self._get_base_directories()
self._create_base_directories()
self.db = SubstitutableDatabase(
os.path.join(self._base_data_dir, "projects.db"), [ProjectDataset])
self.set_current("default", update=False)
def __iter__(self):
for project_ds in ProjectDataset.select():
yield project_ds
def __contains__(self, name):
return ProjectDataset.select().where(
ProjectDataset.name == name).count() > 0
def __len__(self):
return ProjectDataset.select().count()
def __repr__(self):
if len(self) > 20:
return ("Brightway2 projects manager with {} objects, including:"
"{}\nUse `sorted(projects)` to get full list, "
"`projects.report()` to get\n\ta report on all projects."
).format(
len(self), "".join([
"\n\t{}".format(x)
for x in sorted([x.name for x in self])[:10]
]))
else:
return (
"Brightway2 projects manager with {} objects:{}"
"\nUse `projects.report()` to get a report on all projects."
).format(
len(self), "".join([
"\n\t{}".format(x) for x in sorted([x.name for x in self])
]))
### Internal functions for managing projects
def _get_base_directories(self):
eight.wrap_os_environ_io()
envvar = os.getenv("BRIGHTWAY2_DIR")
if envvar:
if not os.path.isdir(envvar):
raise OSError(("BRIGHTWAY2_DIR variable is {}, but this is not"
" a valid directory").format(envvar))
else:
print("Using environment variable BRIGHTWAY2_DIR for data "
"directory:\n{}".format(envvar))
envvar = os.path.abspath(envvar)
logs_dir = os.path.join(envvar, "logs")
create_dir(logs_dir)
return envvar, logs_dir
LABEL = "Brightway2" if sys.version_info < (3, 0) else "Brightway3"
data_dir = appdirs.user_data_dir(LABEL, "pylca")
logs_dir = appdirs.user_log_dir(LABEL, "pylca")
return data_dir, logs_dir
def _create_base_directories(self):
create_dir(self._base_data_dir)
create_dir(self._base_logs_dir)
@property
def current(self):
return self._project_name
def set_current(self, name, writable=True, update=True):
if not self.read_only and lockable() and hasattr(self, "_lock"):
try:
self._lock.release()
except (RuntimeError, ThreadError):
pass
self._project_name = str(name)
# Need to allow writes when creating a new project
# for new metadata stores
self.read_only = False
self.create_project(name)
self._reset_meta()
self._reset_sqlite3_databases()
if not lockable():
pass
elif writable:
self._lock = InterProcessLock(os.path.join(self.dir, "write-lock"))
self.read_only = not self._lock.acquire(timeout=0.05)
if self.read_only:
warnings.warn(READ_ONLY_PROJECT)
else:
self.read_only = True
if not self.read_only and update:
self._do_automatic_updates()
def _do_automatic_updates(self):
"""Run any available automatic updates"""
from .updates import Updates
for update_name in Updates.check_automatic_updates():
print("Applying automatic update: {}".format(update_name))
Updates.do_update(update_name)
def _reset_meta(self):
for obj in config.metadata:
obj.__init__()
def _reset_sqlite3_databases(self):
for relative_path, substitutable_db in config.sqlite3_databases:
substitutable_db.change_path(os.path.join(self.dir, relative_path))
### Public API
@property
def dir(self):
return os.path.join(self._base_data_dir, safe_filename(self.current))
@property
def logs_dir(self):
return os.path.join(self._base_logs_dir, safe_filename(self.current))
@property
def output_dir(self):
"""Get directory for output files.
Uses environment variable ``BRIGHTWAY2_OUTPUT_DIR``; ``preferences['output_dir']``; or directory ``output`` in current project.
Returns output directory path.
"""
eight.wrap_os_environ_io()
ep, pp = os.getenv('BRIGHTWAY2_OUTPUT_DIR'), config.p.get('output_dir')
if ep and os.path.isdir(ep):
return ep
elif pp and os.path.isdir(pp):
return pp
else:
return self.request_directory('output')
def create_project(self, name=None, **kwargs):
name = name or self.current
if not ProjectDataset.select().where(
ProjectDataset.name == name).count():
ProjectDataset.create(data=kwargs, name=name)
create_dir(self.dir)
for dir_name in self._basic_directories:
create_dir(os.path.join(self.dir, dir_name))
create_dir(self.logs_dir)
def copy_project(self, new_name, switch=True):
"""Copy current project to a new project named ``new_name``. If ``switch``, switch to new project."""
if new_name in self:
raise ValueError("Project {} already exists".format(new_name))
fp = os.path.join(self._base_data_dir, safe_filename(new_name))
if os.path.exists(fp):
raise ValueError("Project directory already exists")
project_data = ProjectDataset.select(
ProjectDataset.name == self.current).get().data
ProjectDataset.create(data=project_data, name=new_name)
shutil.copytree(self.dir, fp, ignore=lambda x, y: ["write-lock"])
create_dir(os.path.join(self._base_logs_dir, safe_filename(new_name)))
if switch:
self.set_current(new_name)
def request_directory(self, name):
"""Return the absolute path to the subdirectory ``dirname``, creating it if necessary.
Returns ``False`` if directory can't be created."""
fp = os.path.join(self.dir, str(name))
create_dir(fp)
if not os.path.isdir(fp):
return False
return fp
def _use_temp_directory(self):
"""Point the ProjectManager towards a temporary directory instead of `user_data_dir`.
Used exclusively for tests."""
if not self._is_temp_dir:
self._orig_base_data_dir = self._base_data_dir
self._orig_base_logs_dir = self._base_logs_dir
temp_dir = tempfile.mkdtemp()
self._base_data_dir = os.path.join(temp_dir, "data")
self._base_logs_dir = os.path.join(temp_dir, "logs")
self.db.change_path(':memory:')
self.set_current("default", update=False)
self._is_temp_dir = True
return temp_dir
def _restore_orig_directory(self):
"""Point the ProjectManager back to original directories.
Used exclusively in tests."""
if not self._is_temp_dir:
return
self._base_data_dir = self._orig_base_data_dir
del self._orig_base_data_dir
self._base_logs_dir = self._orig_base_logs_dir
del self._orig_base_logs_dir
self.db.change_path(os.path.join(self._base_data_dir, "projects.db"))
self.set_current("default", update=False)
self._is_temp_dir = False
def delete_project(self, name=None, delete_dir=False):
"""Delete project ``name``, or the current project.
``name`` is the project to delete. If ``name`` is not provided, delete the current project.
By default, the underlying project directory is not deleted; only the project name is removed from the list of active projects. If ``delete_dir`` is ``True``, then also delete the project directory.
If deleting the current project, this function sets the current directory to ``default`` if it exists, or to a random project.
Returns the current project."""
victim = name or self.current
if victim not in self:
raise ValueError("{} is not a project".format(victim))
if len(self) == 1:
raise ValueError("Can't delete only remaining project")
ProjectDataset.delete().where(ProjectDataset.name == victim).execute()
if delete_dir:
dir_path = os.path.join(self._base_data_dir, safe_filename(victim))
assert os.path.isdir(dir_path), "Can't find project directory"
shutil.rmtree(dir_path)
if name is None or name == self.current:
if "default" in self:
self.set_current("default")
else:
self.set_current(next(iter(self)).name)
return self.current
def purge_deleted_directories(self):
"""Delete project directories for projects which are no longer registered.
Returns number of directories deleted."""
registered = {safe_filename(obj.name) for obj in self}
bad_directories = [
os.path.join(self._base_data_dir, dirname)
for dirname in os.listdir(self._base_data_dir)
if os.path.isdir(os.path.join(self._base_data_dir, dirname))
and dirname not in registered
]
for fp in bad_directories:
shutil.rmtree(fp)
return len(bad_directories)
def report(self):
"""Give a report on current projects, including installed databases and file sizes.
Returns tuples of ``(project name, number of databases, size of all databases (GB))``."""
from . import databases
_current = self.current
data = []
def get_dir_size(dirpath):
"""Modified from http://stackoverflow.com/questions/12480367/how-to-generate-directory-size-recursively-in-python-like-du-does.
Does not follow symbolic links"""
return sum(
sum(
os.path.getsize(os.path.join(root, name))
for name in files)
for root, dirs, files in os.walk(dirpath))
names = sorted([x.name for x in self])
for obj in names:
self.set_current(obj, update=False, writable=False)
data.append(
(obj, len(databases), get_dir_size(projects.dir) / 1e9))
self.set_current(_current)
return data
class ProjectManager(Iterable):
_basic_directories = (
"backups",
"intermediate",
"lci",
"processed",
)
_is_temp_dir = False
read_only = False
def __init__(self):
self._base_data_dir, self._base_logs_dir = self._get_base_directories()
self._create_base_directories()
self.db = SubstitutableDatabase(self._base_data_dir / "projects.db",
[ProjectDataset])
columns = {
o.name
for o in self.db._database.get_columns("projectdataset")
}
if "full_hash" not in columns:
src_filepath = self._base_data_dir / "projects.db"
backup_filepath = self._base_data_dir / "projects.backup.db"
shutil.copy(src_filepath, backup_filepath)
MIGRATION_WARNING = """Adding a column to the projects database. A backup copy of this database '{}' was made at '{}'; if you have problems, file an issue, and restore the backup data to use the stable version of Brightway2."""
print(MIGRATION_WARNING.format(src_filepath, backup_filepath))
ADD_FULL_HASH_COLUMN = """ALTER TABLE projectdataset ADD COLUMN "full_hash" integer default 1"""
self.db.execute_sql(ADD_FULL_HASH_COLUMN)
# We don't do this, as the column added doesn't have a default
# value, meaning that one would get error from using the
# development branch alongside the stable branch.
# from playhouse.migrate import SqliteMigrator, migrate
# migrator = SqliteMigrator(self.db._database)
# full_hash = BooleanField(default=True)
# migrate(migrator.add_column("projectdataset", "full_hash", full_hash),)
self.set_current("default", update=False)
def __iter__(self):
for project_ds in ProjectDataset.select():
yield project_ds
def __contains__(self, name):
return ProjectDataset.select().where(
ProjectDataset.name == name).count() > 0
def __len__(self):
return ProjectDataset.select().count()
def __repr__(self):
if len(self) > 20:
return ("Brightway2 projects manager with {} objects, including:"
"{}\nUse `sorted(projects)` to get full list, "
"`projects.report()` to get\n\ta report on all projects."
).format(
len(self),
"".join([
"\n\t{}".format(x)
for x in sorted([x.name for x in self])[:10]
]),
)
else:
return (
"Brightway2 projects manager with {} objects:{}"
"\nUse `projects.report()` to get a report on all projects."
).format(
len(self),
"".join([
"\n\t{}".format(x) for x in sorted([x.name for x in self])
]),
)
### Internal functions for managing projects
def _get_base_directories(self):
envvar = maybe_path(os.getenv("BRIGHTWAY2_DIR"))
if envvar:
if not envvar.is_dir():
raise OSError(("BRIGHTWAY2_DIR variable is {}, but this is not"
" a valid directory").format(envvar))
else:
print("Using environment variable BRIGHTWAY2_DIR for data "
"directory:\n{}".format(envvar))
envvar = envvar.absolute()
logs_dir = envvar / "logs"
create_dir(logs_dir)
return envvar, logs_dir
LABEL = "Brightway3"
data_dir = Path(appdirs.user_data_dir(LABEL, "pylca"))
logs_dir = Path(appdirs.user_log_dir(LABEL, "pylca"))
return data_dir, logs_dir
def _create_base_directories(self):
create_dir(self._base_data_dir)
create_dir(self._base_logs_dir)
@property
def current(self):
return self._project_name
@property
def twofive(self):
return bool(self.dataset.data.get("25"))
def set_current(self, name, writable=True, update=True):
if not self.read_only and lockable() and hasattr(self, "_lock"):
try:
self._lock.release()
except (RuntimeError, ThreadError):
pass
self._project_name = str(name)
# Need to allow writes when creating a new project
# for new metadata stores
self.read_only = False
self.create_project(name)
self.dataset = ProjectDataset.get(
ProjectDataset.name == self._project_name)
self._reset_meta()
self._reset_sqlite3_databases()
self.dataset = ProjectDataset.get(name=name)
if not lockable():
pass
elif writable:
self._lock = InterProcessLock(self.dir / "write-lock")
self.read_only = not self._lock.acquire(timeout=0.05)
if self.read_only:
warnings.warn(READ_ONLY_PROJECT)
else:
self.read_only = True
if not self.read_only and update:
self._do_automatic_updates()
def _do_automatic_updates(self):
"""Run any available automatic updates"""
from .updates import Updates
for update_name in Updates.check_automatic_updates():
print("Applying automatic update: {}".format(update_name))
Updates.do_update(update_name)
def _reset_meta(self):
for obj in config.metadata:
obj.__init__()
def _reset_sqlite3_databases(self):
for relative_path, substitutable_db in config.sqlite3_databases:
substitutable_db.change_path(self.dir / relative_path)
### Public API
@property
def dir(self):
return Path(self._base_data_dir) / safe_filename(
self.current, full=self.dataset.full_hash)
@property
def logs_dir(self):
return Path(self._base_logs_dir) / safe_filename(
self.current, full=self.dataset.full_hash)
@property
def output_dir(self):
"""Get directory for output files.
Uses environment variable ``BRIGHTWAY2_OUTPUT_DIR``; ``preferences['output_dir']``; or directory ``output`` in current project.
Returns output directory path.
"""
ep, pp = (
maybe_path(os.getenv("BRIGHTWAY2_OUTPUT_DIR")),
maybe_path(config.p.get("output_dir")),
)
if ep and ep.is_dir():
return ep
elif pp and pp.is_dir():
return pp
else:
return self.request_directory("output")
def create_project(self, name=None, **kwargs):
name = name or self.current
kwargs["25"] = True
full_hash = kwargs.pop("full_hash", False)
try:
self.dataset = ProjectDataset.get(ProjectDataset.name == name)
except DoesNotExist:
self.dataset = ProjectDataset.create(data=kwargs,
name=name,
full_hash=full_hash)
create_dir(self.dir)
for dir_name in self._basic_directories:
create_dir(self.dir / dir_name)
create_dir(self.logs_dir)
def copy_project(self, new_name, switch=True):
"""Copy current project to a new project named ``new_name``. If ``switch``, switch to new project."""
if new_name in self:
raise ValueError("Project {} already exists".format(new_name))
fp = self._base_data_dir / safe_filename(new_name,
full=self.dataset.full_hash)
if fp.exists():
raise ValueError("Project directory already exists")
project_data = ProjectDataset.get(
ProjectDataset.name == self.current).data
ProjectDataset.create(data=project_data,
name=new_name,
full_hash=self.dataset.full_hash)
shutil.copytree(self.dir, fp, ignore=lambda x, y: ["write-lock"])
create_dir(self._base_logs_dir / safe_filename(new_name))
if switch:
self.set_current(new_name)
def request_directory(self, name):
"""Return the absolute path to the subdirectory ``dirname``, creating it if necessary.
Returns ``False`` if directory can't be created."""
fp = self.dir / str(name)
create_dir(fp)
if not fp.is_dir():
return False
return fp
def _use_temp_directory(self):
"""Point the ProjectManager towards a temporary directory instead of `user_data_dir`.
Used exclusively for tests."""
if not self._is_temp_dir:
self._orig_base_data_dir = self._base_data_dir
self._orig_base_logs_dir = self._base_logs_dir
temp_dir = Path(tempfile.mkdtemp())
self._base_data_dir = temp_dir / "data"
self._base_logs_dir = temp_dir / "logs"
self.db.change_path(":memory:")
self.set_current("default", update=False)
self._is_temp_dir = True
return temp_dir
def _restore_orig_directory(self):
"""Point the ProjectManager back to original directories.
Used exclusively in tests."""
if not self._is_temp_dir:
return
self._base_data_dir = self._orig_base_data_dir
del self._orig_base_data_dir
self._base_logs_dir = self._orig_base_logs_dir
del self._orig_base_logs_dir
self.db.change_path(self._base_data_dir / "projects.db")
self.set_current("default", update=False)
self._is_temp_dir = False
def migrate_project_25(self):
"""Migrate project to Brightway 2.5.
Reprocesses all databases and LCIA objects."""
assert not self.twofive, "Project is already 2.5 compatible"
from .updates import Updates
Updates()._reprocess_all()
self.dataset.data["25"] = True
self.dataset.save()
def delete_project(self, name=None, delete_dir=False):
"""Delete project ``name``, or the current project.
``name`` is the project to delete. If ``name`` is not provided, delete the current project.
By default, the underlying project directory is not deleted; only the project name is removed from the list of active projects. If ``delete_dir`` is ``True``, then also delete the project directory.
If deleting the current project, this function sets the current directory to ``default`` if it exists, or to a random project.
Returns the current project."""
victim = name or self.current
if victim not in self:
raise ValueError("{} is not a project".format(victim))
if len(self) == 1:
raise ValueError("Can't delete only remaining project")
ProjectDataset.delete().where(ProjectDataset.name == victim).execute()
if delete_dir:
dir_path = self._base_data_dir / safe_filename(victim)
assert dir_path.is_dir(), "Can't find project directory"
shutil.rmtree(dir_path)
if name is None or name == self.current:
if "default" in self:
self.set_current("default")
else:
self.set_current(next(iter(self)).name)
return self.current
def purge_deleted_directories(self):
"""Delete project directories for projects which are no longer registered.
Returns number of directories deleted."""
registered = {safe_filename(obj.name) for obj in self}
bad_directories = [
self._base_data_dir / dirname
for dirname in os.listdir(self._base_data_dir)
if (self._base_data_dir /
dirname).is_dir() and dirname not in registered
]
for fp in bad_directories:
shutil.rmtree(fp)
return len(bad_directories)
def report(self):
"""Give a report on current projects, including installed databases and file sizes.
Returns tuples of ``(project name, number of databases, size of all databases (GB))``."""
from . import databases
_current = self.current
data = []
def get_dir_size(dirpath):
"""Modified from http://stackoverflow.com/questions/12480367/how-to-generate-directory-size-recursively-in-python-like-du-does.
Does not follow symbolic links"""
return sum(
sum(os.path.getsize(root / name) for name in files)
for root, dirs, files in os.walk(dirpath))
names = sorted([x.name for x in self])
for obj in names:
self.set_current(obj, update=False, writable=False)
data.append(
(obj, len(databases), get_dir_size(projects.dir) / 1e9))
self.set_current(_current)
return data
def use_short_hash(self):
if not self.dataset.full_hash:
return
try:
old_dir, old_logs_dir = self.dir, self.logs_dir
self.dataset.full_hash = False
if self.dir.exists():
raise OSError("Target directory {} already exists".format(
self.dir))
if self.logs_dir.exists():
raise OSError("Target directory {} already exists".format(
self.logs_dir))
old_dir.rename(self.dir)
old_logs_dir.rename(self.logs_dir)
self.dataset.save()
except Exception as ex:
self.dataset.full_hash = True
raise ex
def use_full_hash(self):
if self.dataset.full_hash:
return
try:
old_dir, old_logs_dir = self.dir, self.logs_dir
self.dataset.full_hash = True
if self.dir.exists():
raise OSError("Target directory {} already exists".format(
self.dir))
if self.logs_dir.exists():
raise OSError("Target directory {} already exists".format(
self.logs_dir))
old_dir.rename(self.dir)
old_logs_dir.rename(self.logs_dir)
self.dataset.save()
except Exception as ex:
self.dataset.full_hash = False
raise ex
if not mailpile_user or mailpile_user == 'root':
usage(2, "Please specify a (non-root) user to launch Mailpile.")
if not re.match(r'^[a-zA-Z0-9\._-]+$', mailpile_user):
usage(2, "That is a strange looking username.")
mailpile_user = pwd.getpwnam(mailpile_user)
if not mailpile_user:
usage(2, "Please specify a (non-root) user to launch Mailpile.")
if not re.match(r'^[a-zA-Z0-9\.:/]+$', url):
usage(2, "That is a strange looking URL.")
mailpile_home = MAILPILE_HOME_PATH % mailpile_user.pw_dir
if not os.path.exists(mailpile_home):
usage(3, "That user has never run Mailpile. Aborting.")
mp_lockfile = os.path.join(mailpile_home, MAILPILE_WORK_LOCK)
mp_lock = InterProcessLock(mp_lockfile)
if not mp_lock.acquire(blocking=False):
# We are happy with this result, don't raise an error.
sys.stderr.write(
"Mailpile is already running for that user. Doing Nothing.\n")
else:
# We will release the lock on exec(), but make sure the user owns
# the lockfile and will be able to take over.
os.chown(mp_lockfile, mailpile_user.pw_uid, mailpile_user.pw_gid)
os.execv('/bin/su',
['/bin/su', '-', mailpile_user.pw_name, '-c', (
'screen -S mailpile -d -m '
'mailpile --idlequit=%d --pid=%s/%s.pid --www=%s --interact'
) % (idlequit, MAILPILE_PIDS_PATH, mailpile_user.pw_name, url)])
class Lock:
"""
A inter-process and inter-thread lock. This reuses uses code from oslo.concurrency
but provides non-blocking acquire. Use the :meth:`singleton` class method to
retrieve an existing instance for thread-safe usage.
"""
_instances: Dict[str, "Lock"] = dict()
_singleton_lock = threading.Lock()
@classmethod
def singleton(cls, name: str, lock_path: Optional[str] = None) -> "Lock":
"""
Retrieve an existing lock object with a given 'name' or create a new one. Use
this method for thread-safe locks.
:param name: Name of lock file.
:param lock_path: Directory for lock files. Defaults to the temporary directory
returned by :func:`tempfile.gettempdir()` if not given.
"""
with cls._singleton_lock:
try:
instance = cls._instances[name]
except KeyError:
instance = cls(name, lock_path)
cls._instances[name] = instance
return instance
def __init__(self, name: str, lock_path: Optional[str] = None) -> None:
self.name = name
dirname = lock_path or tempfile.gettempdir()
lock_path = os.path.join(dirname, name)
self._internal_lock = threading.Semaphore()
self._external_lock = InterProcessLock(lock_path)
self._lock = threading.RLock()
def acquire(self) -> bool:
"""
Attempts to acquire the given lock.
:returns: Whether or not the acquisition succeeded.
"""
with self._lock:
locked_internal = self._internal_lock.acquire(blocking=False)
if not locked_internal:
return False
try:
locked_external = self._external_lock.acquire(blocking=False)
except Exception:
self._internal_lock.release()
raise
else:
if locked_external:
return True
else:
self._internal_lock.release()
return False
def release(self) -> None:
"""Release the previously acquired lock."""
with self._lock:
self._external_lock.release()
self._internal_lock.release()
def locked(self) -> bool:
"""Checks if the lock is currently held by any thread or process."""
with self._lock:
gotten = self.acquire()
if gotten:
self.release()
return not gotten
def locking_pid(self) -> Optional[int]:
"""
Returns the PID of the process which currently holds the lock or ``None``. This
should work on macOS, OpenBSD and Linux but may fail on some platforms. Always
use :meth:`locked` to check if the lock is held by any process.
:returns: The PID of the process which currently holds the lock or ``None``.
"""
with self._lock:
if self._external_lock.acquired:
return os.getpid()
try:
# don't close again in case we are the locking process
self._external_lock._do_open()
lockdata, fmt, pid_index = _get_lockdata()
lockdata = fcntl.fcntl(self._external_lock.lockfile,
fcntl.F_GETLK, lockdata)
lockdata_list = struct.unpack(fmt, lockdata)
pid = lockdata_list[pid_index]
if pid > 0:
return pid
except OSError:
pass
return None
from apscheduler.schedulers import SchedulerNotRunningError # type: ignore
from apscheduler.triggers.date import DateTrigger # type: ignore
from fasteners import InterProcessLock # type: ignore
from deck_chores import __version__
from deck_chores.config import cfg, generate_config
from deck_chores.exceptions import ConfigurationError
from deck_chores.indexes import locking_container_to_services_map
from deck_chores import jobs
import deck_chores.parsers as parse
from deck_chores.utils import from_json, generate_id, log, log_handler
####
lock = InterProcessLock('/tmp/deck-chores.lock')
def there_is_another_deck_chores_container() -> bool:
matched_containers = 0
for container in cfg.client.containers.list():
if container.image.labels.get('org.label-schema.name',
'') == 'deck-chores':
matched_containers += 1
if matched_containers > 1:
return True
return False
####
def main(argv=None):
'''Command line options.'''
# setup argparser
parser = argparse.ArgumentParser()
parser.add_argument('--version',
action='version',
version='%s v%s (%s)' %
(os.path.basename(__file__), __version__, __updated__))
# Actions
action_group = parser.add_argument_group(
"Actions", "A single action must be selected.")
actions = action_group.add_mutually_exclusive_group(required=True)
actions.add_argument("--report",
dest="report",
type=str,
help="Submit the given textual report",
metavar="TEXT")
actions.add_argument("--report-from-file",
dest="report_file",
type=str,
help="Submit the given file as textual report",
metavar="FILE")
actions.add_argument("--cycle",
dest="cycle",
type=str,
help="Cycle the pool with the given ID",
metavar="ID")
actions.add_argument("--disable",
dest="disable",
type=str,
help="Disable the pool with the given ID",
metavar="ID")
actions.add_argument("--enable",
dest="enable",
type=str,
help="Enable the pool with the given ID",
metavar="ID")
# Options
parser.add_argument(
"--keep-reporting",
dest="keep_reporting",
default=0,
type=int,
help="Keep reporting from the specified file with specified interval",
metavar="SECONDS")
parser.add_argument("--random-offset",
dest="random_offset",
default=0,
type=int,
help="Random offset for the reporting interval (+/-)",
metavar="SECONDS")
# Settings
parser.add_argument("--serverhost",
dest="serverhost",
help="Server hostname for remote signature management",
metavar="HOST")
parser.add_argument("--serverport",
dest="serverport",
type=int,
help="Server port to use",
metavar="PORT")
parser.add_argument("--serverproto",
dest="serverproto",
help="Server protocol to use (default is https)",
metavar="PROTO")
parser.add_argument("--serverauthtokenfile",
dest="serverauthtokenfile",
help="File containing the server authentication token",
metavar="FILE")
parser.add_argument("--clientid",
dest="clientid",
help="Client ID to use when submitting issues",
metavar="ID")
# process options
opts = parser.parse_args(argv)
if opts.keep_reporting and not opts.report_file:
print("Error: --keep-reporting is only valid with --report-from-file",
file=sys.stderr)
return 2
serverauthtoken = None
if opts.serverauthtokenfile:
with open(opts.serverauthtokenfile) as f:
serverauthtoken = f.read().rstrip()
reporter = EC2Reporter(opts.serverhost, opts.serverport, opts.serverproto,
serverauthtoken, opts.clientid)
report = None
if opts.cycle:
reporter.cycle(opts.cycle)
return 0
elif opts.enable:
reporter.enable(opts.enable)
return 0
elif opts.disable:
reporter.disable(opts.disable)
return 0
elif opts.report_file:
if opts.keep_reporting:
if opts.random_offset > 0:
random.seed(reporter.clientId)
lock = InterProcessLock(opts.report_file + ".lock")
while True:
if os.path.exists(opts.report_file):
if not lock.acquire(timeout=opts.keep_reporting):
continue
try:
with open(opts.report_file) as f:
report = f.read()
try:
reporter.report(report)
except RuntimeError as e:
# Ignore errors if the server is temporarily unavailable
print("Failed to contact server: %s" % e,
file=sys.stderr)
finally:
lock.release()
random_offset = 0
if opts.random_offset:
random_offset = random.randint(-opts.random_offset,
opts.random_offset)
time.sleep(opts.keep_reporting + random_offset)
else:
with open(opts.report_file) as f:
report = f.read()
else:
report = opts.report
reporter.report(report)
return 0
def write_aggregated_stats(base_dirs, outfile):
'''
Generate aggregated statistics from the given base directories
and write them to the specified output file.
@type base_dirs: list
@param base_dirs: List of AFL base directories
@type outfile: str
@param outfile: Output file for aggregated statistics
'''
# Which fields to add
wanted_fields_total = [
'execs_done', 'execs_per_sec', 'pending_favs', 'pending_total',
'variable_paths', 'unique_crashes', 'unique_hangs'
]
# Which fields to aggregate by mean
wanted_fields_mean = ['exec_timeout']
# Which fields should be displayed per fuzzer instance
wanted_fields_all = ['cycles_done', 'bitmap_cvg']
# Which fields should be aggregated by max
wanted_fields_max = ['last_path']
aggregated_stats = {}
for field in wanted_fields_total:
aggregated_stats[field] = 0
for field in wanted_fields_mean:
aggregated_stats[field] = (0, 0)
for field in wanted_fields_all:
aggregated_stats[field] = []
def convert_num(num):
if '.' in num:
return float(num)
return int(num)
for base_dir in base_dirs:
stats_path = os.path.join(base_dir, "fuzzer_stats")
if os.path.exists(stats_path):
with open(stats_path, 'r') as stats_file:
stats = stats_file.read()
for line in stats.splitlines():
(field_name, field_val) = line.split(':', 1)
field_name = field_name.strip()
field_val = field_val.strip()
if field_name in wanted_fields_total:
aggregated_stats[field_name] += convert_num(field_val)
elif field_name in wanted_fields_mean:
(val, cnt) = aggregated_stats[field_name]
aggregated_stats[field_name] = (val +
convert_num(field_val),
cnt + 1)
elif field_name in wanted_fields_all:
aggregated_stats[field_name].append(field_val)
elif field_name in wanted_fields_max:
num_val = convert_num(field_val)
if (not field_name in aggregated_stats
) or aggregated_stats[field_name] < num_val:
aggregated_stats[field_name] = num_val
# If we don't have any data here, then the fuzzers haven't written any statistics yet
if not aggregated_stats:
return
# Mean conversion
for field_name in wanted_fields_mean:
(val, cnt) = aggregated_stats[field_name]
if cnt:
aggregated_stats[field_name] = float(val) / float(cnt)
else:
aggregated_stats[field_name] = val
# Write out data
fields = []
fields.extend(wanted_fields_total)
fields.extend(wanted_fields_mean)
fields.extend(wanted_fields_all)
fields.extend(wanted_fields_max)
max_keylen = max([len(x) for x in fields])
with InterProcessLock(outfile + ".lock"), open(outfile, 'w') as f:
for field in fields:
if not field in aggregated_stats:
continue
val = aggregated_stats[field]
if isinstance(val, list):
val = " ".join(val)
f.write("%s%s: %s\n" % (field, " " *
(max_keylen + 1 - len(field)), val))
return
def test_try_lock_informatively(tempfile=None):
lock = InterProcessLock(tempfile + '.lck')
lock_path = ensure_unicode(
lock.path) # can be bytes, complicates string formattingetc
t0 = time()
with try_lock_informatively(lock, purpose="happy life") as acquired:
assert_true(lock.acquired)
assert_true(acquired)
assert_greater(
2,
time() -
t0) # should not take any notable time, we cannot be blocking
"""
# InterProcessLock is not re-entrant so nesting should not be used, will result
# in exception on release
with try_lock_informatively(lock, timeouts=[dt, dt*2], proceed_unlocked=True) as acquired:
assert_true(lock.acquired) # due to outer cm
assert_true(acquired) # lock is reentrant apparently
"""
# Let's try in a completely different subprocess
runner = WitlessRunner(env=dict(
os.environ, DATALAD_LOG_LEVEL='info', DATALAD_LOG_TARGET='stderr'))
script1 = Path(tempfile + "-script1.py")
script1_fmt = f"""
from fasteners import InterProcessLock
from time import time
from datalad.support.locking import try_lock_informatively
lock = InterProcessLock({lock_path!r})
with try_lock_informatively(lock, timeouts=[0.05, 0.15], proceed_unlocked={{proceed_unlocked}}) as acquired:
print("Lock acquired=%s" % acquired)
"""
script1.write_text(script1_fmt.format(proceed_unlocked=True))
t0 = time()
res = runner.run([sys.executable, str(script1)],
protocol=StdOutErrCapture)
assert_in('Lock acquired=False', res['stdout'])
assert_in(f'Failed to acquire lock at {lock_path} in 0.05',
res['stderr'])
assert_in(f'Failed to acquire lock at {lock_path} in 0.15',
res['stderr'])
assert_in('proceed without locking', res['stderr'])
assert_greater(time() - t0, 0.19999) # should wait for at least 0.2
try:
import psutil
# PID does not correspond
assert_in('Check following process: PID=', res['stderr'])
assert_in(f'CWD={os.getcwd()} CMDLINE=', res['stderr'])
except ImportError:
pass # psutil was not installed, cannot get list of files
except AssertionError:
# we must have had the other one then
assert_in('failed to determine one', res['stderr'])
if not on_osx:
# so far we had only OSX reporting failing to get PIDs information
# but if it is something else -- re-raise original exception
raise
# in 2nd case, lets try without proceeding unlocked
script1.write_text(script1_fmt.format(proceed_unlocked=False))
t0 = time()
with assert_raises(CommandError) as cme:
runner.run([sys.executable, str(script1)],
protocol=StdOutErrCapture)
assert_in(f"Failed to acquire lock at {lock_path} in 2 attempts.",
str(cme.value))
assert_in(f"RuntimeError", str(cme.value))
assert_false(
cme.value.stdout) # nothing there since print should not happen
assert_in(f'Failed to acquire lock at {lock_path} in 0.05',
cme.value.stderr)
assert_in(f'Failed to acquire lock at {lock_path} in 0.15',
cme.value.stderr)
assert_greater(time() - t0, 0.19999) # should wait for at least 0.2
# now that we left context, should work out just fine
res = runner.run([sys.executable, str(script1)], protocol=StdOutErrCapture)
assert_in('Lock acquired=True', res['stdout'])
assert_not_in(f'Failed to acquire lock', res['stderr'])
assert_not_in('PID', res['stderr'])
if not mailpile_user or mailpile_user == 'root':
usage(2, "Please specify a (non-root) user to launch Mailpile.")
if not re.match(r'^[a-zA-Z0-9\._-]+$', mailpile_user):
usage(2, "That is a strange looking username.")
mailpile_user = pwd.getpwnam(mailpile_user)
if not mailpile_user:
usage(2, "Please specify a (non-root) user to launch Mailpile.")
if not re.match(r'^[a-zA-Z0-9\.:/]+$', url):
usage(2, "That is a strange looking URL.")
mailpile_home = MAILPILE_HOME_PATH % mailpile_user.pw_dir
if not os.path.exists(mailpile_home):
usage(3, "That user has never run Mailpile. Aborting.")
mp_lockfile = os.path.join(mailpile_home, MAILPILE_WORK_LOCK)
mp_lock = InterProcessLock(mp_lockfile)
if not mp_lock.acquire(blocking=False):
# We are happy with this result, don't raise an error.
sys.stderr.write(
"Mailpile is already running for that user. Doing Nothing.\n")
else:
# We will release the lock on exec(), but make sure the user owns
# the lockfile and will be able to take over.
os.chown(mp_lockfile, mailpile_user.pw_uid, mailpile_user.pw_gid)
os.execv('/bin/su', [
'/bin/su', '-', mailpile_user.pw_name, '-c',
('screen -S mailpile -d -m '
'mailpile --idlequit=%d --pid=%s/%s.pid --www=%s --interact') %
(idlequit, MAILPILE_PIDS_PATH, mailpile_user.pw_name, url)
])
def write_aggregated_stats(base_dirs, outfile, cmdline_path=None):
'''
Generate aggregated statistics from the given base directories
and write them to the specified output file.
@type base_dirs: list
@param base_dirs: List of AFL base directories
@type outfile: str
@param outfile: Output file for aggregated statistics
@type cmdline_path: String
@param cmdline_path: Optional command line file to use instead of the
one found inside the base directory.
'''
# Which fields to add
wanted_fields_total = [
'execs_done',
'execs_per_sec',
'pending_favs',
'pending_total',
'variable_paths',
'unique_crashes',
'unique_hangs']
# Which fields to aggregate by mean
wanted_fields_mean = ['exec_timeout']
# Which fields should be displayed per fuzzer instance
wanted_fields_all = ['cycles_done', 'bitmap_cvg']
# Which fields should be aggregated by max
wanted_fields_max = ['last_path']
# Warnings to include
warnings = list()
aggregated_stats = {}
for field in wanted_fields_total:
aggregated_stats[field] = 0
for field in wanted_fields_mean:
aggregated_stats[field] = (0, 0)
for field in wanted_fields_all:
aggregated_stats[field] = []
def convert_num(num):
if '.' in num:
return float(num)
return int(num)
for base_dir in base_dirs:
stats_path = os.path.join(base_dir, "fuzzer_stats")
if not cmdline_path:
cmdline_path = os.path.join(base_dir, "cmdline")
if os.path.exists(stats_path):
with open(stats_path, 'r') as stats_file:
stats = stats_file.read()
for line in stats.splitlines():
(field_name, field_val) = line.split(':', 1)
field_name = field_name.strip()
field_val = field_val.strip()
if field_name in wanted_fields_total:
aggregated_stats[field_name] += convert_num(field_val)
elif field_name in wanted_fields_mean:
(val, cnt) = aggregated_stats[field_name]
aggregated_stats[field_name] = (val + convert_num(field_val), cnt + 1)
elif field_name in wanted_fields_all:
aggregated_stats[field_name].append(field_val)
elif field_name in wanted_fields_max:
num_val = convert_num(field_val)
if (field_name not in aggregated_stats) or aggregated_stats[field_name] < num_val:
aggregated_stats[field_name] = num_val
# If we don't have any data here, then the fuzzers haven't written any statistics yet
if not aggregated_stats:
return
# Mean conversion
for field_name in wanted_fields_mean:
(val, cnt) = aggregated_stats[field_name]
if cnt:
aggregated_stats[field_name] = float(val) / float(cnt)
else:
aggregated_stats[field_name] = val
# Verify fuzzmanagerconf exists and can be parsed
_, cmdline = command_file_to_list(cmdline_path)
target_binary = cmdline[0] if cmdline else None
if target_binary is not None:
if not os.path.isfile("%s.fuzzmanagerconf" % target_binary):
warnings.append("WARNING: Missing %s.fuzzmanagerconf\n" % target_binary)
elif ProgramConfiguration.fromBinary(target_binary) is None:
warnings.append("WARNING: Invalid %s.fuzzmanagerconf\n" % target_binary)
# Look for unreported crashes
failed_reports = 0
for base_dir in base_dirs:
crashes_dir = os.path.join(base_dir, "crashes")
if not os.path.isdir(crashes_dir):
continue
for crash_file in os.listdir(crashes_dir):
if crash_file.endswith(".failed"):
failed_reports += 1
if failed_reports:
warnings.append("WARNING: Unreported crashes detected (%d)\n" % failed_reports)
# Write out data
fields = []
fields.extend(wanted_fields_total)
fields.extend(wanted_fields_mean)
fields.extend(wanted_fields_all)
fields.extend(wanted_fields_max)
max_keylen = max([len(x) for x in fields])
with InterProcessLock(outfile + ".lock"), open(outfile, 'w') as f:
for field in fields:
if field not in aggregated_stats:
continue
val = aggregated_stats[field]
if isinstance(val, list):
val = " ".join(val)
f.write("%s%s: %s\n" % (field, " " * (max_keylen + 1 - len(field)), val))
for warning in warnings:
f.write(warning)
return
class AdaptiveLock:
def __init__(self, timeout: int = 180):
self.timeout = timeout
self.methods: List[str] = ["fcntl", "hard_links", "delay"]
self.lock_instance = None
def lock(self, lock_file):
if self.methods[0] == "hard_links":
self.lock_instance = FluflLock(
lock_file, lifetime=self.timeout
) # seconds after which the lock is broken
try:
self.lock_instance.lock(
timeout=self.timeout) # try for a long time
return
except (FluflLockError, TimeOutError): # timeouts etc.
pass
except OSError: # such as PermissionError
pass
logger.warning("Unable to use hard link-based file locks",
exc_info=True)
self.methods.pop(0)
self.lock(lock_file)
elif self.methods[0] == "fcntl":
self.lock_instance = FcntlLock(lock_file)
acquired = self.lock_instance.acquire(timeout=self.timeout,
delay=1)
if acquired:
return
logger.warning("Unable to use fcntl-based file locks",
exc_info=True)
self.methods.pop(0)
self.lock(lock_file)
else:
# use a random delay to make write collisions unlikely
delay = gauss(20, 5)
if delay > 0:
sleep(delay)
def unlock(self):
if self.methods[0] == "hard_links":
assert isinstance(self.lock_instance, FluflLock)
self.lock_instance.unlock(
unconditionally=True) # do not raise errors in unlock
self.lock_instance = None
elif self.methods[0] == "fcntl":
assert isinstance(self.lock_instance, FcntlLock)
self.lock_instance.release()
def acquire_lock(self):
with InterProcessLock(self.storagepath / 'lockfile'):
yield
class DownloadDirectory:
def __init__(self, filepath, digests):
#: The path to which to save the file after downloading
self.filepath = Path(filepath)
#: Expected hashes of the downloaded data, as a mapping from algorithm
#: names to digests
self.digests = digests
#: The working directory in which downloaded data will be temporarily
#: stored
self.dirpath = self.filepath.with_name(self.filepath.name +
".dandidownload")
#: The file in `dirpath` to which data will be written as it is
#: received
self.writefile = self.dirpath / "file"
#: A `fasteners.InterProcessLock` on `dirpath`
self.lock = None
#: An open filehandle to `writefile`
self.fp = None
#: How much of the data has been downloaded so far
self.offset = None
def __enter__(self):
from fasteners import InterProcessLock
self.dirpath.mkdir(parents=True, exist_ok=True)
self.lock = InterProcessLock(str(self.dirpath / "lock"))
if not self.lock.acquire(blocking=False):
raise RuntimeError(
"Could not acquire download lock for {self.filepath}")
chkpath = self.dirpath / "checksum"
try:
with chkpath.open() as fp:
digests = json.load(fp)
except (FileNotFoundError, ValueError):
digests = {}
matching_algs = self.digests.keys() & digests.keys()
if matching_algs and all(self.digests[alg] == digests[alg]
for alg in matching_algs):
# Pick up where we left off, writing to the end of the file
lgr.debug(
"Download directory exists and has matching checksum; resuming download"
)
self.fp = self.writefile.open("ab")
else:
# Delete the file (if it even exists) and start anew
if not chkpath.exists():
lgr.debug("Starting new download in new download directory")
else:
lgr.debug(
"Download directory found, but digests do not match; starting new download"
)
try:
self.writefile.unlink()
except FileNotFoundError:
pass
self.fp = self.writefile.open("wb")
with chkpath.open("w") as fp:
json.dump(self.digests, fp)
self.offset = self.fp.tell()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.fp.close()
try:
if exc_type is None:
self.writefile.replace(self.filepath)
finally:
self.lock.release()
if exc_type is None:
rmtree(self.dirpath, ignore_errors=True)
self.lock = None
self.fp = None
self.offset = None
return False
def append(self, blob):
self.fp.write(blob)
from config import config
from fasteners import InterProcessLock
ipl = InterProcessLock(config['lock_file'])
def lock_obtained():
print('Lock obtained, starting YeenBot...')
from yeenbot import main
main()
def lock_not_obtained():
print('Lock not obtained, exiting')
# Attempt to acquire the lock
acquired = ipl.acquire(blocking=False)
try:
if acquired:
lock_obtained()
else:
lock_not_obtained()
finally:
if acquired:
ipl.release()
class AdaptiveLock:
timeout = 600
def __init__(self):
self.method = "hard_links"
self.lock_instance = None
self.delay = 0.0
def lock(self, lock_file):
if self.method == "hard_links":
self.lock_instance = FluflLock(
lock_file,
lifetime=60) # seconds after which the lock is broken
try:
self.lock_instance.lock(
timeout=self.timeout) # try for a long time
return
except TimeoutError:
pass
except OSError:
pass
except PermissionError:
pass
logger.warning(
"Unable to use hard link-based file locks. "
"Trying fcntl-based file locks",
exc_info=True)
self.method = "fcntl"
if self.method == "fcntl":
self.lock_instance = FcntlLock(lock_file)
acquired = self.lock_instance.acquire(timeout=self.timeout)
if acquired:
return
logger.warning(
"Unable to use fcntl-based file locks. "
"Disabling file locks",
exc_info=True)
self.method = None
if self.method is None:
# use a random delay to make write collisions unlikely
delay = gauss(10.0, 2.5)
if delay > 0:
sleep(delay)
def unlock(self):
if self.method == "hard_links":
assert isinstance(self.lock_instance, FluflLock)
self.lock_instance.unlock(
unconditionally=True) # do not raise errors in unlock
self.lock_instance = None
elif self.method == "fcntl":
assert isinstance(self.lock_instance, FcntlLock)
self.lock_instance.release()
class Model:
"""The mod management model"""
def __init__(self,
gamePath: Path,
configPath: Path,
cachePath: Path,
ignorelock: bool = False) -> None:
self._gamePath: Path = Path()
self._configPath: Path = Path()
self._cachePath: Path = Path()
self._modsPath: Path = Path()
self._dlcsPath: Path = Path()
self._modList: Dict[Tuple[str, str], Mod] = {}
_cachePath = verifyCachePath(cachePath)
if not _cachePath:
raise InvalidCachePath(cachePath)
self._cachePath = _cachePath
self.updateCallbacks = CallbackList()
self.updateLock = asyncio.Lock()
self.conflicts = ModelConflicts()
self._pool = ProcessPoolExecutor()
self._iteration = 0
if not ignorelock:
self._lock = InterProcessLock(self.lockfile)
if not self._lock.acquire(False):
raise OtherInstanceError(self.lockfile)
self.setPaths(gamePath, configPath)
# TODO: enhancement: watch mod directory for changes
logger.debug('Initialized model')
logger.debug(f'Game path: {self._gamePath}')
logger.debug(f'Config path: {self._configPath}')
logger.debug(f'Cache path: {self._cachePath}')
logger.debug(f'Mods path: {self._modsPath}')
# TODO: incomplete: implement mod installation management
def setPaths(self, gamePath: Path, configPath: Path) -> None:
_gamePath = verifyGamePath(gamePath)
_configPath = verifyConfigPath(configPath)
if self._gamePath == _gamePath and self._configPath == _configPath:
return
if not _gamePath:
raise InvalidGamePath(gamePath)
if not _configPath:
raise InvalidConfigPath(configPath)
modsPath = _gamePath.joinpath('Mods')
_modsPath = verifyModsPath(modsPath)
dlcsPath = _gamePath.joinpath('DLC')
_dlcsPath = verifyDlcsPath(dlcsPath)
if not _modsPath:
raise InvalidModsPath(modsPath)
if not _dlcsPath:
raise InvalidDlcsPath(dlcsPath)
self._gamePath = _gamePath
self._configPath = _configPath
self._modsPath = _modsPath
self._dlcsPath = _dlcsPath
self.lastUpdate = datetime.now(tz=timezone.utc)
self.lastInitialization = datetime.now(tz=timezone.utc)
self._modList = {}
@debounce(25)
async def updateBundledContentsConflicts(self) -> None:
self._iteration += 1
conflicts = await asyncio.get_running_loop().run_in_executor(
self._pool,
partial(ModelConflicts.fromModList, self._modList,
self._iteration))
if conflicts.iteration == self._iteration:
self.conflicts = conflicts
self.updateCallbacks.fire(self)
async def loadInstalledMod(self, path: Path) -> None:
if path.joinpath('.w3mm').is_file():
mod = Mod.from_json(path.joinpath('.w3mm').read_bytes())
mod.enabled = not path.name.startswith('~')
mod.filename = re.sub(r'^(~)', r'', path.name)
if mod.enabled:
enabled = getSettingsValue(
mod.filename, 'Enabled',
self.configpath.joinpath('mods.settings'))
if enabled == '0':
mod.enabled = False
if (mod.filename, mod.target) in self._modList:
logger.bind(path=path).error('Ignoring duplicate MOD')
if not self._modList[(mod.filename, mod.target)].enabled:
self._modList[(mod.filename, mod.target)] = mod
else:
self._modList[(mod.filename, mod.target)] = mod
else:
try:
for mod in await Mod.fromDirectory(path, recursive=False):
mod.installdate = datetime.fromtimestamp(
path.stat().st_ctime, tz=timezone.utc)
mod.target = 'mods'
mod.datatype = 'mod'
mod.enabled = not path.name.startswith('~')
mod.filename = re.sub(r'^(~)', r'', path.name)
if mod.enabled:
enabled = getSettingsValue(
mod.filename, 'Enabled',
self.configpath.joinpath('mods.settings'))
if enabled == '0':
mod.enabled = False
if (mod.filename, mod.target) in self._modList:
logger.bind(path=path).error('Ignoring duplicate MOD')
if not self._modList[(mod.filename,
mod.target)].enabled:
self._modList[(mod.filename, mod.target)] = mod
else:
self._modList[(mod.filename, mod.target)] = mod
await self.update(mod)
except InvalidPathError:
logger.bind(path=path).debug('Invalid MOD')
async def loadInstalledDlc(self, path: Path) -> None:
if path.joinpath('.w3mm').is_file():
mod = Mod.from_json(path.joinpath('.w3mm').read_bytes())
mod.enabled = not all(
file.name.endswith('.disabled') for file in path.glob('**/*')
if file.is_file() and not file.name == '.w3mm')
mod.filename = path.name
self._modList[(mod.filename, mod.target)] = mod
else:
try:
for mod in await Mod.fromDirectory(path, recursive=False):
mod.installdate = datetime.fromtimestamp(
path.stat().st_ctime, tz=timezone.utc)
mod.target = 'dlc'
mod.datatype = 'dlc'
mod.enabled = not all(
file.name.endswith('.disabled')
for file in path.glob('**/*')
if file.is_file() and not file.name == '.w3mm')
mod.filename = path.name
self._modList[(mod.filename, mod.target)] = mod
await self.update(mod)
except InvalidPathError:
logger.bind(path=path).debug('Invalid DLC')
async def loadInstalled(self) -> None:
await asyncio.gather(
*[self.loadInstalledMod(path) for path in self.modspath.iterdir()],
loop=asyncio.get_running_loop())
await asyncio.gather(
*[self.loadInstalledDlc(path) for path in self.dlcspath.iterdir()],
loop=asyncio.get_running_loop())
self.updateBundledContentsConflicts()
self.updateCallbacks.fire(self)
def get(self, mod: ModelIndexType) -> Mod:
return self[mod]
def keys(self) -> KeysView[Tuple[str, str]]:
return self._modList.keys()
def values(self) -> ValuesView[Mod]:
return self._modList.values()
def data(self) -> Dict[Tuple[str, str], Mod]:
return self._modList
async def add(self, mod: Mod) -> None:
# TODO: incomplete: always override compilation trigger mod
if self.modspath in [mod.source, *mod.source.parents]:
raise InvalidSourcePath(
mod.source,
'Invalid mod source: Mods cannot be installed from the mods directory'
)
async with self.updateLock:
if (mod.filename, mod.target) in self._modList:
raise ModExistsError(mod.filename, mod.target)
target = self.getModPath(mod)
if target.exists():
# TODO: incomplete: make sure the mod is tracked by the model
raise ModExistsError(mod.filename, mod.target)
settings = 0
inputs = 0
try:
event_loop = asyncio.get_running_loop()
target.mkdir(parents=True)
# copy mod files
copies = []
logger.bind(name=mod.filename,
path=target).debug('Copying binary files')
for _file in mod.files:
sourceFile = mod.source.joinpath(_file.source)
targetFile = target.joinpath(_file.source)
targetFile.parent.mkdir(parents=True, exist_ok=True)
copies.append((sourceFile, targetFile))
await asyncio.gather(*[
event_loop.run_in_executor(
None, partial(copyfile, _copy[0], _copy[1]))
for _copy in copies
])
copies = []
logger.bind(name=mod.filename,
path=target).debug('Copying content files')
for _content in mod.contents:
sourceFile = mod.source.joinpath(_content.source)
targetFile = target.joinpath(_content.source)
targetFile.parent.mkdir(parents=True, exist_ok=True)
copies.append((sourceFile, targetFile))
await asyncio.gather(*[
event_loop.run_in_executor(
None, partial(copyfile, _copy[0], _copy[1]))
for _copy in copies
])
mod.installed = True
# update settings
logger.bind(name=mod.filename,
path=target).debug('Updating settings')
settings = addSettings(
mod.settings, self.configpath.joinpath('user.settings'))
inputs = addSettings(
mod.inputs, self.configpath.joinpath('input.settings'))
setSettingsValue(mod.filename, 'Enabled', '1',
self.configpath.joinpath('mods.settings'))
await self.update(mod)
except Exception as e:
removeDirectory(target)
if settings:
removeSettings(mod.settings,
self.configpath.joinpath('user.settings'))
if inputs:
removeSettings(mod.inputs,
self.configpath.joinpath('input.settings'))
removeSettingsSection(
mod.filename, self.configpath.joinpath('mods.settings'))
raise e
self._modList[(mod.filename, mod.target)] = mod
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc))
async def update(self, mod: Mod) -> None:
# serialize and store mod structure
target = self.getModPath(mod, True)
try:
with target.joinpath('.w3mm').open(
'w', encoding='utf-8') as modInfoFile:
modSerialized = mod.to_json()
modInfoFile.write(modSerialized)
except Exception as e:
logger.exception(f'Could not update mod: {e}')
async def replace(self, filename: str, target: str, mod: Mod) -> None:
# TODO: incomplete: handle possible conflict with existing mods
async with self.updateLock:
self._modList[(filename, target)] = mod
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc))
async def remove(self, mod: ModelIndexType) -> None:
if await self.disable(mod):
async with self.updateLock:
mod = self[mod]
target = self.getModPath(mod, True)
removeDirectory(target)
try:
removeSettings(mod.settings,
self.configpath.joinpath('user.settings'))
except Exception as e:
logger.bind(name=mod.filename).warning(
f'Could not remove settings from user.settings: {e}')
try:
removeSettings(mod.inputs,
self.configpath.joinpath('input.settings'))
except Exception as e:
logger.bind(name=mod.filename).warning(
f'Could not remove settings from input.settings: {e}')
try:
removeSettingsSection(
mod.filename,
self.configpath.joinpath('mods.settings'))
except Exception as e:
logger.bind(name=mod.filename).warning(
f'Could not remove settings from mods.settings: {e}')
del self._modList[(mod.filename, mod.target)]
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc))
async def enable(self, mod: ModelIndexType) -> bool:
async with self.updateLock:
mod = self[mod]
oldstat = mod.enabled
oldpath = self.getModPath(mod, True)
renamed = False
undo = False
renames = []
settings = 0
inputs = 0
try:
mod.enabled = True
if mod.target == 'mods':
newpath = self.getModPath(mod)
if oldpath != newpath:
oldpath.rename(newpath)
renamed = True
setSettingsValue(mod.filename, 'Enabled', '1',
self.configpath.joinpath('mods.settings'))
if mod.target == 'dlc':
for file in oldpath.glob('**/*'):
while file.is_file() and file.suffix == '.disabled':
file = file.rename(file.with_suffix(''))
renames.append(file)
settings = addSettings(
mod.settings, self.configpath.joinpath('user.settings'))
inputs = addSettings(
mod.inputs, self.configpath.joinpath('input.settings'))
await self.update(mod)
except PermissionError:
logger.bind(path=oldpath).exception(
'Could not enable mod, invalid permissions. Is it open in the explorer?'
)
undo = True
except Exception as e:
logger.exception(f'Could not enable mod: {e}')
mod.enabled = oldstat
undo = True
if undo:
newpath = self.getModPath(mod)
mod.enabled = oldstat
if newpath.is_dir() and newpath != oldpath and renamed:
newpath.rename(oldpath)
for rename in reversed(renames):
rename.rename(
rename.with_suffix(rename.suffix + '.disabled'))
if settings:
removeSettings(mod.settings,
self.configpath.joinpath('user.settings'))
if inputs:
removeSettings(mod.inputs,
self.configpath.joinpath('input.settings'))
if mod.target == 'mods':
setSettingsValue(mod.filename, 'Enabled', '0',
self.configpath.joinpath('mods.settings'))
# TODO: incomplete: handle xml and ini changes
if not undo:
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc))
return True
return False
async def disable(self, mod: ModelIndexType) -> bool:
async with self.updateLock:
mod = self[mod]
oldstat = mod.enabled
oldpath = self.getModPath(mod, True)
renamed = False
undo = False
renames = []
settings = 0
inputs = 0
try:
mod.enabled = False
if mod.target == 'mods':
newpath = self.getModPath(mod)
if oldpath != newpath:
oldpath.rename(newpath)
renamed = True
setSettingsValue(mod.filename, 'Enabled', '0',
self.configpath.joinpath('mods.settings'))
if mod.target == 'dlc':
for file in oldpath.glob('**/*'):
if file.is_file(
) and not file.name == '.w3mm' and not file.suffix == '.disabled':
file = file.rename(
file.with_suffix(file.suffix + '.disabled'))
renames.append(file)
settings = removeSettings(
mod.settings, self.configpath.joinpath('user.settings'))
inputs = removeSettings(
mod.inputs, self.configpath.joinpath('input.settings'))
await self.update(mod)
except PermissionError:
logger.bind(path=oldpath).exception(
'Could not disable mod, invalid permissions. Is it open in the explorer?'
)
undo = True
except Exception as e:
logger.exception(f'Could not disable mod: {e}')
undo = True
if undo:
newpath = self.getModPath(mod)
mod.enabled = oldstat
if newpath.is_dir() and newpath != oldpath and renamed:
newpath.rename(oldpath)
for rename in reversed(renames):
rename.rename(rename.with_suffix(''))
if settings:
addSettings(mod.settings,
self.configpath.joinpath('user.settings'))
if inputs:
addSettings(mod.inputs,
self.configpath.joinpath('input.settings'))
if mod.target == 'mods':
setSettingsValue(mod.filename, 'Enabled', '1',
self.configpath.joinpath('mods.settings'))
# TODO: incomplete: handle xml and ini changes
if not undo:
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc))
return True
return False
async def setFilename(self, mod: ModelIndexType, filename: str) -> None:
async with self.updateLock:
mod = self[mod]
oldname = mod.filename
oldpath = self.getModPath(mod, True)
mod.filename = filename
newpath = self.getModPath(mod)
renamed = False
undo = False
try:
if oldpath != newpath:
oldpath.rename(newpath)
renamed = True
renameSettingsSection(
oldname, filename,
self.configpath.joinpath('mods.settings'))
await self.update(mod)
except PermissionError:
logger.bind(path=oldpath).exception(
'Could not rename mod, invalid permissions. Is it open in the explorer?'
)
undo = True
except Exception as e:
logger.exception(f'Could not rename mod: {e}')
undo = True
if undo:
mod.filename = oldname
if renamed:
newpath.rename(oldpath)
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc), False)
async def setPackage(self, mod: ModelIndexType, package: str) -> None:
async with self.updateLock:
mod = self[mod]
mod.package = package
await self.update(mod)
self.setLastUpdateTime(datetime.now(tz=timezone.utc), False)
async def setCategory(self, mod: ModelIndexType, category: str) -> None:
async with self.updateLock:
mod = self[mod]
mod.category = category
await self.update(mod)
self.setLastUpdateTime(datetime.now(tz=timezone.utc), False)
async def setPriority(self, mod: ModelIndexType, priority: int) -> None:
async with self.updateLock:
mod = self[mod]
mod.priority = priority
if mod.target == 'mods':
setSettingsValue(mod.filename, 'Priority',
str(priority) if priority >= 0 else '',
self.configpath.joinpath('mods.settings'))
await self.update(mod)
self.updateBundledContentsConflicts()
self.setLastUpdateTime(datetime.now(tz=timezone.utc), False)
def setLastUpdateTime(self,
time: datetime,
fireUpdateCallbacks: bool = True) -> None:
self.lastUpdate = time
if fireUpdateCallbacks:
self.updateCallbacks.fire(self)
def getModPath(self, mod: ModelIndexType, resolve: bool = False) -> Path:
if not isinstance(mod, Mod):
mod = self[mod]
basepath = self.gamepath.joinpath(mod.target).resolve()
if not mod.enabled and mod.target == 'mods':
target = basepath.joinpath(f'~{mod.filename}')
else:
target = basepath.joinpath(mod.filename)
if resolve:
if not mod.enabled and target.is_dir() \
and target.parent.joinpath(re.sub(r'^~', r'', target.name)).is_dir():
# if the mod is disabled but there are two directories with each enabled and disabled names,
# resolve to the non-disabled directory
target = target.parent.joinpath(re.sub(r'^~', r'',
target.name))
if not target.is_dir():
if not mod.enabled and target.parent.joinpath(
re.sub(r'^~', r'', target.name)).is_dir():
target = target.parent.joinpath(
re.sub(r'^~', r'', target.name))
if not target.is_dir():
raise ModNotFoundError(mod.filename, mod.target)
return target
def __len__(self) -> int:
return len(self._modList)
def __getitem__(self, mod: ModelIndexType) -> Mod:
if isinstance(mod, int):
return list(self._modList.values())[mod]
if isinstance(mod, tuple) and len(mod) == 2:
if mod not in self._modList:
raise ModNotFoundError(tuple(mod)[0], tuple(mod)[1])
return self._modList[mod]
if isinstance(mod, Mod):
if mod not in self.values():
raise ModNotFoundError(mod.filename, mod.target)
return mod
raise IndexError(f'invalid index type {type(mod)}')
def __iter__(self) -> Iterator[Tuple[str, str]]:
yield from self._modList
@property
def lockfile(self) -> Path:
return self._cachePath.joinpath('w3mm.lock')
@property
def gamepath(self) -> Path:
return self._gamePath
@property
def configpath(self) -> Path:
return self._configPath
@property
def cachepath(self) -> Path:
return self._cachePath
@property
def modspath(self) -> Path:
return self._modsPath
@property
def dlcspath(self) -> Path:
return self._dlcsPath
