def get_codegen_depends(outputwrapper):
"""
Yields all codegen dependencies.
outputwrapper is the CodegenDirWrapper that was passed to generate_all;
it's used to determine the files that have been read.
In addition, all imported python modules are yielded.
"""
# add all files that have been read as depends
for parts in outputwrapper.get_reads():
yield outputwrapper.obj.fsobj.resolve(parts).decode()
module_blacklist = set(depend_module_blacklist())
# add all source files that have been loaded as depends
for module in modules.values():
if module in module_blacklist:
continue
try:
filename = module.__file__
except AttributeError:
# built-in modules don't have __file__, we don't want those as
# depends.
continue
if module.__package__ == '':
continue
if not filename.endswith('.py'):
print("codegeneration depends on non-.py module " + filename)
exit(1)
yield filename
def delete_module(modname, paranoid=None):
""" Delete a module.http://stackoverflow.com/a/1668289
:param modname:
:param paranoid:
:return:
"""
from sys import modules
try:
thismod = modules[modname]
except KeyError:
raise ValueError(modname)
these_symbols = dir(thismod)
if paranoid:
try:
paranoid[:] # sequence support
except:
raise ValueError('must supply a finite list for paranoid')
else:
these_symbols = paranoid[:]
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
if paranoid:
for symbol in these_symbols:
if symbol[:2] == '__': # ignore special symbols
continue
try:
delattr(mod, symbol)
except AttributeError:
pass
def delete_module(modname, paranoid=None):
from sys import modules
try:
thismod = modules[modname]
except KeyError:
raise ValueError(modname)
these_symbols = dir(thismod)
if paranoid is not None:
try:
paranoid[:] # sequence support
except:
raise ValueError('must supply a finite list for paranoid')
else:
these_symbols = paranoid[:]
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
if paranoid is not None:
for symbol in these_symbols:
if symbol[:2] == '__': # ignore special symbols
continue
try:
delattr(mod, symbol)
except AttributeError:
pass
def get_codegen_depends(outputwrapper):
"""
Yields all codegen dependencies.
outputwrapper is the CodegenDirWrapper that was passed to generate_all;
it's used to determine the files that have been read.
In addition, all imported python modules are yielded.
"""
# add all files that have been read as depends
for parts in outputwrapper.get_reads():
yield outputwrapper.obj.fsobj.resolve(parts).decode()
module_blacklist = set(depend_module_blacklist())
# add all source files that have been loaded as depends
for module in modules.values():
if module in module_blacklist:
continue
try:
filename = module.__file__
except AttributeError:
# built-in modules don't have __file__, we don't want those as
# depends.
continue
if module.__package__ == '':
continue
if not filename.endswith('.py'):
print("codegeneration depends on non-.py module " + filename)
exit(1)
yield filename
def delete_module(modname, paranoid=None):
from sys import modules
try:
thismod = modules[modname]
except KeyError:
raise ValueError(modname)
these_symbols = dir(thismod)
if paranoid:
try:
paranoid[:] # sequence support
except:
raise ValueError('must supply a finite list for paranoid')
else:
these_symbols = paranoid[:]
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
if paranoid:
for symbol in these_symbols:
if symbol[:2] == '__': # ignore special symbols
continue
try:
delattr(mod, symbol)
except AttributeError:
pass
def delete_module(modname):
'''a function to remove a module from Python's internal modules
list. This is useful as some tests affect others by importing
modules.'''
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
def del_module(cls, modname):
from sys import modules
try:
thismod = modules[modname]
except KeyError:
raise ValueError(modname)
these_symbols = dir(thismod)
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
def clean_import(modname):
from sys import modules
try:
thismod = modules[modname]
except KeyError:
raise ValueError(modname)
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
delattr(mod, 'Beta')
except AttributeError:
pass
def _delete_module(modname):
from sys import modules
try:
thismod = modules[modname]
del modules[modname]
except KeyError:
pass
for mod in modules.values():
try:
delattr(mod, modname)
except:
pass
def rmmod(_modname: str) -> None:
"""Remove module from the running instance"""
if not isinstance(_modname, str):
raise Exception('rm_mod() only accepts strings - rm_mod(\'_modname\')')
try:
modules[_modname]
except KeyError:
raise ValueError(_modname)
del modules[_modname]
for mod in modules.values():
try:
delattr(mod, _modname)
except AttributeError:
pass
def rmmod(_modname: str):
"""Remove module from the running instance"""
assert isinstance(_modname, str), \
'modexist() only accepts strings - rm_mod(\'_modname\')'
try:
modules[_modname]
except KeyError:
raise ValueError(_modname)
del modules[_modname]
for mod in modules.values():
try:
delattr(mod, _modname)
except AttributeError:
pass
return
def get_codegen_depends(
outputwrapper: CodegenDirWrapper) -> Generator[str, None, None]:
"""
Yields all codegen dependencies.
outputwrapper is the CodegenDirWrapper that was passed to generate_all;
it's used to determine the files that have been read.
In addition, all imported python modules are yielded.
"""
# add all files that have been read as depends
for parts in outputwrapper.get_reads():
# TODO: this assumes that the wrap.obj.fsobj is a fslike.Directory
# this just resolves paths to the output directory
yield outputwrapper.obj.fsobj.resolve(parts).decode()
module_blacklist = set(depend_module_blacklist())
# add all source files that have been loaded as depends
for module in modules.values():
if module in module_blacklist:
continue
try:
filename = module.__file__
except AttributeError:
# built-in modules don't have __file__, we don't want those as
# depends.
continue
if filename is None:
# some modules have __file__ == None, we don't want those either.
continue
if module.__package__ == '':
continue
if not filename.endswith('.py'):
# This usually means that some .so file is imported as module.
# This is not a problem as long as it's not "our" .so file.
# => just handle non-openage non-.py files normally
if 'openage' in module.__name__:
print("codegeneration depends on non-.py module " + filename)
sys.exit(1)
yield filename
def get_codegen_depends(outputwrapper):
"""
Yields all codegen dependencies.
outputwrapper is the CodegenDirWrapper that was passed to generate_all;
it's used to determine the files that have been read.
In addition, all imported python modules are yielded.
"""
# add all files that have been read as depends
for parts in outputwrapper.get_reads():
# TODO: this assumes that the wrap.obj.fsobj is a fslike.Directory
# this just resolves paths to the output directory
yield outputwrapper.obj.fsobj.resolve(parts).decode()
module_blacklist = set(depend_module_blacklist())
# add all source files that have been loaded as depends
for module in modules.values():
if module in module_blacklist:
continue
try:
filename = module.__file__
except AttributeError:
# built-in modules don't have __file__, we don't want those as
# depends.
continue
if filename is None:
# some modules have __file__ == None, we don't want those either.
continue
if module.__package__ == '':
continue
if not filename.endswith('.py'):
# This usually means that some .so file is imported as module.
# This is not a problem as long as it's not "our" .so file.
# => just handle non-openage non-.py files normally
if 'openage' in module.__name__:
print("codegeneration depends on non-.py module " + filename)
sys.exit(1)
yield filename
def _deleteModule(self, modname):
''' Remove the given module name from the exported python modules
:param modname: Module name
:type modname: str
'''
try:
thismod = modules[modname]
except KeyError:
# This module is not imported
raise ValueError(modname)
these_symbols = dir(thismod)
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
def _deleteModule(self, modname):
''' Remove the given module name from the exported python modules
:param modname: Module name
:type modname: str
'''
try:
thismod = modules[modname]
except KeyError:
# This module is not imported
raise ValueError(modname)
these_symbols = dir(thismod)
del modules[modname]
for mod in modules.values():
try:
delattr(mod, modname)
except AttributeError:
pass
def _pickle_moduledict(self, obj):
# save module dictionary as "getattr(module, '__dict__')"
# build index of module dictionaries
try:
modict = self.module_dict_ids
except AttributeError:
modict = {}
from sys import modules
for mod in modules.values():
if isinstance(mod, ModuleType):
modict[id(mod.__dict__)] = mod
self.module_dict_ids = modict
thisid = id(obj)
try:
themodule = modict[thisid]
except KeyError:
return None
from __builtin__ import getattr
return getattr, (themodule, '__dict__')
def _pickle_moduledict(self, obj):
# save module dictionary as "getattr(module, '__dict__')"
# build index of module dictionaries
try:
modict = self.module_dict_ids
except AttributeError:
modict = {}
from sys import modules
for mod in modules.values():
if isinstance(mod, ModuleType):
modict[id(mod.__dict__)] = mod
self.module_dict_ids = modict
thisid = id(obj)
try:
themodule = modict[thisid]
except KeyError:
return None
from __builtin__ import getattr
return getattr, (themodule, '__dict__')
def _pickle_moduledict(self, obj):
try:
modict = self.module_dict_ids
except AttributeError:
modict = {}
from sys import modules
for mod in modules.values():
if isinstance(mod, ModuleType):
try:
modict[id(mod.__dict__)] = mod
except KeyboardInterrupt:
raise
except: # obscure: the above can fail for
# arbitrary reasons, because of the py lib
pass
self.module_dict_ids = modict
thisid = id(obj)
try:
themodule = modict[thisid]
except KeyError:
return None
from __builtin__ import getattr
return getattr, (themodule, '__dict__')
def _pickle_moduledict(self, obj):
try:
modict = self.module_dict_ids
except AttributeError:
modict = {}
from sys import modules
for mod in modules.values():
if isinstance(mod, ModuleType):
try:
modict[id(mod.__dict__)] = mod
except KeyboardInterrupt:
raise
except: # obscure: the above can fail for
# arbitrary reasons, because of the py lib
pass
self.module_dict_ids = modict
thisid = id(obj)
try:
themodule = modict[thisid]
except KeyError:
return None
from __builtin__ import getattr
return getattr, (themodule, '__dict__')
def shell(store, options, jugspace):
'''
shell(store, options, jugspace)
Implement 'shell' command.
Currently depends on Ipython being installed.
'''
try:
import IPython
if IPython.version_info[0] >= 1:
from IPython.terminal.embed import InteractiveShellEmbed
from IPython.terminal.ipapp import load_default_config
else:
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from IPython.frontend.terminal.ipapp import load_default_config
config = load_default_config()
ipshell = InteractiveShellEmbed(config=config,
display_banner=_ipython_banner)
except ImportError:
try:
# Fallback for older Python:
from IPython.Shell import IPShellEmbed
ipshell = IPShellEmbed(banner=_ipython_banner)
except ImportError:
import sys
sys.stderr.write(_ipython_not_found_msg)
sys.exit(1)
def _load_all():
'''
load_all()
Loads all task results.
'''
load_all(jugspace, local_ns)
local_ns = {
'load_all': _load_all,
'value': value,
}
# This is necessary for some versions of Ipython. See:
# http://groups.google.com/group/pylons-discuss/browse_thread/thread/312e3ead5967468a
try:
del jugspace['__builtins__']
except KeyError:
pass
jugspace.update(local_ns)
local_ns['__name__'] = '__jugfile__'
if IPython.version_info[0] >= 5:
from sys import modules
# This is tricky, but the following WOULD NOT work:
# for mod in modules.values():
# ..
# Some modules use https://pypi.python.org/pypi/apipkg which triggers
# name loading when __dict__ is accessed. This may itself import other
# modules, thus raising an error: "RuntimeError: dictionary changed
# size during iteration" Whether this happens depends on exactly which
# modules the user uses/has loaded
modules = list(modules.values())
for mod in modules:
if getattr(mod, '__dict__', None) is jugspace:
break
else:
raise KeyError("Could not find jug module")
ipshell(module=mod, local_ns=local_ns)
else:
ipshell(global_ns=jugspace, local_ns=local_ns)
def run(self, store, options, jugspace, *args, **kwargs):
try:
import IPython
if IPython.version_info[0] >= 1:
from IPython.terminal.embed import InteractiveShellEmbed
from IPython.terminal.ipapp import load_default_config
else:
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from IPython.frontend.terminal.ipapp import load_default_config
config = load_default_config()
ipshell = InteractiveShellEmbed(config=config, display_banner=_ipython_banner)
except ImportError:
try:
# Fallback for older Python:
from IPython.Shell import IPShellEmbed
ipshell = IPShellEmbed(banner=_ipython_banner)
except ImportError:
import sys
sys.stderr.write(_ipython_not_found_msg)
sys.exit(1)
def _load_all():
'''
load_all()
Loads all task results.
'''
load_all(jugspace, local_ns)
reverse_cache = {}
def _invalidate(t):
'''Recursively invalidates its argument, i.e. removes from the store
results of any task which may (even indirectly) depend on its argument.
This is analogous to the ``jug invalidate`` subcommand.
Parameters
----------
t : a Task
Returns
-------
None
'''
from ..task import alltasks
return invalidate(alltasks, reverse_cache, t)
def _get_tasks():
'''Returns a list of all tasks seen by jug
'''
from ..task import alltasks
return alltasks
local_ns = {
'load_all': _load_all,
'value': value,
'invalidate': _invalidate,
'get_tasks': _get_tasks,
}
# This is necessary for some versions of Ipython. See:
# http://groups.google.com/group/pylons-discuss/browse_thread/thread/312e3ead5967468a
try:
del jugspace['__builtins__']
except KeyError:
pass
jugspace.update(local_ns)
local_ns['__name__'] = '__jugfile__'
if IPython.version_info[0] >= 5:
from sys import modules
# This is tricky, but the following WOULD NOT work:
# for mod in modules.values():
# ..
# Some modules use https://pypi.python.org/pypi/apipkg which triggers
# name loading when __dict__ is accessed. This may itself import other
# modules, thus raising an error: "RuntimeError: dictionary changed
# size during iteration" Whether this happens depends on exactly which
# modules the user uses/has loaded
modules = list(modules.values())
for mod in modules:
if getattr(mod, '__dict__', None) is jugspace:
break
else:
raise KeyError("Could not find jug module")
ipshell(module=mod, local_ns=local_ns)
else:
ipshell(global_ns=jugspace, local_ns=local_ns)
async def module_del(self, ctx, module: str):
"""Выгружает модуль
Аргументы:
-----------
module: `str`
Название модуля.
Формат:
- `kurisu.<имя>` / `<имя>`
"""
if ctx.message.author.id != 185459415514742784:
return
importlib.invalidate_caches()
if module.startswith('kurisu.'):
module = module.replace('kurisu.', '')
if not module in dir(globals()['kurisu']):
await ctx.send("А модуль `%s` точно импортирован?" % module)
return
emb = kurisu.prefs.Embeds.new('alert')
emb.add_field(
name="Выгрузка модуля",
value=
"Сенпай, ты же в курсе, что это опасно? Пожалуйста, подтверди действие (Д/н)\nТакже хочу отметить, что __все задачи__, относящиеся к данному модулю будут отменены."
)
emb.set_footer(text="Ожидание ввода...")
mess = await ctx.send(embed=emb)
def check(m):
return (m.content.lower() in [
'д', 'н', 'да', 'нет', 'yes', 'no', 'y', 'n'
]) and (ctx.message.author == m.author)
try:
m = await self.bot.wait_for('message', check=check, timeout=10)
except asyncio.TimeoutError:
m = None
if (m is None) or (m.content.lower() in ['n', 'н', 'no', 'нет']):
emb.clear_fields()
emb.colour = discord.Colour.red()
emb.add_field(name="Выгрузка модуля",
value="Выгрузка модуля %s отменена." % module)
if m is not None:
await m.delete()
emb.set_footer(text="Операция отменена пользователем.")
else:
emb.set_footer(text="Время ожидания вышло.")
await mess.edit(embed=emb)
return
await m.delete()
m = getattr(globals()['kurisu'], module)
mPath = m.__name__
emb.clear_fields()
emb.add_field(name='Вывод', value='```\n\n```')
emb.set_footer(text="Выполняется операция...")
embout = []
tasks = salieri.tasks.allTasks.keys()
mTasks = []
for task in tasks:
if task.startswith(mPath):
mTasks.append(task)
for mTask in mTasks:
t = mTask.split('.')[1:]
task = getattr(getattr(kurisu, t[0]), t[1])
if await salieri.tasks.cancel(task):
embout.append("Задача `%s.%s` отменена." % (t[0], t[1]))
emb.set_field_at(0,
name='Вывод',
value='```\n%s\n```' % '\n'.join(embout))
await mess.edit(embed=emb)
await asyncio.sleep(0.5)
else:
emb.colour = discord.Colour.red()
emb.set_field_at(0,
name="Выгрузка модуля",
value="Не удалось отменить задачу `%s.%s`." %
(t[0], t[1]))
emb.add_field(name='Вывод',
value='```\n%s\n```' % '\n'.join(embout))
emb.set_footer(text="Произошла ошибка.")
await mess.edit(embed=emb)
return
try:
del modules[mPath]
for mod in modules.values():
try:
delattr(mod, module)
except AttributeError:
pass
embout.append("Модуль `%s` выгружен." % mPath)
emb.set_field_at(0,
name='Вывод',
value='```\n%s\n```' % '\n'.join(embout))
await mess.edit(embed=emb)
except:
emb.colour = discord.Colour.red()
emb.set_field_at(
0,
name="Произошла ошибка во время выгрузки модуля `%s`." % mPath)
emb.add_field(name='Вывод',
value='```\n%s\n```' % '\n'.join(embout))
emb.set_footer(text="Произошла ошибка.")
await mess.edit(embed=emb)
return
emb.colour = discord.Colour.green()
emb.set_field_at(0,
name="Выгрузка модуля",
value="Модуль `%s` успешно выгружен." % mPath)
emb.add_field(name='Вывод', value='```\n%s\n```' % '\n'.join(embout))
emb.set_footer(text="Операция выполнена успешно.")
await mess.edit(embed=emb)
from cogs import punishments
from cogs.logging import update_leaderboards
logger = logging.getLogger('discord')
logger.setLevel(logging.INFO)
handler = logging.FileHandler(filename='discord.log',
encoding='utf-8',
mode='w')
handler.setFormatter(
logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)
urllib3.disable_warnings()
global PRELOADED_MODULES
# sys and importlib are ignored here too
PRELOADED_MODULES = set(modules.values())
load_dotenv()
token = os.getenv('DISCORD_TOKEN')
gh_token = os.getenv('GITHUB_TOKEN')
nebula_token = os.getenv('NEBULA_TOKEN')
is_testing = os.getenv('TESTING')
is_testing = True if is_testing == '1' else False
def get_prefix(client, message):
"""Returns the prefix for the specified server"""
if message.guild is None:
return "!"
with open('data/prefixes.json', 'r') as f:
def interact(script='dev.py',hooks=None,**kwargs):
"""
Run a script interactively.
This function call exeuctes a script in a
development mode which specifically provides users with commands (`go`,
`reload`, `repeat`) which streamline development of complex scripts that
require lots of calculation. This effectively takes the place of a debugger,
however it provides a more native coding experience.
dev: start with an error and go nowhere
a fatal flaw here is that running a script with an error in it on the
first execution then you cannot rerun things because you get the
auto-debugger. in fact the auto-debugger prevents you from continuing
to run anything, so errors are fatal before you complete one execution
dev: when we add a pdb.set_trace to a reimport library we cannot run
commands inside the trace:
ValueError: Only callable can be used as callback
this means we cannot easily debug and return to interact
"""
module_host = kwargs.get('module_host',None)
onward_kwargs = kwargs.get('onward',{})
# save preloaded modules
from sys import modules
global preloaded_mods
preloaded_mods = set([i for i in modules.values()])
#! if not module_host or i.__name__!=module_host])
coda = kwargs.pop('coda',None)
# allow subclassing of ReExec
reexec_class = kwargs.pop('reexec_class',ReExec)
# previous method: os.system('python -i %s'%(script))
out = globals()
# allow onward args to be added here
# dev: document this use-case from atgizmo.core.thick.cli
out.update(**onward_kwargs)
# allow for flexible command names in the terminal
for key in ['do','redo','reload']:
if key in kwargs:
reexec_class._names[key] = kwargs[key]
ie = reexec_class(file=script,namespace=out)
sys.ps1 = ">>> "
if hooks:
if not isinstance(hooks,tuple):
raise Exception('hooks must be a tuple')
# hooks are applied in order and transform the outgoing dictionary
# kwargs go through the hook fcuntion
for hook in hooks:
if not callable(hook):
raise Exception('hooks must be callable: %s'%hook)
hook(out,**kwargs)
# nb: this is the site of a previous "coda" functionality which allowed one
# to execute a piece of code after each reexecution. this made the analogy
# between the "iterative reexeuction" method and proper debugging into
# one that is symmetric, simulating the act of stepping through code while
# still retaining the native coding experience
out['__name__'] = '__main__'
# dev: run the code once without main in case there is an error in main,
# then start the interactive session and allow an exception in main to
# continue inside the debugger. this would eliminate a case where an
# exception in __main__ prevents interactive sessions altogether
# let the script know we are ortho in case that is useful when building a
# script that could run with regular CLI arguments or with hardcoded
# tests during development
out['___is_ortho'] = True
# compatible version of execfile
# dev: exec to eval for python <2.7.15. see note above
# dev: the following cannot encounter exceptions or we exit. this means that
# when you start an interact session, the code must be basically perfect
eval(compile(open(script).read(),filename=script,mode='exec'),out,out)
# prepare the interactive session
import code
class InteractiveCommand:
"""
Run functions from the repr hence without parentheses.
Useful for reeexecution commands.
"""
def __init__(self,func,name,prelim=None):
self.func,self.name = func,name
self.prelim = prelim
def __repr__(self):
# briefly considered doing this with @property but this works fine
# currently the prelim feature is deprecated by a subclassed ReExec
# but we retain it here as an option
if self.prelim:
# dev: exec to eval for python <2.7.15. see note above
eval(compile(self.prelim,'<string>','exec'),out,out)
self.func()
# return empty string but we always get a newline
return ''
# apply extra functions if we subclass ReExec to add extra commands
if kwargs.get('commands',[]):
collide = [i for i in kwargs['commands']
if i in reexec_class._names.keys()]
if any(collide):
# dev: if you subclass one of the functions in the parent, then we
# get a name collision that prevents the function from executing
# from the repr
raise Exception('cannot include commands with these names '
'in a subclass of ReExec: %s'%collide)
for cmd in kwargs['commands']:
locals()[cmd] = InteractiveCommand(
prelim=kwargs.get('do_prelim',None),
func=getattr(ie,cmd),name=cmd)
# standard interact gets do and redo
else:
out.update(
# override functions so they can be invoked without parentheses
do = InteractiveCommand(func=ie.do,
name=reexec_class._names['do'],
# code to run before reexecuting a script from the top with do
prelim=kwargs.get('do_prelim',None)),
redo = InteractiveCommand(func=ie.redo,
name=reexec_class._names['redo']),
reload = InteractiveCommand(func=ie.reload,
name=reexec_class._names['reload']),)
#! issue: the following is overwriting key,val so we are testing
for key,val in reexec_class._names.items():
out[val] = out.pop(key)
# consolidate, add tab completion
vars = globals()
# filter out the "key" and "val" keys because they leak into the namespace
vars.update(dict([(i,j) for i,j in locals().items() if i not in [
'key','val','onward_kwargs']]))
vars.update(**vars.pop('out'))
readline.set_completer(rlcompleter.Completer(vars).complete)
readline.parse_and_bind("tab: complete")
# interact
msg = kwargs.get('msg','(interactive mode)')
code.interact(local=vars,banner=msg)
def run(self, store, options, jugspace, *args, **kwargs):
try:
import IPython
if IPython.version_info[0] >= 1:
from IPython.terminal.embed import InteractiveShellEmbed
from IPython.terminal.ipapp import load_default_config
else:
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from IPython.frontend.terminal.ipapp import load_default_config
config = load_default_config()
ipshell = InteractiveShellEmbed(config=config,
display_banner=_ipython_banner)
except ImportError:
try:
# Fallback for older Python:
from IPython.Shell import IPShellEmbed
ipshell = IPShellEmbed(banner=_ipython_banner)
except ImportError:
import sys
sys.stderr.write(_ipython_not_found_msg)
sys.exit(1)
def _load_all():
'''
load_all()
Loads all task results.
'''
load_all(jugspace, local_ns)
reverse_cache = {}
def _invalidate(t):
'''Recursively invalidates its argument, i.e. removes from the store
results of any task which may (even indirectly) depend on its argument.
This is analogous to the ``jug status`` subcommand.
Parameters
----------
t : a Task
Returns
-------
None
'''
from ..task import alltasks
return invalidate(alltasks, reverse_cache, t)
local_ns = {
'load_all': _load_all,
'value': value,
'invalidate': _invalidate,
}
# This is necessary for some versions of Ipython. See:
# http://groups.google.com/group/pylons-discuss/browse_thread/thread/312e3ead5967468a
try:
del jugspace['__builtins__']
except KeyError:
pass
jugspace.update(local_ns)
local_ns['__name__'] = '__jugfile__'
if IPython.version_info[0] >= 5:
from sys import modules
# This is tricky, but the following WOULD NOT work:
# for mod in modules.values():
# ..
# Some modules use https://pypi.python.org/pypi/apipkg which triggers
# name loading when __dict__ is accessed. This may itself import other
# modules, thus raising an error: "RuntimeError: dictionary changed
# size during iteration" Whether this happens depends on exactly which
# modules the user uses/has loaded
modules = list(modules.values())
for mod in modules:
if getattr(mod, '__dict__', None) is jugspace:
break
else:
raise KeyError("Could not find jug module")
ipshell(module=mod, local_ns=local_ns)
else:
ipshell(global_ns=jugspace, local_ns=local_ns)
_code, _dict, _instance, _ilen, _seq, lambda x: x,
(_code, _dict, _instance, _ilen, _seq)):
_aprint(o)
_print('')
_print('asizeof(%s, deep=%s, code=%s)', 'locals()', 'MAX', False)
asizeof(locals(), deep=MAX, code=False, verbose=1)
_print('')
_print('asizeof(%s, deep=%s, code=%s)', 'globals()', 'MAX', False)
asizeof(globals(), deep=MAX, code=False, verbose=2)
_print('')
_print('asizeof(deep=%s, code=%s, *%s)', 'MAX', True, 'sys.modules.values()')
from sys import modules
asizeof(deep=MAX, code=True, verbose=2, *modules.values())
# License file from an earlier version of asizeof() follows:
#---------------------------------------------------------------------
# Copyright (c) 2002-2008 -- ProphICy Semiconductor, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
def harmoniseService(facility, country=None, gc=None, **kwargs):
"""Generic harmonisation function.
>>> harmonise.harmoniseService(facility, country = None, gc = None, **kwargs)
"""
#if facility is None:
# facility = list(FACILITIES.keys())
if not isinstance(facility, string_types):
raise TypeError(
"Wrong type for input service - must be the facility type")
elif not facility in FACILITIES.keys():
raise IOError(
"Service type not recognised - must be a string in the list '%s'" %
list(FACILITIES.keys()))
if country is None:
country = list(COUNTRIES.keys())
if not isinstance(country, string_types) and isinstance(country, Sequence):
for ctry in country:
try:
harmoniseService(facility, country=ctry, gc=gc, **kwargs)
except:
continue
return
elif not isinstance(country, string_types):
raise TypeError(
"Wrong type for input country code - must be the ISO 2-letter string"
)
elif not country in COUNTRIES.keys():
raise IOError(
"Country code not recognised - must be a code of the '%s' area(s)"
% list(COUNTRIES.keys()))
if not (gc is None or isinstance(gc, (string_types, Mapping))):
raise TypeError(
"Coder type not recognised - must be a dictionary or a single string"
)
CC, METADATNAT = None, {}
metadir = FACILITIES[facility].get('code')
# generic name
ccname = "%s%s" % (country, BASENAME.get(facility, ''))
# load country-dedicated module wmmhen available
modname = ccname
fname = '%s.py' % ccname
try:
assert osp.exists(osp.join(__THISDIR, metadir, fname))
# import_module('%s.%s' % (PACKNAME,modname) )
imp = import_module('.%s' % modname, '%s.%s' % (PACKNAME, metadir))
except AssertionError:
logging.warning(
"\n! No country py-file '%s' found - will proceed without !" %
fname)
except ImportError:
logging.warning(
"\n! No country py-module '%s' found - will proceed without !" %
modname)
except:
raise ImportError("No country py-module '%s' loaded" % modname)
else:
logging.warning("\n! Country py-module '%s' found !" % imp.__name__)
try:
assert imp in sysmod.values()
except:
raise ImportError("Country py-module '%s' not loaded correctly" %
imp.__name__)
try:
# assert 'CC' in dir(imp)
CC = getattr(imp, 'CC', None)
except:
logging.warning("\n! Global variable 'CC' not set - use default !")
# try:
# # assert 'META' in dir(imp)
# METADATNAT = getattr(imp, 'META', None)
# assert METADATNAT is not None
# except:
# logging.warning("\n! No default metadata dictionary 'META' available !")
# else:
# logging.warning("\n! Default hard-coded metadata dictionary 'META' found !")
try:
# assert 'harmonise' in dir(imp)
harmonise = getattr(imp, HARMONISE, None)
assert harmonise is not None
except:
logging.warning('\n! Generic formatting/harmonisation methods used !')
harmonise = harmoniseFacilityData
else:
logging.warning(
'\n! Country-specific formatting/harmonisation methods used !')
if 'methods' not in kwargs:
kwargs.update({'methods': {p: None for p in PROCESSES}})
for proc in PROCESSES:
try:
# assert proc in dir(imp)
proc_data = getattr(imp, '%s_data' % proc, None)
assert proc_data is not None
except:
# logging.warning("! no data '%s' method used !" % proc)
pass # proc_data = None
else:
logging.warning("\n! country-specific '%s_data' method loaded !" %
proc)
finally:
kwargs['methods'].update({proc: proc_data})
# load country-dedicated metadata when available
metadata = None
metafname = '%s.json' % ccname
try:
metafname = osp.join(__THISDIR, metadir, metafname)
assert osp.exists(metafname)
with open(metafname, 'r') as fp:
metadata = Json.load(fp)
except (AssertionError, FileNotFoundError):
logging.warning(
"\n! No metadata JSON-file '%s' found - will proceed without !" %
metafname)
else:
logging.warning("! Ad-hoc metadata found - JSON-file '%s' loaded !" %
metafname)
# define the actual metadata: the one loaded, or the default
# metadata = metadata or METADATNAT
if metadata in (None, {}):
raise IOError('No metadata parsed - this cannot end up well')
try:
kwargs.update({'country': {'code': CC or country}})
if 'options' in kwargs.keys():
if 'locate' in kwargs['options'].keys():
kwargs['options']['locate'].update({'gc': gc})
else:
kwargs['options'].update({'locate': {'gc': gc}})
else:
kwargs.update({'options': {'locate': {'gc': gc}}})
res = harmonise(facility, metadata, **kwargs)
except:
raise IOError("Harmonisation process for country '%s' failed..." %
country)
else:
logging.warning("\n! Harmonised data for country '%s' generated !" %
country)
return res
