def initAdditional(self, additionalRegexLibPath='regexLib.json', additionalCodeLibpath='codeLib.py'):
"""Throws key KeyError when the input file structure is incorrect"""
print("\n--------------------Initialising additional regexes-------------------\n")
if additionalRegexLibPath == 'regexLib.json':
additionalRegexLibPath = os.path.join(os.path.dirname(__file__), additionalRegexLibPath)
self.additionalRegexLibPath = additionalRegexLibPath
if additionalCodeLibpath == 'codeLib.py':
loader = machinery.SourceFileLoader('codeLib', os.path.join(os.path.dirname(__file__),additionalCodeLibpath))
self.additionalCodeModule = loader.load_module()
else:
loader = machinery.SourceFileLoader(os.path.basename(additionalCodeLibpath[:-3]), additionalCodeLibpath)
self.additionalCodeModule = loader.load_module()
with open(additionalRegexLibPath,'r') as rl:
self.additionalList = json.load(rl, encoding='utf-8')['additional']
for x in self.additionalList:
reg = self.additionalList[x]['regex']
funct = self.additionalList[x]['code']
if reg is not '':
print("Init regex ", x, " ", reg)
if funct is not '':
self.compiledRegexes.append([re.compile(reg), getattr(self.additionalCodeModule, funct), x])
else:
self.compiledRegexes.append([re.compile(reg), None, x])
else:
print("Regex empty, not compiling")
def base_model(name: str) -> Any:
"""Import and return sqlAlchemy model for given table name."""
# print("Base", name)
if application.PROJECT.conn_manager is None:
raise Exception("Project is not connected yet")
path = _path("%s.mtd" % name, False)
if path is None:
return None
if path.find("system_module/tables") > -1:
path = path.replace(
"system_module/tables",
"%s/cache/%s/sys/file.mtd" % (
config.value("ebcomportamiento/temp_dir"),
application.PROJECT.conn_manager.mainConn().DBName(),
),
)
if path:
path = "%s_model.py" % path[:-4]
if os.path.exists(path):
try:
# FIXME: load_module is deprecated!
# https://docs.python.org/3/library/importlib.html#importlib.machinery.SourceFileLoader.load_module
loader = machinery.SourceFileLoader(name, path)
return loader.load_module() # type: ignore
except Exception as exc:
LOGGER.warning("Error recargando model base:\n%s\n%s", exc,
traceback.format_exc())
pass
return None
def main(args):
logging.info('brokers={}'.format(args.brokers))
logging.info('topic={}'.format(args.topic))
logging.info('rate={}'.format(args.rate))
logging.info('source={}'.format(args.source))
# if a user function is specified, download it and import it
if args.userfunction is not None:
try:
logging.info('downloading user function')
logging.info(args.userfunction)
dl = urllib.urlretrieve(args.userfunction)[0]
loader = importlib.SourceFileLoader('userfunction', dl)
userfunction = types.ModuleType(loader.name)
loader.exec_module(userfunction)
emitter_function = userfunction.user_defined_function
logging.info('user function loaded')
except Exception as e:
logging.error('failed to import user function file')
logging.error(e)
emitter_function = None
else:
logging.info(
'no user function specified, using external file generator')
emitter_function = external_file_generator
logging.info('creating kafka producer')
producer = KafkaProducer(bootstrap_servers=args.brokers)
logging.info('beginning producer loop')
if emitter_function is not None:
for i in emitter_function(args):
producer.send(args.topic, i.encode())
time.sleep(1.0 / args.rate)
logging.info('ending producer loop')
def load_source(name, pathname, file=None):
_LoadSourceCompatibility(name, pathname, file).load_module(name)
module = sys.modules[name]
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
module.__loader__ = machinery.SourceFileLoader(name, pathname)
return module
def import_py_file(module_name, path_to_file):
"""Dynamically imports a python module.
Args:
module_name (str): The name of the module to be imported.
path_to_file (str): The path to the module to be imported.
Returns:
The module object that was imported.
"""
if sys.version_info[0] == 2:
from imp import load_source
import exceptions, warnings
with warnings.catch_warnings(record=True) as w:
imported = load_source(module_name, os.path.abspath(path_to_file))
if w:
mod_name = module_name.replace('.main', '')
if not (type(w[-1].category) == type(exceptions.RuntimeWarning)
or 'Parent module \'apps.' + mod_name +
'\' not found while handling absolute import'
in w[-1].message):
print(w[-1].message)
else:
from importlib import machinery
loader = machinery.SourceFileLoader(module_name,
os.path.abspath(path_to_file))
imported = loader.load_module(module_name)
return imported
def import_from_source(module_filepath):
"""
Import a module from a particular filesystem location.
:param str module_filepath: path to the file that constitutes the module
to import
:return module: module imported from the given location, named as indicated
:raises ValueError: if path provided does not point to an extant file
"""
import sys
if not os.path.exists(module_filepath):
raise ValueError("Path to alleged module file doesn't point to an "
"extant file: '{}'".format(module_filepath))
# Randomly generate module name.
fname_chars = string.ascii_letters + string.digits
name = "".join(random.choice(fname_chars) for _ in range(20))
# Import logic is version-dependent.
if sys.version_info >= (3, 5):
from importlib import util as _il_util
modspec = _il_util.spec_from_file_location(name, module_filepath)
mod = _il_util.module_from_spec(modspec)
modspec.loader.exec_module(mod)
elif sys.version_info < (3, 3):
import imp
mod = imp.load_source(name, module_filepath)
else:
# 3.3 or 3.4
from importlib import machinery as _il_mach
loader = _il_mach.SourceFileLoader(name, module_filepath)
mod = loader.load_module()
return mod
async def set_up(self):
await self.client.wait_until_ready()
self.app_list = [
file for file in os.listdir(app_path)
if os.path.isdir(app_path + file)
]
for app_name in self.app_list:
try:
a_path = app_path + app_name + "/data/config.yml"
with open(a_path, 'r') as f:
data = yaml.load(f)
if data["continue"]:
loader = machinery.SourceFileLoader(
app_name, app_path + app_name + '/worker.py')
app_module = loader.load_module()
instance = app_module.Worker(self.client)
self.continue_app[app_name] = instance
self.commands += data["commands"]
for x in data["commands"]:
self._commands[x] = app_name
except:
import traceback
trace = traceback.format_exc()
await self.client.get_channel(497046680806621184).send(trace)
try:
with open("./datas/agree.txt", "r") as f:
self.agree_users = list(
set([int(i) for i in f.read().split("\n")[1:] if i]))
except FileNotFoundError:
pass
async def command(self, message: discord.Message, command: str, app: str,
point: int):
if message.content == "sigma agree SigmaOS":
return await self.agree(message)
if not message.content == command:
args = re.sub("[{}]".format(command), "", message.content,
len(command))
else:
args = ""
if app in self.continue_app.keys():
if not message.author.id in self.agree_users and not message.content == "sigma contract":
return await self.request_agree(message)
try:
if args:
if args.split()[0] == "help":
return await self.continue_app[app].join(message)
return await self.continue_app[app].command(
message, command, args.split(), point)
except IndexError:
return False
loader = machinery.SourceFileLoader(app, app_path + app + '/worker.py')
app_module = loader.load_module()
instance = app_module.Worker(self.client)
if args:
if args.split()[0] == "help":
await instance.join(message)
try:
return await instance.command(message, command, args.split(),
point)
except IndexError:
return False
def load_program_from_file(filename):
from importlib import machinery as importlib_machinery
import types
loader = importlib_machinery.SourceFileLoader("your_othello_strategy", filename)
module = types.ModuleType(loader.name)
loader.exec_module(module)
return module
def import_file(filename, name=None):
"""
Imports a module from a single file without importing the package that
the file is in.
This is useful for cases where a file needs to be imported from
``setup_package.py`` files without importing the parent package. The
returned module will have the optional ``name`` if given, or else a name
generated from the filename.
"""
# Specifying a traditional dot-separated fully qualified name here
# results in a number of "Parent module '...' not found while
# handling absolute import" warnings. Using the same name, the
# namespaces of the modules get merged together. So, this
# generates an underscore-separated name which is more likely to
# be unique, and it doesn't really matter because the name isn't
# used directly here anyway.
mode = 'r'
if name is None:
basename = os.path.splitext(filename)[0]
name = '_'.join(os.path.relpath(basename).split(os.sep)[1:])
if not os.path.exists(filename):
raise ImportError('Could not import file {0}'.format(filename))
loader = import_machinery.SourceFileLoader(name, filename)
mod = loader.load_module()
return mod
def f(self):
imp = imp2 = None
try:
import imp
except ImportError:
import importlib.machinery as imp2
try:
if imp:
imp.load_source(test_name, fn)
else:
imp2.SourceFileLoader(test_dir, fn)
except example.util.DownloadError:
raise unittest.SkipTest('could not download required data file')
except ExternalProgramMissing as e:
raise unittest.SkipTest(str(e))
except ImportError as e:
raise unittest.SkipTest(str(e))
except topo.AuthenticationRequired as e:
raise unittest.SkipTest(
'cannot download topo data (no auth credentials)')
except Exception as e:
raise e
def _fake_import(fn, name):
from importlib import machinery
m = machinery.SourceFileLoader(name, fn)
try:
sys.modules[name] = m.load_module(name)
except FileNotFoundError:
raise ImportError('file %s not found' % ascii(fn))
def _load_file(moduleName, fileName):
if sys.version_info[:2] >= (3, 3):
from importlib import machinery
return machinery.SourceFileLoader(moduleName, fileName).load_module()
else:
import imp
return imp.load_source(moduleName, fileName)
def import_file(filename, name=None):
"""
Imports a module from a single file as if it doesn't belong to a
particular package.
The returned module will have the optional ``name`` if given, or else
a name generated from the filename.
"""
# Specifying a traditional dot-separated fully qualified name here
# results in a number of "Parent module 'astropy' not found while
# handling absolute import" warnings. Using the same name, the
# namespaces of the modules get merged together. So, this
# generates an underscore-separated name which is more likely to
# be unique, and it doesn't really matter because the name isn't
# used directly here anyway.
mode = 'U' if sys.version_info[0] < 3 else 'r'
if name is None:
basename = os.path.splitext(filename)[0]
name = '_'.join(os.path.relpath(basename).split(os.sep)[1:])
if import_machinery:
loader = import_machinery.SourceFileLoader(name, filename)
mod = loader.load_module()
else:
with open(filename, mode) as fd:
mod = imp.load_module(name, fd, filename, ('.py', mode, 1))
return mod
def read_envInfo(filepath):
'''
This function reads the parameters which will be used
to generate the model grid envelopes.
If the file is not in the right format and does not contain
all the needed infos an error message is returned.
filepath: string
'''
import importlib.machinery as imp
loader = imp.SourceFileLoader(filepath, filepath)
envInfo = loader.load_module()
attr = ['bathyFile', 'hgridFile', \
'e_min_ofs', 'e_max_dep', \
'e_loc_vel', 'e_loc_var', 'e_loc_vmx', \
'e_loc_rmx', 'e_loc_hal', 'e_glo_rmx']
errmsg = False
for a in attr:
if not hasattr(envInfo, a):
errmsg = True
attrerr = a
else:
if getattr(envInfo, a) == "":
errmsg = True
attrerr = a
if errmsg:
raise AttributeError('Attribute ' + attrerr + ' is missing')
return envInfo
def __init__(self, logger, config):
self._logger = logger
self._config = config
# strategy configファイル読み込み
strategy_config_file = self._config['strategy_yaml']
self.strategy_config = yaml.safe_load(
open(strategy_config_file, 'r', encoding='utf-8_sig'))
logger.info("Load strategy configfile: config = {}".format(
strategy_config_file))
# 動的に MyStrategy を読み込んでクラスを上書きする
strategy_py_file = self._config['strategy_py']
module = imm.SourceFileLoader(
'MyStrategy', strategy_py_file).load_module()
logger.info(
"Load MyStrategy class dynamically: module={}".format(module))
strategy_class = getattr(module, 'MyStrategy')
strategy = strategy_class(logger=self._logger, parent=self)
logger.info('Succeeded setup strategy. logic={}, class={}'.format(
self._config['strategy_py'], type(strategy)))
self._strategy = strategy
self._strategy.set_strategy_config(self.strategy_config['parameters'])
# 初回の初期化動作
self.initialize_func = None
try:
self.initialize_func = self._strategy.initialize
except:
pass
if self.initialize_func != None:
self._strategy.initialize()
self._logic_loop_period = self.strategy_config['logic_loop_period']
def __init__(self, folders: Union[str, List[str]]) -> None:
"""Import all menu models into the models namespace.
Instantiation of the namespace requires either a path to the folder that
contains model files or a list of such folders.
Parameters
----------
folders : Union[str, List[str]]
a folder or a list of folders to import models from
"""
if isinstance(folders, str):
folders = [folders]
for folder in folders:
menu_models = [(
f.replace("_model.py", ""),
os.path.abspath(os.path.join(folder, f)),
) for f in os.listdir(folder) if f.endswith("_model.py")]
for model_name, model_file in menu_models:
loader = machinery.SourceFileLoader(model_name, model_file)
spec = util.spec_from_loader(model_name, loader)
if spec is not None:
setattr(self, model_name, util.module_from_spec(spec))
loader.exec_module(getattr(self, model_name))
else:
pass
def test_timestamp_overflow(self):
# When a modification timestamp is larger than 2**32, it should be
# truncated rather than raise an OverflowError.
with source_util.create_modules('_temp') as mapping:
source = mapping['_temp']
compiled = imp.cache_from_source(source)
with open(source, 'w') as f:
f.write("x = 5")
try:
os.utime(source, (2**33 - 5, 2**33 - 5))
except OverflowError:
self.skipTest("cannot set modification time to large integer")
except OSError as e:
if e.errno != getattr(errno, 'EOVERFLOW', None):
raise
self.skipTest(
"cannot set modification time to large integer ({})".
format(e))
loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
mod = loader.load_module('_temp')
# Sanity checks.
self.assertEqual(mod.__cached__, compiled)
self.assertEqual(mod.x, 5)
# The pyc file was created.
os.stat(compiled)
def __initialize_plugins(self, configured_plugins, plugin_module_files):
# Initialize the plugins
for class_name in configured_plugins.keys():
if not plugin_module_files.get(class_name):
raise TidenPluginException(
'Python module not found in plugins/* for configured plugin %s'
% class_name)
plugin_file = plugin_module_files[class_name]
# Load module
loader = machinery.SourceFileLoader(
path.basename(plugin_file)[:-3], plugin_file)
spec = util.spec_from_loader(loader.name, loader)
plugin_module = util.module_from_spec(spec)
loader.exec_module(plugin_module)
preloaded_plugin_config = {
'file': plugin_file,
'class': class_name,
}
# Check mandatory constants in a plugin module
for const in self.mandatory_constants:
preloaded_plugin_config[const] = getattr(plugin_module, const)
# Get plugin options from config
plugin_opts = configured_plugins[class_name]
# Check version if needed
if not plugin_opts.get('version') or \
plugin_opts['version'] == preloaded_plugin_config['TIDEN_PLUGIN_VERSION']:
# Get the instance
preloaded_plugin_config['instance'] \
= getattr(plugin_module, class_name)(class_name, self.config)
self.plugins[class_name] = preloaded_plugin_config
configured_plugins[class_name]['module'] = plugin_file
def load_config(app_root, config_file):
logger.info("Running on Python {}.".format(sys.version))
logger.info("Loading a config file from {}".format(config_file))
version = sys.version_info.major
if version == 3:
from importlib import machinery
import tempfile
config = open(config_file).read()
config = config.replace("@@APP_ROOT@@", app_root)
with tempfile.NamedTemporaryFile(delete=True) as tf:
tf.write(config.encode('utf-8'))
temp_file_name = tf.name
module_ = machinery.SourceFileLoader('Config',
temp_file_name).load_module()
return module_.Config
elif version == 2:
"""
Python 2 does not have importlib.machinery
So far, a naive implementation for loading module.
"""
config = open(config_file).read()
config = config.replace("@@APP_ROOT@@", app_root)
exec(config, globals()) # Config object will be imported
return Config
else:
raise AssertionError
def execute_dsl(
self,
patcher: dag_runner_patcher.DagRunnerPatcher) -> Dict[str, Any]:
self._check_pipeline_dsl_path()
dsl_path = self.flags_dict[labels.PIPELINE_DSL_PATH]
with patcher.patch() as context:
# Simluate python script execution.
# - Need to add the script directory as a first entry of sys.path.
# - Load the script as if we are in __main__ module.
dir_path = os.path.dirname(os.path.realpath(dsl_path))
sys.path.insert(0, dir_path)
loader = machinery.SourceFileLoader('__main__', dsl_path)
try:
loader.exec_module(
import_util.module_from_spec(
import_util.spec_from_loader(loader.name, loader)))
except SystemExit as system_exit: # Swallow normal exit in absl.app.run()
if system_exit.code != 0 and system_exit.code is not None:
raise
sys.path.pop(0)
if not patcher.run_called:
sys.exit('Cannot find ' + patcher.get_runner_class().__name__ +
'.run() in ' + dsl_path)
return context
def main(args: Optional[List[str]] = None) -> None:
parser = argparse.ArgumentParser(
prog="i3pyblocks",
description="A replacement for i3status, written in Python using asyncio.",
)
parser.add_argument(
"-c",
"--config",
help="path to configuration file",
default=None,
required=False,
)
parser.add_argument(
"--version",
action="version",
version=f"%(prog)s {__version__}",
)
parsed_args = parser.parse_args(args=args)
if parsed_args.config:
# Set the config file's __name__ to "__main__", so we can use __main__
# guard in our config files
loader = machinery.SourceFileLoader("__main__", parsed_args.config)
mod = types.ModuleType(loader.name)
loader.exec_module(mod)
else:
from i3pyblocks import utils
utils.asyncio_run(config_example())
def main(args):
"""Configure and start the Kafka stream processor"""
# acquire a SparkSession object
spark = (
sql.SparkSession.builder.appName('kafka-spark-python').getOrCreate())
# if a user function is specified, download it and import it
if args.userfunction is not None:
try:
logging.info('downloading user function')
logging.info(args.userfunction)
dl = urllib.urlretrieve(args.userfunction)[0]
loader = importlib.SourceFileLoader('userfunction', dl)
userfunction = pytypes.ModuleType(loader.name)
loader.exec_module(userfunction)
user_function = functions.udf(userfunction.user_defined_function,
types.StringType())
logging.info('user function loaded')
except Exception as e:
logging.error('failed to import user function file')
logging.error(e)
user_function = None
# configure the operations to read the input topic
records = (
spark.readStream.format('kafka').option(
'kafka.bootstrap.servers',
args.brokers).option('subscribe', args.intopic).load().select(
functions.column('value').cast(
types.StringType()).alias('value'))
# add your data operations here, the raw message is passed along as
# the alias `value`.
#
# for example, to process the message as json and create the
# corresponding objects you could do the following:
#
# .select(
# functions.from_json(
# functions.column('value'), msg_struct).alias('json'))
#
# the following operations would then access the object and its
# properties using the name `json`.
)
# if it exists, add the user function to the stream pipeline
if user_function is not None:
records = (records.select(
user_function(functions.column('value')).alias('value')).where(
'value is not null'))
# configure the output stream
writer = (records.writeStream.format('kafka').option(
'kafka.bootstrap.servers',
args.brokers).option('topic',
args.outtopic).option('checkpointLocation',
'/tmp').start())
# begin processing the input and output topics
writer.awaitTermination()
def __init__(self, args):
self._tz = timezone.utc # utc timezone
self._ts = datetime.now(self._tz).timestamp()
if len(args) != 3:
print("number of args does not match")
exit()
with open(args[1], "r") as f:
jsonData = json.load(f)
self._config = jsonData
print('Configuration file {} loaded'.format(args[1]))
print(json.dumps(self._config, sort_keys=True, indent=4))
# logging configuration
logger = getLogger(__name__)
logger.setLevel(logging.DEBUG)
# create file handler
fh = logging.FileHandler('yourfilename.log')
fh.setLevel(logging.DEBUG)
fh_formatter = logging.Formatter(
'%(levelname)s : %(asctime)s : %(message)s')
fh.setFormatter(fh_formatter)
# create console handler
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch_formatter = logging.Formatter(
'%(levelname)s : %(asctime)s : %(message)s')
ch.setFormatter(ch_formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
self._logger = logger
if "LOG_LEVEL" in self._config:
if self._config["LOG_LEVEL"] in [
"CRITICAL",
"ERROR",
"WARNING",
"INFO",
"DEBUG",
]:
self._logger.setLevel(
eval("logging." + self._config["LOG_LEVEL"]))
self._exchange = Binance(symbol=self._config["SYMBOL"],
apiKey=self._config["APIKEY"],
secret=self._config["SECRET"],
logger=self._logger)
# load your strategy
module = machinery.SourceFileLoader("Strategy", args[2]).load_module()
self._Strategy = module.Strategy(self)
message = "Botter initialized with {}".format(args[1])
self._logger.info(message)
def get_hprams(parent_path):
hparams_file = "{}/hparams.py".format(parent_path)
check_existence(hparams_file)
hparams = machinery.SourceFileLoader('hparams', hparams_file).load_module()
env_hparams = hparams.env_hparams
nn_hparams = hparams.nn_hparams
agent_hparams = hparams.agent_hparams
return env_hparams, nn_hparams, agent_hparams
def _loadNetworksModule(modName, modPath):
""" Loads the module containing the relevant networks """
loader = im.SourceFileLoader(modName, modPath)
mod = types.ModuleType(loader.name)
loader.exec_module(mod)
return mod
def create_loader(pack_name, filepath):
ext = filepath.ext
if ext in ilm.SOURCE_SUFFIXES:
return ilm.SourceFileLoader(pack_name, str(filepath))
if ext in ilm.BYTECODE_SUFFIXES:
return ilm.SourcelessFileLoader(pack_name, str(filepath))
if ext in ilm.EXTENSION_SUFFIXES:
return ilm.ExtensionFileLoader(pack_name, str(filepath))
def __init__(self, path):
self.path = pathlib.Path(path)
name = str(self.path.parent / self.path.stem)
name = name.replace("/", ".")
self.name = re.sub(r"^[\.]+", "", name)
self.module = imm.SourceFileLoader(self.name, path).load_module()
if not hasattr(self.module, "get_parser"):
raise ValueError(f"{path} does not have get_parser()")
def test_bad_syntax(self):
with source_util.create_modules('_temp') as mapping:
with open(mapping['_temp'], 'w') as file:
file.write('=')
loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.load_module('_temp')
self.assertNotIn('_temp', sys.modules)
def load_file_as_module(name):
path = os.path.join(os.path.dirname(bootstrap_file), "%s.py" % name)
if sys.version_info >= (3, 3):
from importlib import machinery
mod = machinery.SourceFileLoader(name, path).load_module()
else:
import imp
mod = imp.load_source(name, path)
return mod
