def _get_settings(self):
self.settings = st.environ(self.environ).settings.get(
self.prefix, None)
if self.settings is None:
raise SettingErrors("Can not find settings.FLASK!")
if self.settings.get("name") is None:
self.settings["name"] = "SwitchTracer"
self.roles = [i[1:] for i in self.settings.keys() if i.startswith("@")]
if not self.roles:
raise SettingErrors("Can not find any ROLEs in settings.FLASK!")
def load_tasks_from_path(packs, reset=False, env=None):
settings = _get_celery_settings(env)
package_path = os.path.dirname(packs)
# print(package_path)
is_organized = eval(os.environ.get("CELERY_ORGANIZED", "0"))
if reset or (not is_organized):
# TODO: RAISE INFO MSG
print("Info: Insert TaskPackage into sys.path")
sys.path.append(package_path)
searching = settings.get("tasks_search_pattern")
if searching is None:
raise SettingErrors("No tasks search pattern!")
# r"^(([_a-zA-Z0-9\.]+\.)?task[_a-zA-Z0-9]*).py$"
pattern = re.compile(searching)
def _import(x):
ret = re.match(pattern, x)
if ret:
import_module(ret.group(1))
if os.path.isdir(packs):
for i in os.listdir(packs):
_import(".".join([os.path.split(packs)[-1], i]))
else:
for i in os.listdir(package_path):
_import(i)
os.environ["CELERY_ORGANIZED"] = "1"
def task_urls_routers(url, records=True, gmap=None, root=None, env=None):
settings = st.environ(env).settings
if not (settings.get("DEFAULT_TASKS_PREFIX")
and settings.get("DEFAULT_TASKS_ROOT")):
raise SettingErrors("Can not find settings.TASKS refers!")
temp = re.match(PATTERN, url)
if temp is None:
return
router_dict = temp.groupdict()
prefix, mprefix, fprefix = router_dict["prefix"], \
router_dict["mprefix"], \
router_dict["fprefix"] or st.environ(env).settings["DEFAULT_TASKS_PREFIX"]
root = root or st.environ(env).settings["DEFAULT_TASKS_ROOT"]
_imported_list = list()
if mprefix:
for i in os.listdir(os.path.join(root, *prefix.split('.'))):
ret = re.match("(%s)\.py" % mprefix, i)
if ret:
_imported_list.append(
import_module(".".join([prefix, ret.group(1)])))
else:
_imported_list.append(import_module(prefix))
def _import(p):
for k in filter(lambda x: x.startswith(fprefix), dir(p)):
task = getattr(p, k)
gmap[task.name] = pickle.dumps(task)
if records and hasattr(gmap, "__setitem__"):
for i in _imported_list:
_import(i)
def connect(self,
purl: dict = None,
query_dict: dict = None,
data: dict = None,
**kwargs):
self.kwargs = kwargs
method, url = self.url
try:
url = url.format(**(purl or dict()))
except (IndexError, KeyError):
raise SettingErrors("Wrong url params from settings.COURIER.")
sender = getattr(requests, method.lower())
if sender is None:
raise SettingErrors("No such Http request method<%s>." % method)
return self.resp_parser(
sender(url=url, params=query_dict, data=data or dict()))
def server_loc(self, key: str):
try:
ret = import_from_path(self.settings["SERVERS"][key])
except IndexError as e:
raise SettingErrors(e)
except ImportedErrors as e:
raise SetupErrors(e)
return ret
def __call__(self, role):
if role not in self.roles:
raise SettingErrors("Can not find ROLE<%s> in settings.FLASK!" %
role)
role = "@%s" % role
app = Flask(self.settings["name"])
app.config.from_mapping(**self.settings[role])
return app
def get_settings(self):
celery_settings = self.settings["CELERY"]
try:
rds = celery_settings["Redis"]
broker = rds["broker_url"]
backend = rds["result_backend"]
except KeyError as e:
raise SettingErrors(e)
self.broker_url = self.resolute_redis_url(broker)
self.backend_url = self.resolute_redis_url(backend)
def set_celery_from_conf(capp=None, broker="Redis", env=None, _return=False):
settings = _get_celery_settings(env)
broker = settings.get(broker, None)
if broker is None:
raise SettingErrors("No such broker!")
capp = capp or Celery(settings.get("routine_name", "SwitchTracer"))
capp.conf.broker_url = broker["broker_url"]
capp.conf.result_backend = broker["result_backend"]
if _return:
return capp
def _get_blue_printer(self):
fmod = self.settings.get("server") or \
st.environ(self.environ).settings["SERVERS"]["flask"]
try:
self.blue_printers = {
i: import_from_path("{flask}.{bprinter}".format(
flask=fmod, bprinter=i)).bluePrinters
for i in self.roles
}
except Exception as e:
raise SettingErrors(e)
def __new__(mcs, name, bases, kw_attrs):
if mcs.__is_single or kw_attrs.get("_is_reload", False):
kw_attrs["VOLUMES"] = kw_attrs.get(
"_default_global_volumes") or st.VOLUMES
settings = st.environ(kw_attrs.get("_environ", None)).settings
if settings.get("DEFAULT_TASKS_URLS") is None:
raise SettingErrors("Can not find settings.TASKS refers!")
kw_attrs["_TASKS"] = kw_attrs.get(
"_default_tasks_urls") or settings["DEFAULT_TASKS_URLS"]
mcs.__is_single = False
kw_attrs["_is_reload"] = False
return type(name, bases, kw_attrs)
def __init__(self, env=None):
self.environ = env
self.settings = st.environ(self.environ).settings.get(
self.__prefix__, None)
if self.settings is None:
raise SettingErrors("Can not find settings.COURIER!")
# TODO: get redis pool
self.redis_pool = None
self.url = self.get_ulr(self.settings.get("master") or self.url_master)
self.fburl = self.settings.get("seed") or self.url_feedback
self.kwargs = dict()
def __init__(self, env):
if self.default_global_registers_map is None or self.default_global_records_list is None:
raise KernelWaresSettingsErrors("No VOLUMES has been Linked!")
setattr(self, "__REGISTERS", self.default_global_registers_map)
setattr(self, "__RECORDS", self.default_global_records_list)
settings = st.environ(env or self.__environ__).settings
self.__dynamic_pool__ = self.__dynamic_pool__ or settings.get(
"DEFAULT_DYNAMIC_POOL_INFO")
if not isinstance(self.__dynamic_pool__, dict):
raise SettingErrors(
"Can Not found dynamic pool info of multiprocessing in settings.RESOLUTER"
)
self.max_pool = self.__max_pool__ or settings.get("DEFAULT_MAX_POOL")
def __getattribute__(self, item):
try:
return super(CourierConfigure, self).__getattribute__(item)
except AttributeError:
raise SettingErrors("Could not find key<%s> in conf" % item)
def resolute_redis_url(self, url):
ret = re.match(self.celery_url_pattern, url)
if ret is None:
raise SettingErrors("Illegal Redis url in settings!")
return ret.groupdict()
def __getitem__(self, key):
v = self._settings.get(key, None)
if v is None:
raise SettingErrors("can Not found '%s' in settings" % key)
return v
def settings(self):
settings = st.environ(self.environ).settings.get(self.prefix)
if settings is None:
raise SettingErrors("Can not find settings.COURIER!")
return settings
def register_bprinter(self, app, role):
if role not in self.roles:
raise SettingErrors("Can not find ROLE<%s> in settings.FLASK!" %
role)
for bp in self.blue_printers[role]:
app.register_blueprint(bp)
def _get_celery_settings(env):
settings = st.environ(env).settings.get("CELERY", None)
if settings is None:
raise SettingErrors("Can not find settings.CELERY!")
return settings
