async def main():
# Read config file
config = Config("config.yaml")
# Configure the database
store = Storage(config.database_filepath)
# Configuration options for the AsyncClient
client_config = AsyncClientConfig(
max_limit_exceeded=0,
max_timeouts=0,
store_sync_tokens=True,
encryption_enabled=config.enable_encryption,
)
# Initialize the matrix client
client = AsyncClient(
config.homeserver_url,
config.user_id,
device_id=config.device_id,
store_path=config.store_filepath,
config=client_config,
)
# Set up event callbacks
callbacks = Callbacks(client, store, config)
client.add_event_callback(callbacks.message, (RoomMessageText,))
client.add_event_callback(callbacks.invite, (InviteMemberEvent,))
# Keep trying to reconnect on failure (with some time in-between)
while True:
try:
# Try to login with the configured username/password
try:
login_response = await client.login(
password=config.user_password,
device_name=config.device_name,
)
# Check if login failed
if type(login_response) == LoginError:
logger.error(f"Failed to login: %s", login_response.message)
return False
except LocalProtocolError as e:
# There's an edge case here where the user enables encryption but hasn't installed
# the correct C dependencies. In that case, a LocalProtocolError is raised on login.
# Warn the user if these conditions are met.
if config.enable_encryption:
logger.fatal(
"Failed to login and encryption is enabled. Have you installed the correct dependencies? "
"https://github.com/poljar/matrix-nio#installation"
)
return False
else:
# We don't know why this was raised. Throw it at the user
logger.fatal("Error logging in: %s", e)
return False
# Login succeeded!
# Sync encryption keys with the server
# Required for participating in encrypted rooms
if client.should_upload_keys:
await client.keys_upload()
logger.info(f"Logged in as {config.user_id}")
await client.sync_forever(timeout=30000, full_state=True)
except (ClientConnectionError, ServerDisconnectedError):
logger.warning("Unable to connect to homeserver, retrying in 15s...")
# Sleep so we don't bombard the server with login requests
sleep(15)
finally:
# Make sure to close the client connection on disconnect
await client.close()
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.network = Network(settings)
self.storage = Storage(settings)
def __init__(
self,
notebook_name,
path,
mask,
format,
twoway=False,
download_only=False,
nodownsync=False,
sleep_on_ratelimit=False,
imageOptions={
"saveImages": False,
"imagesInSubdir": False
},
):
# check auth
if not Storage().getUserToken():
raise Exception("Auth error. There is not any oAuthToken.")
# set path
if not path:
raise Exception("Path to sync directories does not select.")
if not os.path.exists(path):
raise Exception("Path to sync directories does not exist. %s" %
path)
self.path = path
# set mask
if not mask:
mask = "*.*"
self.mask = mask
# set format
if not format:
format = "plain"
self.format = format
if format == "markdown":
self.extension = ".md"
elif format == "html":
self.extension = ".html"
else:
self.extension = ".txt"
self.twoway = twoway
self.download_only = download_only
self.nodownsync = nodownsync
logger.info("Sync Start")
# set notebook
self.notebook_guid, self.notebook_name = self._get_notebook(
notebook_name, path)
# set image options
self.imageOptions = imageOptions
# all is Ok
self.all_set = True
self.sleep_on_ratelimit = sleep_on_ratelimit
def getStorage(self):
if GeekNote.storage:
return GeekNote.storage
GeekNote.storage = Storage()
return GeekNote.storage
def __init__(self):
storage = Storage()
templates_bucket = 'abdp-templates'
folder = 'spark'
DagHandler.template = storage.copy_bucket(
'/app/template/target/tog-0.1.jar', templates_bucket, folder)
def __init__(self, output_file):
# Initialize storage class
self.__parkStorage = Storage()
self.__today = None
self.__output = open(output_file, 'w')
def test_get_draft_non_existent_file():
storage = Storage("some/path")
assert storage.get_draft(user_id, "bib", "2") == None
def get_effective_router(appname):
"return a private copy of the effective router for the specified application"
if not routers or appname not in routers:
return None
return Storage(routers[appname]) # return a copy
def load_routers(all_apps):
"load-time post-processing of routers"
for app in routers.keys():
# initialize apps with routers that aren't present, on behalf of unit tests
if app not in all_apps:
all_apps.append(app)
router = Storage(routers.BASE) # new copy
if app != 'BASE':
for key in routers[app].keys():
if key in ROUTER_BASE_KEYS:
raise SyntaxError, "BASE-only key '%s' in router '%s'" % (
key, app)
router.update(routers[app])
routers[app] = router
router = routers[app]
for key in router.keys():
if key not in ROUTER_KEYS:
raise SyntaxError, "unknown key '%s' in router '%s'" % (key,
app)
if not router.controllers:
router.controllers = set()
elif not isinstance(router.controllers, str):
router.controllers = set(router.controllers)
if router.functions:
router.functions = set(router.functions)
else:
router.functions = set()
if router.languages:
router.languages = set(router.languages)
else:
router.languages = set()
if app != 'BASE':
for base_only in ROUTER_BASE_KEYS:
router.pop(base_only, None)
if 'domain' in router:
routers.BASE.domains[router.domain] = app
if isinstance(router.controllers,
str) and router.controllers == 'DEFAULT':
router.controllers = set()
if os.path.isdir(abspath('applications', app)):
cpath = abspath('applications', app, 'controllers')
for cname in os.listdir(cpath):
if os.path.isfile(abspath(
cpath, cname)) and cname.endswith('.py'):
router.controllers.add(cname[:-3])
if router.controllers:
router.controllers.add('static')
router.controllers.add(router.default_controller)
if router.functions:
router.functions.add(router.default_function)
if isinstance(routers.BASE.applications,
str) and routers.BASE.applications == 'ALL':
routers.BASE.applications = list(all_apps)
if routers.BASE.applications:
routers.BASE.applications = set(routers.BASE.applications)
else:
routers.BASE.applications = set()
for app in routers.keys():
# set router name
router = routers[app]
router.name = app
# compile URL validation patterns
router._acfe_match = re.compile(router.acfe_match)
router._file_match = re.compile(router.file_match)
if router.args_match:
router._args_match = re.compile(router.args_match)
# convert path_prefix to a list of path elements
if router.path_prefix:
if isinstance(router.path_prefix, str):
router.path_prefix = router.path_prefix.strip('/').split('/')
# rewrite BASE.domains as tuples
#
# key: 'domain[:port]' -> (domain, port)
# value: 'application[/controller] -> (application, controller)
# (port and controller may be None)
#
domains = dict()
if routers.BASE.domains:
for (domain, app) in [(d.strip(':'), a.strip('/'))
for (d, a) in routers.BASE.domains.items()]:
port = None
if ':' in domain:
(domain, port) = domain.split(':')
ctlr = None
if '/' in app:
(app, ctlr) = app.split('/')
if app not in all_apps and app not in routers:
raise SyntaxError, "unknown app '%s' in domains" % app
domains[(domain, port)] = (app, ctlr)
routers.BASE.domains = domains
def run(self):
self.set_tid()
self.log = logging.LoggerAdapter(
logging.getLogger(rcEnv.nodename + ".osvcd.dns"), {
"node": rcEnv.nodename,
"component": self.name
})
self.cache = {}
if not os.path.exists(rcEnv.paths.dnsuxsockd):
os.makedirs(rcEnv.paths.dnsuxsockd)
try:
if os.path.isdir(rcEnv.paths.dnsuxsock):
shutil.rmtree(rcEnv.paths.dnsuxsock)
else:
os.unlink(rcEnv.paths.dnsuxsock)
except Exception:
pass
try:
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.bind(rcEnv.paths.dnsuxsock)
self.sock.listen(1)
self.sock.settimeout(self.sock_tmo)
except socket.error as exc:
self.alert("error", "bind %s error: %s", rcEnv.paths.dnsuxsock,
exc)
return
self.log.info("listening on %s", rcEnv.paths.dnsuxsock)
self.zone = "%s." % self.cluster_name.strip(".")
self.suffix = ".%s" % self.zone
self.suffix_len = len(self.suffix)
self.soa_data = {
"origin": self.origin,
"contact": self.contact,
"serial": 1,
"refresh": 7200,
"retry": 3600,
"expire": 432000,
"minimum": 86400,
}
self.soa_content = "%(origin)s %(contact)s %(serial)d %(refresh)d " \
"%(retry)d %(expire)d %(minimum)d" % self.soa_data
self.stats = Storage({
"sessions":
Storage({
"accepted": 0,
"tx": 0,
"rx": 0,
}),
})
while True:
try:
self.do()
except Exception as exc:
self.log.exception(exc)
if self.stopped():
self.log.debug(
"stop event received (%d handler threads to join)",
len(self.threads))
self.join_threads()
self.sock.close()
sys.exit(0)
path + traceback.format_exc())
raise e
p = _params_default(app)
for sym in ('routes_app', 'routes_in', 'routes_out'):
if sym in symbols:
for (k, v) in symbols[sym]:
p[sym].append(compile_regex(k, v))
for sym in ('routes_onerror', 'routes_apps_raw', 'error_handler',
'error_message', 'error_message_ticket', 'default_application',
'default_controller', 'default_function'):
if sym in symbols:
p[sym] = symbols[sym]
if 'routers' in symbols:
p.routers = Storage(symbols['routers'])
for key in p.routers:
if isinstance(p.routers[key], dict):
p.routers[key] = Storage(p.routers[key])
if app is None:
params = p # install base rewrite parameters
thread.routes = params # install default as current routes
#
# create the BASE router if routers in use
#
routers = params.routers # establish routers if present
if isinstance(routers, dict):
routers = Storage(routers)
if routers is not None:
router = _router_default()
async def main_verify() -> None:
global client
# Read config file
config = Config("config.yaml")
# Configure the database
store = Storage(config.database_filepath)
# Configuration options for the AsyncClient
client_config = AsyncClientConfig(
max_limit_exceeded=0,
max_timeouts=0,
store_sync_tokens=True,
encryption_enabled=config.enable_encryption,
)
# Initialize the matrix client
client = AsyncClient(config.homeserver_url,
config.user_id,
device_id=config.device_id,
store_path=config.store_filepath,
config=client_config,
ssl=config.ssl)
# Set up event callbacks
callbacks = Callbacks(client, store, config)
client.add_to_device_callback(callbacks.to_device_callback,
(KeyVerificationEvent, ))
# Keep trying to reconnect on failure (with some time in-between)
error_retries: int = 0
while True:
try:
# Try to login with the configured username/password
try:
login_response = await client.login(
password=config.user_password,
device_name=config.device_name,
)
# Check if login failed
if type(login_response) == LoginError:
logger.error(
f"Failed to login: {login_response.message}, retrying in 15s... ({error_retries})"
)
# try logging in a few times to work around temporary login errors during homeserver restarts
if error_retries < 3:
error_retries += 1
await sleep(15)
continue
else:
return False
else:
error_retries = 0
except LocalProtocolError as e:
# There's an edge case here where the user enables encryption but hasn't installed
# the correct C dependencies. In that case, a LocalProtocolError is raised on login.
# Warn the user if these conditions are met.
if config.enable_encryption:
logger.fatal(
"Failed to login and encryption is enabled. Have you installed the correct dependencies? "
"https://github.com/poljar/matrix-nio#installation")
return False
else:
# We don't know why this was raised. Throw it at the user
logger.fatal(f"Error logging in: {e}")
# Login succeeded!
# Sync encryption keys with the server
# Required for participating in encrypted rooms
if client.should_upload_keys:
await client.keys_upload()
logger.info(f"Logged in as {config.user_id}")
await client.sync_forever(timeout=30000, full_state=True)
except (ClientConnectionError, ServerDisconnectedError, AttributeError,
asyncio.TimeoutError) as err:
logger.debug(err)
logger.warning(
f"Unable to connect to homeserver, retrying in 15s...")
# Sleep so we don't bombard the server with login requests
await sleep(15)
finally:
# Make sure to close the client connection on disconnect
await client.close()
def __init__(self, testing=False):
self.st = Storage(testing=testing)
self.sl = ServerList(testing=testing)
app.url_map.converters["repopath"] = RepositoryPathConverter
app.url_map.converters["apirepopath"] = APIRepositoryPathConverter
Principal(app, use_sessions=False)
tf = app.config["DB_TRANSACTION_FACTORY"]
model_cache = get_model_cache(app.config)
avatar = Avatar(app)
login_manager = LoginManager(app)
mail = Mail(app)
prometheus = PrometheusPlugin(app)
chunk_cleanup_queue = WorkQueue(app.config["CHUNK_CLEANUP_QUEUE_NAME"], tf)
instance_keys = InstanceKeys(app)
ip_resolver = IPResolver(app)
storage = Storage(app, chunk_cleanup_queue, instance_keys, config_provider,
ip_resolver)
userfiles = Userfiles(app, storage)
log_archive = LogArchive(app, storage)
analytics = Analytics(app)
user_analytics = UserAnalytics(app)
billing = Billing(app)
sentry = Sentry(app)
build_logs = BuildLogs(app)
authentication = UserAuthentication(app, config_provider,
OVERRIDE_CONFIG_DIRECTORY)
userevents = UserEventsBuilderModule(app)
superusers = SuperUserManager(app)
signer = Signer(app, config_provider)
instance_keys = InstanceKeys(app)
label_validator = LabelValidator(app)
build_canceller = BuildCanceller(app)
from multiprocessing.pool import Pool
import pandas as pd
from storage import Storage
import os
from get_factors import get_factors
from portfolio_selection import portfolio_selection
from get_position import get_position
from earnings import earnings
from param_effect import extract_data
from get_stock import get_data, split_data
store_path = os.path.join("data", '2013-06-01')
storage = Storage(store_path)
def random_index(stock):
"""
:param stock: T*N
:return: 不重复抽75支股票
"""
import random
index_list = random.sample(list(stock.columns),75)
return index_list
def function(trading_start):
path = os.path.join("data", trading_start)
storage.path = path
# Step 1:
# 获取股价数据,900多能做空的股票
stock_matrix, stock_estimation, stock_trade = get_data(trading_start, '生物制药', window=60)
def filter_url(url,
method='get',
remote='0.0.0.0',
out=False,
app=False,
lang=None,
domain=(None, None),
env=False):
"doctest/unittest interface to regex_filter_in() and regex_filter_out()"
regex_url = re.compile(
r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')
match = regex_url.match(url)
scheme = match.group('scheme').lower()
host = match.group('host').lower()
uri = match.group('uri')
k = uri.find('?')
if k < 0:
k = len(uri)
(path_info, query_string) = (uri[:k], uri[k + 1:])
path_info = urllib.unquote(path_info) # simulate server
e = {
'REMOTE_ADDR': remote,
'REQUEST_METHOD': method,
'WSGI_URL_SCHEME': scheme,
'HTTP_HOST': host,
'REQUEST_URI': uri,
'PATH_INFO': path_info,
'QUERY_STRING': query_string,
#for filter_out request.env use lowercase
'remote_addr': remote,
'request_method': method,
'wsgi_url_scheme': scheme,
'http_host': host
}
request = Storage()
e["applications_parent"] = global_settings.applications_parent
request.env = Storage(e)
request.uri_language = lang
# determine application only
#
if app:
if routers:
return map_url_in(request, e, app=True)
return regex_select(e)
# rewrite outbound URL
#
if out:
(request.env.domain_application,
request.env.domain_controller) = domain
items = path_info.lstrip('/').split('/')
if items[-1] == '':
items.pop() # adjust trailing empty args
assert len(items) >= 3, "at least /a/c/f is required"
a = items.pop(0)
c = items.pop(0)
f = items.pop(0)
if not routers:
return regex_filter_out(uri, e)
acf = map_url_out(request, a, c, f, items)
if items:
url = '%s/%s' % (acf, '/'.join(items))
if items[-1] == '':
url += '/'
else:
url = acf
if query_string:
url += '?' + query_string
return url
# rewrite inbound URL
#
(static, e) = url_in(request, e)
if static:
return static
result = "/%s/%s/%s" % (request.application, request.controller,
request.function)
if request.extension and request.extension != 'html':
result += ".%s" % request.extension
if request.args:
result += " %s" % request.args
if e['QUERY_STRING']:
result += " ?%s" % e['QUERY_STRING']
if request.uri_language:
result += " (%s)" % request.uri_language
if env:
return request.env
return result
def __init__(self, rid, type="data", **kwargs):
Resource.__init__(self, rid, type=type)
self.options = Storage(kwargs)
def test_pickling(self):
""" Test storage pickling """
s = Storage(a=1)
sd = pickle.dumps(s, pickle.HIGHEST_PROTOCOL)
news = pickle.loads(sd)
self.assertEqual(news.a, 1)
def setup():
global storage
storage = Storage(basedir + "some/path")
global user_id
user_id = u'12345'
def instruction_process_engine(self):
ps = r.pubsub(ignore_subscribe_messages=False)
ps.subscribe(config['instruction_channel'])
while True:
if Utils.exit_flag:
msg = 'Thread instruction_process_engine say bye-bye'
print msg
logger.info(msg=msg)
return
threads_status['instruction_process_engine'] = {
'timestamp': ji.Common.ts()
}
msg = dict()
extend_data = dict()
try:
msg = ps.get_message(timeout=config['engine_cycle_interval'])
if msg is None or 'data' not in msg or not isinstance(
msg['data'], basestring):
continue
try:
msg = json.loads(msg['data'])
if msg['action'] == 'pong':
continue
if msg['action'] == 'ping':
# 通过 ping pong 来刷存在感。因为经过实际测试发现，当订阅频道长时间没有数据来往，那么订阅者会被自动退出。
r.publish(config['instruction_channel'],
message=json.dumps({'action': 'pong'}))
continue
except ValueError as e:
log_emit.error(e.message)
continue
if 'node_id' in msg and int(msg['node_id']) != self.node_id:
continue
# 下列语句繁琐写法如 <code>if '_object' not in msg or 'action' not in msg:</code>
if not all([key in msg for key in ['_object', 'action']]):
continue
logger.info(msg=msg)
if msg['_object'] == 'guest':
self.refresh_dom_mapping()
if msg['action'] not in ['create']:
self.dom = self.dom_mapping_by_uuid[msg['uuid']]
assert isinstance(self.dom, libvirt.virDomain)
if msg['action'] == 'create':
t = threading.Thread(target=Guest.create,
args=(self.conn, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'reboot':
Guest.reboot(dom=self.dom)
elif msg['action'] == 'force_reboot':
Guest.force_reboot(dom=self.dom, msg=msg)
elif msg['action'] == 'shutdown':
Guest.shutdown(dom=self.dom)
elif msg['action'] == 'force_shutdown':
Guest.force_shutdown(dom=self.dom)
elif msg['action'] == 'boot':
Guest.boot(dom=self.dom, msg=msg)
elif msg['action'] == 'suspend':
Guest.suspend(dom=self.dom)
elif msg['action'] == 'resume':
Guest.resume(dom=self.dom)
elif msg['action'] == 'delete':
Guest.delete(dom=self.dom, msg=msg)
elif msg['action'] == 'reset_password':
Guest.reset_password(dom=self.dom, msg=msg)
elif msg['action'] == 'attach_disk':
Guest.attach_disk(dom=self.dom, msg=msg)
elif msg['action'] == 'detach_disk':
Guest.detach_disk(dom=self.dom, msg=msg)
elif msg['action'] == 'update_ssh_key':
Guest.update_ssh_key(dom=self.dom, msg=msg)
elif msg['action'] == 'allocate_bandwidth':
t = threading.Thread(target=Guest.allocate_bandwidth,
args=(self.dom, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'adjust_ability':
t = threading.Thread(target=Guest.adjust_ability,
args=(self.dom, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'migrate':
Guest().migrate(dom=self.dom, msg=msg)
elif msg['_object'] == 'disk':
if msg['action'] == 'create':
Storage(storage_mode=msg['storage_mode'],
dfs_volume=msg['dfs_volume']).make_image(
path=msg['image_path'], size=msg['size'])
elif msg['action'] == 'delete':
Storage(storage_mode=msg['storage_mode'],
dfs_volume=msg['dfs_volume']).delete_image(
path=msg['image_path'])
elif msg['action'] == 'resize':
mounted = True if msg['guest_uuid'].__len__(
) == 36 else False
if mounted:
self.refresh_dom_mapping()
self.dom = self.dom_mapping_by_uuid[
msg['guest_uuid']]
# 在线磁盘扩容
if mounted and self.dom.isActive():
# 磁盘大小默认单位为KB，乘以两个 1024，使其单位达到 GiB
msg['size'] = int(msg['size']) * 1024 * 1024
# https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainBlockResize
self.dom.blockResize(disk=msg['device_node'],
size=msg['size'])
Guest.quota(dom=self.dom, msg=msg)
# 离线磁盘扩容
else:
Storage(storage_mode=msg['storage_mode'],
dfs_volume=msg['dfs_volume']).resize_image(
path=msg['image_path'],
size=msg['size'])
elif msg['action'] == 'quota':
self.refresh_dom_mapping()
self.dom = self.dom_mapping_by_uuid[msg['guest_uuid']]
Guest.quota(dom=self.dom, msg=msg)
elif msg['_object'] == 'snapshot':
self.refresh_dom_mapping()
self.dom = self.dom_mapping_by_uuid[msg['uuid']]
if msg['action'] == 'create':
t = threading.Thread(target=Guest.create_snapshot,
args=(self.dom, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'delete':
t = threading.Thread(target=Guest.delete_snapshot,
args=(self.dom, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'revert':
t = threading.Thread(target=Guest.revert_snapshot,
args=(self.dom, msg))
t.setDaemon(False)
t.start()
continue
elif msg['action'] == 'convert':
t = threading.Thread(target=Guest.convert_snapshot,
args=(msg, ))
t.setDaemon(False)
t.start()
continue
elif msg['_object'] == 'os_template_image':
if msg['action'] == 'delete':
Storage(storage_mode=msg['storage_mode'],
dfs_volume=msg['dfs_volume']).delete_image(
path=msg['template_path'])
elif msg['_object'] == 'global':
if msg['action'] == 'refresh_guest_state':
t = threading.Thread(target=Host().refresh_guest_state,
args=())
t.setDaemon(False)
t.start()
continue
if msg['action'] == 'upgrade':
try:
log = self.upgrade(msg['url'])
log_emit.info(msg=log)
except subprocess.CalledProcessError as e:
log_emit.warn(e.output)
self.rollback()
continue
log = self.restart()
log_emit.info(msg=log)
if msg['action'] == 'restart':
log = self.restart()
log_emit.info(msg=log)
else:
err = u'未支持的 _object：' + msg['_object']
log_emit.error(err)
response_emit.success(
_object=msg['_object'],
action=msg['action'],
uuid=msg['uuid'],
data=extend_data,
passback_parameters=msg.get('passback_parameters'))
except KeyError as e:
log_emit.warn(e.message)
if msg['_object'] == 'guest':
if msg['action'] == 'delete':
response_emit.success(
_object=msg['_object'],
action=msg['action'],
uuid=msg['uuid'],
data=extend_data,
passback_parameters=msg.get('passback_parameters'))
except:
# 防止循环线程，在redis连接断开时，混水写入日志
time.sleep(5)
log_emit.error(traceback.format_exc())
response_emit.failure(
_object=msg['_object'],
action=msg.get('action'),
uuid=msg.get('uuid'),
passback_parameters=msg.get('passback_parameters'))
def main():
terminal = StorageTerminal(Storage(StorageTerminal.get_cell_count()),
r'\d{2}:\d{2}', "%H:%M")
terminal.run_storage()
def make_app():
return tornado.web.Application([
(r"/api/v1/file/upload", UploadFile),
(r"/api/v1/file/remove/?(.*)?", RemoveFile),
],
storage=Storage(bucket_name='storage-hub'))
"""
This file is part of the web2py Web Framework
Copyrighted by Massimo Di Pierro <*****@*****.**>
License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
"""
import os
import sys
import platform
from storage import Storage
global_settings = Storage()
settings = global_settings # legacy compatibility
if not hasattr(os, 'mkdir'):
global_settings.db_sessions = True
if global_settings.db_sessions is not True:
global_settings.db_sessions = set()
global_settings.gluon_parent = os.environ.get('web2py_path', os.getcwd())
global_settings.applications_parent = global_settings.gluon_parent
global_settings.app_folders = set()
global_settings.debugging = False
global_settings.is_pypy = hasattr(platform,'python_implementation') and \
platform.python_implementation() == 'PyPy'
def get_stored():
storage = Storage()
stored = storage.get_all()
return [(c['symbol'], pickle.loads(c['df'])) for c in stored]
OPT = Storage({
"add":
Option("--add",
default=None,
action="store",
help="A list member to add to the value pointed by :opt:`--param`. "
"If :opt:`--index` is set, insert the new element at the "
"specified position in the list."),
"color":
Option("--color",
default="auto",
action="store",
dest="color",
help="Colorize output. Possible values are:\n\n"
"* auto: guess based on tty presence\n"
"* always|yes: always colorize\n"
"* never|no: never colorize"),
"config":
Option(
"--config",
default=None,
action="store",
dest="parm_config",
help="The configuration to use as template when creating or "
"installing a service. The value can be ``-`` or ``/dev/stdin`` "
"to read the json-formatted configuration from stdin, or a file "
"path, or uri pointing to a ini-formatted configuration, or a "
"service selector expression (ATTENTION with cloning existing live "
"services that include more than containers, volumes and backend "
"ip addresses ... this could cause disruption on the cloned service)."
),
"debug":
Option(
"--debug",
default=False,
action="store_true",
dest="debug",
help="Increase stream and file log verbosity up to the debug level."),
"daemon":
Option("--daemon",
default=False,
action="store_true",
dest="daemon",
help="A flag inhibiting the command daemonization. Set by the "
"daemonization routine."),
"disable_rollback":
Option("--disable-rollback",
default=False,
action="store_true",
dest="disable_rollback",
help="If set, don't try to rollback resources activated before a "
"start action interrupts on error."),
"discard":
Option("--discard",
default=False,
action="store_true",
dest="discard",
help="Discard the stashed, invalid, configuration file."),
"dry_run":
Option("--dry-run",
default=False,
action="store_true",
dest="dry_run",
help="Show the action execution plan."),
"env":
Option("--env",
default=[],
action="append",
dest="env",
help="Export the uppercased variable in the os environment.\n\n"
"With the create action only, set a env section parameter in "
"the service configuration file. Multiple ``--env <key>=<val>`` "
"can be specified. For all other actions."),
"eval":
Option(
"--eval",
default=False,
action="store_true",
dest="eval",
help="If set with the :cmd:`svcmgr get` action, the printed value of "
":opt:`--param` is evaluated, scoped and dereferenced. If set "
"with the :cmd:`svcmgr set` action, the current value is "
"evaluated before mangling."),
"filter":
Option("--filter",
default="",
action="store",
dest="jsonpath_filter",
help="A JSONPath expression to filter a JSON output."),
"force":
Option("-f",
"--force",
default=False,
action="store_true",
dest="force",
help="Force action, ignore sanity checks."),
"format":
Option(
"--format",
default=None,
action="store",
dest="format",
help="Specify a data formatter. Possible values are json, flat_json, "
"csv or table. csv and table formatters are available only for "
"commands returning tabular data."),
"help":
Option("-h",
"--help",
default=None,
action="store_true",
dest="parm_help",
help="Show this help message and exit."),
"hide_disabled":
Option(
"--hide-disabled",
default=None,
action="store_false",
dest="show_disabled",
help="Do not include the disabled resources. This option supersedes "
"the :kw:`show_disabled` value in the service configuration."),
"impersonate":
Option("--impersonate",
default=None,
action="store",
help="Impersonate a peer node when evaluating keywords."),
"index":
Option("--index",
default=None,
action="store",
type="int",
help="The position in the list pointed by --param where to add "
"the new list element on a set action"),
"interactive":
Option("-i",
"--interactive",
default=False,
action="store_true",
dest="interactive",
help="Prompt the user for a choice instead of using defaults, "
"or failing if no default is defined."),
"interval":
Option("--interval",
default=0,
action="store",
dest="interval",
type="int",
help="with --watch, set the refresh interval. defaults "
"to 0, to refresh on event only."),
"kw":
Option(
"--kw",
action="append",
dest="kw",
help=
"An expression like ``[<section>.]<keyword>[@<scope>][[<index>]]<op><value>`` where\n\n"
"* <section> can be:\n\n"
" * a resource id\n"
" * a resource driver group name (fs, ip, ...). For the set and unset actions only, set the keyword for all matching resources.\n"
"* <op> can be:\n\n"
" * ``=``\n"
" * ``+=``\n"
" * ``-=``\n\n"
"Multiple --kw can be set to apply multiple configuration change "
"in a file with a single write.\n\n"
"Examples:\n\n"
"* app.start=false\n"
" Turn off app start for all app resources\n"
"* app#1.start=true\n"
" Turn on app start for app#1\n"
"* nodes+=node3\n"
" Append node3 to nodes\n"
"* nodes[0]+=node3\n"
" Preprend node3 to nodes\n"),
"leader":
Option(
"--leader",
default=None,
action="store_true",
dest="leader",
help=
"Switch the provision action behaviour to leader, ie provision shared resources that are not provisionned by default."
),
"local":
Option("--local",
default=False,
action="store_true",
dest="local",
help="Execute the service action on the local service "
"instances only, ignoring cluster-wide considerations."),
"master":
Option("--master",
default=False,
action="store_true",
dest="master",
help="Limit the action scope to the master service resources."),
"namespace":
Option(
"--namespace",
action="store",
dest="namespace",
help=
"The namespace to switch to for the action. Namespaces are cluster partitions. A default namespace can be set for the session setting the OSVC_NAMESPACE environment variable."
),
"node":
Option(
"--node",
default="",
action="store",
dest="node",
help=
"The node to send a request to. If not specified the local node is targeted."
),
"nolock":
Option(
"--nolock",
default=False,
action="store_true",
dest="nolock",
help=
"Don't acquire the action lock. Dangerous, but can be useful to set parameters from an action trigger."
),
"nopager":
Option("--no-pager",
default=False,
action="store_true",
dest="nopager",
help="Do not display the command result in a pager."),
"parallel":
Option(
"-p",
"--parallel",
default=False,
action="store_true",
dest="parallel",
help=
"Start actions on specified services in parallel. :kw:`max_parallel` "
"in node.conf limits the number of parallel running subprocesses."),
"param":
Option(
"--param",
default=None,
action="store",
dest="param",
help="An expression like ``[<section>.]<keyword>`` where\n\n"
"* <section> can be:\n\n"
" * a resource id\n"
" * a resource driver group name (fs, ip, ...). For the set and unset actions only, set the keyword for all matching resources."
),
"provision":
Option("--provision",
default=False,
action="store_true",
dest="provision",
help="Provision the service resources after config file creation. "
"Defaults to False."),
"purge_collector":
Option("--purge-collector",
default=False,
action="store_true",
dest="purge_collector",
help="On service delete, also remove the service collector-side"),
"recover":
Option("--recover",
default=False,
action="store_true",
dest="recover",
help="Recover the stashed erroneous configuration file "
"in a :cmd:`svcmgr edit config` command"),
"refresh":
Option("-r",
"--refresh",
default=False,
action="store_true",
dest="refresh",
help="Drop status caches and re-evaluate before printing."),
"remove":
Option(
"--remove",
default=None,
action="store",
help="A list member to drop from the value pointed by :kw:`--param`."),
"resource":
Option(
"--resource",
default=[],
action="append",
help="A resource definition in json dictionary format fed to create "
"or update. The ``rtype`` key point the driver group name, and "
"the ``type`` key the driver name (translated to type in the "
"configuration file section)."),
"restore":
Option(
"--restore",
default=False,
action="store_true",
dest="restore",
help=
"Keep the same service id as the template or config file referenced by the create action. The default behaviour is to generate a new id."
),
"rid":
Option(
"--rid",
default=None,
action="store",
dest="parm_rid",
help=
"A resource specifier expression like ``<spec>[,<spec>]``, where ``<spec>`` can be:\n\n"
"* A resource id\n"
"* A driver group name (app, fs, disk, ...)\n\n"
"Examples:\n\n"
"* ``app``\n"
" all app resources\n"
"* ``container#1,ip#1``\n"
" only container#1 and ip#1\n"),
"sections":
Option("--sections",
action="store",
dest="sections",
help="the comma-separated list of sections to display. "
"if not set, all sections are displayed. sections "
"names are: threads,arbitrators,nodes,services."),
"service":
Option(
"-s",
"--service",
default=None,
action="store",
dest="parm_svcs",
help=
"A service selector expression ``[!]<expr>[<sep>[!]<expr>]`` where:\n\n"
"- ``!`` is the expression negation operator\n\n"
"- ``<sep>`` can be:\n\n"
" - ``,`` OR expressions\n\n"
" - ``+`` AND expressions\n\n"
"- ``<expr>`` can be:\n\n"
" - a shell glob on service names\n\n"
" - ``<param><op><value>`` where:\n\n"
" - ``<param>`` can be:\n\n"
" - ``<rid>:``\n\n"
" - ``<group>:``\n\n"
" - ``<rid>.<key>``\n\n"
" - ``<group>.<key>``\n\n"
" - ``<single value jsonpath expression on the $.monitor.services.<path> dictionary extended under the 'nodes' key by each instance 'status' and 'config' data>``\n\n"
" - ``<op>`` can be:\n\n"
" - ``<`` ``>`` ``<=`` ``>=`` ``=``\n\n"
" - ``~`` with regexp value\n\n"
"Examples:\n\n"
"- ``*dns,ha*+app.timeout>1``\n\n"
"- ``ip:+task:``\n\n"
"- ``!*excluded``\n\n"
"- ``$.avail=warn``\n\n"
"- ``$.nodes.*.status.avail=warn``\n\n"
"Note:\n\n"
"- ``!`` usage requires single quoting the expression to prevent "
"shell history expansion"),
"show_disabled":
Option("--show-disabled",
default=None,
action="store_true",
dest="show_disabled",
help="Include the disabled resources. This option supersedes "
"the :kw:`show_disabled` value in the service configuration."),
"slave":
Option(
"--slave",
default=None,
action="store",
dest="slave",
help="Limit the action to the service resources in the specified, comma-"
"separated, slaves."),
"slaves":
Option("--slaves",
default=False,
action="store_true",
dest="slaves",
help="Limit the action scope to service resources in all slaves."),
"status":
Option(
"--status",
default=None,
action="store",
dest="parm_status",
help=
"Operate only on service with a local instance in the specified availability status "
"(up, down, warn, ...)."),
"subsets":
Option(
"--subsets",
default=None,
action="store",
dest="parm_subsets",
help=
"Limit the action to the resources in the specified, comma-separated, list of subsets."
),
"stats":
Option(
"--stats",
default=False,
action="store_true",
dest="stats",
help=
"Show system resources usage metrics and refresh the information every --interval."
),
"tags":
Option("--tags",
default=None,
action="store",
dest="parm_tags",
help="A comma-separated list of resource tags to limit "
"action to. The ``+`` separator can be used to impose "
"multiple tag conditions. For example, ``tag1+tag2,tag3`` "
"limits the action to resources with both tag1 and"
" tag2, or tag3."),
"template":
Option("--template",
default=None,
action="store",
dest="parm_template",
help="The configuration file template name or id, "
"served by the collector, to use when creating or "
"installing a service."),
"time":
Option(
"--time",
default="300",
action="store",
dest="time",
help="A duration expression like ``1m5s``. The maximum wait time for an "
"async action to finish. Default is 300 seconds."),
"unprovision":
Option(
"--unprovision",
default=False,
action="store_true",
dest="unprovision",
help=
"Unprovision the service resources before config files file deletion. "
"Defaults to False."),
"value":
Option("--value",
default=None,
action="store",
dest="value",
help="The value to set for the keyword pointed by :opt:`--param`"),
"wait":
Option("--wait",
default=False,
action="store_true",
dest="wait",
help="Wait for asynchronous action termination."),
"waitlock":
Option(
"--waitlock",
default="-1",
action="store",
dest="parm_waitlock",
help=
"A duration expression like ``5s``. The maximum wait time when acquiring "
"the service action lock."),
"watch":
Option("-w",
"--watch",
default=False,
action="store_true",
dest="watch",
help="refresh the information every --interval."),
})
def LOAD(c=None,
f='index',
args=None,
vars=None,
extension=None,
target=None,
ajax=False,
ajax_trap=False,
url=None,
user_signature=False,
timeout=None,
times=1,
content='loading...',
**attr):
""" LOAD a component into the action's document
Timing options:
-times: An integer or string ("infinity"/"continuous")
specifies how many times the component is requested
-timeout (milliseconds): specifies the time to wait before
starting the request or the frequency if times is greater than
1 or "infinity".
Timing options default to the normal behavior. The component
is added on page loading without delay.
"""
from html import TAG, DIV, URL, SCRIPT, XML
if args is None:
args = []
vars = Storage(vars or {})
target = target or 'c' + str(random.random())[2:]
attr['_id'] = target
request = current.request
if '.' in f:
f, extension = f.rsplit('.', 1)
if url or ajax:
url = url or URL(request.application,
c,
f,
r=request,
args=args,
vars=vars,
extension=extension,
user_signature=user_signature)
# timing options
if isinstance(times, basestring):
if times.upper() in ("INFINITY", "CONTINUOUS"):
times = "Infinity"
else:
raise TypeError("Unsupported times argument %s" % times)
elif isinstance(times, int):
if times <= 0:
raise ValueError(
"Times argument must be greater than zero, 'Infinity' or None"
)
else:
raise TypeError("Unsupported times argument type %s" % type(times))
if timeout is not None:
if not isinstance(timeout, (int, long)):
raise ValueError("Timeout argument must be an integer or None")
elif timeout <= 0:
raise ValueError(
"Timeout argument must be greater than zero or None")
statement = "web2py_component('%s','%s', %s, %s);" \
% (url, target, timeout, times)
else:
statement = "web2py_component('%s','%s');" % (url, target)
script = SCRIPT(statement, _type="text/javascript")
if not content is None:
return TAG[''](script, DIV(content, **attr))
else:
return TAG[''](script)
else:
if not isinstance(args, (list, tuple)):
args = [args]
c = c or request.controller
other_request = Storage(request)
other_request['env'] = Storage(request.env)
other_request.controller = c
other_request.function = f
other_request.extension = extension or request.extension
other_request.args = List(args)
other_request.vars = vars
other_request.get_vars = vars
other_request.post_vars = Storage()
other_response = Response()
other_request.env.path_info = '/' + \
'/'.join([request.application, c, f] +
map(str, other_request.args))
other_request.env.query_string = \
vars and URL(vars=vars).split('?')[1] or ''
other_request.env.http_web2py_component_location = \
request.env.path_info
other_request.cid = target
other_request.env.http_web2py_component_element = target
other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
other_environment = copy.copy(current.globalenv) # NASTY
other_response._view_environment = other_environment
other_response.generic_patterns = \
copy.copy(current.response.generic_patterns)
other_environment['request'] = other_request
other_environment['response'] = other_response
## some magic here because current are thread-locals
original_request, current.request = current.request, other_request
original_response, current.response = current.response, other_response
page = run_controller_in(c, f, other_environment)
if isinstance(page, dict):
other_response._vars = page
other_response._view_environment.update(page)
run_view_in(other_response._view_environment)
page = other_response.body.getvalue()
current.request, current.response = original_request, original_response
js = None
if ajax_trap:
link = URL(request.application,
c,
f,
r=request,
args=args,
vars=vars,
extension=extension,
user_signature=user_signature)
js = "web2py_trap_form('%s','%s');" % (link, target)
script = js and SCRIPT(js, _type="text/javascript") or ''
return TAG[''](DIV(XML(page), **attr), script)
def set(task):
Storage().set('task', task)
def __call__(self,
c=None,
f='index',
args=None,
vars=None,
extension=None,
target=None,
ajax=False,
ajax_trap=False,
url=None,
user_signature=False,
content='loading...',
**attr):
if args is None:
args = []
vars = Storage(vars or {})
import globals
target = target or 'c' + str(random.random())[2:]
attr['_id'] = target
request = self.environment['request']
if '.' in f:
f, extension = f.rsplit('.', 1)
if url or ajax:
url = url or html.URL(request.application,
c,
f,
r=request,
args=args,
vars=vars,
extension=extension,
user_signature=user_signature)
script = html.SCRIPT('web2py_component("%s","%s")' % (url, target),
_type="text/javascript")
return html.TAG[''](script, html.DIV(content, **attr))
else:
if not isinstance(args, (list, tuple)):
args = [args]
c = c or request.controller
other_request = Storage(request)
other_request['env'] = Storage(request.env)
other_request.controller = c
other_request.function = f
other_request.extension = extension or request.extension
other_request.args = List(args)
other_request.vars = vars
other_request.get_vars = vars
other_request.post_vars = Storage()
other_response = globals.Response()
other_request.env.path_info = '/' + \
'/'.join([request.application, c, f] +
map(str, other_request.args))
other_request.env.query_string = \
vars and html.URL(vars=vars).split('?')[1] or ''
other_request.env.http_web2py_component_location = \
request.env.path_info
other_request.cid = target
other_request.env.http_web2py_component_element = target
other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
other_environment = copy.copy(self.environment)
other_response._view_environment = other_environment
other_response.generic_patterns = \
copy.copy(current.response.generic_patterns)
other_environment['request'] = other_request
other_environment['response'] = other_response
## some magic here because current are thread-locals
original_request, current.request = current.request, other_request
original_response, current.response = current.response, other_response
page = run_controller_in(c, f, other_environment)
if isinstance(page, dict):
other_response._vars = page
other_response._view_environment.update(page)
run_view_in(other_response._view_environment)
page = other_response.body.getvalue()
current.request, current.response = original_request, original_response
js = None
if ajax_trap:
link = html.URL(request.application,
c,
f,
r=request,
args=args,
vars=vars,
extension=extension,
user_signature=user_signature)
js = "web2py_trap_form('%s','%s');" % (link, target)
script = js and html.SCRIPT(js, _type="text/javascript") or ''
return html.TAG[''](html.DIV(html.XML(page), **attr), script)
from storage import Storage
if __name__ == '__main__':
storage = Storage()
bike_list = storage.read()
print(bike_list[0])
async def get_storage(self, plugin, server):
namespace = "{}.{}:".format(plugin.__class__.__name__, server.id)
storage = Storage(namespace, self.redis)
return storage
