def generate_view(query):
if len(query) == 0 and config.get('todo.command.last') == 'quick_create':
add_query = config.get('todo.user.query')
add_tag = config.get('todo.tag.recent')
itemlist.save_todo(add_query,silent=True)
config.update_state(command='', query='')
info = parser.parse(query)
tag = info['tag']
q = info['task']
todos = itemlist.get_todo_list()
# view for pinned items
# pinned items should have unique uuid and different logo
pinned = [t for t in todos if itemlist.feature(t,'pinned') == True]
pinned = [t for t in pinned if (tag is None or t['group'] == tag)]
pinned = [t for t in pinned if (q is None or t['title'].lower().find(q.lower()) >= 0)]
pinned = pinned[::-1]
# view for non-pinned items
normal = [t for t in todos if itemlist.feature(t,'pinned') == False]
normal = [t for t in normal if (tag is None or t['group'] == tag)]
normal = [t for t in normal if (q is None or t['title'].lower().find(q.lower()) >= 0)]
normal = normal[::-1]
feedback_items = []
if len(normal) == 0 and len(pinned) == 0:
feedback_items.append( generate_add_feedbackitem(query, info) )
else:
pinned = map(lambda x: generate_pinned_feedbackitem(x), pinned)
normal = map(lambda x: generate_todo_feedbackitem(x), normal)
feedback_items = pinned + normal
alfred.write(alfred.xml(feedback_items))
def cb_story(args):
# adds tags to the entry properties
request = args["request"]
entry = args["entry"]
config = request.get_configuration()
sep = config.get("tags_separator", ",")
tags = [t.strip() for t in entry.get("tags", "").split(sep)]
tags.sort()
entry["tags_raw"] = tags
form = request.get_form()
try:
flavour = form["flav"].value
except KeyError:
flavour = config.get("default_flavour", "html")
baseurl = config.get("base_url", "")
trigger = config.get("tags_trigger", "tag")
template = config.get("tags_item", '<a href="%(tagurl)s">%(tag)s</a>')
tags = [template % {"base_url": baseurl,
"flavour": flavour,
"tag": tag,
"tagurl": "/".join([baseurl, trigger, tag])}
for tag in tags]
entry["tags"] = ", ".join(tags)
return args
def main():
# change path to launcher
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
launcher_log.info("start XX-Net %s", update_from_github.current_version())
web_control.confirm_xxnet_exit()
setup_win_python.check_setup()
module_init.start_all_auto()
web_control.start()
if has_desktop and config.get(["modules", "launcher", "popup_webui"], 1) == 1:
host_port = config.get(["modules", "launcher", "control_port"], 8085)
webbrowser.open("http://127.0.0.1:%s/" % host_port)
update.start()
if config.get(["modules", "launcher", "show_systray"], 1):
sys_tray.serve_forever()
else:
while True:
time.sleep(100)
module_init.stop_all()
sys.exit()
def main():
""" Initialise and do requested operations. """
try:
# set up the logging system
prepare_logging()
if config.get('version'):
# in version mode, print version and exit
show_version()
elif config.get('help'):
# in help mode, print usage and exit
show_usage()
elif config.get('convert'):
# in converter mode, convert and exit
convert()
else:
# otherwise, start an interpreter session
start_basic()
finally:
# clean up our temp dir if we made one
if plat.temp_dir:
shutil.rmtree(plat.temp_dir)
# avoid sys.excepthook errors when piping output
# http://stackoverflow.com/questions/7955138/addressing-sys-excepthook-error-in-bash-script
try:
sys.stdout.close()
except:
pass
try:
sys.stderr.close()
except:
pass
def set_ini_properties(properties):
aux = []
for p in properties:
line = parse_ini_property(p) + '=' + str(properties[p]) + '\r\n'
aux.append(line)
if len(aux) > 0:
utils.write_file(config.get('ini_path')+config.get('emulator')+'.ini', aux)
def version(self, extended=False):
"""
What version is cobbler?
If extended == False, returns a float for backwards compatibility
If extended == True, returns a dict:
gitstamp -- the last git commit hash
gitdate -- the last git commit date on the builder machine
builddate -- the time of the build
version -- something like "1.3.2"
version_tuple -- something like [ 1, 3, 2 ]
"""
config = ConfigParser()
config.read("/etc/cobbler/version")
data = {}
data["gitdate"] = config.get("cobbler","gitdate")
data["gitstamp"] = config.get("cobbler","gitstamp")
data["builddate"] = config.get("cobbler","builddate")
data["version"] = config.get("cobbler","version")
# dont actually read the version_tuple from the version file
data["version_tuple"] = []
for num in data["version"].split("."):
data["version_tuple"].append(int(num))
if not extended:
# for backwards compatibility and use with koan's comparisons
elems = data["version_tuple"]
return int(elems[0]) + 0.1*int(elems[1]) + 0.001*int(elems[2])
else:
return data
def _getVersionText():
ver = config.verinfo.ver
cur = config.get('__xvmVersion')
msg = ''
if ver is not None:
if utils.compareVersions(ver, cur) == 1:
msg += '{{l10n:ver/newVersion:%s:%s}}\n' % (ver, config.verinfo.message)
if cur.endswith('-dev'):
if config.get('region').lower() == 'ru':
msg += """
<font color='#FF0000'>Внимание!</font>
Установлена тестовая сборка XVM, не рекомендуемая для неопытных пользователей.
Некоторые функции могут работать неправильно, или не работать вообще.
Если вы тестируете XVM, проигнорируйте это сообщение.
<b>Если вы простой пользователь, пожалуйста, используйте стабильную версию с официального сайта мода XVM: <a href='#XVM_SITE_DL#'>www.ModXVM.com</a></b>
"""
else:
msg += """
<font color='#FF0000'>Warning!</font>
You've installed nightly build of XVM, which is not recommended for inexperienced users.
Some functionality may work incorrectly or may not work at all.
If you are testing XVM, you can ignore this message.
<b>If you're just a player and not a tester of XVM, please use a stable version instead of nightly builds. Download the stable version from the official website of XVM: <a href='#XVM_SITE_DL#'>www.ModXVM.com</a></b>
"""
return msg
def testInvalidDomain(self):
# can't be just a bare top-level domain
domain = config.get('domain')
# get top-level domain. e.g.: 'test.hdf.io' -> 'hdf.io'
npos = domain.find('.')
topdomain = domain[npos+1:]
req = self.endpoint + "/"
headers = {'host': topdomain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 403) # 403 == Forbidden
domain = 'two.dots..are.bad.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400) # 400 == bad syntax
domain = 'missingenddot' + topdomain
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400) # 400 == bad syntax
# just a dot is no good
domain = '.' + topdomain
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400) # 400 == bad syntax
domain = '.dot.in.front.is.bad.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400) # 400 == bad syntax
def get_content(cls, abspath, start=None, end=None):
"""
Check if specific files are being requested and return them modified to
contain a bunch of additional files, as specified by config lists above.
"""
serverdir = os.path.dirname(os.path.realpath(__file__)) + "/"
jspath = '../webapp/js'
basepath = os.path.dirname(os.path.realpath(serverdir + jspath))
relpath = abspath[basepath.__len__()+1:]
if relpath == 'js/main.js' and config.get('Optimization', 'optimize_js'):
js = cls.COMBINED_JS
# The default behavior is to cache the combined files in memory
# for as long as the server is running. debug_mode prevents this.
if config.get('App', 'debug_mode'):
cls.COMBINED_JS = ''
return js
elif relpath == 'css/layout.css' and \
config.get('Optimization', 'optimize_css'):
css = cls.COMBINED_CSS
if config.get('App', 'debug_mode'):
cls.COMBINED_CSS = ''
return css
else:
return super(HubbleStaticFileHandler, cls).get_content(abspath, start, end)
def render_string(self, template_name, **kwargs):
"""
Combine all templates found in templates_dir into one string and insert
them into index.html in order to reduce HTTP requests.
"""
if not config.get('Optimization', 'optimize_templates'):
return super(HubbleIndexHandler, self).render_string(template_name, **kwargs)
if self.cached_index != '':
return self.cached_index;
indexfile = open('../webapp/index.html', 'r')
indexhtml = indexfile.read()
indexfile.close()
basepath = os.path.dirname(os.path.realpath(__file__)) + '/../webapp'
minifier = MinCatenator(basepath)
templates = minifier.mincatenate(file_list=config.get('Optimization', 'templates_dir'))
main_div = '<div id="page_content"></div>'.format(templates)
new_main_div = "{}\n{}".format(main_div, templates)
epoch_time = str(time.time())
new_content = indexhtml.replace(main_div, new_main_div).replace('XXX', epoch_time)
# Cache this in memory for as long as the server is running.
if config.get('App', 'debug_mode'):
self.cached_index = new_content
return new_content
def prepare():
""" Initialise unicodepage module. """
codepage = config.get('codepage')
if not codepage:
codepage = '437'
state.console_state.codepage = Codepage(
codepage, box_protect=not config.get('nobox'))
def wrap(*args, **kwargs):
kwargs.update({
"email": config.get("email"),
"password": config.get("password")
})
o = func(*args, **kwargs)
return o()
def make_menu(self):
proxy_stat = self.get_proxy_state()
gae_proxy_checked = win32_adapter.fState.MFS_CHECKED if proxy_stat=="gae" else 0
x_tunnel_checked = win32_adapter.fState.MFS_CHECKED if proxy_stat=="x_tunnel" else 0
pac_checked = win32_adapter.fState.MFS_CHECKED if proxy_stat=="pac" else 0
disable_checked = win32_adapter.fState.MFS_CHECKED if proxy_stat=="disable" else 0
if lang_code == "zh_CN":
menu_options = [(u"设置", None, self.on_show, 0)]
if config.get(["modules", "gae_proxy", "auto_start"], 0) == 1:
menu_options.append((u"全局通过GAEProxy代理", None, self.on_enable_gae_proxy, gae_proxy_checked))
if config.get(["modules", "x_tunnel", "auto_start"], 0) == 1:
menu_options.append((u"全局通过X-Tunnel代理", None, self.on_enable_x_tunnel, x_tunnel_checked))
menu_options += [
(u"全局PAC智能代理", None, self.on_enable_pac, pac_checked),
(u"取消全局代理", None, self.on_disable_proxy, disable_checked),
(u'退出', None, SysTrayIcon.QUIT, False)]
else:
menu_options = [(u"Config", None, self.on_show, 0)]
if config.get(["modules", "gae_proxy", "auto_start"], 0) == 1:
menu_options.append((u"Set Global GAEProxy Proxy", None, self.on_enable_gae_proxy, gae_proxy_checked))
if config.get(["modules", "x_tunnel", "auto_start"], 0) == 1:
menu_options.append((u"Set Global X-Tunnel Proxy", None, self.on_enable_x_tunnel, x_tunnel_checked))
menu_options += [
(u"Set Global PAC Proxy", None, self.on_enable_pac, pac_checked),
(u"Disable Global Proxy", None, self.on_disable_proxy, disable_checked),
(u'Quit', None, SysTrayIcon.QUIT, False)]
return tuple(menu_options)
def send_task(task):
"""Sends task to random free worker"""
free_workers = []
for item in workers:
if not item.is_busy():
free_workers.append(item)
if free_workers:
free_worker = random.choice(free_workers)
print 'Sending task {0} to {1}'.format(
task._id,
free_worker.name
)
task.set_worker(free_worker.name)
result = {
'_id': task._id,
'data': task.data,
'worker_id': free_worker._id,
'direction': 'workers'
}
config.get('storage').publish(Task.CHANNEL, json.dumps(result))
def main():
global log
cfg_file = "tevs.cfg"
out_file = open("summary.csv","w")
config.get(cfg_file)
log = config.logger(util.root("log.txt"))
#pdb.set_trace()
#matches = is_this_like_them("hello there",{"hellc there":"bonjour","yabba":"dabba"})
if not const.use_db:
print "The configuration file indicates no database is in use."
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
print "Generating totals from db %s, user %s" % (const.dbname,
const.dbuser)
print "If there are many vote ops, this may take several minutes."
print "The next output will be the number of vote opportunities."
qs = query_database(dbc)
except db.DatabaseError:
print "Although the configuration file indicates a database is in use,"
print "we could not connect for dbname %s user %s." % (const.dbname,
const.dbuser)
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
return 0
def api(version=None):
global services
if 'ARVADOS_DEBUG' in config.settings():
logging.basicConfig(level=logging.DEBUG)
if not services.get(version):
apiVersion = version
if not version:
apiVersion = 'v1'
logging.info("Using default API version. " +
"Call arvados.api('%s') instead." %
apiVersion)
if 'ARVADOS_API_HOST' not in config.settings():
raise Exception("ARVADOS_API_HOST is not set. Aborting.")
url = ('https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' %
config.get('ARVADOS_API_HOST'))
credentials = CredentialsFromEnv()
# Use system's CA certificates (if we find them) instead of httplib2's
ca_certs = '/etc/ssl/certs/ca-certificates.crt'
if not os.path.exists(ca_certs):
ca_certs = None # use httplib2 default
http = httplib2.Http(ca_certs=ca_certs,
cache=http_cache('discovery'))
http = credentials.authorize(http)
if re.match(r'(?i)^(true|1|yes)$',
config.get('ARVADOS_API_HOST_INSECURE', 'no')):
http.disable_ssl_certificate_validation=True
services[version] = apiclient.discovery.build(
'arvados', apiVersion, http=http, discoveryServiceUrl=url)
return services[version]
def register():
if request.method == 'POST' and config.get('registration', bool):
if 'username' in request.form and 'password' in request.form and \
'email' in request.form:
try:
user = NewUser(request.form['username'],
request.form['password'],
request.form['email'])
user.create()
except exceptions.UserAlreadyExists:
flash(messages.username_already_exists, 'error')
return redirect(url_for('register'))
except:
flash(messages.register_error, 'error')
return redirect(url_for('register'))
if user.is_active():
user.login()
flash(messages.registered_and_logged_in, 'message')
return redirect(url_for('index'))
else:
flash(messages.registered_and_deactivated, 'message')
return redirect(url_for('public_index'))
else:
flash(messages.register_field_error, 'error')
return render_template(get_template('register.html'),
registration_enabled=config.get('registration', bool))
def testGetDomain(self):
filePath = "tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.' + config.get('domain'))
filePath = "somevalue"
domain = getDomain(filePath)
self.assertEqual(domain, 'somevalue.' + config.get('domain'))
def _set_eye_xy_flat_pos(self, x, y, do_party, do_business, in_sky):
#
# Basic formula is pan = atan(x), tilt = atan( (y * sin(pan)) / x )
#
# if y < -0.5:
# y = -0.5
# self.lastXPos = x
# self.lastYPos = y
#y = .125
# Units for X and Y real are "height of the eye" = 1. Scaling out so
# that we can tweak it more here than in touch and can define a reasonable
# addressable area
xr = config.get("xy_scale")["x"] * x
yr = config.get("xy_scale")["y"] * (y + 0.5)
if do_party:
self.p_eye_pos = eyes.xy_to_pnt([xr, yr], True, in_sky)
# pan_rads = math.atan2(xr,1)
# tilt_rads = math.atan2( yr * math.sin(math.fabs(pan_rads)), xr)
# self.p_eye_pos[PAN] = math.degrees(pan_rads)
# self.p_eye_pos[TILT] = math.degrees(tilt_rads) - 90
# if self.p_eye_pos[TILT] < 0:
# self.p_eye_pos[TILT] += 360
# if self.p_eye_pos[TILT] > 180:
# self.p_eye_pos[TILT] = 360-self.p_eye_pos[TILT]
# print "P x=%f y=%f pan=%f tilt=%f" % (xr,yr, self.p_eye_pos[PAN] , self.p_eye_pos[TILT] )
# if self.p_eye_pos[TILT] > 135:
# self.p_eye_pos[TILT] = 135
# if in_sky:
# self.p_eye_pos[PAN] = 360-self.p_eye_pos[PAN]
self._notify_eye_changed(True)
if do_business:
# xr -= 0.25 # This is (roughly) the distance between the lights in light_to_ground units
# pan_rads = math.atan2(xr,1)
# tilt_rads = math.atan2( yr * math.sin(math.fabs(pan_rads)), xr)
# self.b_eye_pos[PAN] = math.degrees(pan_rads)
# self.b_eye_pos[TILT] = math.degrees(tilt_rads) - 90
# if self.b_eye_pos[TILT] < 0:
# self.b_eye_pos[TILT] += 360
# if self.b_eye_pos[TILT] > 180:
# self.b_eye_pos[TILT] = 360-self.b_eye_pos[TILT]
# print "B x=%f y=%f pan=%f tilt=%f" % (xr,yr, self.b_eye_pos[PAN], self.b_eye_pos[TILT])
# if self.b_eye_pos[TILT] > 135:
# self.b_eye_pos[TILT] = 135
# if in_sky:
# self.b_eye_pos[PAN] = 360-self.b_eye_pos[PAN]
self.b_eye_pos = eyes.xy_to_pnt([xr, yr], False, in_sky)
self._notify_eye_changed(False)
def initialize(self):
config = self.iface.getConfig()
cache = config.get("general", "package_cache")
cache_limit = config.get("general", "package_cache_limit")
cache_limit = int(cache_limit) if cache_limit else 0
cache_dir = config.get("directories", "cached_packages_dir")
cache_dir = str(cache_dir) if cache_dir else '/var/cache/pisi/packages'
# If pisi.conf does not have it yet, default is use package cache
if not cache or cache == "True":
enableCache = True
else:
enableCache = False
self.cacheEnabled = enableCache
self.cacheSize = cache_limit
self.settings.cacheGroup.setEnabled(self.cacheEnabled)
self.settings.useCacheCheck.setChecked(enableCache)
self.settings.useCacheSpin.setValue(cache_limit)
self.settings.cacheDirPath.setText(cache_dir)
bandwidth_limit = config.get("general", "bandwidth_limit")
bandwidth_limit = int(bandwidth_limit) if bandwidth_limit else 0
self.settings.useBandwidthLimit.setChecked(not bandwidth_limit == 0)
self.settings.bandwidthSpin.setValue(bandwidth_limit)
def createCampaign():
params = {
"recipients": {
"list_id": config.get().MailChimpListId
},
"type": "regular",
"settings": {
"subject_line": "Olava Weekly Gaming Roundup",
"reply_to": "*****@*****.**",
"from_name": "Olava"
}
}
response = requests.post(
'https://us10.api.mailchimp.com/3.0/campaigns',
auth=(
'apikey',
config.get().MailChimpApiKey
),
json=params
)
try:
response.raise_for_status()
body = response.json()
return body['id']
except requests.exceptions.HTTPError as err:
print("Error: {} {}".format(str(response.status_code), err))
print(json.dumps(response.json(), indent=4))
return -1
except ValueError:
print("Cannot decode json, got %s" % response.text)
return -1
def load_module_menus(self):
global module_menus
module_menus = {}
#config.load()
modules = config.get(['modules'], None)
for module in modules:
values = modules[module]
if module != "launcher" and config.get(["modules", module, "auto_start"], 0) != 1: # skip php_proxy module
continue
#version = values["current_version"]
menu_path = os.path.join(root_path, module, "web_ui", "menu.yaml") # launcher & gae_proxy modules
if not os.path.isfile(menu_path):
continue
#module_menu = yaml.load(file(menu_path, 'r')) # non-i18n
# i18n code lines (Both the locale dir & the template dir are module-dependent)
locale_dir = os.path.abspath(os.path.join(root_path, module, 'lang'))
template_dir = os.path.abspath(os.path.join(root_path, module, 'web_ui'))
jinja2_i18n_helper.ihelper.refresh_env(locale_dir, template_dir)
stream = jinja2_i18n_helper.ihelper.render("menu.yaml", None)
module_menu = yaml.load(stream)
module_menus[module] = module_menu
module_menus = sorted(module_menus.iteritems(), key=lambda (k,v): (v['menu_sort_id']))
def test_global_admin_page(self):
# Load the global administration page.
doc = self.go_as_admin('/global/admin')
assert self.s.status == 200
# Change some settings.
settings_form = doc.first('form', id='save_global')
doc = self.s.submit(settings_form,
sms_number_to_repo=
'{"+198765432109": "haiti", "+8101234567890": "japan"}'
)
assert self.s.status == 200, self.get_admin_page_error_message()
# Reopen the admin page and check if the change took effect on the page.
doc = self.go_as_admin('/global/admin')
assert self.s.status == 200
assert (simplejson.loads(
doc.first('textarea', id='sms_number_to_repo').text) ==
{'+198765432109': 'haiti', '+8101234567890': 'japan'})
# Also check if the change took effect in the config.
assert (config.get('sms_number_to_repo') ==
{'+198765432109': 'haiti', '+8101234567890': 'japan'})
# Change settings again and make sure they took effect.
settings_form = doc.first('form', id='save_global')
doc = self.s.submit(settings_form,
sms_number_to_repo=
'{"+198765432109": "test", "+8101234567890": "japan"}'
)
assert self.s.status == 200, self.get_admin_page_error_message()
assert (config.get('sms_number_to_repo') ==
{'+198765432109': 'test', '+8101234567890': 'japan'})
def prepare():
""" Initialise the memory module """
global field_mem_base, field_mem_start, field_mem_offset
global code_start, stack_size, max_memory, total_memory
# length of field record (by default 128)
file_rec_len = config.get("max-reclen")
# file header (at head of field memory)
file_header_size = 194
# number of file records
num_files = config.get("max-files")
# first field buffer address (workspace size; 3429 for gw-basic)
field_mem_base = config.get("reserved-memory")
# bytes distance between field buffers
field_mem_offset = file_header_size + file_rec_len
# start of 1st field =3945, includes FCB & header header of 1st field
# used by var.py
field_mem_start = field_mem_base + field_mem_offset + file_header_size
# data memory model: start of code section
code_start = field_mem_base + (num_files + 1) * field_mem_offset
# BASIC stack (determined by CLEAR)
# Initially, the stack space should be set to 512 bytes,
# or one-eighth of the available memory, whichever is smaller.
stack_size = 512
# max available memory to BASIC (set by /m)
max_list = config.get("max-memory")
max_list[1] = max_list[1] * 16 if max_list[1] else max_list[0]
max_list[0] = max_list[0] or max_list[1]
max_memory = min(max_list) or 65534
# total size of data segment (set by CLEAR)
total_memory = max_memory
def prepare():
""" Prepare the video modes. """
global video_capabilities, mono_monitor
global colours16, colours16_mono, cga4_palettes, mono_tint
global circle_aspect
video_capabilities = config.get('video')
if video_capabilities == 'tandy':
circle_aspect = (3072, 2000)
else:
circle_aspect = (4, 3)
mono_monitor = config.get('monitor') == 'mono'
if video_capabilities == 'ega' and mono_monitor:
video_capabilities = 'ega_mono'
cga_low = config.get('cga-low')
# set monochrome tint
mono_tint = config.get('mono-tint')
# build colour sets
colours16_mono = tuple(tuple(tint*i//255 for tint in mono_tint)
for i in intensity16_mono)
if mono_monitor:
colours16 = list(colours16_mono)
else:
colours16 = list(colours16_colour)
# initialise the 4-colour CGA palette
# palette 1: Black, Ugh, Yuck, Bleah, choice of low & high intensity
# palette 0: Black, Green, Red, Brown/Yellow, low & high intensity
# tandy/pcjr have high-intensity white, but low-intensity colours
# mode 5 (SCREEN 1 + colorburst on RGB) has red instead of magenta
if video_capabilities in ('pcjr', 'tandy'):
# pcjr does not have mode 5
cga4_palettes = {0: (0, 2, 4, 6), 1: (0, 3, 5, 15), 5: None}
elif cga_low:
cga4_palettes = {0: (0, 2, 4, 6), 1: (0, 3, 5, 7), 5: (0, 3, 4, 7)}
else:
cga4_palettes = {0: (0, 10, 12, 14), 1: (0, 11, 13, 15), 5: (0, 11, 12, 15)}
def toText() :
'''
クロールデータのHTMLファイルをテキストファイルのみに整形して出力します
'''
dir = config.get("crawl", "crawldata.dir.path")
textDir = config.get("crawl", "text.dir.path")
fileUtils.makedirs(textDir)
list = os.listdir(dir)
for file in list:
filePath = dir + file
if os.path.isfile(filePath) :
f = open(filePath,'r', encoding='utf-8-sig')
try :
crawlData = json.load(f)
except:
logger.exception("Could not load %s" % filePath + " ", traceback.format_exc())
continue
f.close()
text = crawlData["html"]
outputFile = textDir + os.path.splitext(file)[0] + ".txt"
f = open(outputFile, 'w', encoding='utf-8-sig')
f.write(text)
f.close()
def start_emulator(options):
emulator_name = config.get('emulator')
emulator_port = config.get('port')
adb_device_name = 'emulator-' + str(emulator_port)
emulator = ['emulator', '-avd', emulator_name]
print "> Created new execution evironment. Wating for emulator..."
# Wait for emulator and clean logcat
emulator.extend(options)
print "> " + ' '.join(emulator)
Popen(emulator)
bootcomplete = None
count = 0
while bootcomplete is not 1:
p = Popen(['adb', '-s', adb_device_name, 'wait-for-device', 'shell', 'getprop', 'sys.boot_completed'], stdout=PIPE)
out=p.communicate()
print out
try:
bootcomplete = int(str(out[0]).strip())
except:
pass
sleep(2)
count+=1
if count % 10 is 0:
print "Waiting boot completed... ", bootcomplete
def set_file_rights(path):
'''Changes group permissions and ownership of file or directory according to configuration.
If compiled_umask is set, will change group permissions of `path` to given value. If `path` is a
directory it will also set all executable bits to one.
If compiled_groupid is set, will change group ownership to given gid.'''
try:
umask = int(config.get('global', 'compiled_umask'))
if os.path.isdir(path):
import stat
umask += stat.S_IXUSR + stat.S_IXGRP + stat.S_IXOTH
except:
umask = None
try:
gid = int(config.get('global', 'compiled_groupid'))
except:
gid = None
# Correcting rights and group ownership, if configured
if umask:
import stat
os.chmod(path, umask)
if gid:
os.chown(path, -1, gid)
def main():
# change path to launcher
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, "readlink", lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
web_control.confirm_xxnet_exit()
setup_win_python.check_setup()
module_init.start_all_auto()
web_control.start()
if has_desktop and config.get(["modules", "launcher", "popup_webui"], 1) == 1:
webbrowser.open("http://127.0.0.1:8085/")
update.start()
if config.get(["modules", "launcher", "show_systray"], 1):
sys_tray.serve_forever()
else:
while True:
time.sleep(100)
module_init.stop_all()
sys.exit()
def getFileName(self, basename):
if config.get("use_h5py"):
if not op.isdir("out"):
os.mkdir("out")
return "out/" + basename + ".h5"
else:
return basename + "." + config.get("domain")
app.router.add_get('/nodeinfo/{statkey}', nodeinfo)
app.router.add_get('/info', info)
app.router.add_post('/register', register)
return app
#
# Main
#
if __name__ == '__main__':
loop = asyncio.get_event_loop()
app = loop.run_until_complete(init(loop))
# create a client Session here so that all client requests
# will share the same connection pool
max_tcp_connections = int(config.get("max_tcp_connections"))
app['client'] = ClientSession(
loop=loop, connector=TCPConnector(limit=max_tcp_connections))
session = aiobotocore.get_session(loop=loop)
app["session"] = session
app["loop"] = loop
asyncio.ensure_future(healthCheck(app), loop=loop)
head_port = config.get("head_port")
log.info("Starting service on port: {}".format(head_port))
log.debug("debug test")
run_app(app, port=int(head_port))
def train(dset, model):
n_epoch = 300
batch_size = 128
loader = data.DataLoader(dataset=db(dset.S_tr, dset.X_tr, dset.y_tr),
batch_size=batch_size,
shuffle=True)
model, recon_loss = get(model)
dualAE = model(dset.X_tr.shape[1], dset.S_tr.shape[1], dset.nclass)
net = dualAE.cuda()
print('-------training V-Net --------')
V_optim = torch.optim.Adam(net.parameters(), lr=0.0005, weight_decay=1e-5)
for epoch in xrange(0):
for i, (_, V, _) in enumerate(loader):
dV = data_corrupt(V, 0.2)
dV = Variable(dV).cuda()
V = Variable(V).cuda()
V_optim.zero_grad()
latent, out = net.V_net_forward(dV)
loss = recon_loss(out, V)
if model == 'CAE':
loss += cae_loss(net.get_VAE_W(), latent) * 0.01
loss.backward()
V_optim.step()
if (i + 1) % 100 == 0:
print('Epoch [%d/40] [%d/%d], Loss: %.5f' %
(epoch + 1, i + 1, len(loader), loss.data[0]))
print('-------training S-Net --------')
S_optim = torch.optim.Adam(net.parameters(), lr=0.00001, weight_decay=1e-5)
for epoch in xrange(0):
for i, (S, _, _) in enumerate(loader):
dS = data_corrupt(S)
dS = Variable(dS).cuda()
S = Variable(S).cuda()
S_optim.zero_grad()
latent, out = net.S_net_forward(dS)
loss = recon_loss(out, S)
if model == 'CAE':
loss += cae_loss(net.get_SAE_W(), latent)
loss.backward()
S_optim.step()
if (i + 1) % 100 == 0:
print('Epoch [%d/40] [%d/%d], Loss: %.5f' %
(epoch + 1, i + 1, len(loader), loss.data[0]))
criter = nn.CrossEntropyLoss()
optim = torch.optim.Adam(net.parameters(), lr=0.001,
weight_decay=1e-5) #0.0005
for epoch in xrange(n_epoch):
for i, (S, V, L) in enumerate(loader):
dS = data_corrupt(S, 0.01)
dS = Variable(dS).cuda()
S, V = Variable(S).cuda(), Variable(V).cuda()
L = Variable(L).cuda()
optim.zero_grad()
latent, pred, out_v, out_s = net(dS)
# reconstruction error
recon_v_loss = recon_loss(out_v, V)
recon_s_loss = recon_loss(out_s, S)
# clf_loss = criter(pred, L)
loss = recon_v_loss + recon_s_loss #+ clf_loss*10
if model == 'CAE':
loss += cae_loss(net.get_SAE_W(), latent) * 0.001
loss.backward()
optim.step()
if (i + 1) % 50 == 0:
print(
'Epoch [%d/%d] [%d/%d], Loss: %.5f --recon_v_loss: %.5f, recon_s_loss: %.5f, clf_loss: %.5f'
% (epoch + 1, n_epoch, i + 1, len(loader), loss.data[0],
recon_v_loss.data[0], recon_s_loss.data[0],
0.0)) #, clf_loss.data[0]))
if (epoch + 1) % 200 == 0:
optim = torch.optim.Adam(net.parameters(),
lr=0.0001,
weight_decay=1e-5)
kNN(net, dset)
return net
def getUsers(self):
fp = open(config.get("ssFile"), 'r')
value = json.load(fp)
fp.close()
return value['port_password']
async def healthCheck(app):
""" Periodic method that pings each active node and verifies it is still healthy.
If node doesn't respond, free up the node slot (the node can re-register if it comes back)'"""
app["last_health_check"] = int(time.time())
# update/initialize root object before starting node updates
headnode_key = getHeadNodeS3Key()
log.info("headnode S3 key: {}".format(headnode_key))
headnode_obj_found = False
head_url = getUrl(app["head_host"], app["head_port"])
nodes = app["nodes"]
while True:
# sleep for a bit
sleep_secs = config.get("head_sleep_time")
await asyncio.sleep(sleep_secs)
now = int(time.time())
log.info("health check {}".format(unixTimeToUTC(now)))
if not headnode_obj_found:
log.info("checking for headnode_key: {}".format(headnode_key))
if await isS3Obj(app, headnode_key):
headnode_obj_found = True
headnode_stats = await getS3ObjStats(app, headnode_key)
log.info("headnode_stats: {}".format(headnode_stats))
else:
# first time hsds has run with this bucket name?
log.warn("need to create headnode obj")
head_state = {}
head_state["created"] = int(time.time())
head_state["id"] = app["id"]
head_state["last_health_check"] = app["last_health_check"]
head_state["head_url"] = head_url
log.info("write head_state to S3: {}".format(head_state))
try:
await putS3JSONObj(app, headnode_key, head_state)
except HTTPInternalServerError as hpe:
# Might be bad AWS config, transient S3 error, or minio not initialized yet...
log.warn(
"HTTPInternalServerError writing head_state: {}: {}".
format(headnode_key, str(hpe)))
continue # start health check on next iteration
head_state = await getS3JSONObj(app, headnode_key)
log.info("head_state: {}".format(head_state))
log.info("elapsed time since last health check: {}".format(
elapsedTime(head_state["last_health_check"])))
if head_state['id'] != app['id']:
log.warn("mis-match bucket head id: {}".format(head_state["id"]))
if now - head_state["last_health_check"] < sleep_secs * 4:
log.warn("other headnode may be active")
continue # skip node checks and loop around again
else:
log.warn(
"other headnode is not active, making this headnode leader"
)
head_state['id'] = app['id']
else:
log.info("head_state id matches S3 Object")
head_state["last_health_check"] = now
app["last_health_check"] = now
head_state["head_url"] = head_url
log.info("write head_state to S3: {}".format(head_state))
await putS3JSONObj(app, headnode_key, head_state)
log.info("putS3JSONObj complete")
fail_count = 0
HEALTH_CHECK_RETRY_COUNT = 1 # times to try before calling a node dead
for node in nodes:
if node["host"] is None:
fail_count += 1
continue
url = getUrl(node["host"], node["port"]) + "/info"
try:
rsp_json = await http_get(app, url)
if "node" not in rsp_json:
log.error("Unexpected response from node")
fail_count += 1
continue
node_state = rsp_json["node"]
node_id = node_state["id"]
if node_id != node['id']:
log.warn("unexpected node_id: {} (expecting: {})".format(
node_id, node['id']))
node['host'] = None
node['id'] = None
fail_count += 1
continue
if 'number' in node_state and node_state['number'] != node[
'node_number']:
msg = "unexpected node_number got {} (expecting: {})"
log.warn(
msg.format(node_state["number"], node['node_number']))
node['host'] = None
node['id'] = None
fail_count += 1
continue
# save off other useful info from the node
app_node_stats = app["node_stats"]
node_stats = {}
for k in NODE_STAT_KEYS:
node_stats[k] = rsp_json[k]
app_node_stats[node_id] = node_stats
# mark the last time we got a response from this node
node["healthcheck"] = unixTimeToUTC(int(time.time()))
node["failcount"] = 0 # rest
except OSError as ose:
log.warn("OSError for req: {}: {}".format(url, str(ose)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("node {}:{} not responding".format(
node["host"], node["port"]))
fail_count += 1
except HTTPInternalServerError as hpe:
log.warn("HTTPInternalServerError for req: {}: {}".format(
url, str(hpe)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("removing {}:{} from active list".format(
node["host"], node["port"]))
fail_count += 1
except TimeoutError as toe:
log.warn("Timeout error for req: {}: {}".format(url, str(toe)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("removing {}:{} from active list".format(
node["host"], node["port"]))
fail_count += 1
log.info("node health check fail_count: {}".format(fail_count))
if fail_count > 0:
if app["cluster_state"] == "READY":
# go back to INITIALIZING state until another node is registered
log.warn(
"Fail_count > 0, Setting cluster_state from READY to INITIALIZING"
)
app["cluster_state"] = "INITIALIZING"
elif fail_count == 0 and app["cluster_state"] != "READY":
log.info("All nodes healthy, changing cluster state to READY")
app["cluster_state"] = "READY"
import threading
import time
from common import logger
import json
import netMonitor
import api
import config
headers = {'content-type': 'application/json'}
fileName = config.get("flow_file")
task_time = config.get("task_time")
class myThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.policy = config.get("flow")
def run(self):
self.taskLoad()
def taskLoad(self):
self.timer_start()
while True:
time.sleep(60)
def timer_start(self):
t = threading.Timer(int(task_time), self.test_func)
t.start()
def getAdminUser():
# Returns the EZID administrator user.
import config
return getByUsername(config.get("auth.admin_username"))
settings = config.get(name='phpipam',
values=[
{
'value':
'url',
'prompt':
"Please enter the full URL of your "
"phpIPAM installation including the API"
" app_id \nex. https://phpipam."
"mycompanyserver.com/api/app_id/ \n"
"URL> ",
'optional':
False,
'sensitive':
False
},
{
'value': 'username',
'prompt':
"Please enter your phpIPAM username: \n"
"Username> ",
'optional': True,
'sensitive': False
},
{
'value': 'password',
'prompt':
"Please enter your phpIPAM password: \n"
"Password> ",
'optional': True,
'sensitive': True
},
])
def __init__(self):
threading.Thread.__init__(self)
self.policy = config.get("flow")
def testPostChunkedRefDataset(self):
print("testPostChunkedRefDataset", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
hdf5_sample_bucket = config.get("hdf5_sample_bucket")
if not hdf5_sample_bucket:
print(
"hdf5_sample_bucket config not set, skipping testChunkedRefDataset"
)
return
s3path = "s3://" + hdf5_sample_bucket + "/data/hdf5test" + "/snp500.h5"
SNP500_ROWS = 3207353
snp500_json = helper.getHDF5JSON("snp500.json")
if not snp500_json:
print(
"snp500.json file not found, skipping testPostChunkedRefDataset"
)
return
if "snp500.h5" not in snp500_json:
self.assertTrue(False)
chunk_dims = [
60000,
] # chunk layout used in snp500.h5 file
chunk_info = snp500_json["snp500.h5"]
dset_info = chunk_info["/dset"]
if "byteStreams" not in dset_info:
self.assertTrue(False)
byteStreams = dset_info["byteStreams"]
# construct map of chunks
chunks = {}
for item in byteStreams:
index = item["index"]
chunk_key = str(index)
chunks[chunk_key] = (item["file_offset"], item["size"])
# get domain
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
rspJson = json.loads(rsp.text)
self.assertTrue("root" in rspJson)
root_uuid = rspJson["root"]
# define types we need
s10_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 10,
"strPad": "H5T_STR_NULLPAD"
}
s4_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 4,
"strPad": "H5T_STR_NULLPAD"
}
fields = ({
'name': 'date',
'type': s10_type
}, {
'name': 'symbol',
'type': s4_type
}, {
'name': 'sector',
'type': 'H5T_STD_I8LE'
}, {
'name': 'open',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'high',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'low',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'volume',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'close',
'type': 'H5T_IEEE_F32LE'
})
datatype = {'class': 'H5T_COMPOUND', 'fields': fields}
data = {
"type": datatype,
"shape": [
SNP500_ROWS,
]
}
layout = {
"class": 'H5D_CHUNKED_REF',
"file_uri": s3path,
"dims": chunk_dims,
"chunks": chunks
}
data['creationProperties'] = {'layout': layout}
req = self.endpoint + '/datasets'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
dset_id = rspJson["id"]
self.assertTrue(helper.validateId(dset_id))
# link new dataset as 'dset'
name = "dset"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_id}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# do a point selection
req = self.endpoint + "/datasets/" + dset_id + "/value"
points = [
1234567,
]
body = {"points": points}
rsp = requests.post(req, data=json.dumps(body), headers=headers)
if rsp.status_code == 404:
print(
"s3object: {} not found, skipping point chunk ref test".format(
s3path))
else:
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), len(points))
item = value[0]
self.assertEqual(item[0], '1998.10.22')
self.assertEqual(item[1], 'MHFI')
self.assertEqual(item[2], 3)
return u + (marks['pos'] - marks['neg']) * diff / (N * N)
# The following variables are used in the remove_dc() method.
# Since we know the DC bias of our circuit, we can compute the
# digital value and simply remove it from all incoming signals.
#
# NOTE: This methodology assumes the DC bias remains constant
# for the entirety of sampling. Any fluctuations in the DC bias
# point can skew the output. There are algorithms that detect
# the DC bias on the fly. This may be worth looking into in the
# future.
#
# The variables are defined outside the function to avoid computing
# them everytime the function is called.
analog_ref_volt = cfg.get('sampling', 'analog_ref_volt') # V
bit_res = cfg.get('sampling', 'bit_res') # bits
dc_bias_volt = cfg.get('sampling', 'dc_bias_volt') # V
max_adc_val = pow(2, bit_res) - 1
# Compute the digital DC bias value
digital_dc_bias = int((max_adc_val / analog_ref_volt) * dc_bias_volt)
def remove_dc(data):
""" Remove the DC bias from a sample
Parameters
----------
data : int
def testPostContiguousDataset(self):
print("testPostContiguousDataset", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
hdf5_sample_bucket = config.get("hdf5_sample_bucket")
if not hdf5_sample_bucket:
print(
"hdf5_sample_bucket config not set, skipping testPostContiguousDataset"
)
return
tall_json = helper.getHDF5JSON("tall.json")
if not tall_json:
print(
"tall.json file not found, skipping testPostContiguousDataset")
return
if "tall.h5" not in tall_json:
self.assertTrue(False)
chunk_info = tall_json["tall.h5"]
if "/g1/g1.1/dset1.1.2" not in chunk_info:
self.assertTrue(False)
dset112_info = chunk_info["/g1/g1.1/dset1.1.2"]
if "byteStreams" not in dset112_info:
self.assertTrue(False)
byteStreams = dset112_info["byteStreams"]
# should be just one element for this contiguous dataset
self.assertTrue(len(byteStreams), 1)
byteStream = byteStreams[0]
dset112_offset = byteStream["file_offset"]
dset112_size = byteStream["size"]
self.assertEqual(dset112_size, 80)
if "/g2/dset2.2" not in chunk_info:
self.assertTrue(False)
dset22_info = chunk_info["/g2/dset2.2"]
if "byteStreams" not in dset22_info:
self.assertTrue(False)
byteStreams = dset22_info["byteStreams"]
self.assertTrue(len(byteStreams), 1)
byteStream = byteStreams[0]
dset22_offset = byteStream["file_offset"]
dset22_size = byteStream["size"]
self.assertEqual(dset22_size, 60)
# get domain
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
rspJson = json.loads(rsp.text)
self.assertTrue("root" in rspJson)
root_uuid = rspJson["root"]
# create dataset fodr /g1/g1.1/dset1.1.2
s3path = "s3://" + hdf5_sample_bucket + "/data/hdf5test" + "/tall.h5"
data = {"type": 'H5T_STD_I32BE', "shape": 20}
layout = {
"class": 'H5D_CONTIGUOUS_REF',
"file_uri": s3path,
"offset": dset112_offset,
"size": dset112_size
}
data['creationProperties'] = {'layout': layout}
req = self.endpoint + '/datasets'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
dset112_id = rspJson["id"]
self.assertTrue(helper.validateId(dset112_id))
# link new dataset as 'dset112'
name = "dset112"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset112_id}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# create dataset for /g2/dset2.2
data = {"type": 'H5T_IEEE_F32BE', "shape": [3, 5]}
layout = {
"class": 'H5D_CONTIGUOUS_REF',
"file_uri": s3path,
"offset": dset22_offset,
"size": dset22_size
}
data['creationProperties'] = {'layout': layout}
req = self.endpoint + '/datasets'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
dset22_id = rspJson["id"]
self.assertTrue(helper.validateId(dset22_id))
# link new dataset as 'dset22'
name = "dset22"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset22_id}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# do a point selection read on dset22
req = self.endpoint + "/datasets/" + dset112_id + "/value"
points = [2, 3, 5, 7, 11, 13, 17, 19]
body = {"points": points}
rsp = requests.post(req, data=json.dumps(body), headers=headers)
if rsp.status_code == 404:
print(
"s3object: {} not found, skipping point read chunk reference contiguous test"
.format(s3path))
return
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("value" in rspJson)
ret_value = rspJson["value"]
self.assertEqual(len(ret_value), len(points))
self.assertEqual(
ret_value,
points) # get back the points since the dataset in the range 0-20
# do a point selection read on dset22
req = self.endpoint + "/datasets/" + dset22_id + "/value"
points = [(0, 0), (1, 1), (2, 2)]
body = {"points": points}
rsp = requests.post(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("value" in rspJson)
ret_value = rspJson["value"]
self.assertEqual(len(ret_value), len(points))
def testPostChunkedRefIndirectDataset(self):
print("testPostChunkedRefIndirectDataset", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
hdf5_sample_bucket = config.get("hdf5_sample_bucket")
if not hdf5_sample_bucket:
print(
"hdf5_sample_bucket config not set, skipping testPostChunkedRefIndirectDataset"
)
return
s3path = "s3://" + hdf5_sample_bucket + "/data/hdf5test" + "/snp500.h5"
SNP500_ROWS = 3207353
snp500_json = helper.getHDF5JSON("snp500.json")
if not snp500_json:
print("snp500.json file not found, skipping testChunkedRefDataset")
return
if "snp500.h5" not in snp500_json:
self.assertTrue(False)
chunk_dims = [
60000,
] # chunk layout used in snp500.h5 file
num_chunks = (SNP500_ROWS // chunk_dims[0]) + 1
chunk_info = snp500_json["snp500.h5"]
dset_info = chunk_info["/dset"]
if "byteStreams" not in dset_info:
self.assertTrue(False)
byteStreams = dset_info["byteStreams"]
self.assertEqual(len(byteStreams), num_chunks)
chunkinfo_data = [(0, 0)] * num_chunks
# fill the numpy array with info from bytestreams data
for i in range(num_chunks):
item = byteStreams[i]
index = item["index"]
chunkinfo_data[index] = (item["file_offset"], item["size"])
# get domain
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
rspJson = json.loads(rsp.text)
self.assertTrue("root" in rspJson)
root_uuid = rspJson["root"]
# create table to hold chunkinfo
# create a dataset to store chunk info
fields = ({
'name': 'offset',
'type': 'H5T_STD_I64LE'
}, {
'name': 'size',
'type': 'H5T_STD_I32LE'
})
chunkinfo_type = {'class': 'H5T_COMPOUND', 'fields': fields}
req = self.endpoint + "/datasets"
# Store 40 chunk locations
chunkinfo_dims = [
num_chunks,
]
payload = {'type': chunkinfo_type, 'shape': chunkinfo_dims}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
chunkinfo_uuid = rspJson['id']
self.assertTrue(helper.validateId(chunkinfo_uuid))
# link new dataset as 'chunks'
name = "chunks"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": chunkinfo_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# write to the chunkinfo dataset
payload = {'value': chunkinfo_data}
req = self.endpoint + "/datasets/" + chunkinfo_uuid + "/value"
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200) # write value
# define types we need
s10_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 10,
"strPad": "H5T_STR_NULLPAD"
}
s4_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 4,
"strPad": "H5T_STR_NULLPAD"
}
fields = ({
'name': 'date',
'type': s10_type
}, {
'name': 'symbol',
'type': s4_type
}, {
'name': 'sector',
'type': 'H5T_STD_I8LE'
}, {
'name': 'open',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'high',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'low',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'volume',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'close',
'type': 'H5T_IEEE_F32LE'
})
datatype = {'class': 'H5T_COMPOUND', 'fields': fields}
data = {
"type": datatype,
"shape": [
SNP500_ROWS,
]
}
layout = {
"class": 'H5D_CHUNKED_REF_INDIRECT',
"file_uri": s3path,
"dims": chunk_dims,
"chunk_table": chunkinfo_uuid
}
data['creationProperties'] = {'layout': layout}
req = self.endpoint + '/datasets'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
dset_id = rspJson["id"]
self.assertTrue(helper.validateId(dset_id))
# link new dataset as 'dset'
name = "dset"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_id}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# do a point selection
req = self.endpoint + "/datasets/" + dset_id + "/value"
points = [
1234567,
]
body = {"points": points}
rsp = requests.post(req, data=json.dumps(body), headers=headers)
if rsp.status_code == 404:
print(
"s3object: {} not found, skipping point read chunk reference indirect test"
.format(s3path))
return
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), len(points))
item = value[0]
self.assertEqual(item[0], '1998.10.22')
self.assertEqual(item[1], 'MHFI')
self.assertEqual(item[2], 3)
