def test__startup_services(self):
class FakeBrokerClient(object):
def __init__(self):
self.messages = {}
def invoke_async(self, msg):
self.messages[msg['service']] = msg
broker_client = FakeBrokerClient()
startup_services = Bunch()
for x in range(10):
name = rand_string()
payload = rand_string()
startup_services[name] = payload
ps = ParallelServer()
ps.broker_client = broker_client
ps.fs_server_config = Bunch()
ps.fs_server_config.startup_services = startup_services
ps.invoke_startup_services()
for expected_service, expected_payload in startup_services.items():
msg = Bunch(broker_client.messages[expected_service])
eq_(msg.action, SERVICE.PUBLISH)
eq_(msg.channel, CHANNEL.STARTUP_SERVICE)
eq_(msg.payload, expected_payload)
eq_(msg.service, expected_service)
ok_(msg.cid.startswith('K'))
self.assertEquals(len(msg.cid), 40)
def gen_updates_rmsprop_by_graves(all_parameters, all_grads,
learning_rate,
rho=0.95, momentum=0.9,
epsilon=0.0001,
get_diffs=False,
what_stats=None
):
updates = []
infos = []
for param_i, grad_i in zip(all_parameters, all_grads):
delta_i = theano.shared(get_p(param_i).get_value() * 0.)
n_i = theano.shared(get_p(param_i).get_value() * 0.)
g_i = theano.shared(get_p(param_i).get_value() * 0.)
n_i_new = rho * n_i + (1 - rho) * grad_i ** 2
g_i_new = rho * g_i + (1 - rho) * grad_i
delta_i_new = momentum * delta_i - get_lr(param_i) * learning_rate * grad_i / T.sqrt(n_i_new - (g_i_new ** 2) + epsilon)
updates.append((n_i, n_i_new))
updates.append((g_i, g_i_new))
updates.append((delta_i, delta_i_new))
update = delta_i_new
updates.append((get_p(param_i), get_p(param_i) + update))
info = Bunch()
info.grad = grad_i
if get_diffs:
info.diff = update
infos.append(info)
return final_result(updates, all_parameters, infos, what_stats)
def main():
vct = CountVectorizer(encoding='ISO-8859-1', min_df=5, max_df=1.0, binary=True, ngram_range=(1, 1),
token_pattern='\\b\\w+\\b', tokenizer=StemTokenizer())
vct_analizer = vct.build_analyzer()
print("Start loading ...")
# data fields: data, bow, file_names, target_names, target
########## NEWS GROUPS ###############
# easy to hard. see "Less is More" paper: http://axon.cs.byu.edu/~martinez/classes/678/Presentations/Clawson.pdf
categories = [['alt.atheism', 'talk.religion.misc'],
['comp.graphics', 'comp.windows.x'],
['comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware'],
['rec.sport.baseball', 'sci.crypt']]
if "imdb" in args.train:
########## IMDB MOVIE REVIEWS ###########
data = Bunch(load_imdb(args.train, shuffle=True, rnd=2356, vct=vct)) # should brind data as is
elif "aviation" in args.train:
raise Exception("We are not ready for that data yet")
elif "20news" in args.train:
########## 20 news groups ######
data = Bunch(load_20newsgroups(categories=categories[0], vectorizer=vct, min_size=50)) # for testing purposes
elif "dummy" in args.train:
########## DUMMY DATA###########
data = Bunch(load_dummy("C:/Users/mramire8/Documents/code/python/data/dummy", shuffle=True,rnd=2356,vct=vct))
else:
raise Exception("We do not know that dataset")
print("Data %s" % args.train)
total = len(data.train.data)
print("Data size %s" % total)
#print(data.train.data[0])
## prepare pool for the sampling
pool = Bunch()
pool.data = data.train.bow.tocsr() # full words, for training
pool.target = data.train.target
pool.predicted = []
pool.remaining = set(range(pool.data.shape[0])) # indices of the pool
bt = randomsampling.BootstrapFromEach(87654321)
for i in range(7):
query_index = bt.bootstrap(pool=pool, k=args.packsize) # get instances from each class
filename = "{0}-P{1}.txt".format(args.train,i)
f = codecs.open(filename, 'a+', 'utf-8')
#print documents in file
random.shuffle(query_index)
for di in query_index:
x = unicode(data.train.data[di].replace("\n","<br>"))
#y = data.train.target[di]
y = data.train.target_names[data.train.target[di]]
#f.write(str(i))
#f.write("\t")
#f.write(str(y))
#f.write("\t")
#f.write(x)
#f.write("\n")
f.close()
pool.remaining.difference_update(query_index) # remove the used ones
def test_sio_list_data_type_input_json(self):
cid = rand_string()
data_format = DATA_FORMAT.JSON
transport = rand_string()
sio_config = {'int_parameters': [rand_string()]} # Not really used but needed
service_sio = Bunch()
service_sio.input_required = ('first_name', 'last_name', List('emails'))
expected_first_name = faker.first_name()
expected_last_name = faker.last_name()
expected_emails = sorted([faker.email(), faker.email()])
r = Request(getLogger(__name__), sio_config)
r.payload = {
'first_name': expected_first_name,
'last_name': expected_last_name,
'emails': expected_emails,
}
r.init(True, cid, service_sio, data_format, transport, {})
eq_(r.input.first_name, expected_first_name)
eq_(r.input.last_name, expected_last_name)
eq_(r.input.emails, expected_emails)
def test_sio_list_data_type_input_xml(self):
cid = rand_string()
data_format = DATA_FORMAT.XML
transport = rand_string()
sio_config = {'int_parameters': [rand_string()]} # Not really used but needed
service_sio = Bunch()
service_sio.input_required = ('first_name', 'last_name', List('emails'))
expected_first_name = faker.first_name()
expected_last_name = faker.last_name()
expected_emails = sorted([faker.email(), faker.email()])
r = Request(getLogger(__name__), sio_config)
r.payload = etree.fromstring("""<request>
<first_name>{}</first_name>
<last_name>{}</last_name>
<emails>
<item>{}</item>
<item>{}</item>
</emails>
</request>""".format(
expected_first_name, expected_last_name, expected_emails[0], expected_emails[1]))
r.init(True, cid, service_sio, data_format, transport, {})
eq_(r.input.first_name, expected_first_name)
eq_(r.input.last_name, expected_last_name)
eq_(r.input.emails, expected_emails)
def process(self, sentence, strs, inspect=False):
if not len(strs):
raise Exception("\"strs\" should at least have one item.")
env = Bunch()
env.sentence = sentence
env.inspect = inspect
# 1. generate SplitBlock list
self.generate_SplitBlock_list(env)
# 2. fix params strs
env.params_strs = ParamsStrs(strs)
env.params_strs.load_env(env.split_block_group)
#env.base_chars_len = chars_len(env.params_strs.original_strs + [env.sentence])
self.fix_params_strs(env)
# 3. generate possible patterns, for multiple fill-able BLANKS.
self.generate_possible_patterns_with_strs_list(env)
# 4. generate possible patterns
self.generate_candidate_patterns_vs_word_groups_s(env)
# 5. generate regular sentence
results = self.generate_regular_sentence(env)
results = SentenceProcess.select_most_fit_sentence(results, read_attr_lambda=lambda item: item[0]) # , base_chars_len=env.base_chars_len)
if env.inspect:
print "[results] len", len(results)
return results
def run(self):
""" Handle action (other then builds) - like rename or delete of project """
result = Bunch()
result.id = self.data["id"]
action_type = self.data["action_type"]
if action_type == ActionType.DELETE:
if self.data["object_type"] == "copr":
self.handle_delete_copr_project()
elif self.data["object_type"] == "build":
self.handle_delete_build()
result.result = ActionResult.SUCCESS
elif action_type == ActionType.LEGAL_FLAG:
self.handle_legal_flag()
elif action_type == ActionType.RENAME:
self.handle_rename(result)
elif action_type == ActionType.CREATEREPO:
self.handle_createrepo(result)
if "result" in result:
if result.result == ActionResult.SUCCESS and \
not getattr(result, "job_ended_on", None):
result.job_ended_on = time.time()
self.frontend_callback.update({"actions": [result]})
def capture(self):
"""
Capture output of the whole build process.
Return stacktrace of Python driver and output of make command, if any.
"""
# The result object containing build success flag and error outputs.
# This gets bound to the context manager variable.
result = Bunch()
self.build_result['capture'] = result
# A temporary file to redirect make output to
self.stream = NamedTemporaryFile()
try:
# Pass execution flow to context manager body,
# effectively running the main build process
yield result
# Signal success if build engine completed
result.success = True
except Exception as ex:
# Signal failure
result.success = False
# Capture Python traceback
result.error = last_error_and_traceback()
# Capture make output
self.stream.seek(0)
result.output = self.stream.read()
def testExcludeWordsInQuery(self, requestsMock):
web.app.template_folder = "../templates"
expectedItems = self.prepareSearchMocks(requestsMock, 1, 1)
with web.app.test_request_context('/'):
response = self.app.get("/api?t=search&q=query+!excluded")
entries, _, _ = newznab.NewzNab(Bunch.fromDict({"name": "forTest", "score": 0, "host": "host"})).parseXml(response.data)
self.assertSearchResults(entries, expectedItems)
calledUrls = sorted([x.url for x in requestsMock.request_history])
self.assertTrue(compare('http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query+!excluded', calledUrls[0]))
response = self.app.get("/api?t=search&q=query+--excluded")
entries, _, _ = newznab.NewzNab(Bunch.fromDict({"name": "forTest", "score": 0, "host": "host"})).parseXml(response.data)
self.assertSearchResults(entries, expectedItems)
calledUrls = sorted([x.url for x in requestsMock.request_history])
self.assertTrue(compare('http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query+!excluded', calledUrls[0]))
self.app.get("/internalapi/search?query=query+!excluded&category=all")
calledUrls = sorted([x.url for x in requestsMock.request_history])
self.assertTrue(compare('http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query+!excluded', calledUrls[0]))
self.app.get("/internalapi/search?query=query+--excluded&category=all")
calledUrls = sorted([x.url for x in requestsMock.request_history])
self.assertTrue(compare('http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query+!excluded', calledUrls[0]))
def _create(self, name, config, **extra):
""" Actually adds a new definition, must be called with self.lock held.
"""
config_no_sensitive = deepcopy(config)
if 'password' in config:
config_no_sensitive['password'] = SECRET_SHADOW
item = Bunch(config=config, config_no_sensitive=config_no_sensitive, is_created=False, impl=None)
# It's optional
conn = extra.get('def_', {'conn':None})['conn']
try:
logger.debug('Creating `%s`', config_no_sensitive)
impl = self.create_impl(config, config_no_sensitive, **extra)
def execute(session, statement):
def execute_impl(**kwargs):
if not session:
raise Exception('Cannot execute the query without a session')
return session.execute(statement, kwargs)
return execute_impl
item.execute = execute(conn, impl)
logger.debug('Created `%s`', config_no_sensitive)
except Exception, e:
logger.warn('Could not create `%s`, config:`%s`, e:`%s`', name, config_no_sensitive, format_exc(e))
def load_atm(time,info,starttime,endtime):
"""Load atmospheric variable from a netcdf file"""
outputdata=Bunch()
for s,v in zip(icar_atm_var,atmvarlist):
atmfile=find_atm_file(time,v,info)
print(atmfile)
sys.stdout.flush()
nc_data=read_nc(atmfile,v,returnNCvar=True)
outputdata[s]=nc_data.data[starttime:endtime,:,info.ymin:info.ymax,info.xmin:info.xmax]
nc_data.ncfile.close()
atmfile=find_atm_file(time,"PS",info)
nc_data=read_nc(atmfile,"PS",returnNCvar=True)
outputdata.ps=nc_data.data[starttime:endtime,info.ymin:info.ymax,info.xmin:info.xmax]
nc_data.ncfile.close()
del nc_data
print(gc.collect())
sys.stdout.flush()
a=read_nc(atmfile,"hyam").data
b=read_nc(atmfile,"hybm").data
p0=read_nc(atmfile,"P0").data
#p_(i,j,k)= A_k * P_0 + B_k P_s(i,j) from http://www.cesm.ucar.edu/models/atm-cam/docs/usersguide/node25.html
outputdata.p = a[np.newaxis,:,np.newaxis,np.newaxis]*p0+b[np.newaxis,:,np.newaxis,np.newaxis]*outputdata.ps[:,np.newaxis,:,:]
outputdata.ntimes=outputdata.p.shape[0]
return outputdata
def convert_atm(data):
output_data=Bunch()
output_data.u = data.u[np.newaxis, ::-1,::-1,:] # m/s
output_data.v = data.v[np.newaxis, ::-1,::-1,:] # m/s
output_data.hgt = data.gph[::-1,:]/g # (m^2/s^2) / (m/s^2) = m
# calculate pressure in Pa from ln(sfc_press) and hybrid sigma coordinates
output_data.p = np.zeros((output_data.u.shape))
ps = np.exp(data.ln_p_sfc[::-1,:]) # Pa
for i in range(len(data.sigma_a)):
# see http://rda.ucar.edu/datasets/ds627.0/docs/Eta_coordinate/
# notes on http://aaron.boone.free.fr/aspdoc/node7.html might help...
output_data.p[0,len(data.sigma_a)-i-1,:,:]=(data.sigma_a[i]*data.P0+data.sigma_b[i]*ps)
psl=ps/((1 - 2.25577E-5*output_data.hgt)**5.25588)
output_data.z=np.zeros(output_data.p.shape)
for i in range(output_data.p.shape[1]):
output_data.z[0,i,...]=((output_data.p[0,i,...]/psl)**(1/5.25588)-1) / (-2.25577E-5)
pii=(100000.0/output_data.p)**(R/cp)
output_data.t=data.t[np.newaxis,::-1,::-1,:]*pii # K (converted to potential temperature)
output_data.qv = data.qv[np.newaxis,::-1,::-1,:] # kg/kg
output_data.cloud = data.cloud[np.newaxis,::-1,::-1,:] # kg/kg
output_data.ice = data.ice[np.newaxis,::-1,::-1,:] # kg/kg
return output_data
def __init__(self, conn, id = None, article_etree = None):
"""
Article
:param conn: Connection-Object
"""
Bunch.__init__(self)
self.conn = conn
self.content_language = None
self.id = id # integer
self.created = None # datetime
self.article_number = None
self.number = None # integer
self.number_pre = None
self.title = None
self.description = None
self.sales_price = None # float
self.sales_price2 = None # float
self.sales_price3 = None # float
self.sales_price4 = None # float
self.sales_price5 = None # float
self.currency_code = None
self.unit_id = None # integer
self.tax_id = None # integer
self.purchase_price = None # float
self.purchase_price_net_gross = None
self.supplier_id = None # integer
if article_etree is not None:
self.load_from_etree(article_etree)
elif id is not None:
self.load()
def test_delete_after_pick_up(self):
support_values = (True, False)
for delete_after_pick_up in support_values:
def ignored(*ignored, **ignored_kwargs):
pass
server = Bunch()
server.parallel_server = Bunch()
server.parallel_server.id = rand_int()
server.parallel_server.hot_deploy_config = Bunch()
server.parallel_server.hot_deploy_config.delete_after_pick_up = delete_after_pick_up
server.parallel_server.odb = Bunch()
server.parallel_server.notify_new_package = ignored
server.parallel_server.odb.hot_deploy = ignored
file_name = '{}.py'.format(uuid4().hex)
processor = BasePickupEventProcessor(uuid4().hex, server)
_os_remove = Mock()
util._os_remove = _os_remove
with NamedTemporaryFile(prefix='zato-test-', suffix=file_name) as tf:
tf.flush()
ret = processor.hot_deploy(tf.name, os.path.basename(tf.name))
self.assertEquals(ret, True)
if delete_after_pick_up:
_os_remove.assert_called_with(tf.name)
else:
self.assertFalse(_os_remove.called)
def __init__(self, conn, id = None, reminder_etree = None):
"""
Reminder
:param conn: Connection-Object
"""
Bunch.__init__(self)
self.conn = conn
self.content_language = None
self.id = id # integer
self.created = None # datetime
self.status = None
self.invoice_id = None # integer
self.contact_id = None # integer
self.reminder_text_id = None # integer
self.reminder_level = None # integer
self.reminder_level_name = None
self.date = None # date
self.label = None
self.subject = None
self.intro = None
self.note = None
self.due_date = None # date
self.total_gross = None # float
self.is_old = None # Steht dieses Flag auf 1, dann gibt es eine aktuellere Mahnung.
if reminder_etree is not None:
self.load_from_etree(reminder_etree)
elif id is not None:
self.load()
def load_atm(time,info):
"""Load atmospheric variable from a netcdf file"""
outputdata=Bunch()
for s,v in zip(icar_atm_var,atmvarlist):
atmfile=find_atm_file(time,v,info)
print(atmfile)
sys.stdout.flush()
nc_data=read_nc(atmfile,v)#,returnNCvar=True)
outputdata[s]=nc_data.data[:,:,info.ymin:info.ymax,info.xmin:info.xmax]
# nc_data.ncfile.close()
nc_data=read_nc(atmfile,"ps")#,returnNCvar=True)
outputdata.ps=nc_data.data[:,info.ymin:info.ymax,info.xmin:info.xmax]
# nc_data.ncfile.close()
del nc_data
print(gc.collect())
sys.stdout.flush()
a=read_nc(atmfile,"a").data
b=read_nc(atmfile,"b").data
p0=read_nc(atmfile,"p0").data
outputdata.p = a[np.newaxis,:,np.newaxis,np.newaxis]*p0+b[np.newaxis,:,np.newaxis,np.newaxis]*outputdata.ps[:,np.newaxis,:,:]
outputdata.ntimes=outputdata.p.shape[0]
return outputdata
def forward(self, bucket):
"""
Receive data bucket from source, run through
transformation machinery and emit to target.
"""
# 1. Map/transform topology address information
if 'transform' in self.channel:
for entrypoint in read_list(self.channel.transform):
try:
transformer = KotoriBootloader.load_entrypoint(entrypoint)
bucket.tdata.update(transformer(bucket.tdata))
except ImportError as ex:
log.error('ImportError "{message}" when loading entrypoint "{entrypoint}"',
entrypoint=entrypoint, message=ex)
# MQTT doesn't prefer leading forward slashes with topic names, let's get rid of them
target_uri_tpl = self.target_uri.path.lstrip('/')
# Compute target bus topic from url matches
target_uri = target_uri_tpl.format(**bucket.tdata)
# Enrich bucket by putting source and target addresses into it
bucket.address = Bunch(source=self.source_address, target=self.target_address)
# 2. Reporting
bucket_logging = Bunch(bucket)
if 'body' in bucket_logging and len(bucket_logging.body) > 100:
bucket_logging.body = bucket_logging.body[:100] + ' [...]'
log.debug('Forwarding bucket to {target} with bucket={bucket}. Effective target uri is {target_uri}',
target=self.channel.target, target_uri=target_uri, bucket=dict(bucket_logging))
# 3. Adapt, serialize and emit appropriately
return self.target_service.emit(target_uri, bucket)
def __init__(self, conn, id = None, contact_etree = None):
"""
Contact
:param conn: Connection-Object
"""
Bunch.__init__(self)
self.conn = conn
self.id = id # integer
self.created = None # datetime
self.client_id = None # integer
self.label = None
self.name = None
self.street = None
self.zip = None
self.city = None
self.state = None
self.country_code = None
self.first_name = None
self.last_name = None
self.salutation = None
self.phone = None
self.fax = None
self.mobile = None
self.email = None
self.www = None
if contact_etree is not None:
self.load_from_etree(contact_etree)
elif id is not None:
self.load()
def __init__(self, conn, id = None, recurring_item_etree = None):
"""
Recurring-Item
:param conn: Connection-Object
"""
Bunch.__init__(self)
self.conn = conn
self.id = id # integer
self.created = None # datetime
self.article_id = None
self.recurring_id = None # integer
self.position = None # integer
self.unit = None
self.quantity = None # float
self.unit_price = None # float
self.tax_name = None
self.tax_rate = None # float
self.title = None
self.description = None
self.total_gross = None # float
self.total_net = None # float
self.reduction = None
self.total_gross_unreduced = None # float
self.total_net_unreduced = None # float
if recurring_item_etree is not None:
self.load_from_etree(recurring_item_etree)
elif id is not None:
self.load()
def __init__(self, *args, **kwargs):
warnings.warn(
"DictContainer is deprecated. Use the Bunch class" " from python-bunch instead.",
DeprecationWarning,
stacklevel=2,
)
Bunch.__init__(self, *args, **kwargs)
def test_uniq(self):
one = Bunch.uniq(TEST_PATH)
two = Bunch.uniq(TEST_PATH, "test")
self.assertTrue(one.path != two.path)
self.assertTrue(one.path != TEST_PATH)
self.assertTrue(one.kind == GHOST)
self.assertTrue(two.kind == "test")
def remote(cmd, creds, curr_dir=None):
"""
runs a command on a remote machine and returns output
"""
if isinstance(creds, dict):
creds = Bunch(creds)
client = paramiko.SSHClient()
args = {'hostname':creds.host, 'username':creds.user}
if 'key_filename' in creds:
creds.key = os.path.expanduser(creds.key_filename[0])
if 'key' in creds:
key_path = os.path.expanduser(creds.key)
key = paramiko.RSAKey.from_private_key_file(key_path)
args['pkey'] = key
if 'password' in creds:
args['password'] = creds.password
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
args['timeout'] = SSH_TIMEOUT
client.connect(**args)
if curr_dir is not None:
make_dirs(curr_dir, creds)
cmd = 'cd "%s" && %s' % (curr_dir, cmd)
stdout, stderr = client.exec_command(cmd)[1:3]
output = stdout.read()
err = stderr.read()
client.close()
if len(err) > 0:
raise Exception(err)
return output
def test_create_channel(self):
channel_item = uuid4().hex
sec_info = uuid4().hex
soap_action = uuid4().hex
url_path = uuid4().hex
match_target = '{}{}{}'.format(soap_action, MISC.SEPARATOR, url_path)
def _dummy_channel_item_from_msg(*ignored):
return channel_item
def _dummy_sec_info_from_msg(*ignored):
return sec_info
msg = Bunch()
msg.soap_action = soap_action
msg.url_path = url_path
ud = url_data.URLData()
ud._channel_item_from_msg = _dummy_channel_item_from_msg
ud._sec_info_from_msg = _dummy_sec_info_from_msg
ud.channel_data = []
ud.url_sec = {}
ud._create_channel(msg, {})
self.assertIn(match_target, ud.url_sec)
eq_(ud.url_sec[match_target], sec_info)
eq_(len(ud.channel_data), 1)
eq_(ud.channel_data[0], channel_item)
def __next__(self):
self.curpos+=1
output_data=Bunch()
filename=self.files[self._curfile]
for v in self._var_names:
if type(v)==str:
curdata=self.load_data(v)
curvarname=v
elif type(v)==list:
cur_operator=v[0]
for varname in v[1:]:
if type(varname)==str:
curvarname=v[1]
break
curdata=self.load_data(v[1])
for curv in v[2:]:
next_data=self.load_data(curv)
cur_operator(curdata,next_data)
output_data[self._short_names[curvarname]]=curdata
output_data.date=self.get_current_date()
return output_data
def load_erai_means(wind_option):
"""docstring for load_erai_means"""
eraid="erai/"
varlist=["p","rh","ta","ua","va","z"]
if (wind_option=="nowind"):
varlist=["p","rh","ta","z"]
outputdata=[]
month_mid_point_doy=(start_day_per_month[1:]+np.array(start_day_per_month[:-1]))*0.5
for month in range(1,13):
curoutput=Bunch(doy=month_mid_point_doy[month-1])
for v in varlist:
if v=="ta":
ta=mygis.read_nc(eraid+"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),v).data
else:
curoutput[v]=mygis.read_nc(eraid+"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),v).data
curoutput["theta"] = ta / units.exner(curoutput.p)
# erai is probably "upside down" so reverse the vertical dimension of all variables
if curoutput.p[0,0,0]<curoutput.p[-1,0,0]:
for v in curoutput.keys():
if v!="doy":
curoutput[v]=curoutput[v][::-1,:,:]
outputdata.append(curoutput)
return outputdata
def generate_majority_vote(n = 200, m = 9):
vote = Bunch()
vote.DESCR = "Toy example to train a network how to do a majority vote."\
"Target is 1 iff there are more 1s than 0s in the input."
vote.data = np.random.randint(0, 2, (n, m))
vote.target = (np.sum(vote.data, 1) > m // 2).reshape(-1,1)
return vote
def load_xor():
xor = Bunch()
xor.DESCR = "The XOR function from logic. A Toy Example for Neural Networks. "\
"Needs at least two hidden units."
xor.data = np.array([[0,0],[0,1],[1,0],[1,1]])
xor.target = np.array([[0, 1, 1, 0]]).T
return xor
def test_on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(self):
no_old_name_msg = uuid4().hex
dummy_lock = DummyLock()
dummy_delete_channel = Dummy_delete_channel(no_old_name_msg)
dummy_create_channel = Dummy_create_channel()
ud = url_data.URLData()
ud.url_sec_lock = dummy_lock
ud._delete_channel = dummy_delete_channel
ud._create_channel = dummy_create_channel
old_name = uuid4().hex
key = uuid4().hex
value = uuid4().hex
for _old_name in(None, old_name):
msg = Bunch()
msg.old_name = old_name
msg[key] = value
ud.on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(msg)
if msg.old_name:
eq_(dummy_delete_channel.msg.old_name, msg.old_name)
eq_(dummy_delete_channel.msg[key], msg[key])
else:
eq_(dummy_delete_channel.msg, no_old_name_msg)
eq_(sorted(dummy_create_channel.msg.items()), sorted(msg.items()))
eq_(dummy_lock.enter_called, True)
def send_matches(self, request, obj, client, lawyers, form):
"""
Sent the matches email.
"""
logger.info('Sending project matches email')
recipient = '"{name}" <{email}>'.format(name=client.get_full_name(), email=client.email)
intro_content = '\r\n'.join(['<h2 style="color: #6c797c !important; font-family: \'Helvetica Neue\', Helvetica, Arial, sans-serif; font-size: 16px; font-weight: normal; line-height: 1.5em; margin: 0 0 25px; padding: 0;">{para}</h2>'.format(para=para) for para in form.cleaned_data.get('intro', '').split("\r\n") if para.strip() != ''])
email = Bunch(
template_name='project_matches',
from_email=SITE_EMAIL,
recipient_list=[recipient],
bcc=['*****@*****.**'],
context={
'lawyers': lawyers,
'to_name': client.first_name,
'intro_content': intro_content,
'url': reverse('dashboard:overview'),
}
)
if settings.DEBUG:
email.pop('bcc')
send_templated_mail(**email)
def __init__(self, conn, id = None, reminder_item_etree = None):
"""
Reminder-Item
:param conn: Connection-Object
"""
Bunch.__init__(self)
self.conn = conn
self.id = id # integer
self.created = None # datetime
self.article_id = None
self.reminder_id = None # integer
self.position = None # integer
self.unit = None
self.quantity = None # float
self.unit_price = None # float
self.title = None
self.description = None
self.total = None # float
if reminder_item_etree is not None:
self.load_from_etree(reminder_item_etree)
elif id is not None:
self.load()
def get_response_data(self):
return Bunch({'id': rand_int()})
def __init__(self, *args, **kwargs):
super(CreateEdit, self).__init__()
self.input = Bunch()
self.input_dict = {}
class CreateEdit(_BaseView):
""" Subclasses of this class will handle the creation/updates of Zato objects.
"""
def __init__(self, *args, **kwargs):
super(CreateEdit, self).__init__()
self.input = Bunch()
self.input_dict = {}
def __call__(self,
req,
initial_input_dict={},
initial_return_data={},
*args,
**kwargs):
""" Handles the request, taking care of common things and delegating
control to the subclass for fetching this view-specific data.
"""
self.input_dict.clear()
self.clear_user_message()
try:
super(CreateEdit, self).__call__(req, *args, **kwargs)
self.set_input()
self.populate_initial_input_dict(initial_input_dict)
input_dict = {'cluster_id': self.cluster_id}
post_id = self.req.POST.get('id')
if post_id:
input_dict['id'] = post_id
input_dict.update(initial_input_dict)
for name in chain(self.SimpleIO.input_required,
self.SimpleIO.input_optional):
if name not in input_dict and name not in self.input_dict:
value = self.input.get(name)
value = self.pre_process_item(name, value)
if value != SKIP_VALUE:
input_dict[name] = value
self.input_dict.update(input_dict)
logger.info('Request self.input_dict %s', self.input_dict)
logger.info('Request self.SimpleIO.input_required %s',
self.SimpleIO.input_required)
logger.info('Request self.SimpleIO.input_optional %s',
self.SimpleIO.input_optional)
logger.info('Request self.input %s', self.input)
logger.info('Request self.req.GET %s', self.req.GET)
logger.info('Request self.req.POST %s', self.req.POST)
logger.info('Sending `%s` to `%s`', self.input_dict,
self.service_name)
response = self.req.zato.client.invoke(self.service_name,
self.input_dict)
if response.ok:
return_data = {'message': self.success_message(response.data)}
return_data.update(initial_return_data)
for name in chain(self.SimpleIO.output_optional,
self.SimpleIO.output_required):
if name not in initial_return_data:
value = getattr(response.data, name, None)
if value:
if isinstance(value, basestring):
value = value.encode('utf-8')
else:
value = str(value)
return_data[name] = value
self.post_process_return_data(return_data)
logger.info('CreateEdit data for frontend `%s`', return_data)
return HttpResponse(dumps(return_data),
content_type='application/javascript')
else:
msg = 'response:`{}`, details.response.details:`{}`'.format(
response, response.details)
logger.error(msg)
raise ZatoException(msg=msg)
except Exception:
return HttpResponseServerError(format_exc())
def pre_process_item(self, name, value):
return value
def success_message(self, item):
raise NotImplementedError('Must be implemented by a subclass')
def post_process_return_data(self, return_data):
return return_data
@property
def verb(self):
if self.form_prefix:
return 'updated'
return 'created'
def __init__(self):
super(Index, self).__init__()
self.input = Bunch()
self.items = []
self.item = None
self.clear_user_message()
def create_configuration(self, **kwargs):
r"""Create a configuration object (set of parameter values) based on a scenario.
:param \**kwargs: See below.
:Keyword arguments:
* *application* -- Application of the intervention. 'itn' for insecticide-treated nets (exposure on day of first feed) or 'irs' (exposure on day of first indoor rest)
* *intervention* -- Type of intervention (characterized by combination of effects). 'dbh' for DBH, 'ins' for insecticide, or 'none' for no intervention.
* *coverage* -- Proportion of mosquitoes exposed on their first feed (or indoor rest). Value between 0 and 1.
* *dose* -- Dose of DBH or insecticide, which determines the efficacies of the different effects. 'low', 'medium', or 'high'
* *num_days* -- Run the simulation for this many days.
* *intervention_day* -- Introduce the intervention on this day. Must be less than num_days.
* *transmission* -- Transmission setting, determined by the biting rate. 'low', 'medium', or 'high'
* *larva_mortality* -- Shape of the relationship between larva mortality and larva population size. 'linear' or 'logistic'
* *human_infectiousness* -- Shape of the relationship between human infectiousness to mosquitoes and human prevalence. 'linear' or 'nonlinear'
"""
kwargs = Bunch(kwargs)
for k in ('intervention', 'coverage', 'dose', 'num_days',
'intervention_day', 'transmission', 'application',
'larva_mortality', 'human_infectiousness'):
if k not in kwargs:
raise ValueError('Need to specify "{}".'.format(k))
# use parameters object as a scaffold for our configuration object
config = copy.deepcopy(self._parameters)
del config.efficacy # we will be putting efficacies under "intervention"
# add some parameters
if kwargs.larva_mortality not in {'linear', 'logistic'}:
raise ValueError(
"Arguments for 'larva_mortality' are 'linear' or 'logistic'.")
if kwargs.human_infectiousness not in {'linear', 'nonlinear'}:
raise ValueError(
"Arguments for 'human_infectiousness' are 'linear' or 'nonlinear'."
)
config.simulation = Bunch(
num_days=kwargs.num_days,
intervention_day=kwargs.intervention_day,
larva_mortality=kwargs.larva_mortality,
human_infectiousness=kwargs.human_infectiousness)
config.intervention = Bunch(coverage=kwargs.coverage,
application=kwargs.application,
efficacy=Bunch(mating=0,
egg=0,
parasite=0,
persistent_mortality=[],
immediate_mortality=0))
# select the transmission level
try:
config.malaria.biting_scaling_factor = self._parameters.malaria.biting_scaling_factor[
kwargs.transmission]
except KeyError:
raise ValueError(
"Arguments for 'transmission' are 'low', 'medium', or 'high'.")
# select index corresponding to the dose level
try:
index = {"low": 0, "medium": 1, "high": 2}[kwargs.dose]
except KeyError:
raise ValueError(
"Arguments for 'dose' are 'low', 'medium', or 'high'.")
# then use dose index and intervention type to set combination of efficacies
itv = kwargs.intervention
if itv == 'dbh':
config.intervention.efficacy.mating = self._parameters.efficacy.mating[
index]
config.intervention.efficacy.egg = self._parameters.efficacy.egg[
index]
config.intervention.efficacy.parasite = self._parameters.efficacy.parasite[
index]
config.intervention.efficacy.persistent_mortality = self._parameters.efficacy.persistent_mortality[
index]
elif itv == 'ins':
config.intervention.efficacy.immediate_mortality = self._parameters.efficacy.immediate_mortality[
index]
elif itv == 'dbh_mating':
config.intervention.efficacy.mating = self._parameters.efficacy.mating[
index]
elif itv == 'dbh_egg':
config.intervention.efficacy.egg = self._parameters.efficacy.egg[
index]
elif itv == 'dbh_parasite':
config.intervention.efficacy.parasite = self._parameters.efficacy.parasite[
index]
elif itv == 'dbh_mortality':
config.intervention.efficacy.persistent_mortality = self._parameters.efficacy.persistent_mortality[
index]
elif itv == 'none':
pass
else:
raise ValueError(
"Arguments for 'intervention' are 'dbh', 'ins', 'dbh_mating', dbh_egg', 'dbh_parasite', dbh_mortality', or 'none'."
)
return config
def __init__(self, parameters_path):
self._parameters_path = parameters_path
with open(parameters_path) as par_file:
self._parameters = json.load(par_file,
object_hook=lambda d: Bunch(d))
def prepareIndexers(self, indexerCount):
config.settings.indexers = []
for i in range(1, indexerCount + 1):
nn = Bunch()
nn.enabled = True
nn.name = "newznab%d" % i
nn.type = "newznab"
nn.host = "http://www.newznab%d.com" % i
nn.apikey = "apikeyindexer.com"
nn.hitLimit = None
nn.timeout = None
nn.score = 0
nn.accessType = "both"
nn.search_ids = ["imdbid", "tvdbid", "rid"]
config.settings.indexers.append(nn)
def get_stats(self,
start,
stop,
service='*',
n=None,
n_type=None,
needs_trends=True,
stats_key_prefix=None,
suffixes=None):
""" Returns statistics for a given interval, as defined by 'start' and 'stop'.
service default to '*' for all services in that period and may be set to return
a one-element list of information regarding that particular service. Setting 'n'
to a positive integer will make it return only top n services.
"""
if not stats_key_prefix:
stats_key_prefix = self.stats_key_prefix
stats_elems = {}
all_services_stats = Bunch({'usage': 0, 'time': 0})
# All mean values
mean_all_services_list = []
# A mean value of all the mean values (mean_all_services_list)
mean_all_services = 0
start = parse(start)
stop = parse(stop)
delta = (stop - start)
if hasattr(delta, 'total_seconds'):
delta_seconds = delta.total_seconds()
else:
delta_seconds = delta.seconds
if not suffixes:
suffixes = self.get_suffixes(start, stop)
# We make several passes. First two passes are made over Redis keys, one gathers the services, if any at all,
# and another one actually collects statistics for each service found. Next pass, a partly optional one,
# computes trends for mean response time and service usage. Another one computes each of the service's
# average rate and updates other attributes basing on values collected in the previous step.
# Optionally, the last one will pick only top n elements of a given type (top mean response time
# or top usage).
# 1st pass
for suffix in suffixes:
keys = self.server.kvdb.conn.keys('{}{}:{}'.format(
stats_key_prefix, service, suffix))
for key in keys:
service_name = key.replace(stats_key_prefix,
'').replace(':{}'.format(suffix),
'')
stats_elem = StatsElem(service_name)
stats_elems[service_name] = stats_elem
# When building statistics, we can't expect there will be data for all the time
# elems built above so to guard against it, this is a dictionary whose keys are the
# said elems and values are mean/usage for each elem. The values will remain
# 0/0.0 if there is no data for the time elem, which may mean that in this
# particular time slice the service wasn't invoked at all.
stats_elem.expected_time_elems = OrderedDict(
(elem, Bunch({
'mean': 0,
'usage': 0.0
})) for elem in suffixes)
# 2nd pass
for service, stats_elem in stats_elems.items():
for suffix in suffixes:
key = '{}{}:{}'.format(stats_key_prefix, service, suffix)
# We can convert all the values to floats here to ease with computing
# all the stuff and convert them still to integers later on, when necessary.
key_values = Bunch((
(name, float(value))
for (name,
value) in self.server.kvdb.conn.hgetall(key).items()))
if key_values:
time = (key_values.usage * key_values.mean)
stats_elem.time += time
mean_all_services_list.append(key_values.mean)
all_services_stats.time += time
all_services_stats.usage += key_values.usage
stats_elem.min_resp_time = min(stats_elem.min_resp_time,
key_values.min)
stats_elem.max_resp_time = max(stats_elem.max_resp_time,
key_values.max)
for attr in ('mean', 'usage'):
stats_elem.expected_time_elems[suffix][
attr] = key_values[attr]
mean_all_services = '{:.0f}'.format(
sp_stats.tmean(
mean_all_services_list)) if mean_all_services_list else 0
# 3rd pass (partly optional)
for stats_elem in stats_elems.values():
stats_elem.mean_all_services = mean_all_services
stats_elem.all_services_time = int(all_services_stats.time)
stats_elem.all_services_usage = int(all_services_stats.usage)
values = stats_elem.expected_time_elems.values()
stats_elem.mean_trend_int = [int(elem.mean) for elem in values]
stats_elem.usage_trend_int = [int(elem.usage) for elem in values]
stats_elem.mean = float('{:.2f}'.format(
sp_stats.tmean(stats_elem.mean_trend_int)))
stats_elem.usage = sum(stats_elem.usage_trend_int)
stats_elem.rate = float('{:.2f}'.format(
sum(stats_elem.usage_trend_int) / delta_seconds))
self.set_percent_of_all_services(all_services_stats, stats_elem)
if needs_trends:
stats_elem.mean_trend = ','.join(
str(elem) for elem in stats_elem.mean_trend_int)
stats_elem.usage_trend = ','.join(
str(elem) for elem in stats_elem.usage_trend_int)
# 4th pass (optional)
if n:
for stats_elem in self.yield_top_n(n, n_type, stats_elems):
yield stats_elem
else:
for stats_elem in stats_elems.values():
yield stats_elem
def add(self, name, item):
with self.update_lock:
self.data[name] = Bunch()
self.data[name].config = item
self.ns_map[name] = self.data[name].config.value
def get_response_data(self):
return Bunch({'name': rand_string()})
myaperturenode = TeapotApertureNode(1, 10, 18, position)
node.addChildNode(myaperturenode, node.EXIT)
position += node.getLength()
n += 1
#----------------------------------------------
# Add the main bunch and lost particles bunch
#----------------------------------------------
print '\nAdding main bunch ...'
Intensity = 170e+10
m0 = mass_proton # protons ...
N_mp = 100000
Particle_distribution_file = 'Input/INITIAL-6D.dat'
macrosize = Intensity / N_mp
bunch = Bunch()
setBunchParamsPTC(bunch)
kin_Energy = bunch.getSyncParticle().kinEnergy()
print ' Momentum: ', bunch.getSyncParticle().momentum(), 'GeV'
print ' Ekin: ', bunch.getSyncParticle().kinEnergy(), 'GeV'
print ' Gamma: ', bunch.getSyncParticle().gamma()
print ' Beta: ', bunch.getSyncParticle().beta()
print ' Charge: ', bunch.charge(), 'e'
print ' Mass: ', bunch.mass(), 'GeV'
bunch_orbit_to_pyorbit(paramsDict["length"], kin_Energy,
Particle_distribution_file, bunch,
N_mp + 1) #read in only first N_mp particles.
print ' Number of particles in the bunch: ', bunch.getSizeGlobal()
ParticleIdNumber().addParticleIdNumbers(
bunch) # Give particles unique number ids
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):
# Emit events about an upcoming subscription
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.1',
'sub_key': ctx.sub_key
})
self.pubsub.emit_about_to_subscribe({
'stage': 'init.ctx',
'data': ctx
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.2',
'sub_key': ctx.sub_key
})
# Endpoint on whose behalf the subscription will be made
endpoint = self.pubsub.get_endpoint_by_id(ctx.endpoint_id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'endpoint',
'data': endpoint,
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.3',
'sub_key': ctx.sub_key
})
with closing(self.odb.session()) as session:
with session.no_autoflush:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage':
'no_ctx_ws_channel_id',
'data':
ctx.ws_channel_id
})
self.pubsub.emit_about_to_subscribe({
'stage':
'sub.sk.4',
'sub_key':
ctx.sub_key
})
if has_subscription(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id):
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'has_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ctx.topic_id': ctx.topic.id,
'ctx.endpoint_id': ctx.endpoint_id,
}
})
raise PubSubSubscriptionExists(
self.cid,
'Endpoint `{}` is already subscribed to topic `{}`'
.format(endpoint.name, ctx.topic.name))
# Is it a WebSockets client?
is_wsx = bool(ctx.ws_channel_id)
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.5',
'sub_key': ctx.sub_key
})
ctx.creation_time = now = utcnow_as_ms()
sub_key = new_sub_key(self.endpoint_type,
ctx.ext_client_id)
self.pubsub.emit_in_subscribe_impl({
'stage': 'new_sk_generated',
'data': {
'sub_key': sub_key,
}
})
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_add_subscription',
'data': {
'is_wsx': is_wsx,
'ctx.creation_time': ctx.creation_time,
'sub_key': sub_key,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Create a new subscription object and flush the session because the subscription's ID
# may be needed for the WSX subscription
ps_sub = add_subscription(session, ctx.cluster_id, sub_key,
ctx)
session.flush()
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_add_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ps_sub': ps_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Common configuration for WSX and broker messages
sub_config = Bunch()
sub_config.topic_name = ctx.topic.name
sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
sub_config.endpoint_name = endpoint.name
sub_config.endpoint_type = self.endpoint_type
sub_config.unsub_on_wsx_close = ctx.unsub_on_wsx_close
sub_config.ext_client_id = ctx.ext_client_id
for name in sub_broker_attrs:
sub_config[name] = getattr(ps_sub, name, None)
#
# At this point there may be several cases depending on whether there are already other subscriptions
# or messages in the topic.
#
# * If there are subscribers, then this method will not move any messages because the messages
# will have been already moved to queues of other subscribers before we are called
#
# * If there are no subscribers but there are messages in the topic then this subscriber will become
# the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
# other subscribers can appear)
#
# * If there are no subscribers and no messages in the topic then this is a no-op
#
move_messages_to_sub_queue(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id,
ctx.sub_pattern_matched,
sub_key, now)
# Subscription's ID is available only now, after the session was flushed
sub_config.id = ps_sub.id
# Update current server's pub/sub config
self.pubsub.add_subscription(sub_config)
if is_wsx:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_wsx_sub',
'data': {
'is_wsx':
is_wsx,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object persists across multiple WSX connections
wsx_sub = add_wsx_subscription(
session, ctx.cluster_id, ctx.is_internal, sub_key,
ctx.ext_client_id, ctx.ws_channel_id, ps_sub.id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_wsx_sub',
'data': {
'wsx_sub':
wsx_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object will be transient - dropped each time a WSX client disconnects
self.pubsub.add_wsx_client_pubsub_keys(
session, ctx.sql_ws_client_id, sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id,
ctx.web_socket.get_peer_info_dict())
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(sub_key)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = sub_key
if is_wsx:
# Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
# note that since this is a new subscription, it is certain that only GD messages may be available,
# never non-GD ones.
ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
sub_key)
gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
sub_key)
self.response.payload.queue_depth = gd_depth + non_gd_depth
else:
# TODO:
# This should be read from that client's delivery task instead of SQL so as to include
# non-GD messages too.
self.response.payload.queue_depth = get_queue_depth_by_sub_key(
session, ctx.cluster_id, sub_key, now)
# Notify workers of a new subscription
sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
# Append information about current server which will let all workers
# know if they should create a subscription object (if they are different) or not.
sub_config.server_receiving_subscription_id = self.server.id
sub_config.server_receiving_subscription_pid = self.server.pid
sub_config.is_api_call = True
logger_pubsub.info(
'Subscription created id=`%s`; t=`%s`; sk=`%s`; patt=`%s`',
sub_config['id'], sub_config['topic_name'],
sub_config['sub_key'], sub_config['sub_pattern_matched'])
self.broker_client.publish(sub_config)
class VaultConnAPI(object):
""" An API through which connections to Vault are established and managed.
"""
def __init__(self, config_list=None, requests_adapter=None):
self.config = Bunch()
self.lock = RLock()
self.requests_adapter = requests_adapter
for config in config_list or []:
self.create(config)
# ################################################################################################################################
def __getitem__(self, name):
return self.config[name]
# ################################################################################################################################
def get(self, name):
return self.config.get(name)
# ################################################################################################################################
def get_client(self, name):
return self.config[name].client
# ################################################################################################################################
def _ping(self, name):
try:
self.config[name].client.ping()
except Exception:
logger.warn('Could not ping Vault connection `%s`, e:`%s`', name, format_exc())
else:
logger.info('Ping OK, Vault connection `%s`', name)
def ping(self, name):
spawn(self._ping, name)
# ################################################################################################################################
def _create(self, config):
conn = _VaultConn(
config.name, config.url, config.token, config.get('service_name'), config.tls_verify, config.timeout,
config.allow_redirects, requests_adapter=self.requests_adapter)
self.config[config.name] = conn
if config.url != UNITTEST.VAULT_URL:
self.ping(config.name)
# ################################################################################################################################
def create(self, config):
with self.lock:
self._create(config)
# ################################################################################################################################
def _delete(self, name):
try:
self.config[name].client.close()
except Exception:
logger.warn(format_exc())
finally:
del self.config[name]
# ################################################################################################################################
def delete(self, name):
with self.lock:
self._delete(name)
# ################################################################################################################################
def edit(self, new_config):
with self.lock:
self._delete(new_config.old_name)
self._create(new_config)
def __init__(self, data={}):
self.data = Bunch.fromDict(data)
self.ns_map = {}
self.update_lock = RLock()
def get_response_data(self):
return Bunch()
def train_eval_fn(FLAGS,
worker_count,
task_index,
is_chief,
target,
init_checkpoint,
train_file,
dev_file,
checkpoint_dir,
is_debug):
graph = tf.Graph()
with graph.as_default():
import json
config = json.load(open(FLAGS.config_file, "r"))
config = Bunch(config)
config.use_one_hot_embeddings = True
config.scope = "bert"
config.dropout_prob = 0.1
config.label_type = "single_label"
if FLAGS.if_shard == "0":
train_size = FLAGS.train_size
epoch = int(FLAGS.epoch / worker_count)
elif FLAGS.if_shard == "1":
train_size = int(FLAGS.train_size/worker_count)
epoch = FLAGS.epoch
init_lr = 2e-5
label_dict = json.load(open(FLAGS.label_id))
num_train_steps = int(
train_size / FLAGS.batch_size * epoch)
num_warmup_steps = int(num_train_steps * 0.1)
num_storage_steps = int(train_size / FLAGS.batch_size)
num_eval_steps = int(FLAGS.eval_size / FLAGS.batch_size)
if is_debug == "0":
num_storage_steps = 2
num_eval_steps = 10
num_train_steps = 10
print("num_train_steps {}, num_eval_steps {}, num_storage_steps {}".format(num_train_steps, num_eval_steps, num_storage_steps))
print(" model type {}".format(FLAGS.model_type))
print(num_train_steps, num_warmup_steps, "=============")
opt_config = Bunch({"init_lr":init_lr/worker_count,
"num_train_steps":num_train_steps,
"num_warmup_steps":num_warmup_steps,
"worker_count":worker_count,
"opt_type":FLAGS.opt_type})
model_io_config = Bunch({"fix_lm":False})
model_io_fn = model_io.ModelIO(model_io_config)
optimizer_fn = optimizer.Optimizer(opt_config)
num_classes = FLAGS.num_classes
model_train_fn = model_fn_builder(config, num_classes, init_checkpoint,
model_reuse=None,
load_pretrained=True,
model_io_fn=model_io_fn,
optimizer_fn=optimizer_fn,
model_io_config=model_io_config,
opt_config=opt_config,
exclude_scope="",
not_storage_params=[],
target="")
model_eval_fn = model_fn_builder(config, num_classes, init_checkpoint,
model_reuse=True,
load_pretrained=True,
model_io_fn=model_io_fn,
optimizer_fn=optimizer_fn,
model_io_config=model_io_config,
opt_config=opt_config,
exclude_scope="",
not_storage_params=[],
target="")
def eval_metric_fn(features, eval_op_dict):
logits = eval_op_dict["logits"]
print(logits.get_shape(), "===logits shape===")
pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32)
prob = tf.nn.softmax(logits)
accuracy = correct = tf.equal(
tf.cast(pred_label, tf.int32),
tf.cast(features["label_ids"], tf.int32)
)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
return {"accuracy":accuracy, "loss":eval_op_dict["loss"],
"pred_label":pred_label, "label_ids":features["label_ids"]}
def train_metric_fn(features, train_op_dict):
logits = train_op_dict["logits"]
print(logits.get_shape(), "===logits shape===")
pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32)
prob = tf.nn.softmax(logits)
accuracy = correct = tf.equal(
tf.cast(pred_label, tf.int32),
tf.cast(features["label_ids"], tf.int32)
)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
return {"accuracy":accuracy, "loss":train_op_dict["loss"],
"train_op":train_op_dict["train_op"]}
name_to_features = {
"input_ids":
tf.FixedLenFeature([FLAGS.max_length], tf.int64),
"input_mask":
tf.FixedLenFeature([FLAGS.max_length], tf.int64),
"segment_ids":
tf.FixedLenFeature([FLAGS.max_length], tf.int64),
"label_ids":
tf.FixedLenFeature([], tf.int64),
}
def _decode_record(record, name_to_features):
"""Decodes a record to a TensorFlow example.
"""
example = tf.parse_single_example(record, name_to_features)
# tf.Example only supports tf.int64, but the TPU only supports tf.int32.
# So cast all int64 to int32.
for name in list(example.keys()):
t = example[name]
if t.dtype == tf.int64:
t = tf.to_int32(t)
example[name] = t
return example
params = Bunch({})
params.epoch = FLAGS.epoch
params.batch_size = FLAGS.batch_size
train_features = tf_data_utils.train_input_fn(train_file,
_decode_record, name_to_features, params, if_shard=FLAGS.if_shard,
worker_count=worker_count,
task_index=task_index)
eval_features = tf_data_utils.eval_input_fn(dev_file,
_decode_record, name_to_features, params, if_shard=FLAGS.if_shard,
worker_count=worker_count,
task_index=task_index)
train_op_dict = model_train_fn(train_features, [], tf.estimator.ModeKeys.TRAIN)
eval_op_dict = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL)
eval_dict = eval_metric_fn(eval_features, eval_op_dict["eval"])
train_dict = train_metric_fn(train_features, train_op_dict["train"])
def eval_fn(eval_dict, sess):
i = 0
total_accuracy = 0
eval_total_dict = {}
while True:
try:
eval_result = sess.run(eval_dict)
for key in eval_result:
if key not in eval_total_dict:
if key in ["pred_label", "label_ids"]:
eval_total_dict[key] = []
eval_total_dict[key].extend(eval_result[key])
if key in ["accuracy", "loss"]:
eval_total_dict[key] = 0.0
eval_total_dict[key] += eval_result[key]
else:
if key in ["pred_label", "label_ids"]:
eval_total_dict[key].extend(eval_result[key])
if key in ["accuracy", "loss"]:
eval_total_dict[key] += eval_result[key]
i += 1
if np.mod(i, num_eval_steps) == 0:
break
except tf.errors.OutOfRangeError:
print("End of dataset")
break
label_id = eval_total_dict["label_ids"]
pred_label = eval_total_dict["pred_label"]
result = classification_report(label_id, pred_label,
target_names=list(label_dict["label2id"].keys()))
print(result, task_index)
eval_total_dict["classification_report"] = result
return eval_total_dict
def train_fn(train_op_dict, sess):
i = 0
cnt = 0
loss_dict = {}
monitoring_train = []
monitoring_eval = []
while True:
try:
[train_result, step] = sess.run([train_op_dict, tf.train.get_global_step()])
for key in train_result:
if key == "train_op":
continue
else:
if np.isnan(train_result[key]):
print(train_loss, "get nan loss")
break
else:
if key in loss_dict:
loss_dict[key] += train_result[key]
else:
loss_dict[key] = train_result[key]
i += 1
cnt += 1
if np.mod(i, num_storage_steps) == 0:
string = ""
for key in loss_dict:
tmp = key + " " + str(loss_dict[key]/cnt) + "\t"
string += tmp
print(string, step)
monitoring_train.append(loss_dict)
eval_finial_dict = eval_fn(eval_dict, sess)
monitoring_eval.append(eval_finial_dict)
for key in loss_dict:
loss_dict[key] = 0.0
cnt = 0
if is_debug == "0":
if i == num_train_steps:
break
except tf.errors.OutOfRangeError:
print("==Succeeded in training model==")
print("===========begin to train============")
sess_config = tf.ConfigProto(allow_soft_placement=False,
log_device_placement=False)
checkpoint_dir = checkpoint_dir if task_index == 0 else None
print("==checkpoint_dir==", checkpoint_dir)
print("start training")
hooks = []
if FLAGS.opt_type == "ps":
sync_replicas_hook = optimizer_fn.opt.make_session_run_hook(is_chief, num_tokens=0)
hooks.append(sync_replicas_hook)
sess = tf.train.MonitoredTrainingSession(master=target,
is_chief=is_chief,
config=sess_config,
hooks=hooks,
checkpoint_dir=checkpoint_dir,
save_checkpoint_steps=num_storage_steps)
elif FLAGS.opt_type == "pai_soar" and pai:
sess = tf.train.MonitoredTrainingSession(master=target,
is_chief=is_chief,
config=sess_config,
hooks=hooks,
checkpoint_dir=checkpoint_dir,
save_checkpoint_steps=num_storage_steps)
elif FLAGS.opt_type == "hvd" and hvd:
bcast_hook = hvd.BroadcastGlobalVariablesHook(0)
hooks.append(bcast_hook)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.visible_device_list = str(hvd.local_rank())
sess = tf.train.MonitoredTrainingSession(checkpoint_dir=checkpoint_dir,
hooks=hooks,
config=sess_config,
save_checkpoint_steps=num_storage_steps)
else:
print("==single sess==")
sess = tf.train.MonitoredTrainingSession(config=sess_config,
hooks=hooks,
checkpoint_dir=checkpoint_dir,
save_checkpoint_steps=num_storage_steps)
step = sess.run(optimizer_fn.global_step)
print(step)
train_fn(train_dict, sess)
if task_index == 0:
print("===========begin to eval============")
eval_finial_dict = eval_fn(eval_dict, sess)
def parse_options():
# strategy: read in command line options
# if an options file is specified, the settings in there override
# what is chosen on the command line. the command line parser options
# are read to parser_options, the config file reader options to config.
# the final options are stored in cl_options.
parser = OptionParser()
group_input = OptionGroup(
parser, "input file specification",
"These files are required to run the simulation. They "
"may either be specified via the command line arguments "
"or in an options file.")
group_sim = OptionGroup(
parser, "simulation options",
"These options modify simulation parameters such as "
"duration, initialisation temperature, etc.")
parser.add_option_group(group_sim)
parser.add_option_group(group_input)
parser.add_option(
"-o",
"--options",
dest="optfile",
type="string",
help=
"path and name of a file specifying input options and simulation options",
metavar=">file<")
parser.add_option("-p",
"--path",
dest="outpath",
type="string",
default='./',
help="destination path for simulation output",
metavar=">path<")
group_input.add_option("-f",
"--forcing",
dest="forcing_file",
type="string",
help="location and name of the atmospheric forcing",
metavar=">file<")
group_input.add_option(
"-z",
"--horizon",
dest="horizon_file",
type="string",
help="location and name of the horizon function file",
metavar=">file<")
group_input.add_option(
"-c",
"--car",
dest="car_file",
type="string",
help="location and name of the car general information file",
metavar=">file<")
group_input.add_option("-b",
"--body",
dest="body_file",
type="string",
help="location and name of the car body file",
metavar=">file<")
group_sim.add_option("-d",
"--duration",
dest="duration",
type="float",
default=2.0,
help="duration of the simulation in hours",
metavar=">duration<")
group_sim.add_option(
"-T",
"--T0",
dest="T0",
type="float",
default=20.0,
help="initialisation temperature of all components in deg. Celsius",
metavar=">T0<")
group_sim.add_option("--date",
dest="date",
type="string",
help="begin of simulation date, format: YYYY-MM-DD",
metavar=">date<")
group_sim.add_option("--time",
dest="time",
type="string",
help="begin of simulation time, format: HH:MM",
metavar=">time<")
(parser_options, args) = parser.parse_args()
cl_options = Bunch()
if not parser_options.optfile is None:
if file_exists(parser_options.optfile):
config = configparser.ConfigParser()
config.read(parser_options.optfile)
print(" * reading options from specified file")
float_keys = ["duration", "t0"]
# copy all options to cl_options structure
for sec in config.sections():
for key in config.options(sec):
if key in float_keys:
if key == 't0': # need to check this since keys are only lower case
cl_options["T0"] = float(config.get(sec, key))
else:
cl_options[key] = float(config.get(sec, key))
else:
cl_options[key] = config.get(sec, key)
else:
print(" error, submitted options file not found!")
print(" {:s}".format(cl_options.optfile))
option_errors = ""
if cl_options == Bunch():
option_errors += " please supply location and name of an options file or other required parameters\n"
else:
if cl_options.forcing_file is None:
option_errors += " please supply location and name of a forcing file\n"
if cl_options.car_file is None:
option_errors += " please supply location and name of a car general information file\n"
if cl_options.body_file is None:
option_errors += " please supply location and name of a car body_file\n"
if not "horizon_file" in cl_options: # a horizon file is no requirement. if not submitted
cl_options.horizon_file = None # it's just ground and sky.
if option_errors != "":
print("")
print(option_errors[:-1])
print(" try --help for more information.")
print("")
print("exiting...")
sys.exit(1)
return cl_options
def edit(self, new_config):
with self.lock:
self._delete(new_config.old_name)
self._create(new_config)
# ################################################################################################################################
if __name__ == '__main__':
name = 'abc'
client_token = '5f763fa3-2872-71ab-4e5d-f1398aca6637'
username = '******'
password = '******'
gh_token = ''
config = Bunch()
config.name = name
config.url = 'http://localhost:49517'
config.token = client_token
config.service_name = 'my.service'
config.tls_verify = True
config.timeout = 20
config.allow_redirects = True
api = VaultConnAPI([config])
import time
time.sleep(0.1)
response1 = api[name].client.authenticate(VAULT.AUTH_METHOD.TOKEN.id, client_token)
logger.info('Response1 %s', response1)
elem = teapot.DriftTEAPOT("a drift")
elem.setLength(lattice_length)
elem.setnParts(n_parts)
teapot_lattice = teapot.TEAPOT_Lattice("teapot_lattice")
teapot_lattice.addNode(elem)
teapot_lattice.initialize()
return teapot_lattice
lattice_length = 248.0 # the length of the drift
n_parts = 1 # number of parts on what the drift will be chopped, or the number of SC nodes
lattice = getLattice(lattice_length,n_parts)
#------------------------------
#Main Bunch init
#------------------------------
b = Bunch()
print "Read Bunch."
runName = "Benchmark_SpaceCharge"
total_macroSize=1.0e+14
b.mass(0.93827231)
energy = 1.0 #Gev
bunch_orbit_to_pyorbit(lattice.getLength(), energy, "Bm_KV_Uniform_10",b)
b.dumpBunch("bunch_init.dat")
b.getSyncParticle().kinEnergy(energy)
nParticlesGlobal = b.getSizeGlobal()
b.macroSize(total_macroSize/nParticlesGlobal)
print total_macroSize/nParticlesGlobal
#-----------------------------------
# Add Direct Force Space Charge node
#-----------------------------------
#tokenizer_en = tfds.features.text.SubwordTextEncoder.load_from_file(file_path.subword_vocab_path)
hyp = {
"num_layers" : 3, #number of transformer blocks
"d_model" : 256, #the projected word vector dimension
"dff" : 512, #feed forward network hidden parameters
"num_heads" : 8, #the number of heads in the multi-headed attention unit
"dropout_rate" : 0.0,
"epsilon_ls" : 0.1, #label_smoothing hyper parameter
"batch_size" : 2,
"epochs" : 2,
"print_chks" : 50,
"input_vocab_size" : 8096, # 8094 + start and end token
"target_vocab_size" : 8096,
"doc_length" : 51200,
"summ_length" : 7000,
"beam_size" : 3,
"test_size" : 0.20,
"copy_gen" : True,
"decay_lr" : False, #decay learning rate
"run_tensorboard" : True,
"from_scratch" : True,
"write_summary_op" : True
}
#print all the model hyperparameters
config = Bunch(hyp)
from bunch import Bunch
PORT = 15170
LISTEN_ADDRESS = "0.0.0.0"
MAX_PACKET_SIZE = 6 * 4
VELOCITY_THRESHOLD = 1
JUMP_THRESHOLD = 5
# Usage: Movement_State.LEFT
MOVEMENT_STATE = Bunch.fromDict({
"LEFT": "Left",
"STILL": "Still",
"RIGHT": "Right"
})
from injection import InjectParts
from injection import JohoTransverse, SNSESpreadDist
from orbit.utils.orbit_mpi_utils import bunch_orbit_to_pyorbit, bunch_pyorbit_to_orbit
print "Start."
xmin = -0.050
xmax = 0.050
ymin = -0.050
ymax = 0.050
foilparams = (xmin, xmax, ymin, ymax)
#------------------------------
#Bunch init
#------------------------------
b = Bunch()
runName = "Test_Injection"
b.mass(0.93827231)
b.macroSize(1.0e+1)
energy = 1.0 #Gev
b.getSyncParticle().kinEnergy(energy)
lostfoilbunch = Bunch()
lostfoilbunch.addPartAttr("LostParticleAttributes")
#------------------------------
#Initial Distribution Functions
#------------------------------
sp = b.getSyncParticle()
print('error')
#test
if __name__ == '__main__':
config_dict = {
"data_train": "<path>/dataset/mnist/mnist.npz",
"num_class": 10,
"input_h": 28,
"input_w": 28,
"input_c": 1,
"batch_size": 5,
"num_iter_per_epoch": 2000000,
}
config_dict.update(keras_format=tf.keras.backend.image_data_format())
config = Bunch(config_dict)
g_config.__init(config)
class Train():
def __init__(self):
self.net = gen_net()
self.optimizer = gen_optimizer()
self.accuracy = (
lambda logits, labels: compute_acc(logits, labels))
self.loss = (lambda logits, labels: compute_loss(logits, labels))
def train_step(self, model, optimizer, images, labels):
with tf.GradientTape() as tape:
teapot_lattice.addNode(elem)
teapot_lattice.initialize()
return teapot_lattice
lattice_length = 248.0 # the length of the lattice
# number of parts into which the drift will be chopped:
n_parts = 1
# also the number of SC nodes
lattice = getLattice(lattice_length, n_parts)
#------------------------------
# Bunch initialization
#------------------------------
b = Bunch()
print "Read Bunch."
runName = "Benchmark_Collimator"
total_macroSize = 1.0e+16
b.mass(consts.mass_proton)
ERef = 1.0 # Gev
print "Reference energy is 1.0 (GeV). \nPlease input desired energy:"
energy = float(raw_input())
print "energy is:", energy
bunch_orbit_to_pyorbit(lattice.getLength(), energy, "Bm_KV_Uniform_10000", b)
b.getSyncParticle().kinEnergy(energy)
nParticlesGlobal = b.getSizeGlobal()
b.macroSize(total_macroSize / nParticlesGlobal)
def get_config_from_json(json_file):
with open(json_file, 'r') as config_file:
config_dict = json.load(config_file)
config = Bunch(config_dict)
return config, config_dict
def on_call_finished(self, invoked_service, response, exception):
now = invoked_service.time.utcnow()
cid = invoked_service.wsgi_environ['zato.request_ctx.{}'.format(
self.request_ctx_cid_key)]
data_key = self.data_pattern.format(cid)
counter_key = self.counter_pattern.format(cid)
source, req_ts_utc, on_target = invoked_service.kvdb.conn.hmget(
data_key, 'source', 'req_ts_utc', 'on_target')
with invoked_service.lock(self.lock_pattern.format(cid)):
data = Bunch()
data.cid = cid
data.resp_ts_utc = now
data.response = response
data.exception = exception
data.ok = False if exception else True
data.source = source
data.target = invoked_service.name
data.req_ts_utc = req_ts_utc
# First store our response and exception (if any)
json_data = dumps(data)
invoked_service.kvdb.conn.hset(data_key,
invoked_service.get_name(),
json_data)
on_target = loads(on_target)
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_target', on_target,
invoked_service)
# We always invoke 'on_target' callbacks, if there are any
self.invoke_callbacks(invoked_service, data, on_target,
self.on_target_channel, cid)
# Was it the last parallel call?
if not invoked_service.kvdb.conn.decr(counter_key):
# Not every subclass will need final callbacks
if self.needs_on_final:
payload = invoked_service.kvdb.conn.hgetall(data_key)
payload['data'] = {}
for key in (key for key in payload.keys()
if key not in JSON_KEYS):
payload['data'][key] = loads(payload.pop(key))
for key in JSON_KEYS:
if key not in ('source', 'data', 'req_ts_utc'):
payload[key] = loads(payload[key])
on_final = payload['on_final']
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_final', on_final,
invoked_service)
self.invoke_callbacks(invoked_service, payload, on_final,
self.on_final_channel, cid)
invoked_service.kvdb.conn.delete(counter_key)
invoked_service.kvdb.conn.delete(data_key)
quads = accLattice.getNodesOfClass(Quad)
#--- The last quad will have zero field.
#--- we will use it in the other examples
#--- to provide tilted exit plane.
#--- It is better to have exit from the 3D region
#--- in the free space.
quads[len(quads) - 1].setParam("dB/dr",0.)
for quad in quads:
quad.setnParts(10)
print "quad =",quad.getName()," gradient[T/m] = %+6.5f "%quad.getParam("dB/dr")
print "========================================================"
bunch_ini = Bunch()
eKin = 1.3
bunch_ini.getSyncParticle().kinEnergy(eKin)
#----------------------------------------------------
# Let's calculate beta functions for 90 deg FODO
# Brho[T*m] = (10/2.998)*beta_relativistic*E[GeV]
# focusing_length[m] = Brho/(g*length_of_quad) g = [T/m] quad gradient
# cappa = 2*focusing_length/L_between_quads
# Twiss beta_max = L*cappa*(cappa + 1)/sqrt(cappa**2 - 1)
# Twiss beta_min = L*cappa*(cappa - 1)/sqrt(cappa**2 - 1)
# L = L_between_quads
#-----------------------------------------------------
L_between_quads = accLattice.getNodePositionsDict()[quads[1]][1] - accLattice.getNodePositionsDict()[quads[0]][1]
print "Distance between quads [m] = ",L_between_quads
def _get_feature_info(obj, request):
params = dict((k.upper(), v) for k, v in request.params.items())
p_bbox = map(float, params.get('BBOX').split(','))
p_width = int(params.get('WIDTH'))
p_height = int(params.get('HEIGHT'))
p_srs = params.get('SRS')
p_info_format = params.get('INFO_FORMAT', b'text/html')
p_x = float(params.get('X'))
p_y = float(params.get('Y'))
p_query_layers = params.get('QUERY_LAYERS').split(',')
p_feature_count = int(params.get('FEATURE_COUNT', GFI_FEATURE_COUNT))
bw = p_bbox[2] - p_bbox[0]
bh = p_bbox[3] - p_bbox[1]
qbox = dict(l=p_bbox[0] + bw * (p_x - GFI_RADIUS) / p_width,
b=p_bbox[3] - bh * (p_y + GFI_RADIUS) / p_height,
r=p_bbox[0] + bw * (p_x + GFI_RADIUS) / p_width,
t=p_bbox[3] - bh * (p_y - GFI_RADIUS) / p_height)
srs = SRS.filter_by(id=int(p_srs.split(':')[-1])).one()
qgeom = geom_from_wkt(("POLYGON((%(l)f %(b)f, %(l)f %(t)f, " +
"%(r)f %(t)f, %(r)f %(b)f, %(l)f %(b)f))") % qbox,
srs.id)
lmap = dict((lyr.keyname, lyr) for lyr in obj.layers)
results = list()
fcount = 0
for lname in p_query_layers:
layer = lmap[lname]
flayer = layer.resource.feature_layer
request.resource_permission(DataScope.read, layer.resource)
request.resource_permission(DataScope.read, flayer)
query = flayer.feature_query()
query.intersects(qgeom)
# Limit number of layer features so that we
# don't overshoot its total number
query.limit(p_feature_count - fcount)
features = list(query())
fcount += len(features)
results.append(
Bunch(keyname=layer.keyname,
display_name=layer.display_name,
feature_layer=flayer,
features=features))
# Needed number of features found, stop search
if fcount >= p_feature_count:
break
if p_info_format == 'application/json':
result = [
dict(
keyname=result.keyname,
display_name=result.display_name,
features=[{
fld.display_name: feature.fields[fld.keyname]
for fld in result.feature_layer.fields
} for feature in result.features],
) for result in results
]
return Response(json.dumps(result, cls=geojson.Encoder),
content_type='application/json',
charset='utf-8')
return Response(render_template(
'nextgisweb:wmsserver/template/get_feature_info_html.mako',
dict(results=results, resource=obj),
request=request),
content_type='text/html',
charset='utf-8')
def parse():
parser = argparse.ArgumentParser(
description='Convert ERAi files to ICAR input forcing files')
parser.add_argument('start_date',
nargs="?",
action='store',
help="Specify starting date (yyyy-mm-dd)",
default="2000-10-01")
parser.add_argument('end_date',
nargs="?",
action='store',
help="Specify end date (yyyy-mm-dd)",
default="2000-10-02")
parser.add_argument('lat_n',
nargs="?",
action='store',
help="northern latitude boundary",
default="60")
parser.add_argument('lat_s',
nargs="?",
action='store',
help="southern latitude boundary",
default="20")
parser.add_argument('lon_e',
nargs="?",
action='store',
help="eastern longitude boundary",
default="-50")
parser.add_argument('lon_w',
nargs="?",
action='store',
help="western longitude boundary",
default="-140")
parser.add_argument('dir',
nargs="?",
action='store',
help="ERAi file location",
default="/glade/p/rda/data/ds627.0/")
parser.add_argument('atmdir',
nargs="?",
action='store',
help="ERAi atmospheric data file location",
default="ei.oper.an.ml/_Y__M_/")
parser.add_argument('sfcdir',
nargs="?",
action='store',
help="ERAi surface data file location",
default="ei.oper.fc.sfc/_Y__M_/")
parser.add_argument('atmfile',
nargs="?",
action='store',
help="ERAi primary atmospheric file",
default="ei.oper.an.ml.regn128sc._Y__M__D__h_")
parser.add_argument('atmuvfile',
nargs="?",
action='store',
help="ERAi U/V atm file",
default="ei.oper.an.ml.regn128uv._Y__M__D__h_")
parser.add_argument('sfcfile',
nargs="?",
action='store',
help="ERAi surface file",
default="ei.oper.fc.sfc.regn128sc._Y__M__D__h_")
parser.add_argument('temp_nc_dir',
nargs="?",
action='store',
help="temporary directory to store netCDF files in",
default="temp_nc_dir")
parser.add_argument('-v',
'--version',
action='version',
version='ERAi2ICAR v' + version)
parser.add_argument('--verbose',
action='store_true',
default=False,
help='verbose output',
dest='verbose')
args = parser.parse_args()
date0 = args.start_date.split("-")
start_date = datetime.datetime(int(date0[0]), int(date0[1]), int(date0[2]))
date0 = args.end_date.split("-")
end_date = datetime.datetime(int(date0[0]), int(date0[1]), int(date0[2]))
if args.temp_nc_dir[-1] != "/":
args.temp_nc_dir += "/"
info = Bunch(lat=[float(args.lat_s), float(args.lat_n)],
lon=[float(args.lon_w), float(args.lon_e)],
start_date=start_date,
end_date=end_date,
atmdir=args.dir + args.atmdir,
sfcdir=args.dir + args.sfcdir,
atmfile=args.atmfile,
uvfile=args.atmuvfile,
sfcfile=args.sfcfile,
nc_file_dir=args.temp_nc_dir,
version=version)
return info
def __init__(cls, classname, bases, nmspc):
# Table name is equal to resource id by default.
# It'll hardlybe ever needed otherwise, but let's keep this
# possibility.
if '__tablename__' not in cls.__dict__:
setattr(cls, '__tablename__', cls.identity)
# Child class can set it's own arguments, let's
# keep it possible. If not set, set our own.
if '__mapper_args__' not in cls.__dict__:
mapper_args = dict()
setattr(cls, '__mapper_args__', mapper_args)
else:
mapper_args = getattr(cls, '__mapper_args__')
if 'polymorphic_identity' not in mapper_args:
mapper_args['polymorphic_identity'] = cls.identity
# For Resource class this variable is not set yet.
Resource = globals().get('Resource', None)
if Resource and cls != Resource:
# Field with external key is needed for child classes, pointing
# to base resource class. May need to be created by hand, but easier to
# create for all together.
if 'id' not in cls.__dict__:
idcol = db.Column('id', db.ForeignKey(Resource.id),
primary_key=True)
idcol._creation_order = Resource.id._creation_order
setattr(cls, 'id', idcol)
# Automatic parent link field detection may not work
# if there are two fields with external key to resource.id.
if 'inherit_condition' not in mapper_args:
mapper_args['inherit_condition'] = (
cls.id == Resource.id)
scope = Bunch()
for base in cls.__mro__:
bscope = base.__dict__.get('__scope__', None)
if bscope is None:
continue
if bscope and not hasattr(bscope, '__iter__'):
bscope = tuple((bscope, ))
for s in bscope:
scope[s.identity] = s
setattr(cls, 'scope', scope)
super(ResourceMeta, cls).__init__(classname, bases, nmspc)
resource_registry.register(cls)
