def instance_status_info():
print('Getting instances status')
for instance in instances:
vm = Instance(instance)
if vm.get_data():
logging.info(vm)
print(vm)
def project_with_instances(built_project, start_stop_cli, request):
cli = start_stop_cli
project = built_project
# This is necessary, because default app config has parameter `stateboard: true`
remove_project_file(project, '.cartridge.yml')
router = Instance('router', 8081, 'localhost:3301')
s1_master = Instance('s1-master', 8082, 'localhost:3302')
s1_replica = Instance('s1-replica', 8083, 'localhost:3303')
s1_replica_2 = Instance('s1-replica-2', 8084, 'localhost:3304')
s2_master = Instance('s2-master', 8085, 'localhost:3305')
p = ProjectWithTopology(
cli,
project,
instances_list=[
router, s1_master, s1_replica, s1_replica_2, s2_master
],
)
request.addfinalizer(lambda: p.stop())
p.start()
return p
def snapshots_cleaner():
print(f'Search and deleting snapshots older than {config.lifetime} days')
for instance in instances:
vm = Instance(instance)
snapshots = vm.get_old_snapshots()
if snapshots:
for snapshot in snapshots:
delete_snap = vm.delete_snapshot(snapshot)
if vm.operation_complete(delete_snap):
pass # logging.info from class Instance returned
def default_project_with_instances(built_default_project, start_stop_cli,
request):
cli = start_stop_cli
project = built_default_project
with open(project.get_cfg_path()) as f:
instances_cfg = yaml.load(f, Loader=yaml.FullLoader)
instances = [
Instance(
name.split('.', maxsplit=1)[1], conf.get('http_port'),
conf.get('advertise_uri')) for name, conf in instances_cfg.items()
if name.startswith('%s.' % project.name)
]
p = ProjectWithTopology(
cli,
project,
instances_list=instances,
)
request.addfinalizer(lambda: p.stop())
p.start()
return p
def project_with_one_joined_instance(cartridge_cmd, built_project,
start_stop_cli, request):
cli = start_stop_cli
project = built_project
instance = Instance('some-instance', 8081, 'localhost:3301')
p = ProjectWithTopology(
cli,
project,
instances_list=[instance],
)
p.start()
rpl = Replicaset('some-replicaset', instances=[instance])
# create replicaset
cmd = [
cartridge_cmd,
'replicasets',
'join',
'--replicaset',
rpl.name,
]
cmd.extend([i.name for i in rpl.instances])
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
# save replicasets
p.set_replicasets([rpl])
request.addfinalizer(lambda: p.stop())
return p
def project_with_instances(built_project, start_stop_cli, request):
cli = start_stop_cli
project = built_project
router = Instance('router', 8081, 'localhost:3301')
s1_master = Instance('s1-master', 8082, 'localhost:3302')
s1_replica = Instance('s1-replica', 8083, 'localhost:3303')
p = ProjectWithTopology(
cli,
project,
instances_list=[router, s1_master, s1_replica],
)
request.addfinalizer(lambda: p.stop())
p.start()
return p
def __call__(self, instances):
instances_encoded = list()
for id_, text, label in instances:
label_encoded = self.transform(label)
instances_encoded.append(Instance(id_, text, label_encoded))
return instances_encoded
def __call__(self, instances):
instances_tokenized = list()
for id_, text, label in tqdm(instances): # tqdm 추가 6.21
text_tokenized = ' '.join(
list(map(lambda x: x[0], self.tokenize(text))))
instances_tokenized.append(Instance(id_, text_tokenized, label))
return instances_tokenized
def __call__(self, instances):
if self.d2v_train:
tagged_documents = self.make_tagged_documents(
instances) # tagged document로 형식 변환
self.train(tagged_documents)
instances_embedded = list()
for id_, text, label in instances:
if self.text_cnn == False:
text_embedded = self.infer_vector(
text.split(' ')) # document to vector
instances_embedded.append(Instance(id_, text_embedded, label))
else:
instances_embedded.append(Instance(id_, text, label))
return instances_embedded
def __call__(self, instances):
instances_tokenized = list()
for id_, text, label in tqdm(instances): # tqdm 추가 6.21
temp = list()
for khaiiiword in self.tokenize(text):
for morph in khaiiiword.morphs:
temp += [morph.lex]
text_tokenized = ' '.join(temp)
instances_tokenized.append(Instance(id_, text_tokenized, label))
return instances_tokenized
def __call__(self, instances, sent_split=True):
instances_tokenized = []
for id_, text, label in instances:
text_tokenized = None
if sent_split:
text_tokenized = '\n'.join(
map(lambda x: ' '.join(self.tokenize(x)),
text.split('\n')))
else:
text_tokenized = ' '.join(self.tokenize(text))
instances_tokenized.append(Instance(id_, text_tokenized, label))
return instances_tokenized
def project_with_replicaset_no_roles(cartridge_cmd, built_project,
start_stop_cli, request):
cli = start_stop_cli
project = built_project
# This is necessary, because default app config has parameter `stateboard: true`
remove_project_file(project, '.cartridge.yml')
instance = Instance('some-instance', 8081, 'localhost:3301')
p = ProjectWithTopology(
cli,
project,
instances_list=[instance],
)
p.start()
rpl = Replicaset('some-rpl', instances=[instance])
cmd = [
cartridge_cmd,
'replicasets',
'join',
'--replicaset',
rpl.name,
]
cmd.extend([i.name for i in rpl.instances])
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
p.set_replicasets([rpl])
request.addfinalizer(lambda: p.stop())
return p
import dynet as dy
from utils import Instance
if __name__ == "__main__":
p = "/Users/enrique/scratch/lstm_polarity/model.dy"
pc = dy.ParameterCollection()
# a = pc.load_param(p, "/vanilla-lstm-builder/_0")
# b = pc.load_param(p, "/vanilla-lstm-builder/_1")
# c = pc.load_param(p, "/vanilla-lstm-builder/_2")
# print(a.value().sum())
# print(b.value().sum())
# print(sum(c.value()))
builder = dy.LSTMBuilder(1, 100, 10, pc)
x = pc.parameters_list()
y = 1
sentence = "To formally prove that increased ROS levels enhance anti-tumour effects of the SG-free diet , the authors crossed Emu-Myc mice with mice deficient for Tigar , a fructose-2 ,6-bisphosphatase , which limits glycolysis and favours pentose phosphate pathways , thus limiting ROS levels XREF_BIBR , XREF_BIBR ( XREF_FIG ) .".lower(
)
words = Instance.normalize(sentence)
a = 1
#x = builder.initial_state().b()
#a = 1
def project_with_vshard_replicasets(cartridge_cmd, built_project,
start_stop_cli, request):
cli = start_stop_cli
project = built_project
# This is necessary, because default app config has parameter `stateboard: true`
remove_project_file(project, '.cartridge.yml')
router = Instance('router', 8081, 'localhost:3301')
hot_master = Instance('hot-master', 8082, 'localhost:3302')
hot_replica = Instance('hot-replica', 8083, 'localhost:3303')
cold_master = Instance('cold-master', 8084, 'localhost:3304')
VSHARD_ROUTER_ROLE = 'vshard-router'
VSHARD_STORAGE_ROLE = 'vshard-storage'
p = ProjectWithTopology(
cli,
project,
instances_list=[router, hot_master, hot_replica, cold_master],
)
p.start()
# replicasets
router_rpl = Replicaset('router', instances=[router])
hot_storage_rpl = Replicaset('hot-storage',
instances=[
hot_master,
hot_replica,
])
cold_storage_rpl = Replicaset('cold-storage', instances=[
cold_master,
])
# router
cmd = [
cartridge_cmd,
'replicasets',
'join',
'--replicaset',
router_rpl.name,
]
cmd.extend([i.name for i in router_rpl.instances])
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
cmd = [
cartridge_cmd,
'replicasets',
'add-roles',
'--replicaset',
router_rpl.name,
VSHARD_ROUTER_ROLE,
]
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
# hot-storage
cmd = [
cartridge_cmd,
'replicasets',
'join',
'--replicaset',
hot_storage_rpl.name,
]
cmd.extend([i.name for i in hot_storage_rpl.instances])
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
cmd = [
cartridge_cmd,
'replicasets',
'add-roles',
'--replicaset',
hot_storage_rpl.name,
'--vshard-group',
'hot',
VSHARD_STORAGE_ROLE,
]
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
# cold-storage
cmd = [
cartridge_cmd,
'replicasets',
'join',
'--replicaset',
cold_storage_rpl.name,
]
cmd.extend([i.name for i in cold_storage_rpl.instances])
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
cmd = [
cartridge_cmd,
'replicasets',
'add-roles',
'--replicaset',
cold_storage_rpl.name,
'--vshard-group',
'cold',
VSHARD_STORAGE_ROLE,
]
rc, output = run_command_and_get_output(cmd, cwd=project.path)
assert rc == 0
# save replicasets
p.set_replicasets([router_rpl, hot_storage_rpl, cold_storage_rpl])
request.addfinalizer(lambda: p.stop())
return p
def snapshots_creater():
print('Creating snapshots')
for instance in instances:
vm = Instance(instance)
if vm.get_data():
if vm.status() != 'STOPPED':
stop_vm = vm.stop()
if vm.operation_complete(stop_vm):
snap_create = vm.create_snapshot()
if vm.operation_complete(snap_create):
if vm.status() != 'RUNNING':
start_vm = vm.start()
if vm.operation_complete(start_vm):
pass # logging.info from class Instance returned
else:
logging.info(f'Instance {vm.name()} already stopped.')
create_snap = vm.create_snapshot()
if vm.operation_complete(create_snap):
pass # logging.info from class Instance returned