def add(self, item):
bookmark = self.find(item)
if not bookmark:
Storage.add(self, item)
self.save()
def test_find_video(self):
Storage.add(self.subject, MediaInfo(path='path1'))
Storage.add(self.subject, MediaInfo(path='path2'))
search_item = MediaInfo(path='path1')
self.assertEqual(self.subject.find(search_item)['path'], 'path1')
def test_add():
st = Storage({})
st.add('a', 5)
key = 'a'
val = st.get(key)
assert val == 5, "Value for the key {} is not equal to expected".format(
key)
def add(self, item):
bookmark = self.find(item)
if not bookmark:
Storage.add(self, item)
self.save()
class ChatBot(object):
def __init__(self, name):
self.name = name
self.storage = Storage()
def get_response(self, text):
return self.storage.find(text)
def train(self, dot_path):
print("Begin train...")
corpus = Corpus()
corpus_data = corpus.load_corpus(dot_path)
for data in corpus_data:
for conversation in data: # conversation表示一个会话
statement_history = []
for text in conversation: # 依次从训练对话中取句子
if statement_history:
self.storage.add(statement_history[-1], text.encode('utf-8'));
statement_history.append(text.encode('utf-8')) # 当前语句加入历史列表
print("End of train!")
def dump(self):
print("Begin dump...")
self.storage.dump()
print("End of dump!")
def printx(self):
self.storage.printx()
def test_find_season_with_wrong_season_attribute(self):
Storage.add(self.subject,
MediaInfo(type='season', path='path3', season='season1'))
Storage.add(self.subject,
MediaInfo(type='season', path='path4', season='season2'))
search_item = MediaInfo(type='season', path='path1', season='season3')
self.assertEqual(self.subject.find(search_item), None)
def test_find_season_with_correct_season_attribute(self):
Storage.add(self.subject,
MediaInfo(type='season', path='path3', season='season1'))
Storage.add(self.subject,
MediaInfo(type='season', path='path4', season='season2'))
search_item = MediaInfo(type='season', path='path4', season='season2')
self.assertEqual(self.subject.find(search_item)['path'], 'path4')
class ColorBuilder(object):
def __init__(self, seed, kernel, printer):
self.__seed = seed
self.__kernel = kernel
self.__storage = Storage()
self.__knownTxs = set((seed.txhash,))
self.__access = rpc_access.fromFile('../data/rpc.ini')
self.__printer = printer
coloredTxs = self.__storage.get_all(seed.color_id)
for tx in coloredTxs:
self.__knownTxs.add(str(tx[0]))
def isColored(self, tx):
for inp in tx.inputs:
if ('txid' in inp) and (inp['txid'] in self.__knownTxs):
return True
return False
def processBlock(self, blockHeight):
txhashes = self.__access.getblock(self.__access.getblockhash(blockHeight))['tx']
for txhash in txhashes:
tx = Transaction(self.__access.getrawtransaction(txhash, 1))
if not self.isColored(tx):
continue
colorVals = list()
self.__printer(txhash + ", inputs: " + str(len(tx.inputs)) + ", outputs: " + str(len(tx.outputs)))
for inp in tx.inputs:
prevHash = inp['txid']
prevTx = Transaction(self.__access.getrawtransaction(prevHash, 1))
# print(str(t1.duration_in_seconds()) + ", inputs: " + str(len(prevTx['vin'])) + ", outputs: " + str(len(prevTx['vout'])))
prevOutIndex = inp['outindx']
colorVal = self.__storage.get(self.__seed.color_id, prevHash, prevOutIndex)
colorVals.append((prevTx.outputs[prevOutIndex], colorVal))
colorOuts = self.__kernel(tx, colorVals)
colorFound = False
for i, colorOut in enumerate(colorOuts):
if colorOut[0] != 0:
self.__storage.add(self.__seed.color_id, txhash, i, colorOut[0], colorOut[1])
colorFound = True
if colorFound:
self.__knownTxs.add(str(txhash))
def build(self):
seedTransaction = self.__access.getrawtransaction(self.__seed.txhash, 1)
for i, seedOut in enumerate(self.__seed.outputs):
self.__storage.add(self.__seed.color_id, self.__seed.txhash, i, seedOut.value, seedOut.label)
block = self.__access.getblock(seedTransaction['blockhash'])
blockHeight = block['height']
while blockHeight <= self.__access.getblockcount():
self.__printer(blockHeight)
self.processBlock(blockHeight)
blockHeight += 1
def test_add(): #Катя
st = Storage({'a': 1, 'b': 2})
key = 'ItsAKEy'
value = 'ItsAValue'
st.add(key, value)
assert st.data[
key] == value, "The key {} is absent or value is not equal to expected one {}".format(
key, value)
print(st)
def test_add():
st = Storage({'a': 1, 'b': 2})
new_key = 'c'
new_value = 3
st.add(new_key, new_value)
assert ('c' in st.data) and (st.data['c'] == 3)
try:
st.add(new_key, new_value)
assert False # make sure exception raises
except Exception as e:
pass
def test_add():
st = Storage({'a': 1, 'b': 2})
key = 'c'
value = 5
st.add(key, value)
assert st.get(key) == 5, "Element is not added"
key = 'b'
value = 4
try:
st.add(key, value)
except KeyError:
assert True
def test_add():
st = Storage({})
key_1, value_1 = 'a', 1
st.add(key_1, value_1)
assert key_1 in st.data.keys(), "Pair not added"
assert st.data[key_1] == value_1, "Added wrong value"
try:
st.add(key_1, value_1)
except KeyError:
pass
else:
assert False, "Exeption about existing key not raised"
def test_add():
st = Storage({'a': 1, 'b': 2})
key = 'c'
val = 3
st.add(key, val)
if st.data['c'] != 3:
return 1
key = 'a'
try:
st.add(key, val)
except Exception:
return 0
return 1
def main():
storage1 = Storage()
storage1.add('some.url', 'some_user', 'some_pass')
storage1.add('second.url', 'DobrijZmej', '******')
storage1.add('other.url', 'DobrijZmej1', '******')
storage1.get_all()
print(storage1.get('some.url'))
storage1.erase('second.url')
storage1.get_all()
storage1.erase('second.url')
def test_add():
st = Storage({'a': 1, 'b': 2})
key, val = 'c', 3
st.add(key, val)
assert st.get(
key) == val, "Value for the key {} is not equal to excepted".format(
key)
key, val = 'b', 20
try:
st.add(key, val)
except KeyError:
pass
else:
raise Exception
def refresh(self):
storage = Storage()
storage.connect()
response = requests.get(self.WORLDOMETER, timeout=30)
soup = BeautifulSoup(response.text, 'html.parser')
main_table = soup.find(id='main_table_countries_today')
today = datetime.today()
rows = main_table.find_all('tr')[1:]
data = []
for row in rows:
tr = row.find('tr')
items = row.find_all('td')
country = items[0].text
total_cases = items[1].text
total_deaths = items[3].text
total_recovered = items[5].text
active_cases = items[6].text
critical_cases = items[7].text
data.append({
'area': country,
'total_cases': self.format_int(total_cases),
'total_deaths': self.format_int(total_deaths),
'total_recovered': self.format_int(total_recovered),
'active_cases': self.format_int(active_cases),
'critical_cases': self.format_int(critical_cases),
'year': today.year,
'month': today.month,
'day': today.day
})
total_cases = data[-1]
total_cases.update({'area': 'Total'})
storage.add(total_cases)
poland_cases = list(filter(lambda item: item['area'] == 'Poland',
data))
if len(poland_cases) == 1:
storage.add(poland_cases[0])
storage.close()
def test_find_episode_with_season_attribute_no_episode_attribute(self):
Storage.add(
self.subject,
MediaInfo(type='episode',
path='path5',
season='season3',
episode="episode1"))
Storage.add(
self.subject,
MediaInfo(type='episode',
path='path6',
season='season4',
episode="episode2"))
search_item = MediaInfo(type='episode', path='path5', season='season3')
self.assertEqual(self.subject.find(search_item), None)
class WeatherFacade:
def __init__(self, config):
self.__storage = Storage()
self.__client = HttpClient(config.proxy, 10)
self.__weather_service = YahooWeatherApi(self.__client)
self.__update_weather()
self.__weather_scheduler = Scheduler(self.__update_weather)
self.__weather_scheduler.start(3600, 00)
def get_weather_forecast(self):
weather = self.__storage.get('weather')
return weather
def __update_weather(self):
weather = self.__weather_service.get_forecast(834463).to_json()
logging.info(u'Updated weather')
self.__storage.add('weather', weather)
class RatesFacade:
def __init__(self, config):
self.__storage = Storage()
self.__client = HttpClient(config.proxy, 10)
self.__rates_service = NbrbRates(self.__client)
self.__update_rates()
self.__rates_scheduler = Scheduler(self.__update_rates)
self.__rates_scheduler.start(3600 * 24, 00, 00)
def get_rate(self):
cur_rate = self.__storage.get('cur_rate')
res = json.loads(cur_rate)
return res['result']
def get_rates(self, currency, tenor, start=None, end=None):
if tenor is not None:
match = re.search('(\\d+)([mMwW])', tenor)
if match is not None:
tenor_i = int(match.group(1))
m_w = match.group(2)
key = 'rates-' + str(
datetime.date.today()) + str(tenor_i) + m_w
res = self.__storage.get(key)
if res is None:
end = datetime.date.today()
start = None
if m_w in ['m', 'M']:
start = utils.month_delta(end, tenor_i * -1)
elif m_w in ['w', 'W']:
start = end - datetime.timedelta(days=7 * tenor_i)
rates = self.__rates_service.get_rates_dynamics(
currency, start, end).to_json()
self.__storage.add(key, rates)
return rates
else:
return res
else:
s_start = utils.string_to_date(start, "%Y-%m-%d")
s_end = utils.string_to_date(end, "%Y-%m-%d")
return self.__rates_service.get_rates_dynamics(
currency, s_start, s_end).to_json()
def __update_rates(self):
cur_rate = self.__rates_service.get_today_rate('USD').to_json()
logging.info(u'Updated rates')
self.__storage.add('cur_rate', cur_rate)
def test_add():
st = Storage({'a': 1, 'b': 2})
# Test One
key = 'c'
value = 3
result = st.add(key, value)
assert result == 0, "The function did not return a 'success' code"
val = st.get(key)
assert val != None, "The added key does not exist in the storage"
assert val == value, "Value for the key {} is not eqial to expected".format(
key)
# Test Two
key = 'b'
value = 4
result = st.add(key, value)
assert result == 404, "The function did not return an 'unsuccess' code"
def test_add():
#try our add function to add new key
st = Storage({'a': 1, 'b': 2})
key = 'c'
val = 3
st = st.add(key, val)
assert st.get(
key) == val, "Value {} for the new key {} added unsuccessfully".format(
val, key)
#try our add function to add existing key
existing_key = 'b'
try:
st = st.add(existing_key, val)
except Exception:
pass
else:
print("After adding existing key {} Exseption not Raised".format(
existing_key))
def test_add():
st = Storage({'a': 1, 'b': 2})
key = 'c'
value = 3
st.add(key, value)
assert st.get(key) == value,\
f"Value for the key {key} is not equal to expected value {value}"
key = 4
value = 'd'
st.add(key, value)
assert st.get(key) == value,\
f"Value for the key {key} is not equal to expected value {value}"
key = 'a'
value_before_add = st.get(key)
value_to_add = -1
try:
st.add(key, value_to_add)
assert False,\
f"Adding an existing key should raise an exception"
except Exception:
pass
value_after_add = st.get(key)
assert value_after_add == value_before_add,\
f"After adding an existing key {key}, value has changed from {value_before_add} to {value_after_add}"
class HashIndex(Index):
"""
Индекс, опирающийся на результаты работы функции hash
совместно с функцией-селектором.
"""
def __init__(self, config, name, selector=lambda x: x):
super(HashIndex, self).__init__(config, name)
self._selector = selector
self._storage = Storage('LL', self._make_buffer())
# словарь для получения позиции в буфере по id объекта
self._id_to_pos = None
# словарь списков id обхектов,
# ключи которого - результаты вычисления hash(selector(x))
self._index = None
def _build(self):
# построение индекса из буфера
self._id_to_pos = {}
self._index = {}
for pos, (id_, hash_) in enumerate(self._storage):
self._id_to_pos[id_] = pos
self._index.setdefault(hash_, []).append(id_)
@_lazy
def indexate(self, data, id_):
if id_ in self._id_to_pos:
raise ValueError('Object with id="%s" already indexed!' % id_)
hash_ = hash(self._selector(data)) # ключ индекса
self._index.setdefault(hash_, []).append(id_)
self._id_to_pos[id_] = self._storage.add((id_, hash_))
@_lazy
def forget(self, id_):
if id_ not in self._id_to_pos:
raise ValueError('Object with id="%s" is not indexed!' % id_)
for l in self._index.values():
try:
l.remove(id_)
except ValueError:
pass
self._storage.remove(self._id_to_pos[id_])
del self._id_to_pos[id_]
@_lazy
def query(self, val):
return self._index.get(hash(val), [])
def test_add():
st = Storage({'a': 1, 'b': 2})
st.add('c', 3)
key = 'c'
val = st.get(key)
assert val == 3, "Value for the key {} is not equal to expected".format(
key)
st.add('a', 5)
key = 'a'
val = st.get(key)
assert val == 1, "Value for the key {} is not equal to expected".format(
key)
st.add(['a', 'b'], 10)
assert (['a', 'b'] in list(st.data.keys())) == False, "Added invalid key"
class StorageTest(unittest.TestCase):
def setUp(self):
self.subject = Storage()
def test_add(self):
self.subject.add(MediaInfo())
self.assertEqual(len(self.subject.items()), 1)
def test_remove(self):
self.subject.add(MediaInfo())
self.subject.remove(MediaInfo())
self.assertEqual(len(self.subject.items()), 0)
def test_load(self):
self.subject.load_storage = lambda : \
[MediaInfo(), MediaInfo()]
self.subject.load()
self.assertEqual(len(self.subject.items()), 2)
def test_save(self):
self.subject.add(MediaInfo())
self.subject.add(MediaInfo())
self.subject.save_storage = lambda(items): \
self.assertEqual(len(items), 2)
self.subject.save()
def test_sanitize(self):
item = MediaInfo(season=None)
Storage.sanitize(item)
self.assertEqual('season' in item, False)
def experiment(args):
save_steps = list(torch.arange(0, int(args.max_steps),
int(args.max_steps) // 10).numpy())
logger = Logger(args.run_name, args)
cuda_is_available = torch.cuda.is_available() and args.cuda
device = torch.device("cuda" if cuda_is_available else "cpu")
args.device = device
torch.manual_seed(args.seed)
if cuda_is_available:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
envs = make_envs(args.env_name, args.num_workers, args.seed)
feudalnet = FeudalNetwork(
num_workers=args.num_workers,
input_dim=envs.single_observation_space.shape,
hidden_dim_manager=args.hidden_dim_manager,
hidden_dim_worker=args.hidden_dim_worker,
n_actions=envs.single_action_space.n,
time_horizon=args.time_horizon,
dilation=args.dilation,
device=device,
mlp=args.mlp,
args=args)
optimizer = torch.optim.RMSprop(feudalnet.parameters(), lr=args.lr,
alpha=0.99, eps=1e-5)
goals, states, masks = feudalnet.init_obj()
x = envs.reset()
step = 0
while step < args.max_steps:
# Detaching LSTMs and goals
feudalnet.repackage_hidden()
goals = [g.detach() for g in goals]
storage = Storage(size=args.num_steps,
keys=['r', 'r_i', 'v_w', 'v_m', 'logp', 'entropy',
's_goal_cos', 'mask', 'ret_w', 'ret_m',
'adv_m', 'adv_w'])
for _ in range(args.num_steps):
action_dist, goals, states, value_m, value_w \
= feudalnet(x, goals, states, masks[-1])
# Take a step, log the info, get the next state
action, logp, entropy = take_action(action_dist)
x, reward, done, info = envs.step(action)
logger.log_episode(info, step)
mask = torch.FloatTensor(1 - done).unsqueeze(-1).to(args.device)
masks.pop(0)
masks.append(mask)
storage.add({
'r': torch.FloatTensor(reward).unsqueeze(-1).to(device),
'r_i': feudalnet.intrinsic_reward(states, goals, masks),
'v_w': value_w,
'v_m': value_m,
'logp': logp.unsqueeze(-1),
'entropy': entropy.unsqueeze(-1),
's_goal_cos': feudalnet.state_goal_cosine(states, goals, masks),
'm': mask
})
step += args.num_workers
with torch.no_grad():
*_, next_v_m, next_v_w = feudalnet(
x, goals, states, mask, save=False)
next_v_m = next_v_m.detach()
next_v_w = next_v_w.detach()
optimizer.zero_grad()
loss, loss_dict = feudal_loss(storage, next_v_m, next_v_w, args)
loss.backward()
torch.nn.utils.clip_grad_norm_(feudalnet.parameters(), args.grad_clip)
optimizer.step()
logger.log_scalars(loss_dict, step)
if len(save_steps) > 0 and step > save_steps[0]:
torch.save({
'model': feudalnet.state_dict(),
'args': args,
'processor_mean': feudalnet.preprocessor.rms.mean,
'optim': optimizer.state_dict()},
f'models/{args.env_name}_{args.run_name}_step={step}.pt')
save_steps.pop(0)
envs.close()
torch.save({
'model': feudalnet.state_dict(),
'args': args,
'processor_mean': feudalnet.preprocessor.rms.mean,
'optim': optimizer.state_dict()},
f'models/{args.env_name}_{args.run_name}_steps={step}.pt')
def test_add():
st = Storage({'a': 1, 'b': 2})
key = 'c'
val = 3
st.add(key, val)
assert st.data[key] == val, "Wrong value added for key {}".format(key)
def main():
password_length = 16
storage = Storage(password_length)
admin, _ = storage.add('admin')
admin.place_secret(read_flag())
UI(storage).interact()
def test_add():
st = Storage()
st.add('a', 1)
assert st.get('a') != 1, "Value for the key {} does not exist"
class TestStorageAPI():
def __init__(self):
db = Database({
"type": 'postgresql',
"host": 'fg-cn-decaf-head1.cs.upb.de',
"port": '5432',
"database": 'decaf_storage',
"user": '******',
"password": '******'
})
#db.drop_all()
db.init_db()
# Configure logging
log_file = LOGFILE
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(log_file)
logger.addHandler(fh)
self.storage = Storage(db, logger=logger)
def test_scenario_insert(self):
# create VNF
code, vnf_id = self.storage.add(Vnf,{
'name': 'name',
'description': 'description',
'path': 'path',
'public': False
})
print('VNF: ', vnf_id)
# create Image
code, image_id = self.storage.add(Image,{
'name': 'name',
'description': 'description',
'location': 'location'
})
print('Image: ', image_id)
# create Flavor
code, flavor_id = self.storage.add(Flavor,{
'name': 'name',
'description': 'description',
'disk': 12,
'ram': 8,
'vcpus': 4
})
print('Flavor: ', flavor_id)
# create VM
code, vm_id = self.storage.add(Vm,{
'name': 'name',
'description': 'description',
'vnf_id': vnf_id,
'flavor_id': flavor_id,
'image_id': image_id,
'image_path': 'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img'
})
print('VM: ', vm_id)
# create Scenario
code, sce_id = self.storage.add(Scenario, {
'name': 'name',
'description': 'description'
})
print('Scenario: ', sce_id)
# add Vnf to Scenario
code, scenario_vnf = self.storage.add(ScenarioVnf, {
'name': 'name',
'description': 'description',
'scenario_id': sce_id,
'vnf_id': vnf_id
})
print('Scenario VNF: ', scenario_vnf)
code, vnfs = self.storage.get(Vnf,
filters={'vnf.uuid': vnf_id},
options=['vms', 'scenario_vnfs.scenario'])
for vnf in vnfs:
print(json.dumps(vnf, cls=StorageJSONEncoder, check_circular=True))
vnf = json.loads(json.dumps(vnf, cls=StorageJSONEncoder, check_circular=True))
print("VMS:\n", json.dumps(vnf['vms']))
class TestStorageAPI():
def __init__(self):
db = Database({
"type": 'postgresql',
"host": 'fg-cn-sandman1.cs.upb.de',
"port": '5432',
"database": 'decaf_storage',
"user": '******',
"password": '******'
})
#db.drop_all()
db.init_db()
# Configure logging
log_file = LOGFILE
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(log_file)
logger.addHandler(fh)
self.storage = Storage(db, logger=logger)
def test_scenario_insert(self):
# create VNF
code, vnf_id = self.storage.add(
Vnf, {
'name': 'name',
'description': 'description',
'path': 'path',
'public': False
})
print('VNF: ', vnf_id)
# create Image
code, image_id = self.storage.add(Image, {
'name': 'name',
'description': 'description',
'location': 'location'
})
print('Image: ', image_id)
# create Flavor
code, flavor_id = self.storage.add(
Flavor, {
'name': 'name',
'description': 'description',
'disk': 12,
'ram': 8,
'vcpus': 4
})
print('Flavor: ', flavor_id)
# create VM
code, vm_id = self.storage.add(
Vm, {
'name':
'name',
'description':
'description',
'vnf_id':
vnf_id,
'flavor_id':
flavor_id,
'image_id':
image_id,
'image_path':
'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img'
})
print('VM: ', vm_id)
# create Scenario
code, sce_id = self.storage.add(Scenario, {
'name': 'name',
'description': 'description'
})
print('Scenario: ', sce_id)
# add Vnf to Scenario
code, scenario_vnf = self.storage.add(
ScenarioVnf, {
'name': 'name',
'description': 'description',
'scenario_id': sce_id,
'vnf_id': vnf_id
})
print('Scenario VNF: ', scenario_vnf)
code, vnfs = self.storage.get(
Vnf,
filters={'vnf.uuid': vnf_id},
options=['vms', 'scenario_vnfs.scenario'])
for vnf in vnfs:
print(json.dumps(vnf, cls=StorageJSONEncoder, check_circular=True))
vnf = json.loads(
json.dumps(vnf, cls=StorageJSONEncoder, check_circular=True))
print("VMS:\n", json.dumps(vnf['vms']))
