def impl_test_apply(self, data_name, test_name):
work_dir = posixpath.join(work_root, test_name)
if posixpath.isdir(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
test_data = posixpath.join(test_data_root, data_name)
music1_dir = posixpath.join(work_dir, 'music1')
result1_dir = posixpath.join(work_dir, 'result1')
music2_dir = posixpath.join(work_dir, 'music2')
result2_dir = posixpath.join(work_dir, 'result2')
create_collection(posixpath.join(test_data, 'image1.json'), result1_dir, music1_dir, 'data.json')
create_collection(posixpath.join(test_data, 'image2.json'), result2_dir, music2_dir, 'data.json')
snapshots1 = Snapshots(result1_dir)
collection1 = Collection(snapshots1, music1_dir)
snapshots2 = Snapshots(result2_dir)
Collection(snapshots1, music2_dir)
collection1.apply_snapshot(snapshots2.load('data.json'))
collection1.remove_unused_pictures()
snapshots1.save(collection1.state, 'data.json')
self.folders_equal(music1_dir, music2_dir)
self.folders_equal(posixpath.join(result1_dir, 'pictures'), posixpath.join(result2_dir, 'pictures'))
self.snapshots_equal(posixpath.join(result1_dir, 'data.json'), posixpath.join(result2_dir, 'data.json'))
def run_pygame() -> None:
"""run game"""
# variables for game
variables = Variables()
# create game window
screen = pygame.display.set_mode(
variables.display.dimensions.width_height())
# create a ship for player
ship = Ship(screen)
# create a bullet group
bullets = Collection(screen)
# create a alien bullets group
aliens = Collection(screen)
# create an alient fleet
game.create_fleet(variables, screen, aliens)
# main logic
while True:
# check for new events
game.check_events(ship, bullets)
# update objects
game.update_objects(ship, aliens, bullets)
# update display
game.update_screen(screen, ship, aliens, bullets)
def __init__(self,docsURLs=[],docsTxts=[],docsColl=[]):
if docsColl:
self.coll = docsColl
elif docsURLs:
if docsTxts:
self.coll = Collection(docsURLs,docsTxts)
else:
self.coll = Collection(docsURLs)
def main():
print 'Creating a List-Implemented Collection...'
collection = Collection(ListCollectionImp)
example(collection)
print 'Creating a Dict-Implemented Collection...'
collection = Collection(DictCollectionImp)
example(collection)
def setUp(self):
# Get the sample JSON data
self.data = requests.get(
"http://samples.openweathermap.org/data/2.5/weather?zip=94040,us&appid=b6907d289e10d714a6e88b30761fae22"
).json()
self.coll = Collection(
getlist=["weather.main", "main.temp", "clouds.all", "doesntExist"])
self.dcoll = Collection()
def test_that_two_collections_concatenation_contains_the_elements_of_the_two_collections(a_list, another_list):
collection = Collection(a_list)
another_collection = Collection(another_list)
new_collection = collection.concat(another_collection)
for element in a_list:
new_collection = new_collection.remove(element)
for element in another_list:
new_collection = new_collection.remove(element)
assert len(new_collection) == 0
def __init__(self, args):
# paths to data files
self.train_path = args.wnut + "/" + const.TRAIN_FILE
self.dev_path = args.wnut + "/" + const.DEV_FILE
self.test_path = args.wnut + "/" + const.TEST_FILE
self.train_path_orth = args.wnut + "/" + const.ORTH_TRAIN_FILE
self.dev_path_orth = args.wnut + "/" + const.ORTH_DEV_FILE
self.test_path_orth = args.wnut + "/" + const.ORTH_TEST_FILE
self.emb_path = args.emb_path
# bells and whistles of the model
self.fine_tune = args.fine_tune
self.oov = args.oov
self.emb_to_use = args.emb
# hyperparameters
self.orth_word_emb_dim = args.orth_word_emb_dim
self.word_emb_dim = 0
# objects to store collection of words, orthographic words, and labels
self.word_collection = Collection("word")
self.orth_word_collection = Collection("orth_word")
self.label_collection = Collection("label")
self.char_collection = Collection("char")
self.orth_char_collection = Collection("orth_char")
self.logger = logger
# vars to store the data from the files
self.words = {}
self.orth_words = {}
self.labels = {}
self.word_indices = {}
self.orth_word_indices = {}
self.label_indices = {}
self.char_indices = {}
self.orth_char_indices = {}
# store all the embeddings in this dict
self.emb = {}
self.emb_table = {}
# lengths of the sentences
self.sequence_lengths = {}
self.word_lengths = {}
self.max_lengths = {}
self.max_sentence_length = 0
self.max_word_length = 0
self.num_labels = 0
# for final train, dev and test data
self.train_data = {}
self.dev_data = {}
self.test_data = {}
def main():
c = Collection(1, 2, 3)
assert c == c.map(lambda x: x)
assert Collection(2) == c.filter(lambda x: x % 2 == 0)
assert 6 == c.reduce(0, lambda x, y: x + y)
assert Collection(1, 2) == c.take(2)
assert Collection(3) == c.drop(2)
assert Collection(0, 0, 0) == Collection(0, 0, 0,
1).take_while(lambda x: x == 0)
assert Collection(1) == Collection(0, 0, 0, 1).drop_while(lambda x: x == 0)
assert Collection(1).search(1)
print('tests passed !')
def test_list():
l = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']
L = Collection(l)
assert list(L) == l
assert list(m for m in L) == l
assert L.values() == l
assert L.keys() == list(range(len(l)))
assert L.items() == [(k, v) for k, v in zip(range(len(l)), l)]
assert len(L) == len(l)
assert 'alpha' in L
assert 'beta' in L
assert 'gamma' in L
assert 'delta' in L
assert 'epsilon' in L
assert 'upsilon' not in L
assert L[0] == 'alpha'
assert L[1] == 'beta'
assert L[2] == 'gamma'
assert L[3] == 'delta'
assert L[4] == 'epsilon'
assert L[-1] == 'epsilon'
assert L[-2] == 'delta'
assert L[-3] == 'gamma'
assert L[-4] == 'beta'
assert L[-5] == 'alpha'
with pytest.raises(IndexError) as exception:
L[5]
assert str(exception.value) == 'list index out of range'
def test_get_random_empty(self):
"""get_random should return None if the collection is empty."""
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
db = Collection(file_name)
self.assertIsNone(db.get_random())
os.remove(file_name)
def __init__(self,
collection=None,
records=None,
dm=None,
metric='geo',
**kwargs):
""" Initialisation:
A fully-formed Collection object can be given, or a list of records.
If a list of records is provided a Collection object is built from it,
with **kwargs being passed to the constructor.
Additionally a distance matrix can be passed, or else one is constructed
using the metric specified (default=geodesic)
"""
if records:
self.collection = Collection(records, **kwargs)
else:
self.collection = collection
if dm is not None:
self.dm = dm
else:
self.dm = self.calc_dm(metric)
self._warnings()
def test_insert_one_item(self):
C = Collection(vtype, utype)
C.append(vertices, indices, uniforms)
C.insert(0, vertices, indices, uniforms)
assert len(C) == 2
assert np.allclose(C[0].indices, indices)
assert np.allclose(C[1].indices, 4 + indices)
def test_append_several_item_2(self):
C = Collection(vtype, utype)
C.append(np.zeros(40, dtype=vtype),
np.zeros(10, dtype=itype),
itemsize=(4, 1))
for i in xrange(10):
assert np.allclose(C[i].indices, 4 * i)
def test_get_from_phrase(self):
"""get_from_phrase should return the comic that is most similar to the phrase."""
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
db = Collection(file_name)
db.add_to_blacklist('or')
db.add_comic({
"number": 1,
"img_url": "https://www.google.com",
"title": "A Title",
"alt": "Some alt-text",
"transcript": "or Hoi hoi or or or or or or"
})
db.add_comic({
"number": 2,
"img_url": "https://www.google.com",
"title": "A Title",
"alt": "Some alt-text",
"transcript": "Hoi hoi mijn vrienden. Zei ik hoi? Hoi"
})
comic = db.get_from_phrase(["The", "hoi", "one"])
self.assertIsNotNone(comic)
self.assertIs(comic["number"], 2)
os.remove(file_name)
def test_generator():
L = Collection(range(5))
l = list(range(5))
assert list(L) == l
assert list(m for m in L) == l
assert L.values() == l
assert L.keys() == list(range(len(l)))
assert L.items() == [(k, v) for k, v in zip(range(len(l)), l)]
assert len(L) == len(l)
assert 0 in L
assert 1 in L
assert 2 in L
assert 3 in L
assert 4 in L
assert 5 not in L
assert L[0] == 0
assert L[1] == 1
assert L[2] == 2
assert L[3] == 3
assert L[4] == 4
assert L[-1] == 4
assert L[-2] == 3
assert L[-3] == 2
assert L[-4] == 1
assert L[-5] == 0
with pytest.raises(IndexError) as exception:
L[5]
assert str(exception.value) == 'range object index out of range'
def test_text_splitter_dict():
t = transfers = '''
# January
bob = $1,000 # from Bob
ted = -$500 # to Ted
# February
carol = $750 # from Carol
alice = -$1250 # to Alice
'''
T = Collection(t, split_lines, cull=True, strip=True, comment='#', sep='=')
d = dict(bob='$1,000', ted='-$500', carol='$750', alice='-$1250')
assert list(m for m in T) == list(d.values())
assert T.values() == list(d.values())
assert T.keys() == list(d.keys())
assert T.items() == list((k, v) for k, v in d.items())
assert len(T) == len(d)
assert '$1,000' in T
assert '-$500' in T
assert '$750' in T
assert '-$1250' in T
assert 'upsilon' not in T
assert T['bob'] == '$1,000'
assert T['ted'] == '-$500'
assert T['carol'] == '$750'
assert T['alice'] == '-$1250'
with pytest.raises(KeyError) as exception:
T['jeff']
assert str(exception.value) == "'jeff'"
def test_get_latest(self):
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
db = Collection(file_name)
latest = 10
for num in range(1, latest + 1):
db.add_comic({
"number": num,
"img_url": "https://www.google.com",
"title": "A Title",
"alt": "Some alt-text",
"transcript": "Hoi hoi"
})
db.add_comic({
"number": 0,
"img_url": "https://www.google.com",
"title": "A Title",
"alt": "Some alt-text",
"transcript": "Hoi hoi"
})
comic = db.get_latest()
self.assertEqual(comic["number"], latest)
os.remove(file_name)
def test_add_comic(self):
"""Adding valid comic structures should not throw an exception."""
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
word = 'woosh'
comic_num = 4
db = Collection(file_name)
db.add_comic({
"number": comic_num,
"img_url": "https://www.google.com",
"title": "A Title",
"alt": "Some alt-text",
"transcript": "{} {}".format(word, word)
})
# Make sure word weights are incremented.
con = sqlite3.connect(file_name)
cursor = con.cursor()
cursor.execute(
'SELECT weight FROM word_weights \
WHERE word_id=(SELECT id FROM words WHERE word=?) \
AND comic_id=?', (word, comic_num))
self.assertEqual(cursor.fetchone()[0], 2)
con.close()
os.remove(file_name)
def create_and_populte_quijote(redis_conn):
"""Load the full text of el quijote and store as independent lines"""
# %%
with open("/home/teo/_data/red-search/quijote.txt",
"rt",
encoding="iso-8859-1") as f_in:
lines = f_in.readlines()
print(len(lines))
docs = [{
"text": line,
"id": i,
"par_num": i
} for i, line in enumerate(lines)]
# %%
cfg = CollectionConfig(name='qxt',
id_fld='id',
text_flds=['text'],
facet_flds=[],
number_flds=['par_num'],
stop_words="el la los las de a es".split(" "))
col = Collection(redis_conn).configure(cfg)
# %%
clear_collection(col)
# %%
com.timeit(lambda: coll.index_documents(col, docs, batch_size=100))
def partitionCollection(K):
"""Partition the collection according to node adjacency.
The actor numbers will be connected to a collection of property numbers,
e.g. 0 [1 [4,12] 2 [6,20]], where 0 is the actor number, 1 and 2 are the
property numbers and 4, 12, 6 and 20 are the element numbers.
"""
sel = getCollection(K)
if len(sel) == 0:
print("Nothing to partition!")
return
if K.obj_type == 'actor':
actor_numbers = K.get(-1,[])
K.clear()
for i in actor_numbers:
K.add(range(sel[int(i)].nelems()),i)
prop = 1
j = 0
for i in K.keys():
p = sel[j].partitionByConnection() + prop
print("Actor %s partitioned in %s parts" % (i,p.max()-p.min()+1))
C = Collection()
C.set(transpose(asarray([p,K[i]])))
K[i] = C
prop += p.max()-p.min()+1
j += 1
K.setType('partition')
def __init__(self, datacube=None, collection=None):
"""Set up the ingester object.
datacube: A datacube instance (which has a database connection and
tile_type and band dictionaries). If this is None the Ingeseter
will create its own datacube instance using the arguments
returned by self.parse_args().
collection: The datacube collection which will accept the ingest.
if this is None the Ingeseter will set up its own collection
using self.datacube.
"""
self.args = self.parse_args()
if self.args.debug:
# Set DEBUG level on the root logger
logging.getLogger().setLevel(logging.DEBUG)
if datacube is None:
self.datacube = IngesterDataCube(self.args)
else:
self.datacube = datacube
self.agdc_root = self.datacube.agdc_root
if collection is None:
self.collection = Collection(self.datacube)
else:
self.collection = collection
def read_data():
# Use test data
test_collection = Collection("test_data/test_collection")
posting_lists = []
for _, pl in enumerate(test_collection):
posting_lists.append(np.unique(np.array(pl[0], dtype=np.uint8)))
return posting_lists
def test_formatting():
l = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']
L = Collection(l)
assert L.render() == 'alpha beta gamma delta epsilon'
L.sep = ', '
assert L.render() == 'alpha, beta, gamma, delta, epsilon'
L.fmt = '<{v}>'
assert L.render() == '<alpha>, <beta>, <gamma>, <delta>, <epsilon>'
L.fmt = lambda k, v: f'{k}: {v}'
assert L.render() == '0: alpha, 1: beta, 2: gamma, 3: delta, 4: epsilon'
d = dict(bob='239-8402',
ted='371-8567',
carol='891-5810',
alice='552-2219')
D = Collection(d)
expected = 'bob: 239-8402, ted: 371-8567, carol: 891-5810, alice: 552-2219'
assert D.render('{k}: {v}', ', ') == expected
class Info:
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
C = Collection([
Info(name='bob', email='*****@*****.**'),
Info(name='ted', email='*****@*****.**'),
Info(name='carol', email='*****@*****.**'),
Info(name='alice', email='*****@*****.**'),
])
result1 = 'Email:\n {}'.format(C.render('{v.name}: {v.email}',
'\n '))
result2 = 'Email:\n {}'.format(
C.render(lambda k, v: f'{v.name}: {v.email}', '\n '))
result3 = 'Email:\n {:{{v.name}}: {{v.email}}|\n }'.format(C)
expected1 = dedent('''
Email:
bob: [email protected]
ted: [email protected]
carol: [email protected]
alice: [email protected]
''').strip()
assert result1 == expected1
assert result2 == expected1
assert result3 == expected1
result4 = '{:{{v.name}}={{v.email}}}'.format(C)
expected4 = '[email protected] [email protected] [email protected] [email protected]'
assert result4 == expected4
def __getattr__(self, key):
if key in self.connection._registered_documents:
document = self.connection._registered_documents[key]
return getattr(self[document.__collection__], key)
else:
if not key in self._collections:
self._collections[key] = Collection(self, key)
return self._collections[key]
def test_add_to_blacklist_new(self):
"""Adding a new word as blacklisted should not throw an exception."""
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
db = Collection(file_name)
db.add_to_blacklist('and')
os.remove(file_name)
def test_construct_new_db(self):
"""The constructor should create a new database if one does not exist."""
file_name = "./{}.sqlite".format(str(uuid.uuid4()))
Collection(file_name)
self.assertTrue(os.path.isfile(file_name))
os.remove(file_name)
def scan_folder(file: os.path) -> Collection:
"""
Checks if the folder is a collection or a manga.
:param file:
:return: Collection
"""
if is_supported_filename(file):
return Collection(file, manage.get_last_index(file))
def simulate(self, n, np, mobility, collections, do_collisions):
parameters = {
'n': n,
'n0': np,
'mobility': mobility,
'do_collisions': do_collisions
}
self.collections = []
# Different cases for number of collections
# Coordinates is an array of relative coordinates on the form [x1, x2, y1, y2]
# Ratio is the ratio of the population each collection gets (based on Iceland's population)
if collections == 4:
coordinates = [[0, 0.5, 0, 0.5], [0, 0.5, 0.5, 1],
[0.5, 1, 0, 0.5], [0.5, 1, 0.5, 1]]
ratios = [0.152, 0.705, 0.037, 0.106]
for coordinate, ratio in zip(coordinates, ratios):
self.collections.append(
Collection(
self, {
'n': int(n * ratio),
'n0': int(np * ratio),
'mobility': mobility,
'do_collisions': do_collisions
}, coordinate))
elif collections == 2:
coordinates = [[0, 0.5, 0, 1], [0.5, 1, 0, 1]]
ratios = [0.64, 0.36]
for coordinate, ratio in zip(coordinates, ratios):
self.collections.append(
Collection(
self, {
'n': int(n * ratio),
'n0': int(np * ratio),
'mobility': mobility,
'do_collisions': do_collisions
}, coordinate))
else:
self.collections = [Collection(self, parameters)]
self.collections_count = len(self.collections)
self.stop_simulation = False
self.loop()
def __getattr__(self, name):
"""Get a collection of this database by name.
Raises InvalidName if an invalid collection name is used.
:Parameters:
- `name`: the name of the collection to get
"""
return Collection(self, name)
def collection(self, name):
if isinstance(name, basestring):
index = self._collections.names[:].index(name)
id = self._collections.ids[index]
else:
id = name
collection = self._load_descriptor(self.protobufs.CollectionDescriptor,
'pydb/collection_{}.bin'.format(id))
return Collection(self, name, collection)
