def test_make_entity(self):
entity = Entity(name='test01', created_user=self._test_user)
entity.save()
self.assertEqual(entity.name, 'test01')
self.assertEqual(list(entity.attrs.all()), [])
self.assertTrue(entity.is_active)
def test_post_create_with_valid_referral_attr(self):
user = self.admin_login()
entity = Entity(name='test-entity', created_user=user)
entity.save()
params = {
'name': 'hoge',
'note': 'fuga',
'is_toplevel': False,
'attrs': [
{'name': 'a', 'type': str(AttrTypeObj), 'ref_ids': [entity.id],
'is_delete_in_chain': True, 'is_mandatory': False, 'row_index': '1'},
{'name': 'b', 'type': str(AttrTypeArrObj), 'ref_ids': [entity.id],
'is_delete_in_chain': True, 'is_mandatory': False, 'row_index': '2'},
],
}
resp = self.client.post(reverse('entity:do_create'),
json.dumps(params),
'application/json')
self.assertEqual(resp.status_code, 200)
created_entity = Entity.objects.get(name='hoge')
self.assertEqual(created_entity.attrs.count(), 2)
self.assertEqual([x.name for x in created_entity.attrs.all()], ['a', 'b'])
self.assertFalse(any([x.is_mandatory for x in created_entity.attrs.all()]))
self.assertTrue(all([x.is_delete_in_chain for x in created_entity.attrs.all()]))
def post(self, request):
E = ED()
E.u, E.k = -1, 1
E.au, E.no_fa, E.inv_name, E.rename = 2, 3, 4, 5
if not request.session.get('is_login', False):
return '', E.au
u = User.get_via_encoded_id(request.session['uid'])
if u is None:
return '', E.au
kwargs: dict = {'pfid': None}
kwargs.update(json.loads(request.body))
if kwargs.keys() != {'name', 'pfid', 'type'}:
return '', E.k
name, pfid, type = kwargs['name'], kwargs['pfid'], kwargs['type']
fa = Entity.get_via_encoded_id(pfid) if pfid is not None else u.root
if fa is None:
return '', E.no_fa
if fa.sons_dup_name(name):
return '', E.rename
e = Entity(name=name, father=fa, type=type)
try:
e.save()
except:
return '', E.u
record_create(u, e)
return e.encoded_id, 0
def test_index_with_objects(self):
user = self.admin_login()
entity = Entity(name='test-entity', created_user=user)
entity.save()
resp = self.client.get(reverse('entity:index'))
self.assertEqual(resp.status_code, 200)
def test_set_parent(self):
entity = Entity(name='test01', created_user=self._test_user)
entity.save()
attr_base = EntityAttr(name='hoge',
created_user=self._test_user,
parent_entity=entity)
attr_base.save()
self.assertEqual(attr_base.parent_entity, entity)
def test_make_attrbase(self):
entity = Entity(name='test01', created_user=self._test_user)
entity.save()
attr_base = EntityAttr(name='hoge',
created_user=self._test_user,
parent_entity=entity)
attr_base.save()
self.assertEqual(attr_base.name, 'hoge')
self.assertTrue(isinstance(attr_base.type, int))
self.assertEqual(attr_base.referral.count(), 0)
def test_index_with_objects(self):
user = self.admin_login()
entity = Entity(name='test-entity', created_user=user)
entity.save()
resp = self.client.get(reverse('entity:index'))
self.assertEqual(resp.status_code, 200)
root = ElementTree.fromstring(resp.content.decode('utf-8'))
self.assertIsNotNone(root.find('.//tbody/tr'))
self.assertEqual(len(root.findall('.//tbody/tr')), 1)
def test_set_parent(self):
entity = Entity(name="test01", created_user=self._test_user)
entity.save()
attr_base = EntityAttr(
name="hoge",
type=AttrTypeValue["object"],
created_user=self._test_user,
parent_entity=entity,
)
attr_base.save()
self.assertEqual(attr_base.parent_entity, entity)
def test_make_attrbase(self):
entity = Entity(name="test01", created_user=self._test_user)
entity.save()
attr_base = EntityAttr(
name="hoge",
type=AttrTypeValue["object"],
created_user=self._test_user,
parent_entity=entity,
)
attr_base.save()
self.assertEqual(attr_base.name, "hoge")
self.assertTrue(isinstance(attr_base.type, int))
self.assertEqual(attr_base.referral.count(), 0)
def test_import_without_permission_parameter(self):
user = User.objects.create(username='******')
entity = Entity(name='origin_name', created_user=user, is_public=False)
entity.save()
EntityResource.import_data_from_request(
{
'id': entity.id,
'name': 'changed_name',
'note': 'changed_note',
'created_user': entity.created_user.username
}, self._test_user)
self.assertEqual(Entity.objects.count(), 1)
self.assertEqual(Entity.objects.last().name, 'origin_name')
def test_import_with_updating_object(self):
entity = Entity(name='test01',
note='note1',
created_user=self._test_user)
entity.save()
EntityResource.import_data_from_request(
{
'id': entity.id,
'name': 'changed_name',
'note': 'changed_note',
'created_user': entity.created_user.username
}, self._test_user)
self.assertEqual(Entity.objects.count(), 1)
self.assertEqual(Entity.objects.last().name, 'changed_name')
self.assertEqual(Entity.objects.last().note, 'changed_note')
self.assertEqual(Entity.objects.last().created_user, self._test_user)
def test_import_with_updating_object(self):
entity = Entity(name="test01",
note="note1",
created_user=self._test_user)
entity.save()
EntityResource.import_data_from_request(
{
"id": entity.id,
"name": "changed_name",
"note": "changed_note",
"created_user": entity.created_user.username,
},
self._test_user,
)
self.assertEqual(Entity.objects.count(), 1)
self.assertEqual(Entity.objects.last().name, "changed_name")
self.assertEqual(Entity.objects.last().note, "changed_note")
self.assertEqual(Entity.objects.last().created_user, self._test_user)
def createEntity(self):
entity = Entity()
entity.brand_name = "foo"
entity.reference = "68625189"
entity.status = True
entity.save()
class ElasticSearchTest(TestCase):
def setUp(self):
self._user = User(username='******')
self._user.save()
self._entity = Entity(name='entity', created_user=self._user)
self._entity.save()
self._entity_attr = EntityAttr(name='test',
type=AttrTypeStr,
is_mandatory=True,
created_user=self._user,
parent_entity=self._entity)
self._entity_attr.save()
def test_get_regex_pattern(self):
# without escape character
p1 = elasticsearch._get_regex_pattern('keyword')
self.assertEqual(p1, '.*[kK][eE][yY][wW][oO][rR][dD].*')
# with escape character
p2 = elasticsearch._get_regex_pattern('key@@@word')
self.assertEqual(p2, '.*[kK][eE][yY]\\@\\@\\@[wW][oO][rR][dD].*')
# with anchor operators
p2 = elasticsearch._get_regex_pattern('^keyword$')
self.assertEqual(p2, '[kK][eE][yY][wW][oO][rR][dD]')
def test_make_query(self):
query = elasticsearch.make_query(
hint_entity=self._entity,
hint_attrs=[{
'name': 'a1',
'keyword': 'hoge|fu&ga'
}, {
'name': 'a2',
'keyword': ''
}],
entry_name='entry1',
)
self.assertEqual(
query, {
'query': {
'bool': {
'filter': [{
'nested': {
'path': 'entity',
'query': {
'term': {
'entity.id': self._entity.id
}
}
}
}, {
'bool': {
'should': [{
'bool': {
'must': [{
'regexp': {
'name':
'.*[eE][nN][tT][rR][yY]1.*'
}
}]
}
}]
}
}, {
'nested': {
'path': 'attr',
'query': {
'bool': {
'should': [{
'term': {
'attr.name': 'a1'
}
}, {
'term': {
'attr.name': 'a2'
}
}]
}
}
}
}, {
'bool': {
'should': [{
'nested': {
'path': 'attr',
'query': {
'bool': {
'filter': [{
'term': {
'attr.name': 'a1'
}
}, {
'bool': {
'should': [{
'match': {
'attr.value':
'hoge'
}
}, {
'regexp': {
'attr.value':
'.*[hH][oO][gG][eE].*'
}
}]
}
}]
}
}
}
}, {
'bool': {
'filter': [{
'nested': {
'path': 'attr',
'query': {
'bool': {
'filter': [{
'term': {
'attr.name':
'a1'
}
}, {
'bool': {
'should': [{
'match': {
'attr.value':
'fu'
}
}, {
'regexp': {
'attr.value':
'.*[fF][uU].*'
}
}]
}
}]
}
}
}
}, {
'nested': {
'path': 'attr',
'query': {
'bool': {
'filter': [{
'term': {
'attr.name':
'a1'
}
}, {
'bool': {
'should': [{
'match': {
'attr.value':
'ga'
}
}, {
'regexp': {
'attr.value':
'.*[gG][aA].*'
}
}]
}
}]
}
}
}
}]
}
}]
}
}],
'should': []
}
}
})
def test_make_query_for_simple(self):
query = elasticsearch.make_query_for_simple('hoge|fuga&1', None, 0)
self.assertEqual(
query, {
'query': {
'bool': {
'must': [{
'bool': {
'should': [{
'bool': {
'should': [{
'bool': {
'must': [{
'regexp': {
'name':
'.*[hH][oO][gG][eE].*'
}
}]
}
}, {
'bool': {
'must': [{
'regexp': {
'name':
'.*[fF][uU][gG][aA].*'
}
}, {
'regexp': {
'name': '.*1.*'
}
}]
}
}]
}
}, {
'bool': {
'filter': {
'nested': {
'path': 'attr',
'query': {
'bool': {
'should': [{
'bool': {
'filter': [{
'regexp': {
'attr.value':
'.*[hH][oO][gG][eE].*'
}
}]
}
}, {
'bool': {
'filter': [{
'regexp': {
'attr.value':
'.*[fF][uU][gG][aA].*'
}
}, {
'regexp': {
'attr.value':
'.*1.*'
}
}]
}
}]
}
},
'inner_hits': {
'_source': ['attr.name']
}
}
}
}
}]
}
}]
}
},
'_source': ['name'],
'sort': [{
'_score': {
'order': 'desc'
},
'name.keyword': {
'order': 'asc'
}
}],
'from':
0,
})
# set hint_entity_name
query = elasticsearch.make_query_for_simple('hoge', 'fuga', 0)
self.assertEqual(
query['query']['bool']['must'][1], {
'nested': {
'path': 'entity',
'query': {
'term': {
'entity.name': 'fuga'
}
}
}
})
# set offset
query = elasticsearch.make_query_for_simple('hoge', 'fuga', 100)
self.assertEqual(query['from'], 100)
def test_make_search_results(self):
entry = Entry.objects.create(name='test_entry',
schema=self._entity,
created_user=self._user)
attr = Attribute.objects.create(name='test_attr',
schema=self._entity_attr,
created_user=self._user,
parent_entry=entry)
attr_value = AttributeValue.objects.create(value='test_attr_value',
created_user=self._user,
parent_attr=attr)
attr.values.add(attr_value)
attr.save()
res = {
'hits': {
'total':
1,
'hits': [{
'_type': 'entry',
'_id': entry.id,
'_source': {
'entity': {
'id': entry.id,
'name': entry.name
},
'name':
entry.name,
'attr': [{
'name': attr.name,
'type': attr.schema.type,
'key': '',
'value': attr_value.value,
'referral_id': '',
'is_readble': True
}],
'is_readble':
True
},
'sort': [entry.name]
}]
}
}
hint_attrs = [{'name': 'test_attr', 'keyword': '', 'is_readble': True}]
results = elasticsearch.make_search_results(self._user, res,
hint_attrs, 100, False)
self.assertEqual(results['ret_count'], 1)
self.assertEqual(results['ret_values'], [{
'entity': {
'id': self._entity.id,
'name': self._entity.name,
},
'entry': {
'id': entry.id,
'name': entry.name
},
'attrs': {
attr.name: {
'type': attr.schema.type,
'value': attr_value.value,
'is_readble': True
}
},
'is_readble': True
}])
def test_make_search_results_for_simple(self):
entry = Entry.objects.create(name='test_entry',
schema=self._entity,
created_user=self._user)
attr = Attribute.objects.create(name='test',
schema=self._entity_attr,
created_user=self._user,
parent_entry=entry)
attr_value = AttributeValue.objects.create(value='test_attr_value',
created_user=self._user,
parent_attr=attr)
entry.attrs.add(attr)
attr.values.add(attr_value)
attr.save()
res = {
'hits': {
'total':
1,
'hits': [{
'_type': 'entry',
'_id': entry.id,
'_source': {
'name': entry.name
},
'inner_hits': {
'attr': {
'hits': {
'total':
1,
'hits': [{
'_type': 'entry',
'_id': entry.id,
'_source': {
'name': attr.name
},
}]
}
}
}
}]
}
}
results = elasticsearch.make_search_results_for_simple(res)
self.assertEqual(results['ret_count'], 1)
self.assertEqual(results['ret_values'], [{
'id': entry.id,
'name': entry.name,
'attr': attr.name,
}])
class ElasticSearchTest(TestCase):
def setUp(self):
self._user = User(username="******")
self._user.save()
self._entity = Entity(name="entity", created_user=self._user)
self._entity.save()
self._entity_attr = EntityAttr(
name="test",
type=AttrTypeStr,
is_mandatory=True,
created_user=self._user,
parent_entity=self._entity,
)
self._entity_attr.save()
def test_get_regex_pattern(self):
# without escape character
p1 = elasticsearch._get_regex_pattern("keyword")
self.assertEqual(p1, ".*[kK][eE][yY][wW][oO][rR][dD].*")
# with escape character
p2 = elasticsearch._get_regex_pattern("key@@@word")
self.assertEqual(p2, ".*[kK][eE][yY]\\@\\@\\@[wW][oO][rR][dD].*")
# with anchor operators
p2 = elasticsearch._get_regex_pattern("^keyword$")
self.assertEqual(p2, "[kK][eE][yY][wW][oO][rR][dD]")
def test_make_query(self):
query = elasticsearch.make_query(
hint_entity=self._entity,
hint_attrs=[
{"name": "a1", "keyword": "hoge|fu&ga"},
{"name": "a2", "keyword": ""},
],
entry_name="entry1",
)
self.assertEqual(
query,
{
"query": {
"bool": {
"filter": [
{
"nested": {
"path": "entity",
"query": {"term": {"entity.id": self._entity.id}},
}
},
{
"bool": {
"should": [
{
"bool": {
"must": [
{
"regexp": {
"name": ".*[eE][nN][tT][rR][yY]1.*"
}
}
]
}
}
]
}
},
{
"nested": {
"path": "attr",
"query": {
"bool": {
"should": [
{"term": {"attr.name": "a1"}},
{"term": {"attr.name": "a2"}},
]
}
},
}
},
{
"bool": {
"should": [
{
"nested": {
"path": "attr",
"query": {
"bool": {
"filter": [
{"term": {"attr.name": "a1"}},
{
"bool": {
"should": [
{
"match": {
"attr.value": "hoge"
}
},
{
"regexp": {
"attr.value": ".*[hH][oO][gG][eE].*"
}
},
]
}
},
]
}
},
}
},
{
"bool": {
"filter": [
{
"nested": {
"path": "attr",
"query": {
"bool": {
"filter": [
{
"term": {
"attr.name": "a1"
}
},
{
"bool": {
"should": [
{
"match": {
"attr.value": "fu"
}
},
{
"regexp": {
"attr.value": ".*[fF][uU].*"
}
},
]
}
},
]
}
},
}
},
{
"nested": {
"path": "attr",
"query": {
"bool": {
"filter": [
{
"term": {
"attr.name": "a1"
}
},
{
"bool": {
"should": [
{
"match": {
"attr.value": "ga"
}
},
{
"regexp": {
"attr.value": ".*[gG][aA].*"
}
},
]
}
},
]
}
},
}
},
]
}
},
]
}
},
],
"should": [],
}
}
},
)
def test_make_query_for_simple(self):
query = elasticsearch.make_query_for_simple("hoge|fuga&1", None, [], 0)
self.assertEqual(
query,
{
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"bool": {
"must": [
{
"regexp": {
"name": ".*[hH][oO][gG][eE].*"
}
}
]
}
},
{
"bool": {
"must": [
{
"regexp": {
"name": ".*[fF][uU][gG][aA].*"
}
},
{"regexp": {"name": ".*1.*"}},
]
}
},
]
}
},
{
"bool": {
"filter": {
"nested": {
"path": "attr",
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"regexp": {
"attr.value": ".*[hH][oO][gG][eE].*"
}
}
]
}
},
{
"bool": {
"filter": [
{
"regexp": {
"attr.value": ".*[fF][uU][gG][aA].*"
}
},
{
"regexp": {
"attr.value": ".*1.*"
}
},
]
}
},
]
}
},
"inner_hits": {"_source": ["attr.name"]},
}
}
}
},
]
}
}
]
}
},
"_source": ["name", "entity"],
"sort": [{"_score": {"order": "desc"}, "name.keyword": {"order": "asc"}}],
"from": 0,
},
)
# set hint_entity_name
query = elasticsearch.make_query_for_simple("hoge", "fuga", [], 0)
self.assertEqual(
query["query"]["bool"]["must"][1],
{"nested": {"path": "entity", "query": {"term": {"entity.name": "fuga"}}}},
)
# set exclude_entity_names
query = elasticsearch.make_query_for_simple("hoge", None, ["fuga"], 0)
self.assertEqual(
query["query"]["bool"]["must_not"][0],
{"nested": {"path": "entity", "query": {"term": {"entity.name": "fuga"}}}},
)
# set offset
query = elasticsearch.make_query_for_simple("hoge", "fuga", [], 100)
self.assertEqual(query["from"], 100)
def test_make_search_results(self):
entry = Entry.objects.create(
name="test_entry", schema=self._entity, created_user=self._user
)
attr = Attribute.objects.create(
name="test_attr",
schema=self._entity_attr,
created_user=self._user,
parent_entry=entry,
)
attr_value = AttributeValue.objects.create(
value="test_attr_value", created_user=self._user, parent_attr=attr
)
attr.values.add(attr_value)
attr.save()
res = {
"hits": {
"total": 1,
"hits": [
{
"_type": "entry",
"_id": entry.id,
"_source": {
"entity": {"id": entry.id, "name": entry.name},
"name": entry.name,
"attr": [
{
"name": attr.name,
"type": attr.schema.type,
"key": "",
"value": attr_value.value,
"referral_id": "",
"is_readble": True,
}
],
"is_readble": True,
},
"sort": [entry.name],
}
],
}
}
hint_attrs = [{"name": "test_attr", "keyword": "", "is_readble": True}]
results = elasticsearch.make_search_results(self._user, res, hint_attrs, 100, False)
self.assertEqual(results["ret_count"], 1)
self.assertEqual(
results["ret_values"],
[
{
"entity": {
"id": self._entity.id,
"name": self._entity.name,
},
"entry": {"id": entry.id, "name": entry.name},
"attrs": {
attr.name: {
"type": attr.schema.type,
"value": attr_value.value,
"is_readble": True,
}
},
"is_readble": True,
}
],
)
def test_make_search_results_for_simple(self):
entry = Entry.objects.create(
name="test_entry", schema=self._entity, created_user=self._user
)
attr = Attribute.objects.create(
name="test",
schema=self._entity_attr,
created_user=self._user,
parent_entry=entry,
)
attr_value = AttributeValue.objects.create(
value="test_attr_value", created_user=self._user, parent_attr=attr
)
entry.attrs.add(attr)
attr.values.add(attr_value)
attr.save()
res = {
"hits": {
"total": 1,
"hits": [
{
"_type": "entry",
"_id": entry.id,
"_source": {
"name": entry.name,
"entity": {
"id": entry.schema.id,
"name": entry.schema.name,
},
},
"inner_hits": {
"attr": {
"hits": {
"total": 1,
"hits": [
{
"_type": "entry",
"_id": entry.id,
"_source": {"name": attr.name},
}
],
}
}
},
}
],
}
}
results = elasticsearch.make_search_results_for_simple(res)
self.assertEqual(results["ret_count"], 1)
self.assertEqual(
results["ret_values"],
[
{
"id": entry.id,
"name": entry.name,
"schema": {
"id": entry.schema.id,
"name": entry.schema.name,
},
"attr": attr.name,
}
],
)
def process_file(self, filename, user):
trip = Trip()
trip.archive_uri = filename
trip.metadata['original_name'] = os.path.splitext(
os.path.split(filename)[1])[0]
filename_data = re.match(RE_FROM_FILENAME,
trip.metadata['original_name'])
# Open the file
mat = sio.loadmat(filename, squeeze_me=True, struct_as_record=False)
# Find where the recording rate syncs with the recorded GPS time.
sync_idx = _acquire_time_sync_idx(mat['GMT_SEC'].data)
if sync_idx is None:
sync_idx = 0 # But load the data.
trip.metadata['time_sync'] = False
else:
trip.metadata['time_sync'] = True
try:
sync_time = datetime(year=mat['DATE_YEAR'].data[sync_idx],
month=mat['DATE_MONTH'].data[sync_idx],
day=mat['DATE_DAY'].data[sync_idx],
hour=mat['GMT_HOUR'].data[sync_idx],
minute=mat['GMT_MINUTE'].data[sync_idx],
second=mat['GMT_SEC'].data[sync_idx],
tzinfo=timezone.utc)
trip.metadata['time_source'] = 'GMT'
except (KeyError, ValueError):
# Not all flights have a StartTimeVec.
if 'StartTimeVec' in mat:
t = mat['StartTimeVec']
sync_time = datetime(year=t[0],
month=t[1],
day=t[2],
hour=t[3],
minute=t[4],
second=t[5],
tzinfo=timezone.utc)
trip.metadata['time_source'] = 'StartTimeVec'
else:
if filename_data:
sync_time = datetime(
year=int(filename_data.group(year_idx)),
month=int(filename_data.group(mon_idx)),
day=int(filename_data.group(day_idx)),
hour=int(filename_data.group(hour_idx)),
minute=int(filename_data.group(min_idx)),
tzinfo=timezone.utc)
trip.metadata['time_source'] = 'Filename'
else:
raise
# In microseconds
sync_offset = (1 / mat['GMT_SEC'].Rate) * SEC_TO_MICROSEC * sync_idx
start_time = sync_time - timedelta(microseconds=sync_offset)
end_time = start_time
# Organize the data by rate and create a time index for each rate.
d = {} # Time series data
m = {} # Meta data
for p in mat:
param = mat[p]
if isinstance(param, sio.matlab.mio5_params.mat_struct):
if param.Rate not in d:
d[param.Rate] = {
'p': {},
't':
pd.date_range(start_time,
periods=len(param.data),
freq='{}U'.format(
int(SEC_TO_MICROSEC *
(1.0 / param.Rate))),
tz=timezone.utc,
name='timestamp'),
'm': {},
}
if d[param.Rate]['t'][-1] > end_time:
end_time = d[param.Rate]['t'][-1].to_datetime()
d[param.Rate]['p'][p] = param.data
d[param.Rate]['m'][p] = {
'rate': param.Rate,
'units': param.Units if len(param.Units) > 0 else None,
'alpha': param.Alpha,
'description': param.Description
}
else:
if isinstance(param, bytes):
param = param.decode('utf-8')
m[p] = param
# Create a dataframe per data rate.
common_id = filename_data.group(id_idx) if filename_data else None
params = {}
for k, v in d.items():
rate_params = v
params[k] = {
'df': pd.DataFrame(rate_params['p'], index=rate_params['t']),
'meta': rate_params['m']
}
if 'ACID' in params[k]['df'].columns:
common_id = params[k]['df']['ACID'].value_counts().index[0]
# Retrieve or create the entity for this trajectory.
try:
entity = Entity.objects.get(common_id=common_id, owner=user)
except Entity.DoesNotExist:
entity = Entity()
entity.common_id = common_id
entity.physical_id = common_id # Not really what is wanted.
entity.owner = user
entity.loader_config = {
'module': __name__,
}
entity.save()
trip.entity = entity
trip.metadata.update(m)
trip.start_datetime = start_time
trip.duration = end_time - start_time
pos = params[1]['df'][['LONP', 'LATP']].join(params[4]['df'][['ALT']],
how='inner')
# All positional values valid and within the USA for the data
# Occasionally the data goes to 0.351562 and/or zero, especially at
# at the end of the flight.
f = ~(((pos['LONP'] >= -0.4) & (pos['LONP'] <= 0.4)) |
((pos['LATP'] >= -0.4) & (pos['LATP'] <= 0.4)))
pos = pos[f]
positions = [
'{} {} {}'.format(p[1].LONP, p[1].LATP, p[1].ALT)
for p in pos.iterrows() if not (p[1].LONP == 0 or p[1].LATP == 0)
]
trip.geometry = GEOSGeometry('SRID=4326;LINESTRING Z ({})'.format(
','.join(positions)))
trip.save()
for k, v in params.items():
tripdata = TripData()
tripdata.dataframe = v['df']
tripdata.metadata = v['meta']
tripdata.trip = trip
tripdata.save()
class ElasticSearchTest(TestCase):
def setUp(self):
self._user = User(username='******')
self._user.save()
self._entity = Entity(name='entity', created_user=self._user)
self._entity.save()
self._entity_attr = EntityAttr(name='test',
type=AttrTypeStr,
is_mandatory=True,
created_user=self._user,
parent_entity=self._entity)
self._entity_attr.save()
def test_get_regex_pattern(self):
# without escape character
p1 = elasticsearch._get_regex_pattern('keyword')
self.assertEqual(p1, '.*[kK][eE][yY][wW][oO][rR][dD].*')
# with escape character
p2 = elasticsearch._get_regex_pattern('key@@@word')
self.assertEqual(p2, '.*[kK][eE][yY]\\@\\@\\@[wW][oO][rR][dD].*')
# with anchor operators
p2 = elasticsearch._get_regex_pattern('^keyword$')
self.assertEqual(p2, '.*^?[kK][eE][yY][wW][oO][rR][dD]$?.*')
def test_make_key_for_each_block_of_keywords(self):
key1 = elasticsearch._make_key_for_each_block_of_keywords(
{'name': 'name'}, 'keyword', True)
self.assertEqual(key1, 'keyword')
key2 = elasticsearch._make_key_for_each_block_of_keywords(
{'name': 'name'}, 'keyword', False)
self.assertEqual(key2, 'keyword_name')
def test_is_matched_keyword(self):
# if it has the same value with a hint
self.assertTrue(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': 'keyword', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': 'keyword'}]
))
# if a hint has ^ and/or $, it matches with the keyword as a regexp
self.assertTrue(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': 'keyword', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': '^keyword'}]
))
self.assertFalse(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': '111keyword', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': '^keyword'}]
))
self.assertTrue(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': 'keyword', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': 'keyword$'}]
))
self.assertFalse(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': 'keyword111', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': 'keyword$'}]
))
# if a hint is blank
self.assertTrue(elasticsearch._is_matched_entry(
attrs=[{'name': 'attr', 'value': 'keyword', 'type': AttrTypeStr.TYPE}],
hint_attrs=[{'name': 'attr', 'keyword': ''}]
))
def test_make_query(self):
query = elasticsearch.make_query(
hint_entity_ids=['1'],
hint_attrs=[{'name': 'a1', 'keyword': 'a'}, {'name': 'a2', 'keyword': ''}],
hint_attr_value=None,
entry_name='entry1',
or_match=False,
)
self.assertEqual(query, {
'query': {
'bool': {
'filter': [
{
'nested': {
'path': 'entity',
'query': {
'bool': {
'should': [
{'term': {'entity.id': 1}}
]
}
}
}
},
{
'bool': {
'should': [
{
'bool': {
'filter': [
{'regexp': {'name': '.*[eE][nN][tT][rR][yY]1.*'}}
]
}
}
]
}
},
{
'nested': {
'path': 'attr',
'query': {
'bool': {
'should': [
{'term': {'attr.name': 'a1'}},
{'term': {'attr.name': 'a2'}}
]
}
}
}
},
{
'nested': {
'path': 'attr',
'query': {
'bool': {
'filter': [
{'term': {'attr.name': 'a1'}},
{
'bool': {
'should': [
{'match': {'attr.value': 'a'}},
{'regexp': {'attr.value': '.*[aA].*'}}
]
}
}
]
}
}
}
}
],
'should': []
}
}
})
def test_make_search_results(self):
entry = Entry.objects.create(name='test_entry',
schema=self._entity,
created_user=self._user)
attr = Attribute.objects.create(name='test_attr',
schema=self._entity_attr,
created_user=self._user,
parent_entry=entry)
attr_value = AttributeValue.objects.create(value='test_attr_value',
created_user=self._user,
parent_attr=attr)
attr.values.add(attr_value)
attr.save()
res = {
'hits': {
'total': 1,
'hits': [
{
'_type': 'entry',
'_id': entry.id,
'_source': {
'entity': {
'id': entry.id,
'name': entry.name
},
'name': entry.name,
'attr': [
{
'name': attr.name,
'type': attr.schema.type,
'key': '',
'value': attr_value.value,
'referral_id': ''
}
]
},
'sort': [entry.name]
}
]
}
}
hint_attrs = [{'name': 'test_attr', 'keyword': ''}]
results = elasticsearch.make_search_results(res, hint_attrs, 100, False)
self.assertEqual(results['ret_count'], 1)
self.assertEqual(results['ret_values'], [
{
'entity': {
'id': self._entity.id,
'name': self._entity.name,
},
'entry': {
'id': entry.id,
'name': entry.name
},
'attrs': {
attr.name:
{
'type': attr.schema.type,
'value': attr_value.value,
}
},
}
])
def create_entity(owner, common_id, metadata=None):
metadata = metadata if metadata is not None else {}
entity = Entity(common_id=common_id, owner=owner, metadata=metadata)
entity.save()
return entity