def test_ix1(self):
root = '/tmp/dix/r1'
if os.path.exists(root):
shutil.rmtree(root)
# import pdb; pdb.set_trace()
ix = dix.IndexId(root, 'name0')
ix.add('one')
ix.add('one')
ix.add('two')
id0 = ix.add('two')
ix.flush()
ix2 = dix.IndexId(root, 'name0')
id1 = ix2.add('two')
print(id0, id1)
ix2.flush()
self.assertEqual(id0, id1)
ix = dix.IndexMap(root, 'map0')
ix.add('one', 'two')
ix.add('one', 'two')
ix.add('two', 'three')
id0 = ix.get('two')
ix.flush()
ix2 = dix.IndexMap(root, 'map0')
id1 = ix2.get('two')
print(id0, id1)
ix2.flush()
self.assertEqual(id0, id1)
ix = dix.IndexList(root, 'list0')
ix.add('one', 'two')
ix.add('one', 'two1')
ix.add('two', 'three')
id0 = ix.get('two')
ix.flush()
ix2 = dix.IndexList(root, 'list0')
id1 = ix2.get('two')
print(id0, id1)
ix2.flush()
self.assertEqual(id0, id1)
id0 = ix.get('one')
id1 = ix2.get('one')
print(id0, id1)
self.assertEqual(id0, id1)
with self.assertRaises(ValueError):
ix0 = dix.IndexList(root, 'list1', sorted=False, unique=True)
ix0.add('one', 'two')
ix0.add('one', 'two')
def get(name, indexType):
trgtRoot = econst.getenv('trgtRoot')
indexRoot = os.path.join(trgtRoot, 'index')
if indexType == 'IndexMap':
return dix.IndexMap(indexRoot, name=name)
if indexType == 'IndexSet':
return dix.IndexSet(indexRoot, name=name)
if indexType == 'IndexId':
return dix.IndexId(indexRoot, name=name)
if indexType == 'IndexList':
return dix.IndexList(indexRoot, name=name)
raise ValueError('Unkown type=%s' % indexType)
def driverTmt(srcRoot, trgtRoot):
indexRoot = os.path.join(trgtRoot, 'index')
dateEventIdSet = dix.IndexSet(indexRoot,
name='dateEventIdSet',
unique=True,
keyType=int)
processedFileList = dix.IndexList(indexRoot,
name='tmt_processedFileList',
unique=True)
fencerId2EventIdList = dix.IndexList(indexRoot,
name='fencerId2eventId',
unique=True,
keyType=int)
eventIdNameMap = dix.IndexMap(indexRoot,
name='eventIdNameMap',
keyType=int)
fencerIdNameMap = dix.IndexMap(indexRoot,
name='fencerIdNameMap',
keyType=int)
engFencerIdfencerId = dix.IndexId(
indexRoot,
name='engFencerIdfencerId',
)
fencerNameEventIdMap = dix.IndexMap(indexRoot,
name='fencerNameEventIdMap',
keyType=str)
eventIdTmtFileSet = dix.IndexSet(indexRoot,
name='eventIdTmtFileSet',
keyType=int)
tmtRoot = os.path.join(srcRoot, 'tmt')
paths = pathlib.Path(tmtRoot)
for path in paths.iterdir():
path = str(path)
if processedFileList.exists(name='tmt', val=path):
print('driverTmt', 'already processed', path)
continue
print('driverTmt', 'processing', path)
with open(path) as fd:
jobj = json.load(fd)
tmt = fileutil.cleanUp(obj=jobj, fctr=fileutil.StrFunctor())
date = strutil.normDate3(tmt['date'])
ts = tmt_core.tmt2ser(tmt)
for event in ts:
fencerId = event['fencer_id']
eventId = event['event_id']
fencerName = event['fencer_name']
eventName = event['event_name']
if not dateEventIdSet.exists(date, eventId):
dateEventIdSet.add(date, eventId)
if fencerName == econst.SpecialNames.FencerExcluded:
print('driverTmt: skipping %s', str(event))
continue
try:
fencerIdNameMap.add(name=fencerId, target=fencerName)
fencerId2EventIdList.add(name=fencerId, val=eventId)
eventIdNameMap.add(name=eventId, target=eventName)
engId = engFencerIdfencerId.add(fencerId)
fencerNameEventIdMap.add((fencerName, eventId), engId)
eventIdTmtFileSet.add(eventId, path)
except KeyError as e:
raise BadData({'path': path})
processedFileList.add(name='tmt', val=path)
def driverDelim(srcRoot, trgtRoot):
indexRoot = os.path.join(trgtRoot, 'index')
processedFileList = dix.IndexList(indexRoot,
name='de_processedFileList',
unique=True)
fencerNameEventIdMap = dix.IndexMap(indexRoot, name='fencerNameEventIdMap')
engFencerIdfencerId = dix.IndexId(indexRoot, name='engFencerIdfencerId')
badFencerengFencerIdMap = dix.IndexMap(indexRoot,
name='badFencerengFencerIdMap',
unique=True)
engIdEventIdSet = dix.IndexSet(indexRoot,
name='engIdEventIdSet',
unique=True,
keyType=int)
eventIdDelemMap = dix.IndexMap(indexRoot,
name='eventIdDelemMap',
unique=True,
hasinvet=False,
keyType=int)
delimRoot = os.path.join(srcRoot, 'delim')
paths = pathlib.Path(delimRoot)
for path in paths.iterdir():
path = str(path)
if processedFileList.exists(name='delim', val=path):
print('driverDelim', 'already processed', path)
continue
print('driverDelim', 'processing', path)
with open(path) as fd:
jobj = json.load(fd)
obj = fileutil.cleanUp(obj=jobj, fctr=fileutil.StrFunctor())
eventId = int(obj['event_id'])
des = de_core.de2ser(de=obj['rounds'])
mat = {'fencer1': [], 'fencer2': [], 'score1': [], 'score2': []}
for (fencer1, fencer2, score1, score2) in des:
try:
key1 = (fencer1, eventId)
key2 = (fencer2, eventId)
updateFencerNameEventId(
keys=[key1, key2],
fencerNameEventIdMap=fencerNameEventIdMap,
engFencerIdfencerId=engFencerIdfencerId,
badFencerengFencerIdMap=badFencerengFencerIdMap)
fencer1 = fencerNameEventIdMap.get(key1)
fencer2 = fencerNameEventIdMap.get(key2)
if not engIdEventIdSet.exists(fencer1, eventId):
engIdEventIdSet.add(fencer1, eventId)
if not engIdEventIdSet.exists(fencer2, eventId):
engIdEventIdSet.add(fencer2, eventId)
except KeyError as e:
raise BadData({'path': path})
# print (fencer1, fencer2, score1, score2)
mat['fencer1'].append(fencer1)
mat['fencer2'].append(fencer2)
mat['score1'].append(score1)
mat['score2'].append(score2)
df = pandas.DataFrame(mat)
if eventIdDelemMap.exists(eventId):
dfp = eventIdDelemMap.get(eventId)
df = dfp.append(df)
df = df.reset_index()
eventIdDelemMap.reset(eventId, df)
else:
eventIdDelemMap.add(eventId, df)
processedFileList.add(name='delim', val=path)