def __init__(self, conf, trainingSet=None, testSet=None, fold='[1]'):
self.config = conf
self.isSaveModel = False
self.isLoadModel = False
self.isOutput = True
self.data = Record(self.config, trainingSet, testSet)
self.foldInfo = fold
self.evalConfig = LineConfig(self.config['evaluation.setup'])
if self.evalConfig.contains('-target'):
self.recType = self.evalConfig['-target']
else:
self.recType = 'track'
if LineConfig(self.config['evaluation.setup']).contains('-cold'):
#evaluation on cold-start users
threshold = int(
LineConfig(self.config['evaluation.setup'])['-cold'])
removedUser = []
for user in self.data.testSet:
if self.data.userRecord.has_key(user) and len(
self.data.userRecord[user]) > threshold:
removedUser.append(user)
for user in removedUser:
del self.data.testSet[user]
if LineConfig(self.config['evaluation.setup']).contains('-sample'):
userList = self.data.testSet.keys()
removedUser = userList[:int(len(userList) * 0.9)]
for user in removedUser:
del self.data.testSet[user]
def query_transactions(self, query: str, update_dicts: bool):
"""
reloads the transactions from the database to reflect the latest
changes. if update is set to true, the dicts of the object are
also updated.
"""
# if all transactions are being loaded, update balance [i can't regex]
update_balance = query.lower().startswith(f'select * from {self.name}') and\
query.lower().count(' where ') == 0
if update_balance:
self.balance = 0.0
db_records = self.database.query(query)
self.records = []
for (t_id, dt_str, amount, category, \
subcategory, business, note) in db_records:
if update_dicts:
self.update_dicts(category, subcategory, business)
# if reloading all
if update_balance:
self.balance += amount
self.records.append(Record(datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S"),
amount,
category,
subcategory,
business,
note,
t_id))
def __init__(self, stdscr, w_x, w_y, w_width, w_height, windows: list,
conf: dict):
"""
initializes the main window. the main window holds the
current account, which acts as a proxy between the ui
and the database.
"""
super().__init__(stdscr, w_x, w_y, w_width, w_height)
# reading column sizes
self.icol = conf['table']['index_length']
self.acol = conf['table']['amount_length']
self.ccol = conf['table']['category_length']
self.sccol = conf['table']['subcategory_length']
self.pcol = conf['table']['payee_length']
self.ncol = conf['table']['note_length']
# other stuff
self.disable_actions = False
self.account = None
self.table_label = 'nothing'
self.windows = windows
# padding on the sides
self.list_width = int(w_width - 2)
self.list_height = int((3 / 4) * self.w_height)
self.clist = CursesList(2, 5, self.list_width, self.list_height, [],
conf['table']['scrollbar-enable'], \
' | '.join(Record.columns(self.icol, self.acol, \
self.ccol, self.sccol, \
self.pcol, self.ncol)))
# shown income, expense
self.table_income = 0.0
self.table_expense = 0.0
self.redraw()
def __process_records(self) -> None:
"""
:return:
"""
if self.__hTreeml is None:
self.log.exception("Does the HTML file exist? Parser couldn't initialize itself")
raise ValueError("HTML failed to load, cannot continue")
table: Tag = self.__hTreeml.find(id="queryResults")
if not table:
self.log.error("Could not find results section in HTML!")
# TODO: raise exception?
return
field_name_row = table.thead.tr
field_names: list = list()
for th in field_name_row.children:
if th != "\n" and "View Record" not in th.string:
field_names.append(th.string)
for tr in table.tbody.children:
if not isinstance(tr, str):
temp_rec: Record = Record()
# There are extraneous '\n' and other strings so get rid of them
# First tag in the list is an <img> element with a link so we discard it
no_strings: List[Tag] = [x for x in tr.contents if not isinstance(x, str)][1:]
for field_name, td in zip(field_names, no_strings):
temp_rec.add_field(field_name, td.string)
self.__records.append(temp_rec)
def text2record(self, text: str, kw_num: int = 7) -> Record:
"""
1. text to structure information
2. convert to Record
Args:
text:
kw_num
Returns: List[Record]
Record
"""
# _text = self.text_help.base_bert_clear(str(text)[:256])
_text = self.text_help.base_bert_clear(
str(text))[:256] # return text after clearing
try:
_text = ','.join(jiagu.summarize(text, 2))
except Exception as e:
_text = ''.join(jiagu.summarize(text, 1))
text_vec = np.zeros(self.word_vec_dim)
_text = self.lac.run([_text])[0]
# [(word, pos, weight), ()]
_text = [(
_text[0][i],
_text[1][i],
self.key_pos_w[v],
) for i, v in enumerate(_text[-1]) if v in self.key_pos_w]
_text = self.key_pos_w_deduplication(_text)[:kw_num]
_flag = False
word_num = 0
if len(_text) > 0:
for item in _text:
try:
word_vec = self.w2v_model.word_vec(item[0]) * np.array(
item[2])
text_vec += word_vec
_flag = True
word_num += 1
except Exception as e:
# logger.debug(e)
pass
if _flag:
text_vec /= word_num
return Record(text=text,
vec=text_vec,
keyword=[(item[0], item[1]) for item in _text])
else:
return Record(text=text, vec=None, keyword=None)
def parse_expense(elements: list, dt: datetime, account: Account) -> Record:
"""
converts the list of strings to a database record
"""
cat = elements[2]
subcat = ''
if ':' in elements[2]:
cat, subcat = elements[2].split(':')
elif elements[2] in account.subcategories:
cat = account.subcategories[elements[2]]
subcat = elements[2]
return Record(dt.replace(microsecond=0), float(elements[0]), cat, \
subcat, elements[1], elements[5])
def __init__(self, conf, trainingSet=None, testSet=None, fold='[1]'):
self.config = conf
self.isSaveModel = False
self.isLoadModel = False
self.isOutput = True
self.data = Record(self.config, trainingSet, testSet)
self.foldInfo = fold
self.evalConfig = LineConfig(self.config['evaluation.setup'])
if self.evalConfig.contains('-target'):
self.recType = self.evalConfig['-target']
else:
self.recType = 'track'
if LineConfig(self.config['evaluation.setup']).contains('-cold'):
#evaluation on cold-start users
threshold = int(
LineConfig(self.config['evaluation.setup'])['-cold'])
removedUser = []
removedTrack = defaultdict(list)
#for user in self.data.testSet:
# if user in self.data.userRecord and len(self.data.userRecord[user])>threshold:
# removedUser.append(user)
for user in self.data.testSet:
if user in self.data.userRecord:
for item in self.data.testSet[user]:
if len(self.data.trackRecord[item]) > threshold:
removedTrack[user].append(item)
for user in removedTrack:
for item in removedTrack[user]:
del self.data.testSet[user][item]
if len(self.data.testSet[user]) == 0:
del self.data.testSet[user]
#for user in removedUser:
# del self.data.testSet[user]
if LineConfig(self.config['evaluation.setup']).contains('-sample'):
userList = list(self.data.testSet.keys())
removedUser = userList[:int(len(userList) * 0.9)]
for user in removedUser:
del self.data.testSet[user]
def _create_records(self, url_list):
"""Creates all of the record objects.
Args:
url_list (list): List of all Urls.
"""
self._records = [
Record(url, deezer_title) for url, deezer_title in url_list
]
if not (self._records):
return
# setup the columns if no csv exists
self._csv_adapter.establish_df(self._records[0].get_all_labels())
def convert_to_record(self, t_datetime: str, amount: str, \
cat: str, subcat: str, business: str, note: str) -> Record:
"""
parses >>rectified<< strings into a Record
"""
try:
record_datetime = datetime.strptime(t_datetime,
'%Y-%m-%d %H:%M:%S')
except ValueError:
return None, f"got wrong datetime format, got {t_datetime}"
# parsing amount
if amount.count('.') > 1:
return None, f"wrong amount format: got {amount}, expected EUR.CENT"
record_amount = float(amount)
return Record(record_datetime, record_amount, cat.strip(), \
subcat.strip(), business.strip(), note.strip()), "success"
def convert_to_record(self, t_date: str, t_time: str, amount: str, \
cat: str, subcat: str, business: str, note: str) -> Record:
"""
parses >>rectified<< strings into a Record
"""
# parsing date
date_lst = t_date.split('/')
if len(date_lst) != 3:
return None, f"wrong date format: got {t_date}, expected MM/DD/YYYY"
# parsing time
time_lst = t_time.split(':')
if len(time_lst) != 2:
return None, f"wrong time format: got {t_time}, expected HH:MM"
record_datetime = datetime(int(date_lst[2]), int(date_lst[0]), int(date_lst[1]),\
int(time_lst[0]), int(time_lst[1]))
# parsing amount
if amount.count('.') > 1:
return None, f"wrong amount format: got {amount}, expected EUR.CENT"
record_amount = float(amount)
return Record(record_datetime, record_amount, cat.strip(), \
subcat.strip(), business.strip(), note.strip()), "success"
def account(database: SQLiteProxy, name: str, initial_balance: str) -> str:
try:
balance_f = float(initial_balance)
except:
return [f"{initial_balance} is not a float value"]
# should stop basic sql injections
if ';' in name:
return ["sneaky but no"]
# this shouldn't be possible anyway but meh
if ' ' in name:
return ["account name cannot contain spaces"]
# other stuff
forbidden, frch = variadic_contains_or(name, '/', '\\','\'', '\"', '!', '?',\
'+', '=', '%', '*', '&', '^',\
'@', '#', '$', '~', '.', '`',\
'[', ']', '(', ')', '[', ']')
if forbidden:
return [f"account name cannot contain {frch}"]
if balance_f < 0:
return [
"initial account balance cannot be negative. are you really that poor?"
]
try:
database.create_table(name)
except SQLiteOperationalError:
return [f"account {name} already exists"]
except:
return [f"could not create account {name}... go figure out why"]
# adding the initial balance
# the account object doesn't get created until we use set account, therefor
# we cannot use the much more convenient call: account.add_transaction(...)
if balance_f > 0:
intial_record = Record(datetime(1, 1, 1, 0, 0, 0, 0), balance_f, '',
'', '', 'initial balance')
database.add_record(name, intial_record)
database.connection.commit()
return [f"successfully added {name} with {balance_f} initial balance"]
class Recommender(object):
def __init__(self, conf, trainingSet=None, testSet=None, fold='[1]'):
self.config = conf
self.isSaveModel = False
self.isLoadModel = False
self.isOutput = True
self.data = Record(self.config, trainingSet, testSet)
self.foldInfo = fold
self.evalConfig = LineConfig(self.config['evaluation.setup'])
if self.evalConfig.contains('-target'):
self.recType = self.evalConfig['-target']
else:
self.recType = 'track'
if LineConfig(self.config['evaluation.setup']).contains('-cold'):
#evaluation on cold-start users
threshold = int(
LineConfig(self.config['evaluation.setup'])['-cold'])
removedUser = {}
for user in self.data.testSet:
if self.data.userRecord.has_key(user) and len(
self.data.userRecord[user]) > threshold:
removedUser[user] = 1
for user in removedUser:
del self.data.testSet[user]
def readConfiguration(self):
self.algorName = self.config['recommender']
self.output = LineConfig(self.config['output.setup'])
self.isOutput = self.output.isMainOn()
self.ranking = LineConfig(self.config['item.ranking'])
def printAlgorConfig(self):
"show algorithm's configuration"
print 'Algorithm:', self.config['recommender']
print 'Training set:', abspath(self.config['record'])
if LineConfig(self.config['evaluation.setup']).contains('-testSet'):
print 'Test set:', abspath(
LineConfig(
self.config['evaluation.setup']).getOption('-testSet'))
#print 'Count of the users in training set: ',len()
self.data.printTrainingSize()
print '=' * 80
def initModel(self):
pass
def buildModel(self):
'build the model (for model-based algorithms )'
pass
def saveModel(self):
pass
def loadModel(self):
pass
def predict(self, user):
return []
def evalRanking(self):
res = [] # used to contain the text of the result
N = 0
threshold = 0
N = int(self.ranking['-topN'])
if N > 100 or N < 0:
print 'N can not be larger than 100! It has been reassigned with 10'
N = 10
res.append(
'userId: recommendations in (itemId, ranking score) pairs, * means the item matches.\n'
)
# predict
recList = {}
userCount = len(self.data.testSet)
rawRes = {}
for i, user in enumerate(self.data.testSet):
itemSet = {}
line = user + ':'
predictedItems = self.predict(user)
recList[user] = predictedItems
if i % 100 == 0:
print self.algorName, self.foldInfo, 'progress:' + str(
i) + '/' + str(userCount)
for item in recList[user]:
if self.data.testSet[user].has_key(item[0]):
line += '*'
line += item + ','
line += '\n'
res.append(line)
currentTime = strftime("%Y-%m-%d %H-%M-%S", localtime(time()))
# output prediction result
if self.isOutput:
fileName = ''
outDir = self.output['-dir']
if self.ranking.contains('-topN'):
fileName = self.config[
'recommender'] + '@' + currentTime + '-top-' + str(
N) + 'items' + self.foldInfo + '.txt'
elif self.ranking.contains('-threshold'):
fileName = self.config[
'recommender'] + '@' + currentTime + '-threshold-' + str(
threshold) + self.foldInfo + '.txt'
FileIO.writeFile(outDir, fileName, res)
print 'The result has been output to ', abspath(outDir), '.'
# output evaluation result
outDir = self.output['-dir']
fileName = self.config[
'recommender'] + '@' + currentTime + '-measure' + self.foldInfo + '.txt'
if self.ranking.contains('-topN'):
self.measure = Measure.rankingMeasure(self.data.testSet, recList,
rawRes, N)
FileIO.writeFile(outDir, fileName, self.measure)
print 'The result of %s %s:\n%s' % (self.algorName, self.foldInfo,
''.join(self.measure))
def execute(self):
self.readConfiguration()
if self.foldInfo == '[1]':
self.printAlgorConfig()
#load model from disk or build model
if self.isLoadModel:
print 'Loading model %s...' % (self.foldInfo)
self.loadModel()
else:
print 'Initializing model %s...' % (self.foldInfo)
self.initModel()
print 'Building Model %s...' % (self.foldInfo)
self.buildModel()
#preict the ratings or item ranking
print 'Predicting %s...' % (self.foldInfo)
self.evalRanking()
#save model
if self.isSaveModel:
print 'Saving model %s...' % (self.foldInfo)
self.saveModel()
return self.measure
def sqlite(terminal, stdscr):
# exception handling
if terminal.windows[statics.WMAIN].account == None:
return ["current account not set"]
if stdscr is None:
return ["cannot query in warmup mode"]
account = terminal.windows[statics.WMAIN].account
db_connection = account.database.connection
terminal.windows[statics.WMAIN].disable_actions = True
potential_table_update = False
query_mode = True
query_history = []
query_surf_index = 0
query_history_buffer = ''
showing_bak = terminal.windows[statics.WMAIN].table_label
terminal.terminal_history.append("query mode activated")
terminal.terminal_history.append(
"> column names: transaction_id(primary key), datetime, "
"amount, category, subcategory, business, note")
terminal.terminal_history.append(
f"> tables: {account.database.list_tables()}")
terminal.terminal_history.append(
">> action menu is disabled: deleting & updating has to be done via terminal"
)
terminal.terminal_history.append(
">> listed records only update on valid select queries")
terminal.terminal_history.append(
">> ctrl + (up|down|pgup|pgdown) can be used to scroll up & down the table"
)
terminal.terminal_history.append(
">> sample query: SELECT * FROM table ORDER BY datetime(datetime) DESC;"
)
terminal.command = ''
terminal.cursor_x = 0
terminal.redraw()
# start accepting input -----------------------------------------------------------
kb_interrupt = False
while query_mode:
try:
input_char = stdscr.get_wch()
kb_interrupt = False
except KeyboardInterrupt:
if kb_interrupt or terminal.command == '':
break
kb_interrupt = True
terminal.command = ''
terminal.cursor_x = 0
terminal.terminal_history.append(
'press ctrl + c again to exit query mode')
terminal.redraw()
continue
except:
continue
# backspace, del --------------------------------------------------------------
if input_char == curses.KEY_BACKSPACE or input_char == '\x7f':
terminal.cursor_x = max(0, terminal.cursor_x - 1)
if terminal.cursor_x == len(terminal.command) - 1:
terminal.command = terminal.command[:terminal.cursor_x]
else:
terminal.command = terminal.command[:terminal.cursor_x] + \
terminal.command[terminal.cursor_x + 1:]
terminal.redraw()
elif input_char == curses.KEY_DC:
if len(terminal.command) > 0 and terminal.cursor_x < len(
terminal.command):
terminal.command = terminal.command[:terminal.cursor_x] + \
terminal.command[terminal.cursor_x + 1:]
terminal.redraw()
# submit ----------------------------------------------------------------------
elif input_char == curses.KEY_ENTER or input_char == '\n':
query = terminal.command.strip()
if query == '':
continue
query_history.append(query)
terminal.terminal_history.append('>>> ' + query)
if query[-1] != ';' or query.count(';') > 1:
terminal.terminal_history.append(
'no semicolons! (or too many)')
terminal.command = ''
terminal.cursor_x = 0
terminal.scroll = 0
query_surf_index = 0
terminal.redraw()
continue
cursor = db_connection.cursor()
try:
cursor.execute(query)
except:
terminal.terminal_history.append('could not execute query')
terminal.command = ''
terminal.cursor_x = 0
terminal.scroll = 0
query_surf_index = 0
terminal.redraw()
continue
if f' {account.name}' not in query:
terminal.terminal_history.append( \
'warning: query does not target current account ({})'. \
format(account.name))
# SELECT command
if query.lower().startswith('select '):
db_items = cursor.fetchall()
custom_records = []
if len(db_items) > 0 and len(db_items[0]) == 7:
for (t_id, dt_str, amount, category, \
subcategory, business, note) in db_items:
custom_records.append(
Record(
datetime.strptime(dt_str,
"%Y-%m-%d %H:%M:%S"), amount,
category, subcategory, business, note, t_id))
terminal.windows[statics.WMAIN].refresh_table_records( \
'custom sql query results', custom_records)
elif len(db_items) > 0 and len(db_items[0]) < 7:
terminal.terminal_history.append(
'unsupported select query, printing...')
for item in db_items:
item_list = [str(x) for x in item]
terminal.terminal_history.append(','.join(item_list))
else:
terminal.terminal_history.append(
'select queries without 7 column results are invalid')
# other commands
else:
potential_table_update = True
db_connection.commit()
terminal.command = ''
terminal.cursor_x = 0
terminal.scroll = 0
query_surf_index = 0
terminal.redraw()
# scrolling terminal ----------------------------------------------------------
elif input_char == curses.KEY_PPAGE:
max_scroll = len(terminal.terminal_history) + 3 - terminal.w_height
# if we can show more than history + 3 reserved lines:
if max_scroll > 0:
terminal.scroll = min(terminal.scroll + 1, max_scroll)
terminal.redraw()
elif input_char == curses.KEY_NPAGE:
terminal.scroll = max(terminal.scroll - 1, 0)
terminal.redraw()
# scrolling table -------------------------------------------------------------
elif input_char == statics.CTRL_PG_UP:
terminal.windows[statics.WMAIN].clist.key_pgup()
terminal.windows[statics.WMAIN].redraw()
terminal.redraw()
elif input_char == statics.CTRL_PG_DOWN:
terminal.windows[statics.WMAIN].clist.key_pgdn()
terminal.windows[statics.WMAIN].redraw()
terminal.redraw()
elif input_char == statics.CTRL_UP:
terminal.windows[statics.WMAIN].clist.key_up()
terminal.windows[statics.WMAIN].redraw()
terminal.redraw()
elif input_char == statics.CTRL_DOWN:
terminal.windows[statics.WMAIN].clist.key_down()
terminal.windows[statics.WMAIN].redraw()
terminal.redraw()
# history surfing -------------------------------------------------------------
elif input_char == curses.KEY_UP:
if len(query_history) != 0:
terminal.scroll = 0
# if we weren't surfing, save the current command in buffer
if query_surf_index == 0:
query_history_buffer = terminal.command
query_surf_index = min(query_surf_index + 1,
len(query_history))
terminal.command = query_history[-query_surf_index]
terminal.cursor_x = len(terminal.command)
terminal.redraw()
elif input_char == curses.KEY_DOWN:
if query_surf_index != 0:
terminal.scroll = 0
query_surf_index -= 1
if query_surf_index == 0:
terminal.command = query_history_buffer
terminal.cursor_x = len(terminal.command)
else:
terminal.command = query_history[-query_surf_index]
terminal.cursor_x = len(terminal.command)
terminal.redraw()
# cursor shift ----------------------------------------------------------------
elif input_char == curses.KEY_LEFT:
terminal.cursor_x = max(0, terminal.cursor_x - 1)
terminal.redraw()
elif input_char == curses.KEY_RIGHT:
terminal.cursor_x = min(len(terminal.command),
terminal.cursor_x + 1)
terminal.redraw()
elif input_char == statics.CTRL_LEFT:
cut_str = terminal.command[:terminal.cursor_x][::-1]
while len(cut_str) != 0 and cut_str[0] == ' ':
cut_str = cut_str[1:]
terminal.cursor_x = max(0, terminal.cursor_x - 1)
next_jump = cut_str.find(' ')
if next_jump == -1:
terminal.cursor_x = 0
else:
terminal.cursor_x = max(0, terminal.cursor_x - next_jump)
terminal.redraw()
elif input_char == statics.CTRL_RIGHT:
cut_str = terminal.command[terminal.cursor_x:]
while len(cut_str) != 0 and cut_str[0] == ' ':
cut_str = cut_str[1:]
terminal.cursor_x = min(terminal.cursor_x + 1,
len(terminal.command))
next_jump = cut_str.find(' ')
if next_jump == -1:
terminal.cursor_x = len(terminal.command)
else:
terminal.cursor_x = min(terminal.cursor_x + next_jump,
len(terminal.command))
cut_str = terminal.command[terminal.cursor_x:]
terminal.redraw()
elif input_char == curses.KEY_HOME:
terminal.cursor_x = 0
terminal.redraw()
elif input_char == curses.KEY_END:
terminal.cursor_x = len(terminal.command)
terminal.redraw()
# normal input ----------------------------------------------------------------
else:
# some command that's not used
if type(input_char) is int:
terminal.terminal_history.append(
f'non standard input: {str(input_char)}')
terminal.redraw()
continue
if input_char == ' ':
# leading spaces don't count
if len(terminal.command) == 0:
terminal.redraw()
continue
if terminal.cursor_x == len(terminal.command):
terminal.command = terminal.command[:terminal.
cursor_x] + input_char
else:
terminal.command = terminal.command[:terminal.cursor_x] + input_char \
+ terminal.command[terminal.cursor_x:]
terminal.cursor_x += 1
terminal.scroll = 0
query_surf_index = 0
terminal.redraw()
# restoring state, updating just to be safe
if potential_table_update:
account.query_transactions(account.full_query, True)
terminal.windows[statics.WMAIN].refresh_table_records(showing_bak)
terminal.windows[statics.WMAIN].disable_actions = False
return ["query mode deactivated"]
feats = []
# Get a large buffer around the intersection address
intersection = None
if args.address:
address = None
if args.latlon:
lat, lon = args.latlon.split(',')
address = args.address, lat, lon
else:
address, lat, lon = util.geocode_address(args.address)
intersection = Record({
'location': {
'latitude': lat,
'longitude': lon
},
'address': address
})
buffer = (intersection.point).buffer(250)
else:
# Convert to 3857 projection and get the bounding box
minx = float(args.minx)
miny = float(args.miny)
maxx = float(args.maxx)
maxy = float(args.maxy)
poly = geojson.Polygon([[[minx, miny], [maxx, miny], [maxx, maxy],
[minx, maxy], [minx, miny]]])
feats.append(geojson.Feature(geometry=poly, properties={}))
class Recommender(object):
def __init__(self, conf, trainingSet=None, testSet=None, fold='[1]'):
self.config = conf
self.isSaveModel = False
self.isLoadModel = False
self.isOutput = True
self.data = Record(self.config, trainingSet, testSet)
self.foldInfo = fold
self.evalConfig = LineConfig(self.config['evaluation.setup'])
if self.evalConfig.contains('-target'):
self.recType = self.evalConfig['-target']
else:
self.recType = 'track'
if LineConfig(self.config['evaluation.setup']).contains('-cold'):
#evaluation on cold-start users
threshold = int(
LineConfig(self.config['evaluation.setup'])['-cold'])
removedUser = []
removedTrack = defaultdict(list)
#for user in self.data.testSet:
# if user in self.data.userRecord and len(self.data.userRecord[user])>threshold:
# removedUser.append(user)
for user in self.data.testSet:
if user in self.data.userRecord:
for item in self.data.testSet[user]:
if len(self.data.trackRecord[item]) > threshold:
removedTrack[user].append(item)
for user in removedTrack:
for item in removedTrack[user]:
del self.data.testSet[user][item]
if len(self.data.testSet[user]) == 0:
del self.data.testSet[user]
#for user in removedUser:
# del self.data.testSet[user]
if LineConfig(self.config['evaluation.setup']).contains('-sample'):
userList = list(self.data.testSet.keys())
removedUser = userList[:int(len(userList) * 0.9)]
for user in removedUser:
del self.data.testSet[user]
def readConfiguration(self):
self.algorName = self.config['recommender']
self.output = LineConfig(self.config['output.setup'])
self.isOutput = self.output.isMainOn()
self.ranking = LineConfig(self.config['item.ranking'])
def printAlgorConfig(self):
"show algorithm's configuration"
print('Algorithm:', self.config['recommender'])
print('Training set:', abspath(self.config['record']))
if LineConfig(self.config['evaluation.setup']).contains('-testSet'):
print(
'Test set:',
abspath(
LineConfig(self.config['evaluation.setup']).getOption(
'-testSet')))
#print 'Count of the users in training set: ',len()
self.data.printTrainingSize()
print('=' * 80)
def initModel(self):
pass
def buildModel(self):
'build the model (for model-based algorithms )'
pass
def saveModel(self):
pass
def loadModel(self):
pass
def predict(self, user):
return []
def evalRanking(self):
res = [] # used to contain the text of the result
N = 0
threshold = 0
top = self.ranking['-topN'].split(',')
top = [int(num) for num in top]
N = int(top[-1])
if N > 100 or N < 0:
print(
'N can not be larger than 100! It has been reassigned with 10')
N = 10
res.append(
'userId: recommendations in (itemId, ranking score) pairs, * means the item matches, $ means the unpop item\n'
)
# predict
recList = {}
userCount = len(self.data.testSet)
for i, user in enumerate(self.data.testSet):
num_pop = 0
line = user + ':'
if user in self.data.userRecord:
predictedItems = self.predict(user)
else:
predictedItems = ['0'] * N
predicted = {}
for k, item in enumerate(predictedItems):
predicted[item] = k
for item in self.data.userRecord[user]:
if item[self.recType] in predicted:
del predicted[item[self.recType]]
predicted = sorted(predicted.items(), key=lambda d: d[1])
predictedItems = [item[0] for item in predicted]
recList[user] = predictedItems[:N]
#print('user', user, 'the recList:', type(self.data.testSet[user]))
if i % 100 == 0:
print(self.algorName, self.foldInfo,
'progress:' + str(i) + '/' + str(userCount))
for item in recList[user]:
if item in self.data.testSet[user]:
line += '*'
if item in self.data.PopTrack:
num_pop += 1
line += '$'
line += item + ','
line += '\n'
res.append(line)
currentTime = strftime("%Y-%m-%d %H-%M-%S", localtime(time()))
# output prediction result
if self.isOutput:
fileName = ''
outDir = self.output['-dir']
if self.ranking.contains('-topN'):
fileName = self.config['recommender'] + '@' + currentTime + '-top-' + self.ranking['-topN']\
+ 'items' + self.foldInfo + '.txt'
FileIO.writeFile(outDir, fileName, res)
print('The result has been output to ', abspath(outDir), '.')
# output evaluation result
outDir = self.output['-dir']
fileName = self.config[
'recommender'] + '@' + currentTime + '-measure' + self.foldInfo + '.txt'
self.measure = Measure.rankingMeasure(self.data.testSet, recList, top,
self.data.getSize(self.recType))
FileIO.writeFile(outDir, fileName, self.measure)
print('The result of %s %s:\n%s' %
(self.algorName, self.foldInfo, ''.join(self.measure)))
def execute(self):
self.readConfiguration()
if self.foldInfo == '[1]':
self.printAlgorConfig()
#load model from disk or build model
if self.isLoadModel:
print('Loading model %s...' % (self.foldInfo))
self.loadModel()
else:
print('Initializing model %s...' % (self.foldInfo))
self.initModel()
print('Building Model %s...' % (self.foldInfo))
self.buildModel()
#preict the ratings or item ranking
print('Predicting %s...' % (self.foldInfo))
self.evalRanking()
#save model
if self.isSaveModel:
print('Saving model %s...' % (self.foldInfo))
self.saveModel()
return self.measure