def main():
path = os.path.dirname(os.path.realpath(os.getcwd()))
data_path = os.path.join(path, 'Data')
years = [str(x) for x in range(2012, 2018)]
seasons = [f"{x}-schedule-scores.shtml" for x in range(2012, 2018)]
test_season = '2019-schedule-scores.shtml'
team_url_csv = os.path.join(data_path, 'mlb_url.csv')
MLBScraper = mlb_scrapping.MLBScraper(years[0], seasons[0], team_url_csv)
games_df = MLBScraper()
for i, year in enumerate(years[1:]):
MLBScraper = mlb_scrapping.MLBScraper(year, seasons[i], team_url_csv)
games_df.append(MLBScraper())
print(list(games_df.columns))
# building test_df
MLBScraper = mlb_scrapping.MLBScraper("2019", test_season, team_url_csv)
test_df = MLBScraper()
test_df = test_df[test_df['team'] == 'ATL']
test_preprocessor = preprocess.Preprocess(test_df)
Xtest, ytest = test_preprocessor.split_xy('attendance')
preprocessor = preprocess.Preprocess(games_df)
X, y = preprocessor.split_xy('attendance')
modeler = Models(X, y)
rf = modeler.fit_random_forest()
print(rf.score(X, y))
print(list(rf.predict(Xtest) - ytest))
print("Scoring for the 2019 braves season")
rmse, r2 = modeler.test(rf, Xtest, ytest)
print(f"RMSE : {rmse}")
print(f'r2 : {r2}')
def build(self):
md = Models(self.seq_length, self.featue_length)
model = md.Attention_after_LSTM()
plot_model(model,
to_file='Attention_after_LSTM_model.png',
show_shapes=True)
return model
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self._init_entries()
def post(self):
object_area = self.request.get("object")
date_area = self.request.get("date")
time_area = self.request.get("time")
message_area = self.request.get("message")
user_input = Models(object=object_area,date=date_area,time=time_area,message=message_area)
user_input.put()
params = {"object":object_area,
"date":date_area,
"time":time_area,
"message":message_area}
self.render_template("your-task.html",params=params)
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(331, 166)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
#define data model as attribute to optimize the Models request
self.model = Models()
self.label = QtWidgets.QLabel(self.centralwidget)
self.label.setGeometry(QtCore.QRect(10, 30, 47, 13))
self.label.setObjectName("label")
self.label_2 = QtWidgets.QLabel(self.centralwidget)
self.label_2.setGeometry(QtCore.QRect(10, 70, 47, 13))
self.label_2.setObjectName("label_2")
self.pushButton = QtWidgets.QPushButton(self.centralwidget)
self.pushButton.setGeometry(QtCore.QRect(60, 100, 75, 23))
self.pushButton.setObjectName("pushButton")
self.pushButton.clicked.connect(self.check_button)
self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
self.pushButton_2.setGeometry(QtCore.QRect(210, 100, 75, 23))
self.pushButton_2.setObjectName("pushButton_2")
# username field
self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
self.textEdit.setGeometry(QtCore.QRect(60, 20, 251, 25))
self.textEdit.setObjectName("textEdit")
# password field
self.textEdit_2 = QtWidgets.QTextEdit(self.centralwidget)
self.textEdit_2.setGeometry(QtCore.QRect(60, 60, 251, 25))
self.textEdit_2.setObjectName("textEdit_2")
MainWindow.setCentralWidget(self.centralwidget)
self.menubar = QtWidgets.QMenuBar(MainWindow)
self.menubar.setGeometry(QtCore.QRect(0, 0, 331, 21))
self.menubar.setObjectName("menubar")
MainWindow.setMenuBar(self.menubar)
self.statusbar = QtWidgets.QStatusBar(MainWindow)
self.statusbar.setObjectName("statusbar")
MainWindow.setStatusBar(self.statusbar)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def add_units(self, units):
ModelManager.empty_model_list(ModelManager.MODEL_UNIT)
for unit_object in units:
unit = Models.Clip()
unit.unit_name = unit_object['unit_name']
unit.type = unit_object['type']
unit.online = unit_object['online']
ModelManager.register_model(unit, ModelManager.MODEL_UNIT)
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self._init_entries()
def main() :
EPOCHS = 50
Model = Models()
generator = Model.make_generator_model()
discriminator = Model.make_discriminator_model()
train(Model = Model, epochs = EPOCHS, generator = generator, discriminator = discriminator)
raw_noise = np.array(DC().preprocessed_set()[0], dtype = np.float32)
raw_noise = make_data_noisy(raw_noise)
predictions = np.array(generator(raw_noise, training = False))
for prediction in predictions :
cv2.imshow("predictions", prediction)
cv2.waitKey(0)
cv2.destroyAllWindows()
with open("prediction.pickle", "wb") as f :
pickle.dump(predictions, f)
def add_units(self, units):
unit_list = []
for unit_object in units:
unit = Models.Unit()
unit.unit_name = unit_object['unit_name']
unit.type = unit_object['type']
unit.online = unit_object['online']
unit_list.append(unit)
ModelManager.register_model(unit, ModelManager.MODEL_UNIT)
self.get_eof_from_unit(unit.unit_name)
self.find_all_existing_clips(unit_list)
def post(self,task_id):
object = self.request.get("object")
date = self.request.get("date")
time = self.request.get("time")
message = self.request.get("message")
task = Models.get_by_id(int(task_id))
task.object = object
task.date = date
task.time = time
task.message = message
task.put()
self.redirect_to("Task-List")
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
self.blog_bucket = BucketManager(self.qiniu)
self.blog_prefix = "raw/"
self._init_blog()
def __init__(self, n_agents, dim_obs, dim_act, batch_size,
capacity, episodes_before_train, load_models=None):
# self.actors = [Actor(dim_obs, dim_act) for i in range(n_agents)]
# self.critics = [Critic(n_agents, dim_obs, dim_act) for i in range(n_agents)]
if load_models is None:
self.models = Models(n_agents, dim_obs, dim_act)
self.actors_target = deepcopy(self.models.actors)
self.critics_target = deepcopy(self.models.critics)
self.critic_optimizer = [Adam(x.parameters(), lr=0.0001) for x in self.models.critics] # 0.001
self.actor_optimizer = [Adam(x.parameters(), lr=0.00001) for x in self.models.actors] # 0.0001
self.memory = ReplayMemory(capacity)
self.var = [1.0 for i in range(n_agents)]
else:
print('Start loading models!')
states = th.load(load_models)
self.models = states['models']
self.critic_optimizer = states['critic_optimizer']
self.actor_optimizer = states['actor_optimizer']
self.critics_target = states['critics_target']
self.actors_target = states['actors_target']
self.memory = states['memory']
self.var = states['var']
print('Models loaded!')
self.n_agents = n_agents
self.n_states = dim_obs
self.n_actions = dim_act
self.batch_size = batch_size
self.use_cuda = th.cuda.is_available()
self.episodes_before_train = episodes_before_train
self.GAMMA = 0.95
self.tau = 0.01
if self.use_cuda:
for x in self.models.actors:
x.cuda()
for x in self.models.critics:
x.cuda()
for x in self.actors_target:
x.cuda()
for x in self.critics_target:
x.cuda()
self.steps_done = 0
self.episode_done = 0
def add_clips(self, clips, unit_name):
unit = self.pending_unit_processing.pop(0)
for clip_object in clips:
clip = Models.Clip()
clip.unit = unit.unit_name
clip.index = clip_object['index']
clip.path = clip_object['path']
clip.clip_in = clip_object['clip_in']
clip.clip_out = clip_object['clip_out']
clip.length = clip_object['length']
clip.calculated_length = clip_object['calculated_length']
clip.fps = clip_object['fps']
ModelManager.register_model(clip, ModelManager.MODEL_CLIP)
if len(self.pending_unit_processing) > 0:
self.melted_telnet_controller.get_unit_clips(
self.pending_unit_processing[0].unit_name, self.add_clips)
else:
self.loaded_callback()
def add_clips(self, clips, unit_name):
clip_models = ModelManager.get_models(ModelManager.MODEL_CLIP)
for clip in clip_models:
if clip.unit == unit_name:
ModelManager.remove_model(clip, ModelManager.MODEL_CLIP)
for clip_object in clips:
clip = Models.Clip()
clip.unit = unit_name
clip.index = clip_object['index']
clip.path = clip_object['path']
clip.clip_in = clip_object['clip_in']
clip.clip_out = clip_object['clip_out']
clip.length = clip_object['length']
clip.calculated_length = clip_object['calculated_length']
clip.fps = clip_object['fps']
ModelManager.register_model(clip, ModelManager.MODEL_CLIP)
if len(clips) == 0:
self.main_controller.get_main_interface_controller().refresh_clips(
)
HIDDEN_SIZE = 32
if __name__ == '__main__':
loader = DataProcess()
train_x, test_x, train_y, test_y = loader.get_data()
train_x_content = train_x[0]
train_x_summary = train_x[1]
test_x_content = test_x[0]
test_x_summary = test_x[1]
raw_label = loader.get_raw_label()
num_words = loader.get_token_nums()
embedding_matrix = loader.get_embedding_matrix()
m = Models(NUM_CLASSES, num_words, MAX_SEQ_LEN_CONTENT,
MAX_SEQ_LEN_SUMMARY, EMBEDDING_SIZE, embedding_matrix)
model = m.multi_channel_model(NUM_FILTERS, FILTER_SIZE, HIDDEN_SIZE)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(0.001),
metrics=['accuracy'])
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=2,
verbose=1,
factor=0.5,
min_lr=1e-5)
history = model.fit(x=[train_x_content, train_x_summary],
y=train_y,
batch_size=BATCH_SIZE,
validation_data=([test_x_content,
import os filename = "LoanPrediction.csv" #change filename dname = os.getcwd() + '/Dataset' filename = os.path.join(dname,filename) data=pd.read_csv(filename) target_column = "Loan_Status" #change target_column y = data[target_column][0:] cols=list(data.columns) cols.remove(target_column) X = data[cols][0:] X_train,X_test,y_train,y_test = make_train_test_split(X,y) model = Models(X_train,X_test,y_train,y_test) #print(model.VanillaLinearRegression()) #print(model.LassoLinearRegression()) #print(model.RidgeLinearRegression()) #print(model.RandomForestRegressor()) #print(model.GradientBoostedRegressor()) #print(model.SupportVectorRegressor()) print(model.VanillaLogisticRegression()) print(model.RandomForestClassifier()) print(model.GradientBoostedClassifier()) print(model.SupportVectorClassifier()) #model.XtremeGradientBoosting()
def get(self):
list = Models.query().fetch()
params = {"list":list}
self.render_template("task-list.html",params=params)
from model import Models # Height, Width field = (11,6) # Frames which we propagate nb_frames = 1 # Number of possible actions nb_actions = 3 # Batch size batch_size = 64 # Discounter gamma = .95 # Number of games number_games = 250 # Initiate Tetris t = Tetris(field = field) # Get model models = Models(nb_frames, field[0], field[1], nb_actions) model = models.get_model() # Agent things agent = Agent(t, model, nb_actions, field, nb_epoch = number_games, gamma=gamma, batch_size=batch_size) agent.train() agent.plot_ma_score() agent.play(t, nb_epoch = 1)
def __init__(self, model_name, device='MYRIAD', extensions=None):
Models.__init__(self, model_name, device, extensions)
class EntryService:
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self.private_list = []
self.all_urls = []
self._init_entries()
def get_tag_url(self, tag):
url = '/search?type=tag&value=%s&start=1&limit=5' % urllib.quote(tag)
return url
def add_private(self, path):
self.private_list.append(path)
self.save_private()
def del_private(self, path):
self.private_list.append(path)
try:
self.private_list.remove(path)
except:
pass
self.save_private()
def save_private(self):
private_path = 'raw/' + config.private_store
with open(private_path, "w") as _file:
pickle.dump(self.private_list, _file)
_file.close()
def _init_entries(self):
private_path = 'raw/' + config.private_store
private_list = []
if os.path.exists(private_path):
with open(private_path, 'r') as _file:
private_list = pickle.load(_file)
self.private_list = private_list
print 'private list', self.private_list
for root, _, files in os.walk(config.entry_dir):
for f in files:
_path = root + '/' + f
self.add_entry(False, _path, _path in private_list)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
public_entries = [e for e in self.entries.values() if not e.private]
self._init_miscellaneous(self.types.add, public_entries)
def add_entry(self, inotified, path, private=False):
entry = self._init_entry(self.types.entry, path, private)
if not entry == None:
self.entries[entry.url] = entry
if inotified:
if private:
self.update_urls()
self.add_private(path)
else:
self._init_miscellaneous(self.types.add, [entry])
else:
self.update_urls()
def update_entry(self, entry, data):
tar = self.entries[entry.url]
if 'title' in data:
tar.name = data.get('title')
#entry.name = data.get('title')
old_private = entry.private
new_private = bool(data.get('private', ''))
changed = False
if 'tags' in data:
old_tags = entry.tags
new_tags = data.get('tags')
if not old_private:
self._init_tag(self.types.delete, entry.url, old_tags)
if not new_private:
self._init_tag(self.types.add, entry.url, new_tags)
tar.tags = new_tags
if old_tags != new_tags: changed = True
if 'cats' in data:
old_cats = entry.categories
new_cats = data.get('cats')
if not old_private:
self._init_category(self.types.delete, entry.url, old_cats)
if not new_private:
self._init_category(self.types.add, entry.url, new_cats)
tar.categories = new_cats
if old_cats != new_cats: changed = True
tar.private = new_private
if old_private != new_private:
self.update_urls()
if new_private:
self.add_private(entry.path)
else:
self.del_private(entry.path)
changed = True
if changed:
self._init_params()
def delete_entry(self, path):
for entry in self.entries.values():
if path == entry.path:
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
page = self._init_entry(self.types.page, path)
if not page == None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path, private=False):
'''
read infomation from md file
'''
url, raw_url, name, date, time, content = self._init_file(
path, entry_type)
if not url == None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = config.raw_url + raw_url
entry.update_url = config.update_url + raw_url
entry.date = date
entry.time = time
entry.private = private
#header, title, categories, tags = extract.parse(entry)
print 'parse', entry.path
with open(entry.path, 'r') as f:
start = f.readline()
if '---' in start.strip():
layout_raw = f.readline()
title_raw = f.readline()
category_raw = f.readline()
tags_raw = f.readline()
#author_raw = f.readline()
end = f.readline()
content = f.read().strip()
#print layout_raw,title_raw,category_raw,tags_raw
title = title_raw.split(':')[1].strip()
categories_str = category_raw.split(':')[1].strip()
categories = categories_str and categories_str.split(
',') or []
tags_str = tags_raw.split(':')[1].strip()[1:-1]
tags = tags_str and tags_str.split(',') or []
header = ''.join([
start, layout_raw, title_raw, category_raw, tags_raw,
end
])
else:
title, categories, tags, header = '', [], [], ''
if title:
entry.name = title
#content = content.replace(header.decode('utf-8'), '')
try:
content = content.decode('utf-8')
except:
pass
entry.content = content
entry.header = header
if config.backend_md:
import markdown
entry.html = markdown.markdown(
content,
extensions=[
'markdown.extensions.codehilite',
'markdown.extensions.toc',
'markdown.extensions.fenced_code',
'markdown.extensions.tables'
])
else:
entry.html = ''
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
if len(tags) > 0:
first_tag = tags[0]
if first_tag.startswith('__'):
entry.author.name = first_tag[2:]
entry.author.url = self.get_tag_url(first_tag)
return entry
return None
def _init_file(self, file_path, entry_type):
"""
#TODO: FIXME: how to determine the publish time of an entry
"""
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content == None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_', '-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
pass
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date == None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(
config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = '/' + prefix + '/'
if entry_type == self.types.page:
url_prefix = '/'
raw_prefix = '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def update_urls(self):
'''
public url noupdate
'''
_list = []
_pub_list = []
for url, entry in self.entries.items():
_list.append(url)
if not entry.private:
_pub_list.append(url)
self.urls = sorted(_pub_list, reverse=True)
self.all_urls = sorted(_list, reverse=True)
def _init_miscellaneous(self, init_type, entries):
'''
1. rebuild index data of the 'entries'
2. refresh the url data
3. refresh the right part of site page
'''
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.update_urls()
self._init_params()
def _init_subscribe(self):
'''
refresh the subscribe last update time
'''
time = None
if self.urls == []:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self, init_type, url, tags):
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
if url not in self.by_tags[tag].urls:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
m_category = self.by_categories[category]
if init_type == self.types.add:
if url not in m_category.urls:
m_category.urls.insert(0, url)
m_category.count += 1
if init_type == self.types.delete:
m_category.count -= 1
m_category.urls.remove(url)
if m_category.count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self, init_type, url):
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
if url not in self.by_months[month].urls:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
'''
refresh the right part of site page
'''
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget(
)
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
#TODO: FIXME: related entries
"""
urls, index = [], 0
try:
index = self.urls.index(url)
except:
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=[], url=None):
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and not url == None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
if about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
#TODO: FIXME: calculate tags' rank
"""
tags = sorted(self.by_tags.values(),
key=lambda v: v.count,
reverse=True)
tags = [t for t in tags if not t.name.startswith('__')]
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls) > 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
return sorted(self.by_categories.values(), key=lambda c: c.name)
def _init_archive_widget(self):
return sorted(self.by_months.values(),
key=lambda m: m.url,
reverse=True)
def _find_by_query(self, query, start, limit):
"""
#TODO: FIXME: how to search in the content of entries
"""
queries = [q.decode('utf-8') for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
if entry.private:
continue
try:
entry.content.index(query)
urls.append(entry.url)
except:
pass
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
if urls == None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total, limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
if entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
page_url = raw_url.replace(config.raw_url,
'').replace(config.raw_suffix,
config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if not page == None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(
config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if not entry == None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self,
archive_type,
url,
start=1,
limit=999999999,
private=False):
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url, config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
_urls = self.urls if private == False else [
e for e in self.all_urls if e not in self.urls
]
urls = [url for url in _urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url,
archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget(
[self.types.archive])
return self.params
def search(self,
search_type,
url,
value='',
start=config.start,
limit=config.limit):
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
if search_type == self.types.tag:
if self.by_tags.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(
self.by_tags.get(value).urls, start, limit)
if search_type == self.types.category:
if self.by_categories.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(
self.by_categories.get(value).urls, start, limit)
if search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start,
limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
self.params.start = start
self.params.limit = limit
return self.params
def error(self, url):
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(
[self.types.blog])
return self.params
def run():
# read data
train_loader, val_loader = data(args.train_data,
args.valid_data,
args.batch_size,
args.workers)
# set up model -- without imagenet pre-training
model = Models(args.arch, 2, gpus).Model()
cudnn.benchmark = True
# resume pretrain model
if args.resume_path is not None:
pretrained_model = torch.load(args.resume_path)
model.load_state_dict(pretrained_model['state_dict'])
best_prec1 = pretrained_model['best_prec1']
print('Load resume model done.')
else:
best_prec1 = 0
print('Best top-1: {:.4f}'.format(best_prec1))
# optimizer
if args.optim == 'SGD':
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optim == 'Adam':
optimizer = torch.optim.Adam(model.parameters(),
args.lr,
weight_decay=args.weight_decay)
# loss function
criterion = nn.CrossEntropyLoss().cuda()
# learning rate optimal
lr_opt = lambda lr, epoch: lr * (0.1 ** (float(epoch) / 20))
model_path = args.model_path
def save_checkpoint(state, is_best, filename='checkpoint'):
torch.save(state, os.path.join(model_path, filename + '_latest.pth.tar'))
if is_best:
shutil.copyfile(os.path.join(model_path, filename + '_latest.pth.tar'),
os.path.join(model_path, filename + '_best.pth.tar'))
# start
for epoch in range(args.start_epoch, args.epochs):
lr_cur = lr_opt(args.lr, epoch) # speed change
for param_group in optimizer.param_groups:
param_group['lr'] = lr_cur
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch, lr_cur)
# evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# remember best prec1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
}, is_best, args.arch)
class EntryService:
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self.private_list = []
self.all_urls = []
self._init_entries()
def get_tag_url(self, tag):
url = '/search?type=tag&value=%s&start=1&limit=5'%urllib.quote(tag)
return url
def add_private(self, path):
self.private_list.append(path)
self.save_private()
def del_private(self, path):
self.private_list.append(path)
try:
self.private_list.remove(path)
except:pass
self.save_private()
def save_private(self):
private_path = 'raw/' + config.private_store
with open(private_path, "w") as _file:
pickle.dump(self.private_list, _file)
_file.close()
def _init_entries(self):
private_path = 'raw/' + config.private_store
private_list = []
if os.path.exists(private_path):
with open(private_path, 'r') as _file:
private_list = pickle.load(_file)
self.private_list = private_list
print 'private list',self.private_list
for root, _, files in os.walk(config.entry_dir):
for f in files:
_path = root + '/' + f
self.add_entry(False, _path, _path in private_list)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
public_entries = [e for e in self.entries.values() if not e.private]
self._init_miscellaneous(self.types.add, public_entries)
def add_entry(self, inotified, path, private=False):
entry = self._init_entry(self.types.entry, path, private)
if not entry == None:
self.entries[entry.url] = entry
if inotified:
if private:
self.update_urls()
self.add_private(path)
else:
self._init_miscellaneous(self.types.add, [entry])
else:
self.update_urls()
def update_entry(self, entry, data):
tar = self.entries[entry.url]
if 'title' in data:
tar.name = data.get('title')
#entry.name = data.get('title')
old_private = entry.private
new_private = bool(data.get('private', ''))
changed = False
if 'tags' in data:
old_tags = entry.tags
new_tags = data.get('tags')
if not old_private:
self._init_tag(self.types.delete, entry.url, old_tags)
if not new_private:
self._init_tag(self.types.add, entry.url, new_tags)
tar.tags = new_tags
if old_tags!=new_tags:changed=True
if 'cats' in data:
old_cats = entry.categories
new_cats = data.get('cats')
if not old_private:
self._init_category(self.types.delete, entry.url, old_cats)
if not new_private:
self._init_category(self.types.add, entry.url, new_cats)
tar.categories = new_cats
if old_cats!=new_cats:changed = True
tar.private = new_private
if old_private!=new_private:
self.update_urls()
if new_private:
self.add_private(entry.path)
else:
self.del_private(entry.path)
changed = True
if changed:
self._init_params()
def delete_entry(self, path):
for entry in self.entries.values():
if path == entry.path:
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
page = self._init_entry(self.types.page, path)
if not page == None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path, private=False):
'''
read infomation from md file
'''
url, raw_url, name, date, time, content = self._init_file(path, entry_type)
if not url == None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = config.raw_url + raw_url
entry.update_url = config.update_url + raw_url
entry.date = date
entry.time = time
entry.private = private
#header, title, categories, tags = extract.parse(entry)
print 'parse',entry.path
with open(entry.path, 'r') as f:
start = f.readline()
if '---' in start.strip():
layout_raw = f.readline()
title_raw = f.readline()
category_raw = f.readline()
tags_raw = f.readline()
#author_raw = f.readline()
end = f.readline()
content = f.read().strip()
#print layout_raw,title_raw,category_raw,tags_raw
title = title_raw.split(':')[1].strip()
categories_str = category_raw.split(':')[1].strip()
categories = categories_str and categories_str.split(',') or []
tags_str = tags_raw.split(':')[1].strip()[1:-1]
tags = tags_str and tags_str.split(',') or []
header = ''.join([start, layout_raw, title_raw, category_raw, tags_raw, end])
else:
title, categories, tags, header = '', [], [], ''
if title:
entry.name = title
#content = content.replace(header.decode('utf-8'), '')
try:
content = content.decode('utf-8')
except:pass
entry.content = content
entry.header = header
if config.backend_md:
import markdown
entry.html = markdown.markdown(content,extensions=['markdown.extensions.codehilite','markdown.extensions.toc','markdown.extensions.fenced_code','markdown.extensions.tables'])
else:
entry.html = ''
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
if len(tags)>0:
first_tag = tags[0]
if first_tag.startswith('__'):
entry.author.name = first_tag[2:]
entry.author.url = self.get_tag_url(first_tag)
return entry
return None
def _init_file(self, file_path, entry_type):
"""
#TODO: FIXME: how to determine the publish time of an entry
"""
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content == None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_' ,'-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
pass
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date == None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = '/' + prefix + '/'
if entry_type == self.types.page:
url_prefix = '/'
raw_prefix = '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def update_urls(self):
'''
public url noupdate
'''
_list = []
_pub_list = []
for url, entry in self.entries.items():
_list.append(url)
if not entry.private:
_pub_list.append(url)
self.urls = sorted(_pub_list, reverse=True)
self.all_urls = sorted(_list, reverse=True)
def _init_miscellaneous(self,init_type, entries):
'''
1. rebuild index data of the 'entries'
2. refresh the url data
3. refresh the right part of site page
'''
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.update_urls()
self._init_params()
def _init_subscribe(self):
'''
refresh the subscribe last update time
'''
time = None
if self.urls == []:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self,init_type, url, tags):
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
if url not in self.by_tags[tag].urls:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
m_category = self.by_categories[category]
if init_type == self.types.add:
if url not in m_category.urls:
m_category.urls.insert(0, url)
m_category.count += 1
if init_type == self.types.delete:
m_category.count -= 1
m_category.urls.remove(url)
if m_category.count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self,init_type, url):
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
if url not in self.by_months[month].urls:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
'''
refresh the right part of site page
'''
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget()
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
#TODO: FIXME: related entries
"""
urls, index = [], 0
try:
index = self.urls.index(url)
except:
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=[], url=None):
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and not url == None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
if about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
#TODO: FIXME: calculate tags' rank
"""
tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
tags = [t for t in tags if not t.name.startswith('__')]
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls)> 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
return sorted(self.by_categories.values(), key=lambda c:c.name)
def _init_archive_widget(self):
return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)
def _find_by_query(self, query, start, limit):
"""
#TODO: FIXME: how to search in the content of entries
"""
queries = [q.decode('utf-8') for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
if entry.private:
continue
try:
entry.content.index(query)
urls.append(entry.url)
except:
pass
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
if urls == None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total,limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
if entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if not page== None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if not entry == None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self, archive_type, url, start=1, limit=999999999, private=False, username=None):
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url,config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
_urls = self.urls if private==False else [ e for e in self.all_urls if e not in self.urls]
urls = [url for url in _urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
if username:
entries = [e for e in entries if e.author.name==username]
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
return self.params
def search(self, search_type, url, value='', start=config.start, limit=config.limit):
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
if search_type == self.types.tag:
if self.by_tags.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
if search_type == self.types.category:
if self.by_categories.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
if search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start, limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
self.params.start = start
self.params.limit = limit
return self.params
def error(self, url):
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
return self.params
class EntryService:
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self._init_entries()
def _init_entries(self):
for root, _, files in os.walk(config.entry_dir):
for f in files:
self.add_entry(False, root + '/' + f)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
self._init_miscellaneous(self.types.add, self.entries.values())
def add_entry(self, inotified, path):
entry = self._init_entry(self.types.entry, path)
if not entry == None:
self.entries[entry.url] = entry
if inotified:
self._init_miscellaneous(self.types.add, [entry])
def delete_entry(self, path):
for entry in self.entries.values():
if path == os.path.abspath(entry.path):
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
page = self._init_entry(self.types.page, path)
if not page == None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path):
url, raw_url, name, date, time, content = self._init_file(path, entry_type)
if not url == None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = raw_url
entry.date = date
entry.time = time
header, title, categories, tags = extract.parse(entry)
entry.content = content
content = content.replace(header, '')
entry.html = markdown.markdown(content)
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
return entry
return None
def _init_file(self, file_path, entry_type):
"""
#TODO: FIXME: how to determine the publish time of an entry
"""
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content == None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_' ,'-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
pass
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date == None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = config.raw_url + '/' + prefix + '/'
if entry_type == self.types.page:
url_prefix = '/'
raw_prefix = config.raw_url + '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def _init_miscellaneous(self,init_type, entries):
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.urls = sorted(self.entries.keys(), reverse=True)
self._init_params()
def _init_subscribe(self):
time = None
if self.urls == []:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self,init_type, url, tags):
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_categories[category].urls.insert(0, url)
self.by_categories[category].count += 1
if init_type == self.types.delete:
self.by_categories[category].count -= 1
self.by_categories[category].urls.remove(url)
if self.by_categories[category].count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self,init_type, url):
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget()
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
#TODO: FIXME: related entries
"""
urls, index = [], 0
try:
index = self.urls.index(url)
except:
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=[], url=None):
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and not url == None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
if about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
#TODO: FIXME: calculate tags' rank
"""
tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls)> 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
return sorted(self.by_categories.values(), key=lambda c:c.name)
def _init_archive_widget(self):
return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)
def _find_by_query(self, query, start, limit):
"""
#TODO: FIXME: how to search in the content of entries
"""
queries = [q.lower() for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
try:
entry.content.index(query)
urls.append(entry.url)
except:
pass
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
if urls == None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total,limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
if entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if not page== None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if not entry == None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self, archive_type, url, start=1, limit=999999999):
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url,config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
urls = [url for url in self.urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
return self.params
def search(self, search_type, url, value='', start=config.start, limit=config.limit):
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
if search_type == self.types.tag:
if self.by_tags.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
if search_type == self.types.category:
if self.by_categories.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
if search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start, limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
return self.params
def error(self, url):
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
return self.params
def get(self,task_id):
task = Models.get_by_id(int(task_id))
params = {"list":task}
self.render_template("delete-task.html",params=params)
class EntryService:
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self._init_entries()
def _init_entries(self):
for root, _, files in os.walk(config.entry_dir):
for f in files:
self.add_entry(False, root + '/' + f)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
self._init_miscellaneous(self.types.add, self.entries.values())
def add_entry(self, inotified, path):
entry = self._init_entry(self.types.entry, path)
if not entry == None:
self.entries[entry.url] = entry
if inotified:
self._init_miscellaneous(self.types.add, [entry])
def delete_entry(self, path):
for entry in self.entries.values():
if path == os.path.abspath(entry.path):
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
page = self._init_entry(self.types.page, path)
if not page == None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path):
url, raw_url, name, date, time, content = self._init_file(path, entry_type)
if not url == None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = raw_url
entry.date = date
entry.time = time
header, title, categories, tags = extract.parse(entry)
entry.content = content
content = content.replace(header, '')
entry.html = markdown.markdown(content)
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
return entry
return None
def _init_file(self, file_path, entry_type):
"""
#TODO: FIXME: how to determine the publish time of an entry
"""
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content == None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_' ,'-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
pass
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date == None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = config.raw_url + '/' + prefix + '/'
if entry_type == self.types.page:
url_prefix = '/'
raw_prefix = config.raw_url + '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def _init_miscellaneous(self,init_type, entries):
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.urls = sorted(self.entries.keys(), reverse=True)
self._init_params()
def _init_subscribe(self):
time = None
if self.urls == []:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self,init_type, url, tags):
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_categories[category].urls.insert(0, url)
self.by_categories[category].count += 1
if init_type == self.types.delete:
self.by_categories[category].count -= 1
self.by_categories[category].urls.remove(url)
if self.by_categories[category].count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self,init_type, url):
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget()
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
#TODO: FIXME: related entries
"""
urls, index = [], 0
try:
index = self.urls.index(url)
except:
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=[], url=None):
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and not url == None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
if about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
#TODO: FIXME: calculate tags' rank
"""
tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls)> 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
return sorted(self.by_categories.values(), key=lambda c:c.name)
def _init_archive_widget(self):
return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)
def _find_by_query(self, query, start, limit):
"""
#TODO: FIXME: how to search in the content of entries
"""
queries = [q.lower() for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
try:
entry.content.index(query)
urls.append(entry.url)
except:
pass
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
if urls == None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total,limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
if entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if not page== None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if not entry == None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self, archive_type, url, start=1, limit=999999999):
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url,config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
urls = [url for url in self.urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
return self.params
def search(self, search_type, url, value='', start=config.start, limit=config.limit):
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
if search_type == self.types.tag:
if self.by_tags.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
if search_type == self.types.category:
if self.by_categories.get(value) == None:
entries = None
else:
entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
if search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start, limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
return self.params
def error(self, url):
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
return self.params
class EntryService(object):
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
self.blog_bucket = BucketManager(self.qiniu)
self.blog_prefix = "raw/"
self._init_blog()
def _init_blog(self):
"""
Initialize blog
- all entries in entry_dir
- all pages in page_dir
- others
"""
cloud_list = {}
ret, _, info = self.blog_bucket.list(config.qiniu_bucket, prefix = "raw/")
file_list = eval(info.text_body)["items"]
for f in file_list:
if f["key"].startswith(self.blog_prefix) and f["mimeType"] == "text/markdown":
file_name = f["key"][len(self.blog_prefix):]
print "Find qiniu file %s" % file_name
cloud_list[file_name] = (f["key"], f["fsize"])
print cloud_list
for root, _, files in os.walk(config.entry_dir):
for f in files:
if f in cloud_list:
print "Find local file %s/%s" % (root, f)
del cloud_list[f]
self.add_entry(False, root + '/' + f)
for f in cloud_list:
info = cloud_list[f]
self.download_file(f, info[0])
print "Download file %s/%s" % (root,f)
self.add_entry(False, root + '/' + f)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
self._init_miscellaneous(self.types.add, self.entries.values())
def download_file(self, f, key):
base_url = 'http://%s/%s' % (config.qiniu_bucket_domain, key)
private_url = self.qiniu.private_download_url(base_url, expires=3600)
urllib.urlretrieve(private_url, "%s/%s" % (config.entry_dir, f))
print(private_url)
def add_entry(self, inotified, path):
"""
Add entry
"""
entry = self._init_entry(self.types.entry, path)
if entry is not None:
self.entries[entry.url] = entry
if inotified:
self._init_miscellaneous(self.types.add, [entry])
def delete_entry(self, path):
"""
Delete entry
"""
for entry in self.entries.values():
if path == os.path.abspath(entry.path):
print "delete success!"
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
"""
Add page
"""
page = self._init_entry(self.types.page, path)
if page is not None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path):
"""
initialize single entry
"""
url, raw_url, name, date, time, content = self._init_file(path, entry_type)
if url is not None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = raw_url
entry.date = date
entry.time = time
header, title, categories, tags = extract.parse(entry)
entry.content = content
content = content.replace(header, '')
entry.html = markdown.markdown(content)
# FIXME How to extract the excerpt of an entry
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
return entry
return None
def _init_file(self, file_path, entry_type):
"""
Initialize single file
"""
# FIXME: how to determine the publish time of an entry
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content is None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_', '-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
print traceback.format_exc()
print file_path
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date is None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = config.raw_url + '/' + prefix + '/'
elif entry_type == self.types.page:
url_prefix = '/'
raw_prefix = config.raw_url + '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def _init_miscellaneous(self, init_type, entries):
"""
Initialize miscellaneous
- tags
- categories
- archives
- all urls
"""
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.urls = sorted(self.entries.keys(), reverse=True)
self._init_params()
def _init_subscribe(self):
"""
Initialize subscriptions
"""
if not self.urls:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self, init_type, url, tags):
"""
Initialize tags
"""
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
"""
Initialize categories
"""
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_categories[category].urls.insert(0, url)
self.by_categories[category].count += 1
if init_type == self.types.delete:
self.by_categories[category].count -= 1
self.by_categories[category].urls.remove(url)
if self.by_categories[category].count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self, init_type, url):
"""
Initialize archives
"""
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
"""
Initialize global params
:return:
"""
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget()
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
Initialize related entries
"""
# FIXME: related entries
try:
index = self.urls.index(url)
except:
print traceback.format_exc()
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=None, url=None):
"""
Initialize abouts widget
:param about_types:
:param url:
:return:
"""
about_types = about_types or []
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and url is not None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
print traceback.format_exc()
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
elif about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
Initialize tags widget
"""
# FIXME: calculate tags' rank
tags = sorted(self.by_tags.values(), key=lambda v: v.count, reverse=True)
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
"""
Initialize recently entries widget
:return:
"""
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
"""
Initialize calender widget
:return:
"""
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls) > 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
"""
Initialize categories widget
:return:
"""
return sorted(self.by_categories.values(), key=lambda c: c.name)
def _init_archive_widget(self):
"""
Initialize archive widget
:return:
"""
return sorted(self.by_months.values(), key=lambda m: m.url, reverse=True)
def _find_by_query(self, query, start, limit):
"""
Find by query
:param query:
:param start:
:param limit:
:return:
"""
# FIXME: how to search in the content of entries
queries = [q.lower() for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
try:
entry.content.index(query)
urls.append(entry.url)
except:
print
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
"""
Find by page
:param urls:
:param start:
:param limit:
:return:
"""
if urls is None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
"""
Pagination
:param pager_type:
:param value:
:param total:
:param start:
:param limit:
:return:
"""
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total, limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
"""
Find content by url
:param entry_type:
:param url:
:return:
"""
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
elif entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
"""
Find the raw content by raw_url
:param raw_url:
:return:
"""
page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if page is not None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if entry is not None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self, archive_type, url, start=1, limit=999999999):
"""
Archives
:param archive_type:
:param url:
:param start:
:param limit:
:return:
"""
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url, config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
urls = [url for url in self.urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
return self.params
def search(self, search_type, url, value='', start=config.start, limit=config.limit):
"""
Search the site
:param search_type:
:param url:
:param value:
:param start:
:param limit:
:return:
"""
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
elif search_type == self.types.tag:
if self.by_tags.get(value) is None:
entries = None
else:
entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
elif search_type == self.types.category:
if self.by_categories.get(value) is None:
entries = None
else:
entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
elif search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start, limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
return self.params
def error(self, url):
"""
Error params
:param url:
:return:
"""
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
return self.params
from utils import Utils
from model import Models, AveragingModels
import pandas as pd
import numpy as np
if __name__ == '__main__':
utils = Utils()
models_ = Models()
X = utils.load_data('./csv/clean_train.csv')
test = utils.load_data('./csv/clean_test.csv')
y = utils.load_data('./csv/target.csv')
test_id = test['Id']
test.drop('Id', axis=1, inplace=True)
models = ['ELASTIC_NET', 'GRADIENT', 'LASSO', 'KERNEL_RIDGE', 'XGB']
base_models = []
for model in models:
base_model = models_.grid_training(X, y, model)
base_models.append(base_model)
averaged_models = AveragingModels(models=(base_models[0], base_models[1],
base_models[2], base_models[3]))
n_folds = 5
score = utils.rmsle_cv(averaged_models, n_folds, X, y)
print(" Averaged base models score: {:.4f} ({:.4f})\n".format(
score.mean(), score.std()))
class Ui_MainWindow(object):
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(331, 166)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
#define data model as attribute to optimize the Models request
self.model = Models()
self.label = QtWidgets.QLabel(self.centralwidget)
self.label.setGeometry(QtCore.QRect(10, 30, 47, 13))
self.label.setObjectName("label")
self.label_2 = QtWidgets.QLabel(self.centralwidget)
self.label_2.setGeometry(QtCore.QRect(10, 70, 47, 13))
self.label_2.setObjectName("label_2")
self.pushButton = QtWidgets.QPushButton(self.centralwidget)
self.pushButton.setGeometry(QtCore.QRect(60, 100, 75, 23))
self.pushButton.setObjectName("pushButton")
self.pushButton.clicked.connect(self.check_button)
self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
self.pushButton_2.setGeometry(QtCore.QRect(210, 100, 75, 23))
self.pushButton_2.setObjectName("pushButton_2")
# username field
self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
self.textEdit.setGeometry(QtCore.QRect(60, 20, 251, 25))
self.textEdit.setObjectName("textEdit")
# password field
self.textEdit_2 = QtWidgets.QTextEdit(self.centralwidget)
self.textEdit_2.setGeometry(QtCore.QRect(60, 60, 251, 25))
self.textEdit_2.setObjectName("textEdit_2")
MainWindow.setCentralWidget(self.centralwidget)
self.menubar = QtWidgets.QMenuBar(MainWindow)
self.menubar.setGeometry(QtCore.QRect(0, 0, 331, 21))
self.menubar.setObjectName("menubar")
MainWindow.setMenuBar(self.menubar)
self.statusbar = QtWidgets.QStatusBar(MainWindow)
self.statusbar.setObjectName("statusbar")
MainWindow.setStatusBar(self.statusbar)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def retranslateUi(self, MainWindow):
_translate = QtCore.QCoreApplication.translate
MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
self.label.setText(_translate("MainWindow", "username"))
self.label_2.setText(_translate("MainWindow", "password"))
self.pushButton.setText(_translate("MainWindow", "check"))
self.pushButton_2.setText(_translate("MainWindow", "exit"))
#controler of button
def check_button(self):
username = self.textEdit.toPlainText()
password = self.textEdit_2.toPlainText()
#print(username,password)
if self.model.get_user(username, password) != None:
#exist value
if self.model.get_user(username, password)[0] == 'admin':
#username is admin so go to admin page
print(self.model.get_user(username, password))
def post(self,task_id):
task = Models.get_by_id(int(task_id))
task.key.delete()
self.redirect_to("Task-List")
sentences_length = [len(sentence) for sentence in train_sentences]
mean_seq_length = np.mean(sentences_length)
max_seq_length = np.max(sentences_length)
print('mean_seq_length = {}, max_seq_length = {}'.format(
mean_seq_length, max_seq_length))
if add_keyword_attention:
keywords, label_key_words = process.extrac_keywords(
train_data, train_label, keyword_num)
for key, value in label_key_words.items():
print(key, ':', value)
keywords_id = [word_vocab[word] for word in list(chain(*keywords))]
else:
keywords_id = None
train_y = process.label_to_onehot(train_labels)
test_y = process.label_to_onehot(test_labels)
print('train size = {}, test size = {}'.format(len(train_sentences),
len(test_sentences)))
print('loading model ...')
models = Models(vocab_size, num_classes, num_features, word_emb_dim,
feature_emb_dim, hidden_dim, num_layers, learning_rate,
keep_prob, add_feature_emb, add_second_attention,
add_keyword_attention)
print('start training model ...')
models.lstm_model(train_sentences, train_y, test_sentences, test_y,
keywords_id, batch_size, num_epochs)
def my_train():
# 是否有GPU加速
use_gpu = torch.cuda.is_available()
dataloader = {
"train": load_data(data_dir_train, image_size, batch_size),
"valid": load_data(data_dir_valid, image_size, batch_size)
}
# 加载模型、损失、优化器等
model = Models(image_dim, n_classes)
loss_f = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
if use_gpu:
model = model.cuda()
for epoch in range(epochs):
print("Epoch {}/{}".format(epoch + 1, epochs))
print("-" * 10)
running_loss = 0.0
train_correct = 0
train_total = 0
# 计时
start_time = time.time()
for i, data in enumerate(dataloader["train"], 0):
train_images, train_labels = data
if use_gpu:
train_images, labels = Variable(train_images.cuda()), Variable(
train_labels.cuda())
else:
train_images, labels = Variable(train_images), Variable(
train_labels)
optimizer.zero_grad() # 梯度置零
outputs = model(train_images) # 模型输出
_, train_predicted = torch.max(outputs.data, 1) # 预测结果
train_correct += (train_predicted == labels.data).sum()
loss = loss_f(outputs, labels) # 计算损失
loss.backward() # 反向传播
optimizer.step() # 单步优化（调整学习率）
running_loss += loss.item()
train_total += train_labels.size(0)
now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
runtime = time.time() - start_time
print('now_time: %s, , time/epoch:%.2fs, time left: %.2fhours' %
(str(now_time), runtime, (epochs - epoch) * runtime / 3600))
print('train %d epoch loss: %.3f acc: %.3f ' %
(epoch + 1, running_loss / train_total,
100 * train_correct / train_total))
writer.add_scalar('Train/Loss', running_loss / train_total, epoch + 1)
writer.add_scalar('Train/Acc', 100 * train_correct / train_total,
epoch + 1)
torch.save(model, 'model.pt')
def __init__(self, model_name, device='CPU', extensions=None):
Models.__init__(self, model_name, device, extensions)
self.model_type = "HEAD"
from data_loader import loader from model import Models from keras.optimizers import Adam from sklearn.model_selection import train_test_split from keras.optimizers import Adam import numpy as np import sys load_data = loader() x_data, y_data, w = load_data.load() print(x_data[:3]) w = w + w - 1 #no. of features # print(w) h = 1 model = Models(h, w) auto_encoder = model.encode_decoder() model.encode_decoder() print(model.encode_decoder().summary()) print(x_data.shape) # train_x,test_x,train_y,test_y=train_test_split(x_data,y_data,test_size=0.1,random_state=30) train_x = x_data[:50] train_y = train_x.copy() test_x = x_data[50:] test_y = test_x.copy() model.fit(train_x, train_y, test_x, test_y) model.save()
import cv2
import numpy as np
from data_loader import loader
from model import Models
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
w = 240
h = 240
c = 3
mod = Models(w, h, c)
auto_encoder = mod.Arch2()
load_img = loader()
auto_encoder.summary()
x_data, y_data = load_img.load('stone', 'paper', 'scissor')
x_data = np.array(x_data, dtype='float') / 255.0
y_data = np.array(y_data, dtype='float') / 255.0
opt = Adam(lr=0.001, decay=0.001 / 50)
train_x, test_x, train_y, test_y = train_test_split(x_data,
y_data,
test_size=0.1,
random_state=30)
auto_encoder.compile(optimizer='adadelta', loss='binary_crossentropy')
auto_encoder.fit(train_x,
train_y,
batch_size=32,
shuffle='true',
epochs=15,
validation_data=(test_x, test_y),
verbose=1)
mean_seq_length, max_seq_length))
if add_keyword_attention:
keywords = text_data.extrac_keywords(keyword_num)
keywords_id = [word_vocab[word] for word in list(chain(*keywords))]
else:
keywords_id = None
train_x, train_y, test_x, test_y, mask_train, mask_test = text_data.data_split(
text_data=sentences,
seq_length=seq_length,
hidden_dim=hidden_dim,
test_rate=test_rate)
print(len(train_x), len(train_y), len(test_x), len(test_y), len(mask_train),
len(mask_test))
train_x = np.array(train_x)
train_y = np.array(train_y)
mask_train = np.array(mask_train)
test_x = np.array(test_x)
test_y = np.array(test_y)
mask_test = np.array(mask_test)
print('loading model ...')
models = Models(vocab_size, num_classes, word_emb_dim, feature_emb_dim,
add_feature, seq_length, hidden_dim, num_features, keep_prob,
add_keyword_attention)
print('start training model ...')
models.lstm_model(train_x, train_y, test_x, test_y, keywords_id, mask_train,
mask_test, num_layers, batch_size, learning_rate, num_epochs)