Exemple #1
0
def main():
    path = os.path.dirname(os.path.realpath(os.getcwd()))
    data_path = os.path.join(path, 'Data')
    years = [str(x) for x in range(2012, 2018)]
    seasons = [f"{x}-schedule-scores.shtml" for x in range(2012, 2018)]
    test_season = '2019-schedule-scores.shtml'
    team_url_csv = os.path.join(data_path, 'mlb_url.csv')
    MLBScraper = mlb_scrapping.MLBScraper(years[0], seasons[0], team_url_csv)
    games_df = MLBScraper()
    for i, year in enumerate(years[1:]):
        MLBScraper = mlb_scrapping.MLBScraper(year, seasons[i], team_url_csv)
        games_df.append(MLBScraper())
    print(list(games_df.columns))
    # building test_df
    MLBScraper = mlb_scrapping.MLBScraper("2019", test_season, team_url_csv)
    test_df = MLBScraper()
    test_df = test_df[test_df['team'] == 'ATL']
    test_preprocessor = preprocess.Preprocess(test_df)
    Xtest, ytest = test_preprocessor.split_xy('attendance')
    preprocessor = preprocess.Preprocess(games_df)
    X, y = preprocessor.split_xy('attendance')
    modeler = Models(X, y)
    rf = modeler.fit_random_forest()
    print(rf.score(X, y))
    print(list(rf.predict(Xtest) - ytest))
    print("Scoring for the 2019 braves season")
    rmse, r2 = modeler.test(rf, Xtest, ytest)
    print(f"RMSE : {rmse}")
    print(f'r2 : {r2}')
Exemple #2
0
    def build(self):
        md = Models(self.seq_length, self.featue_length)
        model = md.Attention_after_LSTM()

        plot_model(model,
                   to_file='Attention_after_LSTM_model.png',
                   show_shapes=True)

        return model
Exemple #3
0
 def __init__(self):
     self.entries = {}
     self.pages = {}
     self.urls = []
     self.by_tags = {}
     self.by_categories = {}
     self.by_months = {}
     self.models = Models()
     self.types = self.models.types()
     self.params = self.models.params()
     self._init_entries()
Exemple #4
0
    def post(self):
        object_area = self.request.get("object")
        date_area = self.request.get("date")
        time_area = self.request.get("time")
        message_area = self.request.get("message")

        user_input = Models(object=object_area,date=date_area,time=time_area,message=message_area)
        user_input.put()

        params = {"object":object_area,
                  "date":date_area,
                  "time":time_area,
                  "message":message_area}

        self.render_template("your-task.html",params=params)
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(331, 166)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")

        #define data model as attribute to optimize the Models request
        self.model = Models()

        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(10, 30, 47, 13))
        self.label.setObjectName("label")

        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(10, 70, 47, 13))
        self.label_2.setObjectName("label_2")

        self.pushButton = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton.setGeometry(QtCore.QRect(60, 100, 75, 23))
        self.pushButton.setObjectName("pushButton")
        self.pushButton.clicked.connect(self.check_button)

        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setGeometry(QtCore.QRect(210, 100, 75, 23))
        self.pushButton_2.setObjectName("pushButton_2")

        # username field
        self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit.setGeometry(QtCore.QRect(60, 20, 251, 25))
        self.textEdit.setObjectName("textEdit")

        # password field
        self.textEdit_2 = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit_2.setGeometry(QtCore.QRect(60, 60, 251, 25))
        self.textEdit_2.setObjectName("textEdit_2")

        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 331, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)
    def add_units(self, units):
        ModelManager.empty_model_list(ModelManager.MODEL_UNIT)

        for unit_object in units:
            unit = Models.Clip()
            unit.unit_name = unit_object['unit_name']
            unit.type = unit_object['type']
            unit.online = unit_object['online']
            ModelManager.register_model(unit, ModelManager.MODEL_UNIT)
Exemple #7
0
 def __init__(self):
     self.entries = {}
     self.pages = {}
     self.urls = []
     self.by_tags = {}
     self.by_categories = {}
     self.by_months = {}
     self.models = Models()
     self.types = self.models.types()
     self.params = self.models.params()
     self._init_entries()
Exemple #8
0
def main() :
    EPOCHS = 50

    Model = Models()

    generator = Model.make_generator_model()
    discriminator = Model.make_discriminator_model()

    train(Model = Model, epochs = EPOCHS, generator = generator, discriminator = discriminator)

    raw_noise = np.array(DC().preprocessed_set()[0], dtype = np.float32)
    raw_noise = make_data_noisy(raw_noise)
    predictions = np.array(generator(raw_noise, training = False))

    for prediction in predictions :
        cv2.imshow("predictions", prediction)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

    with open("prediction.pickle", "wb") as f :
        pickle.dump(predictions, f)
    def add_units(self, units):
        unit_list = []
        for unit_object in units:
            unit = Models.Unit()
            unit.unit_name = unit_object['unit_name']
            unit.type = unit_object['type']
            unit.online = unit_object['online']
            unit_list.append(unit)
            ModelManager.register_model(unit, ModelManager.MODEL_UNIT)
            self.get_eof_from_unit(unit.unit_name)

        self.find_all_existing_clips(unit_list)
Exemple #10
0
    def post(self,task_id):
        object = self.request.get("object")
        date = self.request.get("date")
        time = self.request.get("time")
        message = self.request.get("message")
        task = Models.get_by_id(int(task_id))
        task.object = object
        task.date = date
        task.time = time
        task.message = message
        task.put()

        self.redirect_to("Task-List")
Exemple #11
0
 def __init__(self):
     self.entries = {}
     self.pages = {}
     self.urls = []
     self.by_tags = {}
     self.by_categories = {}
     self.by_months = {}
     self.models = Models()
     self.types = self.models.types()
     self.params = self.models.params()
     
     self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
     self.blog_bucket = BucketManager(self.qiniu)
     self.blog_prefix = "raw/"
     self._init_blog()
Exemple #12
0
    def __init__(self, n_agents, dim_obs, dim_act, batch_size,
                 capacity, episodes_before_train, load_models=None):
        # self.actors = [Actor(dim_obs, dim_act) for i in range(n_agents)]
        # self.critics = [Critic(n_agents, dim_obs, dim_act) for i in range(n_agents)]

        if load_models is None:
            self.models = Models(n_agents, dim_obs, dim_act)
            self.actors_target = deepcopy(self.models.actors)
            self.critics_target = deepcopy(self.models.critics)
            self.critic_optimizer = [Adam(x.parameters(), lr=0.0001) for x in self.models.critics]  # 0.001
            self.actor_optimizer = [Adam(x.parameters(), lr=0.00001) for x in self.models.actors]   # 0.0001
            self.memory = ReplayMemory(capacity)
            self.var = [1.0 for i in range(n_agents)]
        else:
            print('Start loading models!')
            states = th.load(load_models)
            self.models = states['models']
            self.critic_optimizer = states['critic_optimizer']
            self.actor_optimizer = states['actor_optimizer']
            self.critics_target = states['critics_target']
            self.actors_target = states['actors_target']
            self.memory = states['memory']
            self.var = states['var']
            print('Models loaded!')

        self.n_agents = n_agents
        self.n_states = dim_obs
        self.n_actions = dim_act
        self.batch_size = batch_size
        self.use_cuda = th.cuda.is_available()
        self.episodes_before_train = episodes_before_train

        self.GAMMA = 0.95
        self.tau = 0.01

        if self.use_cuda:
            for x in self.models.actors:
                x.cuda()
            for x in self.models.critics:
                x.cuda()
            for x in self.actors_target:
                x.cuda()
            for x in self.critics_target:
                x.cuda()

        self.steps_done = 0
        self.episode_done = 0
    def add_clips(self, clips, unit_name):
        unit = self.pending_unit_processing.pop(0)

        for clip_object in clips:
            clip = Models.Clip()
            clip.unit = unit.unit_name
            clip.index = clip_object['index']
            clip.path = clip_object['path']
            clip.clip_in = clip_object['clip_in']
            clip.clip_out = clip_object['clip_out']
            clip.length = clip_object['length']
            clip.calculated_length = clip_object['calculated_length']
            clip.fps = clip_object['fps']

            ModelManager.register_model(clip, ModelManager.MODEL_CLIP)

        if len(self.pending_unit_processing) > 0:
            self.melted_telnet_controller.get_unit_clips(
                self.pending_unit_processing[0].unit_name, self.add_clips)
        else:
            self.loaded_callback()
    def add_clips(self, clips, unit_name):

        clip_models = ModelManager.get_models(ModelManager.MODEL_CLIP)
        for clip in clip_models:
            if clip.unit == unit_name:
                ModelManager.remove_model(clip, ModelManager.MODEL_CLIP)

        for clip_object in clips:
            clip = Models.Clip()
            clip.unit = unit_name
            clip.index = clip_object['index']
            clip.path = clip_object['path']
            clip.clip_in = clip_object['clip_in']
            clip.clip_out = clip_object['clip_out']
            clip.length = clip_object['length']
            clip.calculated_length = clip_object['calculated_length']
            clip.fps = clip_object['fps']

            ModelManager.register_model(clip, ModelManager.MODEL_CLIP)

        if len(clips) == 0:
            self.main_controller.get_main_interface_controller().refresh_clips(
            )
Exemple #15
0
HIDDEN_SIZE = 32

if __name__ == '__main__':
    loader = DataProcess()
    train_x, test_x, train_y, test_y = loader.get_data()
    train_x_content = train_x[0]
    train_x_summary = train_x[1]
    test_x_content = test_x[0]
    test_x_summary = test_x[1]

    raw_label = loader.get_raw_label()

    num_words = loader.get_token_nums()
    embedding_matrix = loader.get_embedding_matrix()

    m = Models(NUM_CLASSES, num_words, MAX_SEQ_LEN_CONTENT,
               MAX_SEQ_LEN_SUMMARY, EMBEDDING_SIZE, embedding_matrix)
    model = m.multi_channel_model(NUM_FILTERS, FILTER_SIZE, HIDDEN_SIZE)

    model.compile(loss='categorical_crossentropy',
                  optimizer=Adam(0.001),
                  metrics=['accuracy'])

    learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
                                                patience=2,
                                                verbose=1,
                                                factor=0.5,
                                                min_lr=1e-5)
    history = model.fit(x=[train_x_content, train_x_summary],
                        y=train_y,
                        batch_size=BATCH_SIZE,
                        validation_data=([test_x_content,
import os

filename = "LoanPrediction.csv"           #change filename
dname = os.getcwd() + '/Dataset'
filename = os.path.join(dname,filename)
data=pd.read_csv(filename)

target_column = "Loan_Status"                         #change target_column
y = data[target_column][0:]

cols=list(data.columns)
cols.remove(target_column)
X = data[cols][0:]

X_train,X_test,y_train,y_test = make_train_test_split(X,y)
model = Models(X_train,X_test,y_train,y_test)


#print(model.VanillaLinearRegression())
#print(model.LassoLinearRegression())
#print(model.RidgeLinearRegression())
#print(model.RandomForestRegressor())
#print(model.GradientBoostedRegressor())
#print(model.SupportVectorRegressor())

print(model.VanillaLogisticRegression())
print(model.RandomForestClassifier())
print(model.GradientBoostedClassifier())
print(model.SupportVectorClassifier())

#model.XtremeGradientBoosting()
Exemple #17
0
 def get(self):
     list = Models.query().fetch()
     params = {"list":list}
     self.render_template("task-list.html",params=params)
from model import Models

# Height, Width
field = (11,6)
# Frames which we propagate
nb_frames = 1 
# Number of possible actions
nb_actions = 3


# Batch size
batch_size = 64
# Discounter
gamma = .95
# Number of games
number_games = 250


# Initiate Tetris
t = Tetris(field = field)
# Get model
models = Models(nb_frames, field[0], field[1], nb_actions)
model = models.get_model()


# Agent things
agent = Agent(t, model, nb_actions, field, nb_epoch = number_games, gamma=gamma, batch_size=batch_size)
agent.train()
agent.plot_ma_score()
agent.play(t, nb_epoch = 1)
Exemple #19
0
    def __init__(self, model_name, device='MYRIAD', extensions=None):

        Models.__init__(self, model_name, device, extensions)
Exemple #20
0
class EntryService:
    """EntryService."""
    def __init__(self):
        self.entries = {}
        self.pages = {}
        self.urls = []
        self.by_tags = {}
        self.by_categories = {}
        self.by_months = {}
        self.models = Models()
        self.types = self.models.types()
        self.params = self.models.params()
        self.private_list = []
        self.all_urls = []
        self._init_entries()

    def get_tag_url(self, tag):
        url = '/search?type=tag&value=%s&start=1&limit=5' % urllib.quote(tag)
        return url

    def add_private(self, path):
        self.private_list.append(path)
        self.save_private()

    def del_private(self, path):
        self.private_list.append(path)
        try:
            self.private_list.remove(path)
        except:
            pass
        self.save_private()

    def save_private(self):
        private_path = 'raw/' + config.private_store
        with open(private_path, "w") as _file:
            pickle.dump(self.private_list, _file)
            _file.close()

    def _init_entries(self):
        private_path = 'raw/' + config.private_store
        private_list = []
        if os.path.exists(private_path):
            with open(private_path, 'r') as _file:
                private_list = pickle.load(_file)
        self.private_list = private_list
        print 'private list', self.private_list
        for root, _, files in os.walk(config.entry_dir):
            for f in files:
                _path = root + '/' + f
                self.add_entry(False, _path, _path in private_list)
        for root, _, files in os.walk(config.page_dir):
            for f in files:
                self._add_page(root + '/' + f)
        public_entries = [e for e in self.entries.values() if not e.private]
        self._init_miscellaneous(self.types.add, public_entries)

    def add_entry(self, inotified, path, private=False):
        entry = self._init_entry(self.types.entry, path, private)
        if not entry == None:
            self.entries[entry.url] = entry
            if inotified:
                if private:
                    self.update_urls()
                    self.add_private(path)
                else:
                    self._init_miscellaneous(self.types.add, [entry])
            else:
                self.update_urls()

    def update_entry(self, entry, data):
        tar = self.entries[entry.url]
        if 'title' in data:
            tar.name = data.get('title')
            #entry.name = data.get('title')
        old_private = entry.private
        new_private = bool(data.get('private', ''))
        changed = False

        if 'tags' in data:
            old_tags = entry.tags
            new_tags = data.get('tags')
            if not old_private:
                self._init_tag(self.types.delete, entry.url, old_tags)
            if not new_private:
                self._init_tag(self.types.add, entry.url, new_tags)
            tar.tags = new_tags
            if old_tags != new_tags: changed = True
        if 'cats' in data:
            old_cats = entry.categories
            new_cats = data.get('cats')
            if not old_private:
                self._init_category(self.types.delete, entry.url, old_cats)
            if not new_private:
                self._init_category(self.types.add, entry.url, new_cats)
            tar.categories = new_cats
            if old_cats != new_cats: changed = True
        tar.private = new_private
        if old_private != new_private:
            self.update_urls()
            if new_private:
                self.add_private(entry.path)
            else:
                self.del_private(entry.path)
            changed = True

        if changed:
            self._init_params()

    def delete_entry(self, path):
        for entry in self.entries.values():
            if path == entry.path:
                self.entries.pop(entry.url)
                self._init_miscellaneous(self.types.delete, [entry])

    def _add_page(self, path):
        page = self._init_entry(self.types.page, path)
        if not page == None:
            self.pages[page.url] = page

    def _init_entry(self, entry_type, path, private=False):
        '''
        read infomation from md file
        '''
        url, raw_url, name, date, time, content = self._init_file(
            path, entry_type)
        if not url == None:
            entry = self.models.entry(entry_type)
            entry.path = path
            entry.name = name
            entry.url = url
            entry.raw_url = config.raw_url + raw_url
            entry.update_url = config.update_url + raw_url
            entry.date = date
            entry.time = time
            entry.private = private
            #header, title, categories, tags = extract.parse(entry)
            print 'parse', entry.path
            with open(entry.path, 'r') as f:
                start = f.readline()
                if '---' in start.strip():
                    layout_raw = f.readline()
                    title_raw = f.readline()
                    category_raw = f.readline()
                    tags_raw = f.readline()
                    #author_raw = f.readline()
                    end = f.readline()
                    content = f.read().strip()
                    #print layout_raw,title_raw,category_raw,tags_raw
                    title = title_raw.split(':')[1].strip()
                    categories_str = category_raw.split(':')[1].strip()
                    categories = categories_str and categories_str.split(
                        ',') or []
                    tags_str = tags_raw.split(':')[1].strip()[1:-1]
                    tags = tags_str and tags_str.split(',') or []
                    header = ''.join([
                        start, layout_raw, title_raw, category_raw, tags_raw,
                        end
                    ])
                else:
                    title, categories, tags, header = '', [], [], ''

            if title:
                entry.name = title
            #content = content.replace(header.decode('utf-8'), '')
            try:
                content = content.decode('utf-8')
            except:
                pass
            entry.content = content
            entry.header = header
            if config.backend_md:
                import markdown
                entry.html = markdown.markdown(
                    content,
                    extensions=[
                        'markdown.extensions.codehilite',
                        'markdown.extensions.toc',
                        'markdown.extensions.fenced_code',
                        'markdown.extensions.tables'
                    ])
            else:
                entry.html = ''
            entry.excerpt = content[:200] + ' ... ...'
            entry.categories = categories
            entry.tags = tags
            if len(tags) > 0:
                first_tag = tags[0]
                if first_tag.startswith('__'):
                    entry.author.name = first_tag[2:]
                    entry.author.url = self.get_tag_url(first_tag)
            return entry
        return None

    def _init_file(self, file_path, entry_type):
        """
        #TODO: FIXME: how to determine the publish time of an entry
        """
        content, nones = None, [None for _ in xrange(6)]
        try:
            content = codecs.open(file_path, mode='r', encoding='utf-8').read()
        except:
            return nones
        if content == None or len(content.strip()) == 0:
            return nones
        date, mtime = None, None
        name, _ = os.path.splitext(os.path.basename(file_path))
        chars = ['_', '-', '~']
        pattern = r'\d{4}-\d{1,2}-\d{1,2}'
        match = re.search(pattern, name)
        if match:
            y, m, d = match.group().split('-')
            try:
                date = datetime.date(int(y), int(m), int(d))
            except:
                pass
            name = name[len(match.group()):]
            for c in chars:
                if name.startswith(c):
                    name = name[1:]
        stat = os.stat(file_path)
        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
        if date == None:
            date = mtime
        prefix, url_prefix, raw_prefix = date.strftime(
            config.url_date_fmt), '', ''
        if entry_type == self.types.entry:
            url_prefix = config.entry_url + '/' + prefix + '/'
            raw_prefix = '/' + prefix + '/'
        if entry_type == self.types.page:
            url_prefix = '/'
            raw_prefix = '/'
        date = date.strftime(config.date_fmt)
        time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
        url = url_prefix + name + config.url_suffix
        raw_url = raw_prefix + name + config.raw_suffix
        for c in chars:
            name = name.replace(c, ' ')
        return url, raw_url, name, date, time, content

    def update_urls(self):
        '''
        public url noupdate
        '''
        _list = []
        _pub_list = []
        for url, entry in self.entries.items():
            _list.append(url)
            if not entry.private:
                _pub_list.append(url)
        self.urls = sorted(_pub_list, reverse=True)
        self.all_urls = sorted(_list, reverse=True)

    def _init_miscellaneous(self, init_type, entries):
        '''
        1. rebuild index data of the 'entries'
        2. refresh the url data
        3. refresh the right part of site page
        '''
        for entry in entries:
            self._init_tag(init_type, entry.url, entry.tags)
            self._init_category(init_type, entry.url, entry.categories)
            self._init_monthly_archive(init_type, entry.url)
        self.update_urls()
        self._init_params()

    def _init_subscribe(self):
        '''
        refresh the subscribe last update time
        '''
        time = None
        if self.urls == []:
            time = datetime.datetime.now().strftime(config.time_fmt)
        else:
            time = self.entries[self.urls[0]].time
        return self.models.subscribe(time)

    def _init_tag(self, init_type, url, tags):
        for tag in tags:
            if tag not in self.by_tags:
                if init_type == self.types.add:
                    self.by_tags[tag] = self.models.tag(tag, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    if url not in self.by_tags[tag].urls:
                        self.by_tags[tag].urls.insert(0, url)
                        self.by_tags[tag].count += 1
                if init_type == self.types.delete:
                    self.by_tags[tag].count -= 1
                    self.by_tags[tag].urls.remove(url)
                    if self.by_tags[tag].count == 0:
                        self.by_tags.pop(tag)

    def _init_category(self, init_type, url, categories):
        for category in categories:
            if category not in self.by_categories:
                if init_type == self.types.add:
                    self.by_categories[category] = \
                    self.models.category(category, url)
                if init_type == self.types.delete:
                    pass
            else:
                m_category = self.by_categories[category]
                if init_type == self.types.add:
                    if url not in m_category.urls:
                        m_category.urls.insert(0, url)
                        m_category.count += 1
                if init_type == self.types.delete:
                    m_category.count -= 1
                    m_category.urls.remove(url)
                    if m_category.count == 0:
                        self.by_categories.pop(category)

    def _init_monthly_archive(self, init_type, url):
        start = len(config.entry_url) + 1
        end = start + len('/yyyy/mm')
        month = url[start:end]
        if month not in self.by_months:
            if init_type == self.types.add:
                self.by_months[month] = \
                self.models.monthly_archive(self.types.entry, month, url)
            if init_type == self.types.delete:
                pass
        else:
            if init_type == self.types.add:
                if url not in self.by_months[month].urls:
                    self.by_months[month].urls.insert(0, url)
                    self.by_months[month].count += 1
            else:
                self.by_months[month].count -= 1
                self.by_months[month].urls.remove(url)
                if self.by_months[month].count == 0:
                    self.by_months.pop(month)

    def _init_params(self):
        '''
        refresh the right part of site page
        '''
        self.params.subscribe = self._init_subscribe()
        self.params.primary.tags = self._init_tags_widget()
        self.params.primary.recently_entries = self._init_recently_entries_widget(
        )
        self.params.secondary.categories = self._init_categories_widget()
        self.params.secondary.calendar = self._init_calendar_widget()
        self.params.secondary.archive = self._init_archive_widget()

    def _init_related_entries(self, url):
        """
        #TODO: FIXME: related entries
        """
        urls, index = [], 0
        try:
            index = self.urls.index(url)
        except:
            return None
        urls = self.urls[:index]
        urls.extend(self.urls[index + 1:])
        urls = random.sample(urls, min(len(urls), 10))
        return [self.entries.get(url) for url in sorted(urls, reverse=True)]

    def _init_abouts_widget(self, about_types=[], url=None):
        abouts = []
        for about_type in about_types:
            about = self.models.about(about_type)
            if about_type == self.types.entry and not url == None:
                try:
                    i = self.urls.index(url)
                    p, n = i + 1, i - 1
                except:
                    p, n = 999999999, -1
                if p < len(self.urls):
                    url = self.urls[p]
                    about.prev_url = url
                    about.prev_name = self.entries[url].name
                if n >= 0:
                    url = self.urls[n]
                    about.next_url = url
                    about.next_name = self.entries[url].name
            if about_type == self.types.archive:
                about.prev_url = '/'
                about.prev_name = 'main index'
            if about_type == self.types.blog:
                about.prev_url = '/'
                about.prev_name = 'main  index'
                about.next_url = config.archive_url
                about.next_name = 'archives'
            abouts.append(about)
        return abouts

    def _init_tags_widget(self):
        """
        #TODO: FIXME: calculate tags' rank
        """
        tags = sorted(self.by_tags.values(),
                      key=lambda v: v.count,
                      reverse=True)
        tags = [t for t in tags if not t.name.startswith('__')]
        ranks = config.ranks
        div, mod = divmod(len(tags), ranks)
        if div == 0:
            ranks, div = mod, 1
        for r in range(ranks):
            s, e = r * div, (r + 1) * div
            for tag in tags[s:e]:
                tag.rank = r + 1
        return tags

    def _init_recently_entries_widget(self):
        return [self.entries[url] for url in self.urls[:config.recently]]

    def _init_calendar_widget(self):
        date = datetime.datetime.today().strftime(config.date_fmt)
        if len(self.urls) > 0:
            date = self.entries[self.urls[0]].date
        calendar = self.models.calendar(date)
        y, m = calendar.month.split('-')
        for url in self.urls:
            _, _, _, _, d, _ = url.split('/')
            prefix = config.entry_url + '/' + y + '/' + m + '/' + d
            d = int(d)
            if url.startswith(prefix):
                calendar.counts[d] += 1
                if calendar.counts[d] > 1:
                    start = len(config.entry_url)
                    end = start + len('/yyyy/mm/dd')
                    calendar.urls[d] = config.archive_url + url[start:end]
                else:
                    calendar.urls[d] = url
            else:
                break
        return calendar

    def _init_categories_widget(self):
        return sorted(self.by_categories.values(), key=lambda c: c.name)

    def _init_archive_widget(self):
        return sorted(self.by_months.values(),
                      key=lambda m: m.url,
                      reverse=True)

    def _find_by_query(self, query, start, limit):
        """
        #TODO: FIXME: how to search in the content of entries
        """
        queries = [q.decode('utf-8') for q in query.split(' ')]
        urls = []
        for query in queries:
            for entry in self.entries.values():
                if entry.private:
                    continue
                try:
                    entry.content.index(query)
                    urls.append(entry.url)
                except:
                    pass
        return self._find_by_page(sorted(urls), start, limit)

    def _find_by_page(self, urls, start, limit):
        if urls == None or start < 0 or limit <= 0:
            return [], 0
        total = len(urls)
        urls = sorted(urls, reverse=True)
        s, e = (start - 1) * limit, start * limit
        if s > total or s < 0:
            return [], 0
        return [self.entries[url] for url in urls[s:e]], total

    def _paginate(self, pager_type, value, total, start, limit):
        if limit <= 0:
            return self.models.pager(pager_type, value, total, 0, start, limit)
        pages, mod = divmod(total, limit)
        if mod > 0:
            pages += 1
        return self.models.pager(pager_type, value, total, pages, start, limit)

    def find_by_url(self, entry_type, url):
        entry, abouts = None, [self.types.blog]
        if entry_type == self.types.entry:
            entry = self.entries.get(url)
            abouts.insert(0, self.types.entry)
        if entry_type == self.types.page:
            entry = self.pages.get(url)
        self.params.entry = entry
        self.params.entries = self._init_related_entries(url)
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(abouts, url)
        return self.params

    def find_raw(self, raw_url):
        page_url = raw_url.replace(config.raw_url,
                                   '').replace(config.raw_suffix,
                                               config.url_suffix)
        page = self.find_by_url(self.types.page, page_url).entry
        if not page == None and page.raw_url == raw_url:
            return page.content
        entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(
            config.raw_suffix, config.url_suffix)
        entry = self.find_by_url(self.types.entry, entry_url).entry
        if not entry == None and entry.raw_url == raw_url:
            return entry.content
        return None

    def archive(self,
                archive_type,
                url,
                start=1,
                limit=999999999,
                private=False):
        self.params.error = self.models.error(url=url)

        if archive_type == self.types.raw:
            url = url.replace(config.raw_url, config.archive_url)

        entries, count, = [], 0
        archive_url = url.replace(config.archive_url, '').strip('/')
        prefix = url.replace(config.archive_url, config.entry_url)
        pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
        match = re.search(pattern, archive_url)
        if match and match.group() == archive_url or archive_url == '':
            _urls = self.urls if private == False else [
                e for e in self.all_urls if e not in self.urls
            ]
            urls = [url for url in _urls if url.startswith(prefix)]
            entries, _ = self._find_by_page(urls, start, limit)
            count = len(entries)
        else:
            entries = None
        if archive_url == '':
            archive_url = self.types.all

        self.params.entries = entries
        self.params.archive = self.models.archive(archive_type, url,
                                                  archive_url, url, count)
        self.params.primary.abouts = self._init_abouts_widget(
            [self.types.archive])
        return self.params

    def search(self,
               search_type,
               url,
               value='',
               start=config.start,
               limit=config.limit):
        entries, total, abouts = None, 0, [self.types.blog]
        if search_type == self.types.query:
            entries, total = self._find_by_query(value, start, limit)
        if search_type == self.types.tag:
            if self.by_tags.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(
                    self.by_tags.get(value).urls, start, limit)
        if search_type == self.types.category:
            if self.by_categories.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(
                    self.by_categories.get(value).urls, start, limit)
        if search_type == self.types.index:
            entries, total = self._find_by_page(self.urls, start, limit)
            abouts = []
        self.params.error = self.models.error(url=url)
        self.params.entries = entries
        self.params.search = self.models.search(search_type, value, total)
        self.params.pager = self._paginate(search_type, value, total, start,
                                           limit)
        self.params.primary.abouts = self._init_abouts_widget(abouts)
        self.params.start = start
        self.params.limit = limit
        return self.params

    def error(self, url):
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(
            [self.types.blog])
        return self.params
def run():

    # read data
    train_loader, val_loader = data(args.train_data, 
                                    args.valid_data,
                                    args.batch_size, 
                                    args.workers)

    # set up model -- without imagenet pre-training
    model = Models(args.arch, 2, gpus).Model()
    cudnn.benchmark = True

    # resume pretrain model
    if args.resume_path is not None:
        pretrained_model = torch.load(args.resume_path)
        model.load_state_dict(pretrained_model['state_dict'])
        best_prec1 = pretrained_model['best_prec1']
        print('Load resume model done.')
    else:
        best_prec1 = 0
    print('Best top-1: {:.4f}'.format(best_prec1))

    # optimizer
    if args.optim == 'SGD':
        optimizer = torch.optim.SGD(model.parameters(), 
                                    args.lr,
                                    momentum=args.momentum,
                                    weight_decay=args.weight_decay)
    elif args.optim == 'Adam':
        optimizer = torch.optim.Adam(model.parameters(), 
                                     args.lr,
                                     weight_decay=args.weight_decay)

    # loss function
    criterion = nn.CrossEntropyLoss().cuda()

    # learning rate optimal
    lr_opt = lambda lr, epoch: lr * (0.1 ** (float(epoch) / 20))

    model_path = args.model_path
    def save_checkpoint(state, is_best, filename='checkpoint'):
        torch.save(state, os.path.join(model_path, filename + '_latest.pth.tar'))
        if is_best:
            shutil.copyfile(os.path.join(model_path, filename + '_latest.pth.tar'),
                            os.path.join(model_path, filename + '_best.pth.tar'))

    # start 
    for epoch in range(args.start_epoch, args.epochs):
        lr_cur = lr_opt(args.lr, epoch)  # speed change
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr_cur

        # train for one epoch
        train(train_loader, model, criterion, optimizer, epoch, lr_cur)

        # evaluate on validation set
        prec1 = validate(val_loader, model, criterion)

        # remember best prec1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint({
            'epoch': epoch + 1,
            'arch': args.arch,
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
        }, is_best, args.arch)
Exemple #22
0
class EntryService:
    """EntryService."""

    def __init__(self):
        self.entries = {}
        self.pages = {}
        self.urls = []
        self.by_tags = {}
        self.by_categories = {}
        self.by_months = {}
        self.models = Models()
        self.types = self.models.types()
        self.params = self.models.params()
        self.private_list = []
        self.all_urls = []
        self._init_entries()
        
    def get_tag_url(self, tag):
        url = '/search?type=tag&value=%s&start=1&limit=5'%urllib.quote(tag)
        return url
        
    def add_private(self, path):
        self.private_list.append(path)
        self.save_private()
        
    def del_private(self, path):
        self.private_list.append(path)
        try:
            self.private_list.remove(path)
        except:pass
        self.save_private()
        
    def save_private(self):
        private_path = 'raw/' + config.private_store
        with open(private_path, "w") as _file:
            pickle.dump(self.private_list, _file)
            _file.close()

    def _init_entries(self):
        private_path = 'raw/' + config.private_store
        private_list = []
        if os.path.exists(private_path):
            with open(private_path, 'r') as _file:
                private_list = pickle.load(_file)
        self.private_list = private_list
        print 'private list',self.private_list
        for root, _, files in os.walk(config.entry_dir):
            for f in files:
                _path = root + '/' + f
                self.add_entry(False, _path, _path in private_list)
        for root, _, files in os.walk(config.page_dir):
            for f in files:
                self._add_page(root + '/' + f)
        public_entries = [e for e in self.entries.values() if not e.private]
        self._init_miscellaneous(self.types.add, public_entries)

    def add_entry(self, inotified, path, private=False):
        entry = self._init_entry(self.types.entry, path, private)
        if not entry == None:
            self.entries[entry.url] = entry
            if inotified:
                if private:
                    self.update_urls()
                    self.add_private(path)
                else:
                    self._init_miscellaneous(self.types.add, [entry])
            else:
                self.update_urls()
                
    def update_entry(self, entry, data):
        tar = self.entries[entry.url]
        if 'title' in data:
            tar.name = data.get('title')
            #entry.name = data.get('title')
        old_private = entry.private
        new_private = bool(data.get('private', ''))
        changed = False
        
        if 'tags' in data:
            old_tags = entry.tags
            new_tags = data.get('tags')
            if not old_private:
                self._init_tag(self.types.delete, entry.url, old_tags)
            if not new_private:
                self._init_tag(self.types.add, entry.url, new_tags)
            tar.tags = new_tags
            if old_tags!=new_tags:changed=True
        if 'cats' in data:
            old_cats = entry.categories
            new_cats = data.get('cats')
            if not old_private:
                self._init_category(self.types.delete, entry.url, old_cats)
            if not new_private:
                self._init_category(self.types.add, entry.url, new_cats)
            tar.categories = new_cats
            if old_cats!=new_cats:changed = True
        tar.private = new_private
        if old_private!=new_private:
            self.update_urls()
            if new_private:
                self.add_private(entry.path)
            else:
                self.del_private(entry.path)
            changed = True
        
        if changed:
            self._init_params()

    def delete_entry(self, path):
        for entry in self.entries.values():
            if path == entry.path:
                self.entries.pop(entry.url)
                self._init_miscellaneous(self.types.delete, [entry])

    def _add_page(self, path):
        page = self._init_entry(self.types.page, path)
        if not page == None:
            self.pages[page.url] = page

    def _init_entry(self, entry_type, path, private=False):
        '''
        read infomation from md file
        '''
        url, raw_url, name, date, time, content =  self._init_file(path, entry_type)
        if not url == None:
            entry = self.models.entry(entry_type)
            entry.path = path
            entry.name = name
            entry.url = url
            entry.raw_url = config.raw_url + raw_url
            entry.update_url = config.update_url + raw_url
            entry.date = date
            entry.time = time
            entry.private = private
            #header, title, categories, tags = extract.parse(entry)
            print 'parse',entry.path
            with open(entry.path, 'r') as f:
                start = f.readline()
                if '---' in start.strip():
                    layout_raw = f.readline()
                    title_raw = f.readline()
                    category_raw = f.readline()
                    tags_raw = f.readline()
                    #author_raw = f.readline()
                    end = f.readline()
                    content = f.read().strip()
                    #print layout_raw,title_raw,category_raw,tags_raw
                    title = title_raw.split(':')[1].strip()
                    categories_str = category_raw.split(':')[1].strip()
                    categories = categories_str and categories_str.split(',') or []
                    tags_str = tags_raw.split(':')[1].strip()[1:-1]
                    tags = tags_str and tags_str.split(',') or []
                    header = ''.join([start, layout_raw, title_raw, category_raw, tags_raw, end])
                else:
                    title, categories, tags, header = '', [], [], ''
                
            if title:
                entry.name = title
            #content = content.replace(header.decode('utf-8'), '')
            try:
                content = content.decode('utf-8')
            except:pass
            entry.content = content
            entry.header = header
            if config.backend_md:
                import markdown
                entry.html = markdown.markdown(content,extensions=['markdown.extensions.codehilite','markdown.extensions.toc','markdown.extensions.fenced_code','markdown.extensions.tables'])
            else:
                entry.html = ''
            entry.excerpt = content[:200] + ' ... ...'
            entry.categories = categories
            entry.tags = tags
            if len(tags)>0:
                first_tag = tags[0]
                if first_tag.startswith('__'):
                    entry.author.name = first_tag[2:]
                    entry.author.url = self.get_tag_url(first_tag)
            return entry
        return None

    def _init_file(self, file_path, entry_type):
        """
        #TODO: FIXME: how to determine the publish time of an entry
        """
        content, nones = None, [None for _ in xrange(6)]
        try:
            content = codecs.open(file_path, mode='r', encoding='utf-8').read()
        except:
            return nones
        if content == None or len(content.strip()) == 0:
            return nones
        date, mtime = None, None
        name, _ = os.path.splitext(os.path.basename(file_path))
        chars = ['_' ,'-', '~']
        pattern = r'\d{4}-\d{1,2}-\d{1,2}'
        match = re.search(pattern, name)
        if match:
            y, m, d = match.group().split('-')
            try:
                date = datetime.date(int(y), int(m), int(d))
            except:
                pass
            name = name[len(match.group()):]
            for c in chars:
                if name.startswith(c):
                    name = name[1:]
        stat = os.stat(file_path)
        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
        if date == None:
            date = mtime
        prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
        if entry_type == self.types.entry:
            url_prefix = config.entry_url + '/' + prefix + '/'
            raw_prefix = '/' + prefix + '/'
        if entry_type == self.types.page:
            url_prefix = '/'
            raw_prefix = '/'
        date = date.strftime(config.date_fmt)
        time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
        url = url_prefix + name + config.url_suffix
        raw_url = raw_prefix + name + config.raw_suffix
        for c in chars:
            name = name.replace(c, ' ')
        return url, raw_url, name, date, time, content
    
    def update_urls(self):
        '''
        public url noupdate
        '''
        _list = []
        _pub_list = []
        for url, entry in self.entries.items():
            _list.append(url)
            if not entry.private:
                _pub_list.append(url)
        self.urls = sorted(_pub_list, reverse=True)
        self.all_urls = sorted(_list, reverse=True)

    def _init_miscellaneous(self,init_type, entries):
        '''
        1. rebuild index data of the 'entries'
        2. refresh the url data
        3. refresh the right part of site page
        '''
        for entry in entries:
            self._init_tag(init_type, entry.url, entry.tags)
            self._init_category(init_type, entry.url, entry.categories)
            self._init_monthly_archive(init_type, entry.url)
        self.update_urls()
        self._init_params()

    def _init_subscribe(self):
        '''
        refresh the subscribe last update time
        '''
        time = None
        if self.urls == []:
            time = datetime.datetime.now().strftime(config.time_fmt)
        else:
            time = self.entries[self.urls[0]].time
        return self.models.subscribe(time)

    def _init_tag(self,init_type, url, tags):
        for tag in tags:
            if tag not in self.by_tags:
                if init_type == self.types.add:
                    self.by_tags[tag] = self.models.tag(tag, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    if url not in self.by_tags[tag].urls:
                        self.by_tags[tag].urls.insert(0, url)
                        self.by_tags[tag].count += 1
                if init_type == self.types.delete:
                    self.by_tags[tag].count -= 1
                    self.by_tags[tag].urls.remove(url)
                    if self.by_tags[tag].count == 0:
                        self.by_tags.pop(tag)

    def _init_category(self, init_type, url, categories):
        for category in categories:
            if category not in self.by_categories:
                if init_type == self.types.add:
                    self.by_categories[category] = \
                    self.models.category(category, url)
                if init_type == self.types.delete:
                    pass
            else:
                m_category = self.by_categories[category]
                if init_type == self.types.add:
                    if url not in m_category.urls:
                        m_category.urls.insert(0, url)
                        m_category.count += 1
                if init_type == self.types.delete:
                    m_category.count -= 1
                    m_category.urls.remove(url)
                    if m_category.count == 0:
                        self.by_categories.pop(category)

    def _init_monthly_archive(self,init_type, url):
        start = len(config.entry_url) + 1
        end = start + len('/yyyy/mm')
        month = url[start:end]
        if month not in self.by_months:
            if init_type == self.types.add:
                self.by_months[month] = \
                self.models.monthly_archive(self.types.entry, month, url)
            if init_type == self.types.delete:
                pass
        else:
            if init_type == self.types.add:
                if url not in self.by_months[month].urls:
                    self.by_months[month].urls.insert(0, url)
                    self.by_months[month].count += 1
            else:
                self.by_months[month].count -= 1
                self.by_months[month].urls.remove(url)
                if self.by_months[month].count == 0:
                    self.by_months.pop(month)

    def _init_params(self):
        '''
        refresh the right part of site page
        '''
        self.params.subscribe = self._init_subscribe()
        self.params.primary.tags = self._init_tags_widget()
        self.params.primary.recently_entries = self._init_recently_entries_widget()
        self.params.secondary.categories = self._init_categories_widget()
        self.params.secondary.calendar = self._init_calendar_widget()
        self.params.secondary.archive = self._init_archive_widget()

    def _init_related_entries(self, url):
        """
        #TODO: FIXME: related entries
        """
        urls, index = [], 0
        try:
            index = self.urls.index(url)
        except:
            return None
        urls = self.urls[:index]
        urls.extend(self.urls[index + 1:])
        urls = random.sample(urls, min(len(urls), 10))
        return [self.entries.get(url) for url in sorted(urls, reverse=True)]

    def _init_abouts_widget(self, about_types=[], url=None):
        abouts = []
        for about_type in about_types:
            about = self.models.about(about_type)
            if about_type == self.types.entry and not url == None:
                try:
                    i = self.urls.index(url)
                    p, n = i + 1, i - 1
                except:
                    p, n = 999999999, -1
                if p < len(self.urls):
                    url = self.urls[p]
                    about.prev_url = url
                    about.prev_name = self.entries[url].name
                if n >= 0:
                    url = self.urls[n]
                    about.next_url = url
                    about.next_name = self.entries[url].name
            if about_type == self.types.archive:
                about.prev_url = '/'
                about.prev_name = 'main index'
            if about_type == self.types.blog:
                about.prev_url = '/'
                about.prev_name = 'main  index'
                about.next_url = config.archive_url
                about.next_name = 'archives'
            abouts.append(about)
        return abouts

    def _init_tags_widget(self):
        """
        #TODO: FIXME: calculate tags' rank
        """
        tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
        tags = [t for t in tags if not t.name.startswith('__')]
        ranks = config.ranks
        div, mod = divmod(len(tags), ranks)
        if div == 0:
            ranks, div = mod, 1
        for r in range(ranks):
            s, e = r * div, (r + 1) * div
            for tag in tags[s:e]:
                tag.rank = r + 1
        return tags

    def _init_recently_entries_widget(self):
        return [self.entries[url] for url in self.urls[:config.recently]]

    def _init_calendar_widget(self):
        date = datetime.datetime.today().strftime(config.date_fmt)
        if len(self.urls)> 0:
            date = self.entries[self.urls[0]].date
        calendar = self.models.calendar(date)
        y, m = calendar.month.split('-')
        for url in self.urls:
            _, _, _, _, d, _ = url.split('/')
            prefix = config.entry_url + '/' +  y + '/' + m + '/' + d
            d = int(d)
            if url.startswith(prefix):
                calendar.counts[d] += 1
                if calendar.counts[d] > 1:
                    start = len(config.entry_url)
                    end = start + len('/yyyy/mm/dd')
                    calendar.urls[d] = config.archive_url + url[start:end]
                else:
                    calendar.urls[d] = url
            else:
                break
        return calendar

    def _init_categories_widget(self):
        return sorted(self.by_categories.values(), key=lambda c:c.name)

    def _init_archive_widget(self):
        return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)

    def _find_by_query(self, query, start, limit):
        """
        #TODO: FIXME: how to search in the content of entries
        """
        queries = [q.decode('utf-8') for q  in query.split(' ')]
        urls = []
        for query in queries:
            for entry in self.entries.values():
                if entry.private:
                    continue
                try:
                    entry.content.index(query)
                    urls.append(entry.url)
                except:
                    pass
        return self._find_by_page(sorted(urls), start, limit)

    def _find_by_page(self, urls, start, limit):
        if urls == None or start < 0 or limit <= 0:
            return [], 0
        total = len(urls)
        urls = sorted(urls, reverse=True)
        s, e = (start - 1) * limit, start * limit
        if s > total or s < 0:
            return [], 0
        return [self.entries[url] for url in urls[s:e]], total

    def _paginate(self, pager_type, value, total, start, limit):
        if limit <= 0:
            return self.models.pager(pager_type, value, total, 0, start, limit)
        pages, mod = divmod(total,limit)
        if mod > 0:
            pages += 1
        return self.models.pager(pager_type, value, total, pages, start, limit)

    def find_by_url(self, entry_type, url):
        entry, abouts = None, [self.types.blog]
        if entry_type == self.types.entry:
            entry = self.entries.get(url)
            abouts.insert(0, self.types.entry)
        if entry_type == self.types.page:
            entry = self.pages.get(url)
        self.params.entry = entry
        self.params.entries = self._init_related_entries(url)
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(abouts, url)
        return self.params

    def find_raw(self, raw_url):
        page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
        page = self.find_by_url(self.types.page, page_url).entry
        if not page== None and page.raw_url == raw_url:
            return page.content
        entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
        entry = self.find_by_url(self.types.entry, entry_url).entry
        if not entry == None and entry.raw_url == raw_url:
            return entry.content
        return None

    def archive(self, archive_type, url, start=1, limit=999999999, private=False, username=None):
        self.params.error = self.models.error(url=url)

        if archive_type == self.types.raw:
            url = url.replace(config.raw_url,config.archive_url)

        entries, count, = [], 0
        archive_url = url.replace(config.archive_url, '').strip('/')
        prefix =  url.replace(config.archive_url, config.entry_url)
        pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
        match = re.search(pattern, archive_url)
        if match and match.group() == archive_url or archive_url == '':
            _urls = self.urls if private==False else [ e for e in self.all_urls if e not in self.urls]
            urls = [url for url in _urls if url.startswith(prefix)]
            entries, _  =  self._find_by_page(urls, start, limit)
            if username:
                entries = [e for e in entries if e.author.name==username]
            count = len(entries)
        else:
            entries = None
        if archive_url == '':
            archive_url = self.types.all

        self.params.entries = entries
        self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
        self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
        return self.params

    def search(self, search_type, url, value='', start=config.start, limit=config.limit):
        entries, total, abouts = None, 0, [self.types.blog]
        if  search_type == self.types.query:
            entries, total = self._find_by_query(value, start, limit)
        if search_type == self.types.tag:
            if self.by_tags.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
        if search_type == self.types.category:
            if self.by_categories.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
        if search_type == self.types.index:
            entries, total = self._find_by_page(self.urls, start, limit)
            abouts = []
        self.params.error = self.models.error(url=url)
        self.params.entries = entries
        self.params.search = self.models.search(search_type, value, total)
        self.params.pager = self._paginate(search_type, value, total, start, limit)
        self.params.primary.abouts = self._init_abouts_widget(abouts)
        self.params.start = start
        self.params.limit = limit
        return self.params

    def error(self, url):
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
        return self.params
Exemple #23
0
class EntryService:
    """EntryService."""

    def __init__(self):
        self.entries = {}
        self.pages = {}
        self.urls = []
        self.by_tags = {}
        self.by_categories = {}
        self.by_months = {}
        self.models = Models()
        self.types = self.models.types()
        self.params = self.models.params()
        self._init_entries()

    def _init_entries(self):
        for root, _, files in os.walk(config.entry_dir):
            for f in files:
                self.add_entry(False, root + '/' + f)
        for root, _, files in os.walk(config.page_dir):
            for f in files:
                self._add_page(root + '/' + f)
        self._init_miscellaneous(self.types.add, self.entries.values())

    def add_entry(self, inotified, path):
        entry = self._init_entry(self.types.entry, path)
        if not entry == None:
            self.entries[entry.url] = entry
            if inotified:
                self._init_miscellaneous(self.types.add, [entry])

    def delete_entry(self, path):
        for entry in self.entries.values():
            if path == os.path.abspath(entry.path):
                self.entries.pop(entry.url)
                self._init_miscellaneous(self.types.delete, [entry])

    def _add_page(self, path):
        page = self._init_entry(self.types.page, path)
        if not page == None:
            self.pages[page.url] = page

    def _init_entry(self, entry_type, path):
        url, raw_url, name, date, time, content =  self._init_file(path, entry_type)
        if not url == None:
            entry = self.models.entry(entry_type)
            entry.path = path
            entry.name = name
            entry.url = url
            entry.raw_url = raw_url
            entry.date = date
            entry.time = time
            header, title, categories, tags = extract.parse(entry)
            entry.content = content
            content = content.replace(header, '')
            entry.html = markdown.markdown(content)
            entry.excerpt = content[:200] + ' ... ...'
            entry.categories = categories
            entry.tags = tags
            return entry
        return None

    def _init_file(self, file_path, entry_type):
        """
        #TODO: FIXME: how to determine the publish time of an entry
        """
        content, nones = None, [None for _ in xrange(6)]
        try:
            content = codecs.open(file_path, mode='r', encoding='utf-8').read()
        except:
            return nones
        if content == None or len(content.strip()) == 0:
            return nones
        date, mtime = None, None
        name, _ = os.path.splitext(os.path.basename(file_path))
        chars = ['_' ,'-', '~']
        pattern = r'\d{4}-\d{1,2}-\d{1,2}'
        match = re.search(pattern, name)
        if match:
            y, m, d = match.group().split('-')
            try:
                date = datetime.date(int(y), int(m), int(d))
            except:
                pass
            name = name[len(match.group()):]
            for c in chars:
                if name.startswith(c):
                    name = name[1:]
        stat = os.stat(file_path)
        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
        if date == None:
            date = mtime
        prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
        if entry_type == self.types.entry:
            url_prefix = config.entry_url + '/' + prefix + '/'
            raw_prefix = config.raw_url + '/' + prefix + '/'
        if entry_type == self.types.page:
            url_prefix = '/'
            raw_prefix = config.raw_url + '/'
        date = date.strftime(config.date_fmt)
        time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
        url = url_prefix + name + config.url_suffix
        raw_url = raw_prefix + name + config.raw_suffix
        for c in chars:
            name = name.replace(c, ' ')
        return url, raw_url, name, date, time, content

    def _init_miscellaneous(self,init_type, entries):
        for entry in entries:
            self._init_tag(init_type, entry.url, entry.tags)
            self._init_category(init_type, entry.url, entry.categories)
            self._init_monthly_archive(init_type, entry.url)
        self.urls = sorted(self.entries.keys(), reverse=True)
        self._init_params()

    def _init_subscribe(self):
        time = None
        if self.urls == []:
            time = datetime.datetime.now().strftime(config.time_fmt)
        else:
            time = self.entries[self.urls[0]].time
        return self.models.subscribe(time)

    def _init_tag(self,init_type, url, tags):
        for tag in tags:
            if tag not in self.by_tags:
                if init_type == self.types.add:
                    self.by_tags[tag] = self.models.tag(tag, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_tags[tag].urls.insert(0, url)
                    self.by_tags[tag].count += 1
                if init_type == self.types.delete:
                    self.by_tags[tag].count -= 1
                    self.by_tags[tag].urls.remove(url)
                    if self.by_tags[tag].count == 0:
                        self.by_tags.pop(tag)

    def _init_category(self, init_type, url, categories):
        for category in categories:
            if category not in self.by_categories:
                if init_type == self.types.add:
                    self.by_categories[category] = \
                    self.models.category(category, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_categories[category].urls.insert(0, url)
                    self.by_categories[category].count += 1
                if init_type == self.types.delete:
                    self.by_categories[category].count -= 1
                    self.by_categories[category].urls.remove(url)
                    if self.by_categories[category].count == 0:
                        self.by_categories.pop(category)

    def _init_monthly_archive(self,init_type, url):
        start = len(config.entry_url) + 1
        end = start + len('/yyyy/mm')
        month = url[start:end]
        if month not in self.by_months:
            if init_type == self.types.add:
                self.by_months[month] = \
                self.models.monthly_archive(self.types.entry, month, url)
            if init_type == self.types.delete:
                pass
        else:
            if init_type == self.types.add:
                self.by_months[month].urls.insert(0, url)
                self.by_months[month].count += 1
            else:
                self.by_months[month].count -= 1
                self.by_months[month].urls.remove(url)
                if self.by_months[month].count == 0:
                    self.by_months.pop(month)

    def _init_params(self):
        self.params.subscribe = self._init_subscribe()
        self.params.primary.tags = self._init_tags_widget()
        self.params.primary.recently_entries = self._init_recently_entries_widget()
        self.params.secondary.categories = self._init_categories_widget()
        self.params.secondary.calendar = self._init_calendar_widget()
        self.params.secondary.archive = self._init_archive_widget()

    def _init_related_entries(self, url):
        """
        #TODO: FIXME: related entries
        """
        urls, index = [], 0
        try:
            index = self.urls.index(url)
        except:
            return None
        urls = self.urls[:index]
        urls.extend(self.urls[index + 1:])
        urls = random.sample(urls, min(len(urls), 10))
        return [self.entries.get(url) for url in sorted(urls, reverse=True)]

    def _init_abouts_widget(self, about_types=[], url=None):
        abouts = []
        for about_type in about_types:
            about = self.models.about(about_type)
            if about_type == self.types.entry and not url == None:
                try:
                    i = self.urls.index(url)
                    p, n = i + 1, i - 1
                except:
                    p, n = 999999999, -1
                if p < len(self.urls):
                    url = self.urls[p]
                    about.prev_url = url
                    about.prev_name = self.entries[url].name
                if n >= 0:
                    url = self.urls[n]
                    about.next_url = url
                    about.next_name = self.entries[url].name
            if about_type == self.types.archive:
                about.prev_url = '/'
                about.prev_name = 'main index'
            if about_type == self.types.blog:
                about.prev_url = '/'
                about.prev_name = 'main  index'
                about.next_url = config.archive_url
                about.next_name = 'archives'
            abouts.append(about)
        return abouts

    def _init_tags_widget(self):
        """
        #TODO: FIXME: calculate tags' rank
        """
        tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
        ranks = config.ranks
        div, mod = divmod(len(tags), ranks)
        if div == 0:
            ranks, div = mod, 1
        for r in range(ranks):
            s, e = r * div, (r + 1) * div
            for tag in tags[s:e]:
                tag.rank = r + 1
        return tags

    def _init_recently_entries_widget(self):
        return [self.entries[url] for url in self.urls[:config.recently]]

    def _init_calendar_widget(self):
        date = datetime.datetime.today().strftime(config.date_fmt)
        if len(self.urls)> 0:
            date = self.entries[self.urls[0]].date
        calendar = self.models.calendar(date)
        y, m = calendar.month.split('-')
        for url in self.urls:
            _, _, _, _, d, _ = url.split('/')
            prefix = config.entry_url + '/' +  y + '/' + m + '/' + d
            d = int(d)
            if url.startswith(prefix):
                calendar.counts[d] += 1
                if calendar.counts[d] > 1:
                    start = len(config.entry_url)
                    end = start + len('/yyyy/mm/dd')
                    calendar.urls[d] = config.archive_url + url[start:end]
                else:
                    calendar.urls[d] = url
            else:
                break
        return calendar

    def _init_categories_widget(self):
        return sorted(self.by_categories.values(), key=lambda c:c.name)

    def _init_archive_widget(self):
        return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)

    def _find_by_query(self, query, start, limit):
        """
        #TODO: FIXME: how to search in the content of entries
        """
        queries = [q.lower() for q  in query.split(' ')]
        urls = []
        for query in queries:
            for entry in self.entries.values():
                try:
                    entry.content.index(query)
                    urls.append(entry.url)
                except:
                    pass
        return self._find_by_page(sorted(urls), start, limit)

    def _find_by_page(self, urls, start, limit):
        if urls == None or start < 0 or limit <= 0:
            return [], 0
        total = len(urls)
        urls = sorted(urls, reverse=True)
        s, e = (start - 1) * limit, start * limit
        if s > total or s < 0:
            return [], 0
        return [self.entries[url] for url in urls[s:e]], total

    def _paginate(self, pager_type, value, total, start, limit):
        if limit <= 0:
            return self.models.pager(pager_type, value, total, 0, start, limit)
        pages, mod = divmod(total,limit)
        if mod > 0:
            pages += 1
        return self.models.pager(pager_type, value, total, pages, start, limit)

    def find_by_url(self, entry_type, url):
        entry, abouts = None, [self.types.blog]
        if entry_type == self.types.entry:
            entry = self.entries.get(url)
            abouts.insert(0, self.types.entry)
        if entry_type == self.types.page:
            entry = self.pages.get(url)
        self.params.entry = entry
        self.params.entries = self._init_related_entries(url)
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(abouts, url)
        return self.params

    def find_raw(self, raw_url):
        page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
        page = self.find_by_url(self.types.page, page_url).entry
        if not page== None and page.raw_url == raw_url:
            return page.content
        entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
        entry = self.find_by_url(self.types.entry, entry_url).entry
        if not entry == None and entry.raw_url == raw_url:
            return entry.content
        return None

    def archive(self, archive_type, url, start=1, limit=999999999):
        self.params.error = self.models.error(url=url)

        if archive_type == self.types.raw:
            url = url.replace(config.raw_url,config.archive_url)

        entries, count, = [], 0
        archive_url = url.replace(config.archive_url, '').strip('/')
        prefix =  url.replace(config.archive_url, config.entry_url)
        pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
        match = re.search(pattern, archive_url)
        if match and match.group() == archive_url or archive_url == '':
            urls = [url for url in self.urls if url.startswith(prefix)]
            entries, _  =  self._find_by_page(urls, start, limit)
            count = len(entries)
        else:
            entries = None
        if archive_url == '':
            archive_url = self.types.all

        self.params.entries = entries
        self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
        self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
        return self.params

    def search(self, search_type, url, value='', start=config.start, limit=config.limit):
        entries, total, abouts = None, 0, [self.types.blog]
        if  search_type == self.types.query:
            entries, total = self._find_by_query(value, start, limit)
        if search_type == self.types.tag:
            if self.by_tags.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
        if search_type == self.types.category:
            if self.by_categories.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
        if search_type == self.types.index:
            entries, total = self._find_by_page(self.urls, start, limit)
            abouts = []
        self.params.error = self.models.error(url=url)
        self.params.entries = entries
        self.params.search = self.models.search(search_type, value, total)
        self.params.pager = self._paginate(search_type, value, total, start, limit)
        self.params.primary.abouts = self._init_abouts_widget(abouts)
        return self.params

    def error(self, url):
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
        return self.params
Exemple #24
0
 def get(self,task_id):
     task = Models.get_by_id(int(task_id))
     params = {"list":task}
     self.render_template("delete-task.html",params=params)
Exemple #25
0
class EntryService:
    """EntryService."""

    def __init__(self):
        self.entries = {}
        self.pages = {}
        self.urls = []
        self.by_tags = {}
        self.by_categories = {}
        self.by_months = {}
        self.models = Models()
        self.types = self.models.types()
        self.params = self.models.params()
        self._init_entries()

    def _init_entries(self):
        for root, _, files in os.walk(config.entry_dir):
            for f in files:
                self.add_entry(False, root + '/' + f)
        for root, _, files in os.walk(config.page_dir):
            for f in files:
                self._add_page(root + '/' + f)
        self._init_miscellaneous(self.types.add, self.entries.values())

    def add_entry(self, inotified, path):
        entry = self._init_entry(self.types.entry, path)
        if not entry == None:
            self.entries[entry.url] = entry
            if inotified:
                self._init_miscellaneous(self.types.add, [entry])

    def delete_entry(self, path):
        for entry in self.entries.values():
            if path == os.path.abspath(entry.path):
                self.entries.pop(entry.url)
                self._init_miscellaneous(self.types.delete, [entry])

    def _add_page(self, path):
        page = self._init_entry(self.types.page, path)
        if not page == None:
            self.pages[page.url] = page

    def _init_entry(self, entry_type, path):
        url, raw_url, name, date, time, content =  self._init_file(path, entry_type)
        if not url == None:
            entry = self.models.entry(entry_type)
            entry.path = path
            entry.name = name
            entry.url = url
            entry.raw_url = raw_url
            entry.date = date
            entry.time = time
            header, title, categories, tags = extract.parse(entry)
            entry.content = content
            content = content.replace(header, '')
            entry.html = markdown.markdown(content)
            entry.excerpt = content[:200] + ' ... ...'
            entry.categories = categories
            entry.tags = tags
            return entry
        return None

    def _init_file(self, file_path, entry_type):
        """
        #TODO: FIXME: how to determine the publish time of an entry
        """
        content, nones = None, [None for _ in xrange(6)]
        try:
            content = codecs.open(file_path, mode='r', encoding='utf-8').read()
        except:
            return nones
        if content == None or len(content.strip()) == 0:
            return nones
        date, mtime = None, None
        name, _ = os.path.splitext(os.path.basename(file_path))
        chars = ['_' ,'-', '~']
        pattern = r'\d{4}-\d{1,2}-\d{1,2}'
        match = re.search(pattern, name)
        if match:
            y, m, d = match.group().split('-')
            try:
                date = datetime.date(int(y), int(m), int(d))
            except:
                pass
            name = name[len(match.group()):]
            for c in chars:
                if name.startswith(c):
                    name = name[1:]
        stat = os.stat(file_path)
        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
        if date == None:
            date = mtime
        prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
        if entry_type == self.types.entry:
            url_prefix = config.entry_url + '/' + prefix + '/'
            raw_prefix = config.raw_url + '/' + prefix + '/'
        if entry_type == self.types.page:
            url_prefix = '/'
            raw_prefix = config.raw_url + '/'
        date = date.strftime(config.date_fmt)
        time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
        url = url_prefix + name + config.url_suffix
        raw_url = raw_prefix + name + config.raw_suffix
        for c in chars:
            name = name.replace(c, ' ')
        return url, raw_url, name, date, time, content

    def _init_miscellaneous(self,init_type, entries):
        for entry in entries:
            self._init_tag(init_type, entry.url, entry.tags)
            self._init_category(init_type, entry.url, entry.categories)
            self._init_monthly_archive(init_type, entry.url)
        self.urls = sorted(self.entries.keys(), reverse=True)
        self._init_params()

    def _init_subscribe(self):
        time = None
        if self.urls == []:
            time = datetime.datetime.now().strftime(config.time_fmt)
        else:
            time = self.entries[self.urls[0]].time
        return self.models.subscribe(time)

    def _init_tag(self,init_type, url, tags):
        for tag in tags:
            if tag not in self.by_tags:
                if init_type == self.types.add:
                    self.by_tags[tag] = self.models.tag(tag, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_tags[tag].urls.insert(0, url)
                    self.by_tags[tag].count += 1
                if init_type == self.types.delete:
                    self.by_tags[tag].count -= 1
                    self.by_tags[tag].urls.remove(url)
                    if self.by_tags[tag].count == 0:
                        self.by_tags.pop(tag)

    def _init_category(self, init_type, url, categories):
        for category in categories:
            if category not in self.by_categories:
                if init_type == self.types.add:
                    self.by_categories[category] = \
                    self.models.category(category, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_categories[category].urls.insert(0, url)
                    self.by_categories[category].count += 1
                if init_type == self.types.delete:
                    self.by_categories[category].count -= 1
                    self.by_categories[category].urls.remove(url)
                    if self.by_categories[category].count == 0:
                        self.by_categories.pop(category)

    def _init_monthly_archive(self,init_type, url):
        start = len(config.entry_url) + 1
        end = start + len('/yyyy/mm')
        month = url[start:end]
        if month not in self.by_months:
            if init_type == self.types.add:
                self.by_months[month] = \
                self.models.monthly_archive(self.types.entry, month, url)
            if init_type == self.types.delete:
                pass
        else:
            if init_type == self.types.add:
                self.by_months[month].urls.insert(0, url)
                self.by_months[month].count += 1
            else:
                self.by_months[month].count -= 1
                self.by_months[month].urls.remove(url)
                if self.by_months[month].count == 0:
                    self.by_months.pop(month)

    def _init_params(self):
        self.params.subscribe = self._init_subscribe()
        self.params.primary.tags = self._init_tags_widget()
        self.params.primary.recently_entries = self._init_recently_entries_widget()
        self.params.secondary.categories = self._init_categories_widget()
        self.params.secondary.calendar = self._init_calendar_widget()
        self.params.secondary.archive = self._init_archive_widget()

    def _init_related_entries(self, url):
        """
        #TODO: FIXME: related entries
        """
        urls, index = [], 0
        try:
            index = self.urls.index(url)
        except:
            return None
        urls = self.urls[:index]
        urls.extend(self.urls[index + 1:])
        urls = random.sample(urls, min(len(urls), 10))
        return [self.entries.get(url) for url in sorted(urls, reverse=True)]

    def _init_abouts_widget(self, about_types=[], url=None):
        abouts = []
        for about_type in about_types:
            about = self.models.about(about_type)
            if about_type == self.types.entry and not url == None:
                try:
                    i = self.urls.index(url)
                    p, n = i + 1, i - 1
                except:
                    p, n = 999999999, -1
                if p < len(self.urls):
                    url = self.urls[p]
                    about.prev_url = url
                    about.prev_name = self.entries[url].name
                if n >= 0:
                    url = self.urls[n]
                    about.next_url = url
                    about.next_name = self.entries[url].name
            if about_type == self.types.archive:
                about.prev_url = '/'
                about.prev_name = 'main index'
            if about_type == self.types.blog:
                about.prev_url = '/'
                about.prev_name = 'main  index'
                about.next_url = config.archive_url
                about.next_name = 'archives'
            abouts.append(about)
        return abouts

    def _init_tags_widget(self):
        """
        #TODO: FIXME: calculate tags' rank
        """
        tags = sorted(self.by_tags.values(), key=lambda v:v.count, reverse=True)
        ranks = config.ranks
        div, mod = divmod(len(tags), ranks)
        if div == 0:
            ranks, div = mod, 1
        for r in range(ranks):
            s, e = r * div, (r + 1) * div
            for tag in tags[s:e]:
                tag.rank = r + 1
        return tags

    def _init_recently_entries_widget(self):
        return [self.entries[url] for url in self.urls[:config.recently]]

    def _init_calendar_widget(self):
        date = datetime.datetime.today().strftime(config.date_fmt)
        if len(self.urls)> 0:
            date = self.entries[self.urls[0]].date
        calendar = self.models.calendar(date)
        y, m = calendar.month.split('-')
        for url in self.urls:
            _, _, _, _, d, _ = url.split('/')
            prefix = config.entry_url + '/' +  y + '/' + m + '/' + d
            d = int(d)
            if url.startswith(prefix):
                calendar.counts[d] += 1
                if calendar.counts[d] > 1:
                    start = len(config.entry_url)
                    end = start + len('/yyyy/mm/dd')
                    calendar.urls[d] = config.archive_url + url[start:end]
                else:
                    calendar.urls[d] = url
            else:
                break
        return calendar

    def _init_categories_widget(self):
        return sorted(self.by_categories.values(), key=lambda c:c.name)

    def _init_archive_widget(self):
        return sorted(self.by_months.values(), key=lambda m:m.url, reverse=True)

    def _find_by_query(self, query, start, limit):
        """
        #TODO: FIXME: how to search in the content of entries
        """
        queries = [q.lower() for q  in query.split(' ')]
        urls = []
        for query in queries:
            for entry in self.entries.values():
                try:
                    entry.content.index(query)
                    urls.append(entry.url)
                except:
                    pass
        return self._find_by_page(sorted(urls), start, limit)

    def _find_by_page(self, urls, start, limit):
        if urls == None or start < 0 or limit <= 0:
            return [], 0
        total = len(urls)
        urls = sorted(urls, reverse=True)
        s, e = (start - 1) * limit, start * limit
        if s > total or s < 0:
            return [], 0
        return [self.entries[url] for url in urls[s:e]], total

    def _paginate(self, pager_type, value, total, start, limit):
        if limit <= 0:
            return self.models.pager(pager_type, value, total, 0, start, limit)
        pages, mod = divmod(total,limit)
        if mod > 0:
            pages += 1
        return self.models.pager(pager_type, value, total, pages, start, limit)

    def find_by_url(self, entry_type, url):
        entry, abouts = None, [self.types.blog]
        if entry_type == self.types.entry:
            entry = self.entries.get(url)
            abouts.insert(0, self.types.entry)
        if entry_type == self.types.page:
            entry = self.pages.get(url)
        self.params.entry = entry
        self.params.entries = self._init_related_entries(url)
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(abouts, url)
        return self.params

    def find_raw(self, raw_url):
        page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
        page = self.find_by_url(self.types.page, page_url).entry
        if not page== None and page.raw_url == raw_url:
            return page.content
        entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
        entry = self.find_by_url(self.types.entry, entry_url).entry
        if not entry == None and entry.raw_url == raw_url:
            return entry.content
        return None

    def archive(self, archive_type, url, start=1, limit=999999999):
        self.params.error = self.models.error(url=url)

        if archive_type == self.types.raw:
            url = url.replace(config.raw_url,config.archive_url)

        entries, count, = [], 0
        archive_url = url.replace(config.archive_url, '').strip('/')
        prefix =  url.replace(config.archive_url, config.entry_url)
        pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
        match = re.search(pattern, archive_url)
        if match and match.group() == archive_url or archive_url == '':
            urls = [url for url in self.urls if url.startswith(prefix)]
            entries, _  =  self._find_by_page(urls, start, limit)
            count = len(entries)
        else:
            entries = None
        if archive_url == '':
            archive_url = self.types.all

        self.params.entries = entries
        self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
        self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
        return self.params

    def search(self, search_type, url, value='', start=config.start, limit=config.limit):
        entries, total, abouts = None, 0, [self.types.blog]
        if  search_type == self.types.query:
            entries, total = self._find_by_query(value, start, limit)
        if search_type == self.types.tag:
            if self.by_tags.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
        if search_type == self.types.category:
            if self.by_categories.get(value) == None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
        if search_type == self.types.index:
            entries, total = self._find_by_page(self.urls, start, limit)
            abouts = []
        self.params.error = self.models.error(url=url)
        self.params.entries = entries
        self.params.search = self.models.search(search_type, value, total)
        self.params.pager = self._paginate(search_type, value, total, start, limit)
        self.params.primary.abouts = self._init_abouts_widget(abouts)
        return self.params

    def error(self, url):
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
        return self.params
Exemple #26
0
class EntryService(object):
    """EntryService."""

    def __init__(self):
        self.entries = {}
        self.pages = {}
        self.urls = []
        self.by_tags = {}
        self.by_categories = {}
        self.by_months = {}
        self.models = Models()
        self.types = self.models.types()
        self.params = self.models.params()
        
        self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
        self.blog_bucket = BucketManager(self.qiniu)
        self.blog_prefix = "raw/"
        self._init_blog()

    def _init_blog(self):
        """
        Initialize blog
            - all entries in entry_dir
            - all pages in page_dir
            - others
        """
        cloud_list = {}
        ret, _, info = self.blog_bucket.list(config.qiniu_bucket, prefix = "raw/")
        file_list = eval(info.text_body)["items"]
        for f in file_list:
            if f["key"].startswith(self.blog_prefix) and f["mimeType"] == "text/markdown":
                file_name = f["key"][len(self.blog_prefix):]
                print "Find qiniu file %s" %  file_name
                cloud_list[file_name] = (f["key"], f["fsize"])
        print cloud_list

        for root, _, files in os.walk(config.entry_dir):
            for f in files:
                if f in cloud_list:
                    print "Find local file %s/%s" % (root, f)
                    del cloud_list[f]
                self.add_entry(False, root + '/' + f)

        for f in cloud_list:
            info = cloud_list[f]
            self.download_file(f, info[0])
            print "Download file %s/%s" % (root,f)
            self.add_entry(False, root + '/' + f)

        for root, _, files in os.walk(config.page_dir):
            for f in files:
                self._add_page(root + '/' + f)
        self._init_miscellaneous(self.types.add, self.entries.values())

    def download_file(self, f, key):
        base_url = 'http://%s/%s' % (config.qiniu_bucket_domain, key)
        private_url = self.qiniu.private_download_url(base_url, expires=3600)
        urllib.urlretrieve(private_url, "%s/%s" % (config.entry_dir, f))
        print(private_url)

    def add_entry(self, inotified, path):
        """
        Add entry
        """
        entry = self._init_entry(self.types.entry, path)
        if entry is not None:
            self.entries[entry.url] = entry
            if inotified:
                self._init_miscellaneous(self.types.add, [entry])

    def delete_entry(self, path):
        """
        Delete entry
        """
        for entry in self.entries.values():
            if path == os.path.abspath(entry.path):
                print "delete success!"
                self.entries.pop(entry.url)
                self._init_miscellaneous(self.types.delete, [entry])

    def _add_page(self, path):
        """
        Add page
        """
        page = self._init_entry(self.types.page, path)
        if page is not None:
            self.pages[page.url] = page

    def _init_entry(self, entry_type, path):
        """
        initialize single entry
        """
        url, raw_url, name, date, time, content = self._init_file(path, entry_type)
        if url is not None:
            entry = self.models.entry(entry_type)
            entry.path = path
            entry.name = name
            entry.url = url
            entry.raw_url = raw_url
            entry.date = date
            entry.time = time
            header, title, categories, tags = extract.parse(entry)
            entry.content = content
            content = content.replace(header, '')
            entry.html = markdown.markdown(content)
            # FIXME How to extract the excerpt of an entry
            entry.excerpt = content[:200] + ' ... ...'
            entry.categories = categories
            entry.tags = tags
            return entry
        return None

    def _init_file(self, file_path, entry_type):
        """
        Initialize single file
        """
        # FIXME: how to determine the publish time of an entry
        content, nones = None, [None for _ in xrange(6)]
        try:
            content = codecs.open(file_path, mode='r', encoding='utf-8').read()
        except:
            return nones
        if content is None or len(content.strip()) == 0:
            return nones
        date, mtime = None, None
        name, _ = os.path.splitext(os.path.basename(file_path))
        chars = ['_', '-', '~']
        pattern = r'\d{4}-\d{1,2}-\d{1,2}'
        match = re.search(pattern, name)
        if match:
            y, m, d = match.group().split('-')
            try:
                date = datetime.date(int(y), int(m), int(d))
            except:
                print traceback.format_exc()
                print file_path
            name = name[len(match.group()):]
            for c in chars:
                if name.startswith(c):
                    name = name[1:]
        stat = os.stat(file_path)
        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
        if date is None:
            date = mtime
        prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
        if entry_type == self.types.entry:
            url_prefix = config.entry_url + '/' + prefix + '/'
            raw_prefix = config.raw_url + '/' + prefix + '/'
        elif entry_type == self.types.page:
            url_prefix = '/'
            raw_prefix = config.raw_url + '/'
        date = date.strftime(config.date_fmt)
        time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
        url = url_prefix + name + config.url_suffix
        raw_url = raw_prefix + name + config.raw_suffix
        for c in chars:
            name = name.replace(c, ' ')
        return url, raw_url, name, date, time, content

    def _init_miscellaneous(self, init_type, entries):
        """
        Initialize miscellaneous
            - tags
            - categories
            - archives
            - all urls
        """
        for entry in entries:
            self._init_tag(init_type, entry.url, entry.tags)
            self._init_category(init_type, entry.url, entry.categories)
            self._init_monthly_archive(init_type, entry.url)
        self.urls = sorted(self.entries.keys(), reverse=True)
        self._init_params()

    def _init_subscribe(self):
        """
        Initialize subscriptions
        """
        if not self.urls:
            time = datetime.datetime.now().strftime(config.time_fmt)
        else:
            time = self.entries[self.urls[0]].time
        return self.models.subscribe(time)

    def _init_tag(self, init_type, url, tags):
        """
        Initialize tags
        """
        for tag in tags:
            if tag not in self.by_tags:
                if init_type == self.types.add:
                    self.by_tags[tag] = self.models.tag(tag, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_tags[tag].urls.insert(0, url)
                    self.by_tags[tag].count += 1
                if init_type == self.types.delete:
                    self.by_tags[tag].count -= 1
                    self.by_tags[tag].urls.remove(url)
                    if self.by_tags[tag].count == 0:
                        self.by_tags.pop(tag)

    def _init_category(self, init_type, url, categories):
        """
        Initialize categories
        """
        for category in categories:
            if category not in self.by_categories:
                if init_type == self.types.add:
                    self.by_categories[category] = \
                        self.models.category(category, url)
                if init_type == self.types.delete:
                    pass
            else:
                if init_type == self.types.add:
                    self.by_categories[category].urls.insert(0, url)
                    self.by_categories[category].count += 1
                if init_type == self.types.delete:
                    self.by_categories[category].count -= 1
                    self.by_categories[category].urls.remove(url)
                    if self.by_categories[category].count == 0:
                        self.by_categories.pop(category)

    def _init_monthly_archive(self, init_type, url):
        """
        Initialize archives
        """
        start = len(config.entry_url) + 1
        end = start + len('/yyyy/mm')
        month = url[start:end]
        if month not in self.by_months:
            if init_type == self.types.add:
                self.by_months[month] = \
                    self.models.monthly_archive(self.types.entry, month, url)
            if init_type == self.types.delete:
                pass
        else:
            if init_type == self.types.add:
                self.by_months[month].urls.insert(0, url)
                self.by_months[month].count += 1
            else:
                self.by_months[month].count -= 1
                self.by_months[month].urls.remove(url)
                if self.by_months[month].count == 0:
                    self.by_months.pop(month)

    def _init_params(self):
        """
        Initialize global params
        :return:
        """
        self.params.subscribe = self._init_subscribe()
        self.params.primary.tags = self._init_tags_widget()
        self.params.primary.recently_entries = self._init_recently_entries_widget()
        self.params.secondary.categories = self._init_categories_widget()
        self.params.secondary.calendar = self._init_calendar_widget()
        self.params.secondary.archive = self._init_archive_widget()

    def _init_related_entries(self, url):
        """
        Initialize related entries
        """
        # FIXME: related entries
        try:
            index = self.urls.index(url)
        except:
            print traceback.format_exc()
            return None
        urls = self.urls[:index]
        urls.extend(self.urls[index + 1:])
        urls = random.sample(urls, min(len(urls), 10))
        return [self.entries.get(url) for url in sorted(urls, reverse=True)]

    def _init_abouts_widget(self, about_types=None, url=None):
        """
        Initialize abouts widget
        :param about_types:
        :param url:
        :return:
        """
        about_types = about_types or []
        abouts = []
        for about_type in about_types:
            about = self.models.about(about_type)
            if about_type == self.types.entry and url is not None:
                try:
                    i = self.urls.index(url)
                    p, n = i + 1, i - 1
                except:
                    print traceback.format_exc()
                    p, n = 999999999, -1
                if p < len(self.urls):
                    url = self.urls[p]
                    about.prev_url = url
                    about.prev_name = self.entries[url].name
                if n >= 0:
                    url = self.urls[n]
                    about.next_url = url
                    about.next_name = self.entries[url].name
            if about_type == self.types.archive:
                about.prev_url = '/'
                about.prev_name = 'main index'
            elif about_type == self.types.blog:
                about.prev_url = '/'
                about.prev_name = 'main  index'
                about.next_url = config.archive_url
                about.next_name = 'archives'
            abouts.append(about)
        return abouts

    def _init_tags_widget(self):
        """
        Initialize tags widget
        """
        # FIXME: calculate tags' rank
        tags = sorted(self.by_tags.values(), key=lambda v: v.count, reverse=True)
        ranks = config.ranks
        div, mod = divmod(len(tags), ranks)
        if div == 0:
            ranks, div = mod, 1
        for r in range(ranks):
            s, e = r * div, (r + 1) * div
            for tag in tags[s:e]:
                tag.rank = r + 1
        return tags

    def _init_recently_entries_widget(self):
        """
        Initialize recently entries widget
        :return:
        """
        return [self.entries[url] for url in self.urls[:config.recently]]

    def _init_calendar_widget(self):
        """
        Initialize calender widget
        :return:
        """
        date = datetime.datetime.today().strftime(config.date_fmt)
        if len(self.urls) > 0:
            date = self.entries[self.urls[0]].date
        calendar = self.models.calendar(date)
        y, m = calendar.month.split('-')
        for url in self.urls:
            _, _, _, _, d, _ = url.split('/')
            prefix = config.entry_url + '/' + y + '/' + m + '/' + d
            d = int(d)
            if url.startswith(prefix):
                calendar.counts[d] += 1
                if calendar.counts[d] > 1:
                    start = len(config.entry_url)
                    end = start + len('/yyyy/mm/dd')
                    calendar.urls[d] = config.archive_url + url[start:end]
                else:
                    calendar.urls[d] = url
            else:
                break
        return calendar

    def _init_categories_widget(self):
        """
        Initialize categories widget
        :return:
        """
        return sorted(self.by_categories.values(), key=lambda c: c.name)

    def _init_archive_widget(self):
        """
        Initialize archive widget
        :return:
        """
        return sorted(self.by_months.values(), key=lambda m: m.url, reverse=True)

    def _find_by_query(self, query, start, limit):
        """
        Find by query
        :param query:
        :param start:
        :param limit:
        :return:
        """
        # FIXME: how to search in the content of entries
        queries = [q.lower() for q in query.split(' ')]
        urls = []
        for query in queries:
            for entry in self.entries.values():
                try:
                    entry.content.index(query)
                    urls.append(entry.url)
                except:
                    print
        return self._find_by_page(sorted(urls), start, limit)

    def _find_by_page(self, urls, start, limit):
        """
        Find by page
        :param urls:
        :param start:
        :param limit:
        :return:
        """
        if urls is None or start < 0 or limit <= 0:
            return [], 0
        total = len(urls)
        urls = sorted(urls, reverse=True)
        s, e = (start - 1) * limit, start * limit
        if s > total or s < 0:
            return [], 0
        return [self.entries[url] for url in urls[s:e]], total

    def _paginate(self, pager_type, value, total, start, limit):
        """
        Pagination
        :param pager_type:
        :param value:
        :param total:
        :param start:
        :param limit:
        :return:
        """
        if limit <= 0:
            return self.models.pager(pager_type, value, total, 0, start, limit)
        pages, mod = divmod(total, limit)
        if mod > 0:
            pages += 1
        return self.models.pager(pager_type, value, total, pages, start, limit)

    def find_by_url(self, entry_type, url):
        """
        Find content by url
        :param entry_type:
        :param url:
        :return:
        """
        entry, abouts = None, [self.types.blog]
        if entry_type == self.types.entry:
            entry = self.entries.get(url)
            abouts.insert(0, self.types.entry)
        elif entry_type == self.types.page:
            entry = self.pages.get(url)
        self.params.entry = entry
        self.params.entries = self._init_related_entries(url)
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget(abouts, url)
        return self.params

    def find_raw(self, raw_url):
        """
        Find the raw content by raw_url
        :param raw_url:
        :return:
        """
        page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
        page = self.find_by_url(self.types.page, page_url).entry
        if page is not None and page.raw_url == raw_url:
            return page.content

        entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
        entry = self.find_by_url(self.types.entry, entry_url).entry
        if entry is not None and entry.raw_url == raw_url:
            return entry.content
        return None

    def archive(self, archive_type, url, start=1, limit=999999999):
        """
        Archives
        :param archive_type:
        :param url:
        :param start:
        :param limit:
        :return:
        """
        self.params.error = self.models.error(url=url)

        if archive_type == self.types.raw:
            url = url.replace(config.raw_url, config.archive_url)

        entries, count, = [], 0
        archive_url = url.replace(config.archive_url, '').strip('/')
        prefix = url.replace(config.archive_url, config.entry_url)
        pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
        match = re.search(pattern, archive_url)
        if match and match.group() == archive_url or archive_url == '':
            urls = [url for url in self.urls if url.startswith(prefix)]
            entries, _ = self._find_by_page(urls, start, limit)
            count = len(entries)
        else:
            entries = None
        if archive_url == '':
            archive_url = self.types.all

        self.params.entries = entries
        self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
        self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
        return self.params

    def search(self, search_type, url, value='', start=config.start, limit=config.limit):
        """
        Search the site
        :param search_type:
        :param url:
        :param value:
        :param start:
        :param limit:
        :return:
        """
        entries, total, abouts = None, 0, [self.types.blog]
        if search_type == self.types.query:
            entries, total = self._find_by_query(value, start, limit)
        elif search_type == self.types.tag:
            if self.by_tags.get(value) is None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
        elif search_type == self.types.category:
            if self.by_categories.get(value) is None:
                entries = None
            else:
                entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
        elif search_type == self.types.index:
            entries, total = self._find_by_page(self.urls, start, limit)
            abouts = []
        self.params.error = self.models.error(url=url)
        self.params.entries = entries
        self.params.search = self.models.search(search_type, value, total)
        self.params.pager = self._paginate(search_type, value, total, start, limit)
        self.params.primary.abouts = self._init_abouts_widget(abouts)
        return self.params

    def error(self, url):
        """
        Error params
        :param url:
        :return:
        """
        self.params.error = self.models.error(url=url)
        self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
        return self.params
Exemple #27
0
from utils import Utils
from model import Models, AveragingModels
import pandas as pd
import numpy as np

if __name__ == '__main__':
    utils = Utils()
    models_ = Models()

    X = utils.load_data('./csv/clean_train.csv')
    test = utils.load_data('./csv/clean_test.csv')
    y = utils.load_data('./csv/target.csv')

    test_id = test['Id']
    test.drop('Id', axis=1, inplace=True)

    models = ['ELASTIC_NET', 'GRADIENT', 'LASSO', 'KERNEL_RIDGE', 'XGB']
    base_models = []

    for model in models:
        base_model = models_.grid_training(X, y, model)
        base_models.append(base_model)

    averaged_models = AveragingModels(models=(base_models[0], base_models[1],
                                              base_models[2], base_models[3]))

    n_folds = 5

    score = utils.rmsle_cv(averaged_models, n_folds, X, y)
    print(" Averaged base models score: {:.4f} ({:.4f})\n".format(
        score.mean(), score.std()))
class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(331, 166)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")

        #define data model as attribute to optimize the Models request
        self.model = Models()

        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(10, 30, 47, 13))
        self.label.setObjectName("label")

        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(10, 70, 47, 13))
        self.label_2.setObjectName("label_2")

        self.pushButton = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton.setGeometry(QtCore.QRect(60, 100, 75, 23))
        self.pushButton.setObjectName("pushButton")
        self.pushButton.clicked.connect(self.check_button)

        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setGeometry(QtCore.QRect(210, 100, 75, 23))
        self.pushButton_2.setObjectName("pushButton_2")

        # username field
        self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit.setGeometry(QtCore.QRect(60, 20, 251, 25))
        self.textEdit.setObjectName("textEdit")

        # password field
        self.textEdit_2 = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit_2.setGeometry(QtCore.QRect(60, 60, 251, 25))
        self.textEdit_2.setObjectName("textEdit_2")

        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 331, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.label.setText(_translate("MainWindow", "username"))
        self.label_2.setText(_translate("MainWindow", "password"))
        self.pushButton.setText(_translate("MainWindow", "check"))
        self.pushButton_2.setText(_translate("MainWindow", "exit"))

    #controler of button
    def check_button(self):
        username = self.textEdit.toPlainText()
        password = self.textEdit_2.toPlainText()
        #print(username,password)
        if self.model.get_user(username, password) != None:
            #exist value
            if self.model.get_user(username, password)[0] == 'admin':
                #username is admin so go to admin page
                print(self.model.get_user(username, password))
Exemple #29
0
 def post(self,task_id):
     task = Models.get_by_id(int(task_id))
     task.key.delete()
     self.redirect_to("Task-List")
Exemple #30
0
sentences_length = [len(sentence) for sentence in train_sentences]
mean_seq_length = np.mean(sentences_length)
max_seq_length = np.max(sentences_length)
print('mean_seq_length = {}, max_seq_length = {}'.format(
    mean_seq_length, max_seq_length))

if add_keyword_attention:
    keywords, label_key_words = process.extrac_keywords(
        train_data, train_label, keyword_num)
    for key, value in label_key_words.items():
        print(key, ':', value)
    keywords_id = [word_vocab[word] for word in list(chain(*keywords))]
else:
    keywords_id = None

train_y = process.label_to_onehot(train_labels)
test_y = process.label_to_onehot(test_labels)

print('train size = {}, test size = {}'.format(len(train_sentences),
                                               len(test_sentences)))

print('loading model ...')
models = Models(vocab_size, num_classes, num_features, word_emb_dim,
                feature_emb_dim, hidden_dim, num_layers, learning_rate,
                keep_prob, add_feature_emb, add_second_attention,
                add_keyword_attention)
print('start training model ...')
models.lstm_model(train_sentences, train_y, test_sentences, test_y,
                  keywords_id, batch_size, num_epochs)
Exemple #31
0
def my_train():
    # 是否有GPU加速
    use_gpu = torch.cuda.is_available()

    dataloader = {
        "train": load_data(data_dir_train, image_size, batch_size),
        "valid": load_data(data_dir_valid, image_size, batch_size)
    }

    # 加载模型、损失、优化器等
    model = Models(image_dim, n_classes)
    loss_f = torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    if use_gpu:
        model = model.cuda()

    for epoch in range(epochs):
        print("Epoch {}/{}".format(epoch + 1, epochs))
        print("-" * 10)

        running_loss = 0.0
        train_correct = 0
        train_total = 0

        # 计时
        start_time = time.time()

        for i, data in enumerate(dataloader["train"], 0):
            train_images, train_labels = data
            if use_gpu:
                train_images, labels = Variable(train_images.cuda()), Variable(
                    train_labels.cuda())
            else:
                train_images, labels = Variable(train_images), Variable(
                    train_labels)

            optimizer.zero_grad()  # 梯度置零
            outputs = model(train_images)  # 模型输出
            _, train_predicted = torch.max(outputs.data, 1)  # 预测结果

            train_correct += (train_predicted == labels.data).sum()

            loss = loss_f(outputs, labels)  # 计算损失
            loss.backward()  # 反向传播
            optimizer.step()  # 单步优化(调整学习率)

            running_loss += loss.item()
            train_total += train_labels.size(0)

        now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        runtime = time.time() - start_time
        print('now_time: %s, , time/epoch:%.2fs, time left: %.2fhours' %
              (str(now_time), runtime, (epochs - epoch) * runtime / 3600))
        print('train %d epoch loss: %.3f  acc: %.3f ' %
              (epoch + 1, running_loss / train_total,
               100 * train_correct / train_total))

        writer.add_scalar('Train/Loss', running_loss / train_total, epoch + 1)
        writer.add_scalar('Train/Acc', 100 * train_correct / train_total,
                          epoch + 1)

    torch.save(model, 'model.pt')
 def __init__(self, model_name, device='CPU', extensions=None):
     
     Models.__init__(self, model_name, device, extensions)
     self.model_type = "HEAD"
from data_loader import loader
from model import Models
from keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam
import numpy as np
import sys

load_data = loader()
x_data, y_data, w = load_data.load()
print(x_data[:3])
w = w + w - 1  #no. of features
# print(w)
h = 1
model = Models(h, w)
auto_encoder = model.encode_decoder()
model.encode_decoder()

print(model.encode_decoder().summary())

print(x_data.shape)
# train_x,test_x,train_y,test_y=train_test_split(x_data,y_data,test_size=0.1,random_state=30)

train_x = x_data[:50]
train_y = train_x.copy()
test_x = x_data[50:]
test_y = test_x.copy()

model.fit(train_x, train_y, test_x, test_y)

model.save()
import cv2
import numpy as np
from data_loader import loader
from model import Models
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
w = 240
h = 240
c = 3

mod = Models(w, h, c)
auto_encoder = mod.Arch2()
load_img = loader()
auto_encoder.summary()
x_data, y_data = load_img.load('stone', 'paper', 'scissor')
x_data = np.array(x_data, dtype='float') / 255.0
y_data = np.array(y_data, dtype='float') / 255.0
opt = Adam(lr=0.001, decay=0.001 / 50)
train_x, test_x, train_y, test_y = train_test_split(x_data,
                                                    y_data,
                                                    test_size=0.1,
                                                    random_state=30)
auto_encoder.compile(optimizer='adadelta', loss='binary_crossentropy')
auto_encoder.fit(train_x,
                 train_y,
                 batch_size=32,
                 shuffle='true',
                 epochs=15,
                 validation_data=(test_x, test_y),
                 verbose=1)
    mean_seq_length, max_seq_length))

if add_keyword_attention:
    keywords = text_data.extrac_keywords(keyword_num)
    keywords_id = [word_vocab[word] for word in list(chain(*keywords))]
else:
    keywords_id = None

train_x, train_y, test_x, test_y, mask_train, mask_test = text_data.data_split(
    text_data=sentences,
    seq_length=seq_length,
    hidden_dim=hidden_dim,
    test_rate=test_rate)

print(len(train_x), len(train_y), len(test_x), len(test_y), len(mask_train),
      len(mask_test))
train_x = np.array(train_x)
train_y = np.array(train_y)
mask_train = np.array(mask_train)
test_x = np.array(test_x)
test_y = np.array(test_y)
mask_test = np.array(mask_test)

print('loading model ...')
models = Models(vocab_size, num_classes, word_emb_dim, feature_emb_dim,
                add_feature, seq_length, hidden_dim, num_features, keep_prob,
                add_keyword_attention)
print('start training model ...')
models.lstm_model(train_x, train_y, test_x, test_y, keywords_id, mask_train,
                  mask_test, num_layers, batch_size, learning_rate, num_epochs)