Example #1
0
def get_feeditems(url, min_timestamp=None):
    entries = FeedItemModel.query(FeedItemModel.feed_link == url)
    entries.order(FeedItemModel.published)
    # filter on timestamps
    if min_timestamp is not None and parse_timestamp(min_timestamp) is not None:
        entries = entries.filter(FeedItemModel.timestamp > parse_timestamp(min_timestamp))

    # Get individual items
    items = []
    for item in entries:
        print("Iterating an item:", item.title)
        items.append(feeditem_from_model(item))

    return items
Example #2
0
def get_feeditems(url, min_timestamp=None):
    entries = FeedItemModel.query(FeedItemModel.feed_link == url)
    entries.order(FeedItemModel.published)
    # filter on timestamps
    if (min_timestamp is not None and
        parse_timestamp(min_timestamp) is not None):
        entries = entries.filter(FeedItemModel.timestamp >\
                                parse_timestamp(min_timestamp))

    # Get individual items
    items = []
    for item in entries:
        print("Iterating an item:", item.title)
        items.append(feeditem_from_model(item))

    return items
Example #3
0
 def update_db(self, _id, json_body):
     for note in json_body.get('notes', []):
         note.setdefault('author', self.uid)
         if 'timestamp' in note:
             note['timestamp'] = util.parse_timestamp(note['timestamp'])
         else:
             note['timestamp'] = datetime.datetime.utcnow()
     self.dbc.update({'_id': _id}, {'$set': util.mongo_dict(json_body)})
Example #4
0
 def update_db(self, _id, json_body):
     for note in json_body.get('notes', []):
         note.setdefault('author', self.uid)
         if 'timestamp' in note:
             note['timestamp'] = util.parse_timestamp(note['timestamp'])
         else:
             note['timestamp'] = datetime.datetime.utcnow()
     self.dbc.update({'_id': _id}, {'$set': util.mongo_dict(json_body)})
Example #5
0
def is_modified(vegobjekt):
    # import pdb; pdb.set_trace()
    log = logging.getLogger("geofencebroker")
    next_date = parse_timestamp(vegobjekt["metadata"]["sist_modifisert"])

    table_vegobjekter = db.get_table("vegobjekter")
    geofence = table_vegobjekter.find_one(id=vegobjekt.get("id"))
    if not geofence:
        log.warn("vegobjekt not found in database")
        return False

    prev_date = parse_timestamp(geofence["sist_modifisert"])

    if next_date > prev_date:
        # 'vegobjekt' has been modified
        return True
    elif next_date < prev_date:
        log.warn("next_date < prev_data: (%s < %s)" % (next_date, prev_date))
        log.warn("Most likely a bug!!")
    return False
Example #6
0
 def post(self, sid):
     """Create a new Acquisition."""
     json_body = self._post()
     _id = bson.ObjectId(sid)
     session = self.app.db.sessions.find_one({'_id': _id}, ['permissions', 'public'])
     if not session:
         self.abort(404, 'no such session')
     if not self.superuser_request and util.user_perm(session['permissions'], self.uid).get('access') != 'admin':
         self.abort(400, 'must be session admin to create acquisition')
     json_body['session'] = _id
     json_body['permissions'] = session['permissions']
     json_body['public'] = session.get('public', False)
     json_body['files'] = []
     if 'timestamp' in json_body:
         json_body['timestamp'] = util.parse_timestamp(json_body['timestamp'])
     return {'_id': str(self.dbc.insert(json_body))}
Example #7
0
 def post(self, pid):
     """Create a new Session."""
     json_body = self._post()
     _id = bson.ObjectId(pid)
     project = self.app.db.projects.find_one({'_id': _id}, ['group', 'permissions', 'public'])
     if not project:
         self.abort(404, 'no such project')
     if not self.superuser_request and util.user_perm(project['permissions'], self.uid).get('access') != 'admin':
         self.abort(400, 'must be project admin to create session')
     json_body['project'] = _id
     json_body['group'] = project['group']
     json_body['permissions'] = project['permissions']
     json_body['public'] = project.get('public', False)
     json_body['files'] = []
     if 'timestamp' in json_body:
         json_body['timestamp'] = util.parse_timestamp(json_body['timestamp'])
     return {'_id': str(self.dbc.insert(json_body))}
Example #8
0
 def post(self, sid):
     """Create a new Acquisition."""
     json_body = self._post()
     _id = bson.ObjectId(sid)
     session = self.app.db.sessions.find_one({'_id': _id},
                                             ['permissions', 'public'])
     if not session:
         self.abort(404, 'no such session')
     if not self.superuser_request and util.user_perm(
             session['permissions'], self.uid).get('access') != 'admin':
         self.abort(400, 'must be session admin to create acquisition')
     json_body['session'] = _id
     json_body['permissions'] = session['permissions']
     json_body['public'] = session.get('public', False)
     json_body['files'] = []
     if 'timestamp' in json_body:
         json_body['timestamp'] = util.parse_timestamp(
             json_body['timestamp'])
     return {'_id': str(self.dbc.insert(json_body))}
Example #9
0
 def parse_js(js):
     thumb_url = self.last_logo_url
     url = js['url']
     for _dict in js['image']:
         if _dict['size'] == 'extralarge':
             thumb_url = _dict['#text']
             # Replace 'png' with 'jpg' in the URL.
             # This is a hack to allow Discord embeds to work.
             thumb_url = thumb_url[:-3] + 'jpg'
     data = {}
     data['account_created'] = js['registered']['unixtime']
     data['account_created'] = util.parse_timestamp(
         data['account_created'])
     data['scrobbles'] = js['playcount']
     data['country'] = js['country']
     if data['country'] == '': del data['country']
     data['age'] = js['age']
     if data['age'] == '0': del data['age']
     return thumb_url, url, data
Example #10
0
 def post(self, pid):
     """Create a new Session."""
     json_body = self._post()
     _id = bson.ObjectId(pid)
     project = self.app.db.projects.find_one(
         {'_id': _id}, ['group', 'permissions', 'public'])
     if not project:
         self.abort(404, 'no such project')
     if not self.superuser_request and util.user_perm(
             project['permissions'], self.uid).get('access') != 'admin':
         self.abort(400, 'must be project admin to create session')
     json_body['project'] = _id
     json_body['group'] = project['group']
     json_body['permissions'] = project['permissions']
     json_body['public'] = project.get('public', False)
     json_body['files'] = []
     if 'timestamp' in json_body:
         json_body['timestamp'] = util.parse_timestamp(
             json_body['timestamp'])
     return {'_id': str(self.dbc.insert(json_body))}
Example #11
0
def demo():
    logger.info(color("bold_yellow") + "尝试启动更新器,等待其执行完毕。若版本有更新,则会干掉这个进程并下载更新文件,之后重新启动进程...(请稍作等待)")

    dlc_path = os.path.realpath("auto_updater.py")
    p = subprocess.Popen(
        [
            dlc_path,
            "--pid",
            str(os.getpid()),
            "--version",
            str(now_version),
            "--cwd",
            os.getcwd(),
            "--exe_name",
            os.path.realpath("DNF蚊子腿小助手.exe"),
        ],
        cwd="utils",
        shell=True,
        creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    p.wait()

    if p.returncode != 0:
        last_modify_time = parse_timestamp(os.stat(dlc_path).st_mtime)
        logger.error(f"DLC出错了,错误码为{p.returncode},DLC最后一次修改时间为{last_modify_time}")

        uploader = Uploader()
        netdisk_latest_dlc_info = uploader.find_latest_dlc_version()
        latest_version_time = parse_time(netdisk_latest_dlc_info.time)

        if latest_version_time > last_modify_time:
            logger.info(
                f"网盘中最新版本dlc上传于{latest_version_time}左右,在当前版本最后修改时间{last_modify_time}之后,有可能已经修复dlc的该问题,将尝试更新dlc为最新版本"
            )
            uploader.download_file(netdisk_latest_dlc_info, "utils")
        else:
            logger.warning(f"网盘中最新版本dlc上传于{latest_version_time}左右,在当前版本最后修改时间{last_modify_time}之前,请耐心等待修复该问题的新版本发布~")
def main():
    dirs = os.listdir(os.path.join(BASE_DIR, '_api'))
    if len(dirs) == 0:
        print("Springnote backup not found.")
        return
    elif len(dirs) == 1:
        subdomain = dirs[0]
    else:
        subdomain = None
        while subdomain not in dirs:
            subdomain = input("Choose ID ({}) : ".format(', '.join(dirs)))
    makedirs(os.path.join(DOKUWIKI_ROOT, subdomain, 'data', 'attic'), exist_ok=True)
    makedirs(os.path.join(DOKUWIKI_ROOT, subdomain, 'data', 'meta'), exist_ok=True)
    makedirs(os.path.join(DOKUWIKI_ROOT, subdomain, 'data', 'pages'), exist_ok=True)
    pages = _load(subdomain, 'pages')
    tree = E.pages()
    node_by_id = {}
    dangling_nodes = {}
#    pages = {_['page']['identifier']: _['page'] for _ in pages}  # No need yet
    print('0 / {} pages'.format(len(pages)))
    for i, entry in enumerate(pages):
        id_ = entry['page']['identifier']
        page = _load(subdomain, 'pages/{}'.format(id_))

        # tree
        if id_ not in node_by_id:
            node = E.page(id=str(id_))
            node_by_id[id_] = node
        elif id_ in dangling_nodes:
            node = node_by_id[id_]
            del dangling_nodes[id_]
        else:
            raise ValueError('Multiple pages with a same ID: {}'.format(id_))
        node.set('title', entry['page']['title'])
        parent_id = entry['page'].get('relation_is_part_of', None)
        if not parent_id:
            # the root
            tree.append(node)
        elif parent_id in node_by_id:
            node_by_id[parent_id].append(node)
        else:
            assert parent_id not in dangling_nodes
            parent = E.page(node, id=str(parent_id))
            node_by_id[parent_id] = dangling_nodes[parent_id] = parent

        # content
        xhtml = b'''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html version="-//W3C//DTD XHTML 1.1//EN"
    xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.w3.org/1999/xhtml
    http://www.w3.org/MarkUp/SCHEMA/xhtml11.xsd"
    >
  <head>
    <title>{title}</title>
  </head>
  <body>
    {body}
  </body>
</html>
'''.format(
            title=page['page']['title'].encode('utf8'),
            body=page['page']['source'].encode('utf8'))
        # TODO: save private pages in a different path
        _save(subdomain, 'pages/{}.xhtml'.format(id_), xhtml)
        # revisions
        revisions_data = _load(subdomain, 'pages/{}/revisions'.format(id_))
        if revisions_data:
            revisions = sorted(revisions_data,
                    key=lambda _: _['revision']['date_created'])
            data = []
            for k, revision in enumerate(revisions):
                description = revision['revision']['description']
                revision = _load(subdomain, 'pages/{}/revisions/{}'.format(
                    id_, revision['revision']['identifier']))
                timestamp = util.parse_timestamp(
                        revision['revision']['date_created'])
                data.append('\t'.join([
                    str(int(timestamp)),
                    '127.0.0.1',
                    'E' if k else 'C',  # for 'edit' and 'create' respectively
                    str(id_),
                    revision['revision']['creator'] or '',
                    description]))
                source = revision['revision']['source'] or ''
                path = os.path.join(DOKUWIKI_ROOT, subdomain, 'data', 'attic',
                        '{}.{}.xhtml.gz'.format(id_, timestamp))
                with gzip.open(path, 'wb+') as f:
                    f.write(source.encode('utf8'))
            _save(subdomain, 'meta/{}.changes'.format(id_), '\n'.join(data))

        # TODO: attachments
        print(b'{} / {} pages'.format(i + 1, len(pages)))

    print('Saving tree structures...')
    lost_and_found = E.page(id='lost_and_found', title='Lost and Found')
    if dangling_nodes:
        for id_, node in dangling_nodes.items():
            lost_and_found.append(node)
        tree.append(lost_and_found)
    # NOTE: The tree is sorted by title by default; maybe we need to
    #       double-check it
    for node in list(tree.getiterator()):
        if node.tag != 'page':
            continue
        path = []
        parent = node.getparent()
        while parent.tag == 'page':
            path.insert(0, parent.get('id'))
            parent = parent.getparent()
        node.set('path', ':'.join(path))
    new_tree = E.pages()
    for node in list(tree.getiterator()):
        if node.tag != 'page':
            continue
        new_tree.append(node)
    _save(subdomain, 'meta/_tree.xml', lxml.etree.tostring(new_tree,
        pretty_print=True, encoding='utf8'))
    # tree with public pages only
    for node in list(new_tree.getiterator()):
        if node.tag in ['pages']:
            continue
        id_ = node.get('id')
        if id_ in ['lost_and_found']:
            continue
        collaboration = _load(subdomain,
            'pages/{}/collaboration'.format(int(id_)))
        # The page is public when any of the following is true:
        # - collaboration.json is an empty list
        # - There exists {"access_rights": "reader",
        #                 "rights_holder": "everybody", ...}
        is_public = not collaboration or any(
            _['collaboration']['access_rights'] == 'reader' and
            _['collaboration']['rights_holder'] == 'everybody'
            for _ in collaboration)
        if not is_public:
            for child in node:
                node.remove(child)
                lost_and_found.append(child)
            node.getparent().remove(node)
    # TODO: Sort by title?
    _save(subdomain, 'pages/_tree.xml', lxml.etree.tostring(new_tree,
        pretty_print=True, encoding='utf8'))
Example #13
0
def do_classify(incoming_data, x_request_id):
    '''wrapper of classify
    '''
    result = {'code': 1, 'message': ''}
    logger.debug('<%s>, [do_classify] enter, incoming_data=%s' % (x_request_id, incoming_data))

    tag = incoming_data['tag']
    seq = incoming_data['seq']
    pois = incoming_data['pois']
    algo_type = incoming_data.get('algo_type', 'gmm')
    logger.info('<%s>, [classify gmm] params: tag=%s, seq=%s, algo_type=%s' %(x_request_id, tag, seq, algo_type))

    # parse seq
    seq = [util.parse_timestamp(e) for e in seq]

    # classify
    models = []
    for poi in pois:
        try:
            model = dao.get_model_by_tag_lable(algo_type, tag, poi)
        except IndexError:
            logger.error('tag=%s, label=%s can not find model' % (tag, poi))
            result['message'] = '<%s>, [do_classify] Can not find model whose tag=%s, label=%s ' % (x_request_id ,tag, poi)
            raise ValueError(result['message'])
        models.append(model)

    _models = []
    labels = []
    for model in models:
        labels.append(model.get('eventLabel'))
        _model = {
            'nMix': model.get('nMix'),
            'covarianceType': model.get('covarianceType'),
            'nIter': model.get('nIter'),
            'count': model.get('count'),
            'params': model.get('params'),
        }
        _models.append(_model)
    my_predictor = Predictor(_models)
    logger.debug('-------\n\n_models:\n %s\n' % (_models))

    score_results = []
    seq_scores = my_predictor.scores(seq)
    logger.debug('seq_scores: %s' % (seq_scores))
    for scores in seq_scores:
        score_result = {}
        for index, score in enumerate(scores):
            if labels[index] in score_result:
                score_result[labels[index]] += score
            else:
                score_result[labels[index]] = score
        score_results.append(score_result)
    logger.debug('!!!!!! score_result:%s' %(score_result))
    # store seq in db
    for index, timestamp in enumerate(seq):
        event_label = max(score_results[index].iterkeys(), key=lambda key: score_results[index][key])  # max prob key is labelt
        dao.save_train_data(timestamp, event_label)
        logger.info('<%s> [classify] store timestamp=%s, label=%s to db success' % (x_request_id, timestamp, event_label))

    logger.info('<%s> [classify gmm] success' % (x_request_id))
    logger.debug('<%s> [classify gmm] result: %s' % (x_request_id, score_results))

    result['code'] = 0
    result['message'] = 'success'
    result['result'] = score_results

    return result
def do_classify(incoming_data, x_request_id):
    '''wrapper of classify
    '''
    result = {'code': 1, 'message': ''}
    logger.debug('<%s>, [do_classify] enter, incoming_data=%s' % (x_request_id, incoming_data))

    tag = incoming_data['tag']
    seq = incoming_data['seq']
    pois = incoming_data['pois']
    algo_type = incoming_data.get('algo_type', 'gmm')
    logger.info('<%s>, [classify gmm] params: tag=%s, seq=%s, algo_type=%s' %(x_request_id, tag, seq, algo_type))

    # parse seq
    seq = [util.parse_timestamp(e) for e in seq]

    # classify
    models = []
    for poi in pois:
        try:
            model = dao.get_model_by_tag_lable(algo_type, tag, poi)
        except IndexError:
            logger.error('tag=%s, label=%s can not find model' % (tag, poi))
            result['message'] = '<%s>, [do_classify] Can not find model whose tag=%s, label=%s ' % (x_request_id ,tag, poi)
            raise ValueError(result['message'])
        models.append(model)

    _models = []
    labels = []
    for model in models:
        labels.append(model.get('eventLabel'))
        _model = {
            'nMix': model.get('nMix'),
            'covarianceType': model.get('covarianceType'),
            'nIter': model.get('nIter'),
            'count': model.get('count'),
            'params': model.get('params'),
        }
        _models.append(_model)
    my_predictor = Predictor(_models)
    logger.debug('-------\n\n_models:\n %s\n' % (_models))

    score_results = []
    seq_scores = my_predictor.scores(seq)
    logger.debug('seq_scores: %s' % (seq_scores))
    for scores in seq_scores:
        score_result = {}
        for index, score in enumerate(scores):
            if labels[index] in score_result:
                score_result[labels[index]] += score
            else:
                score_result[labels[index]] = score
        score_results.append(score_result)
    logger.debug('!!!!!! score_result:%s' %(score_result))
    # store seq in db
    for index, timestamp in enumerate(seq):
        event_label = max(score_results[index].iterkeys(), key=lambda key: score_results[index][key])  # max prob key is labelt
        dao.save_train_data(timestamp, event_label)
        logger.info('<%s> [classify] store timestamp=%s, label=%s to db success' % (x_request_id, timestamp, event_label))

    logger.info('<%s> [classify gmm] success' % (x_request_id))
    logger.debug('<%s> [classify gmm] result: %s' % (x_request_id, score_results))

    result['code'] = 0
    result['message'] = 'success'
    result['result'] = score_results

    return result
Example #15
0
def test_parse_timestamp():
    assert parse_timestamp(1628222400.0) == now_for_test
Example #16
0
    def download_file(self,
                      fileinfo: FileInFolder,
                      download_dir: str,
                      overwrite=True,
                      show_log=True,
                      download_only_if_server_version_is_newer=True) -> str:
        """
        下载最新版本压缩包到指定目录,并返回最终压缩包的完整路径
        """
        make_sure_dir_exists(download_dir)

        download_dir = os.path.realpath(download_dir)
        target_path = StrWrapper(os.path.join(download_dir, fileinfo.name))

        if download_only_if_server_version_is_newer and os.path.isfile(
                target_path.value):
            # 仅在服务器版本比本地已有文件要新的时候才重新下载
            # 由于蓝奏云时间显示不精确,将其往前一分钟,避免同一文件下次检查时其蓝奏云时间显示为xx分钟前,解析后会有最多一分钟内的误差,而导致不必要的重新下载
            # 比如本次是x分y秒检查并更新,下次检查时是x+6分y+10秒,此时解析蓝奏云时间得到上传时间为x分y+10秒,就会产生额外的不必要下载
            server_version_upload_time = parse_time(
                fileinfo.time) - timedelta(minutes=1)
            local_version_last_modify_time = parse_timestamp(
                os.stat(target_path.value).st_mtime)

            get_log_func(
                logger.info, show_log
            )(f"{fileinfo.name} 本地修改时间为:{local_version_last_modify_time} 网盘版本上传时间为:{server_version_upload_time}"
              )

            if server_version_upload_time <= local_version_last_modify_time:
                # 暂无最新版本,无需重试
                get_log_func(
                    logger.info, show_log
                )(color("bold_cyan") +
                  f"当前设置了对比修改时间参数,网盘中最新版本 {fileinfo.name} 上传于{server_version_upload_time}左右,在当前版本{local_version_last_modify_time}之前,无需重新下载"
                  )
                return target_path.value

        def after_downloaded(file_name):
            """下载完成后的回调函数"""
            target_path.value = file_name
            get_log_func(logger.info, show_log)(f"最终下载文件路径为 {file_name}")

        get_log_func(logger.info, show_log)(f"即将开始下载 {target_path.value}")
        callback = None
        if show_log: callback = self.show_progress
        retCode = self.down_file_by_url(fileinfo.url,
                                        "",
                                        download_dir,
                                        callback=callback,
                                        downloaded_handler=after_downloaded,
                                        overwrite=overwrite)
        if retCode != LanZouCloud.SUCCESS:
            get_log_func(logger.error, show_log)(f"下载失败,retCode={retCode}")
            if retCode == LanZouCloud.NETWORK_ERROR:
                get_log_func(
                    logger.warning, show_log
                )(color("bold_yellow") +
                  ("蓝奏云api返回网络错误,这很可能是由于dns的问题导致的\n"
                   "分别尝试在浏览器中访问下列两个网页,是否一个打的开一个打不开?\n"
                   "https://fzls.lanzoux.com/s/djc-helper\n"
                   "https://fzls.lanzous.com/s/djc-helper\n"
                   "\n"
                   "如果是这样,请按照下面这个链接,修改本机的dns,使用阿里、腾讯、百度、谷歌dns中的任意一个应该都可以解决。\n"
                   "https://www.ypojie.com/9830.html\n"
                   "\n"
                   "如果两个都打不开,大概率是蓝奏云挂了-。-可选择忽略后面的弹框,继续运行旧版本,或者手动去QQ群或github下载最新版本"
                   ))
            raise Exception("下载失败")

        return target_path.value
Example #17
0
    def download_file_in_folder(
            self,
            folder: Folder,
            name: str,
            download_dir: str,
            overwrite=True,
            show_log=True,
            try_compressed_version_first=False,
            cache_max_seconds=600,
            download_only_if_server_version_is_newer=True) -> str:
        """
        下载网盘指定文件夹的指定文件到本地指定目录,并返回最终本地文件的完整路径
        """
        def _download(fname: str) -> str:
            return with_cache(
                cache_name_download,
                os.path.join(folder.name, fname),
                cache_max_seconds=cache_max_seconds,
                cache_miss_func=lambda: self.
                download_file(self.find_file(folder, fname),
                              download_dir,
                              overwrite=overwrite,
                              show_log=show_log,
                              download_only_if_server_version_is_newer=
                              download_only_if_server_version_is_newer),
                cache_validate_func=lambda target_path: os.path.isfile(
                    target_path),
            )

        if try_compressed_version_first:
            # 先尝试获取压缩版本
            compressed_filename = self.get_compressed_version_filename(name)
            try:
                get_log_func(logger.info,
                             show_log)(color("bold_green") +
                                       f"尝试优先下载压缩版本 {compressed_filename}")

                # 记录下载前的最近修改时间
                before_download_last_modify_time = None
                old_compressed_filepath = os.path.join(download_dir,
                                                       compressed_filename)
                if os.path.isfile(old_compressed_filepath):
                    before_download_last_modify_time = parse_timestamp(
                        os.stat(old_compressed_filepath).st_mtime)

                # 下载压缩版本
                compressed_filepath = _download(compressed_filename)

                # 记录下载完成后的最近修改时间
                after_download_last_modify_time = parse_timestamp(
                    os.stat(compressed_filepath).st_mtime)

                # 解压缩
                dirname = os.path.dirname(compressed_filepath)
                target_path = os.path.join(dirname, name)

                need_decompress = True
                if before_download_last_modify_time is not None and before_download_last_modify_time == after_download_last_modify_time and os.path.exists(
                        target_path):
                    # 如果前后修改时间没有变动,说明没有实际发生下载,比如网盘版本与当前本地版本一致,如果此时目标文件已经解压过,将不再尝试解压
                    need_decompress = False

                if need_decompress:
                    decompress_file_with_lzma(compressed_filepath, target_path)
                else:
                    get_log_func(logger.info, show_log)(
                        f"{compressed_filepath}未发生改变,且目标文件已存在,无需尝试解压缩")
                # 返回解压缩的文件路径
                return target_path
            except Exception as e:
                get_log_func(logger.error, show_log)(
                    f"下载压缩版本 {compressed_filename} 失败,将尝试普通版本~", exc_info=e)

        # 下载普通版本
        return _download(name)
def main():
    if not os.path.exists(os.path.join(BASE_DIR, '_api')):
        print("Springnote backup not found.")
        return
    dirs = os.listdir(os.path.join(BASE_DIR, '_api'))
    if len(dirs) == 0:
        print("Springnote backup not found.")
        return
    elif len(dirs) == 1:
        subdomain = dirs[0]
    else:
        subdomain = None
        while subdomain not in dirs:
            subdomain = input("Choose ID ({}) : ".format(
                ', '.join(sorted(dirs))))

    print("Initializing...")
    util.makedirs(SAVE_ROOT, exist_ok=True)

    pages = util.load_resource(subdomain, 'pages')
    print("Converting XHTML to MediaWiki...")
    output_paths = [
        os.path.join(SAVE_ROOT, '{}.xml'.format(subdomain)),
        os.path.join(SAVE_ROOT, '{}.private.xml'.format(subdomain))]
    output_fp = [open(_, 'w+') for _ in output_paths]
    for fp in output_fp:
        fp.write('<mediawiki>\n')
    id2title = {str(_['page']['identifier']): _['page']['title']
            for _ in pages}
    n = 0
    titles = set()
    for entry in pages:
        id_ = entry['page']['identifier']
        fp = output_fp[0] if util.is_public(subdomain, id_) else output_fp[1]

        page = util.load_resource(subdomain, 'pages/{}'.format(id_))
        title = page['page']['title']
        new_title = title
        for k, v in MEDIAWIKI_NG.items():
            new_title = new_title.replace(k, v)
        j = 1
        new_title_ = new_title
        while new_title_ in titles:
            j += 1
            new_title_ = new_title + ' ({})'.format(j)
        new_title = new_title_
        titles.add(new_title)
        revisions_data = util.load_resource(subdomain,
                'pages/{}/revisions'.format(id_))
        if revisions_data:
            revisions = sorted(revisions_data,
                    key=lambda _: _['revision']['date_created'])
        else:
            revisions = []
        fp.write('<page>\n')
        fp.write(u'<title>{}</title>\n'.format(
            escape(new_title)).encode('utf-8'))
        wiki_text = ''
        for revision in revisions:
            timestamp = util.parse_timestamp(
                        revision['revision']['date_created'])
            timestamp = datetime.datetime.utcfromtimestamp(timestamp)\
                    .strftime('%Y-%m-%dT%H:%M:%SZ')
            revision_data = util.load_resource(subdomain,
                'pages/{}/revisions/{}'.format(
                    id_, revision['revision']['identifier']))
            source = revision_data['revision']['source']
            wiki_text = (sabal2mediawiki(source, id2title) if source
                    else wiki_text)  # previous revision
            fp.write('<revision>\n')
            fp.write('<timestamp>{}</timestamp>\n'.format(timestamp))
            if revision['revision'].get('creator'):
                fp.write('<contributor><username>{}'
                    '</username></contributor>\n'.format(
                        escape(revision['revision']['creator'])))
            if revision['revision'].get('description'):
                fp.write(u'<comment>{}</comment>\n'\
                    .format(escape(revision['revision']['description']))\
                    .encode('utf-8'))
            fp.write(u'<text>{}</text>\n'\
                    .format(escape(wiki_text)).encode('utf-8'))
            fp.write('</revision>\n')
        fp.write('</page>\n')
        n += 1
        print(b"{} / {} pages - {}".format(
            n, len(pages), title.encode('utf8')))
    for fp in output_fp:
        fp.write('</mediawiki>')
        fp.close()
    print("Converted XML at:")
    for _ in output_paths:
        print(_)