Beispiel #1
0
def feature_about():
    # 获取特征列表
    feature_dict = NewsUtil.get_feature()
    # 获取新闻中出现特征后最近的5个词及其属性
    logger.info("In Prepare Raw News...")
    raw_news_data = CommonUtil.read_excel(RAW_NEWS_DEMO_PATH)
    raw_news_table = raw_news_data.sheet_by_index(0)
    raw_news_rows = raw_news_table.nrows
    segmentor = Segmentor()  # 初始化实例
    segmentor.load_with_lexicon(cws_model_path,
                                CFETSFX_LEXICON_PATH)  # 加载模型,第二个参数是您的外部词典文件路径
    feature_about_list = list()
    for rowN in range(0, raw_news_rows):
        news_content = raw_news_table.cell_value(rowN, 2)
        sentences = SentenceSplitter.split(news_content)
        for sentence in sentences:
            print(sentence)
            # 分词
            words = segmentor.segment(sentence)
            print(list(words))
            for word_index in range(0, len(words)):
                word = words[word_index]
                for feature_word in feature_dict.values():
                    if feature_word in word:
                        about_list = list()
                        count = 0
                        while word_index < len(words) and count < 6:
                            about_list.append(words[word_index])
                            count += 1
                            word_index += 1
                        feature_about_list.append(about_list)
                        print(about_list)
                        break
    segmentor.release()
    CommonUtil.write_csv(FEATURE_ABOUT_PATH, feature_about_list)
Beispiel #2
0
def get_feature_value():
    feature_vector_list = list()
    feature_name_list = list()
    for feature_name in FEATURE_NAME_LIST:
        feature_name_list.append(feature_name)
        feature_list = CommonUtil.read_csv(MarketDataCrawler.MARKET_DATA_PATH +
                                           '/' + feature_name + '.csv')
        feature_dict = dict()
        for feature_i in range(1, len(feature_list)):
            date = CommonUtil.get_datetime_from_string_(
                feature_list[feature_i][0]).date()
            # 开盘价
            feature_value = float(feature_list[feature_i][1])
            feature_dict[date] = feature_value
        for value_key in valueDict.keys():
            if value_key in feature_dict.keys():
                feature_value = feature_dict[value_key]
            else:
                feature_value = 'N/A'
            if value_key in featureValue.keys():
                feature_items = featureValue[value_key]
            else:
                feature_items = list()
            feature_items.append(feature_value)
            featureValue[value_key] = feature_items
    feature_name_list.append(VALUE_NAME)
    feature_name_list.insert(0, 'DATE')
    feature_vector_list.append(feature_name_list)
    for key in featureValue.keys():
        feature_items = featureValue[key]
        feature_items.append(valueDict[key])
        feature_items.insert(0, key)
        feature_vector_list.append(feature_items)
    CommonUtil.write_csv('../files/marketdata/FEATURE_VECTOR.csv',
                         feature_vector_list)
Beispiel #3
0
def news_sentiment():
    logger.info("In News Sentiment...")
    count = 1
    for mapped_news in newsMappedList:
        feature_vector_item = list()
        news_index = mapped_news[0]
        news_time = mapped_news[1]
        feature_vector_item.append(news_index)
        feature_vector_item.append(news_time)
        feature_vector = list()
        keyword_sentiment_dict = dict()
        # 下标从0开始,减1
        news_mapped = newsList[news_index - 1]
        for mapped_news_index in range(2, len(mapped_news)):
            keyword = mapped_news[mapped_news_index]
            sentiment_result = BaiduNLPProcessor.sentiment_classify(
                news_mapped[2])
            keyword_sentiment_dict[keyword] = sentiment_result
        keys = featureDict.keys()
        for key in keys:
            if featureDict[key] in keyword_sentiment_dict.keys():
                feature_vector.append(keyword_sentiment_dict[featureDict[key]])
            else:
                feature_vector.append(0)
        feature_vector_item.append(feature_vector)
        newsItemList.append(feature_vector_item)
        feature_vector.insert(0, news_time)
        newsFeatureList.append(feature_vector)
        logger.info(count)
        count += 1
    CommonUtil.write_csv(NEWS_ITEM_PATH, newsItemList)
    CommonUtil.write_csv(NEWS_FEATURE_PATH, newsFeatureList)
    logger.info("News Sentiment Done!")
def link_vec(NEWS_VEC, PRICE_VEC, p):
    ALL_VEC = []
    # print(NEWS_VEC[0][1])
    # print(PRICE_VEC[0][0])
    for each_price_vec in PRICE_VEC:
        flag = 0
        for each_news_vec in NEWS_VEC:
            # print("each_news_vec[0] , each_price_vec[0]", each_news_vec[0] , each_price_vec[0])
            if each_news_vec[0] == each_price_vec[0]:
                print("matched!")
                vec_of_all = each_news_vec[::]
                vec_of_all.append(each_price_vec[1])
                vec_of_all.append(each_price_vec[2])
                ALL_VEC.append(vec_of_all)
                flag = 1
                break
        if flag == 0:
            print("not matched! Created!")
            list_temp = []
            list_temp.append(each_price_vec[0])
            list_temp += [0] * (len(each_news_vec) - 1)
            list_temp.append(each_price_vec[1])
            list_temp.append(each_price_vec[2])
            ALL_VEC.append(list_temp)

    CommonUtil.write_csv(p, ALL_VEC)
Beispiel #5
0
def get_value_list():
    value_list = CommonUtil.read_csv(MarketDataCrawler.MARKET_DATA_PATH + '/' +
                                     VALUE_NAME + '.csv')
    for value_i in range(1, len(value_list)):
        date = CommonUtil.get_datetime_from_string_(
            value_list[value_i][0]).date()
        # 收盘价
        value = float(value_list[value_i][2])
        valueDict[date] = value
Beispiel #6
0
def get_news_item(start_date, end_date):
    # 从最新到最远获取新闻
    reverse_year = int(end_date[0:4])
    reverse_month = int(end_date[4:6])
    reverse_day = int(end_date[6:8])
    reverse_pattern = (reverse_year, reverse_month, reverse_day, 23, 59, 59,
                       99, 99, 99)
    reverse_cursor = CommonUtil.convert_date_to_long(reverse_pattern)
    logger.info("reverseCursor is %s" % reverse_cursor)
    finished_year = int(start_date[0:4])
    finished_month = int(start_date[4:6])
    finished_day = int(start_date[6:8])
    finished_pattern = (finished_year, finished_month, finished_day, 0, 0, 0,
                        0, 0, 0)
    finished_cursor = CommonUtil.convert_date_to_long(finished_pattern)
    logger.info("finishedCursor is %s" % finished_cursor)
    # 需要爬数据的网址
    url_pattern = 'https://api-prod.wallstreetcn.com/apiv1/content/lives?' \
                  'channel=weex-channel,gold-channel,gold-forex-channel,' \
                  'forex-channel,goldc-channel,oil-channel&client=pc'
    news_limit = 100
    cursor = reverse_cursor
    file_content = ''
    # 页面计数器
    page_num = 0
    # 循环开始
    while int(cursor) > int(finished_cursor):
        page_num += 1
        url = url_pattern + "&cursor=" + str(cursor) + "&" + str(news_limit)
        logger.info(url)
        page = requests.Session().get(url)
        page.encoding = 'utf-8'
        if page.status_code == 200:
            data_all = json.loads(page.text)
            res_data = data_all['data']
            data_items = res_data['items']
            cursor = res_data['next_cursor']
            for item_i in range(len(data_items)):
                display_time = data_items[item_i]['display_time']
                context_text = data_items[item_i]['content_text']
                context = context_text.strip().replace('\n', '')
                context = context.replace('\r', '')
                time = CommonUtil.convert_long_to_date(display_time)
                file_content = file_content + time + "," + context + "\n"
                # print(item_i+1, ": " , time, ", ", context_text)
        CommonUtil.save_to_file(
            '../files/wallstreetcn_%s_%s.csv' % (start_date, end_date),
            file_content)
        file_content = ''
        # 无下一页数据时退出循环
        if cursor == '':
            break
    logger.info("Finished With %s Pages Crawled." % page_num)
Beispiel #7
0
def news_segment():
    logger.info("In Segment News...")
    count = 1
    for news_item in newsList:
        word_list = BaiduNLPProcessor.lexer(news_item[2])
        word_list.insert(0, news_item[0])
        word_list.insert(1, CommonUtil.get_string_from_datetime(news_item[1]))
        newsSegmentationList.append(word_list)
        logger.info(count)
        count += 1
    CommonUtil.write_csv(SEGMENTED_NEWS_PATH, newsSegmentationList)
    logger.info("Segment News...Done!")
Beispiel #8
0
def adjust_feature_vector():
    feature_vector_list = CommonUtil.read_csv(
        '../files/marketdata/FEATURE_VECTOR.csv')
    pre_item = feature_vector_list[0]
    current_item = pre_item
    for vector_i in range(1, len(feature_vector_list)):
        current_item = feature_vector_list[vector_i]
        for i in range(1, len(current_item)):
            if current_item[i] == 'N/A':
                current_item[i] = pre_item[i]
        feature_vector_list[vector_i] = current_item
        pre_item = current_item
    CommonUtil.write_csv('../files/marketdata/ADJUSTED_FEATURE_VECTOR.csv',
                         feature_vector_list)
Beispiel #9
0
    def check_connection(self, url):
        """
        Check connection status to the server currently  selected by user and
        show a status box indicating current operation.

        :param url: The link of the server chose by user.This string could be
            a domain name or the IP address of a server.

            .. seealso:: :attr:`link` in
                :meth:`~util.common.CommonUtil.check_connection`.
        :type url: str
        :return: A flag indicating connection status is good or not.

            .. seealso:: :meth:`~util.common.CommonUtil.check_connection`. in
                :class:`~util.common.CommonUtil` class.
        :rtype: int
        """
        print("Checking Server Status...")
        conn = CommonUtil.check_connection(url)
        if conn:
            self.statusinfo[0][1] = "OK"
            self.statusinfo[0][2] = "GREEN"
        else:
            self.statusinfo[0][1] = "Error"
            self.statusinfo[0][2] = "RED"
        return conn
Beispiel #10
0
def _delete_local_unused_branch(branch_name=None):
    """
    批量删除本地无用分支
    :param branch_name: 需要删除的branch,支持正则表达式
    """
    if not branch_name:
        print("Error: 请指定要删除的branchName,支持正则,例如:python3 git.py del '7.22|earning|feature'")
        return
    dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
    print(dirs_arr)
    print()
    err_output = open(os.devnull, 'w')  # 隐藏错误输出
    for d in dirs_arr:
        result = os.path.join(Constants.YY_ROOT_DIR, d)
        os.chdir(result)
        try:
            result = subprocess.Popen("git branch | grep -E -i '%s' | xargs git branch -D" % branch_name,
                                      shell=True, stdout=subprocess.PIPE, stderr=err_output)
            print("%s: " % d)
            while result.poll() is None:
                line = result.stdout.readline().strip().decode("utf-8")
                if line:
                    print(line, end='')
                    print()
            print()
        except Exception as e:
            print("del exception = %s" % e, end='')
            print()
Beispiel #11
0
    def __init__(self):
        """
        Initialize a new TUI session.

        * Load server list from a configuration file under working directory.
        * Try to load the hosts data file under working directory if it
          exists.

        .. note:: IF hosts data file does not exists correctly in current
            working directory, a warning message box would popup. And
            operations to change the hosts file on current system could be
            done only until a new data file has been downloaded.

        .. seealso:: :meth:`~tui.curses_d.CursesDaemon.session_daemon` method
            in :class:`~tui.curses_d.CursesDaemon`.

        .. seealso:: :meth:`~gui.hostsutil.HostsUtil.init_main` in
            :class:`~gui.hostsutil.HostsUtil` class.
        """
        super(HostsUtil, self).__init__()
        # Set mirrors
        self.settings[0][2] = CommonUtil.set_network("network.conf")
        # Read data file and set function list
        try:
            self.set_platform()
            RetrieveData.unpack()
            RetrieveData.connect_db()
            self.set_info()
            self.set_func_list()
        except IOError:
            self.messagebox("No data file found! Press F6 to get data file "
                            "first.", 1)
        except BadZipfile:
            self.messagebox("Incorrect Data file! Press F6 to get a new data "
                            "file first.", 1)
Beispiel #12
0
def prepare_raw_news():
    logger.info("In Prepare Raw News...")
    raw_news_data = CommonUtil.read_excel(RAW_NEWS_PATH)
    raw_news_table = raw_news_data.sheet_by_index(0)
    raw_news_rows = raw_news_table.nrows
    for rowN in range(0, raw_news_rows):
        news_item = list()
        news_index = int(raw_news_table.cell_value(rowN, 0))
        news_time = CommonUtil.get_datetime_from_cell(
            raw_news_table.cell_value(rowN, 1))
        news_content = raw_news_table.cell_value(rowN, 2)
        news_item.append(news_index)
        news_item.append(news_time)
        news_item.append(news_content)
        newsList.append(news_item)
    logger.info("Prepare Raw News...Done!")
Beispiel #13
0
    def check_connection(self, url):
        """
        Check connection status to the server currently  selected by user and
        show a status box indicating current operation.

        :param url: The link of the server chose by user.This string could be
            a domain name or the IP address of a server.

            .. seealso:: :attr:`link` in
                :meth:`~util.common.CommonUtil.check_connection`.
        :type url: str
        :return: A flag indicating connection status is good or not.

            .. seealso:: :meth:`~util.common.CommonUtil.check_connection`. in
                :class:`~util.common.CommonUtil` class.
        :rtype: int
        """
        self.messagebox("Checking Server Status...")
        conn = CommonUtil.check_connection(url)
        if conn:
            self.statusinfo[0][1] = "OK"
            self.statusinfo[0][2] = "GREEN"
        else:
            self.statusinfo[0][1] = "Error"
            self.statusinfo[0][2] = "RED"
        self.status()
        return conn
Beispiel #14
0
 def set_info(self):
     """
     Set the information of the current local data file.
     """
     info = RetrieveData.get_info()
     build = info["Buildtime"]
     self.hostsinfo["Version"] = info["Version"]
     self.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
Beispiel #15
0
 def get_custom_conf_path(cls):
     if not CommonUtil.check_platform()[0] == "Windows":
         path = os.path.expanduser('~/.custom.hosts')
         if not os.path.isfile(path):
             path = os.path.expanduser('~/custom.hosts')
         if os.path.isfile(path):
             return path
     return None
Beispiel #16
0
def reduce_feature_vector():
    logger.info("In Reduce Feature Vector...")
    prepare_feature()
    origin_feature_num = len(featureDict.keys())
    global featureVectorList
    reduced_feature_vector_list = list()
    feature_list = list()
    feature_count_threshold = 2
    file_path = FEATURE_VECTOR_PATH + '_' + str(
        PRICE_SAMPLE_MINUTE) + CSV_FILE_SUFFIX
    featureVectorList = CommonUtil.read_csv(file_path)
    feature_count_dict = dict()
    feature_count_list = [0] * origin_feature_num
    is_title = True
    for feature_vector in featureVectorList:
        if is_title:
            is_title = False
        else:
            for feature_value_index in range(0, origin_feature_num):
                if feature_vector[feature_value_index] != '0.0':
                    feature_count_list[feature_value_index] += 1
    feature_index = 0
    for key in featureDict.keys():
        feature_count = feature_count_list[feature_index]
        feature_count_dict[key] = feature_count
        if feature_count >= feature_count_threshold:
            feature_list.append(feature_index)
        feature_index += 1
    logger.info(str('Reduce Feature Vector to: ' + str(len(feature_list))))
    feature_list.append(origin_feature_num)
    # 拼装计数超过阈值的特征向量
    for feature_vector in featureVectorList:
        reduced_feature_vector = list()
        for feature_value_index in range(0, origin_feature_num + 1):
            if feature_value_index in feature_list:
                try:
                    reduced_feature_vector.append(
                        feature_vector[feature_value_index])
                except IndexError:
                    logger.error(feature_vector)
                    logger.error(feature_value_index)
        reduced_feature_vector_list.append(reduced_feature_vector)
    file_path = REDUCED_FEATURE_VECTOR_PATH + '_' + str(
        PRICE_SAMPLE_MINUTE) + CSV_FILE_SUFFIX
    CommonUtil.write_csv(file_path, reduced_feature_vector_list)
    logger.info("Reduce Feature Vector Done!")
Beispiel #17
0
 def run(self):
     """
     Start operations to check the network connection with a specified
     server.
     """
     self.trigger.emit(-1)
     status = CommonUtil.check_connection(self.link)
     self.trigger.emit(status)
Beispiel #18
0
 def get_custom_conf_path(cls):
     if not CommonUtil.check_platform()[0] == "Windows":
         path = os.path.expanduser('~/.custom.hosts')
         if not os.path.isfile(path):
             path = os.path.expanduser('~/custom.hosts')
         if os.path.isfile(path):
             return path
     return None
Beispiel #19
0
    def process_bar(self, done, block, total, mode=1):
        """
        Draw `Process Bar` at the bottom which is used to indicate progress of
        downloading operation.

        .. note:: This method is a callback function responses to
            :meth:`urllib.urlretrieve` method while fetching hosts data
            file.

        :param done: Block count of packaged retrieved.
        :type done: int
        :param block: Block size of the data pack retrieved.
        :type block: int
        :param total: Total size of the hosts data file.
        :type total: int
        :param mode: A flag indicating the status of `Process Bar`.
            The default value of `mode` is `1`.

            ====  ====================================
            mode  `Process Bar` status
            ====  ====================================
            1     Downloading operation is processing.
            0     Not downloading.
            ====  ====================================
        :type mode: int
        """
        screen = self._stdscr.subwin(2, 80, 20, 0)
        screen.bkgd(' ', curses.color_pair(4))
        normal = curses.A_NORMAL
        line_width = 76
        prog_len = line_width - 20
        # Progress Bar
        if mode:
            done *= block
            prog = 1.0 * prog_len * done / total
            progress = ''.join(['=' * int(prog), '-' * int(2 * prog % 2)])
            progress = progress.ljust(prog_len)
            total = CommonUtil.convert_size(total).ljust(7)
            done = CommonUtil.convert_size(done).rjust(7)
        else:
            progress = ' ' * prog_len
            done = total = 'N/A'.center(7)
        # Show Progress
        prog_bar = "[%s] %s | %s" % (progress, done, total)
        screen.addstr(1, 2, prog_bar, normal)
        screen.refresh()
Beispiel #20
0
 def set_info(self):
     """
     Set the information of the current local data file.
     """
     info = RetrieveData.get_info()
     build = info["Buildtime"]
     self.hostsinfo["Version"] = info["Version"]
     self.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
Beispiel #21
0
 def set_info(cls):
     """
     Set the information of the current local data file.
     Modified from tui.hostsutil.HostsUtil.set_info()
     """
     info = RetrieveData.get_info()
     build = info["Buildtime"]
     cls.hostsinfo["Version"] = info["Version"]
     cls.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
Beispiel #22
0
    def set_progress(self, done, block, total):
        """
        Send message to the main dialog to set the progress bar.

        :param done: Block count of packaged retrieved.
        :type done: int
        :param block: Block size of the data pack retrieved.
        :type block: int
        :param total: Total size of the hosts data file.
        :type total: int
        """
        done = done * block
        if total <= 0:
            total = self.filesize
        prog = 100 * done / total
        done = CommonUtil.convert_size(done)
        total = CommonUtil.convert_size(total)
        text = unicode(_translate(
            "Util", "Downloading: %s / %s", None)) % (done, total)
        self.prog_trigger.emit(prog, text)
Beispiel #23
0
    def set_progress(self, done, block, total):
        """
        Send message to the main dialog to set the progress bar.

        :param done: Block count of packaged retrieved.
        :type done: int
        :param block: Block size of the data pack retrieved.
        :type block: int
        :param total: Total size of the hosts data file.
        :type total: int
        """
        done = done * block
        if total <= 0:
            total = self.filesize
        prog = 100 * done / total
        done = CommonUtil.convert_size(done)
        total = CommonUtil.convert_size(total)
        text = unicode(_translate("Util", "Downloading: %s / %s",
                                  None)) % (done, total)
        self.prog_trigger.emit(prog, text)
Beispiel #24
0
 def set_info(self):
     """
     Set the information of the current local data file.
     """
     info = RetrieveData.get_info()
     ver = info["Version"]
     self._cur_ver = ver
     self.set_label_text(self.ui.labelVersionData, ver)
     build = info["Buildtime"]
     build = CommonUtil.timestamp_to_date(build)
     self.set_label_text(self.ui.labelReleaseData, unicode(build))
Beispiel #25
0
    def check_connection(cls):
        """
        Check connection status to the servers.
        Modified from tui.curses_d.CursesDaemon.check_connection()
        """
        cls.show_status("Checking Server Status ...")
        cls.settings[0][2] = CommonUtil.set_network(cls.netfile)
        srv_id = 0
        for  srv_id,server in enumerate(cls.settings[0][2]):
            cls.show_status("Trying to connect to "+server["tag"]+"...")
            url = server["test_url"]
            conn = CommonUtil.check_connection(url)
            if conn :
                cls.show_status("Connected!")
                cls.settings[0][1] = srv_id
                return
            cls.show_status("Failed.")

        cls.show_status("Can not connect to any server!")
        cls.show_status("Script exits now...")
        sys.exit(1)
Beispiel #26
0
def decay_influence(dt_news_time, dt_current_time):
    delta_seconds = CommonUtil.get_interval_seconds(dt_current_time,
                                                    dt_news_time)
    if delta_seconds > NEWS_INFLUENCE_DACAY_THRESHOLD * 60:
        return 0
    # 0-60秒
    if delta_seconds <= NEWS_INFLUENCE_MOST * 60:
        influence_score = 1 / (NEWS_INFLUENCE_MOST * 60) * delta_seconds
    else:
        influence_score = NEWS_INFLUENCE_DACAY_THRESHOLD/(NEWS_INFLUENCE_DACAY_THRESHOLD - NEWS_INFLUENCE_MOST) \
                          - 1/(NEWS_INFLUENCE_DACAY_THRESHOLD - NEWS_INFLUENCE_MOST) * delta_seconds / 60
    return round(influence_score * FEATURE_VECTOR_SCALE,
                 CURRENCY_PAIR_PRECISION)
Beispiel #27
0
 def set_platform(cls):
     """
     Set the information about current operating system.
     Modified from tui.hostsutil.HostsUtil.set_platform()
     """
     system, hostname, path, encode, flag = CommonUtil.check_platform()
     cls.platform = system
     cls.hostname = hostname
     cls.hosts_path = path
     if encode == "win_ansi":
         cls.sys_eol = "\r\n"
     else:
         cls.sys_eol = "\n"
Beispiel #28
0
 def set_platform(self):
     """
     Set the information of current operating system platform.
     """
     system, hostname, path, encode, flag = CommonUtil.check_platform()
     self.platform = system
     self.hostname = hostname
     self.hosts_path = path
     self.plat_flag = flag
     if encode == "win_ansi":
         self.sys_eol = "\r\n"
     else:
         self.sys_eol = "\n"
Beispiel #29
0
 def set_platform(self):
     """
     Set the information of current operating system platform.
     """
     system, hostname, path, encode, flag = CommonUtil.check_platform()
     self.platform = system
     self.hostname = hostname
     self.hosts_path = path
     self.plat_flag = flag
     if encode == "win_ansi":
         self.sys_eol = "\r\n"
     else:
         self.sys_eol = "\n"
Beispiel #30
0
def prepare_feature():
    logger.info("In Prepare Feature...")
    # 获取sheet
    feature_data = CommonUtil.read_excel(FEATURE_PATH)
    feature_table = feature_data.sheet_by_index(0)
    # 获取总行数
    feature_rows = feature_table.nrows
    # 获取总列数
    # feature_cols = feature_table.ncols
    for rowNum in range(1, feature_rows):
        key = feature_table.cell_value(rowNum, 0)
        value = feature_table.cell_value(rowNum, 1)
        featureDict[key] = value
    logger.info("Prepare Feature...Done!")
Beispiel #31
0
def _checkout_branch(branch_name=None):
    """
    批量检出指定分支,支持正则表达式
    :param branch_name 分支名,不可为空
    """
    if not branch_name:
        print("请输入要切换的分支名称,支持正则")
        return
    dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
    # dirs_arr = ['pluginlivebasemedia', 'ycloud', 'entmobile', 'entlive', 'livebasebiz']
    for d in dirs_arr:
        result = os.path.join(Constants.YY_ROOT_DIR, d)
        os.chdir(result)
        try:
            print("%s: " % d)
            # 先检查本地分支
            try:
                local_branch = subprocess.check_output('git branch | grep %s' % branch_name, shell=True).decode()
                local_branch_arr = list(filter(None, local_branch.split("\n")))  # type: list
                if len(local_branch_arr) == 1:
                    if '*' not in local_branch_arr[0]:
                        subprocess.call('git checkout %s' % local_branch_arr[0], shell=True)  # 检出匹配到的本地分支
                    else:
                        print("当前分支 %s 已经是目标分支,无需切换" % local_branch_arr[0])
                    continue
                elif len(local_branch_arr) > 1:
                    print("匹配到多个本地分支,请细化正则表达式后重试")
                    continue
            except:
                # 再检查远程分支
                subprocess.check_output(['git', 'fetch'])
                try:
                    remote_branch = subprocess.check_output('git branch -r | grep %s' % branch_name,
                                                            shell=True).decode()
                    remote_branch_arr = list(filter(None, remote_branch.split("\n")))  # type: list
                    if len(remote_branch_arr) == 1:
                        first_remote_branch = remote_branch_arr[0]
                        format_remote_branch = first_remote_branch.split("/")[-1]
                        subprocess.call('git checkout -b %s %s' % (format_remote_branch, first_remote_branch),
                                        shell=True)  # 检出匹配到的远程分支
                        continue
                    elif len(remote_branch_arr) > 1:
                        print("匹配到多个远程分支,请细化正则表达式后重试")
                        continue
                except:
                    print("未匹配到任何分支,请检查正则表达式是否正确")

        except Exception as e:
            print(e, end='')
Beispiel #32
0
    def check_writable(self):
        """
        Check if current session has write privileges to the hosts file.

        .. note:: IF current session does not has the write privileges to the
            hosts file of current system, a warning message box would popup.

        .. note:: ALL operation would change the `hosts` file on current
            system could only be done while current session has write
            privileges to the file.
        """
        self._writable = CommonUtil.check_privileges()[1]
        if not self._writable:
            print("Please check if you have writing\n"
                            "privileges to the hosts file!")
Beispiel #33
0
 def set_platform(self):
     """
     Set the information about current operating system.
     """
     system, hostname, path, encode, flag = CommonUtil.check_platform()
     color = "GREEN" if flag else "RED"
     self.platform = system
     self.statusinfo[1][1] = system
     self.hostname = hostname
     self.hosts_path = path
     self.statusinfo[1][2] = color
     if encode == "win_ansi":
         self.sys_eol = "\r\n"
     else:
         self.sys_eol = "\n"
Beispiel #34
0
 def set_platform(self):
     """
     Set the information about current operating system.
     """
     system, hostname, path, encode, flag = CommonUtil.check_platform()
     color = "GREEN" if flag else "RED"
     self.platform = system
     self.statusinfo[1][1] = system
     self.hostname = hostname
     self.hosts_path = path
     self.statusinfo[1][2] = color
     if encode == "win_ansi":
         self.sys_eol = "\r\n"
     else:
         self.sys_eol = "\n"
Beispiel #35
0
def get_feature():
    # 特征字典[AAAA:黄金]
    feature_dict = dict()
    # 获取sheet
    feature_data = CommonUtil.read_excel(FEATURE_PATH)
    feature_table = feature_data.sheet_by_index(0)
    # 获取总行数
    feature_rows = feature_table.nrows
    # 获取总列数
    # feature_cols = feature_table.ncols
    for rowNum in range(1, feature_rows):
        key = feature_table.cell_value(rowNum, 0)
        value = feature_table.cell_value(rowNum, 1)
        feature_dict[key] = value
    return feature_dict
Beispiel #36
0
def get_market_data(i_type, i_count, s_data_type):
    url_pattern = 'https://forexdata.wallstreetcn.com/kline?prod_code=' + s_data_type + \
                  '&candle_period=' + str(i_type) + \
                  '&fields=time_stamp,open_px,close_px,high_px,low_px,ma5,ma10,ma20,ma60,upper,mid,lower,diff,dea,' \
                  'macd,k,d,j,rsi6,rsi12,rsi24&data_count=' + str(i_count)  # 需要爬数据的网址
    logger.info(url_pattern)
    page = requests.Session().get(url_pattern)
    page.encoding = 'utf-8'
    file_content = ''
    if page.status_code == 200:
        data_all = json.loads(page.text)
        res_data = data_all['data']
        candle_data = res_data['candle']
        # 处理标题
        data_fields = candle_data['fields']
        for item_i in range(len(data_fields)):
            file_content += data_fields[item_i] + ','
        file_content += '\n'
        # 处理数据
        data_list = candle_data[s_data_type]
        for item_i in range(len(data_list)):
            data_items = data_list[item_i]
            data_item = ''
            for item_j in range(len(data_items)):
                # 日期格式转换
                if item_j == 0:
                    data_item += CommonUtil.convert_long_to_date(
                        data_items[item_j]) + ','
                else:
                    data_item += str(data_items[item_j]) + ','
            file_content += data_item + '\n'
        logger.info("Finished With %s Items Crawled." % (len(data_list)))
        CommonUtil.save_to_file(MARKET_DATA_PATH + '/%s.csv' % s_data_type,
                                file_content)
    else:
        logger.warning("Response Code is %s, Please Check!" % page.status_code)
Beispiel #37
0
    def check_writable(self):
        """
        Check if current session is ran with root privileges.

        .. note:: IF current session does not has the write privileges to the
            hosts file of current system, a warning message box would popup.

        .. note:: ALL operation would change the `hosts` file on current
            system could only be done while current session has write
            privileges to the file.
        """
        writable = CommonUtil.check_privileges()[1]
        self._writable = writable
        if not writable:
            self.warning_permission()
Beispiel #38
0
    def check_writable(self):
        """
        Check if current session is ran with root privileges.

        .. note:: IF current session does not has the write privileges to the
            hosts file of current system, a warning message box would popup.

        .. note:: ALL operation would change the `hosts` file on current
            system could only be done while current session has write
            privileges to the file.
        """
        writable = CommonUtil.check_privileges()[1]
        self._writable = writable
        if not writable:
            self.warning_permission()
Beispiel #39
0
def _list_branch():
    dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
    print(dirs_arr)
    print()
    for d in dirs_arr:
        path = os.path.join(Constants.YY_ROOT_DIR, d)
        print(path)
        os.chdir(path)
        try:
            result = subprocess.check_output('git branch', shell=True).decode()
            if result:
                print("%s: " % d)
                print(result, end='')
                print()
        except Exception as e:
            print(e, end='')
Beispiel #40
0
 def weixin_login(self):
     url = "https://mp.weixin.qq.com/cgi-bin/bizlogin?action=startlogin"
     params = {
         'username': self.username,
         'pwd': CommonUtil.md5(self.password),
         'imgcode': '',
         'f': 'json'
     }
     response = self.session.post(url,
                                  data=params,
                                  headers=self.headers,
                                  verify=False)
     if response.status_code == 200:
         target = response.content.decode('utf-8')
         print(target)
         self.get_weixin_login_qrcode()
Beispiel #41
0
    def check_writable(self):
        """
        Check if current session has write privileges to the hosts file.

        .. note:: IF current session does not has the write privileges to the
            hosts file of current system, a warning message box would popup.

        .. note:: ALL operation would change the `hosts` file on current
            system could only be done while current session has write
            privileges to the file.
        """
        self._writable = CommonUtil.check_privileges()[1]
        if not self._writable:
            self.messagebox(
                "Please check if you have writing\n"
                "privileges to the hosts file!", 1)
            exit()
    def launch_tui(cls):
        """
        Start a Text-based User Interface (TUI) session of
        `Hosts Setup Utility`.

        .. note:: This is a `classmethod`.
        """

        # Set code page for Windows
        system = CommonUtil.check_platform()[0]
        cp = "437"
        if system == "Windows":
            chcp = os.popen("chcp")
            cp = chcp.read().split()[-1]
            chcp.close()
            os.popen("chcp 437")

        main = tui.HostsUtil()
        main.start()

        # Restore the default code page for Windows
        if system == "Windows":
            os.popen("chcp " + cp)
Beispiel #43
0
    def init_main(self):
        """
        Set up the elements on the main dialog. Check the environment of
        current operating system and current session.

        * Load server list from a configuration file under working directory.
        * Try to load the hosts data file under working directory if it
          exists.

        .. note:: IF hosts data file does not exists correctly in current
            working directory, a warning message box would popup. And
            operations to change the hosts file on current system could be
            done only until a new data file has been downloaded.

        .. seealso:: Method :meth:`~tui.hostsutil.HostsUtil.__init__` in
            :class:`~tui.hostsutil.HostsUtil` class.
        """
        self.ui.SelectMirror.clear()
        self.set_version()
        # Set mirrors
        self.mirrors = CommonUtil.set_network("network.conf")
        self.set_mirrors()
        # Read data file and set function list
        try:
            RetrieveData.unpack()
            RetrieveData.connect_db()
            self.set_func_list(1)
            self.refresh_func_list()
            self.set_info()
        except IOError:
            self.warning_no_datafile()
        except BadZipfile:
            self.warning_incorrect_datafile()
        # Check if current session have root privileges
        self.check_writable()
        self.init_flag += 1
Beispiel #44
0
        for name_format in excludes:
            ex_files.extend(glob.glob(file_path(name_format)))
        for name_format in includes:
            files = glob.glob(file_path(name_format))
            for src_file in files:
                if src_file not in ex_files:
                    tar_path = os.path.join(prefix + VERSION,
                                            src_file[rel_len:])
                    tar.add(src_file, tar_path)
                    print "compressing: %s" % src_file
        tar.close()
        exit(1)

from util import CommonUtil

system = CommonUtil.check_platform()[0]
if system == "Windows":
    # Build binary executables for Windows
    import struct
    import zipfile
    from distutils.core import setup
    import py2exe

    # Set working directories
    WORK_DIR = SCRIPT_DIR + "work/"
    DIR_NAME = "HostsTool"
    DIST_DIR = WORK_DIR + DIR_NAME + '/'
    WIN_OPTIONS = {
        "includes": ["sip"],
        "excludes": ["_scproxy", "_sysconfigdata"],
        "dll_excludes": ["MSVCP90.dll"],
Beispiel #45
0
    def messagebox(msg, mode=0):
        """
        Draw a `Message Box` with :attr:`msg` in a specified type defined by
        :attr:`mode`.

        .. note:: This is a `static` method.

        :param msg: The information to be displayed in message box.
        :type msg: str
        :param mode: A flag indicating the type of message box to be
            displayed. The default value of `mode` is `0`.

            ====  ===========================================================
            mode  Message Box Type
            ====  ===========================================================
            0     A simple message box showing a message without any buttons.
            1     A message box with an `OK` button for user to confirm.
            2     A message box with `OK` and `Cancel` buttons for user to
                  choose.
            ====  ===========================================================
        :type mode: int
        :return: A flag indicating the choice made by user.

            ======  =====================================================
            Return  Condition
            ======  =====================================================
            0       #. No button is pressed while :attr:`mode` is `0`.
                    #. `OK` button pressed while :attr:`mode` is `1`.
                    #. `Cancel` button pressed while :attr:`mode` is `2`.
            1       `OK` button pressed while :attr:`mode` is `2`.
            ======  =====================================================
        :rtype: int
        """
        pos_x = 24 if mode == 0 else 20
        pos_y = 10
        width = 30 if mode == 0 else 40
        height = 2
        messages = CommonUtil.cut_message(msg, width - 4)
        height += len(messages)
        if mode:
            height += 2
        # Draw Shadow
        shadow = curses.newwin(height, width, pos_y + 1, pos_x + 1)
        shadow.bkgd(' ', curses.color_pair(8))
        shadow.refresh()
        # Draw Subwindow
        screen = curses.newwin(height, width, pos_y, pos_x)
        screen.box()
        screen.bkgd(' ', curses.color_pair(2))
        screen.keypad(1)
        # Set local variable
        normal = curses.A_NORMAL
        select = curses.A_REVERSE
        # Insert messages
        for i in range(len(messages)):
            screen.addstr(1 + i, 2, messages[i].center(width - 4), normal)
        if mode == 0:
            screen.refresh()
        else:
            # Draw subwindow frame
            line_height = 1 + len(messages)
            screen.hline(line_height, 1, curses.ACS_HLINE, width - 2)
            screen.addch(line_height, 0, curses.ACS_SSSB)
            screen.addch(line_height, width - 1, curses.ACS_SBSS)
            tab = 0
            key_in = None
            while key_in != 27:
                if mode == 1:
                    choices = ["OK"]
                elif mode == 2:
                    choices = ["OK", "Cancel"]
                else:
                    return 0
                for i, item in enumerate(choices):
                    item_str = ''.join(['[', item, ']'])
                    tab_pos_x = 6 + 20 * i if mode == 2 else 18
                    screen.addstr(line_height + 1, tab_pos_x, item_str,
                                  select if i == tab else normal)
                screen.refresh()
                key_in = screen.getch()
                if mode == 2:
                    # OK or Cancel
                    if key_in in [9, curses.KEY_LEFT, curses.KEY_RIGHT]:
                        tab = [1, 0][tab]
                    if key_in in [ord('a'), ord('c')]:
                        key_in -= (ord('a') - ord('A'))
                    if key_in in [ord('C'), ord('O')]:
                        return [ord('C'), ord('O')].index(key_in)
                if key_in in [10, 32]:
                    return not tab
            return 0