Пример #1
0
def QA_SU_save_financial_files():
    """本地存储financialdata
    """
    download_financialzip()
    coll = DATABASE.financial
    coll.create_index([("code", ASCENDING), ("report_date", ASCENDING)],
                      unique=True)
    for item in os.listdir(download_path):
        if item[0:4] != 'gpcw':
            print(
                "file ", item,
                " is not start with gpcw , seems not a financial file , ignore!"
            )
            continue

        date = int(item.split('.')[0][-8:])
        print('QUANTAXIS NOW SAVING {}'.format(date))
        if coll.find({'report_date': date}).count() < 100:

            print(coll.find({'report_date': date}).count())
            data = QA_util_to_json_from_pandas(
                parse_filelist([item]).reset_index().drop_duplicates(
                    subset=['code', 'report_date']).sort_index())
            try:
                coll.insert_many(data, ordered=False)

            except Exception as e:
                if isinstance(e, MemoryError):
                    coll.insert_many(data, ordered=True)
                elif isinstance(e, pymongo.bulk.BulkWriteError):
                    pass
        else:
            print('ALL READY IN DATABASE')

    print('SUCCESSFULLY SAVE/UPDATE FINANCIAL DATA')
Пример #2
0
def QA_SU_save_financial_files():
    """本地存储financialdata
    """
    download_financialzip()
    coll = DATABASE.financial
    coll.create_index(
        [("code", ASCENDING), ("report_date", ASCENDING)], unique=True)
    for item in os.listdir(download_path):
        if item[0:4] != 'gpcw':
            print(
                "file ", item, " is not start with gpcw , seems not a financial file , ignore!")
            continue

        date = int(item.split('.')[0][-8:])
        print('QUANTAXIS NOW SAVING {}'.format(date))
        if coll.find({'report_date': date}).count() < 3600:

            print(coll.find({'report_date': date}).count())
            data = QA_util_to_json_from_pandas(parse_filelist([item]).reset_index(
            ).drop_duplicates(subset=['code', 'report_date']).sort_index())
            # data["crawl_date"] = str(datetime.date.today())
            try:
                coll.insert_many(data, ordered=False)

            except Exception as e:
                if isinstance(e, MemoryError):
                    coll.insert_many(data, ordered=True)
                elif isinstance(e, pymongo.bulk.BulkWriteError):
                    pass
        else:
            print('ALL READY IN DATABASE')

    print('SUCCESSFULLY SAVE/UPDATE FINANCIAL DATA')
Пример #3
0
def QA_SU_save_financial_files(fromtdx=False):
    """本地存储financialdata
    """
    if (fromtdx):
        download_financialzip_fromtdx()
    else:
        download_financialzip()

    coll = DATABASE.financial
    coll.create_index([("code", ASCENDING), ("report_date", ASCENDING)],
                      unique=True)
    for item in os.listdir(download_path):
        if item[0:4] != 'gpcw':
            print(
                "file ", item,
                " is not start with gpcw , seems not a financial file , ignore!"
            )
            continue

        date = int(item.split('.')[0][-8:])
        print('QUANTAXIS NOW SAVING {}'.format(date))
        print('在数据库中的条数 {}'.format(coll.find({'report_date': date}).count()))
        try:
            data = QA_util_to_json_from_pandas(
                parse_filelist([item]).reset_index().drop_duplicates(
                    subset=['code', 'report_date']).sort_index())
            print('即将更新的条数 {}'.format(len(data)))
            # data["crawl_date"] = str(datetime.date.today())
            try:
                for d in data:
                    coll.update_one(
                        {
                            'code': d['code'],
                            'report_date': d['report_date']
                        }, {'$set': d},
                        upsert=True)

            except Exception as e:
                if isinstance(e, MemoryError):
                    coll.insert_many(data, ordered=True)
                elif isinstance(e, pymongo.bulk.BulkWriteError):
                    pass
        except Exception as e:
            print('似乎没有数据')

    print('SUCCESSFULLY SAVE/UPDATE FINANCIAL DATA')