Пример #1
0
def export_users_schema_process():
    dump_date = get_dump_date(MONGO, ARCHIVES_BASE_FOLDER)
    if is_dump_date_valid(dump_date):
        export_destination = os.path.join(ARCHIVES_BASE_FOLDER,
                                          f"{MONGO}-{USERS}-{dump_date}.csv")
        mongodb = MongoDB()
        mongodb.export_users_schema(export_destination, USERS_SCHEMA)
Пример #2
0
def restore_dump_process():
    dump_date = get_dump_date(MONGO, ARCHIVES_BASE_FOLDER)
    if is_dump_date_valid(dump_date):
        mongodb = MongoDB()
        bson_file = os.path.join(
            get_dump_folder_endpoint(ARCHIVES_BASE_FOLDER, MONGO, dump_date),
            f'{USERS}.bson')
        mongodb.restore_db(bson_file, USERS_SCHEMA)
Пример #3
0
 def __init__():
     find_time = int((time.time())*1000)-300000
     filetime = time.strftime("%Y%m%d%H%M%S", time.localtime())
     filename = 'inc_app_{}.dat'.format(filetime)
     # f = open(r'./app_full.dat', 'a', encoding='utf-8')
     mog_content = MongoDB(environment=environment,db_name=db_name_contents).client
     mog_topic = MongoDB(environment=environment,db_name=db_name_topic).client
     mys=MysqlDB(environment=environment,db_name=da_name_sql).getClient()
     cur = mys.cursor()
Пример #4
0
    def mongodb(self) -> MongoDB:
        if not self.has_mongodb():
            raise NoConnectionError('No MongoDB configuration')

        mongodb = MongoDB(self._mongodb_configuration)
        mongodb.connect()

        self._instances.append(mongodb)

        return mongodb
Пример #5
0
def remove_duplicates_process():
    dump_date = get_dump_date(MONGO, ARCHIVES_BASE_FOLDER)
    if is_dump_date_valid(dump_date):
        mongodb = MongoDB()
        mongodb.connect()
        mongodb.remove_duplicates()
        mongodb.disconnect()
Пример #6
0
def drop_database_process():
    dump_date = get_dump_date(MONGO, ARCHIVES_BASE_FOLDER)
    if is_dump_date_valid(dump_date):
        mongodb = MongoDB()
        mongodb.connect()
        mongodb.drop_database()
        mongodb.disconnect()
Пример #7
0
def remove_documents_with_null_values_process():
    dump_date = get_dump_date(MONGO, ARCHIVES_BASE_FOLDER)
    if is_dump_date_valid(dump_date):
        mongodb = MongoDB()
        mongodb.connect()
        mongodb.remove_documents_with_null_values()
        mongodb.disconnect()
Пример #8
0
 def start_requests(self):
     self.client = MongoDB(environment=environment, db_name=db_name).client
     zds = self.client.dailypops.hotword.find({"article_state": 0})
     for k in zds:
         parm = k.get("hotword", "")
         url = "http://newssearch.chinadaily.com.cn/rest/en/search?keywords={}&sort=dp&page=0&curType=story&type=&channel=&source=".format(
             parm)
         yield scrapy.Request(url=url,
                              callback=self.parse,
                              meta={"data": k})
Пример #9
0
import uvicorn
from fastapi import FastAPI, Path, Query, HTTPException
from starlette.responses import JSONResponse
from typing import Optional
from fastapi.middleware.cors import CORSMiddleware

from database.mongodb import MongoDB
from config.development import config
from model.Tree import createShoptreeModel, updateShoptreeModel, createNameshoptreeModel, updateNameshoptreeModel

mongo_config = config["mongo_config"]
mongo_db = MongoDB(
    mongo_config["host"],
    mongo_config["port"],
    mongo_config["user"],
    mongo_config["password"],
    mongo_config["auth_db"],
    mongo_config["db"],
    mongo_config["collection"],
)
mongo_db._connect()

mongo_config_area = config["mongo_config_area"]
mongo_db_area = MongoDB(
    mongo_config_area["host"],
    mongo_config_area["port"],
    mongo_config_area["user"],
    mongo_config_area["password"],
    mongo_config_area["auth_db"],
    mongo_config_area["db"],
    mongo_config_area["collection"],
Пример #10
0
#?  {
#?         "host": "localhost",
#?         "port": 27017,
#?         "user": "******",
#?         "password": "******",
#?         "auth_db": "admin",
#?         "db": "bakerypj",
#?         "collection": "bakery",
#? }

mongo_db = MongoDB(
    mongo_config["host"],
    mongo_config["port"],
    mongo_config["user"],
    mongo_config["password"],
    mongo_config["auth_db"],
    mongo_config["db"],
    mongo_config["collection"],
)
mongo_db._connect()  #!เชื่อมต่อกับDBแล้ว

app = FastAPI()  #!ทำการใช้API

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)
Пример #11
0
import uvicorn
from fastapi import FastAPI, Path, Query, HTTPException
from starlette.responses import JSONResponse
from typing import Optional
from fastapi.middleware.cors import CORSMiddleware

from database.mongodb import MongoDB
from config.development import config
from model.student import createStudentModel, updateStudentModel

mongo_config = config["mongo_config"]
mongo_db = MongoDB(  # เรียกใช้MongoDBเเละส่งค่า
    mongo_config["host"],
    mongo_config["port"],
    mongo_config["user"],
    mongo_config["password"],
    mongo_config["auth_db"],
    mongo_config["db"],
    mongo_config["collection"],
)
mongo_db._connect()  # mongodb

app = FastAPI()

app.add_middleware(  # ตัวคั่นกลาง
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)
Пример #12
0
from starlette.responses import JSONResponse
from typing import Optional
from fastapi.middleware.cors import CORSMiddleware

from database.mongodb import MongoDB
from config.development import config
from model.student import createStudentModel, updateStudentModel


mongo_config = config["mongo_config"]

mongo_db = MongoDB(  # import MongoDB จาก mongodb.py
    mongo_config["host"],
    mongo_config["port"],
    mongo_config["user"],
    mongo_config["password"],
    mongo_config["auth_db"],
    mongo_config["db"],
    mongo_config["collection"],
)
mongo_db._connect()  # เชื่อมกับ mongodb

app = FastAPI()

app.add_middleware(  # middleare เหมือนตัวคั่นกลาง เป็นส่วนนึงของ api
    CORSMiddleware,
    allow_origins=["*"],  # * คือ ทุกคนสามารถเข้าถึงได้หมด
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)
Пример #13
0
mongo_config = config["mongo_config"]
print("Mongo_config", mongo_config)

# "host" :"location",
# "port" : "27017",
# "user" : "root",
# "password" : "root",
# "auth" : "admin",
# "db" : "waterpark",
# "collection" : "rides",

mongo_db = MongoDB(
    mongo_config["host"],
    mongo_config["port"],
    mongo_config["user"],
    mongo_config["password"],
    mongo_config["auth_db"],
    mongo_config["db"],
    mongo_config["collection"],
)
mongo_db._connect()  # กับdb

# use API
app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
Пример #14
0
def search():
    client = MongoDB(environment=environment, db_name=db_name).client
    zds = client.dailypops.hotword.find({"question_state": 0}).limit(1000)
    # driver = webdriver.Firefox(options=options)
    # driver = webdriver.Firefox()
    path = r'C:\Users\EDZ\Documents\WeChat Files\wodexinwolai\FileStorage\File\2019-05/chromedriver'

    chrome_options = Options()
    chrome_options.add_argument('--headless')
    driver = webdriver.Chrome(executable_path=path,
                              chrome_options=chrome_options)
    begin_url = "https://www.google.com/"
    driver.get(begin_url)
    driver.find_element_by_id("gb_70").click()
    driver.find_element_by_id("identifierId").send_keys(
        "*****@*****.**")
    driver.find_element_by_id("identifierNext").click()
    time.sleep(3)
    driver.find_element_by_xpath("//input[@name='password']").send_keys(
        "jiexin88")
    driver.find_element_by_id("passwordNext").click()
    time.sleep(5)
    for k in zds:
        # url = 'https://www.google.com/search?sxsrf=ACYBGNRgCAf2dRIVd6dwrtD4B82G2GPK7A%3A1569392173168&ei=LQaLXe35CcmmmAX5k50o&q=trump&oq={}&gs_l=psy-ab.3..35i39l2j0i131j0i3j0i131j0j0i3j0i131l2j0.4634.7621..8636...1.2..3.398.1517.0j7j1j1......0....1..gws-wiz.....10..0i71j0i67j0i131i67j35i362i39j0i131i273j0i273.jn_vf2Z0qbo&ved=0ahUKEwitxPq3qevkAhVJE6YKHflJBwUQ4dUDCAs&uact=5'
        url = "https://www.google.com/search?biw=1536&bih=890&ei=ZomJXceaOtCbmAWyi7egCg&q={}&oq=commp&gs_l=psy-ab.3.1.0i67l3j0i10l7.8182.9766..12758...0.0..0.135.565.0j5......0....1..gws-wiz.......0.KfvdJE90Egw"
        print('参数', k)
        hotword = k.get("hotword", "")
        hotword_id = k.get("hotword_id", "")
        event_id = k.get("event_id", "")
        hotword = hotword.split(" ")
        print('hotword_list', hotword)
        parms = '+'.join(hotword)
        # parms = 'competition'
        print('parms', parms)
        url = url.format(parms)
        # url_ = url.format(parms)

        print('url_', url)
        driver.get(url)
        # driver.find_element_by_class_name("related-question-pair").click()
        response = driver.page_source
        # print(response)
        html = etree.HTML(response)
        titles = html.xpath(
            '//div[@class="related-question-pair"]//div[@class="match-mod-horizontal-padding hide-focus-ring cbphWd"]//text()'
        )
        print('titles', titles)
        contents = Selector(text=response).xpath(
            '//div[@class="related-question-pair"]//div[@class="gy6Qzb kno-ahide"]'
        ).extract()
        for title, content in zip(titles, contents):
            con = Selector(text=content).xpath(
                '//div[contains(@class,"mod")]//text()').extract()
            con = ' '.join(con)
            items = {}
            items['question_id'] = md5_(title + hotword_id)
            items['event_id'] = event_id
            items['hotword_id'] = hotword_id
            items['question'] = title
            items['answer'] = con
            items['source'] = ''
            items['release_time'] = '2019-09-25'
            items['time_stamp'] = int(time.time())
            items['entity'] = []
            items['label'] = []
            items['static_page'] = 0
            items['nlp_state'] = 0
            print(items)
            client.dailypops.question.update(
                {'question_id': items['question_id']}, items, True)
        s1 = {'hotword_id': hotword_id}
        s2 = {'$set': {'question_state': 1}}
        client.dailypops.hotword.update(s1, s2)
        time.sleep(3)