Exemple #1
0
def connect_db():
    engine = db_connect()
    create_table(engine)
    sessionFactory = sessionmaker(bind=engine)
    global SESSION
    SESSION = scoped_session(sessionFactory)
    print('Connect db')
Exemple #2
0
 def __init__(self):
     """
     Initializes database connection and sessionmaker. Creates songs table.
     """
     engine = db_connect()
     create_table(engine)
     self.Session = sessionmaker(bind=engine)
def insert_item(item):
    engine = db_connect()
    create_table(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    for data in item:
        job = JobInfo(**data)
        if data['sponsored']:
            exist_row = session.query(JobInfo).filter(
                JobInfo.title == data['title'],
                JobInfo.sponsored == data['sponsored']).all()
            if not exist_row:
                session.add(job)
                session.commit()
            else:
                query = session.query(JobInfo).filter(
                    JobInfo.title == data['title'],
                    JobInfo.sponsored == data['sponsored'])
                for instance in query:
                    add_filter_salary(data, instance, query, session)

        if data['sponsored'] is None:
            exist_post_url = session.query(JobInfo).filter(
                JobInfo.post_url == data['post_url']).all()
            if not exist_post_url:
                session.add(job)
                session.commit()
            else:
                query = session.query(JobInfo).filter(
                    JobInfo.post_url == data['post_url'])
                for instance in query:
                    add_filter_salary(data, instance, query, session)
    session.close()
 def __init__(self):
     """
     Initializes database connection and sessionmaker
     Creates tables
     """
     engine = db_connect()
     create_table(engine)
     self.factory = sessionmaker(bind=engine)
 def __init__(self):
     """
     Initializes database connection and sessionmaker.
     Creates artists table.
     """
     engine = db_connect()
     create_table(engine)
     self.Session = sessionmaker(bind=engine)
 def do_startdb(self, arg):
     """Creates all needed tables for the local database."""
     # TODO : Reset DB before.
     models.create_table()
     print("Done!")
     print(
         'If you want to be able to run this program without "sudo", please run the following command:'
     )
     print("sudo chmod +r " + models.db_filename)
Exemple #7
0
    def __init__(self):
        """
        初始化class
        """

        engine = db_connect()
        create_table(engine)
        sessionFactory = sessionmaker(bind=engine)
        self.Session = scoped_session(sessionFactory)
        self.number = 1
        self.print_lock = threading.Lock()
    def __init__(self, keyword, staring_salary, contract_only):
        super().__init__(keyword, staring_salary, contract_only)
        self.all_jobs = []
        self.engine = db_connect()
        create_table(self.engine)
        self.Session = sessionmaker(bind=self.engine)
        self.session = self.Session()

        t = Thread(target=self.run, args=())
        t.start()
        t.join()
Exemple #9
0
    def __init__(self):
        """
        初始化class
        """

        engine = db_connect()
        create_table(engine)
        sessionFactory = sessionmaker(bind=engine)
        self.Session = scoped_session(sessionFactory)
        self.number = 1
        self.print_lock = threading.Lock()
    def __init__(self, keyword, staring_salary, contract_only):
        super().__init__(keyword, staring_salary, contract_only)
        self.df = self.create_pandas_df()
        indeed_uk = Thread(target=self.get_indeed_uk_jobs, args=(self.indeed_uk_job_links,))
        reed = Thread(target=self.get_reed_jobs, args=())
        total_jobs = Thread(target=self.get_jobs, args=(self.total_job_links,))
        cw_jobs = Thread(target=self.get_jobs, args=(self.cw_job_links,))
        # cv_library_jobs = Thread(target=self.get_cvlibrary_jobs, args=(self.cvlibrary_job_links,))

        indeed_uk.start()
        reed.start()
        total_jobs.start()
        cw_jobs.start()
        # cv_library_jobs.start()

        self.TOTAL_JOB_COUNT = len(self.total_job_links) + len(self.cw_job_links) + len(self.indeed_uk_job_links) + \
                               + len(self.reed_job_links)
        # + len(self.cvlibrary_job_links)

        indeed_uk.join()
        reed.join()
        total_jobs.join()
        cw_jobs.join()
        # cv_library_jobs.join()

        job_count_by_site = {
            'total_cw_jobs': [len(self.total_job_links) + len(self.cw_job_links), self.total_cw_relevant_jobs],
            'indeed_uk_jobs': [len(self.indeed_uk_job_links), self.indeed_uk_relevant_jobs],
            'reed_jobs': [len(self.reed_job_links), self.reed_relevant_jobs],
            'cvlibrary_jobs': None,  # [len(self.cvlibrary_job_links), self.cvlibrary_relevant_jobs]
        }
        self.df = self.df.drop_duplicates()
        engine = db_connect()
        create_table(engine)
        self.df.to_sql(name=self.db_format, con=engine, index=False)
        self.to_excel(df=self.df, keyword=self.keyword)
        self.get_job_counts(total_count=self.TOTAL_JOB_COUNT,
                            relevant_count=JobsCounter.RELEVANT_JOBS_COUNT,
                            job_count_by_site=job_count_by_site)
Exemple #11
0
 def POST(self,text):
     if web.ctx.session.login==True:
         num = int(text.split('--')[1])
         tablename = text.split('--')[0]
         f = DynamicForm()
         custom_form(f,num)
         if not f.validates():
             msg = ERR_COL_EDIT
             return web.template.render('templates/',base='base',globals={'session':web.ctx.session}).columns(f,msg)
         else:
             fnames = [models.any2str(f["name" + str(i)].value) for i in range(num)]
             fattrs = [models.any2str(f["attr" + str(i)].value) for i in range(num)]
             attrs = {models.any2str('PK'):models.any2str(f["primarykey"].value)}
             msg = models.create_table(models.any2str(tablename),fnames,fattrs,attrs)
             if msg!='':
                 return web.template.render('templates/',base='base',globals={'session':web.ctx.session}).columns(f,msg)
             else:
                 raise web.seeother('/../')
     else:
         raise web.seeother("/../")
 def __init__(self):
     engine = create_engine(URL(**db_settings.DATABASE))
     create_table(engine)
     self.Session = sessionmaker(bind=engine)
Exemple #13
0
                  'weixin_oil_cross', 'weixin_lng_con', 'weixin_cnpc_news', 'weixin_energy_express',\
                  'weixin_petro_trading', \
                  'weixin_hai_bei', 'weixin_offshore_energy', 'weixin_hai_bo', 'weixin_crsl', 'weixin_oil_cubic',\
                  'weixin_oil_link']
    table_name_pro = ['news_oil_oe_pro', 'world_oil_pro', 'hart_energy_pro', 'cnpc_news_pro', 'oilfield_tech_pro', \
                      'oil_and_gas_pro', 'in_en_storage_pro', 'jpt_latest_pro', 'energy_voice_pro', 'gulf_oil_gas_pro', \
                      'energy_pedia_pro', 'up_stream_pro', 'oil_price_pro', 'inen_tech_pro', 'inen_newenergy_pro', \
                      'drill_contractor_pro', 'rog_tech_pro', 'natural_gas_pro', 'rig_zone_pro', 'offshore_tech_pro', \
                      'energy_year_pro', 'energy_china_pro', 'china_five_pro', 'offshore_energy_pro', 'jwn_energy_pro', \
                      'iran_oilgas_pro', 'neng_yuan_pro', 'wood_mac_pro', 'rystad_energy_pro', 'westwood_energy_pro',\
                      'iea_news_pro', 'weixin_data_pro', 'weixin_oil_cross_pro', 'weixin_lng_con_pro',\
                      'weixin_cnpc_news_pro', 'weixin_energy_express_pro', 'weixin_petro_trading_pro', \
                      'weixin_hai_bei_pro', 'weixin_offshore_energy_pro', 'weixin_hai_bo_pro', 'weixin_crsl_pro',\
                      'weixin_oil_cubic_pro', 'weixin_oil_link_pro']
    engine = db_connect()
    create_table(engine)
    cate_file = 'input_data/categories_list.xlsx'
    df_dicts = read_xlsx(cate_file)

    # ==================== generate all the keyword and category pair==================================
    # country section
    df_dicts['country'].columns = [
        'region', 'country', 'key_words_chinese', 'key_words_english'
    ]  ## rename cols
    country_keywords_pair = gen_keywords_pair(df_dicts['country'], 2, [3, 4])
    # region section
    df_dicts['region'].columns = [
        'region', 'chinese_keywords', 'english_keywords'
    ]
    region_df = df_dicts['region']
    region_df.columns = ['region', 'chinese_keywords', 'english_keywords']
Exemple #14
0
# *-* coding:utf-8 *-*
'''
@author: ioiogoo
@date: 17-1-7 下午2:18
'''
from bobao_news import Bobao_new
from freebuf_news import Freebuf_new
from hacker_news import Hacker_new
from jobbole_news import Jobbole_new
from gevent.pool import Pool
from gevent import monkey
monkey.patch_all()
from models import create_table, models

for model in models:
    create_table(model)


def crawl():
    news = [Bobao_new, Freebuf_new, Hacker_new, Jobbole_new]
    p = Pool()
    p.map(lambda x: x().handle(), news)


if __name__ == '__main__':
    crawl()
Exemple #15
0
 def __init__(self):
     self.engine = models.db_connect()
     models.create_table(self.engine)
     self.Session = sessionmaker(bind=self.engine)
Exemple #16
0
import os
from flask import Flask
from flask import render_template, request, g, redirect, url_for
import models
from forms import ratingForm
from models import db
from models import rating
import sqlite3
import peewee

app = Flask(__name__)
app.secret_key="eqvrioubheqirbvq"

models.create_table()

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/item', methods=['GET', 'POST'])
def main():
    return render_template('item.html')
    
@app.route('/post-rating', methods=['POST'])
def post():
    s = "Arrows"
    rOne = request.form.get('replyOne')
    rTwo = request.form.get('replyTwo')
    rThree = request.form.get('replyThree')
    conn = sqlite3.connect('data/project_db.db')
    c = conn.cursor()
Exemple #17
0
from flask_login import LoginManager, UserMixin, login_user, login_required, logout_user, current_user
from users.routes import user_api_bluepring
import models

app = Flask(__name__)
login_manager = LoginManager()
login_manager.init_app(app)

app.config.update(
    dict(
        SECRET_KEY="thisissecret",
        SQLALCHEMY_DATABASE_URI='sqlite://///app/login.db',
    ))

models.init_app(app)
models.create_table(app)

app.register_blueprint(user_api_bluepring)


@login_manager.user_loader
def load_user(user_id):
    return models.User.query.get(int(user_id))


@app.route('/')
def hello_world():
    return 'Hello World!'


if __name__ == '__main__':
Exemple #18
0
 def __init__(self):
     #Lets just assure table order is created
     create_table(engine)
 def __init__(self):
     engine = db_connect()
     create_table(engine)
     self.Session = sessionmaker(bind=engine)
from OpenSSL import SSL
import os

app = Flask(__name__)
api = Api(app)

# ssl
context = SSL.Context(SSL.SSLv23_METHOD)
cer = os.path.join(os.path.dirname(__file__), 'resources/my.com.crt')
key = os.path.join(os.path.dirname(__file__), 'resources/my.com.key')
# cer = '~/resources/my.com.crt'
# key = '~/resources/my.com.key'

# db
engine = db_connect()
create_table(engine)
session = sessionmaker(bind=engine)()


class Price(Resource):
    def get(self, book_id):
        book_info = session.query(Info).filter_by(book_id=book_id).all()
        if book_info:
            price = [i.price for i in book_info]
            return price
        else:
            return {}

api.add_resource(Price, '/<string:book_id>')

if __name__ == '__main__':
Exemple #21
0
 def __init__(self):
     engine = db_connect()
     create_table(engine)
     self.Session = sessionmaker(bind=engine)
Exemple #22
0
 def __init__(self):
     #Lets just assure table order is created
     create_table(engine)