def connect_db():
engine = db_connect()
create_table(engine)
sessionFactory = sessionmaker(bind=engine)
global SESSION
SESSION = scoped_session(sessionFactory)
print('Connect db')
def __init__(self):
"""
Initializes database connection and sessionmaker. Creates songs table.
"""
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
def insert_item(item):
engine = db_connect()
create_table(engine)
Session = sessionmaker(bind=engine)
session = Session()
for data in item:
job = JobInfo(**data)
if data['sponsored']:
exist_row = session.query(JobInfo).filter(
JobInfo.title == data['title'],
JobInfo.sponsored == data['sponsored']).all()
if not exist_row:
session.add(job)
session.commit()
else:
query = session.query(JobInfo).filter(
JobInfo.title == data['title'],
JobInfo.sponsored == data['sponsored'])
for instance in query:
add_filter_salary(data, instance, query, session)
if data['sponsored'] is None:
exist_post_url = session.query(JobInfo).filter(
JobInfo.post_url == data['post_url']).all()
if not exist_post_url:
session.add(job)
session.commit()
else:
query = session.query(JobInfo).filter(
JobInfo.post_url == data['post_url'])
for instance in query:
add_filter_salary(data, instance, query, session)
session.close()
def __init__(self):
"""
Initializes database connection and sessionmaker
Creates tables
"""
engine = db_connect()
create_table(engine)
self.factory = sessionmaker(bind=engine)
def __init__(self):
"""
Initializes database connection and sessionmaker.
Creates artists table.
"""
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
def do_startdb(self, arg):
"""Creates all needed tables for the local database."""
# TODO : Reset DB before.
models.create_table()
print("Done!")
print(
'If you want to be able to run this program without "sudo", please run the following command:'
)
print("sudo chmod +r " + models.db_filename)
def __init__(self):
"""
初始化class
"""
engine = db_connect()
create_table(engine)
sessionFactory = sessionmaker(bind=engine)
self.Session = scoped_session(sessionFactory)
self.number = 1
self.print_lock = threading.Lock()
def __init__(self, keyword, staring_salary, contract_only):
super().__init__(keyword, staring_salary, contract_only)
self.all_jobs = []
self.engine = db_connect()
create_table(self.engine)
self.Session = sessionmaker(bind=self.engine)
self.session = self.Session()
t = Thread(target=self.run, args=())
t.start()
t.join()
def __init__(self):
"""
初始化class
"""
engine = db_connect()
create_table(engine)
sessionFactory = sessionmaker(bind=engine)
self.Session = scoped_session(sessionFactory)
self.number = 1
self.print_lock = threading.Lock()
def __init__(self, keyword, staring_salary, contract_only):
super().__init__(keyword, staring_salary, contract_only)
self.df = self.create_pandas_df()
indeed_uk = Thread(target=self.get_indeed_uk_jobs, args=(self.indeed_uk_job_links,))
reed = Thread(target=self.get_reed_jobs, args=())
total_jobs = Thread(target=self.get_jobs, args=(self.total_job_links,))
cw_jobs = Thread(target=self.get_jobs, args=(self.cw_job_links,))
# cv_library_jobs = Thread(target=self.get_cvlibrary_jobs, args=(self.cvlibrary_job_links,))
indeed_uk.start()
reed.start()
total_jobs.start()
cw_jobs.start()
# cv_library_jobs.start()
self.TOTAL_JOB_COUNT = len(self.total_job_links) + len(self.cw_job_links) + len(self.indeed_uk_job_links) + \
+ len(self.reed_job_links)
# + len(self.cvlibrary_job_links)
indeed_uk.join()
reed.join()
total_jobs.join()
cw_jobs.join()
# cv_library_jobs.join()
job_count_by_site = {
'total_cw_jobs': [len(self.total_job_links) + len(self.cw_job_links), self.total_cw_relevant_jobs],
'indeed_uk_jobs': [len(self.indeed_uk_job_links), self.indeed_uk_relevant_jobs],
'reed_jobs': [len(self.reed_job_links), self.reed_relevant_jobs],
'cvlibrary_jobs': None, # [len(self.cvlibrary_job_links), self.cvlibrary_relevant_jobs]
}
self.df = self.df.drop_duplicates()
engine = db_connect()
create_table(engine)
self.df.to_sql(name=self.db_format, con=engine, index=False)
self.to_excel(df=self.df, keyword=self.keyword)
self.get_job_counts(total_count=self.TOTAL_JOB_COUNT,
relevant_count=JobsCounter.RELEVANT_JOBS_COUNT,
job_count_by_site=job_count_by_site)
def POST(self,text):
if web.ctx.session.login==True:
num = int(text.split('--')[1])
tablename = text.split('--')[0]
f = DynamicForm()
custom_form(f,num)
if not f.validates():
msg = ERR_COL_EDIT
return web.template.render('templates/',base='base',globals={'session':web.ctx.session}).columns(f,msg)
else:
fnames = [models.any2str(f["name" + str(i)].value) for i in range(num)]
fattrs = [models.any2str(f["attr" + str(i)].value) for i in range(num)]
attrs = {models.any2str('PK'):models.any2str(f["primarykey"].value)}
msg = models.create_table(models.any2str(tablename),fnames,fattrs,attrs)
if msg!='':
return web.template.render('templates/',base='base',globals={'session':web.ctx.session}).columns(f,msg)
else:
raise web.seeother('/../')
else:
raise web.seeother("/../")
def __init__(self):
engine = create_engine(URL(**db_settings.DATABASE))
create_table(engine)
self.Session = sessionmaker(bind=engine)
'weixin_oil_cross', 'weixin_lng_con', 'weixin_cnpc_news', 'weixin_energy_express',\
'weixin_petro_trading', \
'weixin_hai_bei', 'weixin_offshore_energy', 'weixin_hai_bo', 'weixin_crsl', 'weixin_oil_cubic',\
'weixin_oil_link']
table_name_pro = ['news_oil_oe_pro', 'world_oil_pro', 'hart_energy_pro', 'cnpc_news_pro', 'oilfield_tech_pro', \
'oil_and_gas_pro', 'in_en_storage_pro', 'jpt_latest_pro', 'energy_voice_pro', 'gulf_oil_gas_pro', \
'energy_pedia_pro', 'up_stream_pro', 'oil_price_pro', 'inen_tech_pro', 'inen_newenergy_pro', \
'drill_contractor_pro', 'rog_tech_pro', 'natural_gas_pro', 'rig_zone_pro', 'offshore_tech_pro', \
'energy_year_pro', 'energy_china_pro', 'china_five_pro', 'offshore_energy_pro', 'jwn_energy_pro', \
'iran_oilgas_pro', 'neng_yuan_pro', 'wood_mac_pro', 'rystad_energy_pro', 'westwood_energy_pro',\
'iea_news_pro', 'weixin_data_pro', 'weixin_oil_cross_pro', 'weixin_lng_con_pro',\
'weixin_cnpc_news_pro', 'weixin_energy_express_pro', 'weixin_petro_trading_pro', \
'weixin_hai_bei_pro', 'weixin_offshore_energy_pro', 'weixin_hai_bo_pro', 'weixin_crsl_pro',\
'weixin_oil_cubic_pro', 'weixin_oil_link_pro']
engine = db_connect()
create_table(engine)
cate_file = 'input_data/categories_list.xlsx'
df_dicts = read_xlsx(cate_file)
# ==================== generate all the keyword and category pair==================================
# country section
df_dicts['country'].columns = [
'region', 'country', 'key_words_chinese', 'key_words_english'
] ## rename cols
country_keywords_pair = gen_keywords_pair(df_dicts['country'], 2, [3, 4])
# region section
df_dicts['region'].columns = [
'region', 'chinese_keywords', 'english_keywords'
]
region_df = df_dicts['region']
region_df.columns = ['region', 'chinese_keywords', 'english_keywords']
# *-* coding:utf-8 *-*
'''
@author: ioiogoo
@date: 17-1-7 下午2:18
'''
from bobao_news import Bobao_new
from freebuf_news import Freebuf_new
from hacker_news import Hacker_new
from jobbole_news import Jobbole_new
from gevent.pool import Pool
from gevent import monkey
monkey.patch_all()
from models import create_table, models
for model in models:
create_table(model)
def crawl():
news = [Bobao_new, Freebuf_new, Hacker_new, Jobbole_new]
p = Pool()
p.map(lambda x: x().handle(), news)
if __name__ == '__main__':
crawl()
def __init__(self):
self.engine = models.db_connect()
models.create_table(self.engine)
self.Session = sessionmaker(bind=self.engine)
import os
from flask import Flask
from flask import render_template, request, g, redirect, url_for
import models
from forms import ratingForm
from models import db
from models import rating
import sqlite3
import peewee
app = Flask(__name__)
app.secret_key="eqvrioubheqirbvq"
models.create_table()
@app.route('/')
def index():
return render_template('index.html')
@app.route('/item', methods=['GET', 'POST'])
def main():
return render_template('item.html')
@app.route('/post-rating', methods=['POST'])
def post():
s = "Arrows"
rOne = request.form.get('replyOne')
rTwo = request.form.get('replyTwo')
rThree = request.form.get('replyThree')
conn = sqlite3.connect('data/project_db.db')
c = conn.cursor()
from flask_login import LoginManager, UserMixin, login_user, login_required, logout_user, current_user
from users.routes import user_api_bluepring
import models
app = Flask(__name__)
login_manager = LoginManager()
login_manager.init_app(app)
app.config.update(
dict(
SECRET_KEY="thisissecret",
SQLALCHEMY_DATABASE_URI='sqlite://///app/login.db',
))
models.init_app(app)
models.create_table(app)
app.register_blueprint(user_api_bluepring)
@login_manager.user_loader
def load_user(user_id):
return models.User.query.get(int(user_id))
@app.route('/')
def hello_world():
return 'Hello World!'
if __name__ == '__main__':
def __init__(self):
#Lets just assure table order is created
create_table(engine)
def __init__(self):
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
from OpenSSL import SSL
import os
app = Flask(__name__)
api = Api(app)
# ssl
context = SSL.Context(SSL.SSLv23_METHOD)
cer = os.path.join(os.path.dirname(__file__), 'resources/my.com.crt')
key = os.path.join(os.path.dirname(__file__), 'resources/my.com.key')
# cer = '~/resources/my.com.crt'
# key = '~/resources/my.com.key'
# db
engine = db_connect()
create_table(engine)
session = sessionmaker(bind=engine)()
class Price(Resource):
def get(self, book_id):
book_info = session.query(Info).filter_by(book_id=book_id).all()
if book_info:
price = [i.price for i in book_info]
return price
else:
return {}
api.add_resource(Price, '/<string:book_id>')
if __name__ == '__main__':
def __init__(self):
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
def __init__(self):
#Lets just assure table order is created
create_table(engine)
