def __init__(self):
self.log = Log('./logs',
'schedule-push.log',
dividelevel=0,
loglevel="info")
self.mydqldb = db.Connection(host=conf.MYSQL_HOST,
user=conf.MYSQL_USER,
password=conf.MYSQL_PASSWD,
database=conf.MYSQL_DB,
time_zone='+8:00',
max_idle_time=252)
def __init__(self, queue):
multiprocessing.Process.__init__(self)
self.log = Log("./logs",
name="producer-queue.log",
dividelevel=0,
loglevel="info")
self.queue = queue
self.mydqldb = db.Connection(host=conf.MYSQL_HOST,
user=conf.MYSQL_USER,
password=conf.MYSQL_PASSWD,
database=conf.MYSQL_DB,
time_zone='+8:00',
max_idle_time=252)
def __init__(self):
# author database:
self.connection = db.Connection(config.authordb["host"],
config.authordb["db"],
config.authordb["user"],
config.authordb["password"],
config.authordb["schema"])
# Creates any missing tables that are expected to exist in the application database.
# Does NOT verify whether the current columns are accurate.
with self.connection.db.cursor() as cursor:
helpers.Make_tables(cursor)
# PROD DATABASE with article info:
self.PROD = db.Connection(config.rxdb["host"], config.rxdb["db"],
config.rxdb["user"], config.rxdb["password"],
config.rxdb["schema"])
# Requests HTML configuration
self.session = HTMLSession(mock_browser=False)
self.session.headers['User-Agent'] = config.user_agent
self.log = Logger()
def __init__(self, idx, queue):
multiprocessing.Process.__init__(self)
self.log = Log("./logs",
name="consumer-%s-log" % idx,
dividelevel=0,
loglevel="debug")
self.idx = idx
self.queue = queue
self.wechat_key = None
self.pubnum = None
self.last_wechat_key_time = int(time.time())
self.craler_handler = CrawlerWeixin()
self.mydqldb = db.Connection(host=conf.MYSQL_HOST,
user=conf.MYSQL_USER,
password=conf.MYSQL_PASSWD,
database=conf.MYSQL_DB,
time_zone='+8:00',
max_idle_time=252)
def _getIssueDataFromApi(self, project, project_name, start_date):
earliest_date = datetime.datetime.now()
conn = db.Connection(self.__config['database'])
start = 0
total = 50
while start <= total:
url = 'https://' + self.__config['server']['Domain'] + self.__config[
'server']['ApiPath'] + 'search?jql=status%20%3D%20' + self.__config[
'server'][
'Status'] + '%20AND%20project%20%3D%20' + project + '&startAt=' + str(
start)
auth = (self.__config['server']['User'],
self.__config['server']['Password'])
data = common.makeApiRequest(url, auth, False)
total = data['total']
start += data['maxResults']
(issues,
accounts) = self.__extractIssueData(project, project_name, data)
update_date = None
try:
update_date = datetime.datetime.strptime(
issues[-1:][0]['updated_date'][:23],
'%Y-%m-%dT%H:%M:%S.%f')
except:
update_date = datetime.datetime.now()
conn.addUsers(accounts)
conn.addIssues(issues)
for issue in issues:
self._getChangelogDataFromApi(issue['issue_key'], conn)
if update_date < start_date:
break
conn.closeConnection()
return issues
def _getDataFromApi(self, project_name, project, start_date, end_date):
earliest_date = datetime.datetime.now()
counter = 0
conn = db.Connection(self.__config['database'])
while earliest_date > start_date and counter < self._loop_limit:
url = 'https://' + self.__config['server'][
'Domain'] + self.__config['server'][
'ApiPath'] + project + '?state=ALL&limit=' + str(
self._page_size) + '&start=' + str(
self._page_size * counter)
auth = (self.__config['server']['User'],
self.__config['server']['Password'])
data = common.makeApiRequest(url, auth, False)
(activities, users) = self.__extractPRData(project_name, data)
conn.addUsers(users)
earliest_date = conn.addActivities(activities)
for activity in activities:
url = 'https://' + self.__config['server'][
'Domain'] + self.__config['server'][
'ApiPath'] + project + '/' + str(
activity['id']) + '/activities'
data = common.makeApiRequest(url, auth, False)
(reviewers, comments,
users) = self.__extractActData(activity['id'], data)
conn.addUsers(users)
conn.addReviewers(reviewers)
conn.addComments(comments)
counter += 1
conn.closeConnection()
def handle_event():
event = request.json
if event['type'] == 'url_verification':
return Response(event['challenge'], mimetype='text/plain')
elif event['type'] == 'event_callback':
inner_event = event['event']
if inner_event['type'] == 'reaction_added':
route = EMOJI_ROUTES.get(inner_event['reaction'])
if route:
message = get_message_from_item(inner_event['item'])
conn = db.Connection()
if not conn.has_run(inner_event):
message = get_message_from_item(inner_event['item'])
user = inner_event['user']
channel = inner_event['item']['channel']
route(channel, user, message)
conn.mark_run(inner_event)
else:
logging.info('Unknown emoji:', inner_event['reaction'])
else:
logging.warn('Unknown inner event type:', inner_event['type'])
else:
logging.warn('Unknown outer event type:', event['type'])
return ''
"""Definition of web service and initialization of the server.
This is the entrypoint for the application, the script called
when the server is started and the router for all user requests.
"""
import re
import bottle
import config
import db
import endpoints
import helpers
import models
connection = db.Connection(config.db["host"], config.db["db"],
config.db["user"], config.db["password"])
# - ROUTES -
# paper query endpoint
@bottle.get('/v1/papers')
def index():
query = bottle.request.query.q
timeframe = bottle.request.query.timeframe
category_filter = bottle.request.query.getall(
'category') # multiple params possible
metric = bottle.request.query.metric
entity = "papers"
page = bottle.request.query.page
page_size = bottle.request.query.page_size
#! /usr/bin/env python
# coding:utf-8
# author:jzh
import tornado.ioloop, dao, extractor, traceback, db
import tornado.web, conf, utils
from log import Log
mydqldb = db.Connection(host=conf.MYSQL_HOST,
user=conf.MYSQL_USER,
password=conf.MYSQL_PASSWD,
database=conf.MYSQL_DB,
time_zone='+8:00',
max_idle_time=252)
print mydqldb.query("select * from wechat_key;")
def main():
records = read_csv("./resources/initial.csv")
connection = db.Connection()
with connection.connect() as conn:
for record in records:
add(conn, record)
print("\nnot found in memory... try in db\n")
city_dic = db_connector.getCity(code)
if bool(city_dic) == True:
city_code = city_dic.get("code", "")
city_name = city_dic.get("city", "")
city_pop = city_dic.get("population", "")
city_country = city_dic.get("country", "")
new_city = city.City(city_code, city_name, city_pop, city_country)
cities.append(new_city)
if __name__ == "__main__":
welcomeMessage()
db_connector = db.Connection()
cities = []
my_option = int(1)
while my_option != 0:
try:
my_option = int(menuOptions())
except:
print("invalid option... please enter valid option")
continue
print("you selected option " + str(my_option) + "\n")
if my_option == 1:
getCity(db_connector, cities)
elif my_option == 2:
#转换成新的时间格式(2016-05-05 20:28:54)
#dt = time.strftime("%Y-%m-%d %H:%M:%S",time_local)
dt = time.strftime("%Y%m%d", time_local)
if not os.path.exists(os.path.join(_DIR, dt)):
os.mkdir(os.path.join(_DIR, dt))
dbinfo = {
'host': '127.0.0.1',
'user': '******',
'passwd': '123',
'db': 'data',
'port': 3306,
'charset': 'utf8'
}
conn = db.Connection(**dbinfo)
#r = requests.post(URL, headers=HEADERS, data=PAYLOAD)
#for page in range(PAGE_S,PAGE_E):
for page in range(PAGE_S, PAGE_E):
r = requests.post(URL.format(page=page), headers=HEADERS)
if r.status_code != 200:
print(r.status_code)
print("error:page:{}".format(page))
continue
if len(r.text) == 0:
print(len(r.text))
print("error:page:{},done".format(page))
continue
parser.add_argument('cmd', default='conn_test', help='What the script should do', choices=choicesList)
parser.add_argument('-n', '--number', type=int, default=100 , help='Number of things to add')
parser.add_argument('--name' , default='user',nargs = '?',help='Default name that users will have')
parser.add_argument('--table', default='actives', help='What table to describe (used with desc_table)')
parser.add_argument('unused', nargs='*', help='Irrelevant info ')
def parseEnvFile(fileName='../.env'):
f = open(fileName, "r")
conf = []
for line in f:
if not len(line) == 0:
conf.append(tuple(map(lambda x: x.strip('\n'), line.split("="))))
return dict(conf)
if __name__ == '__main__':
env = parseEnvFile()
host,user,pw = env['DEV_DB_HOST'], env['DEV_DB_USER'],env['DEV_DB_PASS']
args = parser.parse_args()
database = db.Connection(host=host,user=user,passwd=pw)
if args.cmd == choicesList[0]: # desc_table
database.describeTable(args.table)
elif args.cmd == choicesList[1]: # gen_users
database.genUsers(n=args.number, name=args.name)
elif args.cmd == choicesList[2]: # gen_requests
database.genRequests(args.number)
elif args.cmd == choicesList[3]: # gen_transactions
database.genTransactions(args.number)
