def get(self):
try:
webData = self.request.body
logging.info("Oa2LoginHandle, webdata is: %s" % (webData))
user = self.get_argument("user", None)
pwd = self.get_argument("pwd", None)
state = self.get_argument("state", None)
client_id = self.get_argument("client_id", None)
response_type = self.get_argument("response_type", None)
scope = self.get_argument("scope", None)
redirect_uri = self.get_argument("redirect_uri", None)
if user != None:
resp = 'errors'
# resp = OtherAuthServer().login(user, pwd) #对用户名和密码进行校验
if 'errors' in resp:
self.write("user name or password is wrong")
else:
# 返回code
code = user
redirect_uri = redirect_uri + '?code=' + code + '&state=' + state
logging.info("Oa2LoginHandle, redirect_uri is: %s" %
(redirect_uri))
self.write(redirect_uri)
# self.redirect(redirect_uri)
else:
self.write("user name can't be empty")
except Exception, Argument:
self.write(Argument.message)
logging.error(Argument.message)
def calling():
time.sleep(10)
# Set the request parameters
logging.info("expense allocation rest call")
url = 'https://dev67363.service-now.com/api/323921/budget_use_case_exp_alloc'
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
response = requests.request('POST',
url,
headers=headers,
allow_redirects=False,
auth=('admin', 'K3w1sTVeDtZl'))
print(response.text)
time.sleep(10)
logging.info("expense allocation rest call")
url = 'https://dev67363.service-now.com/api/323921/budget_use_case_budget_calc'
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
response = requests.request('POST',
url,
headers=headers,
allow_redirects=False,
auth=('admin', 'K3w1sTVeDtZl'))
print(response.text)
def get_post_list(cnx,
category_code: int,
current_seq=9999999999999,
limit_count=20) -> list:
sql = (f" SELECT b.subject, \n"
f" a.name AS category_name,\n"
f" c.name AS user_name,\n"
f" b.create_timestamp AS post_create_time,\n"
f" b.seq AS post_id,\n"
f" b.category_code\n"
f" FROM respac.post_category a\n"
f" JOIN respac.post b\n"
f" ON (a.code = b.category_code)\n"
f" JOIN respac.user c\n"
f" ON (b.user_code = c.code)\n"
f" WHERE a.code = {category_code}\n"
f" AND b.seq < {current_seq}\n"
f" ORDER BY b.category_code, b.seq DESC\n"
f" LIMIT {limit_count} \n"
f" ")
logging.info(sql)
result = select(cnx, sql)
logging.info(result)
if result is None:
logging.error("DB 연결에 실패하였습니다.")
return []
return result
def write_to_json_file():
"""
爬虫数据写入json文件,
而不是直接写入数据库
"""
logging.info("enter write_to_json_file...")
#获得最大页数，确定爬虫范围
last_page_number = get_last_web_page_number()
#打开文件
current_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))+"/json"
yyyy = time.strftime("%Y")
mm = time.strftime("%m")
dd = time.strftime("%d")
folder = os.path.join(current_dir, yyyy, mm, dd)
if not os.path.exists(folder):
os.makedirs(folder)
os.chdir(folder)
logging.debug("max page is :%s"%last_page_number)
for i in range(1,last_page_number+1):
with open("news_%s.json"%i, "w+") as news:
#获取页面
page = get_web_page_by_url(URL.format(page_number=i))
#解析
for item_json_data in load_and_parse(page):
#写入文件
news.write(item_json_data+"\n")
logging.info("leaving write_to_json_file...")
def clean_death_workers(self):
'''定期检查worker的心跳信息，及时清除死亡worker'''
workers, tasks = self.workerManager.clean_death_workers()
logging.info("death workers:%s; relatedTasks:%s", workers, tasks)
for task in tasks:
self.taskManager.fail_task(task.uuid, TaskStatus.notReturned)
return workers
def parseViewformat(df, responseStr, topic):
# remove first row
if df.size > 1:
filter_df = df.iloc[1:]
filter_df.loc[:, 'SPC_0_break'] = \
filter_df['timestamp_view_previous'].astype(str) + '(' + filter_df['SPC_0_previous'].astype(
str) + ')' \
+ ' -> ' + filter_df['timestamp_view'].astype(str) + '(' + filter_df['SPC_0'].astype(str) + ')'
df_list = filter_df[[
'SPC_0_break', 'timestamp_view_date', 'timestamp_view_date_previous'
]].values.tolist()
if len(df_list) == 0:
logging.info('無' + topic)
else:
firstDate = ''
# filter_df_first = ''
responseStr = responseStr + insertTextIndent(topic + '：', '20', '1')
filter_df_first = filter_df.iloc[0].astype(str)
for df_row in df_list:
if firstDate == '':
firstDate = filter_df_first['timestamp_view_date_previous']
responseStr = responseStr + insertTextIndent(
firstDate, '50', '1')
elif df_row[2] != firstDate:
firstDate = df_row[2]
responseStr = responseStr + insertTextIndent(
df_row[1], '50', '1')
responseStr = responseStr + insertTextIndent(df_row[0], '70', '1')
return responseStr
def main():
browser = webdriver.Chrome()
# 将浏览器最大化显示
browser.maximize_window()
# 设置宽高
browser.set_window_size(1400, 900)
browser.get('https://www.baidu.com')
# browser.find_element_by_id("kw").send_keys("selenium")
# t1 = browser.find_element_by_tag_name("input").send_keys("selenium 1111")\
# xpath: attributer （属性）
# input 标签下 id =kw 的元素
# browser.find_element_by_xpath("//input[@id='kw']").send_keys("selenium")
# xpath: idRelative （id相关性）
# 在/form/span/input 层级标签下有个 div 标签的 id=fm 的元素
browser.find_element_by_xpath("//div[@class='s_form_wrapper soutu-env-nomac soutu-env-index']/form/span/input").send_keys("selenium")
browser.find_element_by_id("su").click()
logging.info(browser.title)
# logging.info(t1)
time.sleep(1)
# 后退
browser.back()
time.sleep(1)
# 前进
browser.forward()
time.sleep(1)
browser.quit()
def close_all(error):
if hasattr(g, 'mariadb'):
g.mariadb.close()
logging.info("DB Connection closed")
if hasattr(g, 'redis'):
g.redis.close()
logging.info("Redis Connection closed")
def run_loop(self, stop_event, queue=None):
'''
run handler in endless loop in one dedicate process
'''
# need to config QueueHandler in worker process
if queue is not None:
h = QueueHandler(queue)
root = original_logging.getLogger()
root.addHandler(h)
# set level of queue handler to DEBUG to accept all logs
root.setLevel(original_logging.DEBUG)
# after this, all log will be put into the queue use QueueHandler
# logging.info("subprocess for handler started", self._handler_key)
self._stop_event = stop_event
self._connection.set_stop_condition(self._stop_condition)
self._main()
logging.info("loop stopped for handler", self._handler_key)
if queue is not None:
# close queue
queue.close()
self._before_stop()
sys.exit(0)
def _reconnect(args, msg, e):
if len(args) == 0 or not isinstance(args[0], RabbitMQBlockingClient):
raise MQError("unsupported decorator, it should be applied to RabbitMQBlockingClient methods")
self = args[0]
########################################################
logging.error("enter reconnection")
########################################################
init_func = getattr(self, "_init")
client_configs = getattr(self, "_client_configs")
########################################################
logging.error("begin reconnection")
########################################################
logging.error(\
"message queue connection error: %s, %s reconnecting..."\
% (msg, e))
time.sleep(5)
try:
########################################################
logging.error("test init_func")
########################################################
init_func(client_configs)
logging.info("message queue reconnection succeeded")
########################################################
logging.error("test init_func")
########################################################
return True
except AMQPChannelError, e:
try:
close_func = getattr(self, "close")
close_func()
except Exception, e:
logging.error(e)
logging.error(traceback.format_exc())
def clean_death_workers(self):
'''定期检查worker的心跳信息，及时清除死亡worker'''
workers,tasks = self.workerManager.clean_death_workers()
logging.info("death workers:%s; relatedTasks:%s", workers, tasks)
for task in tasks:
self.taskManager.fail_task(task.uuid, TaskStatus.notReturned)
return workers
def _init_thread(self):
logging.debug ("")
if self._geoip is None:
logging.error("OS error: no geoip set")
return
if self._switch is None:
logging.error("OS error: no switch set")
return
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(self._server_cert, self._server_key)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
sock.bind((self._listen_addr, self._listen_port))
logging.info ("DOT input start from :"+str( self._listen_addr)+":"+str( self._listen_port))
sock.listen(self._maxlisten)
with context.wrap_socket(sock, server_side=True) as ssock:
while True:
try:
conn, addr = ssock.accept()
start_new_thread(self._decoder_thread,(conn, addr))
except OSError as err:
logging.error("OS error: {0}".format(err))
self._countererror=self._countererror+1
def publish(self, message, priority=0):
if priority < 0:
raise MQError("priority %d should be >= 0" % priority)
if priority >= self._priority_level:
logging.warn("priority has been changed due to exceeding", \
message=message, original_priority=priority, \
changed_priority=self._priority_level - 1)
priority = self._priority_level - 1
if priority < 0:
priority = 0
#determine properties
properties = pika.BasicProperties()
properties.content_type = self._content_type
if self._persistent:
properties.delivery_mode = 2
if self._with_timestamp:
properties.timestamp = datetime2timestamp(
datetime.datetime.utcnow())
#get group_id
if self._group_mode:
if self._content_type != "text/json":
#Note: here we just support hashable object, e.g., str/unicode etc.
group_hash = message
else:
if not ("__group_hash" in message.keys()):
raise MQError(
"__group_hash is required if message type is in group_mode"
)
group_hash = message["__group_hash"]
group_int_hash = hash(group_hash)
group_id = group_int_hash % self._group_counts[priority]
else:
group_id = 0
#the message is changed here
if self._content_type == "text/json" and "__group_hash" in message.keys(
):
message.pop("__group_hash")
#determine body
if self._content_type == "text/json":
body = dumps_jsonx(message)
else:
body = message
#determine routing_key
routing_key = self.generate_routing_name(self._binding_prefix,
priority, group_id)
try:
self._channel.basic_publish(exchange=self._exchange,
routing_key=routing_key,
body=body,
properties=properties)
except AttributeError, e:
logging.info('basic publish raise AttributeError: %s' % e)
raise pika.exceptions.AMQPConnectionError
def getBestBlockCount():
maxHeight = -1
for seed in config['seeds']:
height = neoapi.getCurrentHeight('http://' + seed)
if maxHeight < height:
maxHeight = height
logging.info('[getBestBlockCount] maxheight: {0}'.format(maxHeight))
return maxHeight
def getBestBlockCount():
maxHeight = -1
for seed in SEEDS:
height = neoapi.getCurrentHeight('http://' + seed)
if maxHeight < height:
maxHeight = height
logging.info('[{0}] getBestBlockCount maxheight: {1}'.format(type, maxHeight))
return maxHeight
def sample_vid_proto(vid_proto, stride=10):
new_vid = {}
new_vid['video'] = vid_proto['video']
new_vid['root_path'] = vid_proto['root_path']
idx = np.arange(0, len(vid_proto['frames']), stride)
logging.info("Sampling video by 1 / {}.".format(stride))
new_vid['frames'] = [vid_proto['frames'][i] for i in idx]
return new_vid
def on_no_motion():
logging.info("on_no_motion")
print("Recording stopping")
x = Thread(target=am.mountMoth, args=())
x.start()
x.join()
print("Recording stopped")
d.transfer_audio(config.am_mount_path, config.local_audio_path)
def check_accuracy(self):
'''
Checks whether the desired accuracy for the boundary values was reached.
It calculates the difference between the solution of the simulation
and the given boundary values at the right border and compares its
maximum against the tolerance.
If set by the user it also calculates some kind of consistency error
that shows how "well" the spline functions comply with the system
dynamic given by the vector field.
'''
# this is the solution of the simulation
a = self.sim_data[0][0]
b = self.sim_data[0][-1]
xt = self.sim_data[1]
# get boundary values at right border of the interval
if self.constraints:
bv = self._dyn_sys_orig.boundary_values
x_sym = self._dyn_sys_orig.states
else:
bv = self.dyn_sys.boundary_values
x_sym = self.dyn_sys.states
xb = dict([(k, v[1]) for k, v in bv.items() if k in x_sym])
# what is the error
logging.debug(40*"-")
logging.debug("Ending up with: Should Be: Difference:")
err = np.empty(xt.shape[1])
for i, xx in enumerate(x_sym):
err[i] = abs(xb[xx] - xt[-1][i])
logging.debug(str(xx)+" : %f %f %f"%(xt[-1][i], xb[xx], err[i]))
logging.debug(40*"-")
#if self._ierr:
ierr = self._parameters['ierr']
eps = self._parameters['eps']
if ierr:
# calculate maximum consistency error on the whole interval
maxH = auxiliary.consistency_error((a,b), self.eqs.trajectories.x, self.eqs.trajectories.u, self.eqs.trajectories.dx, self.dyn_sys.f_num)
reached_accuracy = (maxH < ierr) and (max(err) < eps)
logging.debug('maxH = %f'%maxH)
else:
# just check if tolerance for the boundary values is satisfied
reached_accuracy = max(err) < eps
if reached_accuracy:
logging.info(" --> reached desired accuracy: "+str(reached_accuracy))
else:
logging.debug(" --> reached desired accuracy: "+str(reached_accuracy))
self.reached_accuracy = reached_accuracy
def run_test(self):
logging.info("Ping test started")
serverNum = len(config.serverList)
executor = ThreadPoolExecutor(max_workers=serverNum)
f_list = []
for lst_id in range(0, serverNum):
f_list.append(
executor.submit(doPingTestFor, lst_id, self.server_lst,
self.dbname))
wait(f_list)
def assign_task(self, worker, task):
"""task分配给worker"""
logging.info("%s -> %s",task.get_uuid(), str(worker))
ok = False
self.lock.acquire()
if worker.get_uuid() not in self.workerTask:
self.workerTask[worker.get_uuid()] = task
ok = True
self.lock.release()
return ok
def assign_task(self, worker, task):
"""task分配给worker"""
logging.info("%s -> %s", task.get_uuid(), str(worker))
ok = False
self.lock.acquire()
if worker.get_uuid() not in self.workerTask:
self.workerTask[worker.get_uuid()] = task
ok = True
self.lock.release()
return ok
def register_worker(self, worker):
'''注册作业节点'''
logging.info("%s come in", worker)
status = "OK"
if worker is not None:
self.workerManager.add_worker(worker)
else:
status = "Invalid"
# logging.info(self.workerManager.get_workers())
return status
def register_worker(self, worker):
'''注册作业节点'''
logging.info("%s come in", worker)
status = "OK"
if worker is not None:
self.workerManager.add_worker(worker)
else:
status = "Invalid"
# logging.info(self.workerManager.get_workers())
return status
def __del__(self):
"""
Deletes user.
"""
if self.delete_user:
url = FB_HOST + "/%s" % self.id
params = {'access_token': self.app_access_token}
response = requests.delete(url, params=params)
logging.info("delete user: %s" % response.text)
else:
logging.info("delete user: false")
def __del__(self):
"""
Deletes user.
"""
if self.delete_user:
url = FB_HOST + "/%s" % self.id
params = {'access_token': self.app_access_token}
response = requests.delete(url, params=params)
logging.info("delete user: %s" % response.text)
else:
logging.info("delete user: false")
def schedule_next(self):
logging.info('tasks: %s',self.taskManager.get_tasks_stats())
task = self.taskManager.next_task()
worker = self.workerManager.next_worker()
self.workerManager.assign_task(worker, task)
try:
proxy = common.RPCServerProxy.get_proxy(worker)
proxy.assign_task(task)
except Exception,e:
traceback.print_exc()
self.remove_worker(worker)
def _init_thread(self):
logging.debug ("")
with socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) as sock:
sock.bind((self._listen_addr, self._listen_port))
logging.info ("UDP input start from :"+str( self._listen_addr)+":"+str( self._listen_port))
while True:
try:
msg,conn = sock.recvfrom(self._buffersize)
start_new_thread(self._decoder_thread,(sock,conn, msg))
except OSError as err:
logging.error("OS error: {0}".format(err))
def schedule_next(self):
logging.info('tasks: %s', self.taskManager.get_tasks_stats())
task = self.taskManager.next_task()
worker = self.workerManager.next_worker()
self.workerManager.assign_task(worker, task)
try:
proxy = common.RPCServerProxy.get_proxy(worker)
proxy.assign_task(task)
except Exception, e:
traceback.print_exc()
self.remove_worker(worker)
def save_session(data):
try:
redis_session = RedisSession.RedisSession()
access_token = str(uuid.uuid4())
redis_session.save_session(access_token, data)
session['access_token'] = access_token
logging.info(session)
except Exception as e:
logging.error(e)
return False
return True
def _execute_task(self, task):
logging.info(task)
cmd = task.get_cmd()
logging.info(cmd)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out,err = p.communicate()
if p.returncode == 0:
logging.info("success")
logging.info(out)
else:
logging.info("failed")
result = TaskResult(task.get_uuid(), p.returncode)
return result
def on_motion():
logging.info("on_motion")
print("Recording starting")
x = Thread(target=am.unmountMoth, args=())
y = Thread(target=c.click,
args=(config.photo_count_on_motion,
config.photo_count_delay_sec))
x.start()
y.start()
x.join()
y.join()
print("Recording started")
sleep(30)
def _init_thread(self):
logging.debug ("")
server_address = (self._listen_addr, self._listen_port)
httpd = HTTPServer(server_address, Webserver)
httpd.socket = ssl.wrap_socket (httpd.socket, certfile=self._server_cert ,keyfile=self._server_key, server_side=True)
logging.info ("DOH input start from :"+str( self._listen_addr)+":"+str( self._listen_port))
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
logging.error('Stopping httpd...\n')
def listentoclient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if data:
msgqueue.put(data)
else:
logging.info('Client disconnected')
client.close()
except IOError:
client.close()
return False
def publish(self, message, priority=0):
if priority < 0:
raise MQError("priority %d should be >= 0" % priority)
if priority >= self._priority_level:
logging.warn("priority has been changed due to exceeding", \
message=message, original_priority=priority, \
changed_priority=self._priority_level - 1)
priority = self._priority_level - 1
if priority < 0:
priority = 0
#determine properties
properties = pika.BasicProperties()
properties.content_type = self._content_type
if self._persistent:
properties.delivery_mode = 2
if self._with_timestamp:
properties.timestamp = datetime2timestamp(datetime.datetime.utcnow())
#get group_id
if self._group_mode:
if self._content_type != "text/json":
#Note: here we just support hashable object, e.g., str/unicode etc.
group_hash = message
else:
if not ("__group_hash" in message.keys()):
raise MQError("__group_hash is required if message type is in group_mode")
group_hash = message["__group_hash"]
group_int_hash = hash(group_hash)
group_id = group_int_hash % self._group_counts[priority]
else:
group_id = 0
#the message is changed here
if self._content_type == "text/json" and "__group_hash" in message.keys():
message.pop("__group_hash")
#determine body
if self._content_type == "text/json":
body = dumps_jsonx(message)
else:
body = message
#determine routing_key
routing_key = self.generate_routing_name(self._binding_prefix, priority, group_id)
try:
self._channel.basic_publish(exchange=self._exchange, routing_key=routing_key, body=body, properties=properties)
except AttributeError, e:
logging.info('basic publish raise AttributeError: %s' % e)
raise pika.exceptions.AMQPConnectionError
def _decoder_thread(self,sock,conn, txdata):
try:
logging.debug ("")
host, port = conn
if self._geoip is None:
ok= True
text= "NO GEOIP"
else:
ok,text = self._geoip(host)
if ok == True:
self._conterrequests=self._conterrequests+1
self._conterrequest=self._conterrequest+1
data =self._switch(txdata)
if data is not None:
isblock,rxdata,blockname,dname = data
sock.sendto(rxdata , conn)
logging.info("IP : "+str(host)+":"+str(port)+" :"+ text)
if isblock==True:
logging.info("Domain : "+str(dname)+ " is block true : blockname " + str(blockname))
else:
logging.info("Domain : "+str(dname)+ " is block false ")
else:
logging.info("IP : "+str(host)+": "+ text)
except OSError as err:
logging.error("OS error: {0}".format(err))
self._countererror=self._countererror +1
self._conterrequest=self._conterrequest-1
def get_post_count(cnx, category_code: int, limit_count=20.0):
sql = (f"SELECT COUNT(*) AS total_count\n"
f"from respac.post a\n"
f"WHERE a.category_code = {category_code}\n")
logging.info(sql)
result = select(cnx, sql)
logging.info(result)
if result is None:
logging.error("DB 연결에 실패하였습니다.")
return []
return math.ceil(int(result[0]['total_count']) / limit_count)
def db_connect():
# Construct connection string
try:
conn = mysql.connector.connect(**config)
logging.info("Connection established")
return conn
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
logging.error("Something is wrong with the user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
logging.error("Database does not exist")
else:
logging.error(err)
return None
def post(self):
inventory_details = api.payload
order_number = inventory_details["orderID"]
order_url = "http://cb-order-service:7070/v4/api/orders/" + \
order_number + "/detail/internal"
order_details = requests.request('GET', order_url).json()
logging.info("order_details")
logging.info(str(order_details))
order_amount = order_details["data"]["orderAmount"]
url = 'https://dev67363.service-now.com/api/now/table/fm_expense_line'
payload = {
"short_description": "---CBSTest---",
"user": "******",
"amount": order_amount
}
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
logging.info(str(payload))
response = requests.request('POST',
url,
headers=headers,
json=payload,
auth=('admin', 'K3w1sTVeDtZl'))
logging.info(str(response))
threadVar = threading.Thread(target=calling)
threadVar.start()
return {"result": "expense line added in servicenow"}, 201
def get_worker(port):
ip = utils.IPGetter.get_ip_address()
master_info = NodeInfo(name="master",ip=conf.MASTER_IP,
port=conf.MASTER_PORT, status=NodeStatus.working)
node_info = NodeInfo(name="worker", ip=ip, port=port, status=NodeStatus.idle)
if common.doesServiceExist(node_info.ip, node_info.port):
logging.warning("%s:%s already been used! change another port" % (node_info.ip, node_info.port))
exit(1)
worker_node = WorkerNode(master_info, node_info)
logging.info("%s startup" % worker_node.get_node_info())
return worker_node
def process_queue():
""""simulate to perform some work (for testing)"""
while True:
if msgqueue.empty():
logging.debug("empty queue")
else:
msg = msgqueue.get()
msgqueue.task_done()
logging.info("tcp-msg: %s" % str(msg))
if "exit" in msg:
break
time.sleep(1)
logging.info("finished")
def delete_policy_group(self, policy_group_id):
'''
return: ok, error_message
'''
params = {
'action': 'policygroup_del',
'policygroupid': policy_group_id
}
r = requests.post(WEBROUTER_ABTEST_API, params=params, timeout=TIMEOUT)
logging.info('>>> r: {}.'.format(r))
if r.status_code != requests.codes.ok or r.json()['code'] != 200:
return False, r.json()['desc']
return True, ''
def is_valid(self):
'''
return: ok, error_message
'''
params = {'action': 'policygroup_check'}
r = requests.post(
WEBROUTER_ABTEST_API,
params=params,
data=self.to_json(),
timeout=TIMEOUT)
logging.info('>>> r: {}.'.format(r))
if r.status_code != requests.codes.ok or r.json()['code'] != 200:
return False, r.json()['desc']
return True, ''
def deactive_mountpoint(self, mountpoint):
'''
mountpoint: for example, a.b.c/d (presume host, i.e, `a.b.c`, is present)
return: ok, error_message
'''
params = {
'action': 'runtime_del',
'hostname': self.__normalize_mountpoint(mountpoint)
}
r = requests.post(WEBROUTER_ABTEST_API, params=params, timeout=TIMEOUT)
logging.info('>>> r: {}.'.format(r))
if r.status_code != requests.codes.ok or r.json()['code'] != 200:
return False, r.json()['desc']
return True, ''
def add_policy_group(self, abtest_policy_group):
'''
abtest_policy_group: ABTestPolicyGroup
return: ok, policy_group_id/error_message
'''
params = {'action': 'policygroup_set'}
r = requests.post(
WEBROUTER_ABTEST_API,
params=params,
data=abtest_policy_group.to_json(),
timeout=TIMEOUT)
logging.info('>>> r.text: {}.'.format(r.text))
if r.status_code != requests.codes.ok or r.json()['code'] != 200:
return False, r.json()['desc']
return True, r.json()['data']['groupid']
def before_request():
# py2.7设置编码
reload(sys)
sys.setdefaultencoding('utf8')
# 记录access_log
row = [
request.headers.get('X-Forwarded-For', request.remote_addr),
request.path,
request.url,
request.headers.get('Referer', ''),
];
strlog = "access_info(ip/path/url/referer) \t"
for v in row:
strlog += v + "\t"
logging.info(strlog)
def get_policy_group(self, policy_group_id):
'''
return: ok, ABTestPolicyGroup/error_message
'''
params = {'action': 'policygroup_get', 'policygroupid': policygroup_id}
r = requests.post(WEBROUTER_ABTEST_API, params=params, timeout=TIMEOUT)
logging.info('>>> r: {}.'.format(r))
if r.status_code != requests.codes.ok or r.json()['code'] != 200:
return False, r.json()['desc']
policys = []
for policy_id in r.json['data']['group']:
ok, policy_or_err = self.get_policy(policy_id)
if not ok:
return False, policy_or_err
policys.append(policy)
return True, ABTestPolicyGroup(policys)
def write_to_mongo_db():
"""
爬虫数据写入json文件,
而不是直接写入数据库
"""
logging.info("enter write_mongo_db")
#获得最大页数，确定爬虫范围
last_page_number = get_last_web_page_number()
logging.debug("max page is :%s"%last_page_number)
for i in range(1,last_page_number+1):
#获取页面
page = get_web_page_by_url(URL.format(page_number=i))
#解析页面
for item_raw_str in load_and_parse(page):
#写入mongo数据库
item_json_data = json.loads(item_raw_str)
mongo.insert_news(item_json_data)
logging.info("leaving write_to_mongo_db")
def solve(self):
'''
This is the main loop.
While the desired accuracy has not been reached, the collocation system will
be set up and solved with a iteratively raised number of spline parts.
Returns
-------
callable
Callable function for the system state.
callable
Callable function for the input variables.
'''
# do the first iteration step
logging.info("1st Iteration: {} spline parts".format(self.eqs.trajectories.n_parts_x))
self._iterate()
# this was the first iteration
# now we are getting into the loop
self.nIt = 1
while not self.reached_accuracy and self.nIt < self._parameters['maxIt']:
# raise the number of spline parts
self.eqs.trajectories._raise_spline_parts()
if self.nIt == 1:
logging.info("2nd Iteration: {} spline parts".format(self.eqs.trajectories.n_parts_x))
elif self.nIt == 2:
logging.info("3rd Iteration: {} spline parts".format(self.eqs.trajectories.n_parts_x))
elif self.nIt >= 3:
logging.info("{}th Iteration: {} spline parts".format(self.nIt+1, self.eqs.trajectories.n_parts_x))
# start next iteration step
self._iterate()
# increment iteration number
self.nIt += 1
# as a last, if there were any constraints to be taken care of,
# we project the unconstrained variables back on the original constrained ones
if self.constraints:
self.constrain()
# return the found solution functions
return self.eqs.trajectories.x, self.eqs.trajectories.u
def run_loop(self, stop_event, queue=None):
'''
run handler in endless loop in one dedicate process
'''
# need to config QueueHandler in worker process
if queue is not None:
h = QueueHandler(queue)
root = original_logging.getLogger()
root.addHandler(h)
# set level of queue handler to DEBUG to accept all logs
root.setLevel(original_logging.DEBUG)
# after this, all log will be put into the queue use QueueHandler
# logging.info("subprocess for handler started", self._handler_key)
self._stop_event = stop_event
self._connection.set_stop_condition(self._stop_condition)
self._main()
logging.info("loop stopped for handler", self._handler_key)
if queue is not None:
# close queue
queue.close()
self._before_stop()
sys.exit(0)
def finish_task(self, taskUuid):
"""完成一个作业"""
logging.info("task finished: %s", str(taskUuid))
self.lock.acquire()
self.runningTasks.pop(taskUuid, None)
self.lock.release()
def test():
logging.info("Fetching app_access_token...")
app_access_token = FacebookUserAccess.get_app_access_token()
logging.info("Generating test user...")
testuser = TestUser(app_access_token=app_access_token, permissions='email, manage_pages')
logging.info("Getting short term user token...")
assert(testuser.user_access.access_token)
logging.info("Getting long term user token...")
assert(testuser.user_access.long_term_access_token)
logging.info("Getting access code...")
assert(testuser.user_access.access_code)
logging.info("Getting user permissions...")
assert(testuser.user_access.get_permissions())
logging.info("Getting page tokens...")
assert(not testuser.user_access.get_page_permissions())
# -*-coding=utf-8-*- __author__ = 'jiangdapng([email protected])' import socket from log import logging class IPGetter(object): @classmethod def get_ip_address(cls): ip = socket.gethostbyname(socket.gethostname()) return ip if __name__ == '__main__': logging.info(IPGetter.get_ip_address())
def unconstrain(self, constraints):
'''
This method is used to enable compliance with desired box constraints given by the user.
It transforms the vectorfield by projecting the constrained state variables on
new unconstrained ones.
Parameters
----------
constraints : dict
The box constraints for the state variables
'''
# save constraints
self.constraints = constraints
# backup the original constrained system
self._dyn_sys_orig = copy.deepcopy(self.dyn_sys)
# get symbolic vectorfield
# (as sympy matrix toenable replacement method)
x = sp.symbols(self.dyn_sys.states)
u = sp.symbols(self.dyn_sys.inputs)
ff_mat = sp.Matrix(self.dyn_sys.f_sym(x, u))
# get neccessary information form the dynamical system
a = self.dyn_sys.a
b = self.dyn_sys.b
boundary_values = self.dyn_sys.boundary_values
# handle the constraints by projecting the constrained state variables
# on new unconstrained variables using saturation functions
for k, v in self.constraints.items():
# check if boundary values are within saturation limits
xk = self.dyn_sys.states[k]
xa, xb = self.dyn_sys.boundary_values[xk]
if not ( v[0] < xa < v[1] ) or not ( v[0] < xb < v[1] ):
logging.error('Boundary values have to be strictly within the saturation limits!')
logging.info('Please have a look at the documentation, \
especially the example of the constrained double intgrator.')
raise ValueError('Boundary values have to be strictly within the saturation limits!')
# calculate saturation function expression and its derivative
yk = sp.Symbol(xk)
m = 4.0/(v[1] - v[0])
psi = v[1] - (v[1]-v[0])/(1. + sp.exp(m * yk))
#dpsi = ((v[1]-v[0])*m*sp.exp(m*yk))/(1.0+sp.exp(m*yk))**2
dpsi = (4. * sp.exp(m * yk))/(1. + sp.exp(m * yk))**2
# replace constrained variables in vectorfield with saturation expression
# x(t) = psi(y(t))
ff_mat = ff_mat.replace(sp.Symbol(xk), psi)
# update vectorfield to represent differential equation for new
# unconstrained state variable
#
# d/dt x(t) = (d/dy psi(y(t))) * d/dt y(t)
# <==> d/dt y(t) = d/dt x(t) / (d/dy psi(y(t)))
ff_mat[k] /= dpsi
# update boundary values for new unconstrained variable
boundary_values[xk] = ( (1./m) * np.log((xa - v[0]) / (v[1] - xa)),
(1./m) * np.log((xb - v[0]) / (v[1] - xb)) )
# create a callable function for the new symbolic vectorfield
ff = np.asarray(ff_mat).flatten().tolist()
xu = self.dyn_sys.states + self.dyn_sys.inputs
_f_sym = sp.lambdify(xu, ff, modules='sympy')
def f_sym(x, u):
xu = np.hstack((x,u))
return _f_sym(*xu)
# create a new unconstrained system
xa = [boundary_values[x][0] for x in self.dyn_sys.states]
xb = [boundary_values[x][1] for x in self.dyn_sys.states]
ua = [boundary_values[u][0] for u in self.dyn_sys.inputs]
ub = [boundary_values[u][1] for u in self.dyn_sys.inputs]
self.dyn_sys = DynamicalSystem(f_sym , a, b, xa, xb, ua, ub)
def publish(self, message_type, message, priority=0):
logging.info('published message: %s' % message)
if not (message_type in self._proxies.keys()):
self._add_proxy(message_type)
return self._proxies[message_type].publish(message, priority)
