def _fdsserver_is_ready(self):
'''
check fdsserver is ready
return True if state is ready
'''
cmd = "netstat -napt 2>&1 | grep 10000 | grep FDSServer"
data = os.popen(cmd).read()
if not data:
Debug.error("query fdsserver state error, command: %s!" % cmd)
return False
Debug.debug("FDSServer: %s" % data)
return True
def health_check(ipport):
''' health check
returns:
0 -port is opened and listening
2 -port is not opened
'''
cmd = "netstat -napu 2>&1 | grep " + ipport + " | grep PluginContain"
data = os.popen(cmd).read()
if not data:
Debug.error("query udp sip state error, command: %s!" % cmd)
return SERVICE_CRITICAL
Debug.debug("port status: %s" % data)
return SERVICE_PASSING
def get_service_ipport(servicename, serviceid):
consul_ipport = get_consul_address()
reqfile = "/v1/catalog/service/" + servicename
jdata = http_get_request(consul_ipport, reqfile)
Debug.debug("query service: %s" % json_dumps4(jdata))
'''
{
"Node": "linux-eagle-42",
"Datacenter": "dc2",
"CreateIndex": 72590,
"ServiceName": "intf-udp_sip",
"TaggedAddresses": {
"wan": "10.180.30.42",
"lan": "10.180.30.42"
},
"ModifyIndex": 72590,
"ServicePort": 5060,
"ServiceID": "[email protected]:5060",
"ServiceAddress": "10.180.30.42",
"Address": "10.180.30.42",
"ServiceTags": [
"version:GMPC17"
],
"NodeMeta": {
"consul-network-segment": ""
},
"ServiceEnableTagOverride": false,
"ID": "460a16cb-f28d-5fdf-40e7-92e960c440f0"
},
'''
# extract port
for it in jdata:
if ("ServiceID" in it.keys() and "ServiceAddress" in it.keys() and "ServicePort" in it.keys()):
# service_id = it["ServiceID"];
ipaddr = it["ServiceAddress"];
port = it["ServicePort"]
if not ipaddr:
# FIXME
ipaddr = "127.0.0.1"
if serviceid == it["ServiceID"]:
print("service Id: %s" % serviceid)
state = health_check(ipaddr + ":" + str(port))
print("service state: %d" % state)
return state
def http_put_request(ipport, reqfile, jdata):
''' request http server, return JSON response '''
Debug.debug("PUT http://{0}{1} : {2}".format(
':'.join(str(n) for n in ipport), reqfile, jdata))
resp = ""
try:
httpClient = httplib.HTTPConnection(*ipport, timeout=100)
headers = {"Content-type": "application/json"}
httpClient.request('PUT', reqfile, json_dumps(jdata), headers)
response = httpClient.getresponse()
Debug.debug("HTTP response: %s" % response.status)
resp = response.read().decode()
except Exception as err:
Debug.error("%%Error post_req: %s" % str(err))
finally:
if httpClient:
httpClient.close()
return json_parse(resp)
def service_is_registered(self):
consul_ipport = get_consul_address();
reqfile = "/v1/catalog/service/" + self._service_name
Debug.debug("service_check service: {0}".format(self._service_name))
jdata = http_get_request(consul_ipport, reqfile)
''' http response shall like:
[{
...
"ServiceID": "xxx",
"ServiceName": "{component}",
"ServiceTags": ["hostname"],
"ServiceAddress": "{ip}",
"ServicePort": {port},
...
}, ...]
'''
if not type(jdata) is type([]):
Debug.error("Bad data type(requied []): %s" % str(type(jdata)))
return False;
if len(jdata) <= 0 or not "ServiceName" in jdata[0].keys():
return False
return True
#encoding:utf-8
import urllib2
from bs4 import BeautifulSoup
import re
import threading
from fiction.models import *
import string
from debug import Debug
from models import *
from utils.hashs import create_nid
#for the thread to crawler all the fiction information from qidian.com
from crawler_config import ALL_PATTERN
DG = Debug()
DG.debug = True
def get_book_infor(host, name, url, type_lock=False):
DG.trace('yes now')
"""get book information(avatar, introduction, click times, type, completed words number,
and all the chapter in website:host
Args:
host: fiction website
book_url: book url located in host
content_tag: div area contains the fiction's information that we want
content_class: indetification flags
avatar_tag: the fictions' avatar
avatar_class: indetification
intro_tag, intro_class
Return:
