def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception, e:
logging.error("Error: %s : %s", func.__name__, str(e))
if not hasattr(e, 'errorCode'):
out.failureMessage("Sorry, operation has failed!!!.")
tools.exit()
errorCode = int(e.errorCode)
# auth-token error, re-auth
if errorCode == 9:
storage = Storage()
storage.removeUser()
GeekNote()
return func(*args, **kwargs)
elif errorCode == 3:
out.failureMessage("Sorry, you do not have permissions to do this operation.")
else:
return False
tools.exit()
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# Non-Pretty HTML output
contentHTML = str(BeautifulSoup(contentHTML))
else:
contentHTML = Editor.HTMLEscape(content)
return Editor.wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exit()
def _reliable_op(*args, **kwargs):
argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
fname = func.func_name
msg = 'MQ - -> %s(%s)' % (fname, ','.join('%s=%s' % entry \
for entry in zip(argnames[1:],\
args[1:]) + kwargs.items()))
trans_times = 0
while True:
trans_times += 1
if trans_times > MAX_RETRY:
logging.error('rabbitmq connection lost!')
return None
try:
###############################################################
logging.warn('function:%s' %(func.func_name))
###############################################################
return func(*args, **kwargs)
except pika.exceptions.AMQPConnectionError, e:
logging.error("AMQPConnectionError: %s" % e)
logging.error(traceback.format_exc())
if not _reconnect(args, msg, e):
raise
except pika.exceptions.AMQPChannelError, e:
try:
if len(args) == 0 or not isinstance(args[0], RabbitMQBlockingClient):
raise MQError("unsupported decorator, it should be applied to RabbitMQBlockingClient methods")
close_func = getattr(args[0], "close")
close_func()
except Exception, e:
logging.error(e)
logging.error(traceback.format_exc())
def RepoListParser(self, url):
try:
repoListHtml = self.session.get(url)
except Exception as e:
logging.error(e)
return
repoListHtmlSoup = BeautifulSoup(repoListHtml.text, 'html5lib')
repoListTables = repoListHtmlSoup.table
if repoListTables == None:
return
for oneTr in repoListTables.select('tr'):
tds = oneTr.find_all('td')
if len(tds) == 0:
continue
repoPathTd = tds[0]
repoOwnerTd = tds[2]
newRepo = {}
newRepo['path'] = self.UrlBase+repoPathTd.a['href']
newRepo['owner'] = repoOwnerTd.a.get_text()
self.AllRepos.append(newRepo)
nextPage = repoListHtmlSoup.find('div', id="paginator")
if len(nextPage) == 0:
return 0
page_a = nextPage.select('a')
for one_a in page_a:
if repr(one_a.get_text()) == repr(u'下一页'):
pageUrl = one_a['href']
self.RepoListParser(self.RepoListUrl+pageUrl)
def checkVersion(self):
versionOK = self.getUserStore().checkVersion("Python EDAMTest",
UserStoreConstants.EDAM_VERSION_MAJOR,
UserStoreConstants.EDAM_VERSION_MINOR)
if not versionOK:
logging.error("Old EDAM version")
return tools.exitErr()
def plot(self):
'''
Plot the calculated trajectories and show interval error functions.
This method calculates the error functions and then calls
the :py:func:`visualisation.plotsim` function.
'''
try:
import matplotlib
except ImportError:
logging.error('Matplotlib is not available for plotting.')
return
if self.constraints:
sys = self._dyn_sys_orig
else:
sys = self.dyn_sys
# calculate the error functions H_i(t)
max_con_err, error = auxiliary.consistency_error((sys.a, sys.b),
self.eqs.trajectories.x,
self.eqs.trajectories.u,
self.eqs.trajectories.dx,
sys.f_num, len(self.sim_data[0]), True)
H = dict()
for i in self.eqs.trajectories._eqind:
H[i] = error[:,i]
visualisation.plot_simulation(self.sim_data, H)
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for cteating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# remove all new-lines characters in html
contentHTML = re.sub(r'\n', r'', contentHTML)
else:
contentHTML = HTMLEscape(content)
return wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exit()
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception, e:
logging.error("Error: %s : %s", func.__name__, str(e))
if not hasattr(e, 'errorCode'):
out.failureMessage("Sorry, operation has failed!!!.")
tools.exitErr()
errorCode = int(e.errorCode)
# auth-token error, re-auth
if errorCode == 9:
storage = Storage()
storage.removeUser()
GeekNote()
return func(*args, **kwargs)
elif errorCode == 3:
out.failureMessage("Sorry, you do not have permissions "
"to do this operation.")
# Rate limited
# Patched because otherwise if you get rate limited you still keep
# hammering the server on scripts
elif errorCode == 19:
print("\nRate Limit Hit: Please wait %s seconds before continuing" %
str(e.rateLimitDuration))
tools.exitErr()
else:
return False
tools.exitErr()
def _add_proxy(self, message_type):
import DefaultConfig
message_config = RabbitMQBlockingClient._get_message_config(message_type, DefaultConfig.MESSAGE_CONFIGS)
proxy = RabbitMQBlockingProxy(self._blocking_connection, self._aux_store, message_config)
key = message_config["message_type"]
if key in self._proxies.keys():
logging.error("proxy has been added", key)
raise MQError("proxy has been added", key)
self._proxies[key] = proxy
def auth(self):
GNA = GeekNoteAuth()
self.authToken = GNA.getToken()
userInfo = self.getUserInfo()
if not isinstance(userInfo, object):
logging.error("User info not get")
return False
self.getStorage().createUser(self.authToken, userInfo)
return True
def allowAccess(self):
response = self.loadPage(self.url['base'],
self.url['access'],
"GET",
{'oauth_token': self.tmpOAuthToken})
logging.debug(response.data)
tree = html.fromstring(response.data);
token = "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[0]}) + "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[1]})
sourcePage = tree.xpath("//input[@name='_sourcePage']/@value")[0]
fp = tree.xpath("//input[@name='__fp']/@value")[0]
targetUrl = tree.xpath("//input[@name='targetUrl']/@value")[0]
logging.debug(token);
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
access = self.postData['access']
access['oauth_token'] = self.tmpOAuthToken
access['oauth_callback'] = ""
access['embed'] = 'false'
access['suggestedNotebookName'] = 'Geeknote'
access['supportLinkedSandbox'] = ''
access['analyticsLoginOrigin'] = 'Other'
access['clipperFlow'] = 'false'
access['showSwitchService'] = 'true'
access['_sourcePage'] = sourcePage
access['__fp'] = fp
access['targetUrl'] = targetUrl
response = self.loadPage(self.url['base'],
self.url['access'],
"POST", access, token)
if response.status != 302:
logging.error("Unexpected response status on allowing "
"access 302 != %s", response.status)
logging.error(response.data)
tools.exitErr()
responseData = self.parseResponse(response.location)
if 'oauth_verifier' not in responseData:
logging.error("OAuth verifier not found")
tools.exitErr()
self.verifierToken = responseData['oauth_verifier']
logging.debug("OAuth verifier token take")
def reply(self, message, response):
properties = pika.BasicProperties(correlation_id=message["__correlation_id"])
if self._rpc_reply_content_type == "text/json":
response = dumps_jsonx(response)
try:
self._channel.basic_publish(exchange="", routing_key=message["__reply_to"], body=response, properties=properties)
except AttributeError, e:
logging.error('basic publish raise AttributeError: %s' % e)
raise pika.exceptions.AMQPConnectionError
def FileListParser(self, url, fileOwner):
try:
fileListHtml = self.session.get(url)
except Exception as e:
logging.error(e)
newRepo = {}
newRepo['path'] = url
newRepo['owner'] = fileOwner
if self.FileLock.acquire() == True:
self.AllRepos.append(newRepo)
self.FileLock.release()
fileListHtmlSoup = BeautifulSoup(fileListHtml.text, 'html5lib')
fileListTables = fileListHtmlSoup.table
if fileListTables == None:
return 0
fileListTrs = fileListTables.select('tr')
if len(fileListTrs) == 0:
return 0
for oneTr in fileListTrs[1:]:
# print "level--->%s" %oneTr.select('select[class="file-level-type-select"]')[0].option['value']#.select('option["selected="selected"]')
previewA = oneTr.select('span[class="dirent-name"]')[0].select('a')[0]
previewPath = previewA['href']
pathList = repr(previewPath).split('/')
if len(pathList) < 4:
continue
path = pathList[1]+'/'+pathList[2]+'/'+pathList[3]
if path == 'sys/seafadmin/repo':
# print 'this is a dir'
self.FileListParser(self.UrlBase+previewPath, fileOwner)
else:
# print 'find a new file'
fileName = previewA.get_text()
downloadPath = oneTr.select('a')[2]['href']
newFile = {}
newFile['fileName'] = fileName
newFile['owner'] = fileOwner
newFile['previewPath'] = self.UrlBase+previewPath
newFile['downloadPath'] = self.UrlBase+downloadPath
tagSpan = oneTr.select('span[class="file-tags"]')
level = oneTr.select('select[class="file-level-type-select"]')[0].select('option[selected="selected"]')[0]['value']
newFile['level'] = level
if len(tagSpan) > 0:
newFile['tags'] = tagSpan[0].get_text()
else:
newFile['tags'] = ''
if self.FileLock.acquire():
self.AllFiles.append(newFile)
self.FileLock.release()
def im_transform(image, size=-1, scale=1., mean_values=[0.,0.,0.]):
assert len(mean_values) == 3
if size > 0:
try:
patch = cv2.resize(image, (size, size))
except:
logging.error("Unable to resize image.")
print image.shape
raise
else:
patch = image
trans_img = (np.asarray(patch) - np.asarray(mean_values)) * scale
return trans_img.swapaxes(1,2).swapaxes(0,1)
def loadPage(self, url, uri=None, method="GET", params=""):
if not url:
logging.error("Request URL undefined")
tools.exitErr()
if not uri:
urlData = urlparse(url)
url = urlData.netloc
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params:
params = urlencode(params)
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie": '; '.join([key + '=' + self.cookies[key] for key in self.cookies.keys()])
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
logging.debug("Request URL: %s:/%s > %s # %s", url,
uri, unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(url)
conn.request(method, uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
logging.debug("Response : %s > %s",
response.status,
response.getheaders())
result = tools.Struct(status=response.status,
location=response.getheader('location', None),
data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
# delete cookies whose content is "deleteme"
for key in self.cookies.keys():
if self.cookies[key] == "deleteme":
del self.cookies[key]
return result
def login(self):
response = self.loadPage(self.url['base'], self.url['login'], "GET", {'oauth_token': self.tmpOAuthToken})
if response.status != 200:
logging.error("Unexpected response status on login 200 != %s", response.status)
tools.exit()
if not self.cookies.has_key('JSESSIONID'):
logging.error("Not found value JSESSIONID in the response cookies")
tools.exit()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth']%self.tmpOAuthToken
response = self.loadPage(self.url['base'], self.url['login']+";jsessionid="+self.cookies['JSESSIONID'], "POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exit()
logging.debug("Success authorize, redirect to access page")
def getOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_token=self.tmpOAuthToken, oauth_verifier=self.verifierToken))
if response.status != 200:
logging.error("Unexpected response status on getting oauth token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth token not found")
tools.exit()
logging.debug("OAuth token take : %s", responseData['oauth_token'])
self.OAuthToken = responseData['oauth_token']
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
TODO: add media support
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format == 'markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# Non-Pretty HTML output
contentHTML = str(BeautifulSoup(contentHTML, 'html.parser'))
elif format == 'html':
# Html to ENML http://dev.evernote.com/doc/articles/enml.php
ATTR_2_REMOVE = ["id"
, "class"
#, "on*"
, "accesskey"
, "data"
, "dynsrc"
, "tabindex"
];
soup = BeautifulSoup(content, 'html.parser')
for tag in soup.findAll():
if hasattr(tag, 'attrs'):
map(lambda x: tag.attrs.pop(x, None), \
[k for k in tag.attrs.keys() \
if k in ATTR_2_REMOVE \
or k.find('on') == 0])
contentHTML = str(soup)
else:
contentHTML = Editor.HTMLEscape(content)
return Editor.wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exit()
def getTmpOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_callback="https://"+self.url['base']))
if response.status != 200:
logging.error("Unexpected response status on get temporary oauth_token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth temporary not found")
tools.exit()
self.tmpOAuthToken = responseData['oauth_token']
logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
def handleTwoFactor(self):
self.code = out.GetUserAuthCode()
self.postData['tfa']['code'] = self.code
response = self.loadPage(self.url['base'], self.url['tfa'] + ";jsessionid=" + self.cookies['JSESSIONID'], "POST", self.postData['tfa'])
if not response.location and response.status == 200:
if self.incorrectCode < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect two factor code')
out.preloader.setMessage('Authorize...')
self.incorrectCode += 1
return self.handleTwoFactor()
else:
logging.error("Incorrect two factor code")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
storage = Storage()
extras = storage.getUserprop('markdown2_extras')
#contentHTML = markdown.markdown(content).encode("utf-8")
contentHTML = markdown.markdown(
content, extras=extras).encode("utf-8")
soup = BeautifulSoup(contentHTML, 'html.parser')
Editor.checklistInSoupToENML(soup)
# Non-Pretty HTML output
contentHTML = str(soup)
#
# For the 'pre' format, simply wrap the content with a 'pre' tag. Do
# perform any parsing/mutation.
#
elif format=='pre':
contentHTML = u''.join(('<pre>', content, '</pre>')).encode("utf-8")
else:
contentHTML = Editor.HTMLEscape(content)
contentHTML = contentHTML.replace('[x]','<en-todo checked="true"></en-todo>')
contentHTML = contentHTML.replace('[ ]','<en-todo></en-todo>')
return Editor.wrapENML(contentHTML)
except:
import traceback
traceback.print_exc()
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exitErr()
def allowAccess(self):
self.postData['access']['oauth_token'] = self.tmpOAuthToken
self.postData['access']['oauth_callback'] = "https://"+self.url['base']
response = self.loadPage(self.url['base'], self.url['access'], "POST", self.postData['access'])
if response.status != 302:
logging.error("Unexpected response status on allowing access 302 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.location)
if not responseData.has_key('oauth_verifier'):
logging.error("OAuth verifier not found")
tools.exit()
self.verifierToken = responseData['oauth_verifier']
logging.debug("OAuth verifier token take")
def matlab_engine(fun_file, input_list, eng=None):
'''matlab enginer wrapper
return_val = fun(input_list)
'''
import matlab.engine
script_dirname = os.path.abspath(os.path.dirname(fun_file))
fun_name = stem(fun_file)
if eng is None:
eng = matlab.engine.start_matlab('-nodisplay -nojvm -nosplash -nodesktop')
else:
logging.debug("Use opened Matlab session: {}".format(eng))
try:
eng.cd(script_dirname)
func = getattr(eng, fun_name)
result = func(input_list)
except matlab.engine.EngineError, e:
# Use new engine instead
logging.error("Existing engine no response: {}".format(eng))
raise e
def login(self):
response = self.loadPage(self.url['base'],
self.url['login'],
"GET",
{'oauth_token': self.tmpOAuthToken})
# parse hpts and hptsh from page content
hpts = re.search('.*\("hpts"\)\.value.*?"(.*?)"', response.data)
hptsh = re.search('.*\("hptsh"\)\.value.*?"(.*?)"', response.data)
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
self.postData['login']['hpts'] = hpts and hpts.group(1) or ""
self.postData['login']['hptsh'] = hptsh and hptsh.group(1) or ""
response = self.loadPage(self.url['base'],
self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
"POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
if response.status == 302:
# the user has enabled two factor auth
return self.handleTwoFactor()
logging.debug("Success authorize, redirect to access page")
def getTmpOAuthToken(self):
response = self.loadPage(self.url['base'],
self.url['token'],
"GET",
self.getTokenRequestData(
oauth_callback="https://" + self.url['base']))
if response.status != 200:
logging.error("Unexpected response status on get "
"temporary oauth_token 200 != %s", response.status)
raise OAuthError('OAuth token request failed')
responseData = self.parseResponse(response.data)
if 'oauth_token' not in responseData:
logging.error("OAuth temporary not found")
raise OAuthError('OAuth token request failed')
self.tmpOAuthToken = responseData['oauth_token']
logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
def plot(self, show=True, ret_array=False):
'''
Plots the spline function or returns an array with its values at
some points of the spline interval.
Parameters
----------
show : bool
Whethter to plot the spline's curve or not.
ret_array : bool
Wheter to return an array with values of the spline at points
of the interval.
'''
if not show and not ret_array:
# nothing to do here...
return
elif self._prov_flag:
# spline cannot be plotted, because there are no numeric
# values for its polynomial coefficients
logging.error("There are no numeric values for the spline's\
polynomial coefficients.")
return
# create array of values
tt = np.linspace(self.a, self.b, 1000, endpoint=True)
St = [self.f(t) for t in tt]
if show:
try:
import matplotlib.pyplot as plt
plt.plot(tt,St)
plt.show()
except ImportError:
logging.error('Could not import matplotlib for plotting the curve.')
if ret_array:
return St
def getSearchBySphinx(self, q, index = '*'):
"""
从sphinx搜索结果
Args:
q - 搜索词
index - 需要使用的索引, default: '*'
Returns:
Id list.
"""
res = self.sp.query(q, index)
lastError = self.sp.getLastError()
lastWarning = self.sp.getLastWarning()
if lastError:
logging.error('getSearchBySphinx has error: %s' % (lastError))
if lastWarning:
logging.warning('getSearchBySphinx has warning: %s' % (lastWarning))
return res
def reliable_op(func):
"""
if message queue is broken, will try to reconnect...
"""
def _reconnect(args, msg, e):
if len(args) == 0 or not isinstance(args[0], RabbitMQBlockingClient):
raise MQError("unsupported decorator, it should be applied to RabbitMQBlockingClient methods")
self = args[0]
########################################################
logging.error("enter reconnection")
########################################################
init_func = getattr(self, "_init")
client_configs = getattr(self, "_client_configs")
########################################################
logging.error("begin reconnection")
########################################################
logging.error(\
"message queue connection error: %s, %s reconnecting..."\
% (msg, e))
time.sleep(5)
try:
########################################################
logging.error("test init_func")
########################################################
init_func(client_configs)
logging.info("message queue reconnection succeeded")
########################################################
logging.error("test init_func")
########################################################
return True
except AMQPChannelError, e:
try:
close_func = getattr(self, "close")
close_func()
except Exception, e:
logging.error(e)
logging.error(traceback.format_exc())
logging.error('Exception', e)
logging.error(traceback.format_exc())
def crawlStock(stockidList, k):
try:
tstr = ""
datalist = []
for t in tempList:
if tstr == "":
tstr = k + t.strip()
else:
tstr = tstr + "," + k + t.strip()
#logging.info(tstr)
url = "http://qt.gtimg.cn/q=" + tstr
#print url
response = urllib2.urlopen(url)
html = response.read()
#logging.info(html)
lines = html.strip().split(";")
if "" in lines:
lines.remove("")
for line in lines:
lv = line.strip().split("=")[1].replace('"', '').split('~')
datalist.append(lv)
analyseData1(datalist, k)
except Exception, e:
logging.error(traceback.format_exc())
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception, e:
logging.error("Error: %s : %s", func.__name__, str(e))
if not hasattr(e, 'errorCode'):
out.failureMessage("Sorry, operation has failed!!!.")
tools.exitErr()
errorCode = int(e.errorCode)
# auth-token error, re-auth
if errorCode == 9:
storage = Storage()
storage.removeUser()
GeekNote()
return func(*args, **kwargs)
elif errorCode == 3:
out.failureMessage("Sorry, you do not have permissions "
"to do this operation.")
# Rate limited
# Patched because otherwise if you get rate limited you still keep
# hammering the server on scripts
elif errorCode == 19:
print(
"\nRate Limit Hit: Please wait %s seconds before continuing"
% str(e.rateLimitDuration))
tools.exitErr()
else:
return False
tools.exitErr()
def login(self):
response = self.loadPage(self.url['base'],
self.url['login'],
"GET",
{'oauth_token': self.tmpOAuthToken})
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
response = self.loadPage(self.url['base'],
self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
"POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
if response.status == 302:
# the user has enabled two factor auth
return self.handleTwoFactor()
logging.debug("Success authorize, redirect to access page")
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content,"utf-8")
# add 2 space before new line in paragraph for cteating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# remove all new-lines characters in html
contentHTML = re.sub(r'\n', r'', contentHTML)
else:
contentHTML = HTMLEscape(content)
return wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html. Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. Content must be an UTF-8 encode.")
return tools.exit()
def textToENML(content, raise_ex=False):
"""
Create an ENML format of note.
"""
pdb.set_trace()
if not isinstance(content, str):
content = ""
try:
content = unicode(content,"utf-8")
# add 2 space before new line in paragraph for cteating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
#contentHTML, md = convert_markdown(content)
#meta = md.Meta
# remove all new-lines characters in html
contentHTML = re.sub(r'\n', r'', content)
return wrapENML(contentHTML)
except:
raise
if raise_ex:
raise Exception("Error while parsing text to html. Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. Content must be an UTF-8 encode.")
return tools.exit()
def checkJumpAndDuplicatedRecordFromSPC(coreName, machineId):
responseData = initResponse()
result = {}
result['status'] = ''
responseStr = ''
jsonObject = {}
try:
jsonObject = queryLastSPC(coreName, machineId)
data = praseJsonFormat(jsonObject)
# logging.info(data)
df = pd.DataFrame(data).iloc[0].astype(str)
# 取第一筆
timestamp_iso = df['timestamp_iso']
RD_618 = df['RD_618']
SPC_0 = df['SPC_0']
if RD_618 == SPC_0:
return 1, RD_618 # true, 模次正常
return 0, RD_618 # false, 模次亂跳或是模次重複
except:
logging.error("checkJumpAndDuplicatedRecordFromSPC exception")
responseData = setErrorResponse(responseData, '400', 'exception')
return parseResponseJson(responseData, result, 'result')
def insert(con: mysql.connector.MySQLConnection, sql: str) -> int:
try:
cursor = con.cursor()
cursor.execute(sql)
con.commit()
cursor.close()
except mysql.connector.IntegrityError as e:
logging.error(f'DB insert 데이터가 중복되었습니다. sql: {sql}')
logging.error(f'{e}')
# 중복
return 1
except Exception as e:
logging.error(f'DB insert 중 에러가 발생하였습니다. sql: {sql}')
logging.error(f'{e}')
# 에러
return 2
return 0
def db_connect():
# Construct connection string
try:
conn = mysql.connector.connect(**config)
logging.info("Connection established")
return conn
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
logging.error("Something is wrong with the user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
logging.error("Database does not exist")
else:
logging.error(err)
return None
#encoding=utf-8
import threading
from log import logging
import os
try:
from bs4 import BeautifulSoup
except Exception:
logging.error('import [BeautifulSoup] error')
os._exit()
class WorkThread(threading.Thread):
session = ''
AllRepos = []
AllFiles = []
UrlBase = ''
RepoListUrl = '' #UrlBase+'/sys/seafadmin/'
FileLock = threading.Lock()
RepoLock = threading.Lock()
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
if self.RepoLock.acquire():
if len(self.AllRepos) == 0:
self.RepoLock.release()
break
def parse(self):
self.INP_DATA = {}
if self.CMD is None:
out.printAbout()
return False
if self.CMD == "autocomplete":
# подставляем аргументы для автозаполнения
# делаем смещение на 1 аргумент, т.к. 1 это autocomplete
self.__init__(self.sys_argv[1:])
self.printAutocomplete()
return False
if self.CMD == "--help":
self.printHelp()
return False
if not self.COMMANDS.has_key(self.CMD):
self.printErrorCommand()
return False
if "--help" in self.INP:
self.printHelp()
return False
# Подготовка данных
for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
# установка значений по умолчаеию
if params.has_key('default'):
self.INP_DATA[arg] = params['default']
# замена altName во входящих аргументах на полные
if params.has_key('altName') and params['altName'] in self.INP:
self.INP[self.INP.index(params['altName'])] = arg
activeArg = None
ACTIVE_CMD = None
# проверяем и подставляем первый адгумент по умолчанию
if self.COMMANDS[self.CMD].has_key('firstArg'):
firstArg = self.COMMANDS[self.CMD]['firstArg']
if len(self.INP) > 0:
# смотрим что первое знаение не аргумент по умолчанию, а другой аргумент
if self.INP[0] not in (self.CMD_ARGS.keys() + self.CMD_FLAGS.keys()):
self.INP = [firstArg, ] + self.INP
else:
self.INP = [firstArg, ]
for item in self.INP:
# Проверяем что ожидаем аргумент
if activeArg is None:
# Действия для аргумента
if self.CMD_ARGS.has_key(item):
activeArg = item
ACTIVE_CMD = self.CMD_ARGS[activeArg]
# Действия для флага
elif self.CMD_FLAGS.has_key(item):
self.INP_DATA[item] = self.CMD_FLAGS[item]["value"]
# Ошибка параметр не найден
else:
self.printErrorArgument(item)
return False
else:
activeArgTmp = None
# Значения является параметром
if self.CMD_ARGS.has_key(item) or self.CMD_FLAGS.has_key(item):
# "Активный" аргумент имеет параметр emptyValue
if ACTIVE_CMD.has_key("emptyValue"):
activeArgTmp = item # запоминаем новый "активный" аргумент
item = ACTIVE_CMD['emptyValue'] # подменяем значение на emptyValue
# Ошибка, "активный" аргумент не имеет значений
else:
self.printErrorArgument(activeArg, item)
return False
if ACTIVE_CMD.has_key("type"):
convType = ACTIVE_CMD['type']
if convType not in (int, str):
logging.error("Unsupported argument type: %s", convType)
return False
try:
item = convType(item)
except:
self.printErrorArgument(activeArg, item)
return False
self.INP_DATA[activeArg] = item
activeArg = activeArgTmp # тут или пусто, или новый "активный" аргумент
# если остались "активные" аршументы
if activeArg is not None:
# если есть параметр emptyValue
if ACTIVE_CMD.has_key("emptyValue"):
self.INP_DATA[activeArg] = ACTIVE_CMD['emptyValue']
# инече ошибка
else:
self.printErrorArgument(activeArg, "")
return False
# проверка, присутствует ли необходимый аргумент запросе
for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
if params.has_key('required') and arg not in self.INP:
self.printErrorReqArgument(arg)
return False
# trim -- and ->_
self.INP_DATA = dict([key.lstrip("-").replace("-", "_"), val] for key, val in self.INP_DATA.items() )
return self.INP_DATA
def textToENML(content, raise_ex=False, format="markdown", rawmd=False):
"""
Transform formatted text to ENML
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
content = re.sub(r"([^\r\n])([\r\n])([^\r\n])", r"\1 \n\3",
content)
# content = re.sub(r'\r\n', '\n', content)
if format == "pre":
# For the 'pre' format, simply wrap the content with a 'pre' tag.
# Do not perform any further parsing/mutation.
contentHTML = u"".join(
("<pre>", content, "</pre>")).encode("utf-8")
elif format == "markdown":
# Markdown format https://daringfireball.net/projects/markdown/basics
extras = None
if not rawmd:
storage = Storage()
extras = storage.getUserprop("markdown2_extras")
content = Editor.HTMLEscapeTag(content)
contentHTML = markdown.markdown(content, extras=extras)
soup = BeautifulSoup(contentHTML, "html.parser")
Editor.checklistInSoupToENML(soup)
contentHTML = str(soup)
elif format == "html":
# Html to ENML http://dev.evernote.com/doc/articles/enml.php
soup = BeautifulSoup(content, "html.parser")
ATTR_2_REMOVE = [
"id",
"class",
# "on*",
"accesskey",
"data",
"dynsrc",
"tabindex",
]
for tag in soup.findAll():
if hasattr(tag, "attrs"):
map(
lambda x: tag.attrs.pop(x, None),
[
k for k in tag.attrs.keys()
if k in ATTR_2_REMOVE or k.find("on") == 0
],
)
contentHTML = str(soup)
else:
# Plain text format
contentHTML = Editor.HTMLEscape(content)
tmpstr = ""
for l in contentHTML.split("\n"):
if l == "":
tmpstr = tmpstr + u"<div><br/></div>"
else:
tmpstr = tmpstr + u"<div>" + l + u"</div>"
contentHTML = tmpstr.encode("utf-8")
contentHTML = contentHTML.replace(
"[x]", '<en-todo checked="true"></en-todo>')
contentHTML = contentHTML.replace("[ ]", "<en-todo></en-todo>")
return Editor.wrapENML(contentHTML)
except:
import traceback
traceback.print_exc()
if raise_ex:
raise Exception("Error while parsing text to html.")
logging.error("Error while parsing text to html.")
out.failureMessage("Error while parsing text to html.")
return tools.exitErr()
def textToENML(content, raise_ex=False, format='markdown', rawmd=False):
"""
Transform formatted text to ENML
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
content = re.sub(r'\r\n', '\n', content)
if format == 'pre':
# For the 'pre' format, simply wrap the content with a 'pre' tag.
# Do not perform any further parsing/mutation.
contentHTML = u''.join(
('<pre>', content, '</pre>')).encode("utf-8")
elif format == 'markdown':
# Markdown format https://daringfireball.net/projects/markdown/basics
storage = Storage()
extras = storage.getUserprop('markdown2_extras')
if not rawmd:
content = Editor.HTMLEscapeTag(content)
contentHTML = markdown.markdown(content, extras=extras)
soup = BeautifulSoup(contentHTML, 'html.parser')
Editor.checklistInSoupToENML(soup)
contentHTML = str(soup)
elif format == 'html':
# Html to ENML http://dev.evernote.com/doc/articles/enml.php
soup = BeautifulSoup(content, 'html.parser')
ATTR_2_REMOVE = [
"id",
"class",
# "on*",
"accesskey",
"data",
"dynsrc",
"tabindex"
]
for tag in soup.findAll():
if hasattr(tag, 'attrs'):
map(lambda x: tag.attrs.pop(x, None), [
k for k in tag.attrs.keys()
if k in ATTR_2_REMOVE or k.find('on') == 0
])
contentHTML = str(soup)
else:
# Plain text format
contentHTML = Editor.HTMLEscape(content)
tmpstr = ''
for l in contentHTML.split('\n'):
if l == '':
tmpstr = tmpstr + u'<div><br/></div>'
else:
tmpstr = tmpstr + u'<div>' + l + u'</div>'
contentHTML = tmpstr.encode("utf-8")
contentHTML = contentHTML.replace(
'[x]', '<en-todo checked="true"></en-todo>')
contentHTML = contentHTML.replace('[ ]', '<en-todo></en-todo>')
return Editor.wrapENML(contentHTML)
except:
import traceback
traceback.print_exc()
if raise_ex:
raise Exception("Error while parsing text to html.")
logging.error("Error while parsing text to html.")
out.failureMessage("Error while parsing text to html.")
return tools.exitErr()
def set_coefficients(self, free_coeffs=None, coeffs=None):
'''
This function is used to set up numerical values either for all the spline's coefficients
or its independent ones.
Parameters
----------
free_coeffs : numpy.ndarray
Array with numerical values for the free coefficients of the spline.
coeffs : numpy.ndarray
Array with coefficients of the polynomial spline parts.
'''
# deside what to do
if coeffs is None and free_coeffs is None:
# nothing to do
pass
elif coeffs is not None and free_coeffs is None:
# set all the coefficients for the spline's polynomial parts
#
# first a little check
if not (self.n == coeffs.shape[0]):
logging.error(
'Dimension mismatch in number of spline parts ({}) and \
rows in coefficients array ({})'.format(
self.n, coeffs.shape[0]))
raise ValueError(
'Dimension mismatch in number of spline parts ({}) and \
rows in coefficients array ({})'.format(
self.n, coeffs.shape[0]))
elif not (coeffs.shape[1] == 4):
logging.error(
'Dimension mismatch in number of polynomial coefficients (4) and \
columns in coefficients array ({})'.format(
coeffs.shape[1]))
# elif not (self._indep_coeffs.size == coeffs.shape[1]):
# logging.error('Dimension mismatch in number of free coefficients ({}) and \
# columns in coefficients array ({})'.format(self._indep_coeffs.size, coeffs.shape[1]))
# raise ValueError
# set coefficients
self._coeffs = coeffs
# update polynomial parts
for k in xrange(self.n):
self._P[k] = np.poly1d(self._coeffs[k])
elif coeffs is None and free_coeffs is not None:
# a little check
if not (self._indep_coeffs.size == free_coeffs.size):
logging.error('Got {} values for the {} independent coefficients.'\
.format(free_coeffs.size, self._indep_coeffs.size))
raise ValueError('Got {} values for the {} independent coefficients.'\
.format(free_coeffs.size, self._indep_coeffs.size))
# set the numerical values
self._indep_coeffs = free_coeffs
# update the spline coefficients and polynomial parts
for k in xrange(self.n):
coeffs_k = self._dep_array[k].dot(
free_coeffs) + self._dep_array_abs[k]
self._coeffs[k] = coeffs_k
self._P[k] = np.poly1d(coeffs_k)
else:
# not sure...
logging.error(
'Not sure what to do, please either pass `coeffs` or `free_coeffs`.'
)
raise TypeError(
'Not sure what to do, please either pass `coeffs` or `free_coeffs`.'
)
# now we have numerical values for the coefficients so we can set this to False
self._prov_flag = False
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
# exit preloader
tools.exit('exit', exit_status_code)
if __name__ == "__main__":
main()
def allowAccess(self):
response = self.loadPage(
self.url["base"],
self.url["access"],
"GET",
{"oauth_token": self.tmpOAuthToken},
)
logging.debug(response.data)
tree = html.fromstring(response.data)
token = (
"&"
+ urlencode(
{
"csrfBusterToken": tree.xpath(
"//input[@name='csrfBusterToken']/@value"
)[0]
}
)
+ "&"
+ urlencode(
{
"csrfBusterToken": tree.xpath(
"//input[@name='csrfBusterToken']/@value"
)[1]
}
)
)
sourcePage = tree.xpath("//input[@name='_sourcePage']/@value")[0]
fp = tree.xpath("//input[@name='__fp']/@value")[0]
targetUrl = tree.xpath("//input[@name='targetUrl']/@value")[0]
logging.debug(token)
if response.status != 200:
logging.error(
"Unexpected response status " "on login 200 != %s", response.status
)
tools.exitErr()
if "JSESSIONID" not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
access = self.postData["access"]
access["oauth_token"] = self.tmpOAuthToken
access["oauth_callback"] = ""
access["embed"] = "false"
access["suggestedNotebookName"] = "Geeknote"
access["supportLinkedSandbox"] = ""
access["analyticsLoginOrigin"] = "Other"
access["clipperFlow"] = "false"
access["showSwitchService"] = "true"
access["_sourcePage"] = sourcePage
access["__fp"] = fp
access["targetUrl"] = targetUrl
response = self.loadPage(
self.url["base"], self.url["access"], "POST", access, token
)
if response.status != 302:
logging.error(
"Unexpected response status on allowing " "access 302 != %s",
response.status,
)
logging.error(response.data)
tools.exitErr()
responseData = self.parseResponse(response.location)
if "oauth_verifier" not in responseData:
logging.error("OAuth verifier not found")
tools.exitErr()
self.verifierToken = responseData["oauth_verifier"]
logging.debug("OAuth verifier token take")
def export_basic_inf(data_list, workbook, is_exsit):
start_row = 1 # 数据起始excel行号
if not is_exsit:
worksheet = workbook.add_sheet(basic_inf_sheet_name,
cell_overwrite_ok=True)
worksheet.write(0, 0, "序号", style_title)
worksheet.write(0, 1, "公司名称", style_title)
worksheet.write(0, 2, "法定代表人", style_title)
worksheet.write(0, 3, "注册资本", style_title)
worksheet.write(0, 4, "实缴资本", style_title)
worksheet.write(0, 5, "经营状态", style_title)
worksheet.write(0, 6, "成立日期", style_title)
worksheet.write(0, 7, "统一社会信用代码", style_title)
worksheet.write(0, 8, "纳税人识别号", style_title)
worksheet.write(0, 9, "注册号", style_title)
worksheet.write(0, 10, "组织机构代码", style_title)
worksheet.write(0, 11, "企业类型", style_title)
worksheet.write(0, 12, "所属行业", style_title)
worksheet.write(0, 13, "人员规模", style_title)
worksheet.write(0, 14, "营业期限", style_title)
worksheet.write(0, 15, "企业地址", style_title)
worksheet.write(0, 16, "经营范围", style_title)
# worksheet.write(0, 3, "电话", style_title)
# worksheet.write(0, 4, "地址", style_title)
# worksheet.write(0, 5, "邮箱", style_title)
# worksheet.write(0, 6, "注册号", style_title)
# worksheet.write(0, 7, "统一社会信用代码", style_title)
# worksheet.write(0, 8, "注册资本", style_title)
# worksheet.write(0, 9, "成立日期", style_title)
# worksheet.write(0, 10, "企业类型", style_title)
# worksheet.write(0, 11, "经营范围", style_title)
# worksheet.write(0, 12, "公司住所", style_title)
# worksheet.write(0, 13, "营业期限", style_title)
# worksheet.write(0, 14, "企业状态", style_title)
# 设置表格列宽
for i in range(17):
if i == 0:
continue
elif i == 16:
worksheet.col(
i).width = 256 * 200 # 设置第二列宽度，256为衡量单位，50表示50个字符宽度
else:
worksheet.col(i).width = 256 * 20
else:
start_row = read_excel_rows(spider_result_file_name,
basic_inf_sheet_name)
worksheet = workbook.get_sheet(basic_inf_sheet_name)
# 定义excel表头列表
basic_info_title = [
'序号', '公司名称', '法定代表人/法人/负责人/经营者/投资人/执行事务合伙人', '注册资本/开办资金', '实缴资本',
'经营状态/登记状态', '成立日期', '统一社会信用代码', '纳税人识别号', '注册号', '组织机构代码',
'企业类型/社会组织类型', '所属行业', '人员规模', '营业期限/证书有效期/有效期', '企业地址/住所/地址',
'经营范围/业务范围'
]
for _response in data_list:
worksheet.write(start_row, 0, start_row, style)
# 公司名称
company_name = _response.find(class_="content")
if company_name is not None:
company_name = company_name.select('h1')[0].text.replace(
'\n', '').replace(
' ', '') if len(company_name.select('h1')) > 0 else '无'
print('公司名称：' + company_name)
worksheet.write(start_row, 1, company_name, style) # 将信息输入表格
basic_informatiion_array = _response.select(
"#Cominfo > table > tbody > tr") # 符号">"为上一个标签下的直接子标签
if len(basic_informatiion_array) == 0:
logging.error('未找到该公司基本信息============{}'.format(company_name))
for basic_tr in basic_informatiion_array: # 循环tr
basic_td_array = [
data for data in basic_tr.children if data != ' '
]
i = 0
while i < len(basic_td_array): # 循环td
key = basic_td_array[i].text.replace('\n', '').replace(' ', '')
if len(basic_td_array[i + 1].select('h2')) > 0:
value = basic_td_array[i + 1].select('h2')[0].text.replace(
'\n', '').replace(' ', '')
else:
value = basic_td_array[i + 1].text.replace(
'\n', '').replace(' ',
'').replace('查看地图',
'').replace('附近企业', '')
# 查出表格的标题在自定义的basic_info_title中的索引位置，为写入excel的列索引。
index = [k for k, x in enumerate(basic_info_title) if key in x]
if len(index) > 0:
print(key + '：' + value)
worksheet.write(start_row, index[0], value,
style) # 将信息输入表格
i += 2
start_row += 1
# if len(basic_informatiion_array) > 0:
# # 法人
# legal_person = '-'
# if len(basic_informatiion_array[0].select('td')) > 1:
# # legal_person_item = basic_informatiion_array[0].select('td')[0].text.replace('\n', '').replace(' ', '')
# # if len(basic_informatiion_array[0].select('td')[1].select('h2')) > 0 and ('法定代表人' in legal_person_item or '经营者' in legal_person_item):
# if len(basic_informatiion_array[0].select('td')[1].select('h2')) > 0:
# legal_person = basic_informatiion_array[0].select('td')[1].select('h2')[0].text.replace('\n', '').replace(' ', '')
# print('法人：' + legal_person)
# worksheet.write(start_row, 2, legal_person, style) # 将信息输入表格
# # 注册资本
# registration_capital = '-'
# if len(basic_informatiion_array[0].select('td')) > 3:
# registration_capital_item = basic_informatiion_array[0].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '注册资本' in registration_capital_item:
# registration_capital = basic_informatiion_array[0].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('注册资本：' + registration_capital)
# worksheet.write(start_row, 3, registration_capital, style) # 将信息输入表格
# # 实缴资本
# real_capital = '-'
# if len(basic_informatiion_array) > 1 and len(basic_informatiion_array[1].select('td')) > 1:
# real_capital_item = basic_informatiion_array[1].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '实缴资本' in real_capital_item:
# real_capital = basic_informatiion_array[1].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('实缴资本：' + real_capital)
# worksheet.write(start_row, 4, real_capital, style) # 将信息输入表格
# # 经营状态
# operating_status = '-'
# if len(basic_informatiion_array) > 2 and len(basic_informatiion_array[2].select('td')) > 1:
# operating_status_item = basic_informatiion_array[2].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '经营状态' in operating_status_item:
# operating_status = basic_informatiion_array[2].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('经营状态：' + operating_status)
# worksheet.write(start_row, 5, operating_status, style) # 将信息输入表格
# # 成立日期
# establishment_date = '-'
# if len(basic_informatiion_array) > 2 and len(basic_informatiion_array[2].select('td')) > 3:
# establishment_date_item = basic_informatiion_array[2].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '成立日期' in establishment_date_item:
# establishment_date = basic_informatiion_array[2].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('成立日期：' + establishment_date)
# worksheet.write(start_row, 6, establishment_date, style) # 将信息输入表格
# # 统一社会信用代码
# unified_credit_code = '-'
# if len(basic_informatiion_array) > 3 and len(basic_informatiion_array[3].select('td')) > 1:
# unified_credit_code_item = basic_informatiion_array[3].select('td')[0].text.replace('\n', '').replace(' ','')
# if '统一社会信用代码' in unified_credit_code_item:
# unified_credit_code = basic_informatiion_array[3].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('统一社会信用代码：' + unified_credit_code)
# worksheet.write(start_row, 7, unified_credit_code, style) # 将信息输入表格
# # 纳税人识别号
# taxpayer_identification = '-'
# if len(basic_informatiion_array) > 3 and len(basic_informatiion_array[3].select('td')) > 3:
# taxpayer_identification_item = basic_informatiion_array[3].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '纳税人识别号' in taxpayer_identification_item:
# taxpayer_identification = basic_informatiion_array[3].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('纳税人识别号：' + taxpayer_identification)
# worksheet.write(start_row, 8, taxpayer_identification, style) # 将信息输入表格
# # 注册号
# registration_number = '-'
# if len(basic_informatiion_array) > 4 and len(basic_informatiion_array[4].select('td')) > 1:
# registration_number_item = basic_informatiion_array[4].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '注册号' in registration_number_item:
# registration_number = basic_informatiion_array[4].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('注册号：' + registration_number)
# worksheet.write(start_row, 9, registration_number, style) # 将信息输入表格
# # 组织机构代码
# organization_code = '-'
# if len(basic_informatiion_array) > 4 and len(basic_informatiion_array[4].select('td')) > 3:
# organization_code_item = basic_informatiion_array[4].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '组织机构代码' in organization_code_item:
# organization_code = basic_informatiion_array[4].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('组织机构代码：' + organization_code)
# worksheet.write(start_row, 10, organization_code, style) # 将信息输入表格
# # 企业类型
# company_type = '-'
# if len(basic_informatiion_array) > 5 and len(basic_informatiion_array[5].select('td')) > 1:
# company_type_item = basic_informatiion_array[5].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '企业类型' in company_type_item:
# company_type = basic_informatiion_array[5].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('企业类型：' + company_type)
# worksheet.write(start_row, 11, company_type, style) # 将信息输入表格
# # 所属行业
# industry = '-'
# if len(basic_informatiion_array) > 5 and len(basic_informatiion_array[5].select('td')) > 3:
# industry_item = basic_informatiion_array[5].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '所属行业' in industry_item:
# industry = basic_informatiion_array[5].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('所属行业：' + industry)
# worksheet.write(start_row, 12, industry, style) # 将信息输入表格
# # 人员规模
# employees = '-'
# if len(basic_informatiion_array) > 9 and len(basic_informatiion_array[9].select('td')) > 1:
# employees_item = basic_informatiion_array[9].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '人员规模' in employees_item:
# employees = basic_informatiion_array[9].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('人员规模：' + employees)
# worksheet.write(start_row, 13, employees, style) # 将信息输入表格
# # 营业期限
# business_term = '-'
# if len(basic_informatiion_array) > 9 and len(basic_informatiion_array[9].select('td')) > 3:
# business_term_item = basic_informatiion_array[9].select('td')[2].text.replace('\n', '').replace(' ', '')
# if '营业期限' in business_term_item:
# business_term = basic_informatiion_array[9].select('td')[3].text.replace('\n', '').replace(' ', '')
# print('营业期限：' + business_term)
# worksheet.write(start_row, 14, business_term, style) # 将信息输入表格
# # 企业地址
# adress = '-'
# if len(basic_informatiion_array) > 10 and len(basic_informatiion_array[10].select('td')) > 1:
# adress_item = basic_informatiion_array[10].select('td')[0].text.replace('\n','').replace(' ', '')
# if '企业地址' in adress_item:
# adress = basic_informatiion_array[10].select('td')[1].text.replace('查看地图', '').replace('附近企业', '').replace('\n', '').replace(' ', '')
# print('企业地址：' + adress)
# worksheet.write(start_row, 15, adress, style) # 将信息输入表格
# # 经营范围
# business_scope = '-'
# if len(basic_informatiion_array) > 11 and len(basic_informatiion_array[11].select('td')) > 1:
# business_scope_item = basic_informatiion_array[11].select('td')[0].text.replace('\n', '').replace(' ', '')
# if '经营范围' in business_scope_item:
# business_scope = basic_informatiion_array[11].select('td')[1].text.replace('\n', '').replace(' ', '')
# print('经营范围：' + business_scope)
# worksheet.write(start_row, 16, business_scope, style) # 将信息输入表格
# start_row += 1
# 公司名称
# # company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
# if len(company_name) > 0:
# company_name = company_name.strip()
# else:
# company_name = '--'
# print('公司名称：' + company_name)
# worksheet.write(start_row, 1, company_name, style) # 将信息输入表格
# 法人
# legal_person = re.findall('<a class="oper" href=".*?">(.*?\s*)</a>', _response)
# if len(legal_person) > 0:
# legal_person = legal_person.strip()
# else:
# legal_person = '--'
# print('法人：' + legal_person)
# worksheet.write(start_row, 2, legal_person, style) # 将信息输入表格
# 电话
# telephone = re.findall('<a href="tel:.*?" class="phone a-decoration">(.*?\s*)</a>', _response)
# if len(telephone) > 0:
# telephone = telephone[0].strip()
# else:
# telephone = '--'
# print('电话：' + telephone)
# worksheet.write(start_row, 3, telephone, style) # 将信息输入表格
# 地址
# address = re.findall('</div> <div class="address">(\s*.*?\s*)</div> </div>', _response)
# if len(address) > 0:
# address = address[0].strip()
# else:
# address = '--'
# print('地址：' + address)
# worksheet.write(start_row, 4, address, style) # 将信息输入表格
# 邮箱
# email = re.findall('<a href="javascript:;" class="email a-decoration">(.*?\s*)</a>', _response)
# if len(email) > 0:
# email = email[0].strip()
# else:
# email = '--'
# print('邮箱：' + email)
# worksheet.write(start_row, 5, email, style) # 将信息输入表格
# basic_list = re.findall('</div> <div class="basic-item-right">(\s*.*?\s*)</div>', _response)
# if len(basic_list) > 0:
# # 注册号
# registration_number = basic_list[0].strip()
# else:
# registration_number = '--'
# print('注册号：' + registration_number)
# worksheet.write(start_row, 6, registration_number, style) # 将信息输入表格
# if len(basic_list) > 1:
# # 统一社会信用代码
# credit_ode = basic_list[1].strip()
# else:
# credit_ode = '--'
# print('统一社会信用代码：' + credit_ode)
# worksheet.write(start_row, 7, credit_ode, style) # 将信息输入表格
# 注册资本
# if len(basic_list) > 2:
# registration_capital = basic_list[2].strip()
# else:
# registration_capital = '--'
# print('注册资本：' + registration_capital)
# worksheet.write(start_row, 8, registration_capital, style) # 将信息输入表格
# if len(basic_list) > 3:
# # 成立日期
# registration_date = basic_list[3].strip()
# else:
# registration_date = '--'
# print('成立日期：' + registration_date)
# worksheet.write(start_row, 9, registration_date, style) # 将信息输入表格
# if len(basic_list) > 4:
# # 企业类型
# company_type = basic_list[4].strip()
# else:
# company_type = '--'
# print('企业类型：' + company_type)
# worksheet.write(start_row, 10, company_type, style) # 将信息输入表格
# if len(basic_list) > 5:
# # 经营范围
# business_scope = basic_list[5].strip()
# else:
# business_scope = '--'
# print('经营范围：' + business_scope)
# worksheet.write(start_row, 11, business_scope, style) # 将信息输入表格
# if len(basic_list) > 6:
# # 公司住所
# company_address = basic_list[6].strip()
# else:
# company_address = '--'
# print('公司住所：' + company_address)
# worksheet.write(start_row, 12, company_address, style) # 将信息输入表格
# if len(basic_list) > 7:
# # 营业期限
# business_term = basic_list[7].strip()
# else:
# business_term = '--'
# print('营业期限：' + business_term)
# worksheet.write(start_row, 13, business_term, style) # 将信息输入表格
# if len(basic_list) > 8:
# # 企业状态
# enterprise_state = basic_list[8].strip()
# else:
# enterprise_state = '--'
# print('企业状态：' + enterprise_state)
# worksheet.write(start_row, 14, enterprise_state, style) # 将信息输入表格
# start_row += 1
# print('----------------------------------------------------------------------')
return worksheet
def export_partners(data_list,workbook, is_exsit):
start_row = 1 # 数据起始excel行号
order_number = 0 # 数据起始序号
if not is_exsit:
worksheet = workbook.add_sheet(partners_sheet_name,cell_overwrite_ok=True)
worksheet.write(0, 0, u"序号", style_title)
worksheet.write(0, 1, u"公司名称", style_title)
worksheet.write(0, 2, u"股东名称", style_title)
worksheet.write(0, 3, u"持股比例", style_title)
worksheet.write(0, 4, u"认缴出资额", style_title)
worksheet.write(0, 5, u"认缴出资日期", style_title)
worksheet.write(0, 6, u"实缴出资额", style_title)
worksheet.write(0, 7, u"实缴出资日期", style_title)
# Setting row height and column width 设置宽和高 xlwt中是行和列都是从0开始计算的
worksheet.col(1).width = 256 * 50
worksheet.col(2).width = 256 * 50
worksheet.col(3).width = 256 * 20
worksheet.col(4).width = 256 * 30
worksheet.col(5).width = 256 * 30
worksheet.col(6).width = 256 * 30
worksheet.col(7).width = 256 * 30
else:
start_row = read_excel_rows(spider_result_file_name, partners_sheet_name)
worksheet = workbook.get_sheet(partners_sheet_name)
order_number = int(get_merged_cells_value(spider_result_file_name,partners_sheet_name, start_row-1, 0)) # 序号位置是第一列
# 定义excel表头列表
partner_title = ['序号', '公司名称', '股东名称', '持股比例', '认缴出资额', '认缴出资日期', '实缴出资额', '实缴出资日期']
for _response in data_list:
order_number += 1
# 公司名称
company_name = _response.find(class_="content")
if company_name is not None:
company_name = company_name.select('h1')[0].text.replace('\n', '').replace(' ', '') if len(company_name.select('h1')) > 0 else '无'
print('公司名称：' + company_name)
partner_array = _response.select("#partnerslist > table > tbody > tr")
if len(partner_array) == 0:
if len(_response.select("#partnerslist > div")) > 1 and len(_response.select("#partnerslist > div")[1].select('table')) > 0:
partner_array =_response.select("#partnerslist > div")[1].select('table')[0].select('tbody > tr')
if len(partner_array)-1 > 0:
worksheet.write_merge(start_row, start_row + len(partner_array)-2, 0, 0, order_number, style_merge) # 序号
worksheet.write_merge(start_row, start_row + len(partner_array)-2, 1, 1, company_name, style_merge) # 合并公司名称单元格
else:
worksheet.write(start_row, 0, order_number, style)
worksheet.write(start_row, 1, company_name, style)
worksheet.write(start_row, 2, '--', style)
worksheet.write(start_row, 3, '--', style)
worksheet.write(start_row, 4, '--', style)
worksheet.write(start_row, 5, '--', style)
worksheet.write(start_row, 6, '--', style)
worksheet.write(start_row, 7, '--', style)
start_row += 1
if len(partner_array) == 0:
logging.error('未找到该公司股东信息============{}'.format(company_name))
if len(partner_array) > 0:
partner_th = [data for data in partner_array[0].children if data != ' '][1:] # 股东表格的标题行th,去除序号th
partner_tbale_th = [th.text.replace('\n', '').replace(' ', '') for th in partner_th]
# print('表格标题行=========={}'.format(partner_tbale_th))
if len(partner_array) > 1:
for i in range(1, len(partner_array)):
partner_row = [data for data in partner_array[i].children if data != ' '][1:] # 数据行，排除序号列td
for j in range(len(partner_row)):
# 去除th标题中的单位，否则自定义的partner_title中查询不到。列如：认缴出资额(万元)--》认缴出资额。
title = partner_tbale_th[j].split('(')[0] if '(' in partner_tbale_th[j] else partner_tbale_th[j]
unit = (partner_tbale_th[j].split('(')[1] if '(' in partner_tbale_th[j] else '').replace(')', '')
# 查出股东表格的标题行在自定义的partner_title中的索引位置，为写入excel的列索引。
index = [k for k,x in enumerate(partner_title) if title in x]
if len(index) > 0:
v = partner_row[j].text.replace('\n', '').replace(' ', '').replace('持股详情>', '')
if '-' != v:
v += unit
print(partner_title[index[0]]+'：'+v)
worksheet.write(start_row, index[0], v, style) # 将信息输入表格
elif len(partner_row[j].select('h3')) > 0:
partner_name = partner_row[j].select('h3')[0].text.replace('\n', '').replace(' ', '')
print('股东名称：' + partner_name)
worksheet.write(start_row, 2, partner_name, style) # 将信息输入表格
start_row += 1
# #股东及出资信息
# partner_name = '-'
# if len(partner_array[i].select('td')) > 1 and len(partner_array[i].select('td')[1].select('h3')) > 0:
# partner_name = partner_array[i].select('td')[1].select('h3')[0].text.replace('\n', '').replace(' ', '')
# worksheet.write(start_row, 2, partner_name, style) # 将信息输入表格
# print('股东名称：' + partner_name)
#
# #持股比例
# stock_rate= partner_array[i].select('td')[4].text.replace('\n', '').replace(' ', '')
# if len(partner) == 10 and partner[4].text[0:5] == '持股比例':
# stock_rate = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '')
# if '%' in stock_rate:
# stock_rate = stock_rate.split('%')[0]+'%'
# worksheet.write(start_row, 3, stock_rate, style) # 将信息输入表格
# print('持股比例：' + stock_rate)
#
# # 认缴出资额(万元)
# if len(partner) == 5 and partner[3].text[0:5] =='认缴出资额':
# money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if money != '-':
# money =money+partner[3].text[5:]
# elif len(partner) == 6 and partner[3].text[0:5] =='认缴出资额':
# money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if money != '-':
# money =money+partner[3].text[5:]
# elif len(partner) == 7 and partner[3].text[0:5] =='认缴出资额':
# money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if money != '-':
# money = money + partner[4].text[5:]
# elif len(partner) == 8 and partner[4].text[0:5] =='认缴出资额':
# money = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if money != '-':
# money = money + partner[4].text[5:]
# elif len(partner) == 10 and partner[5].text[0:5] =='认缴出资额':
# money = partner_array[i].select('td')[9].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if money != '-':
# money = money + partner[5].text[5:]
# else:
# money = '--'
# worksheet.write(start_row, 4, money, style) # 将信息输入表格
# print('认缴出资额：' + money)
#
# #认缴出资日期
# if len(partner) == 5 and partner[3].text[0:6] =='认缴出资日期':
# time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# elif len(partner) == 6 and partner[4].text[0:6] =='认缴出资日期':
# time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# elif len(partner) == 7 and partner[3].text[0:6] =='认缴出资日期':
# time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# elif len(partner) == 8 and partner[3].text[0:6] =='认缴出资日期':
# time = partner_array[i].select('td')[7].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# elif len(partner) == 10 and partner[6].text[0:6] =='认缴出资日期':
# time = partner_array[i].select('td')[10].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# else:
# time = '--'
# worksheet.write(start_row, 5, time, style) # 将信息输入表格
# print('认缴出资日期：' + time)
#
# #实缴出资额(万元)
# if len(partner) == 7 and partner[5].text[0:5] =='实缴出资额':
# real_money = partner_array[i].select('td')[7].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if real_money != '-':
# real_money=real_money+partner[5].text[5:]
# elif len(partner) == 8 and partner[6].text[0:5] =='实缴出资额':
# real_money = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if real_money != '-':
# real_money=real_money+partner[6].text[5:]
# elif len(partner) == 10 and partner[7].text[0:5] =='实缴出资额':
# real_money = partner_array[i].select('td')[11].text.replace('\n', '').replace(' ', '').replace('<br>', '')
# if real_money != '-':
# real_money=real_money+partner[7].text[5:]
# else:
# real_money = '--'
# worksheet.write(start_row, 6, real_money, style) # 将信息输入表格
# print('实缴出资额：' + real_money)
#
# #实缴出资日期
# if len(partner) == 7 and partner[6].text =='实缴出资日期' :
# real_time = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '').replace('<br>','')
# elif len(partner) == 8 and partner[7].text =='实缴出资日期':
# real_time = partner_array[i].select('td')[9].text.replace('\n', '').replace(' ', '').replace('<br>','')
# elif len(partner) == 10 and partner[8].text =='实缴出资日期':
# real_time = partner_array[i].select('td')[12].text.replace('\n', '').replace(' ', '').replace('<br>','')
# else:
# real_time = '--'
# worksheet.write(start_row, 7, real_time, style) # 将信息输入表格
# print('实缴出日期：' + real_time)
# start_row += 1
# 公司名称
# company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
# if len(company_name) > 0:
# company_name = company_name[0].strip()
# else:
# company_name = '--'
# print('公司名称：' + company_name)
# print(_response)
# list = re.findall('<div class="stock-item">(.*?\s*\n*.*?\s*\n*.*?\s*\n*)</div>\n?\s*</div>\n?\s*</div>\n?\s*</div>', _response)
# for partner in list:
# print(partner)
# partner_list = re.findall('<div class="stock-text">(.*?\s*)<', partner)
# #股东名称
# partner_name = re.findall(' <div class="stock-title"> <span > <a class="text-blue a-decoration"( href=".*?")? >(.*?\s*)</a>', partner)
# if len(partner_name) > 0:
# partner_name = partner_name[0][1].strip()
# else:
# partner_name = '--'
# print('股东名称：' + partner_name)
# worksheet.write(start_row, 2, partner_name,style) # 将信息输入表格
# if len(partner_list) > 0:
# #持股比例
# stock_rate = partner_list[0].strip()
# else:
# stock_rate = '--'
# print('持股比例：' + stock_rate)
# worksheet.write(start_row, 3, stock_rate,style) # 将信息输入表格
# if len(partner_list) > 1:
# #股东类型
# stock_type = partner_list[1].strip()
# else:
# stock_type = '--'
# print('股东类型：' + stock_type)
# worksheet.write(start_row, 4, stock_type,style) # 将信息输入表格
# if len(partner_list) > 2:
# #认缴出资额(万元)
# money = partner_list[2].strip()
# else:
# money = '--'
# print('认缴出资额(万元)：' + money)
# worksheet.write(start_row, 5, money,style) # 将信息输入表格
# if len(partner_list) > 3:
# #认缴出资日期
# time = partner_list[3].strip()
# else:
# time = '--'
# print('认缴出资日期：' + time)
# worksheet.write(start_row, 6, time,style) # 将信息输入表格
# start_row += 1
# print('----------------------------------------------------------------------')
return worksheet
# response1 = retry_crawl(url, is_proxy)
# else:
# try:
# response1 = requests.get(url, headers=get_headers(), timeout=spider_timeout)
# except Exception as e:
# response1 = retry_crawl(url, is_proxy)
# _response1 = response1.text
# 此处使用selenium爬取页面，解决页面返回没有数据的问题
_response1 = selenium_crawl(url, is_proxy)
if _response1 is not None:
_soup = BeautifulSoup(_response1, 'lxml')
# print(_soup)
data_list.append(_soup)
print("{}=============抓取成功！".format(name))
except Exception as e:
logging.error(name + '=========================抓取该公司的信息异常')
error_data_list.append(name)
logging.error(e)
# print(str(e))
# logging.exception(e)
# 导出excel
if len(data_list) > 0 or len(error_data_list) > 0:
print(
'==================正在写入excel文件，请勿关闭程序！=================='
)
export_excel(data_list, error_data_list)
time.sleep(
random.randint(crawl_interval_mintime,
crawl_interval_maxtime)) # 每隔5到20秒
else:
print('====================本程序只能在外网环境下运行====================')
if result == 0:
return True
os.system('ps -ef | grep "./neo-cli" | awk \'{print $2}\' | xargs kill')
return True
def restartRecently():
if timedelta(minutes=START_SILENT) < datetime.now() - lastRestartTimestamp:
return True
return False
while True:
if not isLocalRunning():
startLocalNode()
continue
time.sleep(INTERVAL)
localBlockCount = getLocalBlockCount()
bestBlockCount = getBestBlockCount()
if localBlockCount < 0 or bestBlockCount < 0:
logging.error(
'[wrongheight] wrong height, localheight: {0}, bestheight: {1}'.
format(localBlockCount, bestBlockCount))
continue
if RESTART_THRESHOLD < bestBlockCount - localBlockCount and not restartRecently(
):
restart_cnt += 1
logging.warning(
'[restart] restarting, restart_cnt: {0}, localheight: {1}, bestheight: {2}'
.format(restart_cnt, localBlockCount, bestBlockOount))
stopLocalNode()
def export_excel(data, error_data):
if check_file(spider_result_file_name):
# 读取文件的时候需要将formatting_info参数设置为True，默认是False；保留样式，不然获取合并的单元格数组为空
old_workbook = xlrd.open_workbook(spider_result_file_name,
formatting_info=True)
workbook = copy(old_workbook) # 拷贝一份原来的excel
else:
# 创建输出excel文件
workbook = xlwt.Workbook(encoding="utf-8")
# 导出企业基本信息
if len(data) > 0:
print(
"==============================企业基本信息=============================="
)
try:
export_basic_inf(
data, workbook,
check_sheet_exsit(spider_result_file_name,
basic_inf_sheet_name))
except Exception as e:
logging.error('===================写入企业基本信息异常===================')
# logging.exception(e)
logging.error(e)
# 导出企业股东信息
print(
"==============================企业股东信息=============================="
)
try:
export_partners(
data, workbook,
check_sheet_exsit(spider_result_file_name,
partners_sheet_name))
except Exception as e:
logging.error('===================写入企业股东信息异常===================')
# logging.exception(e)
logging.error(e)
# 导出企业主要人员s
print(
"==============================企业主要人员=============================="
)
try:
export_key_personnel(
data, workbook,
check_sheet_exsit(spider_result_file_name,
key_personnel_sheet_name))
except Exception as e:
logging.error('===================写入企业主要人员异常===================')
# logging.exception(e)
logging.error(e)
# 导出抓取失败企业名称
if len(error_data) > 0:
print(
"==============================写入抓取失败企业=============================="
)
try:
export_error_data(
error_data, workbook,
check_sheet_exsit(spider_result_file_name,
error_data_sheet_name))
except Exception as e:
logging.error('===================写入抓取失败企业异常===================')
logging.error(e)
# logging.exception(e)
workbook.save(spider_result_file_name)
def parse(self):
self.INP_DATA = {}
if self.CMD is None:
out.printAbout()
return False
if self.CMD == "autocomplete":
# substitute arguments for AutoComplete
# 1 offset to make the argument as 1 is autocomplete
self.__init__(self.sys_argv[1:])
self.printAutocomplete()
return False
if self.CMD == "--help":
self.printHelp()
return False
if self.CMD not in self.COMMANDS:
self.printErrorCommand()
return False
if "--help" in self.INP:
self.printHelp()
return False
# prepare data
for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
# set values by default
if 'default' in params:
self.INP_DATA[arg] = params['default']
# replace `altName` entered arguments on full
if 'altName' in params and params['altName'] in self.INP:
self.INP[self.INP.index(params['altName'])] = arg
activeArg = None
ACTIVE_CMD = None
# check and insert first argument by default
if 'firstArg' in self.COMMANDS[self.CMD]:
firstArg = self.COMMANDS[self.CMD]['firstArg']
if len(self.INP) > 0:
# Check that first argument is a default argument
# and another argument.
if self.INP[0] not in (self.CMD_ARGS.keys() +
self.CMD_FLAGS.keys()):
self.INP = [
firstArg,
] + self.INP
else:
self.INP = [
firstArg,
]
for item in self.INP:
# check what are waiting the argument
if activeArg is None:
# actions for the argument
if item in self.CMD_ARGS:
activeArg = item
ACTIVE_CMD = self.CMD_ARGS[activeArg]
# actions for the flag
elif item in self.CMD_FLAGS:
self.INP_DATA[item] = self.CMD_FLAGS[item]["value"]
# error. parameter is not found
else:
self.printErrorArgument(item)
return False
else:
activeArgTmp = None
# values it is parameter
if item in self.CMD_ARGS or item in self.CMD_FLAGS:
# active argument is "emptyValue"
if "emptyValue" in ACTIVE_CMD:
activeArgTmp = item # remember the new "active" argument
item = ACTIVE_CMD[
'emptyValue'] # set the active atgument to emptyValue
# Error, "active" argument has no values
else:
self.printErrorArgument(activeArg, item)
return False
if 'type' in ACTIVE_CMD:
convType = ACTIVE_CMD['type']
if convType not in (int, str):
logging.error("Unsupported argument type: %s",
convType)
return False
try:
item = convType(item)
except:
self.printErrorArgument(activeArg, item)
return False
self.INP_DATA[activeArg] = item
activeArg = activeArgTmp # this is either a new "active" argument or emptyValue.
# if there are still active arguments
if activeArg is not None:
# if the active argument is emptyValue
if 'emptyValue' in ACTIVE_CMD:
self.INP_DATA[activeArg] = ACTIVE_CMD['emptyValue']
# An error argument
else:
self.printErrorArgument(activeArg, "")
return False
# check whether there is a necessary argument request
for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
if 'required' in params and arg not in self.INP:
self.printErrorReqArgument(arg)
return False
# trim -- and ->_
self.INP_DATA = dict([key.lstrip("-").replace("-", "_"), val]
for key, val in self.INP_DATA.items())
return self.INP_DATA
def export_key_personnel(data_list, workbook, is_exsit):
start_row = 1 # 数据起始excel行号
order_number = 0 # 数据起始序号
if not is_exsit:
worksheet = workbook.add_sheet(key_personnel_sheet_name,
cell_overwrite_ok=True)
worksheet.write(0, 0, "序号", style_title)
worksheet.write(0, 1, "公司名称", style_title)
worksheet.write(0, 2, "姓名", style_title)
worksheet.write(0, 3, "职务", style_title)
# 设置表格列宽
for i in range(4):
if i == 0:
continue
else:
worksheet.col(i).width = 256 * 20
else:
start_row = read_excel_rows(spider_result_file_name,
key_personnel_sheet_name)
worksheet = workbook.get_sheet(key_personnel_sheet_name)
order_number = int(
get_merged_cells_value(spider_result_file_name,
key_personnel_sheet_name, start_row - 1,
0)) # 序号位置是第一列
for _response in data_list:
order_number += 1
# 公司名称
company_name = _response.find(class_="content")
if company_name is not None:
company_name = company_name.select('h1')[0].text.replace(
'\n', '').replace(
' ', '') if len(company_name.select('h1')) > 0 else '无'
print('公司名称：' + company_name)
personnel_array = _response.select(
"#Mainmember > table > tbody > tr") # 包含了标题tr
# print(personnel_array)
if len(personnel_array) - 1 > 0:
worksheet.write_merge(start_row,
start_row + len(personnel_array) - 2, 0, 0,
order_number, style_merge) # 合并序号单元格
worksheet.write_merge(start_row,
start_row + len(personnel_array) - 2, 1, 1,
company_name, style_merge) # 合并公司名称单元格
else:
worksheet.write(start_row, 0, order_number, style)
worksheet.write(start_row, 1, company_name, style)
worksheet.write(start_row, 2, '--', style)
worksheet.write(start_row, 3, '--', style)
start_row += 1
if len(personnel_array) == 0:
logging.error('未找到该公司主要人员============{}'.format(company_name))
if len(personnel_array) > 1:
for i in range(1, len(personnel_array)):
# 姓名
if len(personnel_array[i].select('td')) > 1 and len(
personnel_array[i].select('td')[1].select('h3')) > 0:
name = personnel_array[i].select('td')[1].select(
'h3')[0].text.replace('\n', '').replace(' ', '')
print('姓名：' + name)
worksheet.write(start_row, 2, name, style) # 将信息输入表格
# 职务
if len(personnel_array[i].select('td')) > 2:
job = personnel_array[i].select('td')[2].text.replace(
'\n', '').replace(' ', '')
print('职务：' + job)
worksheet.write(start_row, 3, job, style) # 将信息输入表格
start_row += 1
# order_number += 1
# 公司名称
# company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
# if len(company_name) > 0:
# company_name = company_name[0].strip()
# else:
# company_name = '--'
# print('公司名称：' + company_name)
# personnel_list = re.findall('<a class="employee-item" href=".*?" style="display: block;">(\s*.*?\s*)</a>', _response)
# # print(personnel_list)
# if len(personnel_list) > 0:
# worksheet.write_merge(start_row, start_row + len(personnel_list)-1, 0, 0, order_number, style_merge) # 合并序号单元格
# worksheet.write_merge(start_row, start_row + len(personnel_list)-1, 1, 1, company_name, style_merge) # 合并公司名称单元格
# else:
# worksheet.write(start_row, 0, order_number, style)
# worksheet.write(start_row, 1, company_name, style)
# worksheet.write(start_row, 2, '--', style)
# worksheet.write(start_row, 3, '--', style)
# start_row += 1
#
# for personnel in personnel_list:
# # 姓名
# name = re.findall('<div class="employee-name">(\s*.*?\s*)</div>', personnel)
# if len(name) > 0:
# name = name[0].strip()
# else:
# name = '--'
# print('姓名：' + name)
# worksheet.write(start_row, 2, name, style) # 将信息输入表格
# # 职务
# job = re.findall('<div class="employee-job">(\s*.*?\s*)</div>', personnel)
# if len(job) > 0:
# job = job[0].strip()
# else:
# job = '--'
# print('职务：' + job)
# worksheet.write(start_row, 3, job, style) # 将信息输入表格
# start_row += 1
# print('----------------------------------------------------------------------')
return worksheet
def genDailyReportSimplification(startTs, endTs):
br = '<br/>'
responseData = initResponse()
result = {}
result['status'] = ''
responseStr = ''
# list of data collector
listOfCollector = ['A01', 'A03', 'A05', 'A06']
validStartTime = transferDatetime(startTs)
validEndTime = transferDatetime(endTs)
responseStr = responseStr + "[數據採集回報 " + validStartTime.strftime("%Y/%m/%d %H:%M:%S") \
+ " - " + validEndTime.strftime("%Y/%m/%d %H:%M:%S") + "]" + br
try:
for machineId in listOfCollector:
coreName = 'spc'
spcRows = getTotalRowsBetweenTwoTimestamp(validStartTime,
validEndTime, 10, 1)
response = queryInfoBetweenTimestampSPC0(coreName, machineId,
validStartTime,
validEndTime, spcRows)
doc = praseJsonFormat(response)
df = pd.DataFrame(doc)
responseStr = responseStr + machineId + br
# query燈號狀態
lightResponse = queryLightStatus(machineId)
machineStatus = parseLightStatus(lightResponse)
if df.empty:
logging.info(coreName + "/" + machineId + ":" + startTs + '-' +
validEndTime.strftime("%Y/%m/%d %H:%M:%S") +
' DataFrame is empty!')
responseStr = responseStr + insertTextIndent(
'目前機台狀態：' + machineStatus + '，' + '此區間沒有生產數據', '20', '1')
else:
df.loc[:, 'SPC_0_previous'] = df['SPC_0']
# shift 是往下移一個
df['SPC_0_previous'] = df['SPC_0_previous'].shift(1).fillna(
'-1').astype(int)
df.loc[:, 'timestamp_previous'] = df['timestamp'].shift(
1).fillna(-1).astype(float)
# for顯示使用
df.loc[:, 'timestamp_view_temp'] = df[
'timestamp_iso'].str.replace('T', ' ').str.replace(
'Z', '').astype(str)
df.loc[:, 'timestamp_view'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%H:%M:%S')
# last
df.loc[:, 'timestamp_view_last'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%m/%d %H:%M:%S')
# date
df.loc[:, 'timestamp_view_date'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%m/%d')
df.loc[:, 'timestamp_view_date_previous'] = df[
'timestamp_view_date'].shift(1).fillna('-1')
df.loc[:,
'timestamp_view_previous'] = df['timestamp_view'].shift(
1).fillna('-1')
# 最新模次
df_lastest = df.iloc[-1:].astype(str)
lastTime = df_lastest['timestamp'].values[0]
lastTimeStr = df_lastest['timestamp_view_last'].values[0]
lastSPC = df_lastest['SPC_0'].values[
0] + '(' + lastTimeStr + ')'
responseStr = responseStr + insertTextIndent(
'目前機台狀態：' + machineStatus, '20', '1')
responseStr = responseStr + insertTextIndent(
lastSPC, '50', '1')
# 生產模次間隔超過五分鐘
df['timestamp_previous_addInterval'] = df[
'timestamp_previous'] + int(PRODUCTION_DURING_SEC)
df_broken_timestamp = df[df['timestamp'].astype(float) >
df['timestamp_previous_addInterval']]
if df_broken_timestamp.size > 1:
filter_df = df_broken_timestamp.iloc[1:]
times_df = filter_df.groupby([
"timestamp_view_date_previous"
]).size().to_frame(name='count')
if times_df.size > 0:
responseStr = responseStr + insertTextIndent(
'生產模次間隔超過五分鐘：', '20', '1')
for index, row in times_df.iterrows():
responseStr = responseStr + insertTextIndent(
index + ' 發生 ' + str(row['count']) + ' 次',
'50', '1')
responseStr = responseStr + br
return responseStr
except:
logging.error("genDailyReport exception")
responseData = setErrorResponse(responseData, '400', 'exception')
return parseResponseJson(responseData, result, 'result')
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
# content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
content = re.sub(r'\r\n', '\n', content)
if format == 'markdown':
storage = Storage()
extras = storage.getUserprop('markdown2_extras')
content = Editor.HTMLEscapeTag(content)
contentHTML = markdown.markdown(content, extras=extras)
soup = BeautifulSoup(contentHTML, 'html.parser')
Editor.checklistInSoupToENML(soup)
# Non-Pretty HTML output
contentHTML = str(soup)
elif format == 'pre':
#
# For the 'pre' format, simply wrap the content with a 'pre' tag. Do
# perform any parsing/mutation.
#
contentHTML = u''.join(
('<pre>', content, '</pre>')).encode("utf-8")
elif format == 'html':
# Html to ENML http://dev.evernote.com/doc/articles/enml.php
ATTR_2_REMOVE = [
"id",
"class",
# "on*",
"accesskey",
"data",
"dynsrc",
"tabindex"
]
soup = BeautifulSoup(content, 'html.parser')
for tag in soup.findAll():
if hasattr(tag, 'attrs'):
map(lambda x: tag.attrs.pop(x, None), [
k for k in tag.attrs.keys()
if k in ATTR_2_REMOVE or k.find('on') == 0
])
contentHTML = str(soup)
else:
contentHTML = Editor.HTMLEscape(content)
tmpstr = ''
for l in contentHTML.split('\n'):
if l == '':
tmpstr = tmpstr + u'<div><br/></div>'
else:
tmpstr = tmpstr + u'<div>' + l + u'</div>'
contentHTML = tmpstr.encode("utf-8")
contentHTML = contentHTML.replace(
'[x]', '<en-todo checked="true"></en-todo>')
contentHTML = contentHTML.replace('[ ]', '<en-todo></en-todo>')
return Editor.wrapENML(contentHTML)
except:
import traceback
traceback.print_exc()
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exitErr()
def queryLightStatus(data_collector, lightVersion, isInsert):
responseData = initResponse()
result = {}
result['status'] = ''
pingStatus = ''
connectionStatus = ''
machineStatus = ''
# 紀錄lightstatus到solr
lightstatusData = {}
# dashboard light color
if lightVersion == 'dashboard':
pingFail = "gray"
iconDisconnected = "icon_disconnected"
iconIdle = "icon_idle"
auto = "green"
halfAutoOrManual = "yellow"
# oee light color
else:
pingFail = "red"
iconDisconnected = "gray"
iconIdle = "yellow"
auto = "green"
halfAutoOrManual = "green"
try:
lightstatusData['machine_id'] = data_collector
nowDatetime = datetime.now()
nowTimestamp = int(nowDatetime.timestamp())
lightstatusData['timestamp'] = nowTimestamp
lightstatusData['id'] = data_collector + str(nowTimestamp)
lightstatusData['timestamp_iso'] = nowDatetime.strftime(
"%Y-%m-%dT%H:%M:%SZ")
#list of data collector
listOfCollector = ['A01', 'A03', 'A05', 'A06']
if data_collector not in listOfCollector:
result['lightColor'] = 'none'
result['status'] = "no match machine id"
responseData = setErrorResponse(responseData, '404',
'collector machine_id not found')
return parseResponseJson(responseData, result, 'result')
#先判斷是否ping的到
pingStatus = checkPing(data_collector)
if pingStatus == "False":
result['lightColor'] = pingFail
result['status'] = result['status'] + "ping=fail; 請檢查控制器或網路"
lightstatusData['light_color'] = pingFail
if isInsert == True:
addLightStatusData('lightstatus', data_collector,
[lightstatusData])
return parseResponseJson(responseData, result, 'result')
else:
result['status'] = result['status'] + "ping=success; "
#再判斷是否連結的到collector
connectionStatus = checkConnect(data_collector)
if connectionStatus == "False":
result['lightColor'] = iconDisconnected
result['status'] = result['status'] + "connection=fail; 請檢查控制器"
lightstatusData['light_color'] = iconDisconnected
if isInsert == True:
addLightStatusData('lightstatus', data_collector,
[lightstatusData])
return parseResponseJson(responseData, result, 'result')
else:
result['status'] = result['status'] + "connection=success; "
# query machine status
jsonObject = {}
jsonObject = queryMachinestate('machinestatus')
data = praseJsonFormat(jsonObject)
if len(data) > 0:
machineStatus = data[0]['MachineState']
result['status'] = result[
'status'] + "machineStatus=" + machineStatus + "; "
else:
result['status'] = result['status'] + "machineStatus=none(no data)"
if machineStatus == "全自動":
# 全自動才判斷生產狀態
# query spc status
jsonObject = {}
jsonObject = queryLastSPC("spc", data_collector)
data = praseJsonFormat(jsonObject)
if len(data) > 0:
spc_0 = data[0]['SPC_0']
result['status'] = result['status'] + "spc_0=" + str(
spc_0) + "; "
timestamp_iso = data[0]['timestamp_iso']
timestamp = data[0]['timestamp']
result['status'] = result[
'status'] + "timestamp=" + timestamp_iso.replace(
'T', ' ').replace('Z', '')
now = datetime.now().timestamp()
if float(timestamp
) + PRODUCTION_DURING_SEC < now: # 超過5分鐘沒有生產 -> 待機中
result['lightColor'] = iconIdle
lightstatusData['light_color'] = iconIdle
else:
result['lightColor'] = auto
lightstatusData['light_color'] = auto
else:
result['status'] = result['status'] + "spc_0=none(no data)"
else: #手動/半自動
result['lightColor'] = halfAutoOrManual
lightstatusData['light_color'] = halfAutoOrManual
if isInsert == True:
addLightStatusData('lightstatus', data_collector,
[lightstatusData])
if lightVersion == 'oee':
# add machine status for TPM APP UI
if result['lightColor'] == auto or result[
'lightColor'] == halfAutoOrManual:
result['lightStatus'] = 'run'
elif result['lightColor'] == pingFail:
result['lightStatus'] = 'down'
elif result['lightColor'] == iconDisconnected:
result['lightStatus'] = 'issued'
elif result['lightColor'] == iconIdle:
result['lightStatus'] = 'idle'
return parseResponseJson(responseData, result, 'result')
except:
logging.error("queryLightStatus exception")
setErrorResponse(responseData, '400', 'exception')
return parseResponseJson(responseData, result, 'result')
def write_into_influxdb(host_dict):
"""
Wirte data into influxdb.
:param host_dict (dict) - as blew
{
'name': 'instance-00000163',
'uuid': 'dbbc272d-60b0-4380-9d23-4c011896e7f4',
'region': 'wuxi-test-compute144',
'client': influxdb_client
}
"""
host_name = host_dict['name']
host_uuid = host_dict['uuid']
region = host_dict['region']
client = host_dict['client']
db_list = []
logging.debug("begin to collect %s data." % host_name)
begin_time = time.time()
try:
prefix_cmd = "virsh qemu-agent-command " + host_name
#query cpu stat
(status, result) = commands.getstatusoutput(
prefix_cmd +
" '{\"execute\":\"guest-query-cpu-usage\", \"arguments\":{\"delay\":1}}'"
)
if status == 0:
result = json.loads(result)
record = get_influxdb_record(host_uuid, region, "cpu", "usage",
"total", result['return']['usage'])
db_list.append(record)
#query mem stat
(status, result) = commands.getstatusoutput(
prefix_cmd + " '{\"execute\":\"guest-query-mem-usage\"}'")
if status == 0:
result = json.loads(result)
record = get_influxdb_record(host_uuid, region, "mem", "usage",
"total", result['return']['usage'])
db_list.append(record)
#query disk stat
(status, result) = commands.getstatusoutput(
prefix_cmd +
" '{\"execute\":\"guest-query-disk-stat\", \"arguments\":{\"delay\":1}}'"
)
if status == 0:
result = json.loads(result)
result = result['return']['disk_stat']
for item in result:
record = get_influxdb_record(host_uuid, region, "disk",
'rd_ops', item['name'],
item['rd_ops'])
db_list.append(record)
record = get_influxdb_record(host_uuid, region, "disk",
'wr_ops', item['name'],
item['wr_ops'])
db_list.append(record)
record = get_influxdb_record(host_uuid, region, "disk",
'rd_octet', item['name'],
item['rd_octet'])
db_list.append(record)
record = get_influxdb_record(host_uuid, region, "disk",
'wr_octet', item['name'],
item['wr_octet'])
db_list.append(record)
#query network stat
(status, result) = commands.getstatusoutput(
prefix_cmd +
" '{\"execute\":\"guest-query-net-stat\", \"arguments\":{\"delay\":1}}'"
)
if status == 0:
result = json.loads(result)
result = result['return']['net_stat']
for item in result:
record = get_influxdb_record(host_uuid, region, "network",
'receive', item['name'],
item['receive'])
db_list.append(record)
record = get_influxdb_record(host_uuid, region, "network",
'send', item['name'],
item['send'])
db_list.append(record)
#write data to influxdb
client.write(db_list)
except Exception:
logging.error("failed to collect %s data." % host_name)
logging.error(traceback.print_exc())
end_time = time.time()
spend_time = int(end_time - begin_time)
logging.debug("finished to collect %s data.took time: %d" %
(host_name, spend_time))
def genDailyReport(startTs, endTs):
br = '<br/>'
responseData = initResponse()
result = {}
result['status'] = ''
responseStr = ''
# list of data collector
listOfCollector = ['A01', 'A03', 'A05', 'A06']
validStartTime = transferDatetime(startTs)
validEndTime = transferDatetime(endTs)
responseStr = responseStr + "[數據採集回報 " + validStartTime.strftime("%Y/%m/%d %H:%M:%S")\
+ " - " + validEndTime.strftime("%Y/%m/%d %H:%M:%S") + "]" + br
try:
for machineId in listOfCollector:
coreName = 'spc'
spcRows = getTotalRowsBetweenTwoTimestamp(validStartTime,
validEndTime, 10, 1)
response = queryInfoBetweenTimestampSPC0(coreName, machineId,
validStartTime,
validEndTime, spcRows)
doc = praseJsonFormat(response)
df = pd.DataFrame(doc)
responseStr = responseStr + machineId + br
#query燈號狀態
lightResponse = queryLightStatus(machineId)
machineStatus = parseLightStatus(lightResponse)
if df.empty:
logging.info(coreName + "/" + machineId + ":" + startTs + '-' +
validEndTime.strftime("%Y/%m/%d %H:%M:%S") +
' DataFrame is empty!')
responseStr = responseStr + insertTextIndent(
'最新模次：' + machineStatus + '，' + '此區間沒有生產數據', '20', '1')
else:
df.loc[:, 'SPC_0_previous'] = df['SPC_0']
# shift 是往下移一個
df['SPC_0_previous'] = df['SPC_0_previous'].shift(1).fillna(
'-1').astype(int)
df.loc[:, 'timestamp_previous'] = df['timestamp'].shift(
1).fillna(-1).astype(float)
#for顯示使用
df.loc[:, 'timestamp_view_temp'] = df[
'timestamp_iso'].str.replace('T', ' ').str.replace(
'Z', '').astype(str)
df.loc[:, 'timestamp_view'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%H:%M:%S')
#last
df.loc[:, 'timestamp_view_last'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%Y/%m/%d %H:%M:%S')
#date
df.loc[:, 'timestamp_view_date'] = pd.to_datetime(
df['timestamp_view_temp'],
errors='coerce').dt.strftime('%Y/%m/%d')
df.loc[:, 'timestamp_view_date_previous'] = df[
'timestamp_view_date'].shift(1).fillna('-1')
df.loc[:,
'timestamp_view_previous'] = df['timestamp_view'].shift(
1).fillna('-1')
# 最新模次
df_lastest = df.iloc[-1:].astype(str)
lastTime = df_lastest['timestamp'].values[0]
lastTimeStr = df_lastest['timestamp_view_last'].values[0]
lastSPC = lastTimeStr + '(' + df_lastest['SPC_0'].values[
0] + ')'
responseStr = responseStr + insertTextIndent(
'最新模次：' + machineStatus + '，' + lastSPC, '20', '1')
#連線訊息
dcCoreName = 'dc_event'
dcRows = getTotalRowsBetweenTwoTimestamp(
validStartTime, validEndTime, 3, 4)
validStartTimeStr = convertDatetimeToString(
validStartTime, "%Y-%m-%dT%H:%M:%SZ")
validEndTimeStr = convertDatetimeToString(
validEndTime, "%Y-%m-%dT%H:%M:%SZ")
dcResponse = queryDCLogBetweenTimestamp(
dcCoreName, machineId, validStartTimeStr, validEndTimeStr,
dcRows)
dcDoc = praseJsonFormat(dcResponse)
docDF = pd.DataFrame(dcDoc)
if docDF.empty:
logging.info(dcCoreName + "/" + machineId + ":" + startTs +
'-' + endTs +
' connection dataframe is empty!')
else:
docFalseDF = docDF.where(docDF['result'] == False).dropna(
).drop_duplicates('timestamp').copy()
# for顯示使用
docFalseDF.loc[:, 'timestamp_view_temp'] = docFalseDF[
'timestamp'].str.replace('T', ' ').str.replace(
'Z', '').astype(str)
docFalseDF.loc[:, 'timestamp_view_month'] = pd.to_datetime(
docFalseDF['timestamp_view_temp'],
errors='coerce').dt.strftime('%Y/%m/%d %H:%M:%S')
if docFalseDF['result'].count() == 0:
logging.info('未發生斷線問題')
else: # 發生斷線問題
docFirstDate = ''
lastIndex = 0
# 刪除下一筆連續中斷的連線
for index, row in docFalseDF.iterrows():
if lastIndex + 1 == index:
docFalseDF = docFalseDF.drop(index)
lastIndex = index
# 找出下一筆恢復連線
# shift 是往上移一個
docDF.loc[:, 'result_next'] = docDF['result'].shift(
-1).astype(bool).copy()
docSameDF = docDF[(docDF['result_next'] !=
docDF['result'])]
# 紀錄下一個falseIndex
dict = {}
lastFalseIndex = 0
for index, row in docSameDF.iterrows():
if row['result_next'] == False:
lastFalseIndex = index + 1
elif lastFalseIndex != 0 | (
lastFalseIndex == 0 & row['result'] ==
False): #第一筆是斷線的時候, 加入第一筆斷線的訊息到dict
dict[lastFalseIndex] = index + 1
for index, row in docFalseDF.iterrows():
responseStr = responseStr + insertTextIndent(
'中斷連線：' + row['timestamp_view_month'], '20',
'0')
isFalseExist = index in dict
if isFalseExist == True:
if index != dict[index]:
nextIndex = dict[index]
docRow = docDF.ix[[nextIndex]].copy()
docRowResult = docRow['result'].to_string(
index=False)
docRow.loc[:,
'timestamp_view_temp'] = docRow[
'timestamp'].str.replace(
'T', ' ').str.replace(
'Z', '').astype(str)
docRow.loc[:,
'timestamp_view_month'] = pd.to_datetime(
docRow[
'timestamp_view_temp'],
errors='coerce'
).dt.strftime(
'%Y/%m/%d %H:%M:%S')
docRowMonth = docRow[
'timestamp_view_month'].to_string(
index=False)
responseStr = responseStr + insertTextIndent(
'恢復連線：' + docRowMonth, '20', '0')
# 模次不連續
df.loc[:, 'SPC_0_previous_add1'] = df['SPC_0_previous'] + 1
df_broken_spc_0 = df[(df['SPC_0_previous_add1'] !=
df['SPC_0'])]
# 多加一個條件, 排除停機後, spc_0跟上一筆相同的中斷模次
df_broken_spc_0 = df_broken_spc_0[
(df_broken_spc_0['SPC_0'] !=
df_broken_spc_0['SPC_0_previous']) |
((df_broken_spc_0['SPC_0'] ==
df_broken_spc_0['SPC_0_previous']) &
(df_broken_spc_0['SPC_0'] != df_broken_spc_0['RD_618']))]
responseStr = parseViewformat(df_broken_spc_0, responseStr,
'模次不連續')
# 模次時間超過五分鐘
df['timestamp_previous_addInterval'] = df[
'timestamp_previous'] + int(PRODUCTION_DURING_SEC)
df_broken_timestamp = df[df['timestamp'].astype(float) >
df['timestamp_previous_addInterval']]
responseStr = parseViewformat(df_broken_timestamp, responseStr,
'模次生產時間超過5分鐘')
responseStr = responseStr + br
return responseStr
except:
logging.error("genDailyReport exception")
responseData = setErrorResponse(responseData, '400', 'exception')
return parseResponseJson(responseData, result, 'result')
def main(args=None):
try:
# if terminal
if config.IS_IN_TERMINAL:
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# if input stream
else:
COMMAND, ARGS = modifyArgsByStdinStream()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
def loadPage(self,
url,
uri=None,
method="GET",
params="",
additionalParams=""):
if not url:
logging.error("Request URL undefined")
tools.exitErr()
if not url.startswith("http"):
url = "https://" + url
urlData = urlparse(url)
if not uri:
url = "%s://%s" (urlData.scheme, urlData.netloc)
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params:
params = urlencode(params) + additionalParams
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie":
'; '.join(
[key + '=' + self.cookies[key] for key in self.cookies.keys()])
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
if self._proxy is None or proxy_bypass(urlData.hostname):
host = urlData.hostname
port = urlData.port
real_host = real_port = None
else:
host = self._proxy.hostname
port = self._proxy.port
real_host = urlData.hostname
real_port = urlData.port
logging.debug("Request URL: %s:/%s > %s # %s", url, uri,
unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(host, port)
if real_host is not None:
conn.set_tunnel(real_host, real_port, headers=self._proxy_auth)
if config.DEBUG:
conn.set_debuglevel(1)
conn.request(method, url + uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
logging.debug("Response : %s > %s", response.status,
response.getheaders())
result = tools.Struct(status=response.status,
location=response.getheader('location', None),
data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
# delete cookies whose content is "deleteme"
for key in self.cookies.keys():
if self.cookies[key] == "deleteme":
del self.cookies[key]
return result
def unconstrain(self, constraints):
'''
This method is used to enable compliance with desired box constraints given by the user.
It transforms the vectorfield by projecting the constrained state variables on
new unconstrained ones.
Parameters
----------
constraints : dict
The box constraints for the state variables
'''
# save constraints
self.constraints = constraints
# backup the original constrained system
self._dyn_sys_orig = copy.deepcopy(self.dyn_sys)
# get symbolic vectorfield
# (as sympy matrix toenable replacement method)
x = sp.symbols(self.dyn_sys.states)
u = sp.symbols(self.dyn_sys.inputs)
ff_mat = sp.Matrix(self.dyn_sys.f_sym(x, u))
# get neccessary information form the dynamical system
a = self.dyn_sys.a
b = self.dyn_sys.b
boundary_values = self.dyn_sys.boundary_values
# handle the constraints by projecting the constrained state variables
# on new unconstrained variables using saturation functions
for k, v in self.constraints.items():
# check if boundary values are within saturation limits
xk = self.dyn_sys.states[k]
xa, xb = self.dyn_sys.boundary_values[xk]
if not (v[0] < xa < v[1]) or not (v[0] < xb < v[1]):
logging.error(
'Boundary values have to be strictly within the saturation limits!'
)
logging.info('Please have a look at the documentation, \
especially the example of the constrained double intgrator.'
)
raise ValueError(
'Boundary values have to be strictly within the saturation limits!'
)
# calculate saturation function expression and its derivative
yk = sp.Symbol(xk)
m = 4.0 / (v[1] - v[0])
psi = v[1] - (v[1] - v[0]) / (1. + sp.exp(m * yk))
#dpsi = ((v[1]-v[0])*m*sp.exp(m*yk))/(1.0+sp.exp(m*yk))**2
dpsi = (4. * sp.exp(m * yk)) / (1. + sp.exp(m * yk))**2
# replace constrained variables in vectorfield with saturation expression
# x(t) = psi(y(t))
ff_mat = ff_mat.replace(sp.Symbol(xk), psi)
# update vectorfield to represent differential equation for new
# unconstrained state variable
#
# d/dt x(t) = (d/dy psi(y(t))) * d/dt y(t)
# <==> d/dt y(t) = d/dt x(t) / (d/dy psi(y(t)))
ff_mat[k] /= dpsi
# update boundary values for new unconstrained variable
boundary_values[xk] = ((1. / m) * np.log(
(xa - v[0]) / (v[1] - xa)), (1. / m) * np.log(
(xb - v[0]) / (v[1] - xb)))
# create a callable function for the new symbolic vectorfield
ff = np.asarray(ff_mat).flatten().tolist()
xu = self.dyn_sys.states + self.dyn_sys.inputs
_f_sym = sp.lambdify(xu, ff, modules='sympy')
def f_sym(x, u):
xu = np.hstack((x, u))
return _f_sym(*xu)
# create a new unconstrained system
xa = [boundary_values[x][0] for x in self.dyn_sys.states]
xb = [boundary_values[x][1] for x in self.dyn_sys.states]
ua = [boundary_values[u][0] for u in self.dyn_sys.inputs]
ub = [boundary_values[u][1] for u in self.dyn_sys.inputs]
self.dyn_sys = DynamicalSystem(f_sym, a, b, xa, xb, ua, ub)
#logging.info(tstr)
url = "http://qt.gtimg.cn/q=" + tstr
response = urllib2.urlopen(url)
html = response.read()
lines = html.strip().split(";")
lines.remove("")
#time.sleep(100)
logging.info(len(lines))
for line in lines:
lv = line.strip().split("=")[1].replace('"',
'').split('~')
datalist.append(lv)
#logging.info(tempList)
except Exception, e:
logging.error(e)
tempList = []
if (len(tempList) > 0):
try:
tstr = ""
for t in tempList:
if tstr == "":
tstr = "s_sh" + t.strip()
else:
tstr = tstr + "," + "s_sh" + t.strip()
url = "http://qt.gtimg.cn/q=" + tstr
response = urllib2.urlopen(url)
html = response.read()
lines = html.split("\r\n")
for lv in lines:
lv = html.split("=")[1].replace('"', '').split('~')