Esempio n. 1
0
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception, e:
                logging.error("Error: %s : %s", func.__name__, str(e))

                if not hasattr(e, 'errorCode'):
                    out.failureMessage("Sorry, operation has failed!!!.")
                    tools.exit()

                errorCode = int(e.errorCode)

                # auth-token error, re-auth
                if errorCode == 9:
                    storage = Storage()
                    storage.removeUser()
                    GeekNote()
                    return func(*args, **kwargs)

                elif errorCode == 3:
                    out.failureMessage("Sorry, you do not have permissions to do this operation.")

                else:
                    return False

                tools.exit()
Esempio n. 2
0
    def textToENML(content, raise_ex=False, format='markdown'):
        """
        Create an ENML format of note.
        """
        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            # add 2 space before new line in paragraph for creating br tags
            content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
            if format=='markdown':
              contentHTML = markdown.markdown(content).encode("utf-8")
              # Non-Pretty HTML output
              contentHTML = str(BeautifulSoup(contentHTML)) 
            else:
              contentHTML = Editor.HTMLEscape(content)
            return Editor.wrapENML(contentHTML)
        except:
            if raise_ex:
                raise Exception("Error while parsing text to html."
                                " Content must be an UTF-8 encode.")

            logging.error("Error while parsing text to html. "
                          "Content must be an UTF-8 encode.")
            out.failureMessage("Error while parsing text to html. "
                               "Content must be an UTF-8 encode.")
            return tools.exit()
    def _reliable_op(*args, **kwargs):
        argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
        fname = func.func_name
        msg = 'MQ - -> %s(%s)' % (fname, ','.join('%s=%s' % entry \
                                                for entry in zip(argnames[1:],\
                                                args[1:]) + kwargs.items()))
        trans_times = 0
        while True:
            trans_times += 1
            if trans_times > MAX_RETRY:
                logging.error('rabbitmq connection lost!')
                return None
            try:
###############################################################
                logging.warn('function:%s' %(func.func_name))
###############################################################
                return func(*args, **kwargs)

            except pika.exceptions.AMQPConnectionError, e:
                logging.error("AMQPConnectionError: %s" % e)
                logging.error(traceback.format_exc())
                if not _reconnect(args, msg, e):
                    raise
            except pika.exceptions.AMQPChannelError, e:
                try:
                    if len(args) == 0 or not isinstance(args[0], RabbitMQBlockingClient):
                        raise MQError("unsupported decorator, it should be applied to RabbitMQBlockingClient methods")
                    close_func = getattr(args[0], "close")
                    close_func()
                except Exception, e:
                    logging.error(e)
                    logging.error(traceback.format_exc())
Esempio n. 4
0
 def RepoListParser(self, url):
     try:
         repoListHtml = self.session.get(url)
     except Exception as e:
         logging.error(e)
         return
     repoListHtmlSoup = BeautifulSoup(repoListHtml.text, 'html5lib')
     repoListTables = repoListHtmlSoup.table
     
     if repoListTables == None:
         return
     
     for oneTr in repoListTables.select('tr'):
         tds = oneTr.find_all('td')
         if len(tds) == 0:
             continue
         
         repoPathTd = tds[0]
         repoOwnerTd = tds[2]
         newRepo = {}
         newRepo['path'] = self.UrlBase+repoPathTd.a['href']
         newRepo['owner'] = repoOwnerTd.a.get_text()
         
         self.AllRepos.append(newRepo)
         
     nextPage = repoListHtmlSoup.find('div', id="paginator")
     if len(nextPage) == 0:
         return 0
     
     page_a = nextPage.select('a')
     for one_a in page_a:
         if repr(one_a.get_text()) == repr(u'下一页'):
             pageUrl = one_a['href'] 
             self.RepoListParser(self.RepoListUrl+pageUrl)
Esempio n. 5
0
 def checkVersion(self):
     versionOK = self.getUserStore().checkVersion("Python EDAMTest",
                                    UserStoreConstants.EDAM_VERSION_MAJOR,
                                    UserStoreConstants.EDAM_VERSION_MINOR)
     if not versionOK:
         logging.error("Old EDAM version")
         return tools.exitErr()
Esempio n. 6
0
    def plot(self):
        '''
        Plot the calculated trajectories and show interval error functions.

        This method calculates the error functions and then calls
        the :py:func:`visualisation.plotsim` function.
        '''

        try:
            import matplotlib
        except ImportError:
            logging.error('Matplotlib is not available for plotting.')
            return

        if self.constraints:
            sys = self._dyn_sys_orig
        else:
            sys = self.dyn_sys
            
        # calculate the error functions H_i(t)
        max_con_err, error = auxiliary.consistency_error((sys.a, sys.b), 
                                                          self.eqs.trajectories.x,
                                                          self.eqs.trajectories.u, 
                                                          self.eqs.trajectories.dx, 
                                                          sys.f_num, len(self.sim_data[0]), True)
        
        H = dict()
        for i in self.eqs.trajectories._eqind:
            H[i] = error[:,i]

        visualisation.plot_simulation(self.sim_data, H)
Esempio n. 7
0
def textToENML(content, raise_ex=False, format='markdown'):
    """
    Create an ENML format of note.
    """
    if not isinstance(content, str):
        content = ""
    try:
        content = unicode(content, "utf-8")
        # add 2 space before new line in paragraph for cteating br tags
        content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
        if format=='markdown':
          contentHTML = markdown.markdown(content).encode("utf-8")
          # remove all new-lines characters in html
          contentHTML = re.sub(r'\n', r'', contentHTML)
        else:
          contentHTML = HTMLEscape(content)
        return wrapENML(contentHTML)
    except:
        if raise_ex:
            raise Exception("Error while parsing text to html."
                            " Content must be an UTF-8 encode.")

        logging.error("Error while parsing text to html. "
                      "Content must be an UTF-8 encode.")
        out.failureMessage("Error while parsing text to html. "
                           "Content must be an UTF-8 encode.")
        return tools.exit()
Esempio n. 8
0
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception, e:
                logging.error("Error: %s : %s", func.__name__, str(e))

                if not hasattr(e, 'errorCode'):
                    out.failureMessage("Sorry, operation has failed!!!.")
                    tools.exitErr()

                errorCode = int(e.errorCode)

                # auth-token error, re-auth
                if errorCode == 9:
                    storage = Storage()
                    storage.removeUser()
                    GeekNote()
                    return func(*args, **kwargs)

                elif errorCode == 3:
                    out.failureMessage("Sorry, you do not have permissions "
                                       "to do this operation.")

                # Rate limited
                # Patched because otherwise if you get rate limited you still keep
                # hammering the server on scripts
                elif errorCode == 19:
                    print("\nRate Limit Hit: Please wait %s seconds before continuing" %
                          str(e.rateLimitDuration))
                    tools.exitErr()

                else:
                    return False

                tools.exitErr()
 def _add_proxy(self, message_type):
     import DefaultConfig
     message_config = RabbitMQBlockingClient._get_message_config(message_type, DefaultConfig.MESSAGE_CONFIGS)
     proxy = RabbitMQBlockingProxy(self._blocking_connection, self._aux_store, message_config)
     key = message_config["message_type"]
     if key in self._proxies.keys():
         logging.error("proxy has been added", key)
         raise MQError("proxy has been added", key)
     self._proxies[key] = proxy
Esempio n. 10
0
    def auth(self):
        GNA = GeekNoteAuth()
        self.authToken = GNA.getToken()
        userInfo = self.getUserInfo()
        if not isinstance(userInfo, object):
            logging.error("User info not get")
            return False

        self.getStorage().createUser(self.authToken, userInfo)
        return True
Esempio n. 11
0
    def allowAccess(self):
        response = self.loadPage(self.url['base'],
                                 self.url['access'],
                                 "GET",
                                 {'oauth_token': self.tmpOAuthToken})

        logging.debug(response.data)
        tree = html.fromstring(response.data);
        token = "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[0]}) + "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[1]})
        sourcePage = tree.xpath("//input[@name='_sourcePage']/@value")[0]
        fp = tree.xpath("//input[@name='__fp']/@value")[0]
        targetUrl = tree.xpath("//input[@name='targetUrl']/@value")[0]
        logging.debug(token);

        if response.status != 200:
            logging.error("Unexpected response status "
                          "on login 200 != %s", response.status)
            tools.exitErr()

        if 'JSESSIONID' not in self.cookies:
            logging.error("Not found value JSESSIONID in the response cookies")
            tools.exitErr()

        access = self.postData['access']
        access['oauth_token'] = self.tmpOAuthToken
        access['oauth_callback'] = ""
        access['embed'] = 'false'
        access['suggestedNotebookName'] = 'Geeknote'
        access['supportLinkedSandbox'] = ''
        access['analyticsLoginOrigin'] = 'Other'
        access['clipperFlow'] = 'false'
        access['showSwitchService'] = 'true'
        access['_sourcePage'] = sourcePage
        access['__fp'] = fp
        access['targetUrl'] = targetUrl

        response = self.loadPage(self.url['base'],
                                 self.url['access'],
                                 "POST", access, token)

        if response.status != 302:
            logging.error("Unexpected response status on allowing "
                          "access 302 != %s", response.status)
            logging.error(response.data)
            tools.exitErr()

        responseData = self.parseResponse(response.location)
        if 'oauth_verifier' not in responseData:
            logging.error("OAuth verifier not found")
            tools.exitErr()

        self.verifierToken = responseData['oauth_verifier']

        logging.debug("OAuth verifier token take")
    def reply(self, message, response):
        properties = pika.BasicProperties(correlation_id=message["__correlation_id"])

        if self._rpc_reply_content_type == "text/json":
            response = dumps_jsonx(response)

        try:
            self._channel.basic_publish(exchange="", routing_key=message["__reply_to"], body=response, properties=properties)
        except AttributeError, e:
            logging.error('basic publish raise AttributeError: %s' % e)
            raise pika.exceptions.AMQPConnectionError
Esempio n. 13
0
    def FileListParser(self, url, fileOwner):
        try:
            fileListHtml = self.session.get(url)
        except Exception as e:
            logging.error(e)
            newRepo = {}
            newRepo['path'] = url
            newRepo['owner'] = fileOwner
            if self.FileLock.acquire() == True:
                self.AllRepos.append(newRepo)
                self.FileLock.release()
            
        fileListHtmlSoup = BeautifulSoup(fileListHtml.text, 'html5lib')

        fileListTables = fileListHtmlSoup.table
        if fileListTables == None:
            return 0
        fileListTrs = fileListTables.select('tr')
        if len(fileListTrs) == 0:
            return 0
        
        for oneTr in fileListTrs[1:]:
#             print "level--->%s" %oneTr.select('select[class="file-level-type-select"]')[0].option['value']#.select('option["selected="selected"]')
            previewA = oneTr.select('span[class="dirent-name"]')[0].select('a')[0]
            previewPath = previewA['href']
            pathList = repr(previewPath).split('/')
            if len(pathList) < 4:
                continue
            path = pathList[1]+'/'+pathList[2]+'/'+pathList[3]
            
            if path == 'sys/seafadmin/repo':
#                 print 'this is a dir'
                self.FileListParser(self.UrlBase+previewPath, fileOwner)
            else:
#                 print 'find a new file'
                fileName = previewA.get_text()
                downloadPath = oneTr.select('a')[2]['href']
                newFile = {}
                newFile['fileName'] = fileName
                newFile['owner'] = fileOwner
                newFile['previewPath'] = self.UrlBase+previewPath
                newFile['downloadPath'] = self.UrlBase+downloadPath
                tagSpan = oneTr.select('span[class="file-tags"]')
                level = oneTr.select('select[class="file-level-type-select"]')[0].select('option[selected="selected"]')[0]['value']
                newFile['level'] = level
                if len(tagSpan) > 0:
                    newFile['tags'] = tagSpan[0].get_text()
                else:
                    newFile['tags'] = ''
                    
                if self.FileLock.acquire():
                    self.AllFiles.append(newFile)
                    self.FileLock.release()
Esempio n. 14
0
def im_transform(image, size=-1, scale=1., mean_values=[0.,0.,0.]):
    assert len(mean_values) == 3
    if size > 0:
        try:
            patch = cv2.resize(image, (size, size))
        except:
            logging.error("Unable to resize image.")
            print image.shape
            raise
    else:
        patch = image
    trans_img = (np.asarray(patch) - np.asarray(mean_values)) * scale
    return trans_img.swapaxes(1,2).swapaxes(0,1)
Esempio n. 15
0
    def loadPage(self, url, uri=None, method="GET", params=""):
        if not url:
            logging.error("Request URL undefined")
            tools.exitErr()

        if not uri:
            urlData = urlparse(url)
            url = urlData.netloc
            uri = urlData.path + '?' + urlData.query

        # prepare params, append to uri
        if params:
            params = urlencode(params)
            if method == "GET":
                uri += ('?' if uri.find('?') == -1 else '&') + params
                params = ""

        # insert local cookies in request
        headers = {
            "Cookie": '; '.join([key + '=' + self.cookies[key] for key in self.cookies.keys()])
        }

        if method == "POST":
            headers["Content-type"] = "application/x-www-form-urlencoded"

        logging.debug("Request URL: %s:/%s > %s # %s", url,
                      uri, unquote(params), headers["Cookie"])

        conn = httplib.HTTPSConnection(url)
        conn.request(method, uri, params, headers)
        response = conn.getresponse()
        data = response.read()
        conn.close()

        logging.debug("Response : %s > %s",
                      response.status,
                      response.getheaders())
        result = tools.Struct(status=response.status,
                              location=response.getheader('location', None),
                              data=data)

        # update local cookies
        sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
        for key in sk:
            self.cookies[key] = sk[key].value
        # delete cookies whose content is "deleteme"
        for key in self.cookies.keys():
            if self.cookies[key] == "deleteme":
                del self.cookies[key]

        return result
Esempio n. 16
0
    def login(self):
        response = self.loadPage(self.url['base'], self.url['login'], "GET", {'oauth_token': self.tmpOAuthToken})

        if response.status != 200:
            logging.error("Unexpected response status on login 200 != %s", response.status)
            tools.exit()

        if not self.cookies.has_key('JSESSIONID'):
            logging.error("Not found value JSESSIONID in the response cookies")
            tools.exit()

        # get login/password
        self.username, self.password = out.GetUserCredentials()

        self.postData['login']['username'] = self.username
        self.postData['login']['password'] = self.password
        self.postData['login']['targetUrl'] = self.url['oauth']%self.tmpOAuthToken
        response = self.loadPage(self.url['base'], self.url['login']+";jsessionid="+self.cookies['JSESSIONID'], "POST", 
            self.postData['login'])

        if not response.location and response.status == 200:
            if self.incorrectLogin < 3:
                out.preloader.stop()
                out.printLine('Sorry, incorrect login or password')
                out.preloader.setMessage('Authorize...')
                self.incorrectLogin += 1
                return self.login()
            else:
                logging.error("Incorrect login or password")

        if not response.location:
            logging.error("Target URL was not found in the response on login")
            tools.exit()

        logging.debug("Success authorize, redirect to access page")
Esempio n. 17
0
    def getOAuthToken(self):
        response = self.loadPage(self.url['base'], self.url['token'], "GET",  
            self.getTokenRequestData(oauth_token=self.tmpOAuthToken, oauth_verifier=self.verifierToken))

        if response.status != 200:
            logging.error("Unexpected response status on getting oauth token 200 != %s", response.status)
            tools.exit()

        responseData = self.parseResponse(response.data)
        if not responseData.has_key('oauth_token'):
            logging.error("OAuth token not found")
            tools.exit()

        logging.debug("OAuth token take : %s", responseData['oauth_token'])
        self.OAuthToken = responseData['oauth_token']
Esempio n. 18
0
    def textToENML(content, raise_ex=False, format='markdown'):
        """
        Create an ENML format of note.
        TODO: add media support
        """
        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            # add 2 space before new line in paragraph for creating br tags
            content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
            if format == 'markdown':
                contentHTML = markdown.markdown(content).encode("utf-8")
                # Non-Pretty HTML output
                contentHTML = str(BeautifulSoup(contentHTML, 'html.parser'))
            elif format == 'html':
                # Html to ENML http://dev.evernote.com/doc/articles/enml.php
                ATTR_2_REMOVE = ["id"
                                 , "class"
                                 #, "on*"
                                 , "accesskey"
                                 , "data"
                                 , "dynsrc"
                                 , "tabindex"
                ];
                soup = BeautifulSoup(content, 'html.parser')
                
                for tag in soup.findAll():
                    if hasattr(tag, 'attrs'):
                        map(lambda x: tag.attrs.pop(x, None), \
                            [k for k in tag.attrs.keys() \
                             if k in ATTR_2_REMOVE \
                             or k.find('on') == 0])
                contentHTML = str(soup)
            else:
                contentHTML = Editor.HTMLEscape(content)
            return Editor.wrapENML(contentHTML)
        except:
            if raise_ex:
                raise Exception("Error while parsing text to html."
                                " Content must be an UTF-8 encode.")

            logging.error("Error while parsing text to html. "
                          "Content must be an UTF-8 encode.")
            out.failureMessage("Error while parsing text to html. "
                               "Content must be an UTF-8 encode.")

            return tools.exit()
Esempio n. 19
0
    def getTmpOAuthToken(self):
        response = self.loadPage(self.url['base'], self.url['token'], "GET", 
            self.getTokenRequestData(oauth_callback="https://"+self.url['base']))

        if response.status != 200:
            logging.error("Unexpected response status on get temporary oauth_token 200 != %s", response.status)
            tools.exit()

        responseData = self.parseResponse(response.data)
        if not responseData.has_key('oauth_token'):
            logging.error("OAuth temporary not found")
            tools.exit()

        self.tmpOAuthToken = responseData['oauth_token']

        logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
Esempio n. 20
0
    def handleTwoFactor(self):
        self.code = out.GetUserAuthCode()
        self.postData['tfa']['code'] = self.code
        response = self.loadPage(self.url['base'], self.url['tfa'] + ";jsessionid=" + self.cookies['JSESSIONID'], "POST", self.postData['tfa'])
        if not response.location and response.status == 200:
            if self.incorrectCode < 3:
                out.preloader.stop()
                out.printLine('Sorry, incorrect two factor code')
                out.preloader.setMessage('Authorize...')
                self.incorrectCode += 1
                return self.handleTwoFactor()
            else:
                logging.error("Incorrect two factor code")

        if not response.location:
            logging.error("Target URL was not found in the response on login")
            tools.exitErr()
Esempio n. 21
0
    def textToENML(content, raise_ex=False, format='markdown'):
        """
        Create an ENML format of note.
        """
        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            # add 2 space before new line in paragraph for creating br tags
            content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
            if format=='markdown':
              storage = Storage()
              extras = storage.getUserprop('markdown2_extras')
              #contentHTML = markdown.markdown(content).encode("utf-8")
              contentHTML = markdown.markdown(
                      content, extras=extras).encode("utf-8")

              soup = BeautifulSoup(contentHTML, 'html.parser')
              Editor.checklistInSoupToENML(soup)
              # Non-Pretty HTML output
              contentHTML = str(soup)
            #
            # For the 'pre' format, simply wrap the content with a 'pre' tag. Do
            # perform any parsing/mutation.
            #
            elif format=='pre':
              contentHTML = u''.join(('<pre>', content, '</pre>')).encode("utf-8")
            else:
              contentHTML = Editor.HTMLEscape(content)

            contentHTML = contentHTML.replace('[x]','<en-todo checked="true"></en-todo>')
            contentHTML = contentHTML.replace('[ ]','<en-todo></en-todo>')

            return Editor.wrapENML(contentHTML)
        except:
            import traceback
            traceback.print_exc()
            if raise_ex:
                raise Exception("Error while parsing text to html."
                                " Content must be an UTF-8 encode.")

            logging.error("Error while parsing text to html. "
                          "Content must be an UTF-8 encode.")
            out.failureMessage("Error while parsing text to html. "
                               "Content must be an UTF-8 encode.")
            return tools.exitErr()
Esempio n. 22
0
    def allowAccess(self):

        self.postData['access']['oauth_token'] = self.tmpOAuthToken
        self.postData['access']['oauth_callback'] = "https://"+self.url['base']
        response = self.loadPage(self.url['base'], self.url['access'], "POST", self.postData['access'])

        if response.status != 302:
            logging.error("Unexpected response status on allowing access 302 != %s", response.status)
            tools.exit()

        responseData = self.parseResponse(response.location)
        if not responseData.has_key('oauth_verifier'):
            logging.error("OAuth verifier not found")
            tools.exit()

        self.verifierToken = responseData['oauth_verifier']

        logging.debug("OAuth verifier token take")
Esempio n. 23
0
def matlab_engine(fun_file, input_list, eng=None):
    '''matlab enginer wrapper
    return_val = fun(input_list)
    '''
    import matlab.engine
    script_dirname = os.path.abspath(os.path.dirname(fun_file))
    fun_name = stem(fun_file)
    if eng is None:
        eng = matlab.engine.start_matlab('-nodisplay -nojvm -nosplash -nodesktop')
    else:
        logging.debug("Use opened Matlab session: {}".format(eng))
    try:
        eng.cd(script_dirname)
        func = getattr(eng, fun_name)
        result = func(input_list)
    except matlab.engine.EngineError, e:
        # Use new engine instead
        logging.error("Existing engine no response: {}".format(eng))
        raise e
Esempio n. 24
0
    def login(self):
        response = self.loadPage(self.url['base'],
                                 self.url['login'],
                                 "GET",
                                 {'oauth_token': self.tmpOAuthToken})

        # parse hpts and hptsh from page content
        hpts = re.search('.*\("hpts"\)\.value.*?"(.*?)"', response.data)
        hptsh = re.search('.*\("hptsh"\)\.value.*?"(.*?)"', response.data)

        if response.status != 200:
            logging.error("Unexpected response status "
                          "on login 200 != %s", response.status)
            tools.exitErr()

        if 'JSESSIONID' not in self.cookies:
            logging.error("Not found value JSESSIONID in the response cookies")
            tools.exitErr()

        # get login/password
        self.username, self.password = out.GetUserCredentials()

        self.postData['login']['username'] = self.username
        self.postData['login']['password'] = self.password
        self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
        self.postData['login']['hpts'] = hpts and hpts.group(1) or ""
        self.postData['login']['hptsh'] = hptsh and hptsh.group(1) or ""
        response = self.loadPage(self.url['base'],
                                 self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
                                 "POST",
                                 self.postData['login'])

        if not response.location and response.status == 200:
            if self.incorrectLogin < 3:
                out.preloader.stop()
                out.printLine('Sorry, incorrect login or password')
                out.preloader.setMessage('Authorize...')
                self.incorrectLogin += 1
                return self.login()
            else:
                logging.error("Incorrect login or password")

        if not response.location:
            logging.error("Target URL was not found in the response on login")
            tools.exitErr()

        if response.status == 302:
            # the user has enabled two factor auth
            return self.handleTwoFactor()


        logging.debug("Success authorize, redirect to access page")
Esempio n. 25
0
    def getTmpOAuthToken(self):
        response = self.loadPage(self.url['base'],
                                 self.url['token'],
                                 "GET",
                                 self.getTokenRequestData(
                                     oauth_callback="https://" + self.url['base']))

        if response.status != 200:
            logging.error("Unexpected response status on get "
                          "temporary oauth_token 200 != %s", response.status)
            raise OAuthError('OAuth token request failed')

        responseData = self.parseResponse(response.data)
        if 'oauth_token' not in responseData:
            logging.error("OAuth temporary not found")
            raise OAuthError('OAuth token request failed')

        self.tmpOAuthToken = responseData['oauth_token']

        logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
Esempio n. 26
0
 def plot(self, show=True, ret_array=False):
     '''
     Plots the spline function or returns an array with its values at
     some points of the spline interval.
     
     Parameters
     ----------
     
     show : bool
         Whethter to plot the spline's curve or not.
     
     ret_array : bool
         Wheter to return an array with values of the spline at points
         of the interval.
         
     '''
     
     if not show and not ret_array:
         # nothing to do here...
         return
     elif self._prov_flag:
         # spline cannot be plotted, because there are no numeric
         # values for its polynomial coefficients
         logging.error("There are no numeric values for the spline's\
                         polynomial coefficients.")
         return
     
     # create array of values
     tt = np.linspace(self.a, self.b, 1000, endpoint=True)
     St = [self.f(t) for t in tt]
     
     if show:
         try:
             import matplotlib.pyplot as plt
             plt.plot(tt,St)
             plt.show()
         except ImportError:
             logging.error('Could not import matplotlib for plotting the curve.')
     
     if ret_array:
         return St
Esempio n. 27
0
	def getSearchBySphinx(self, q, index = '*'):
		"""
		从sphinx搜索结果

		Args:
			q 		- 搜索词
			index 	- 需要使用的索引, default: '*'

		Returns:
			Id list.
		"""
		res = self.sp.query(q, index)

		lastError = self.sp.getLastError()
		lastWarning = self.sp.getLastWarning()
		if lastError:
			logging.error('getSearchBySphinx has error: %s' % (lastError))
		if lastWarning:
			logging.warning('getSearchBySphinx has warning: %s' % (lastWarning))
		
		return res
def reliable_op(func):
    """
    if message queue is broken, will try to reconnect...
    """
    def _reconnect(args, msg, e):
        if len(args) == 0 or not isinstance(args[0], RabbitMQBlockingClient):
            raise MQError("unsupported decorator, it should be applied to RabbitMQBlockingClient methods")
        self = args[0]
########################################################
        logging.error("enter reconnection")
########################################################
        init_func = getattr(self, "_init")
        client_configs = getattr(self, "_client_configs")
        
########################################################
        logging.error("begin reconnection")
########################################################
        logging.error(\
                      "message queue connection error: %s, %s reconnecting..."\
         % (msg, e))
        time.sleep(5)
        try:
########################################################
            logging.error("test init_func")
########################################################
            init_func(client_configs)
            logging.info("message queue reconnection succeeded")
########################################################
            logging.error("test init_func")
########################################################
            return True
        except AMQPChannelError, e:
            try:
                close_func = getattr(self, "close")
                close_func()
            except Exception, e:
                logging.error(e)
                logging.error(traceback.format_exc())
            logging.error('Exception', e)
            logging.error(traceback.format_exc())
Esempio n. 29
0
def crawlStock(stockidList, k):
    try:
        tstr = ""
        datalist = []
        for t in tempList:
            if tstr == "":
                tstr = k + t.strip()
            else:
                tstr = tstr + "," + k + t.strip()
        #logging.info(tstr)
        url = "http://qt.gtimg.cn/q=" + tstr
        #print url
        response = urllib2.urlopen(url)
        html = response.read()
        #logging.info(html)
        lines = html.strip().split(";")
        if "" in lines:
            lines.remove("")
        for line in lines:
            lv = line.strip().split("=")[1].replace('"', '').split('~')
            datalist.append(lv)
        analyseData1(datalist, k)
    except Exception, e:
        logging.error(traceback.format_exc())
Esempio n. 30
0
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception, e:
                logging.error("Error: %s : %s", func.__name__, str(e))

                if not hasattr(e, 'errorCode'):
                    out.failureMessage("Sorry, operation has failed!!!.")
                    tools.exitErr()

                errorCode = int(e.errorCode)

                # auth-token error, re-auth
                if errorCode == 9:
                    storage = Storage()
                    storage.removeUser()
                    GeekNote()
                    return func(*args, **kwargs)

                elif errorCode == 3:
                    out.failureMessage("Sorry, you do not have permissions "
                                       "to do this operation.")

                # Rate limited
                # Patched because otherwise if you get rate limited you still keep
                # hammering the server on scripts
                elif errorCode == 19:
                    print(
                        "\nRate Limit Hit: Please wait %s seconds before continuing"
                        % str(e.rateLimitDuration))
                    tools.exitErr()

                else:
                    return False

                tools.exitErr()
Esempio n. 31
0
    def login(self):
        response = self.loadPage(self.url['base'],
                                 self.url['login'],
                                 "GET",
                                 {'oauth_token': self.tmpOAuthToken})

        if response.status != 200:
            logging.error("Unexpected response status "
                          "on login 200 != %s", response.status)
            tools.exitErr()

        if 'JSESSIONID' not in self.cookies:
            logging.error("Not found value JSESSIONID in the response cookies")
            tools.exitErr()

        # get login/password
        self.username, self.password = out.GetUserCredentials()

        self.postData['login']['username'] = self.username
        self.postData['login']['password'] = self.password
        self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
        response = self.loadPage(self.url['base'],
                                 self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
                                 "POST",
                                 self.postData['login'])

        if not response.location and response.status == 200:
            if self.incorrectLogin < 3:
                out.preloader.stop()
                out.printLine('Sorry, incorrect login or password')
                out.preloader.setMessage('Authorize...')
                self.incorrectLogin += 1
                return self.login()
            else:
                logging.error("Incorrect login or password")

        if not response.location:
            logging.error("Target URL was not found in the response on login")
            tools.exitErr()

        if response.status == 302:
            # the user has enabled two factor auth
            return self.handleTwoFactor()


        logging.debug("Success authorize, redirect to access page")
Esempio n. 32
0
def textToENML(content, raise_ex=False, format='markdown'):
    """
    Create an ENML format of note.
    """
    if not isinstance(content, str):
        content = ""
    try:
        content = unicode(content,"utf-8")
        # add 2 space before new line in paragraph for cteating br tags
        content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
        if format=='markdown':
          contentHTML = markdown.markdown(content).encode("utf-8")
          # remove all new-lines characters in html
          contentHTML = re.sub(r'\n', r'', contentHTML)
        else:
          contentHTML = HTMLEscape(content)
        return wrapENML(contentHTML)
    except:
        if raise_ex:
            raise Exception("Error while parsing text to html. Content must be an UTF-8 encode.")
            
        logging.error("Error while parsing text to html. Content must be an UTF-8 encode.")
        out.failureMessage("Error while parsing text to html. Content must be an UTF-8 encode.")
        return tools.exit()
Esempio n. 33
0
def textToENML(content, raise_ex=False):
    """
    Create an ENML format of note.
    """
    pdb.set_trace()    
    if not isinstance(content, str):
        content = ""
    try:
        content = unicode(content,"utf-8")
        # add 2 space before new line in paragraph for cteating br tags
        content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
        #contentHTML, md = convert_markdown(content)
        #meta = md.Meta
        # remove all new-lines characters in html
        contentHTML = re.sub(r'\n', r'', content)
        return wrapENML(contentHTML)
    except:
        raise        
        if raise_ex:
            raise Exception("Error while parsing text to html. Content must be an UTF-8 encode.")

        logging.error("Error while parsing text to html. Content must be an UTF-8 encode.")
        out.failureMessage("Error while parsing text to html. Content must be an UTF-8 encode.")
        return tools.exit()
Esempio n. 34
0
def checkJumpAndDuplicatedRecordFromSPC(coreName, machineId):
    responseData = initResponse()
    result = {}
    result['status'] = ''
    responseStr = ''
    jsonObject = {}
    try:
        jsonObject = queryLastSPC(coreName, machineId)
        data = praseJsonFormat(jsonObject)
        # logging.info(data)
        df = pd.DataFrame(data).iloc[0].astype(str)

        # 取第一筆
        timestamp_iso = df['timestamp_iso']
        RD_618 = df['RD_618']
        SPC_0 = df['SPC_0']

        if RD_618 == SPC_0:
            return 1, RD_618  # true, 模次正常
        return 0, RD_618  # false, 模次亂跳或是模次重複
    except:
        logging.error("checkJumpAndDuplicatedRecordFromSPC exception")
        responseData = setErrorResponse(responseData, '400', 'exception')
        return parseResponseJson(responseData, result, 'result')
Esempio n. 35
0
def insert(con: mysql.connector.MySQLConnection, sql: str) -> int:
    try:
        cursor = con.cursor()
        cursor.execute(sql)
        con.commit()
        cursor.close()
    except mysql.connector.IntegrityError as e:
        logging.error(f'DB insert 데이터가 중복되었습니다. sql: {sql}')
        logging.error(f'{e}')
        # 중복
        return 1
    except Exception as e:
        logging.error(f'DB insert 중 에러가 발생하였습니다. sql: {sql}')
        logging.error(f'{e}')
        # 에러
        return 2

    return 0
Esempio n. 36
0
def db_connect():
    # Construct connection string
    try:
        conn = mysql.connector.connect(**config)
        logging.info("Connection established")
        return conn
    except mysql.connector.Error as err:
        if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
            logging.error("Something is wrong with the user name or password")
        elif err.errno == errorcode.ER_BAD_DB_ERROR:
            logging.error("Database does not exist")
        else:
            logging.error(err)
        return None
Esempio n. 37
0
#encoding=utf-8
import threading
from log import logging
import os

try:
    from bs4 import BeautifulSoup
except Exception:
    logging.error('import [BeautifulSoup] error')
    os._exit()


class WorkThread(threading.Thread):
    session = ''
    AllRepos = []
    AllFiles = []
    UrlBase = ''
    RepoListUrl = ''  #UrlBase+'/sys/seafadmin/'

    FileLock = threading.Lock()
    RepoLock = threading.Lock()

    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        while True:
            if self.RepoLock.acquire():
                if len(self.AllRepos) == 0:
                    self.RepoLock.release()
                    break
Esempio n. 38
0
    def parse(self):
        self.INP_DATA = {}

        if self.CMD is None:
            out.printAbout()
            return False

        if self.CMD == "autocomplete":
            # подставляем аргументы для автозаполнения
            # делаем смещение на 1 аргумент, т.к. 1 это autocomplete
            self.__init__(self.sys_argv[1:])
            self.printAutocomplete()
            return False

        if self.CMD == "--help":
            self.printHelp()
            return False

        if not self.COMMANDS.has_key(self.CMD):
            self.printErrorCommand()
            return False

        if "--help" in self.INP:
            self.printHelp()
            return False

        # Подготовка данных
        for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
            # установка значений по умолчаеию
            if params.has_key('default'):
                self.INP_DATA[arg] = params['default']

            # замена altName во входящих аргументах на полные
            if params.has_key('altName') and params['altName'] in self.INP:
                self.INP[self.INP.index(params['altName'])] = arg

        activeArg = None
        ACTIVE_CMD = None
        # проверяем и подставляем первый адгумент по умолчанию
        if self.COMMANDS[self.CMD].has_key('firstArg'):
            firstArg = self.COMMANDS[self.CMD]['firstArg']
            if len(self.INP) > 0:
                # смотрим что первое знаение не аргумент по умолчанию, а другой аргумент
                if self.INP[0] not in (self.CMD_ARGS.keys() + self.CMD_FLAGS.keys()):
                    self.INP = [firstArg, ] + self.INP
            else:
                self.INP = [firstArg, ]
        

        for item in self.INP:
            # Проверяем что ожидаем аргумент
            if activeArg is None:
                # Действия для аргумента
                if self.CMD_ARGS.has_key(item):
                    activeArg = item
                    ACTIVE_CMD = self.CMD_ARGS[activeArg]

                # Действия для флага
                elif self.CMD_FLAGS.has_key(item):
                    self.INP_DATA[item] = self.CMD_FLAGS[item]["value"]

                # Ошибка параметр не найден
                else:
                    self.printErrorArgument(item)
                    return False

            else:
                activeArgTmp = None
                # Значения является параметром
                if self.CMD_ARGS.has_key(item) or self.CMD_FLAGS.has_key(item):
                    # "Активный" аргумент имеет параметр emptyValue
                    if ACTIVE_CMD.has_key("emptyValue"):
                        activeArgTmp = item # запоминаем новый "активный" аргумент
                        item = ACTIVE_CMD['emptyValue'] # подменяем значение на emptyValue
                    # Ошибка, "активный" аргумент не имеет значений
                    else:
                        self.printErrorArgument(activeArg, item)
                        return False

                if ACTIVE_CMD.has_key("type"):
                    convType = ACTIVE_CMD['type']
                    if convType not in (int, str):
                        logging.error("Unsupported argument type: %s", convType)
                        return False

                    try:
                        item = convType(item)
                    except:
                        self.printErrorArgument(activeArg, item)
                        return False

                self.INP_DATA[activeArg] = item
                activeArg = activeArgTmp # тут или пусто, или новый "активный" аргумент

        # если остались "активные" аршументы
        if activeArg is not None:
            # если есть параметр emptyValue
            if ACTIVE_CMD.has_key("emptyValue"):
                self.INP_DATA[activeArg] = ACTIVE_CMD['emptyValue']

            # инече ошибка
            else:
                self.printErrorArgument(activeArg, "")
                return False

        # проверка, присутствует ли необходимый аргумент запросе
        for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
            if params.has_key('required') and arg not in self.INP:
                self.printErrorReqArgument(arg)
                return False

        # trim -- and ->_
        self.INP_DATA = dict([key.lstrip("-").replace("-", "_"), val] for key, val in self.INP_DATA.items() )
        return self.INP_DATA
Esempio n. 39
0
    def textToENML(content, raise_ex=False, format="markdown", rawmd=False):
        """
        Transform formatted text to ENML
        """

        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            # add 2 space before new line in paragraph for creating br tags
            content = re.sub(r"([^\r\n])([\r\n])([^\r\n])", r"\1  \n\3",
                             content)
            # content = re.sub(r'\r\n', '\n', content)

            if format == "pre":
                # For the 'pre' format, simply wrap the content with a 'pre' tag.
                # Do not perform any further parsing/mutation.
                contentHTML = u"".join(
                    ("<pre>", content, "</pre>")).encode("utf-8")
            elif format == "markdown":
                # Markdown format https://daringfireball.net/projects/markdown/basics
                extras = None

                if not rawmd:
                    storage = Storage()
                    extras = storage.getUserprop("markdown2_extras")
                    content = Editor.HTMLEscapeTag(content)

                contentHTML = markdown.markdown(content, extras=extras)

                soup = BeautifulSoup(contentHTML, "html.parser")
                Editor.checklistInSoupToENML(soup)
                contentHTML = str(soup)
            elif format == "html":
                # Html to ENML http://dev.evernote.com/doc/articles/enml.php
                soup = BeautifulSoup(content, "html.parser")
                ATTR_2_REMOVE = [
                    "id",
                    "class",
                    # "on*",
                    "accesskey",
                    "data",
                    "dynsrc",
                    "tabindex",
                ]

                for tag in soup.findAll():
                    if hasattr(tag, "attrs"):
                        map(
                            lambda x: tag.attrs.pop(x, None),
                            [
                                k for k in tag.attrs.keys()
                                if k in ATTR_2_REMOVE or k.find("on") == 0
                            ],
                        )
                contentHTML = str(soup)
            else:
                # Plain text format
                contentHTML = Editor.HTMLEscape(content)

                tmpstr = ""
                for l in contentHTML.split("\n"):
                    if l == "":
                        tmpstr = tmpstr + u"<div><br/></div>"
                    else:
                        tmpstr = tmpstr + u"<div>" + l + u"</div>"

                contentHTML = tmpstr.encode("utf-8")
                contentHTML = contentHTML.replace(
                    "[x]", '<en-todo checked="true"></en-todo>')
                contentHTML = contentHTML.replace("[ ]", "<en-todo></en-todo>")

            return Editor.wrapENML(contentHTML)

        except:
            import traceback

            traceback.print_exc()
            if raise_ex:
                raise Exception("Error while parsing text to html.")
            logging.error("Error while parsing text to html.")
            out.failureMessage("Error while parsing text to html.")
            return tools.exitErr()
Esempio n. 40
0
    def textToENML(content, raise_ex=False, format='markdown', rawmd=False):
        """
        Transform formatted text to ENML
        """

        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            content = re.sub(r'\r\n', '\n', content)

            if format == 'pre':
                # For the 'pre' format, simply wrap the content with a 'pre' tag.
                # Do not perform any further parsing/mutation.
                contentHTML = u''.join(
                    ('<pre>', content, '</pre>')).encode("utf-8")
            elif format == 'markdown':
                # Markdown format https://daringfireball.net/projects/markdown/basics
                storage = Storage()
                extras = storage.getUserprop('markdown2_extras')

                if not rawmd:
                    content = Editor.HTMLEscapeTag(content)

                contentHTML = markdown.markdown(content, extras=extras)

                soup = BeautifulSoup(contentHTML, 'html.parser')
                Editor.checklistInSoupToENML(soup)
                contentHTML = str(soup)
            elif format == 'html':
                # Html to ENML http://dev.evernote.com/doc/articles/enml.php
                soup = BeautifulSoup(content, 'html.parser')
                ATTR_2_REMOVE = [
                    "id",
                    "class",
                    # "on*",
                    "accesskey",
                    "data",
                    "dynsrc",
                    "tabindex"
                ]

                for tag in soup.findAll():
                    if hasattr(tag, 'attrs'):
                        map(lambda x: tag.attrs.pop(x, None), [
                            k for k in tag.attrs.keys()
                            if k in ATTR_2_REMOVE or k.find('on') == 0
                        ])
                contentHTML = str(soup)
            else:
                # Plain text format
                contentHTML = Editor.HTMLEscape(content)

                tmpstr = ''
                for l in contentHTML.split('\n'):
                    if l == '':
                        tmpstr = tmpstr + u'<div><br/></div>'
                    else:
                        tmpstr = tmpstr + u'<div>' + l + u'</div>'

                contentHTML = tmpstr.encode("utf-8")
                contentHTML = contentHTML.replace(
                    '[x]', '<en-todo checked="true"></en-todo>')
                contentHTML = contentHTML.replace('[ ]', '<en-todo></en-todo>')

            return Editor.wrapENML(contentHTML)

        except:
            import traceback
            traceback.print_exc()
            if raise_ex:
                raise Exception("Error while parsing text to html.")
            logging.error("Error while parsing text to html.")
            out.failureMessage("Error while parsing text to html.")
            return tools.exitErr()
Esempio n. 41
0
    def set_coefficients(self, free_coeffs=None, coeffs=None):
        '''
        This function is used to set up numerical values either for all the spline's coefficients
        or its independent ones.
        
        Parameters
        ----------
        
        free_coeffs : numpy.ndarray
            Array with numerical values for the free coefficients of the spline.
        
        coeffs : numpy.ndarray
            Array with coefficients of the polynomial spline parts.
        '''

        # deside what to do
        if coeffs is None and free_coeffs is None:
            # nothing to do
            pass

        elif coeffs is not None and free_coeffs is None:
            # set all the coefficients for the spline's polynomial parts
            #
            # first a little check
            if not (self.n == coeffs.shape[0]):
                logging.error(
                    'Dimension mismatch in number of spline parts ({}) and \
                            rows in coefficients array ({})'.format(
                        self.n, coeffs.shape[0]))
                raise ValueError(
                    'Dimension mismatch in number of spline parts ({}) and \
                            rows in coefficients array ({})'.format(
                        self.n, coeffs.shape[0]))
            elif not (coeffs.shape[1] == 4):
                logging.error(
                    'Dimension mismatch in number of polynomial coefficients (4) and \
                            columns in coefficients array ({})'.format(
                        coeffs.shape[1]))
            # elif not (self._indep_coeffs.size == coeffs.shape[1]):
            #     logging.error('Dimension mismatch in number of free coefficients ({}) and \
            #                 columns in coefficients array ({})'.format(self._indep_coeffs.size, coeffs.shape[1]))
            #     raise ValueError

            # set coefficients
            self._coeffs = coeffs

            # update polynomial parts
            for k in xrange(self.n):
                self._P[k] = np.poly1d(self._coeffs[k])

        elif coeffs is None and free_coeffs is not None:
            # a little check
            if not (self._indep_coeffs.size == free_coeffs.size):
                logging.error('Got {} values for the {} independent coefficients.'\
                                .format(free_coeffs.size, self._indep_coeffs.size))
                raise ValueError('Got {} values for the {} independent coefficients.'\
                                .format(free_coeffs.size, self._indep_coeffs.size))

            # set the numerical values
            self._indep_coeffs = free_coeffs

            # update the spline coefficients and polynomial parts
            for k in xrange(self.n):
                coeffs_k = self._dep_array[k].dot(
                    free_coeffs) + self._dep_array_abs[k]
                self._coeffs[k] = coeffs_k
                self._P[k] = np.poly1d(coeffs_k)
        else:
            # not sure...
            logging.error(
                'Not sure what to do, please either pass `coeffs` or `free_coeffs`.'
            )
            raise TypeError(
                'Not sure what to do, please either pass `coeffs` or `free_coeffs`.'
            )

        # now we have numerical values for the coefficients so we can set this to False
        self._prov_flag = False
Esempio n. 42
0
        if COMMAND == 'notebook-remove':
            Notebooks().remove(**ARGS)

        # Tags
        if COMMAND == 'tag-list':
            Tags().list(**ARGS)

        if COMMAND == 'tag-create':
            Tags().create(**ARGS)

        if COMMAND == 'tag-edit':
            Tags().edit(**ARGS)

        if COMMAND == 'tag-remove':
            Tags().remove(**ARGS)

    except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
        if e.message:
            exit_status_code = e.message

    except Exception, e:
        traceback.print_exc()
        logging.error("App error: %s", str(e))

    # exit preloader
    tools.exit('exit', exit_status_code)

if __name__ == "__main__":
    main()
Esempio n. 43
0
    def allowAccess(self):
        response = self.loadPage(
            self.url["base"],
            self.url["access"],
            "GET",
            {"oauth_token": self.tmpOAuthToken},
        )

        logging.debug(response.data)
        tree = html.fromstring(response.data)
        token = (
            "&"
            + urlencode(
                {
                    "csrfBusterToken": tree.xpath(
                        "//input[@name='csrfBusterToken']/@value"
                    )[0]
                }
            )
            + "&"
            + urlencode(
                {
                    "csrfBusterToken": tree.xpath(
                        "//input[@name='csrfBusterToken']/@value"
                    )[1]
                }
            )
        )
        sourcePage = tree.xpath("//input[@name='_sourcePage']/@value")[0]
        fp = tree.xpath("//input[@name='__fp']/@value")[0]
        targetUrl = tree.xpath("//input[@name='targetUrl']/@value")[0]
        logging.debug(token)

        if response.status != 200:
            logging.error(
                "Unexpected response status " "on login 200 != %s", response.status
            )
            tools.exitErr()

        if "JSESSIONID" not in self.cookies:
            logging.error("Not found value JSESSIONID in the response cookies")
            tools.exitErr()

        access = self.postData["access"]
        access["oauth_token"] = self.tmpOAuthToken
        access["oauth_callback"] = ""
        access["embed"] = "false"
        access["suggestedNotebookName"] = "Geeknote"
        access["supportLinkedSandbox"] = ""
        access["analyticsLoginOrigin"] = "Other"
        access["clipperFlow"] = "false"
        access["showSwitchService"] = "true"
        access["_sourcePage"] = sourcePage
        access["__fp"] = fp
        access["targetUrl"] = targetUrl

        response = self.loadPage(
            self.url["base"], self.url["access"], "POST", access, token
        )

        if response.status != 302:
            logging.error(
                "Unexpected response status on allowing " "access 302 != %s",
                response.status,
            )
            logging.error(response.data)
            tools.exitErr()

        responseData = self.parseResponse(response.location)
        if "oauth_verifier" not in responseData:
            logging.error("OAuth verifier not found")
            tools.exitErr()

        self.verifierToken = responseData["oauth_verifier"]

        logging.debug("OAuth verifier token take")
Esempio n. 44
0
def export_basic_inf(data_list, workbook, is_exsit):
    start_row = 1  # 数据起始excel行号
    if not is_exsit:
        worksheet = workbook.add_sheet(basic_inf_sheet_name,
                                       cell_overwrite_ok=True)
        worksheet.write(0, 0, "序号", style_title)
        worksheet.write(0, 1, "公司名称", style_title)
        worksheet.write(0, 2, "法定代表人", style_title)
        worksheet.write(0, 3, "注册资本", style_title)
        worksheet.write(0, 4, "实缴资本", style_title)
        worksheet.write(0, 5, "经营状态", style_title)
        worksheet.write(0, 6, "成立日期", style_title)
        worksheet.write(0, 7, "统一社会信用代码", style_title)
        worksheet.write(0, 8, "纳税人识别号", style_title)
        worksheet.write(0, 9, "注册号", style_title)
        worksheet.write(0, 10, "组织机构代码", style_title)
        worksheet.write(0, 11, "企业类型", style_title)
        worksheet.write(0, 12, "所属行业", style_title)
        worksheet.write(0, 13, "人员规模", style_title)
        worksheet.write(0, 14, "营业期限", style_title)
        worksheet.write(0, 15, "企业地址", style_title)
        worksheet.write(0, 16, "经营范围", style_title)

        # worksheet.write(0, 3, "电话", style_title)
        # worksheet.write(0, 4, "地址", style_title)
        # worksheet.write(0, 5, "邮箱", style_title)
        # worksheet.write(0, 6, "注册号", style_title)
        # worksheet.write(0, 7, "统一社会信用代码", style_title)
        # worksheet.write(0, 8, "注册资本", style_title)
        # worksheet.write(0, 9, "成立日期", style_title)
        # worksheet.write(0, 10, "企业类型", style_title)
        # worksheet.write(0, 11, "经营范围", style_title)
        # worksheet.write(0, 12, "公司住所", style_title)
        # worksheet.write(0, 13, "营业期限", style_title)
        # worksheet.write(0, 14, "企业状态", style_title)
        # 设置表格列宽
        for i in range(17):
            if i == 0:
                continue
            elif i == 16:
                worksheet.col(
                    i).width = 256 * 200  # 设置第二列宽度,256为衡量单位,50表示50个字符宽度
            else:
                worksheet.col(i).width = 256 * 20
    else:
        start_row = read_excel_rows(spider_result_file_name,
                                    basic_inf_sheet_name)
        worksheet = workbook.get_sheet(basic_inf_sheet_name)

    # 定义excel表头列表
    basic_info_title = [
        '序号', '公司名称', '法定代表人/法人/负责人/经营者/投资人/执行事务合伙人', '注册资本/开办资金', '实缴资本',
        '经营状态/登记状态', '成立日期', '统一社会信用代码', '纳税人识别号', '注册号', '组织机构代码',
        '企业类型/社会组织类型', '所属行业', '人员规模', '营业期限/证书有效期/有效期', '企业地址/住所/地址',
        '经营范围/业务范围'
    ]

    for _response in data_list:
        worksheet.write(start_row, 0, start_row, style)
        # 公司名称
        company_name = _response.find(class_="content")
        if company_name is not None:
            company_name = company_name.select('h1')[0].text.replace(
                '\n', '').replace(
                    ' ', '') if len(company_name.select('h1')) > 0 else '无'
        print('公司名称:' + company_name)
        worksheet.write(start_row, 1, company_name, style)  # 将信息输入表格

        basic_informatiion_array = _response.select(
            "#Cominfo > table > tbody > tr")  # 符号">"为上一个标签下的直接子标签

        if len(basic_informatiion_array) == 0:
            logging.error('未找到该公司基本信息============{}'.format(company_name))

        for basic_tr in basic_informatiion_array:  # 循环tr
            basic_td_array = [
                data for data in basic_tr.children if data != ' '
            ]
            i = 0
            while i < len(basic_td_array):  # 循环td
                key = basic_td_array[i].text.replace('\n', '').replace(' ', '')
                if len(basic_td_array[i + 1].select('h2')) > 0:
                    value = basic_td_array[i + 1].select('h2')[0].text.replace(
                        '\n', '').replace(' ', '')
                else:
                    value = basic_td_array[i + 1].text.replace(
                        '\n', '').replace(' ',
                                          '').replace('查看地图',
                                                      '').replace('附近企业', '')
                # 查出表格的标题在自定义的basic_info_title中的索引位置,为写入excel的列索引。
                index = [k for k, x in enumerate(basic_info_title) if key in x]
                if len(index) > 0:
                    print(key + ':' + value)
                    worksheet.write(start_row, index[0], value,
                                    style)  # 将信息输入表格
                i += 2
        start_row += 1

        # if len(basic_informatiion_array) > 0:
        #     # 法人
        #     legal_person = '-'
        #     if len(basic_informatiion_array[0].select('td')) > 1:
        #         # legal_person_item = basic_informatiion_array[0].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         # if len(basic_informatiion_array[0].select('td')[1].select('h2')) > 0 and ('法定代表人' in legal_person_item or '经营者' in legal_person_item):
        #         if len(basic_informatiion_array[0].select('td')[1].select('h2')) > 0:
        #             legal_person = basic_informatiion_array[0].select('td')[1].select('h2')[0].text.replace('\n', '').replace(' ', '')
        #     print('法人:' + legal_person)
        #     worksheet.write(start_row, 2, legal_person, style)  # 将信息输入表格
        #     # 注册资本
        #     registration_capital = '-'
        #     if len(basic_informatiion_array[0].select('td')) > 3:
        #         registration_capital_item = basic_informatiion_array[0].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '注册资本' in registration_capital_item:
        #             registration_capital = basic_informatiion_array[0].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('注册资本:' + registration_capital)
        #     worksheet.write(start_row, 3, registration_capital, style)  # 将信息输入表格
        #     # 实缴资本
        #     real_capital = '-'
        #     if len(basic_informatiion_array) > 1 and len(basic_informatiion_array[1].select('td')) > 1:
        #         real_capital_item = basic_informatiion_array[1].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '实缴资本' in real_capital_item:
        #             real_capital = basic_informatiion_array[1].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('实缴资本:' + real_capital)
        #     worksheet.write(start_row, 4, real_capital, style)  # 将信息输入表格
        #     # 经营状态
        #     operating_status = '-'
        #     if len(basic_informatiion_array) > 2 and len(basic_informatiion_array[2].select('td')) > 1:
        #         operating_status_item = basic_informatiion_array[2].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '经营状态' in operating_status_item:
        #             operating_status = basic_informatiion_array[2].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('经营状态:' + operating_status)
        #     worksheet.write(start_row, 5, operating_status, style)  # 将信息输入表格
        #     # 成立日期
        #     establishment_date = '-'
        #     if len(basic_informatiion_array) > 2 and len(basic_informatiion_array[2].select('td')) > 3:
        #         establishment_date_item = basic_informatiion_array[2].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '成立日期' in establishment_date_item:
        #             establishment_date = basic_informatiion_array[2].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('成立日期:' + establishment_date)
        #     worksheet.write(start_row, 6, establishment_date, style)  # 将信息输入表格
        #     # 统一社会信用代码
        #     unified_credit_code = '-'
        #     if len(basic_informatiion_array) > 3 and len(basic_informatiion_array[3].select('td')) > 1:
        #         unified_credit_code_item = basic_informatiion_array[3].select('td')[0].text.replace('\n', '').replace(' ','')
        #         if '统一社会信用代码' in unified_credit_code_item:
        #             unified_credit_code = basic_informatiion_array[3].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('统一社会信用代码:' + unified_credit_code)
        #     worksheet.write(start_row, 7, unified_credit_code, style)  # 将信息输入表格
        #     # 纳税人识别号
        #     taxpayer_identification = '-'
        #     if len(basic_informatiion_array) > 3 and len(basic_informatiion_array[3].select('td')) > 3:
        #         taxpayer_identification_item = basic_informatiion_array[3].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '纳税人识别号' in taxpayer_identification_item:
        #             taxpayer_identification = basic_informatiion_array[3].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('纳税人识别号:' + taxpayer_identification)
        #     worksheet.write(start_row, 8, taxpayer_identification, style)  # 将信息输入表格
        #     # 注册号
        #     registration_number = '-'
        #     if len(basic_informatiion_array) > 4 and len(basic_informatiion_array[4].select('td')) > 1:
        #         registration_number_item = basic_informatiion_array[4].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '注册号' in registration_number_item:
        #             registration_number = basic_informatiion_array[4].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('注册号:' + registration_number)
        #     worksheet.write(start_row, 9, registration_number, style)  # 将信息输入表格
        #     # 组织机构代码
        #     organization_code = '-'
        #     if len(basic_informatiion_array) > 4 and len(basic_informatiion_array[4].select('td')) > 3:
        #         organization_code_item = basic_informatiion_array[4].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '组织机构代码' in organization_code_item:
        #             organization_code = basic_informatiion_array[4].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('组织机构代码:' + organization_code)
        #     worksheet.write(start_row, 10, organization_code, style)  # 将信息输入表格
        #     # 企业类型
        #     company_type = '-'
        #     if len(basic_informatiion_array) > 5 and len(basic_informatiion_array[5].select('td')) > 1:
        #         company_type_item = basic_informatiion_array[5].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '企业类型' in company_type_item:
        #             company_type = basic_informatiion_array[5].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('企业类型:' + company_type)
        #     worksheet.write(start_row, 11, company_type, style)  # 将信息输入表格
        #     # 所属行业
        #     industry = '-'
        #     if len(basic_informatiion_array) > 5 and len(basic_informatiion_array[5].select('td')) > 3:
        #         industry_item = basic_informatiion_array[5].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '所属行业' in industry_item:
        #             industry = basic_informatiion_array[5].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('所属行业:' + industry)
        #     worksheet.write(start_row, 12, industry, style)  # 将信息输入表格
        #     # 人员规模
        #     employees = '-'
        #     if len(basic_informatiion_array) > 9 and len(basic_informatiion_array[9].select('td')) > 1:
        #         employees_item = basic_informatiion_array[9].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '人员规模' in employees_item:
        #             employees = basic_informatiion_array[9].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('人员规模:' + employees)
        #     worksheet.write(start_row, 13, employees, style)  # 将信息输入表格
        #     # 营业期限
        #     business_term = '-'
        #     if len(basic_informatiion_array) > 9 and len(basic_informatiion_array[9].select('td')) > 3:
        #         business_term_item = basic_informatiion_array[9].select('td')[2].text.replace('\n', '').replace(' ', '')
        #         if '营业期限' in business_term_item:
        #             business_term = basic_informatiion_array[9].select('td')[3].text.replace('\n', '').replace(' ', '')
        #     print('营业期限:' + business_term)
        #     worksheet.write(start_row, 14, business_term, style)  # 将信息输入表格
        #     # 企业地址
        #     adress = '-'
        #     if len(basic_informatiion_array) > 10 and len(basic_informatiion_array[10].select('td')) > 1:
        #         adress_item = basic_informatiion_array[10].select('td')[0].text.replace('\n','').replace(' ', '')
        #         if '企业地址' in adress_item:
        #             adress = basic_informatiion_array[10].select('td')[1].text.replace('查看地图', '').replace('附近企业', '').replace('\n', '').replace(' ', '')
        #     print('企业地址:' + adress)
        #     worksheet.write(start_row, 15, adress, style)  # 将信息输入表格
        #     # 经营范围
        #     business_scope = '-'
        #     if len(basic_informatiion_array) > 11 and len(basic_informatiion_array[11].select('td')) > 1:
        #         business_scope_item = basic_informatiion_array[11].select('td')[0].text.replace('\n', '').replace(' ', '')
        #         if '经营范围' in business_scope_item:
        #             business_scope = basic_informatiion_array[11].select('td')[1].text.replace('\n', '').replace(' ', '')
        #     print('经营范围:' + business_scope)
        #     worksheet.write(start_row, 16, business_scope, style)  # 将信息输入表格
        # start_row += 1

        # 公司名称
        # # company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
        # if len(company_name) > 0:
        #     company_name = company_name.strip()
        # else:
        #     company_name = '--'
        # print('公司名称:' + company_name)
        # worksheet.write(start_row, 1, company_name, style)  # 将信息输入表格
        # 法人
        # legal_person = re.findall('<a class="oper" href=".*?">(.*?\s*)</a>', _response)
        # if len(legal_person) > 0:
        #     legal_person = legal_person.strip()
        # else:
        #     legal_person = '--'
        # print('法人:' + legal_person)
        # worksheet.write(start_row, 2, legal_person, style)  # 将信息输入表格
        # 电话
        # telephone = re.findall('<a href="tel:.*?" class="phone a-decoration">(.*?\s*)</a>', _response)
        # if len(telephone) > 0:
        #     telephone = telephone[0].strip()
        # else:
        #     telephone = '--'
        # print('电话:' + telephone)
        # worksheet.write(start_row, 3, telephone, style)  # 将信息输入表格
        # 地址
        # address = re.findall('</div> <div class="address">(\s*.*?\s*)</div> </div>', _response)
        # if len(address) > 0:
        #     address = address[0].strip()
        # else:
        #     address = '--'
        # print('地址:' + address)
        # worksheet.write(start_row, 4, address, style)  # 将信息输入表格
        # 邮箱
        # email = re.findall('<a href="javascript:;" class="email a-decoration">(.*?\s*)</a>', _response)
        # if len(email) > 0:
        #     email = email[0].strip()
        # else:
        #     email = '--'
        # print('邮箱:' + email)
        # worksheet.write(start_row, 5, email, style)  # 将信息输入表格
        # basic_list = re.findall('</div> <div class="basic-item-right">(\s*.*?\s*)</div>', _response)
        # if len(basic_list) > 0:
        #     # 注册号
        #     registration_number = basic_list[0].strip()
        # else:
        #     registration_number = '--'
        # print('注册号:' + registration_number)
        # worksheet.write(start_row, 6, registration_number, style)  # 将信息输入表格
        # if len(basic_list) > 1:
        #     # 统一社会信用代码
        #     credit_ode = basic_list[1].strip()
        # else:
        #     credit_ode = '--'
        # print('统一社会信用代码:' + credit_ode)
        # worksheet.write(start_row, 7, credit_ode, style)  # 将信息输入表格
        # 注册资本
        # if len(basic_list) > 2:
        #     registration_capital = basic_list[2].strip()
        # else:
        #     registration_capital = '--'
        # print('注册资本:' + registration_capital)
        # worksheet.write(start_row, 8, registration_capital, style)  # 将信息输入表格
        # if len(basic_list) > 3:
        #     # 成立日期
        #     registration_date = basic_list[3].strip()
        # else:
        #     registration_date = '--'
        # print('成立日期:' + registration_date)
        # worksheet.write(start_row, 9, registration_date, style)  # 将信息输入表格
        # if len(basic_list) > 4:
        #     # 企业类型
        #     company_type = basic_list[4].strip()
        # else:
        #     company_type = '--'
        # print('企业类型:' + company_type)
        # worksheet.write(start_row, 10, company_type, style)  # 将信息输入表格
        # if len(basic_list) > 5:
        #     # 经营范围
        #     business_scope = basic_list[5].strip()
        # else:
        #     business_scope = '--'
        # print('经营范围:' + business_scope)
        # worksheet.write(start_row, 11, business_scope, style)  # 将信息输入表格
        # if len(basic_list) > 6:
        #     # 公司住所
        #     company_address = basic_list[6].strip()
        # else:
        #     company_address = '--'
        # print('公司住所:' + company_address)
        # worksheet.write(start_row, 12, company_address, style)  # 将信息输入表格
        # if len(basic_list) > 7:
        #     # 营业期限
        #     business_term = basic_list[7].strip()
        # else:
        #     business_term = '--'
        # print('营业期限:' + business_term)
        # worksheet.write(start_row, 13, business_term, style)  # 将信息输入表格
        # if len(basic_list) > 8:
        #     # 企业状态
        #     enterprise_state = basic_list[8].strip()
        # else:
        #     enterprise_state = '--'
        # print('企业状态:' + enterprise_state)
        # worksheet.write(start_row, 14, enterprise_state, style)  # 将信息输入表格
        # start_row += 1
        # print('----------------------------------------------------------------------')
    return worksheet
Esempio n. 45
0
def export_partners(data_list,workbook, is_exsit):
    start_row = 1   # 数据起始excel行号
    order_number = 0    # 数据起始序号
    if not is_exsit:
        worksheet = workbook.add_sheet(partners_sheet_name,cell_overwrite_ok=True)
        worksheet.write(0, 0, u"序号", style_title)
        worksheet.write(0, 1, u"公司名称", style_title)
        worksheet.write(0, 2, u"股东名称", style_title)
        worksheet.write(0, 3, u"持股比例", style_title)
        worksheet.write(0, 4, u"认缴出资额", style_title)
        worksheet.write(0, 5, u"认缴出资日期", style_title)
        worksheet.write(0, 6, u"实缴出资额", style_title)
        worksheet.write(0, 7, u"实缴出资日期", style_title)

        # Setting row height and column width 设置宽和高 xlwt中是行和列都是从0开始计算的
        worksheet.col(1).width = 256 * 50
        worksheet.col(2).width = 256 * 50
        worksheet.col(3).width = 256 * 20
        worksheet.col(4).width = 256 * 30
        worksheet.col(5).width = 256 * 30
        worksheet.col(6).width = 256 * 30
        worksheet.col(7).width = 256 * 30
    else:
        start_row = read_excel_rows(spider_result_file_name, partners_sheet_name)
        worksheet = workbook.get_sheet(partners_sheet_name)
        order_number = int(get_merged_cells_value(spider_result_file_name,partners_sheet_name, start_row-1, 0))  # 序号位置是第一列

    # 定义excel表头列表
    partner_title = ['序号', '公司名称', '股东名称', '持股比例', '认缴出资额', '认缴出资日期', '实缴出资额', '实缴出资日期']

    for _response in data_list:
        order_number += 1
        # 公司名称
        company_name = _response.find(class_="content")
        if company_name is not None:
            company_name = company_name.select('h1')[0].text.replace('\n', '').replace(' ', '') if len(company_name.select('h1')) > 0 else '无'
        print('公司名称:' + company_name)

        partner_array = _response.select("#partnerslist > table > tbody > tr")
        if len(partner_array) == 0:
            if len(_response.select("#partnerslist > div")) > 1 and len(_response.select("#partnerslist > div")[1].select('table')) > 0:
                partner_array =_response.select("#partnerslist > div")[1].select('table')[0].select('tbody > tr')

        if len(partner_array)-1 > 0:
            worksheet.write_merge(start_row, start_row + len(partner_array)-2, 0, 0, order_number, style_merge)   # 序号
            worksheet.write_merge(start_row, start_row + len(partner_array)-2, 1, 1, company_name, style_merge)  # 合并公司名称单元格
        else:
            worksheet.write(start_row, 0, order_number, style)
            worksheet.write(start_row, 1, company_name, style)
            worksheet.write(start_row, 2, '--', style)
            worksheet.write(start_row, 3, '--', style)
            worksheet.write(start_row, 4, '--', style)
            worksheet.write(start_row, 5, '--', style)
            worksheet.write(start_row, 6, '--', style)
            worksheet.write(start_row, 7, '--', style)
            start_row += 1

        if len(partner_array) == 0:
            logging.error('未找到该公司股东信息============{}'.format(company_name))

        if len(partner_array) > 0:
            partner_th = [data for data in partner_array[0].children if data != ' '][1:]  # 股东表格的标题行th,去除序号th
            partner_tbale_th = [th.text.replace('\n', '').replace(' ', '') for th in partner_th]
            # print('表格标题行=========={}'.format(partner_tbale_th))

        if len(partner_array) > 1:
            for i in range(1, len(partner_array)):
                partner_row = [data for data in partner_array[i].children if data != ' '][1:]   # 数据行,排除序号列td
                for j in range(len(partner_row)):
                    # 去除th标题中的单位,否则自定义的partner_title中查询不到。列如:认缴出资额(万元)--》认缴出资额。
                    title = partner_tbale_th[j].split('(')[0] if '(' in partner_tbale_th[j] else partner_tbale_th[j]
                    unit = (partner_tbale_th[j].split('(')[1] if '(' in partner_tbale_th[j] else '').replace(')', '')
                    # 查出股东表格的标题行在自定义的partner_title中的索引位置,为写入excel的列索引。
                    index = [k for k,x in enumerate(partner_title) if title in x]
                    if len(index) > 0:
                        v = partner_row[j].text.replace('\n', '').replace(' ', '').replace('持股详情>', '')
                        if '-' != v:
                            v += unit
                        print(partner_title[index[0]]+':'+v)
                        worksheet.write(start_row, index[0], v, style)  # 将信息输入表格
                    elif len(partner_row[j].select('h3')) > 0:
                        partner_name = partner_row[j].select('h3')[0].text.replace('\n', '').replace(' ', '')
                        print('股东名称:' + partner_name)
                        worksheet.write(start_row, 2, partner_name, style)  # 将信息输入表格
                start_row += 1

            # #股东及出资信息
            # partner_name = '-'
            # if len(partner_array[i].select('td')) > 1 and len(partner_array[i].select('td')[1].select('h3')) > 0:
            #     partner_name = partner_array[i].select('td')[1].select('h3')[0].text.replace('\n', '').replace(' ', '')
            # worksheet.write(start_row, 2, partner_name, style)  # 将信息输入表格
            # print('股东名称:' + partner_name)
            #
            # #持股比例
            # stock_rate= partner_array[i].select('td')[4].text.replace('\n', '').replace(' ', '')
            # if len(partner) == 10 and partner[4].text[0:5] == '持股比例':
            #     stock_rate = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '')
            # if '%' in stock_rate:
            #     stock_rate = stock_rate.split('%')[0]+'%'
            # worksheet.write(start_row, 3, stock_rate, style)  # 将信息输入表格
            # print('持股比例:' + stock_rate)
            #
            # # 认缴出资额(万元)
            # if len(partner) == 5 and partner[3].text[0:5] =='认缴出资额':
            #     money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if money != '-':
            #             money =money+partner[3].text[5:]
            # elif len(partner) == 6 and partner[3].text[0:5] =='认缴出资额':
            #     money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if money != '-':
            #             money =money+partner[3].text[5:]
            # elif len(partner) == 7 and partner[3].text[0:5] =='认缴出资额':
            #     money = partner_array[i].select('td')[5].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if money != '-':
            #         money = money + partner[4].text[5:]
            # elif len(partner) == 8 and partner[4].text[0:5] =='认缴出资额':
            #     money = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if money != '-':
            #         money = money + partner[4].text[5:]
            # elif len(partner) == 10 and partner[5].text[0:5] =='认缴出资额':
            #     money = partner_array[i].select('td')[9].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if money != '-':
            #         money = money + partner[5].text[5:]
            # else:
            #     money = '--'
            # worksheet.write(start_row, 4, money, style)  # 将信息输入表格
            # print('认缴出资额:' + money)
            #
            # #认缴出资日期
            # if len(partner) == 5 and partner[3].text[0:6] =='认缴出资日期':
            #     time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            # elif len(partner) == 6 and partner[4].text[0:6] =='认缴出资日期':
            #     time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            # elif len(partner) == 7 and partner[3].text[0:6] =='认缴出资日期':
            #     time = partner_array[i].select('td')[6].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            # elif len(partner) == 8 and partner[3].text[0:6] =='认缴出资日期':
            #     time = partner_array[i].select('td')[7].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            # elif len(partner) == 10 and partner[6].text[0:6] =='认缴出资日期':
            #     time = partner_array[i].select('td')[10].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            # else:
            #     time = '--'
            # worksheet.write(start_row, 5, time, style)  # 将信息输入表格
            # print('认缴出资日期:' + time)
            #
            # #实缴出资额(万元)
            # if len(partner) == 7 and partner[5].text[0:5] =='实缴出资额':
            #     real_money = partner_array[i].select('td')[7].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if real_money != '-':
            #         real_money=real_money+partner[5].text[5:]
            # elif len(partner) == 8 and partner[6].text[0:5] =='实缴出资额':
            #     real_money = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if real_money != '-':
            #         real_money=real_money+partner[6].text[5:]
            # elif len(partner) == 10 and partner[7].text[0:5] =='实缴出资额':
            #     real_money = partner_array[i].select('td')[11].text.replace('\n', '').replace(' ', '').replace('<br>', '')
            #     if real_money != '-':
            #         real_money=real_money+partner[7].text[5:]
            # else:
            #     real_money = '--'
            # worksheet.write(start_row, 6, real_money, style)  # 将信息输入表格
            # print('实缴出资额:' + real_money)
            #
            # #实缴出资日期
            # if len(partner) == 7  and partner[6].text =='实缴出资日期' :
            #     real_time = partner_array[i].select('td')[8].text.replace('\n', '').replace(' ', '').replace('<br>','')
            # elif len(partner) == 8 and partner[7].text =='实缴出资日期':
            #     real_time = partner_array[i].select('td')[9].text.replace('\n', '').replace(' ', '').replace('<br>','')
            # elif len(partner) == 10 and partner[8].text =='实缴出资日期':
            #     real_time = partner_array[i].select('td')[12].text.replace('\n', '').replace(' ', '').replace('<br>','')
            # else:
            #     real_time = '--'
            # worksheet.write(start_row, 7, real_time, style)  # 将信息输入表格
            # print('实缴出日期:' + real_time)
            # start_row += 1

            # 公司名称
            # company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
            # if len(company_name) > 0:
            #     company_name = company_name[0].strip()
            # else:
            #     company_name = '--'
            # print('公司名称:' + company_name)
            # print(_response)
            # list = re.findall('<div class="stock-item">(.*?\s*\n*.*?\s*\n*.*?\s*\n*)</div>\n?\s*</div>\n?\s*</div>\n?\s*</div>', _response)
            # for partner in list:
            # print(partner)
            # partner_list = re.findall('<div class="stock-text">(.*?\s*)<', partner)
            # #股东名称
            # partner_name = re.findall(' <div class="stock-title"> <span > <a class="text-blue a-decoration"( href=".*?")? >(.*?\s*)</a>', partner)
            # if len(partner_name) > 0:
            #     partner_name = partner_name[0][1].strip()
            # else:
            #     partner_name = '--'
            # print('股东名称:' + partner_name)
            # worksheet.write(start_row, 2, partner_name,style)  # 将信息输入表格
            # if len(partner_list) > 0:
            #     #持股比例
            #     stock_rate = partner_list[0].strip()
            # else:
            #     stock_rate = '--'
            # print('持股比例:' + stock_rate)
            # worksheet.write(start_row, 3, stock_rate,style)  # 将信息输入表格
            # if len(partner_list) > 1:
            #     #股东类型
            #     stock_type = partner_list[1].strip()
            # else:
            #     stock_type = '--'
            # print('股东类型:' + stock_type)
            # worksheet.write(start_row, 4, stock_type,style)  # 将信息输入表格
            # if len(partner_list) > 2:
            #     #认缴出资额(万元)
            #     money = partner_list[2].strip()
            # else:
            #     money = '--'
            # print('认缴出资额(万元):' + money)
            # worksheet.write(start_row, 5, money,style)  # 将信息输入表格
            # if len(partner_list) > 3:
            #     #认缴出资日期
            #     time = partner_list[3].strip()
            # else:
            #     time = '--'
            # print('认缴出资日期:' + time)
            # worksheet.write(start_row, 6, time,style)  # 将信息输入表格
            # start_row += 1
        # print('----------------------------------------------------------------------')
    return worksheet
Esempio n. 46
0
                 #         response1 = retry_crawl(url, is_proxy)
                 # else:
                 #     try:
                 #         response1 = requests.get(url, headers=get_headers(), timeout=spider_timeout)
                 #     except Exception as e:
                 #         response1 = retry_crawl(url, is_proxy)
                 # _response1 = response1.text
                 # 此处使用selenium爬取页面,解决页面返回没有数据的问题
                 _response1 = selenium_crawl(url, is_proxy)
                 if _response1 is not None:
                     _soup = BeautifulSoup(_response1, 'lxml')
                 # print(_soup)
                 data_list.append(_soup)
                 print("{}=============抓取成功!".format(name))
             except Exception as e:
                 logging.error(name + '=========================抓取该公司的信息异常')
                 error_data_list.append(name)
                 logging.error(e)
                 # print(str(e))
                 # logging.exception(e)
             # 导出excel
             if len(data_list) > 0 or len(error_data_list) > 0:
                 print(
                     '==================正在写入excel文件,请勿关闭程序!=================='
                 )
                 export_excel(data_list, error_data_list)
             time.sleep(
                 random.randint(crawl_interval_mintime,
                                crawl_interval_maxtime))  # 每隔5到20秒
 else:
     print('====================本程序只能在外网环境下运行====================')
Esempio n. 47
0
    if result == 0:
        return True
    os.system('ps -ef | grep "./neo-cli" | awk \'{print $2}\' | xargs kill')
    return True


def restartRecently():
    if timedelta(minutes=START_SILENT) < datetime.now() - lastRestartTimestamp:
        return True
    return False


while True:
    if not isLocalRunning():
        startLocalNode()
        continue
    time.sleep(INTERVAL)
    localBlockCount = getLocalBlockCount()
    bestBlockCount = getBestBlockCount()
    if localBlockCount < 0 or bestBlockCount < 0:
        logging.error(
            '[wrongheight] wrong height, localheight: {0}, bestheight: {1}'.
            format(localBlockCount, bestBlockCount))
        continue
    if RESTART_THRESHOLD < bestBlockCount - localBlockCount and not restartRecently(
    ):
        restart_cnt += 1
        logging.warning(
            '[restart] restarting, restart_cnt: {0}, localheight: {1}, bestheight: {2}'
            .format(restart_cnt, localBlockCount, bestBlockOount))
        stopLocalNode()
Esempio n. 48
0
def export_excel(data, error_data):
    if check_file(spider_result_file_name):
        # 读取文件的时候需要将formatting_info参数设置为True,默认是False;保留样式,不然获取合并的单元格数组为空
        old_workbook = xlrd.open_workbook(spider_result_file_name,
                                          formatting_info=True)
        workbook = copy(old_workbook)  # 拷贝一份原来的excel
    else:
        # 创建输出excel文件
        workbook = xlwt.Workbook(encoding="utf-8")
    # 导出企业基本信息
    if len(data) > 0:
        print(
            "==============================企业基本信息=============================="
        )
        try:
            export_basic_inf(
                data, workbook,
                check_sheet_exsit(spider_result_file_name,
                                  basic_inf_sheet_name))
        except Exception as e:
            logging.error('===================写入企业基本信息异常===================')
            # logging.exception(e)
            logging.error(e)
        # 导出企业股东信息
        print(
            "==============================企业股东信息=============================="
        )
        try:
            export_partners(
                data, workbook,
                check_sheet_exsit(spider_result_file_name,
                                  partners_sheet_name))
        except Exception as e:
            logging.error('===================写入企业股东信息异常===================')
            # logging.exception(e)
            logging.error(e)
        # 导出企业主要人员s
        print(
            "==============================企业主要人员=============================="
        )
        try:
            export_key_personnel(
                data, workbook,
                check_sheet_exsit(spider_result_file_name,
                                  key_personnel_sheet_name))
        except Exception as e:
            logging.error('===================写入企业主要人员异常===================')
            # logging.exception(e)
            logging.error(e)
    # 导出抓取失败企业名称
    if len(error_data) > 0:
        print(
            "==============================写入抓取失败企业=============================="
        )
        try:
            export_error_data(
                error_data, workbook,
                check_sheet_exsit(spider_result_file_name,
                                  error_data_sheet_name))
        except Exception as e:
            logging.error('===================写入抓取失败企业异常===================')
            logging.error(e)
            # logging.exception(e)
    workbook.save(spider_result_file_name)
Esempio n. 49
0
    def parse(self):
        self.INP_DATA = {}

        if self.CMD is None:
            out.printAbout()
            return False

        if self.CMD == "autocomplete":
            # substitute arguments for AutoComplete
            # 1 offset to make the argument as 1 is autocomplete
            self.__init__(self.sys_argv[1:])
            self.printAutocomplete()
            return False

        if self.CMD == "--help":
            self.printHelp()
            return False

        if self.CMD not in self.COMMANDS:
            self.printErrorCommand()
            return False

        if "--help" in self.INP:
            self.printHelp()
            return False

        # prepare data
        for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
            # set values by default
            if 'default' in params:
                self.INP_DATA[arg] = params['default']

            # replace `altName` entered arguments on full
            if 'altName' in params and params['altName'] in self.INP:
                self.INP[self.INP.index(params['altName'])] = arg

        activeArg = None
        ACTIVE_CMD = None
        # check and insert first argument by default
        if 'firstArg' in self.COMMANDS[self.CMD]:
            firstArg = self.COMMANDS[self.CMD]['firstArg']
            if len(self.INP) > 0:
                # Check that first argument is a default argument
                # and another argument.
                if self.INP[0] not in (self.CMD_ARGS.keys() +
                                       self.CMD_FLAGS.keys()):
                    self.INP = [
                        firstArg,
                    ] + self.INP
            else:
                self.INP = [
                    firstArg,
                ]

        for item in self.INP:
            # check what are waiting the argument
            if activeArg is None:
                # actions for the argument
                if item in self.CMD_ARGS:
                    activeArg = item
                    ACTIVE_CMD = self.CMD_ARGS[activeArg]

                # actions for the flag
                elif item in self.CMD_FLAGS:
                    self.INP_DATA[item] = self.CMD_FLAGS[item]["value"]

                # error. parameter is not found
                else:
                    self.printErrorArgument(item)
                    return False

            else:
                activeArgTmp = None
                # values it is parameter
                if item in self.CMD_ARGS or item in self.CMD_FLAGS:
                    # active argument is "emptyValue"
                    if "emptyValue" in ACTIVE_CMD:
                        activeArgTmp = item  # remember the new "active" argument
                        item = ACTIVE_CMD[
                            'emptyValue']  # set the active atgument to emptyValue
                    # Error, "active" argument has no values
                    else:
                        self.printErrorArgument(activeArg, item)
                        return False

                if 'type' in ACTIVE_CMD:
                    convType = ACTIVE_CMD['type']
                    if convType not in (int, str):
                        logging.error("Unsupported argument type: %s",
                                      convType)
                        return False

                    try:
                        item = convType(item)
                    except:
                        self.printErrorArgument(activeArg, item)
                        return False

                self.INP_DATA[activeArg] = item
                activeArg = activeArgTmp  # this is either a new "active" argument or emptyValue.

        # if there are still active arguments
        if activeArg is not None:
            # if the active argument is emptyValue
            if 'emptyValue' in ACTIVE_CMD:
                self.INP_DATA[activeArg] = ACTIVE_CMD['emptyValue']

            # An error argument
            else:
                self.printErrorArgument(activeArg, "")
                return False

        # check whether there is a necessary argument request
        for arg, params in (self.CMD_ARGS.items() + self.CMD_FLAGS.items()):
            if 'required' in params and arg not in self.INP:
                self.printErrorReqArgument(arg)
                return False

        # trim -- and ->_
        self.INP_DATA = dict([key.lstrip("-").replace("-", "_"), val]
                             for key, val in self.INP_DATA.items())
        return self.INP_DATA
Esempio n. 50
0
def export_key_personnel(data_list, workbook, is_exsit):
    start_row = 1  # 数据起始excel行号
    order_number = 0  # 数据起始序号
    if not is_exsit:
        worksheet = workbook.add_sheet(key_personnel_sheet_name,
                                       cell_overwrite_ok=True)
        worksheet.write(0, 0, "序号", style_title)
        worksheet.write(0, 1, "公司名称", style_title)
        worksheet.write(0, 2, "姓名", style_title)
        worksheet.write(0, 3, "职务", style_title)
        # 设置表格列宽
        for i in range(4):
            if i == 0:
                continue
            else:
                worksheet.col(i).width = 256 * 20
    else:
        start_row = read_excel_rows(spider_result_file_name,
                                    key_personnel_sheet_name)
        worksheet = workbook.get_sheet(key_personnel_sheet_name)
        order_number = int(
            get_merged_cells_value(spider_result_file_name,
                                   key_personnel_sheet_name, start_row - 1,
                                   0))  # 序号位置是第一列

    for _response in data_list:
        order_number += 1
        # 公司名称
        company_name = _response.find(class_="content")
        if company_name is not None:
            company_name = company_name.select('h1')[0].text.replace(
                '\n', '').replace(
                    ' ', '') if len(company_name.select('h1')) > 0 else '无'
        print('公司名称:' + company_name)
        personnel_array = _response.select(
            "#Mainmember > table > tbody > tr")  # 包含了标题tr
        # print(personnel_array)
        if len(personnel_array) - 1 > 0:
            worksheet.write_merge(start_row,
                                  start_row + len(personnel_array) - 2, 0, 0,
                                  order_number, style_merge)  # 合并序号单元格
            worksheet.write_merge(start_row,
                                  start_row + len(personnel_array) - 2, 1, 1,
                                  company_name, style_merge)  # 合并公司名称单元格
        else:
            worksheet.write(start_row, 0, order_number, style)
            worksheet.write(start_row, 1, company_name, style)
            worksheet.write(start_row, 2, '--', style)
            worksheet.write(start_row, 3, '--', style)
            start_row += 1

        if len(personnel_array) == 0:
            logging.error('未找到该公司主要人员============{}'.format(company_name))

        if len(personnel_array) > 1:
            for i in range(1, len(personnel_array)):
                # 姓名
                if len(personnel_array[i].select('td')) > 1 and len(
                        personnel_array[i].select('td')[1].select('h3')) > 0:
                    name = personnel_array[i].select('td')[1].select(
                        'h3')[0].text.replace('\n', '').replace(' ', '')
                print('姓名:' + name)
                worksheet.write(start_row, 2, name, style)  # 将信息输入表格
                # 职务
                if len(personnel_array[i].select('td')) > 2:
                    job = personnel_array[i].select('td')[2].text.replace(
                        '\n', '').replace(' ', '')
                print('职务:' + job)
                worksheet.write(start_row, 3, job, style)  # 将信息输入表格
                start_row += 1

        # order_number += 1
        # 公司名称
        # company_name = re.findall('<div class="company-name">(.*?\s*)<', _response)
        # if len(company_name) > 0:
        #     company_name = company_name[0].strip()
        # else:
        #     company_name = '--'
        # print('公司名称:' + company_name)
        # personnel_list = re.findall('<a class="employee-item" href=".*?" style="display: block;">(\s*.*?\s*)</a>', _response)
        # # print(personnel_list)
        # if len(personnel_list) > 0:
        #     worksheet.write_merge(start_row, start_row + len(personnel_list)-1, 0, 0, order_number, style_merge)  # 合并序号单元格
        #     worksheet.write_merge(start_row, start_row + len(personnel_list)-1, 1, 1, company_name, style_merge)  # 合并公司名称单元格
        # else:
        #     worksheet.write(start_row, 0, order_number, style)
        #     worksheet.write(start_row, 1, company_name, style)
        #     worksheet.write(start_row, 2, '--', style)
        #     worksheet.write(start_row, 3, '--', style)
        #     start_row += 1
        #
        # for personnel in personnel_list:
        #     # 姓名
        #     name = re.findall('<div class="employee-name">(\s*.*?\s*)</div>', personnel)
        #     if len(name) > 0:
        #         name = name[0].strip()
        #     else:
        #         name = '--'
        #     print('姓名:' + name)
        #     worksheet.write(start_row, 2, name, style)  # 将信息输入表格
        #     # 职务
        #     job = re.findall('<div class="employee-job">(\s*.*?\s*)</div>', personnel)
        #     if len(job) > 0:
        #         job = job[0].strip()
        #     else:
        #         job = '--'
        #     print('职务:' + job)
        #     worksheet.write(start_row, 3, job, style)  # 将信息输入表格
        #     start_row += 1
        # print('----------------------------------------------------------------------')
    return worksheet
Esempio n. 51
0
def genDailyReportSimplification(startTs, endTs):
    br = '<br/>'
    responseData = initResponse()
    result = {}
    result['status'] = ''
    responseStr = ''
    # list of data collector
    listOfCollector = ['A01', 'A03', 'A05', 'A06']

    validStartTime = transferDatetime(startTs)
    validEndTime = transferDatetime(endTs)

    responseStr = responseStr + "[數據採集回報 " + validStartTime.strftime("%Y/%m/%d %H:%M:%S") \
                  + " - " + validEndTime.strftime("%Y/%m/%d %H:%M:%S") + "]" + br

    try:
        for machineId in listOfCollector:
            coreName = 'spc'
            spcRows = getTotalRowsBetweenTwoTimestamp(validStartTime,
                                                      validEndTime, 10, 1)
            response = queryInfoBetweenTimestampSPC0(coreName, machineId,
                                                     validStartTime,
                                                     validEndTime, spcRows)
            doc = praseJsonFormat(response)
            df = pd.DataFrame(doc)

            responseStr = responseStr + machineId + br
            # query燈號狀態
            lightResponse = queryLightStatus(machineId)
            machineStatus = parseLightStatus(lightResponse)

            if df.empty:
                logging.info(coreName + "/" + machineId + ":" + startTs + '-' +
                             validEndTime.strftime("%Y/%m/%d %H:%M:%S") +
                             ' DataFrame is empty!')
                responseStr = responseStr + insertTextIndent(
                    '目前機台狀態:' + machineStatus + ',' + '此區間沒有生產數據', '20', '1')
            else:
                df.loc[:, 'SPC_0_previous'] = df['SPC_0']
                # shift 是往下移一個
                df['SPC_0_previous'] = df['SPC_0_previous'].shift(1).fillna(
                    '-1').astype(int)
                df.loc[:, 'timestamp_previous'] = df['timestamp'].shift(
                    1).fillna(-1).astype(float)

                # for顯示使用
                df.loc[:, 'timestamp_view_temp'] = df[
                    'timestamp_iso'].str.replace('T', ' ').str.replace(
                        'Z', '').astype(str)
                df.loc[:, 'timestamp_view'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%H:%M:%S')
                # last
                df.loc[:, 'timestamp_view_last'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%m/%d %H:%M:%S')
                # date
                df.loc[:, 'timestamp_view_date'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%m/%d')
                df.loc[:, 'timestamp_view_date_previous'] = df[
                    'timestamp_view_date'].shift(1).fillna('-1')

                df.loc[:,
                       'timestamp_view_previous'] = df['timestamp_view'].shift(
                           1).fillna('-1')

                # 最新模次
                df_lastest = df.iloc[-1:].astype(str)
                lastTime = df_lastest['timestamp'].values[0]
                lastTimeStr = df_lastest['timestamp_view_last'].values[0]
                lastSPC = df_lastest['SPC_0'].values[
                    0] + '(' + lastTimeStr + ')'

                responseStr = responseStr + insertTextIndent(
                    '目前機台狀態:' + machineStatus, '20', '1')
                responseStr = responseStr + insertTextIndent(
                    lastSPC, '50', '1')

                # 生產模次間隔超過五分鐘
                df['timestamp_previous_addInterval'] = df[
                    'timestamp_previous'] + int(PRODUCTION_DURING_SEC)
                df_broken_timestamp = df[df['timestamp'].astype(float) >
                                         df['timestamp_previous_addInterval']]
                if df_broken_timestamp.size > 1:
                    filter_df = df_broken_timestamp.iloc[1:]
                    times_df = filter_df.groupby([
                        "timestamp_view_date_previous"
                    ]).size().to_frame(name='count')
                    if times_df.size > 0:
                        responseStr = responseStr + insertTextIndent(
                            '生產模次間隔超過五分鐘:', '20', '1')
                        for index, row in times_df.iterrows():
                            responseStr = responseStr + insertTextIndent(
                                index + ' 發生 ' + str(row['count']) + ' 次',
                                '50', '1')

            responseStr = responseStr + br
        return responseStr
    except:
        logging.error("genDailyReport exception")
        responseData = setErrorResponse(responseData, '400', 'exception')
        return parseResponseJson(responseData, result, 'result')
Esempio n. 52
0
    def textToENML(content, raise_ex=False, format='markdown'):
        """
        Create an ENML format of note.
        """
        if not isinstance(content, str):
            content = ""
        try:
            content = unicode(content, "utf-8")
            # add 2 space before new line in paragraph for creating br tags
            # content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1  \n\3', content)
            content = re.sub(r'\r\n', '\n', content)

            if format == 'markdown':
                storage = Storage()
                extras = storage.getUserprop('markdown2_extras')

                content = Editor.HTMLEscapeTag(content)
                contentHTML = markdown.markdown(content, extras=extras)

                soup = BeautifulSoup(contentHTML, 'html.parser')
                Editor.checklistInSoupToENML(soup)

                # Non-Pretty HTML output
                contentHTML = str(soup)
            elif format == 'pre':
                #
                # For the 'pre' format, simply wrap the content with a 'pre' tag. Do
                # perform any parsing/mutation.
                #
                contentHTML = u''.join(
                    ('<pre>', content, '</pre>')).encode("utf-8")
            elif format == 'html':
                # Html to ENML http://dev.evernote.com/doc/articles/enml.php
                ATTR_2_REMOVE = [
                    "id",
                    "class",
                    # "on*",
                    "accesskey",
                    "data",
                    "dynsrc",
                    "tabindex"
                ]
                soup = BeautifulSoup(content, 'html.parser')

                for tag in soup.findAll():
                    if hasattr(tag, 'attrs'):
                        map(lambda x: tag.attrs.pop(x, None), [
                            k for k in tag.attrs.keys()
                            if k in ATTR_2_REMOVE or k.find('on') == 0
                        ])
                contentHTML = str(soup)
            else:
                contentHTML = Editor.HTMLEscape(content)

                tmpstr = ''
                for l in contentHTML.split('\n'):
                    if l == '':
                        tmpstr = tmpstr + u'<div><br/></div>'
                    else:
                        tmpstr = tmpstr + u'<div>' + l + u'</div>'

                contentHTML = tmpstr.encode("utf-8")

            contentHTML = contentHTML.replace(
                '[x]', '<en-todo checked="true"></en-todo>')
            contentHTML = contentHTML.replace('[ ]', '<en-todo></en-todo>')

            return Editor.wrapENML(contentHTML)
        except:
            import traceback
            traceback.print_exc()
            if raise_ex:
                raise Exception("Error while parsing text to html."
                                " Content must be an UTF-8 encode.")
            logging.error("Error while parsing text to html. "
                          "Content must be an UTF-8 encode.")
            out.failureMessage("Error while parsing text to html. "
                               "Content must be an UTF-8 encode.")
            return tools.exitErr()
Esempio n. 53
0
def queryLightStatus(data_collector, lightVersion, isInsert):
    responseData = initResponse()
    result = {}
    result['status'] = ''
    pingStatus = ''
    connectionStatus = ''
    machineStatus = ''
    # 紀錄lightstatus到solr
    lightstatusData = {}

    # dashboard light color
    if lightVersion == 'dashboard':
        pingFail = "gray"
        iconDisconnected = "icon_disconnected"
        iconIdle = "icon_idle"
        auto = "green"
        halfAutoOrManual = "yellow"
    # oee light color
    else:
        pingFail = "red"
        iconDisconnected = "gray"
        iconIdle = "yellow"
        auto = "green"
        halfAutoOrManual = "green"

    try:
        lightstatusData['machine_id'] = data_collector
        nowDatetime = datetime.now()
        nowTimestamp = int(nowDatetime.timestamp())
        lightstatusData['timestamp'] = nowTimestamp
        lightstatusData['id'] = data_collector + str(nowTimestamp)
        lightstatusData['timestamp_iso'] = nowDatetime.strftime(
            "%Y-%m-%dT%H:%M:%SZ")

        #list of data collector
        listOfCollector = ['A01', 'A03', 'A05', 'A06']
        if data_collector not in listOfCollector:
            result['lightColor'] = 'none'
            result['status'] = "no match machine id"
            responseData = setErrorResponse(responseData, '404',
                                            'collector machine_id not found')
            return parseResponseJson(responseData, result, 'result')

        #先判斷是否ping的到
        pingStatus = checkPing(data_collector)
        if pingStatus == "False":
            result['lightColor'] = pingFail
            result['status'] = result['status'] + "ping=fail; 請檢查控制器或網路"
            lightstatusData['light_color'] = pingFail
            if isInsert == True:
                addLightStatusData('lightstatus', data_collector,
                                   [lightstatusData])
            return parseResponseJson(responseData, result, 'result')
        else:
            result['status'] = result['status'] + "ping=success; "

        #再判斷是否連結的到collector
        connectionStatus = checkConnect(data_collector)
        if connectionStatus == "False":
            result['lightColor'] = iconDisconnected
            result['status'] = result['status'] + "connection=fail; 請檢查控制器"
            lightstatusData['light_color'] = iconDisconnected
            if isInsert == True:
                addLightStatusData('lightstatus', data_collector,
                                   [lightstatusData])
            return parseResponseJson(responseData, result, 'result')
        else:
            result['status'] = result['status'] + "connection=success; "

        # query machine status
        jsonObject = {}
        jsonObject = queryMachinestate('machinestatus')
        data = praseJsonFormat(jsonObject)
        if len(data) > 0:
            machineStatus = data[0]['MachineState']
            result['status'] = result[
                'status'] + "machineStatus=" + machineStatus + "; "
        else:
            result['status'] = result['status'] + "machineStatus=none(no data)"

        if machineStatus == "全自動":
            # 全自動才判斷生產狀態
            # query spc status
            jsonObject = {}
            jsonObject = queryLastSPC("spc", data_collector)
            data = praseJsonFormat(jsonObject)
            if len(data) > 0:
                spc_0 = data[0]['SPC_0']
                result['status'] = result['status'] + "spc_0=" + str(
                    spc_0) + "; "
                timestamp_iso = data[0]['timestamp_iso']
                timestamp = data[0]['timestamp']
                result['status'] = result[
                    'status'] + "timestamp=" + timestamp_iso.replace(
                        'T', ' ').replace('Z', '')
                now = datetime.now().timestamp()
                if float(timestamp
                         ) + PRODUCTION_DURING_SEC < now:  # 超過5分鐘沒有生產 -> 待機中
                    result['lightColor'] = iconIdle
                    lightstatusData['light_color'] = iconIdle
                else:
                    result['lightColor'] = auto
                    lightstatusData['light_color'] = auto
            else:
                result['status'] = result['status'] + "spc_0=none(no data)"
        else:  #手動/半自動
            result['lightColor'] = halfAutoOrManual
            lightstatusData['light_color'] = halfAutoOrManual
        if isInsert == True:
            addLightStatusData('lightstatus', data_collector,
                               [lightstatusData])
        if lightVersion == 'oee':
            # add machine status for TPM APP UI
            if result['lightColor'] == auto or result[
                    'lightColor'] == halfAutoOrManual:
                result['lightStatus'] = 'run'
            elif result['lightColor'] == pingFail:
                result['lightStatus'] = 'down'
            elif result['lightColor'] == iconDisconnected:
                result['lightStatus'] = 'issued'
            elif result['lightColor'] == iconIdle:
                result['lightStatus'] = 'idle'
        return parseResponseJson(responseData, result, 'result')
    except:
        logging.error("queryLightStatus exception")
        setErrorResponse(responseData, '400', 'exception')
        return parseResponseJson(responseData, result, 'result')
Esempio n. 54
0
def write_into_influxdb(host_dict):
    """
    Wirte data into influxdb.

    :param host_dict (dict) - as blew
        {
            'name': 'instance-00000163',
            'uuid': 'dbbc272d-60b0-4380-9d23-4c011896e7f4',
            'region': 'wuxi-test-compute144',
            'client': influxdb_client
        }
    """
    host_name = host_dict['name']
    host_uuid = host_dict['uuid']
    region = host_dict['region']
    client = host_dict['client']
    db_list = []

    logging.debug("begin to collect %s data." % host_name)
    begin_time = time.time()
    try:
        prefix_cmd = "virsh qemu-agent-command " + host_name
        #query cpu stat
        (status, result) = commands.getstatusoutput(
            prefix_cmd +
            " '{\"execute\":\"guest-query-cpu-usage\", \"arguments\":{\"delay\":1}}'"
        )
        if status == 0:
            result = json.loads(result)
            record = get_influxdb_record(host_uuid, region, "cpu", "usage",
                                         "total", result['return']['usage'])
            db_list.append(record)

        #query mem stat
        (status, result) = commands.getstatusoutput(
            prefix_cmd + " '{\"execute\":\"guest-query-mem-usage\"}'")
        if status == 0:
            result = json.loads(result)
            record = get_influxdb_record(host_uuid, region, "mem", "usage",
                                         "total", result['return']['usage'])
            db_list.append(record)

        #query disk stat
        (status, result) = commands.getstatusoutput(
            prefix_cmd +
            " '{\"execute\":\"guest-query-disk-stat\", \"arguments\":{\"delay\":1}}'"
        )
        if status == 0:
            result = json.loads(result)
            result = result['return']['disk_stat']
            for item in result:
                record = get_influxdb_record(host_uuid, region, "disk",
                                             'rd_ops', item['name'],
                                             item['rd_ops'])
                db_list.append(record)
                record = get_influxdb_record(host_uuid, region, "disk",
                                             'wr_ops', item['name'],
                                             item['wr_ops'])
                db_list.append(record)
                record = get_influxdb_record(host_uuid, region, "disk",
                                             'rd_octet', item['name'],
                                             item['rd_octet'])
                db_list.append(record)
                record = get_influxdb_record(host_uuid, region, "disk",
                                             'wr_octet', item['name'],
                                             item['wr_octet'])
                db_list.append(record)

        #query network stat
        (status, result) = commands.getstatusoutput(
            prefix_cmd +
            " '{\"execute\":\"guest-query-net-stat\", \"arguments\":{\"delay\":1}}'"
        )
        if status == 0:
            result = json.loads(result)
            result = result['return']['net_stat']
            for item in result:
                record = get_influxdb_record(host_uuid, region, "network",
                                             'receive', item['name'],
                                             item['receive'])
                db_list.append(record)
                record = get_influxdb_record(host_uuid, region, "network",
                                             'send', item['name'],
                                             item['send'])
                db_list.append(record)

        #write data to influxdb
        client.write(db_list)
    except Exception:
        logging.error("failed to collect %s data." % host_name)
        logging.error(traceback.print_exc())

    end_time = time.time()
    spend_time = int(end_time - begin_time)
    logging.debug("finished to collect %s data.took time: %d" %
                  (host_name, spend_time))
Esempio n. 55
0
def genDailyReport(startTs, endTs):
    br = '<br/>'
    responseData = initResponse()
    result = {}
    result['status'] = ''
    responseStr = ''
    # list of data collector
    listOfCollector = ['A01', 'A03', 'A05', 'A06']

    validStartTime = transferDatetime(startTs)
    validEndTime = transferDatetime(endTs)

    responseStr = responseStr + "[數據採集回報 " + validStartTime.strftime("%Y/%m/%d %H:%M:%S")\
                   + " - " + validEndTime.strftime("%Y/%m/%d %H:%M:%S") + "]" + br

    try:
        for machineId in listOfCollector:
            coreName = 'spc'
            spcRows = getTotalRowsBetweenTwoTimestamp(validStartTime,
                                                      validEndTime, 10, 1)
            response = queryInfoBetweenTimestampSPC0(coreName, machineId,
                                                     validStartTime,
                                                     validEndTime, spcRows)
            doc = praseJsonFormat(response)
            df = pd.DataFrame(doc)

            responseStr = responseStr + machineId + br
            #query燈號狀態
            lightResponse = queryLightStatus(machineId)
            machineStatus = parseLightStatus(lightResponse)

            if df.empty:
                logging.info(coreName + "/" + machineId + ":" + startTs + '-' +
                             validEndTime.strftime("%Y/%m/%d %H:%M:%S") +
                             ' DataFrame is empty!')
                responseStr = responseStr + insertTextIndent(
                    '最新模次:' + machineStatus + ',' + '此區間沒有生產數據', '20', '1')
            else:
                df.loc[:, 'SPC_0_previous'] = df['SPC_0']
                # shift 是往下移一個
                df['SPC_0_previous'] = df['SPC_0_previous'].shift(1).fillna(
                    '-1').astype(int)
                df.loc[:, 'timestamp_previous'] = df['timestamp'].shift(
                    1).fillna(-1).astype(float)

                #for顯示使用
                df.loc[:, 'timestamp_view_temp'] = df[
                    'timestamp_iso'].str.replace('T', ' ').str.replace(
                        'Z', '').astype(str)
                df.loc[:, 'timestamp_view'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%H:%M:%S')

                #last
                df.loc[:, 'timestamp_view_last'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%Y/%m/%d %H:%M:%S')

                #date
                df.loc[:, 'timestamp_view_date'] = pd.to_datetime(
                    df['timestamp_view_temp'],
                    errors='coerce').dt.strftime('%Y/%m/%d')
                df.loc[:, 'timestamp_view_date_previous'] = df[
                    'timestamp_view_date'].shift(1).fillna('-1')

                df.loc[:,
                       'timestamp_view_previous'] = df['timestamp_view'].shift(
                           1).fillna('-1')

                # 最新模次
                df_lastest = df.iloc[-1:].astype(str)
                lastTime = df_lastest['timestamp'].values[0]
                lastTimeStr = df_lastest['timestamp_view_last'].values[0]
                lastSPC = lastTimeStr + '(' + df_lastest['SPC_0'].values[
                    0] + ')'

                responseStr = responseStr + insertTextIndent(
                    '最新模次:' + machineStatus + ',' + lastSPC, '20', '1')

                #連線訊息
                dcCoreName = 'dc_event'
                dcRows = getTotalRowsBetweenTwoTimestamp(
                    validStartTime, validEndTime, 3, 4)
                validStartTimeStr = convertDatetimeToString(
                    validStartTime, "%Y-%m-%dT%H:%M:%SZ")
                validEndTimeStr = convertDatetimeToString(
                    validEndTime, "%Y-%m-%dT%H:%M:%SZ")
                dcResponse = queryDCLogBetweenTimestamp(
                    dcCoreName, machineId, validStartTimeStr, validEndTimeStr,
                    dcRows)
                dcDoc = praseJsonFormat(dcResponse)
                docDF = pd.DataFrame(dcDoc)

                if docDF.empty:
                    logging.info(dcCoreName + "/" + machineId + ":" + startTs +
                                 '-' + endTs +
                                 ' connection dataframe is empty!')
                else:
                    docFalseDF = docDF.where(docDF['result'] == False).dropna(
                    ).drop_duplicates('timestamp').copy()
                    # for顯示使用
                    docFalseDF.loc[:, 'timestamp_view_temp'] = docFalseDF[
                        'timestamp'].str.replace('T', ' ').str.replace(
                            'Z', '').astype(str)

                    docFalseDF.loc[:, 'timestamp_view_month'] = pd.to_datetime(
                        docFalseDF['timestamp_view_temp'],
                        errors='coerce').dt.strftime('%Y/%m/%d %H:%M:%S')

                    if docFalseDF['result'].count() == 0:
                        logging.info('未發生斷線問題')
                    else:  # 發生斷線問題
                        docFirstDate = ''
                        lastIndex = 0
                        # 刪除下一筆連續中斷的連線
                        for index, row in docFalseDF.iterrows():
                            if lastIndex + 1 == index:
                                docFalseDF = docFalseDF.drop(index)
                            lastIndex = index
                        # 找出下一筆恢復連線
                        # shift 是往上移一個
                        docDF.loc[:, 'result_next'] = docDF['result'].shift(
                            -1).astype(bool).copy()
                        docSameDF = docDF[(docDF['result_next'] !=
                                           docDF['result'])]
                        # 紀錄下一個falseIndex
                        dict = {}
                        lastFalseIndex = 0
                        for index, row in docSameDF.iterrows():
                            if row['result_next'] == False:
                                lastFalseIndex = index + 1
                            elif lastFalseIndex != 0 | (
                                    lastFalseIndex == 0 & row['result'] ==
                                    False):  #第一筆是斷線的時候, 加入第一筆斷線的訊息到dict
                                dict[lastFalseIndex] = index + 1

                        for index, row in docFalseDF.iterrows():
                            responseStr = responseStr + insertTextIndent(
                                '中斷連線:' + row['timestamp_view_month'], '20',
                                '0')
                            isFalseExist = index in dict
                            if isFalseExist == True:
                                if index != dict[index]:
                                    nextIndex = dict[index]
                                    docRow = docDF.ix[[nextIndex]].copy()
                                    docRowResult = docRow['result'].to_string(
                                        index=False)
                                    docRow.loc[:,
                                               'timestamp_view_temp'] = docRow[
                                                   'timestamp'].str.replace(
                                                       'T', ' ').str.replace(
                                                           'Z', '').astype(str)
                                    docRow.loc[:,
                                               'timestamp_view_month'] = pd.to_datetime(
                                                   docRow[
                                                       'timestamp_view_temp'],
                                                   errors='coerce'
                                               ).dt.strftime(
                                                   '%Y/%m/%d %H:%M:%S')
                                    docRowMonth = docRow[
                                        'timestamp_view_month'].to_string(
                                            index=False)
                                    responseStr = responseStr + insertTextIndent(
                                        '恢復連線:' + docRowMonth, '20', '0')

                # 模次不連續
                df.loc[:, 'SPC_0_previous_add1'] = df['SPC_0_previous'] + 1
                df_broken_spc_0 = df[(df['SPC_0_previous_add1'] !=
                                      df['SPC_0'])]
                # 多加一個條件, 排除停機後, spc_0跟上一筆相同的中斷模次
                df_broken_spc_0 = df_broken_spc_0[
                    (df_broken_spc_0['SPC_0'] !=
                     df_broken_spc_0['SPC_0_previous']) |
                    ((df_broken_spc_0['SPC_0'] ==
                      df_broken_spc_0['SPC_0_previous']) &
                     (df_broken_spc_0['SPC_0'] != df_broken_spc_0['RD_618']))]
                responseStr = parseViewformat(df_broken_spc_0, responseStr,
                                              '模次不連續')

                # 模次時間超過五分鐘
                df['timestamp_previous_addInterval'] = df[
                    'timestamp_previous'] + int(PRODUCTION_DURING_SEC)
                df_broken_timestamp = df[df['timestamp'].astype(float) >
                                         df['timestamp_previous_addInterval']]
                responseStr = parseViewformat(df_broken_timestamp, responseStr,
                                              '模次生產時間超過5分鐘')

            responseStr = responseStr + br
        return responseStr
    except:
        logging.error("genDailyReport exception")
        responseData = setErrorResponse(responseData, '400', 'exception')
        return parseResponseJson(responseData, result, 'result')
Esempio n. 56
0
def main(args=None):
    try:
        # if terminal
        if config.IS_IN_TERMINAL:
            sys_argv = sys.argv[1:]
            if isinstance(args, list):
                sys_argv = args

            sys_argv = tools.decodeArgs(sys_argv)

            COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None

            aparser = argparser(sys_argv)
            ARGS = aparser.parse()

        # if input stream
        else:
            COMMAND, ARGS = modifyArgsByStdinStream()

        # error or help
        if COMMAND is None or ARGS is False:
            return tools.exit()

        logging.debug("CLI options: %s", str(ARGS))

        # Users
        if COMMAND == 'user':
            User().user(**ARGS)

        if COMMAND == 'login':
            User().login(**ARGS)

        if COMMAND == 'logout':
            User().logout(**ARGS)

        if COMMAND == 'settings':
            User().settings(**ARGS)

        # Notes
        if COMMAND == 'create':
            Notes().create(**ARGS)

        if COMMAND == 'edit':
            Notes().edit(**ARGS)

        if COMMAND == 'remove':
            Notes().remove(**ARGS)

        if COMMAND == 'show':
            Notes().show(**ARGS)

        if COMMAND == 'find':
            Notes().find(**ARGS)

        # Notebooks
        if COMMAND == 'notebook-list':
            Notebooks().list(**ARGS)

        if COMMAND == 'notebook-create':
            Notebooks().create(**ARGS)

        if COMMAND == 'notebook-edit':
            Notebooks().edit(**ARGS)

        if COMMAND == 'notebook-remove':
            Notebooks().remove(**ARGS)

        # Tags
        if COMMAND == 'tag-list':
            Tags().list(**ARGS)

        if COMMAND == 'tag-create':
            Tags().create(**ARGS)

        if COMMAND == 'tag-edit':
            Tags().edit(**ARGS)

        if COMMAND == 'tag-remove':
            Tags().remove(**ARGS)

    except (KeyboardInterrupt, SystemExit, tools.ExitException):
        pass

    except Exception, e:
        traceback.print_exc()
        logging.error("App error: %s", str(e))
Esempio n. 57
0
    def loadPage(self,
                 url,
                 uri=None,
                 method="GET",
                 params="",
                 additionalParams=""):
        if not url:
            logging.error("Request URL undefined")
            tools.exitErr()

        if not url.startswith("http"):
            url = "https://" + url
        urlData = urlparse(url)
        if not uri:
            url = "%s://%s" (urlData.scheme, urlData.netloc)
            uri = urlData.path + '?' + urlData.query

        # prepare params, append to uri
        if params:
            params = urlencode(params) + additionalParams
            if method == "GET":
                uri += ('?' if uri.find('?') == -1 else '&') + params
                params = ""

        # insert local cookies in request
        headers = {
            "Cookie":
            '; '.join(
                [key + '=' + self.cookies[key] for key in self.cookies.keys()])
        }

        if method == "POST":
            headers["Content-type"] = "application/x-www-form-urlencoded"

        if self._proxy is None or proxy_bypass(urlData.hostname):
            host = urlData.hostname
            port = urlData.port
            real_host = real_port = None
        else:
            host = self._proxy.hostname
            port = self._proxy.port
            real_host = urlData.hostname
            real_port = urlData.port

        logging.debug("Request URL: %s:/%s > %s # %s", url, uri,
                      unquote(params), headers["Cookie"])

        conn = httplib.HTTPSConnection(host, port)

        if real_host is not None:
            conn.set_tunnel(real_host, real_port, headers=self._proxy_auth)
        if config.DEBUG:
            conn.set_debuglevel(1)

        conn.request(method, url + uri, params, headers)
        response = conn.getresponse()
        data = response.read()
        conn.close()

        logging.debug("Response : %s > %s", response.status,
                      response.getheaders())
        result = tools.Struct(status=response.status,
                              location=response.getheader('location', None),
                              data=data)

        # update local cookies
        sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
        for key in sk:
            self.cookies[key] = sk[key].value
        # delete cookies whose content is "deleteme"
        for key in self.cookies.keys():
            if self.cookies[key] == "deleteme":
                del self.cookies[key]

        return result
Esempio n. 58
0
    def unconstrain(self, constraints):
        '''
        This method is used to enable compliance with desired box constraints given by the user.
        It transforms the vectorfield by projecting the constrained state variables on
        new unconstrained ones.

        Parameters
        ----------

        constraints : dict
            The box constraints for the state variables
        '''

        # save constraints
        self.constraints = constraints

        # backup the original constrained system
        self._dyn_sys_orig = copy.deepcopy(self.dyn_sys)

        # get symbolic vectorfield
        # (as sympy matrix toenable replacement method)
        x = sp.symbols(self.dyn_sys.states)
        u = sp.symbols(self.dyn_sys.inputs)
        ff_mat = sp.Matrix(self.dyn_sys.f_sym(x, u))

        # get neccessary information form the dynamical system
        a = self.dyn_sys.a
        b = self.dyn_sys.b
        boundary_values = self.dyn_sys.boundary_values

        # handle the constraints by projecting the constrained state variables
        # on new unconstrained variables using saturation functions
        for k, v in self.constraints.items():
            # check if boundary values are within saturation limits
            xk = self.dyn_sys.states[k]
            xa, xb = self.dyn_sys.boundary_values[xk]

            if not (v[0] < xa < v[1]) or not (v[0] < xb < v[1]):
                logging.error(
                    'Boundary values have to be strictly within the saturation limits!'
                )
                logging.info('Please have a look at the documentation, \
                              especially the example of the constrained double intgrator.'
                             )
                raise ValueError(
                    'Boundary values have to be strictly within the saturation limits!'
                )

            # calculate saturation function expression and its derivative
            yk = sp.Symbol(xk)
            m = 4.0 / (v[1] - v[0])
            psi = v[1] - (v[1] - v[0]) / (1. + sp.exp(m * yk))

            #dpsi = ((v[1]-v[0])*m*sp.exp(m*yk))/(1.0+sp.exp(m*yk))**2
            dpsi = (4. * sp.exp(m * yk)) / (1. + sp.exp(m * yk))**2

            # replace constrained variables in vectorfield with saturation expression
            # x(t) = psi(y(t))
            ff_mat = ff_mat.replace(sp.Symbol(xk), psi)

            # update vectorfield to represent differential equation for new
            # unconstrained state variable
            #
            #      d/dt x(t) = (d/dy psi(y(t))) * d/dt y(t)
            # <==> d/dt y(t) = d/dt x(t) / (d/dy psi(y(t)))
            ff_mat[k] /= dpsi

            # update boundary values for new unconstrained variable
            boundary_values[xk] = ((1. / m) * np.log(
                (xa - v[0]) / (v[1] - xa)), (1. / m) * np.log(
                    (xb - v[0]) / (v[1] - xb)))

        # create a callable function for the new symbolic vectorfield
        ff = np.asarray(ff_mat).flatten().tolist()
        xu = self.dyn_sys.states + self.dyn_sys.inputs
        _f_sym = sp.lambdify(xu, ff, modules='sympy')

        def f_sym(x, u):
            xu = np.hstack((x, u))
            return _f_sym(*xu)

        # create a new unconstrained system
        xa = [boundary_values[x][0] for x in self.dyn_sys.states]
        xb = [boundary_values[x][1] for x in self.dyn_sys.states]
        ua = [boundary_values[u][0] for u in self.dyn_sys.inputs]
        ub = [boundary_values[u][1] for u in self.dyn_sys.inputs]

        self.dyn_sys = DynamicalSystem(f_sym, a, b, xa, xb, ua, ub)
Esempio n. 59
0
                    #logging.info(tstr)
                    url = "http://qt.gtimg.cn/q=" + tstr
                    response = urllib2.urlopen(url)
                    html = response.read()
                    lines = html.strip().split(";")
                    lines.remove("")

                    #time.sleep(100)
                    logging.info(len(lines))
                    for line in lines:
                        lv = line.strip().split("=")[1].replace('"',
                                                                '').split('~')
                        datalist.append(lv)
                    #logging.info(tempList)
                except Exception, e:
                    logging.error(e)
                tempList = []
        if (len(tempList) > 0):
            try:
                tstr = ""
                for t in tempList:
                    if tstr == "":
                        tstr = "s_sh" + t.strip()
                    else:
                        tstr = tstr + "," + "s_sh" + t.strip()
                url = "http://qt.gtimg.cn/q=" + tstr
                response = urllib2.urlopen(url)
                html = response.read()
                lines = html.split("\r\n")
                for lv in lines:
                    lv = html.split("=")[1].replace('"', '').split('~')