コード例 #1
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def export2lmdb(self, lmdb_host, lmdb_dbname):
        """ 匯出IMDB資料庫的資料到LMDB

        :param lmdb_host:
        :param lmdb_dbname:
        :return:
        """

        logger.info('export to lmdb')

        since = 0
        i = 0

        while True:
            movies = self.DbOperator.get_movies_to_export_lmdb(since, limit=self.DbOperator.LIMIT)
            if movies:
                for movie in movies:
                    movieid = movie[0]
                    imdbid = 'tt%07d' % int(movie[1])
                    i += 1
                    try:
                        imdbmovie = self.IMDbObj.get_movie(imdbid, movieid)
                        imdbmovie.save2db(lmdb_host, lmdb_dbname)
                        logger.info(
                            '%d, %s, %s, %s, %s' % (i, movieid, imdbid, imdbmovie['url'], imdbmovie['posterurl'])
                        )
                    except Exception as e:
                        logger.error('save db error: %s \r\n %s' % (imdbid, str(e)))

                since += self.DbOperator.LIMIT
            else:
                break
コード例 #2
0
ファイル: Googlect.py プロジェクト: zzhsec/farmscan_domain
 def parser_subject(self):
     try:
         #certsearch?include_expired=true&include_subdomains=true&domain=jd.com
         url = '{0}/certsearch?domain={1}&include_expired=true&include_subdomains=true'.format(
             self.website, self.domain)
         if self.req(url):
             result = (self.result[6:-1]).replace("\n", "").replace(
                 "[", "").replace("]", "").split(",")[-4]
             total_page = (self.result[6:-1]).replace("\n", "").replace(
                 "[", "").replace("]", "").split(",")[-1]
             current_page = (self.result[6:-1]).replace("\n", "").replace(
                 "[", "").replace("]", "").split(",")[-2]
             self.token = ast.literal_eval(result)
             rawres = parser(self.result, self.domain)
             domains = rawres.hostnames()
             if domains != None:
                 self.dns_names.extend(domains)
             '''
             while current_page < total_page:#重复请求,页面未变,该如何修改页面呢?
                 url = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p={0}".format(self.token)
                 if self.req(url):
                     print "xxxxx"
                     current_page = \
                     (self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
                     print current_page
                     rawres = parser(self.result, self.domain)
                     domains = rawres.hostnames()
                     self.dns_names.extend(domains)
                 else:
                     break
             '''
     except Exception as e:
         logger.error("Error in {0}: {1}".format(
             __file__.split('/')[-1], e))
         return
コード例 #3
0
 def req(self, url):
     headers = {
         'User-Agent':
         'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
         'Accept':
         'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'en-GB,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
     }
     try:
         resp = requests.get(url,
                             headers=headers,
                             timeout=self.timeout,
                             proxies=self.proxy)
         if resp.status_code == 200:
             if hasattr(resp, "text"):
                 return resp.text
             else:
                 return resp.conten
         else:
             return None
     except Exception as e:
         logger.error("Error in {0}: {1}".format(
             __file__.split('/')[-1], e))
         return None
コード例 #4
0
def main():

    logger.info(MSG_STARTING)
    try:
        config = get_config()

        logger.info(config['app']['name'])
        logger.info(MSG_GET_PAYLOAD)
        payload = get_payload()

        if config['debug']['postulant']:
            logger.info(payload)

        logger.info(MSG_GET_FIELDS_PAYLOAD)
        fields_payload = get_fieds_payload()

        logger.info(MSG_VALIDATE_PAYLOAD)
        validate_payload(fields_payload, payload)

        code = payload['code']
        validate_code_response_200(code)

    except QACliException as error:
        logger.error(str(error))

    logger.info(MSG_ENDING)
コード例 #5
0
 def post_info(self, data, api, res_str):
     try:
         response = requests.post(
             api,
             data=json.dumps(data),
             headers={'auth-token': self.auth_header_val})
         # 1. 字典序列化;2. 带请求头 content-type:   application/json
         if res_str == 'Client_Info': data = {'Client_Info': '....'}
         print_info = '[{date_time}]POST[{res_str}]:{data} to server'.format(
             date_time=datetime.datetime.now().strftime(
                 '%Y-%m-%d %H:%M:%S'),
             res_str=res_str,
             data=data)
         print(print_info)
         logger.info(print_info)
         # 获得返回结果
         rep = json.loads(response.text)
         return rep
     except requests.ConnectionError as e:
         msg = traceback.format_exc()
         rep = {'code': 3, 'msg': msg}
         logger.error(msg)
         print(rep)
         return rep
     except ValueError as e:
         msg = traceback.format_exc()
         rep = {'code': 3, 'msg': msg}
         logger.error(msg)
         print(rep)
         return rep
コード例 #6
0
    def check(self):
        logger.info("Doing Zone Transfer Check ...")
        try:
            self.get_ns_server()
            self.get_ns_server_nslookup()
        except Exception as e:
            logger.error("Error in {0}: {1}".format(__file__.split('/')[-1],e))
            return False

        if len(self.nsservers) == 0:
            logger.info("None NS Server found for {0}.".format(self.domain))
            return False
        else:
            for _ in self.nsservers:
                has_zone_transfer = self.axfr_check(self.domain, _)

            if has_zone_transfer != 0 and len(self.results) != 0:
                logger.info("Zone Transfer Detected for {0}".format(self.domain))
                #__file__  current file,the file contains current code.
                fp = open(os.path.join(os.path.dirname(os.path.dirname(__file__)), "output", "{0}_zone_transfer.txt".format(self.domain)), "wb")
                fp.writelines(self.results)
                fp.close()
                for item in self.results:
                    print item
                return True
            if has_zone_transfer == 0 or len(self.results) == 0:
                logger.info("Zone Transfer False")
                return False
コード例 #7
0
 def run(self):
     try:
         timestemp = time.time()
         url = "{0}?0.{1}&callback=&k={2}&page=1&order=default&sort=desc&action=moreson&_={3}&verify={4}".format(
             self.url, timestemp, self.domain, timestemp, self.verify)
         #response = req.get(url,proxies=self.proxy).content
         # no proxy needed for this class
         response = req.get(url).content
         result = json.loads(response)
         if result.get('status') == '1':
             for item in result.get('data'):
                 if is_domain(item.get('domain')):
                     self.domain_name.append(item.get('domain'))
         elif result.get('status') == 3:
             logger.warning("chaxun.la api block our ip...")
             logger.info("input you verify_code")
             # print('get verify_code():', self.verify)
             # self.verify_code()
             # self.run()
         self.domain_name = list(set(self.domain_name))
     except Exception as e:
         logger.error("Error in {0}: {1}".format(__file__.split('/')[-1], e))
     finally:
         logger.info("{0} found {1} domains".format(self.engine_name, len(self.domain_name)))
         return self.domain_name,self.smiliar_domain_name,self.email
コード例 #8
0
    def push(cls, msg):
        """ Send push notifications
        https://about.pushed.co/docs/api#api-method-push
        """

        logger.info('Pushing message via Pushed service...')

        response = requests.post(cls.API_URL + 'push',
                                 data={
                                     'app_key': app_conf.pushed_app_key,
                                     'app_secret': app_conf.pushed_app_secret,
                                     'target_type': 'app',
                                     'content': msg,
                                 })

        response_json = json.loads(response.text)

        if 'error' in response_json:
            error_msg = response_json['error']['message']
            logger.error(f'Message not sent. Error: {error_msg}')
            return

        response_type = response_json['response']['type']

        if response_type == 'shipment_successfully_sent':
            logger.info('Message successfully delivered to Pushed service')
        else:
            logger.error(f'Wrong response type from Pushed: {response_type}')
コード例 #9
0
    def evaluate_config(self, cfg, **kwargs):
        x_train, x_test, y_train, y_test = cfg['x_train'], cfg['x_test'], cfg[
            'y_train'], cfg['y_test']
        params = cfg['params']
        self.set_params(params)

        logger.info('Testing:' + str(params))
        #history = [x for x in x_train.values], [y for y in y_train.values]
        #predictions = []
        errors = []
        scores = []
        # Test the model on the train set
        model_fit = self.fit(x_train, y=y_train)
        train_predictions = model_fit.predict(x_train)
        predictions = model_fit.predict(x_test)

        if predictions is None or len(predictions) <= 0:
            logger.error('No predictions made for config {}'.format(
                str(params)))
        else:
            return {
                'y': y_test,
                'y_pred': np.array(predictions),
                'y_train': y_train,
                'y_train_pred': np.array(train_predictions),
                'model': str(self),
                'params': self.get_params(),
                'errors': errors,
                'scores': scores
            }
コード例 #10
0
    def evaluate_config(self, cfg, **kwargs):
        x_train, x_test, y_train, y_test = cfg['x_train'], cfg['x_test'], cfg[
            'y_train'], cfg['y_test']
        params = cfg['params']
        self.set_params(params)

        logger.info('Testing:' + str(params))
        history = [x for x in x_train.values], [y for y in y_train.values]
        predictions = []
        errors = []
        scores = []
        # Test the model on the train set
        model_fit = self.fit(x_train, y=y_train)
        train_predictions = model_fit.predict(x_train)
        predictions = model_fit.predict(x_test)
        if False:
            for i in range(len(x_test)):
                # logger.info('grid_search {} {} {}/{}'.format(self.name, str(order), i+1, len(x_test)))
                yhat = 0
                try:
                    _left = len(x_test) - i
                    model_fit = self.fit(history[0],
                                         y=history[1],
                                         params=params)
                    scores.append(model_fit.score(history[0], y=history[1]))
                    # Save test set prediction
                    forecast = model_fit.predict(
                        x_test[i:]
                    )  # ToDO: Save model, use new model only if score increases
                    yhat = float(
                        forecast[0])  # Forecast next element of the test set
                    if np.isnan(yhat):
                        yhat = 0
                except Exception as e:
                    errors.append('Error at step {} for config {}: {}'.format(
                        i, str(params), str(e)))
                    pass
                finally:
                    predictions.append(yhat)  # add forecasted y to predictions
                    history[0].append(
                        x_test.iloc[i].values
                    )  # Add an element from test set to history
                    history[1].append(
                        y_test.iloc[i]
                    )  # Add an element from test set to history

        if predictions is None or len(predictions) <= 0:
            logger.error('No predictions made for config {}'.format(
                str(params)))
        else:
            return {
                'y': y_test,
                'y_pred': np.array(predictions),
                'y_train': y_train,
                'y_train_pred': np.array(train_predictions),
                'model': str(self),
                'params': params,
                'errors': errors,
                'scores': scores
            }
コード例 #11
0
 def fit(self, x, **kwargs):
     params = kwargs.get('params')
     try:
         if params.get('beta'):
             # Use Augmented Dickey-Fuller test: try to reject the null hypothesis that the series
             # has a unit root, i.e. it's stationary. Statsmodels implementation has issues with window
             # size lower than 6, so adapt it for now.
             # Using the p-value, check if it's possible to reject the null hypothesis at 5 % significance
             # level and than choose Simple Exponential Smoothing or Holt's Exponential Smoothing (additional
             # trend factor).
             r = adfuller(x) if x.shape[0] > 6 else adfuller(x, maxlag=4)
             pvalue = r[1]
             if pvalue < 0.05:
                 self.model = ExponentialSmoothing(x, trend=None, seasonal=None) \
                     .fit(smoothing_level=params.get('alpha'))
             else:
                 self.model = ExponentialSmoothing(x, trend='additive', seasonal='additive',
                                                  seasonal_periods=params.get('seasonal_periods')) \
                     .fit(smoothing_level=params.get('alpha'), smoothing_slope=self.params.get('beta'))
         else:
             self.model = ExponentialSmoothing(x, trend=None, seasonal=None) \
                 .fit(smoothing_level=params.get('alpha'))
         self.params = params
         return self.model
     except (ValueError, np.linalg.linalg.LinAlgError):
         logger.error(
             'Exponential Smoothing convergence error (a:{},b:{})'.format(
                 params.get('alpha'), params.get('beta')))
         return None
コード例 #12
0
    def get_related_domains(self):
        if self.hash_codes > 0:
            main_of_domain = tldextract.extract(self.domain).domain
            for hash in self.hash_codes:
                try:
                    url = self.detail_url.format(hash)
                    resp = req.get(url,
                                   timeout=self.timeout,
                                   proxies=self.proxy,
                                   verify=False)
                    if resp:
                        formated_string = (self.result[6:-1]).replace("\n", "")
                        tmplist = ast.literal_eval(
                            formated_string)  #把list格式的string转换成list

                        main_of_cn_domain = tmplist[0][1][1].split(",")[-1]

                        if "CN\u003d" in main_of_cn_domain:  #有可能响应的内容为空,判断一下
                            main_of_cn_domain = main_of_cn_domain.replace(
                                "CN\u003d", "")
                            main_of_cn_domain = tldextract.extract(
                                main_of_cn_domain).domain
                            if main_of_domain in main_of_cn_domain:  #判断cn中的域名是否和要查询的域名相似
                                self.related_domain_name.extend(
                                    tmplist[0][1][-1])
                        else:
                            continue
                except Exception, e:
                    logger.error("Error in {0}: {1}".format(
                        __file__.split('/')[-1], e))
                    return
コード例 #13
0
    def parser_subject(self):
        try:
            url = self.search_api.format(self.domain)
            if self.req(url):
                formated_string = (self.result[6:-1]).replace(
                    "\n", "").replace("[", "").replace("]", "")
                self.token = formated_string.split(",")[-4]
                #print result
                #total_page = (self.result[6:-1]).replace("\n","").replace("[","").replace("]","").split(",")[-1]
                #current_page = (self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
                #it seems that no need to switch pages, the main purpose of this API is to get the related domains. it's enough using items in one page

                rawres = parser(self.result, self.domain)
                domains = rawres.hostnames()
                if domains != None:
                    self.domain_name.extend(domains)

                for item in formated_string.split(","):  #
                    if len(
                            item
                    ) >= 40 and " " not in item:  #sSWg6vIw46sI1eNhDlilAaanXC9htQlVuMuHJWqyNr8=
                        item = item.strip("\"")
                        #item = unicode(item).decode()
                        item = item.replace("\\u003d", "=")
                        self.hash_codes.append(item)
        except Exception as e:
            logger.error("Error in {0}: {1}".format(
                __file__.split('/')[-1], e))
            return
コード例 #14
0
ファイル: jd.py プロジェクト: wliustc/simpleSignin
    def sign(self):
        jobs_failed = []

        for job_class in self.job_list:
            job = job_class(self)

            # 默认使用移动设备User-agent,否则使用PC版User-Agent
            # if job.is_mobile:
            #     job.session.headers.update({
            #         'User-Agent': self.user.ua
            #     })
            # else:
            #     job.session.headers.update({
            #         'User-Agent': self.user.ua_pc})

            try:
                job.run()
            except Exception as e:
                logger.error('# 任务运行出错: ' + repr(e))
                traceback.print_exc()

            if not job.job_success:
                jobs_failed.append(job.job_name)

        print('=================================')
        print('= 任务数: {}; 失败数: {}'.format(len(self.job_list), len(jobs_failed)))
        if jobs_failed:
            print('= 失败的任务: {}'.format(jobs_failed))
        else:
            print('= 全部成功 ~')
        print('=================================')
        return len(jobs_failed) == 0
コード例 #15
0
 def do_search(self):
     try:
         url = "http://{0}/search?num={1}&start={2}&hl=en&meta=&q={3}".format(
             self.server, self.quantity, self.counter, self.word)
     except Exception, e:
         logger.error("Error in {0}: {1}".format(
             __file__.split('/')[-1], e))
コード例 #16
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def import_listfile(self):
        """ 將IMDB的資料庫文字檔匯進DB

        如果匯入時發生問題,則利用上次的備份檔進行還原

        :return:
        """
        logger.info('import listfile to db')

        try:
            self.make_csvdir()
            imdbpy2sql_path = os.path.normpath('%s/updateimdb/bin/' %
                                               DIR_CRONTAB)
            cmd = 'python %s/imdbpy2sql.py -d %s -u %s -c %s -i table' \
                  % (imdbpy2sql_path, self.Downloader.get_download_dir_path(), self.DbOperator.URI, self.get_csvdir())
            subprocess.check_call(cmd, shell=True)
        except Exception:
            logger.error(
                'error occurred during import listfile to db, try to restore the older db'
            )
            self.DbBackup.restoredb(
                self.DbOperator.DB,
                '%s/%s.bak' % (self.DbBackup.BACKUPDIR, self.DbOperator.DB))
            logger.info('restore success!')
            raise

        logger.info('import listfile to db is success!')
コード例 #17
0
def new_request(wrapUrl, timeout=None):
    # create dict to save request data
    kwargs = dict(wrapUrl.kwargs)
    method = kwargs['method']

    # check if exist timeout, else use default value
    if timeout is not None:
        kwargs['timeout'] = timeout
    else:
        kwargs['timeout'] = DEFAULT_TIMEOUT

    kwargs = define_request(wrapUrl._url, **kwargs)

    try:
        session = requests.Session()
        request = session.request(method, wrapUrl._url, **kwargs)

    except Exception as e:
        logger.debug(e)
    finally:
        if request is not None:
            if request.status_code != 200:
                logger.error("%s (%s)" % (request.reason, request.status_code))
            else:
                return request
コード例 #18
0
 def do_search(self):
     try:
         url = "https://{0}/customsearch/v1?key={1}&highRange={2}&lowRange={3}&cx={4}&start={5}&q={6}".format(
             self.server, self.api_key, self.highRange, self.lowRange,
             self.cse_id, self.counter, self.word)
     except Exception, e:
         logger.error(e)
コード例 #19
0
 def do_search(self):
     try:
         url = "http://{0}/search?text={1}&numdoc=50&lr={2}".format(
             self.server, self.word,
             self.counter)  #  %40=@ 搜索内容如:@meizu.com;在关键词前加@有何效果呢?,测试未发现不同
     except Exception, e:
         logger.error(e)
コード例 #20
0
ファイル: search_shodan.py プロジェクト: hiw0rld/Teemo
 def do_search(self):
     try:
         api = shodan.Shodan(self.apikey)
         self.results = api.search(self.word)
         self.totalresults += str(self.results)
     except shodan.APIError, e:
         logger.error(e)
コード例 #21
0
 def do_search(self):
     try:
         url = "http://api.cognitive.microsoft.com/bing/v7.0/search?q={0}&mkt=en-us".format(
             self.word, self.counter)  # 这里的pn参数是条目数
     except Exception, e:
         logger.error("Error in {0}: {1}".format(
             __file__.split('/')[-1], e))
コード例 #22
0
ファイル: Censys.py プロジェクト: zhangmanhui/teemo
    def search(self):
        temp_domains = []
        try:
            main_of_domain = tldextract.extract(self.domain).domain
            c = censys.certificates.CensysCertificates(
                api_id=self.api_id, api_secret=self.api_secret)

            # iterate over certificates that match a search
            fields = ["parsed.subject_dn",
                      "parsed.fingerprint_sha256"]  #parsed.issuer_dn
            for cert in c.search("{0}".format(self.domain), fields=fields):
                #print cert["parsed.subject_dn"]
                cn_domain = cert["parsed.subject_dn"].split(",")[-1].split(
                    "=")[-1]  #cn一定是在最后吗
                main_of_cn_domain = tldextract.extract(cn_domain).domain

                if main_of_domain in main_of_cn_domain:
                    detail = c.view(
                        cert["parsed.fingerprint_sha256"]
                    )  #print c.view("a762bf68f167f6fbdf2ab00fdefeb8b96f91335ad6b483b482dfd42c179be076")
                    #print detail
                    #print detail["parsed"]["names"]
                    temp_domains.extend(detail["parsed"]["names"])
                    temp_domains = list(set(temp_domains))
        except Exception, e:
            logger.error("Error in {0}: {1}".format(
                __file__.split('/')[-1], e))
コード例 #23
0
def build_wordlist(wordlist_file):
    # 读入字典文件
    fd = open(wordlist_file, "rb")
    logger.info("start wordlist build...")
    global raw_words
    raw_words = fd.readlines()
    logger.info("This dictionary contains %s rows" % len(raw_words))
    print "This dictionary contains %s rows..." % len(raw_words)
    if len(raw_words) == 0:
        logger.error("This dictionary is empty...")
    fd.close()

    found_resume = False
    words = Queue.Queue()

    for word in raw_words:
        word = word.rstrip()

        # 这功能暂时没开

        if conf['resume'] is not None:
            if found_resume:
                words.put(word)
            else:
                if word == conf['resume']:
                    found_resume = True
                    print "Resuming wordlist from : %s" % conf['resume']

        else:
            words.put(word)

    logger.info("wordlist build is complete...")
    return words
コード例 #24
0
 def do_search(self):
     try:
         url = "http://{0}/web?q={1}&pu=100&page={2}".format(
             self.server, self.word,
             self.counter)  #  %40=@ 搜索内容如:@meizu.com;在关键词前加@有何效果呢?,测试未发现不同
     except Exception, e:
         logger.error(e)
コード例 #25
0
    def req(self, req_method, url, params=None):
        params = params or {}
        headers = {
            'User-Agent':
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
            'Accept':
            'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-GB,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'Referer': 'https://dnsdumpster.com'
        }

        try:
            if req_method == 'GET':
                resp = req.get(url,
                               headers=headers,
                               timeout=self.timeout,
                               proxies=self.proxy)
            else:
                resp = req.post(url,
                                data=params,
                                headers=headers,
                                timeout=self.timeout,
                                proxies=self.proxy)
            if hasattr(resp, "text"):
                return resp.text
            else:
                return resp.content
        except Exception as e:
            logger.error("Error in {0}: {1}".format(
                __file__.split('/')[-1], e))
            return None
コード例 #26
0
ファイル: zonetransfer.py プロジェクト: BenDerPan/Teemo
    def check(self):
        logger.info("Doing Zone Transfer Check ...")
        try:
            self.get_ns_server()
            self.get_ns_server_nslookup()
        except Exception as e:
            logger.error(e)

        if len(self.nsservers) == 0:
            logger.info("None NS Server found for {0}.".format(self.domain))
        else:
            for _ in self.nsservers:
                has_zone_transfer = self.axfr_check(self.domain, _)

            if has_zone_transfer != 0 and len(self.results) != 0:
                logger.info("Zone Transfer Detected for {0}".format(
                    self.domain))
                fp = open(
                    "..\\output\\{0}_zone_transfer.txt".format(self.domain),
                    "wb")
                fp.writelines(self.results)
                fp.close()
                for item in self.results:
                    print item
            if has_zone_transfer == 0 or len(self.results) == 0:
                logger.info("Zone Transfer False")
コード例 #27
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def update_movie_imdbid(self):
        """ 更新IMDB資料庫電影的IMDBID

        如果更新某部電影的IMDBID,出現錯誤連續達到10次,則不繼續嘗試更新IMDBID(可能會是網路等問題)

        :return:
        """

        logger.info('update imdb_id field')

        count = 0
        max_try = 10

        while True:
            movies = self.DbOperator.get_null_imdbid_movies()
            if len(movies) == 0:
                break

            for movie in movies:
                try_times = 0
                count += 1

                try:
                    logger.info('%s: %s' % (count, self.get_imdbid_result(movie[0])))
                except Exception:
                    try_times += 1
                    time.sleep(3)
                    if try_times == max_try:
                        logger.error(traceback.format_exc())
                        return

        logger.info('import db to table is success!')
コード例 #28
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def export2lmdb(self, lmdb_host, lmdb_dbname):
        """ 匯出IMDB資料庫的資料到LMDB

        :param lmdb_host:
        :param lmdb_dbname:
        :return:
        """

        logger.info('export to lmdb')

        since = 0
        i = 0

        while True:
            movies = self.DbOperator.get_movies_to_export_lmdb(
                since, limit=self.DbOperator.LIMIT)
            if movies:
                for movie in movies:
                    movieid = movie[0]
                    imdbid = 'tt%07d' % int(movie[1])
                    i += 1
                    try:
                        imdbmovie = self.IMDbObj.get_movie(imdbid, movieid)
                        imdbmovie.save2db(lmdb_host, lmdb_dbname)
                        logger.info('%d, %s, %s, %s, %s' %
                                    (i, movieid, imdbid, imdbmovie['url'],
                                     imdbmovie['posterurl']))
                    except Exception as e:
                        logger.error('save db error: %s \r\n %s' %
                                     (imdbid, str(e)))

                since += self.DbOperator.LIMIT
            else:
                break
コード例 #29
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def update_movie_imdbid(self):
        """ 更新IMDB資料庫電影的IMDBID

        如果更新某部電影的IMDBID,出現錯誤連續達到10次,則不繼續嘗試更新IMDBID(可能會是網路等問題)

        :return:
        """

        logger.info('update imdb_id field')

        count = 0
        max_try = 10

        while True:
            movies = self.DbOperator.get_null_imdbid_movies()
            if len(movies) == 0:
                break

            for movie in movies:
                try_times = 0
                count += 1

                try:
                    logger.info('%s: %s' %
                                (count, self.get_imdbid_result(movie[0])))
                except Exception:
                    try_times += 1
                    time.sleep(3)
                    if try_times == max_try:
                        logger.error(traceback.format_exc())
                        return

        logger.info('import db to table is success!')
コード例 #30
0
def ph_request(url=None, header={}, value=None):

    if url is None:
        logger.error("URL is not found...")
        exit(0)
    else:
        logger.info("Target url is {}".format(url))

    if len(header) == 0:
        logger.warning("Header is empty...")
        header = {
            'User-Agent':
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36'
        }

    try:
        driver = webdriver.PhantomJS(executable_path=set_phantomjs_path())

    except WebDriverException:
        logger.error("phantomjs path error...")
        exit(0)

    try:
        driver.get(url)
        time.sleep(3)

    finally:
        return driver.page_source
コード例 #31
0
 def do_search(self):
     try:
         url = "https://{0}/?q={1}".format(self.server, self.word)
         #must use https
     except Exception, e:
         logger.error("Error in {0}: {1}".format(
             __file__.split('/')[-1], e))
コード例 #32
0
ファイル: update.py プロジェクト: wqqsukida/ops_client
 def utask_process(self):
     utask_res = {"utask_id": self.utask_id, "status_code": 5,
                  "run_time": "", "message": "",}
     start_time = datetime.datetime.now()
     try:
         img_file = self.get_img()  # 获取img文件
         res = subprocess.Popen(
             'sudo {update_cmd} dera update-fw {node} -f {img_file} -y 2>&1'.
             format(
                 update_cmd=self.cmd_file,
                 node=self.node,
                 img_file=img_file
             ),
             shell=True,
             stdout=subprocess.PIPE
         )
         res.wait()
         res_stdout = res.stdout.read().decode('utf-8')
         end_time = datetime.datetime.now()
         run_time = end_time - start_time
         utask_res["run_time"] = str(run_time)
         if 'Success' in res_stdout:
             utask_res["status_code"] = 2
         else:
             utask_res["status_code"] = 3
         utask_res["message"] = res_stdout
     except Exception as e:
         end_time = datetime.datetime.now()
         run_time = end_time - start_time
         utask_res["run_time"] = str(run_time)
         utask_res["status_code"] = 3
         msg = traceback.format_exc()
         utask_res["message"] = msg
         logger.error(msg)
     json.dump(utask_res, open(self.task_res_path, 'w'))
コード例 #33
0
ファイル: manager.py プロジェクト: nickatnight/srvrdj
 def destroy_nodes(self):
     L.error("DESTROYING ALL NODES FOR MANAGER %s"%self.name)
     for node in self.nodes:
         L.warning("KILLING NODE: %s"%node)
         try:
             call("docker-machine kill %s && sleep 10"%node)
             call("docker-machine rm %s"%node)
         except: pass
コード例 #34
0
ファイル: data.py プロジェクト: LoRexxar/Feigong
 def PostData(self, payload):
     data = payload
     try:
         r = self.s.post(self.url, data=data, headers=self.headers)
     except:
         logger.error("Time out...")
         exit(0)
     return r.text.encode('utf-8')
コード例 #35
0
ファイル: data.py プロジェクト: LoRexxar/Feigong
 def GetLen(self, payload):
     data = quote(payload)
     try:
         r = self.s.get(self.url + "?" + data, headers=self.headers)
     except:
         logger.error("Time out...")
         exit(0)
     lens = len(r.text.encode('utf-8'))
     return lens
コード例 #36
0
ファイル: data.py プロジェクト: LoRexxar/Feigong
 def PostBuildData(self, payload, llen):
     data = payload
     try:
         r = self.s.post(self.url, data=data, headers=self.headers)
     except:
         logger.error("Time out...")
         exit(0)
     lens = len(r.text.encode('utf-8'))
     # print r.text.encode('utf-8')
     if lens == llen:
         return True
     else:
         return False
コード例 #37
0
ファイル: data.py プロジェクト: LoRexxar/Feigong
 def PostTimeData(self, payload, dtime):
     data = payload
     ptime = time.time()
     try:
         r = self.s.post(self.url, data=data, headers=self.headers)
     except:
         logger.error("Time out...")
         exit(0)
     rr = r.text.encode('utf-8')
     ntime = time.time()
     if ntime - ptime > dtime:
         return True
     else:
         return False
コード例 #38
0
ファイル: bruter.py プロジェクト: LoRexxar/BScanner
def dir_bruter(word_queue, target_url, stime, extensions=None, pbar=None):

    while not word_queue.empty():

        pbar.update(1)
        
        attempt = word_queue.get()

        attempt_list = []

        # 检查是否有文件扩展名,如果没有就是我们要暴力破解的路径
        # if "." not in attempt:
        # 	attempt_list.append("%s/" % attempt)
        # else:
        attempt_list.append("%s" % attempt)

        # 如果我们想暴力扩展
        if extensions:
            for extension in extensions:
                if extension == ".swp":
                    attempt_list.append("/.%s%s" % (attempt.strip('/'), extension))
                else:
                    attempt_list.append("%s%s" % (attempt, extension))

        # 迭代我们想要尝试的文件列表
        for brute in attempt_list:

            url = "%s%s" % (target_url, urllib.quote(brute))
            # print url
            try:
                headers = {}
                headers["User-Agent"] = conf['ua']
                r = urllib2.Request(url, headers=headers)
                # pbar.update(1)
                try:
                    response = urllib2.urlopen(r, timeout=2)
                except:
                    logger.error("Time out...")
                    continue  # 有可能卡死

                # 请求完成后睡眠
                time.sleep(stime)

                if response.code != 404:
                    logger.info("Get !!!!" + url)
                    tqdm.write("[%d] => %s" % (response.code, url))

            except urllib2.URLError, e:
                if hasattr(e, 'code') and e.code != 404:
                    tqdm.write("!!! %d => %s" % (e.code, url))
コード例 #39
0
ファイル: dealpayload.py プロジェクト: LoRexxar/Feigong
def normal_injection(select=None, source=None, conditions=None, limit=0, dealpayload=None, data=None, isStrings=False, isCount=False, sqlirequest=None):

    # 生成随机字符串padding
    padding = random_string()

    payload = dealpayload.construct_normal_payload(select=select, source=source, conditions=conditions, limit=limit, padding=padding)

    if sqlirequest == "GET":
        result = data.GetData(payload)
    elif sqlirequest == "POST":
        result = data.PostData(payload)
    else:
        logger.error("sqlirequest error...")
        exit(0)

    if isCount:
        return int(UnpackFunction(result, padding))
    elif isStrings:
        return UnpackFunction(result, padding)
    else:
        logger.error("Something error...")
コード例 #40
0
ファイル: pi-alarm.py プロジェクト: fradaloisio/pi-alarm
def main():
    # main

    # activate API
    if api.lunch():
        logger.info("API started")
    else:
        logger.error("API failed start")

    GPIO.setmode(GPIO.BOARD)
    pir = 26
    GPIO.setup(pir, GPIO.IN)
    logger.info("Application started")
    while True:
        while active.getActive():
            if GPIO.input(pir):
                time.sleep(1)
                if GPIO.input(pir):
                    active.setDetection("true")
                    sender.emailAlert()
                    active.setDetection("false")
                    time.sleep(0.2)
コード例 #41
0
ファイル: imdb_manager.py プロジェクト: codeguycool/CEC
    def import_listfile(self):
        """ 將IMDB的資料庫文字檔匯進DB

        如果匯入時發生問題,則利用上次的備份檔進行還原

        :return:
        """
        logger.info('import listfile to db')

        try:
            self.make_csvdir()
            imdbpy2sql_path = os.path.normpath('%s/updateimdb/bin/' % DIR_CRONTAB)
            cmd = 'python %s/imdbpy2sql.py -d %s -u %s -c %s -i table' \
                  % (imdbpy2sql_path, self.Downloader.get_download_dir_path(), self.DbOperator.URI, self.get_csvdir())
            subprocess.check_call(cmd, shell=True)
        except Exception:
            logger.error('error occurred during import listfile to db, try to restore the older db')
            self.DbBackup.restoredb(self.DbOperator.DB, '%s/%s.bak' % (self.DbBackup.BACKUPDIR, self.DbOperator.DB))
            logger.info('restore success!')
            raise

        logger.info('import listfile to db is success!')
コード例 #42
0
ファイル: dealpayload.py プロジェクト: LoRexxar/Feigong
def time_injection(select=None, source=None, conditions=None, limit=0, dealpayload=None, data=None, times=0, isNumber=False, isStrings=False, isCount=False, sqlirequest=None):
    """
    使用二分法注入逻辑注入数据
    """
    chartbl = []

    if isNumber:
        chartbl = NumberInjectionList
    elif isStrings:
        chartbl = CharInjectionList
    elif isCount:
        chartbl = CountInjectionList
    else:
        logger.error("injection target error...")
        exit(0)

    while 1 != len(chartbl):
        position = (len(chartbl) >> 1)
        posValue = chartbl[position]

        payload = dealpayload.construct_time_payload(select=select, source=source, conditions=conditions, limit=limit, compare=posValue)
        # logger.debug("testing payload:" + payload)

        if sqlirequest == "GET":
            result = data.GetTimeData(payload, times)
        elif sqlirequest == "POST":
            result = data.PostTimeData(payload, times)
        else:
            logger.error("sqlirequest error...")
            exit(0)

        if result:
            if type(chartbl) != xrange:
                chartbl = chartbl[position:]
            else:
                # xrange() - extended virtual charset used for memory/space optimization
                chartbl = xrange(chartbl[position], chartbl[-1] + 1)
        else:
            if type(chartbl) != xrange:
                chartbl = chartbl[:position]
            else:
                chartbl = xrange(chartbl[0], chartbl[position])

        # 判断结果
        if len(chartbl) == 1:
            # logger.debug("injection success,the chartbl[0]+1 is %d", chartbl[0]+1)
            if isCount & chartbl[0] == 100:
                logger.error("Count or Length >100...")
                return 100
            return chartbl[0] + 1
コード例 #43
0
ファイル: commands.py プロジェクト: nickatnight/srvrdj
def error_out(context, name, message, e=None):
    if context.obj['json']: return json.dumps({name: message})
    L.error('%s:\t%s'%(name, message))
    if isinstance(e, Exception): raise e
コード例 #44
0
ファイル: content.py プロジェクト: LoRexxar/Feigong
    def run_content(self):

        if len(self.columns_name) == 0:
            SqliColumns.get_columns(self)

        # 循环解包,进入注入
        for database_name in self.columns_name:
            for table_name in self.columns_name[database_name]:

                # 获取数据的条数,如果小于设置的self.content_count,那需要设置条数等于self.content_count
                content_counts = self.get_content_count(database_name, table_name)
                if content_counts == 0:
                    logger.warning('Database %s Table %s is empty...' % (database_name, table_name))
                    continue
                elif content_counts != self.content_count:
                    logger.debug('Database %s Table %s content amount change to %d' % (database_name, table_name, content_counts))
                    self.content_count = content_counts
                else:
                    pass

                # 声明一个表储存数据
                content = PrettyTable(list(self.columns_name[database_name][table_name]))
                content.padding_width = 1
                content.align = "r"

                # 每个表都要注入指定条数那么多次
                for limits in xrange(self.content_count):

                    # 声明一个队列,储存返回的值
                    result = Queue.Queue()

                    # 声明线程队列、结果队列和最终插入table的数据队列
                    threads = []
                    results = []
                    contents = []

                    # 开始多线程的注入
                    logger.debug("Start multithreading Sqli...")
                    for column_name in self.columns_name[database_name][table_name]:
                        # 开始一个线程注入一个字段
                        try:
                            t = threading.Thread(target=self.get_content, name='thread for %s' % column_name,
                                                 args=(result, database_name, table_name, column_name, limits))
                            t.start()
                        except:
                            logger.error('Thread error...')

                        threads.append(t)

                    # 等待所有线程结束
                    for t in threads:
                        t.join()

                    # 注入处理返回数据,插入content中的一条
                    while not result.empty():
                        results.append(result.get())

                    # 处理返回的数据
                    for i in list(self.columns_name[database_name][table_name]):
                        for item in results:
                            if item[0] == i:
                                contents.append(item[1])
                            else:
                                continue

                    # 插入数据
                    content_str = ','.join(contents)
                    logger.info("Sqli success content is %s" % content_str)
                    content.add_row(contents)

                # 输出表
                logger.debug("Database %s Table %s sqli success..." % (database_name, table_name))
                print "[*] Database %s Table %s content:" % (database_name, table_name)
                print content
コード例 #45
0
ファイル: options.py プロジェクト: LoRexxar/BScanner
def oparser():
    # 处理参数
    parser = OptionParser()
    parser.version = conf['version']

    parser.add_option("--version", "-v", dest="showVersion", action="store_true",
                      help="show program's version and exit")

    # 必选参数
    target = OptionGroup(parser, "Target", "At least one of these options has to be provided to define the target(s)")

    target.add_option("-u", dest="url", help="Target URL")
    target.add_option("--test", dest="test", action="store_true", help="auto test")

    # 可选参数
    opt = OptionGroup(parser, "Options", "Optional parameters")

    opt.add_option("-t", "--thread", dest="thread", type="int", help="thread number(default 30)")
    opt.add_option("-e", "--ext", dest="extensions", action="store_true",
                   help="Whether to use file extension(default false)")
    opt.add_option("-f", "--filename", dest="filename", help="Scanner dictionary (default ./php.txt)")
    opt.add_option("-s", "--sleeptime", dest="stime", type="int", help="Each request of sleep time (default 1)")
    opt.add_option("-l", dest="loglevel", type="int", help="log level(1-5) "
                                                           "1, CRITICAL; "
                                                           "2, ERROR(default); "
                                                           "3, WARN; "
                                                           "4, INFO; "
                                                           "5, DEBUG;")

    parser.add_option_group(target)
    parser.add_option_group(opt)

    (args, _) = parser.parse_args(sys.argv)

    if args.showVersion:
        print parser.version
        print "-- By LoRexxar"
        exit(0)

    if not (args.url or args.test):
        errMsg = "missing a mandatory option (-u) or (--test), "
        errMsg += "use -h for basic or --help for advanced help"
        parser.error(errMsg)

    conf['url'] = (args.url.rstrip('/').rstrip('"').rstrip('\'') if args.url else "http://testphp.vulnweb.com/")

    # 处理下url用来log的名字
    name = re.findall("[\w\.-]+", conf['url'])
    try:
        conf['name'] = (name[1] if len(name) == 2 else name[0])
    except IndexError:
        errMsg = "url input error!"
        logger.error("url matching fail!")
        parser.error(errMsg)

    conf['thread'] = (args.thread if args.thread else 30)

    if conf['thread'] < 1 or conf['thread'] > 50:
        # 线程数为0-50
        errMsg = "thread value error (1-50)"
        parser.error(errMsg)

    # 设置睡眠时间
    conf['stime'] = (args.stime if args.stime else 1)

    if conf['stime'] < 0 or conf['stime'] > 10:
        # 睡眠时间为0-10
        errMsg = "time value error (0-10)"
        parser.error(errMsg)

    conf['loglevel'] = (args.loglevel if args.loglevel else 2)
    if conf['loglevel'] < 1 or conf['loglevel'] > 5:
        # loglevel: 1-5
        errMsg = "loglevel value error(input 1-5)"
        parser.error(errMsg)

    if conf['loglevel'] == 1:
        conf['loglevel'] = logging.CRITICAL
    elif conf['loglevel'] == 2:
        conf['loglevel'] = logging.ERROR
    elif conf['loglevel'] == 3:
        conf['loglevel'] = logging.WARN
    elif conf['loglevel'] == 4:
        conf['loglevel'] = logging.INFO
    elif conf['loglevel'] == 5:
        conf['loglevel'] = logging.DEBUG
    else:
        conf['loglevel'] = logging.ERROR

    # 开启log
    log(conf['name'], conf['loglevel'])

    # 设置扫描器字典

    wordlist_file = (args.filename if args.filename else "./dic/php.txt")
    word_queue = build_wordlist(wordlist_file)

    extensions = ['.bak', '.orig', '.inc', '.swp', '~']

    # 进度条
    pbar = tqdm(total=word_queue.qsize(), leave=False)

    # 开始扫描
    if args.extensions:
        tqdm.write('start scanning with extensions...')
        for i in range(conf['thread']):
            t = threading.Thread(target=dir_bruter, args=(word_queue, conf['url'], conf['stime'], extensions, pbar))
            t.start()
    else:
        tqdm('start scanning...')
        for i in range(conf['thread']):
            t = threading.Thread(target=dir_bruter, args=(word_queue, conf['url'], conf['stime'], None, pbar))
            t.start()