def get_nickname(self): """获取用户昵称""" url = 'https://weibo.cn/%s/info' % (self.user['id']) selector = self.parser.deal_html(url, self.config['cookie']) nickname = selector.xpath('//title/text()')[0] nickname = nickname[:-3] if nickname == u'登录 - 新' or nickname == u'新浪': write_log(self.config['since_date']) sys.exit(u'cookie错误或已过期,请按照README中方法重新获取') self.user['nickname'] = nickname
def log_opened_connection(self, ip: str, version: str, dbs: List[str], dumped_dbs: List[str]): dumped_str = "" if len(dumped_dbs) > 1: dumped_str = "\n".join([f"[{ip}]: Dumped '{d_db}'" for d_db in dumped_dbs]) + "\n" write_log(f"{self.options.out}/{self.options.file}", f"------------------------------------------------\n" f"[{ip}]: opened connection.\n" f"\t├─ Version: {version}\n" f"\t└─ Databases: {dbs}\n" + dumped_str)
def main(): # Get the arguments from the command-line except the filename argv = sys.argv[1:] params = get_params(argv) print('Program started, input file: %s' % params['filepath']) if 'debug' in params.keys(): if params['debug'] == '1': print('Setting debug level: Debug') logging.getLogger().setLevel(logging.DEBUG) else: print('Setting debug level: Warnings') logging.getLogger().setLevel(logging.WARNING) else: logging.getLogger().setLevel(logging.INFO) params['data_params'], type_dict = reader.get_data_settings(params['config_file']) header, raw_data = reader.get_data(params) type_selector, converters = reader.validate_data_format(params['data_params'], header) key_fails, failed_rows, surveys,quality_assessment, out_keys,meta, raw_surveys = data_processor.process_rows(raw_data, header, params, converters) if 'logfile' in params: if params['logfile'].endswith('txt'): writer.write_log(logfiler.write_txt_log(params, key_fails, raw_data, failed_rows, meta), params['logfile'], 'Log:') elif params['logfile'].endswith('csv'): writer.write_log(logfiler.create_pbi_log(surveys,quality_assessment, meta['attribute_quality'], meta, failed_rows,params), params['logfile'], ['Key', 'Value']) else: logging.CRITICAL('Logfile not written, wrong file extension provided') out_header = surveys[0].keys() if params['outfile'].endswith('.csv'): writer.write_data(surveys, out_header, params,rounding=9,raw_surveys=raw_surveys) if len(quality_assessment) > 0: writer.write_data(quality_assessment, quality_assessment[0].keys(), params, sub_file='group_qual') if len(meta['attribute_quality']) > 0 and 'logfile' in params: writer.write_data(meta['attribute_quality'], meta['attribute_quality'][0].keys(), params, sub_file='attr_qual') else: logging.critical("Your output filename: %s must end with .csv" % params['outfile'])
if args.input_ips is not None: ips = parse_str_ips(args.input_ips) options = ConnectionWorkerOptions(out=args.input_out, file=DEFAULT_OUT_FILE, dump=args.input_dump) print(options) # Create out dir if not exists if not os.path.exists(options.out): os.makedirs(options.out) write_log( f"{options.out}/{options.file}", f"=========== | {datetime.now():%Y-%m-%d %H:%M:%S} | ===========\n") print_banner() q = Queue() for j in range(workers): worker = ConnectionWorker(q, j, options) worker.setDaemon(True) worker.start() for ip in ips: print(f"Enqueued ip: {ip}") q.put(ip)