def get_nickname(self):
"""获取用户昵称"""
url = 'https://weibo.cn/%s/info' % (self.user['id'])
selector = self.parser.deal_html(url, self.config['cookie'])
nickname = selector.xpath('//title/text()')[0]
nickname = nickname[:-3]
if nickname == u'登录 - 新' or nickname == u'新浪':
write_log(self.config['since_date'])
sys.exit(u'cookie错误或已过期,请按照README中方法重新获取')
self.user['nickname'] = nickname
def log_opened_connection(self, ip: str, version: str,
dbs: List[str], dumped_dbs: List[str]):
dumped_str = ""
if len(dumped_dbs) > 1:
dumped_str = "\n".join([f"[{ip}]: Dumped '{d_db}'" for d_db in dumped_dbs]) + "\n"
write_log(f"{self.options.out}/{self.options.file}",
f"------------------------------------------------\n"
f"[{ip}]: opened connection.\n"
f"\t├─ Version: {version}\n"
f"\t└─ Databases: {dbs}\n" +
dumped_str)
def main():
# Get the arguments from the command-line except the filename
argv = sys.argv[1:]
params = get_params(argv)
print('Program started, input file: %s' % params['filepath'])
if 'debug' in params.keys():
if params['debug'] == '1':
print('Setting debug level: Debug')
logging.getLogger().setLevel(logging.DEBUG)
else:
print('Setting debug level: Warnings')
logging.getLogger().setLevel(logging.WARNING)
else:
logging.getLogger().setLevel(logging.INFO)
params['data_params'], type_dict = reader.get_data_settings(params['config_file'])
header, raw_data = reader.get_data(params)
type_selector, converters = reader.validate_data_format(params['data_params'], header)
key_fails, failed_rows, surveys,quality_assessment, out_keys,meta, raw_surveys = data_processor.process_rows(raw_data, header, params, converters)
if 'logfile' in params:
if params['logfile'].endswith('txt'):
writer.write_log(logfiler.write_txt_log(params, key_fails, raw_data, failed_rows, meta), params['logfile'], 'Log:')
elif params['logfile'].endswith('csv'):
writer.write_log(logfiler.create_pbi_log(surveys,quality_assessment, meta['attribute_quality'], meta, failed_rows,params), params['logfile'], ['Key', 'Value'])
else:
logging.CRITICAL('Logfile not written, wrong file extension provided')
out_header = surveys[0].keys()
if params['outfile'].endswith('.csv'):
writer.write_data(surveys, out_header, params,rounding=9,raw_surveys=raw_surveys)
if len(quality_assessment) > 0:
writer.write_data(quality_assessment, quality_assessment[0].keys(), params, sub_file='group_qual')
if len(meta['attribute_quality']) > 0 and 'logfile' in params:
writer.write_data(meta['attribute_quality'], meta['attribute_quality'][0].keys(), params,
sub_file='attr_qual')
else:
logging.critical("Your output filename: %s must end with .csv" % params['outfile'])
if args.input_ips is not None:
ips = parse_str_ips(args.input_ips)
options = ConnectionWorkerOptions(out=args.input_out,
file=DEFAULT_OUT_FILE,
dump=args.input_dump)
print(options)
# Create out dir if not exists
if not os.path.exists(options.out):
os.makedirs(options.out)
write_log(
f"{options.out}/{options.file}",
f"=========== | {datetime.now():%Y-%m-%d %H:%M:%S} | ===========\n")
print_banner()
q = Queue()
for j in range(workers):
worker = ConnectionWorker(q, j, options)
worker.setDaemon(True)
worker.start()
for ip in ips:
print(f"Enqueued ip: {ip}")
q.put(ip)