def periodic_check_url(url, interval): """ This task check an url `url` in intervals of `interval`. It reschedule itself every run to act periodicaly. Since, huey periodic_task works minimum with one minute. This method handles exceptions and logs the ones that it is unable to handle. Keeps trying to recheck every url. - In case of `Timeout`, waits twice the time before the next check. - For `ConnectionError` or `HTTPError`, it doubles the check interval permanentely to reduce the number of errors, but doesn't stop tracking the url. - Only in case of `MissingSchema` (invalid url) that the system stops tracking. Since it's invalid, there is no point on checking again. An approach could be try to correct the url with some variants, but I thought better to stick to the main task. """ # recall this task to make it run as a periodic one # (huey limitation: minimum precision of minutes for periodical tasks) res = periodic_check_url.schedule(args=(url, interval), delay=interval) check_stats = {'size': None, 'status_code': None, 'error': None} kw = {'url': url, 'stream': True, 'timeout': max(interval / 10, 20)} try: response = session.get(**kw) check_stats['size'] = get_size(response) check_stats['status_code'] = response.status_code except (ConnectionError, HTTPError) as e: # e.g. DNS failure, refused connection, etc # invalid HTTP response # doubles the waiting to recheck this url res.revoke() periodic_check_url.schedule(args=(url, interval * 2), delay=(interval * 2)) check_stats['error'] = e check_stats['size'] = get_size(e.response) if isinstance(e, HTTPError): check_stats['status_code'] = e.response.status_code return check_stats except Timeout as e: # timeout: adds some more waiting, but doesn't increase the waiting # in the chain of calls res.revoke() check_stats['error'] = e check_stats['size'] = get_size(e.response) periodic_check_url.schedule(args=(url, interval), delay=(interval * 2)) return check_stats except MissingSchema as e: # invalid url: don't ever retry it check_stats['error'] = e res.revoke() return check_stats except Exception as e: # unexpected error check_stats['error'] = e logger.error(str(e)) return check_stats
def doConnect(self): """ Executed to connect the gateway to the server.""" try: # Create UDP socket self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) #self.udp_socket.bind(('127.0.0.1', 4844)) self.udp_socket.settimeout(poll_time * 4) # Get implemented driver list drivers_list = {} for driver_name, (_, driver_version) in registered_drivers.items(): drivers_list.update({driver_name: {'version': driver_version}}) # Generate request telegram register_id = self.get_new_message_id() register_request = { "ID": register_id, "REGISTER": { "server_key": 'server key obtained from the user', "gateway": { "version": version, "heartbeat": poll_time, }, "drivers": drivers_list } } # Send register request self.udp_socket.sendto( json.dumps(register_request).encode('utf8'), self.server_address) except Exception as e: self.doCleanup() self.status = GatewayStatus.ERROR self.error_msg = 'Exception during setup: ' + str(e) logger.error(self.error_msg) raise Exception(self.error_msg) # Get response before defined timeout (4 x poll_time) try: data, address = self.udp_socket.recvfrom(4096) if data != None and address == self.server_address: response_json = json.loads(data.decode('utf-8')) if response_json["ID"] == register_id: # Connection succeded if (response_json["REGISTER"] == 'Success'): # From now on, no timeout is set in the socket self.udp_socket.setblocking(0) self.last_poll_sent = time.time() self.last_poll_received = time.time() self.status = GatewayStatus.CONNECTED logger.debug("Gateway connected") return else: logger.debug("Datagram id or ip does not match") except Exception as e: logger.debug("Invalid json datagram received: " + str(e)) # Connection failed self.status = GatewayStatus.ERROR
def func(self): # headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"} while True: url, savepath = self.taskqueue.get() block = 1 << 14 # 16k try: req = request.Request(url, headers=USER_AGENT) with request.urlopen(req, timeout=60) as response: with open(savepath, "wb") as f: for data in iter(partial(response.read, block), b""): f.write(data) except socket.timeout: logger.warning("下载超时:{}".format(url)) os.remove(savepath) self.count += 1 self.taskqueue.put((url, savepath)) continue except Exception as e: logger.error("出错:{}".format(e)) logger.error("出错url:{}".format(url)) self.count += 1 self.taskqueue.put((url, savepath)) finally: # response.close() self.taskqueue.task_done() self.count -= 1 logger.info("下载完成:{}".format(savepath)) logger.debug("当前队列任务数:{}".format(self.count))
def GUIrunPeriod(): try: q = [IGuiPeriodCaller.qsize(), IGuiCaller.qsize(), IGuiIdleCaller.qsize()] if q[0] > 100 or q[1] > 100 or q[2] > 100: logging.info("IGuiPeriodCaller=%d(d %d) IGuiCaller=%d(d %d) IGuiIdleCaller=%d(d %d)" % ( q[0], q[0] - oq[0], q[1], q[1] - oq[1], q[2], q[2] - oq[2])) oq[:] = q[:] # Replace content in-place n = 0 t = time.time() try: while True: Run = IGuiPeriodCaller.get(0) try: n += 1 t0 = time.time() Run() t0 = time.time() - t0 if t0 > 0.1: logging.info("Run=%r took %r", Run, t0) except Exception, E: logging.error("GUIrun/period: %s\nRun = %r\nTraceback: %s", E, Run, traceback.format_exc()) except Queue.Empty: pass tt = time.time() if tt - t > 0.05: logging.error("GUIrunPrriod: proceed %d for %r", n, tt - t) GUIrun(False) except Queue.Empty: pass return True
def get_ms_average_score_welcome(player_obj: object) -> str: """Returns welcome message telling the player's average time.""" average_records = None for difficulty in speed_challenge.data.SC_DIFFICULTIES: average_records = player_obj.get_sc_average_records(difficulty) if average_records is not None: break if (difficulty is None) or (average_records is None): log_all(difficulty, average_records, log_level=40) logger.error( "get_ms_high_score_welcome: Found no difficulty or average_records." ) return SC_WelcomeUtils.get_ms_short_welcome() avg_score = int(mean(average_records)) avg_score = SC_EndGame.format_score_as_minutes_seconds(avg_score) ms_average_score = random.choice( speed_challenge.data.MT_AVERAGE_SCORE).format( difficulty, avg_score) ms_beat_it = random.choice(speed_challenge.data.MT_BEAT_IT) speech_list = (ms_average_score, 1, ms_beat_it) return get_linear_nlg(speech_list)
def main(): logger.info('Initialize') data_sink = Observable() builder = Builder(config) builder.bind(data_sink) leds = Leds() while True: try: prices = [entry[1:] for entry in get_dummy_data() ] if config.dummy_data else fetch_prices() data_sink.update_observers(prices) leds.off() time.sleep(config.refresh_interval) except Exception as e: logger.error(str(e)) leds.on() time.sleep(10) except KeyboardInterrupt: logger.info('Exit') data_sink.close() exit()
def select_version_id(self, version_id=None): """ param: version_id 默认为None, version_id 为非None 时为创建MC 目录构造。 """ if version_id is None: vers = os.listdir(self.versions) if len(vers) == 0: logger.error("{} 没有游戏。".format(self.gameDir)) sys.exit(1) else: logger.info("当前版本:{}".format(vers)) vers.sort() self.version_id = vers[-1] logger.info("默认选择版本:{}".format(self.version_id)) self.client_jar = joinpath(self.versions, self.version_id, self.version_id + ".jar") self.client_json = joinpath(self.versions, self.version_id, self.version_id + ".json") self.server_jar = joinpath(self.serverDir, "server-" + self.version_id + ".jar") else: self.version_id = version_id self.client_jar = joinpath(self.versions, self.version_id, self.version_id + ".jar") self.client_json = joinpath(self.versions, self.version_id, self.version_id + ".json") self.server_jar = joinpath(self.serverDir, "server-" + self.version_id + ".jar")
def __get_gameDir(self): if path.exists(self.gameDir): pass else: logger.error('游戏目录不存在: {} 或者 当前没有游戏。'.format(self.gameDir)) sys.exit(1)
def FREE(self): self.locked -= 1 self.thread = None if self.locked != 0: # print "!ERROR! Main free not before MAIN!" logging.error("Main free not before MAIN!") sys.exit(0)
def send_email(from_='*****@*****.**', to='', subject='', message='', server='', port=25, username='', password='', use_ssl=False, starttls=False): logger.debug('Emailing %s' % to) if not server: return msg = MIMEText(message) msg['Subject'] = subject msg['From'] = from_ msg['To'] = to if (use_ssl and not starttls): s = smtplib.SMTP_SSL(host=server, port=port) else: s = smtplib.SMTP(server, port) if (starttls): s.ehlo() s.starttls() s.ehlo() if (not username == '' and not password == ''): s.login(username, password) try: s.sendmail(from_, [to], msg.as_string()) logger.info('Email sent to %s' % (to)) except Exception as e: logger.error('Email sending produced exception %r' % e) s.quit()
def check_data(self): wait.until(EC.presence_of_element_located(cn.cus_name)) con_name = self.driver.find_element(*cn.con_name).text con_tel = self.driver.find_element(*cn.con_tel).text cert_type = self.driver.find_element(*cn.cert_type).text cert_num = self.driver.find_element(*cn.cert_num).text con_addr = self.driver.find_element(*cn.con_addr).text taxpayer = self.driver.find_element(*cn.taxpayer).text cus_id = self.driver.find_element(*cn.cus_id).text data = con_name.strip() +\ con_tel.strip() + \ cert_type.strip() + \ cert_num.strip() + \ con_addr.strip() + \ taxpayer.strip() data_dict_new = str(data_dict['联系人']) + \ str(data_dict['联系电话']) + \ str(data_dict['证件类型']) + \ str(data_dict['证件号码']) + \ str(data_dict['联系地址']) + \ str(data_dict['一般纳税人']) if data == data_dict_new: print('校验一致') logger.info('校验一致') else: logger.error('校验不一致') return cus_id
def error_handler(update: Update, context: CallbackContext) -> None: logger.error(msg="Exception occurred while handling an update:", exc_info=context.error) tb_str = ''.join( traceback.format_exception(None, context.error, context.error.__traceback__)) update_str = update.to_dict() if isinstance(update, Update) else str(update) msg = ( f'An exception was raised while handling an update\n' f'<pre>update = {html.escape(json.dumps(update_str, indent=2, ensure_ascii=False))}' '</pre>\n\n' f'<pre>context.chat_data = {html.escape(str(context.chat_data))}</pre>\n\n' f'<pre>context.user_data = {html.escape(str(context.user_data))}</pre>\n\n' f'<pre>{html.escape(tb_str)}</pre>') # os.getenv('username') # Windows msg = msg.replace(os.getlogin(), "USERNAME") msg = msg.replace(os.getcwd(), ".") if len(msg) > 4000: print('Sending Error!') piece_len = len(msg) // 4000 msgs = [msg[4000 * i:4000 * (i + 1)] for i in range(0, piece_len)] + [msg[piece_len * 4000:]] for index, i_msg in enumerate(msgs): context.bot.send_message(chat_id=channel_token, text=f'Page {index}/{len(msgs) - 1}\n\n' + i_msg, parse_mode='HTML') else: context.bot.send_message(chat_id=channel_token, text=msg, parse_mode='HTML')
def __init__(self, username=None): self.username = username self.usercache = DotDict() if self.username != None: self.user_conf = CONF / (self.username + ".json") # 有token(xbox), 且没过期 self.is_xbox_expires() if self.usercache: xsts = self.get_xsts_token(self.usercache.xbox_token) else: code = self.microsoft_account_login() jdata = self.get_access_token(code) logger.debug("access_token ↓") logger.debug(pformat(jdata)) # get xbox token xbox = self.get_xbox_token(jdata["access_token"]) # XSTS xsts = self.get_xsts_token(xbox["Token"]) xsts_token = xsts["Token"] xsts_uhs = xsts["DisplayClaims"]["xui"][0]["uhs"] # MC accesstoken expires 8小时 if self.is_mc_expires(): logger.debug("MC accesstoken 首次请求OR过期重新请求。") # mc access token mc_j = self.get_mc_token(xsts_token, xsts_uhs) # check microsoft acounnts 是否拥有MC, 否则退出。 if not self.check_mc(mc_j["access_token"]): msg = "你的微软账号没有MC" logger.error(msg) raise AuthorizedError(msg) # 添加 MC 启动的 access_token profile = self.get_mc_profile(mc_j["access_token"]) # MC username self.username = profile["name"] self.usercache.username = profile["name"] self.usercache.uuid = profile["id"] self.usercache.mc_access_token = mc_j["access_token"] self.usercache.mc_timestamp = int(time.time()) # 落盘 self.save() # MC accesstoken 没过期。直接使用 else: logger.debug("MC accesstoken 没过期直接使用。")
def execute_read_query(connection, query): try: cursor = connection.cursor() cursor.execute(query) return cursor.fetchall() except Exception as e: logger.error('error {0} occurred while reading; query = {1}'.format( str(e), query))
def execute_query(connection, query): try: connection.cursor().execute(query) connection.commit() except Exception as e: logger.error( 'error {0} occurred while processing query; query = {1}'.format( str(e), query))
def format_report_short(self,): report = self._format_report_short() try: d = simplejson.loads(self.data['logdata'])[0] filename = d['FILENAME'] user = d['USER'] report = report.replace('EXTRA', ' File "{0}", user "{1}".'.format(filename, user)) except Exception as e: logger.error('Could not extract SMB filename from log data: %r' % (self.data)) return report
def delete(page_mark_serie_id: str) -> None: """ delete the record in dynamodb """ try: DYNAMO_TABLE.delete_item( Key=dict(serie_id=page_mark_serie_id), Exists=True, ) except ValidationError as e: logger.error(f"serie {page_mark_serie_id} does not exists in db.") raise e
def cancel_button_handler(call): chatId = call.message.chat.id messageId = call.message.message_id try: if call.message and call.data == 'cancel': with data.create_connection(dbname) as connection: edit_message(chatId, messageId, util._cancel(chatId, connection)) except Exception as e: logger.error(str(e))
def get(serie_id: str) -> Union[None, PageMark]: """ Retrieves a page mark object from db or returns None if no matching key is found. """ try: item = DYNAMO_TABLE.get_item(Key=dict(serie_id=serie_id))['Item'] except ClientError: logger.error('failed to get ', exc_info=True) return None except KeyError: return None return PageMark.deserialize(item)
def try_get_asynearch(): try: response = requests.get(test_url, headers=headers) html = response.text if '上一页' in html: return True else: return False except Exception as e: logger.error(e) return
def init_app(env, **kwargs): try: setup_logging('INFO') if env == '': raise TypeError('Supplied Env value is empty') # (Checks, if we are passing the env value as empty and raises exception) logger.info('Starting APP in {} mode'.format(env)) return create_app() except ValueError as e: logger.error('Gunicorn server failed to start due to ValueError.{}'.format(e)) except Exception as e: logger.error('Gunicorn server failed to start. {}'.format(e))
def get_ms_set_table_bounds(lower_bound: int, upper_bound: int) -> str: """Returns message about setting upper & lower table bound.""" if (lower_bound is not None) and (upper_bound is not None): return FPUpperLowerBounds.get_ms_set_upper_lower_bounds(lower_bound, upper_bound) elif (lower_bound is not None): return FPUpperLowerBounds.get_ms_set_lower_bound(lower_bound) elif (upper_bound is not None): return FPUpperLowerBounds.get_ms_set_upper_bound(upper_bound) logger.error(f"get_ms_table_range_input: End reached.") return ''
def create_connection(name): try: connection = psycopg2.connect(database=name, user=user, password=password, host='127.0.0.1', port='5432') return connection except Exception as e: logger.error( 'error {0} occurred while creating connection to database'.format( str(e)))
def check_answer(handler_input) -> bool: """Returns boolean if user answer is correct.""" table_1, table_2 = QuestionAttr.get_question_tables(handler_input, True) answer = SlotUtils.get_slot_val_by_name(handler_input, 'answer') if not answer: logger.error("check_answer: No answer found.") ## NOTE: if this error is being thrown, need to incorporate ## FallbackUtils.return_unknown_slot_response() return False return (table_1 * table_2 == int(answer))
def count_down(self, c, demonise=True): logger.info('get task redraw pixbuf') if demonise: try: sst = Thread(target=self.cc, args=(self, c)) sst.setDaemon(1) sst.start() # self.cc(c) except: logger.error(traceback.format_exc()) else: self.cc(self, c)
def load(self): try: setup_logging('INFO') logger.info('Starting APP in PROD mode') return create_app() #self.application except Exception as e: logger.error('Falcon APP not loaded.{}'.format(e)) raise falcon.HTTPError( status="503 APP Unavailable", title='Atlantis APP', description='Atlantis falcon APP failed to start', code=503)
def doCleanup(self): """ Executed to setup the driver.""" try: # Clean drivers self.clean_drivers() # Clean gateway if self.udp_socket: self.udp_socket.close() self.udp_socket = None self.server_address = None self.message_id = 0 except Exception as e: logger.error('Exception during cleanup: ' + str(e))
def edit_message(chatId, messageId, message): try: Bot.edit_message_text(message, chatId, message_id=messageId) return True except Exception as e: e = str(e) if 'Forbidden: bot was kicked from the group chat' in e or \ 'Forbidden: bot was blocked by the user' in e: with data.create_connection(dbname) as connection: _clear(chatId, connection) elif not 'Bad Request: message is not modified' in e: logger.error('Unknown error: {0}'.format(e)) return False
def image_provider(config): providerByName = { "local": LocalProvider, "consolegrid": ConsoleGridProvider, } normalize = lambda s: s.strip().lower() names = map(normalize, config.provider_spec.split(",")) instances = map(lambda name: providerByName[name](), names) logger.debug("Creating with component providers: %s" % str(instances)) if len(instances) == 0: logger.error("No image providers specified. Ice will run, but will not \ find grid images for your ROMs. If this wasnt intentional, \ see config.txt.") return CombinedProvider(*instances)
def __exit__(self, exc_type=None, exc_value=None, traceback=None): with self.lock: self.locked -= 1 if self.thread != thread.get_ident(): # print "!ERROR! Thread free not locked lock!" logging.error("Thread free not locked lock!") sys.exit(0) else: if self.locked == 0: if self.calllock: self.thread = None self.calllock = True # gtk.gdk.threads_leave() return None
def transform_page_to_beautiful_soup(url: str) -> Tuple[BeautifulSoup, Dict[str, str]]: """ request a web page and transforms it to a beautiful soup object""" try: bakaupdate_response = requests.get( url, headers=base_headers().update({ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', }) ) except Exception as e: logger.error(f'Failed to request {url}. Error {e}') raise e bakaupdate_response.raise_for_status() return BeautifulSoup(bakaupdate_response.content, features="lxml"), bakaupdate_response.cookies.get_dict()
def get_windows_img(self): """ 在这里我们把file_path这个参数写死,直接保存到我们项目根目录的一个文件夹.\Screenshots下 """ file_path = os.path.dirname(os.path.abspath('.')) + '/screenshots/' rq = time.strftime('%Y%m%d%H%M', time.localtime(time.time())) screen_name = file_path + rq + '.png' print(screen_name) try: self.driver.get_screenshot_as_file(screen_name) logger.error("Had take screenshot and save to folder : /screenshots") except NameError as e: logger.error("Failed to take screenshot! %s" % e) self.get_windows_img()
def __enter__(self, callLock=True, orig=""): with self.lock: DoLock = (thread.get_ident() != self.thread) if self.warn and self.mainthread != thread.get_ident(): logging.error("GUI accessed from wrong thread! %s\nTraceback: %s", orig, "".join(traceback.format_stack())) if DoLock: if callLock: # gtk.gdk.threads_enter() self.calllock = True else: self.calllock = False with self.lock: self.thread = thread.get_ident() self.locked += 1 return None
def read(url, referer): content = "" ua = choose() headers = { 'User-Agent': ua, 'Referer': referer, 'Content-Type': 'text/html', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding': 'gzip' # 以gzip下载html,降低网络资源负荷 } cookies = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies)) req_referer = urllib2.Request( url=referer, headers=headers ) opener.open(req_referer, timeout=10) # 用于记录cookie req = urllib2.Request( url=url, headers=headers ) try: result = opener.open(req, timeout=20) import StringIO import gzip predata = result.read() stream = StringIO.StringIO(predata) gzipper = gzip.GzipFile(fileobj=stream) try: content = gzipper.read() print "OK" except: #如果服务器不支持gzip,那么就直接下载网页 content = predata result.close() except urllib2.HTTPError, e: logger.error("Error Code: %s. Reason: %s. Request URL: %s", str(e.code), str(e.reason), url)
def create_incident(cls, type_, data=None): print '{0}: {1}'.format(type_, data) logger.debug('Creating incident type: {0}'.format(type_)) if type_ == 'ftp.login_attempt': IncidentFTPLogin(data=data, write_object=True) elif type_ == 'http.login_attempt': IncidentHTTPLogin(data=data, write_object=True) elif type_ == 'ssh.login_attempt': IncidentSSHLogin(data=data, write_object=True) elif type_ == 'smb.file_open': IncidentSMBFileOpen(data=data, write_object=True) elif type_ == 'scans.host_portscan': IncidentHostPortScan(data=data, write_object=True) elif type_ == 'scans.network_portscan': IncidentNetworkPortScan(data=data, write_object=True) elif type_ == 'telnet.login_attempt': IncidentTelnetLogin(data=data, write_object=True) elif type_ == 'httpproxy.login_attempt': IncidentHTTProxy(data=data, write_object=True) elif type_ == 'mysql.login_attempt': IncidentMySQLLogin(data=data, write_object=True) elif type_ == 'mssql.login_attempt': IncidentMSSQLLogin(data=data, write_object=True) elif type_ == 'tftp.action': IncidentTFTP(data=data, write_object=True) elif type_ == 'ntp.monlist': IncidentNTPMonList(data=data, write_object=True) elif type_ == 'vnc.login_attempt': IncidentVNCLogin(data=data, write_object=True) elif type_ == 'snmp.cmd': IncidentSNMP(data=data, write_object=True) elif type_ == 'rdp.login_attempt': IncidentRDPLogin(data=data, write_object=True) elif type_ == 'sip.login_attempt': IncidentSIPLogin(data=data, write_object=True) else: logger.error('Unknown incident type: {0}'.format(type_))
def GUIrun(clean=True): try: t = time.time() n = 0 while True: Run = IGuiCaller.get(0) try: n += 1 t0 = time.time() Run() t0 = time.time() - t0 if t0 > 0.1: logging.info("Run=%r took %r", Run, t0) except Exception, E: logging.error("GUIrun/caller: %s\nRun = %r\nTraceback: %s", E, Run, traceback.format_exc()) tt = time.time() if tt - t > 0.05: logging.error("GUIrun/caller: proceed %d for %r; left=%d", n, tt - t, IGuiCaller.qsize()) break except Queue.Empty: pass try: t = time.time() n = 0 while True: Run = IGuiIdleCaller.get(0) try: n += 1 t0 = time.time() Run() t0 = time.time() - t0 if t0 > 0.1: logging.info("Run=%r took %r", Run, t0) except Exception, E: logging.error("GUIrun/idle: %s\nRun = %r\nTraceback: %s", E, Run, traceback.format_exc()) tt = time.time() if tt - t > 0.05: logging.error("GUIrun/idle: proceed %d for %r; left=%d", n, tt - t, IGuiIdleCaller.qsize()) break except Queue.Empty: pass if clean: with IdleCallerLock: IdleCaller[0] = IGuiCaller.qsize() > 0 or IGuiIdleCaller.qsize() > 0 return IdleCaller[0]
def _say(data): try: ss = subprocess.Popen(data) ss.wait() except: logger.error(traceback.format_exc())
import StringIO import gzip predata = result.read() stream = StringIO.StringIO(predata) gzipper = gzip.GzipFile(fileobj=stream) try: content = gzipper.read() print "OK" except: #如果服务器不支持gzip,那么就直接下载网页 content = predata result.close() except urllib2.HTTPError, e: logger.error("Error Code: %s. Reason: %s. Request URL: %s", str(e.code), str(e.reason), url) except urllib2.URLError, e: logger.error("Error Reason: %s. Request URL: %s", str(e.reason), url) return content class RedirectHandler(urllib2.HTTPRedirectHandler): def __init__(self): pass def http_error_302(self, req, fp, code, msg, headers): return headers['Content-Length'] def get_data_size(url, referer): ua = choose() headers = {