def parse_html(html): soup = BeautifulSoup(html, 'lxml') tables = soup.find('div', id='wrapper').find_all('table') for table in tables: img = table.find('img')['src'] name = table.select('.pl2')[0].a['title'] string = unicode(table.find('p').string) lst = string.split(' / ') author = lst[0].strip() publisher = lst[-3].strip() date = lst[-2].strip() price = lst[-1].strip() credit = unicode(table.select('.rating_nums')[0].string) try: desc = unicode(table.select('.inq')[0].string) except IndexError: desc = '无' global id id += 1 yield { 'id': id, 'img': img, 'name': name, 'author': author, 'publisher': publisher, 'date': date, 'price': price, 'credit': float(credit), 'desc': desc }
def safe_unicode(obj, *args): """ return the unicode representation of obj """ try: return unicode(obj, *args) except UnicodeDecodeError: # obj is byte string ascii_text = str(obj).encode('string_escape') return unicode(ascii_text)
def _write_status(self): b = game.get_board() cbm = self._chess_camera.current_colored_board_mask() cbm_board = pawn_board_from_colored_board_mask(cbm) pgn = game.pgn() formatted_datetime = time.strftime("%x %X") text = unicode("\n\n").join( [formatted_datetime, unicode(cbm_board), unicode(b), pgn]) with io.open(lib_path('status.txt'), 'w', encoding='utf8') as f: f.write(text)
def __unicode__(self) -> str: """ Gets the unicode of the string representation of the dictionary. :return: The unicode of the string representation of the dictionary. :rtype: str """ return idna.unicode(repr(self.__range_key_dict))
def image_limiter(): with open('location.json', 'r') as file: result = json.loads(file.read()) images_total = result[0]['Images total'] processed = {"pages": "?", "result": []} percentual = {} for zone in result[1:]: for z in zone: per = ((100 * zone[z][0]['count']) / images_total) percentual[z] = f"{'%.2f'%(per)}%" zone_limit = ceil(per * limit / 100) processed['result'].append(zone[z][1:zone_limit]) processed['result'] = [j for i in processed['result'] for j in i] with open('percentual.json', 'w+', encoding='utf8') as file: result = json.dumps(percentual, indent=3, ensure_ascii=False) file.write(unicode(result)) file.close() with open('processed.json', 'w+', encoding='utf8') as file: result = json.dumps(processed, indent=3) file.write(result) file.close() download_images()
def check_light(): s = requests.Session() r = s.get('https://dom.jch-online.com/ru').text r = r[r.find('<meta name="csrf-token" content="') + 33:] aut_token = r[:r.find('"')] data = { 'utf8': '✓', 'authenticity_token': aut_token, 'username': '******', 'password': '******', } s.post('https://dom.jch-online.com/ru/login', data=data) headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', 'Cookie': '_jchbyt_session_v27=88e239f6df19e1abb7c8dea93766133f', } r = s.get('https://dom.jch-online.com/ru/accounts', headers=headers).text r = r[r.find('<meta name="csrf-token" content="') + 33:] xcsrf_token = r[:r.find('"')] r = r[r.find('<li class="account selected" data-param="') + 41:] access = r[:r.find('"')] headers.update({'X-CSRF-Token': xcsrf_token, 'X-ACCESS-TOKEN': access}) r = s.post('https://dom.jch-online.com/ru/accounts/643486/saldo.json', headers=headers) r = unicode(r.content, 'utf-8') status = r[r.find(':') + 2:r.find(',') - 1] value = r[r.find('"sum":"') + 7:r.find('","ac')] return status + ' ' + value
def format_str(content): content = unicode(content,'utf-8') content_str = '' for i in content: if is_chinese(i): content_str = content_str+i return content_str
def sendmail(subject, msg, toaddrs, fromaddr, smtpaddr, password): ''''' @subject:邮件主题 @msg:邮件内容 @toaddrs:收信人的邮箱地址 @fromaddr:发信人的邮箱地址 @smtpaddr:smtp服务地址,可以在邮箱看,比如163邮箱为smtp.163.com @password:发信人的邮箱密码 ''' mail_msg = MIMEMultipart() if not isinstance(subject, unicode): subject = unicode(subject, 'utf-8') mail_msg['Subject'] = subject mail_msg['From'] = fromaddr mail_msg['To'] = ','.join(toaddrs) mail_msg.attach(MIMEText(msg, 'html', 'utf-8')) try: s = smtplib.SMTP() s.connect(smtpaddr) # 连接smtp服务器 s.login(fromaddr, password) # 登录邮箱 s.sendmail(fromaddr, toaddrs, mail_msg.as_string()) # 发送邮件 s.quit() except Exception as e: print("Error: unable to send email") print(traceback.format_exc())
def safe_str(obj): """ return the byte string representation of obj """ try: return str(obj) except UnicodeEncodeError: # obj is unicode return unicode(obj).encode('unicode_escape')
def export_as_csv(modeladmin, request, queryset): """ Generic csv export admin action. based on http://djangosnippets.org/snippets/1697/ """ opts = modeladmin.model._meta field_names = set([field.name for field in opts.fields]) if fields: fieldset = set(fields) field_names = field_names & fieldset elif exclude: excludeset = set(exclude) field_names = field_names - excludeset response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename=%s.csv' % unicode( opts).replace('.', '_') writer = csv.writer(response) if header: writer.writerow(list(field_names)) for obj in queryset: writer.writerow([getattr(obj, field) for field in field_names]) return response
def get(self, response, format=None): user = unicode(response.user.id) final_expenses_v02 = make_a_list(response) x_data, y_data = map(list, zip(*final_expenses_v02)) data = { "labels": x_data, "chartdata": y_data, } return Response(data)
def health_check(request, **kwargs): if request.version == 'v1': referer = request.META.get('HTTP_REFERER') if referer: print(referer) else: print("No referrer") content = { 'user': unicode(request.user), # django.contrib.auth.User` instance. 'auth': unicode(request.auth), # None } print(content) return Response(data={'msg': 'Ok'}, status=status.HTTP_200_OK) elif request.version == 'v2': return Response(data={'msg': 'Not implemented yet'}, status=status.HTTP_404_NOT_FOUND)
def _run(self): if not self._stop_event.isSet(): try: self._fn() except Exception: logger = get_logger(unicode(self._fn)) logger.exception("Failed to execute PeriodicTask.") finally: gevent.spawn_later(self._period, self._run)
def facebook_handler(request, stage): if stage == "login": request.log_action("facebook.login") if 'next' in request.GET: request.session["fb_next"] = request.GET['next'] request.session.modified = True elif "HTTP_REFERER" in request.META: request.session["fb_next"] = request.META["HTTP_REFERER"] request.session.modified = True return redirect(login_url(fbview_url(request, "auth"))) elif stage == "auth": request.log_action("facebook.auth") return redirect( auth_url(fbview_url(request, "done"), ["public_profile", "email", "user_friends"])) elif stage == "done": next_uri = "/" if "fb_next" in request.session: next_uri = request.session["fb_next"] del request.session["fb_next"] request.session.modified = True code = request.GET.get("code") if code: try: token = loads(url_read(token_url(request, code))) data = loads(url_read(profile_url(token))) except Exception as e: request.log_action("facebook.url_read.exception", {"message": unicode(e)}) raise try: profile = Profile.objects.get(fb_id=data.get("id")) request_profile = request.get_user().profile if request.user.is_authenticated(): request.log_action("facebook.merge", {"id": data.get("id")}, profile) profile.merge_from_other(request_profile) user = profile.user request.set_user(user) except Profile.DoesNotExist: user = request.get_user() profile = user.profile profile.update_from_fb_data(token, data) request.log_action("facebook.connect", {"data": data}, profile) return redirect(next_uri) else: request.log_action("facebook.error", {"params": request.GET}) return redirect("/fb_error")
def slugify(value): """ Normalizes string, converts to lowercase, removes non-alpha characters, and converts spaces to hyphens. From Django's "django/template/defaultfilters.py". Copied from: https://gist.github.com/berlotto/6295018 """ _slugify_strip_re = re.compile(r'[^\w\s-]') _slugify_hyphenate_re = re.compile(r'[-\s]+') if not isinstance(value, unicode): value = unicode(value) value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = unicode(_slugify_strip_re.sub('', value).strip().lower()) return _slugify_hyphenate_re.sub('-', value)
def post(self, request): serializer = AuthenticationSerializer(data=request.data) serializer.is_valid(raise_exception=True) user = serializer.validated_data['user'] token, created = Token.objects.get_or_create(user=user) userObject = User.objects.get(email=user) us = UserSerializer(userObject) content = {'token': unicode(token.key), 'user': us.data} return Response(content)
def bot_respond(message, fi): message = message.lower() if find_if_there(unflexible_with_sites, message) == True: # string = GhurboFlight.detect_flight_query(message) # if string!="": # string=GhurboFlight.ask_info # print (GhurboFlight.ask_info) # return unicode(string) #### string=GhurboFlight.ask_info fi.id = 0 string = FlightConversationTemplate.ask_info.__getitem__(0) return unicode(string) print(fi.departure) if fi.id == 0 and FlightConversationTemplate.detect_place( message) != fi.departure and fi.departure != "": string = FlightConversationTemplate.ask_info.__getitem__(2) return string if fi.id == 0 and FlightConversationTemplate.detect_place( message) is not None: string = FlightConversationTemplate.ask_info.__getitem__(1) fi.departure = FlightConversationTemplate.detect_place(message) return unicode(string) if message in greeting_responses: hour = datetime.datetime.now().hour print(hour) if (hour > 6 and hour < 12): string = greeting_responses[message].__getitem__(0) elif (hour >= 12 and hour < 18): string = greeting_responses[message].__getitem__(1) elif hour >= 18 and hour < 20: string = greeting_responses[message].__getitem__(2) else: string = greeting_responses[message].__getitem__(3) string = string + bot_welcome reply = unicode(string) return reply string = GhurboFlight.detect_flight_query(message) if (string != ""): return unicode(string) string = bot_sorry return unicode(string)
def slugify(s): slug = unicodedata.normalize('NFKD', unicode(s)) slug = slug.encode('ascii', 'ignore').lower() slug = re.sub(r"[^a-z0-9]+", '_', slug.decode("utf-8")).strip('_') # To avoid slug collision, append hash of raw string m = md5() m.update(s.encode('utf8')) slug = '{0}_{1}'.format(slug, m.hexdigest()) return slug
def sub_emoji(text): try: from idna import unicode text = unicode(text, "utf-8") except TypeError as e: pass try: highpoints = re.compile(u'[\U00010000-\U0010ffff]') except re.error: highpoints = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]') return highpoints.sub(u'', text)
def get_token(self, api_key, api_secret): flickr = flickrapi.FlickrAPI(api_key, api_secret, store_token=False) flickr.get_request_token(oauth_callback='oob') authorize_url = flickr.auth_url(perms=u'write') print(authorize_url) verifier = unicode(raw_input('Verifier code: ')) flickr.get_access_token(verifier) return flickr.token_cache.token.token, flickr.token_cache.token.token_secret
class TechCrunchArticleLoader(ItemLoader): default_input_processor = MapCompose(lambda s: unicode(s, "utf-8"), unicode.strip) default_output_processor = Join() title_in = MapCompose(unicode.strip, unicode.title) title_out = Join() text_in = MapCompose(unicode.strip) text_out = Join() tags_in = MapCompose(unicode.strip) tags_out = Join(separator=u'; ')
def show_metadata(request, id): file_name = request.POST.get("transformed_file", "") if file_name == "" or not experiment.file_exists(id, file_name): return render(request, '404.html') file_content = unicode(experiment.read_file(id, file_name), "utf-8") return render( request, 'nix/metadata.html', { 'experiment_id': id, 'transformed_files': experiment.get_json_ld_files(id), 'selected_file': file_name, 'file_content': file_content })
def get_id(self): """get user id from profile file, if not exist, it will generate a uuid for the user. """ if self.username is not None: try: with open(PROFILE_FILE) as f: user_profiles = json.load(f) if self.username in user_profiles: return user_profiles[self.username][1] except IOError: pass except ValueError: pass return unicode(uuid.uuid4())
def converstation(message, fi): message = message.lower() # res=detect_place(message) # if res is not None: # string=ask_info.__getitem__(1) # fi.departure=res # return string res = BotArchitecture.detect_date(message) if res is not None: fi.date = res string = ask_info.__getitem__(2) return string res = detect_class(message) if res is not None: string = ask_info.__getitem__(3) fi.passenger_class = res return string # # if fi.id>6: # return unicode("okay") # string=ask_info.__getitem__(fi.id); # if fi.id==1: # res= detect_place(message) # fi.departure=res # fi.id=fi.id+1 # elif fi.id==2: # res = detect_place(message) # fi.destination=res # fi.id=fi.id+1 # elif fi.id==3: # res = BotArchitecture.detect_date(message) # fi.date=res # fi.id=fi.id+1 # elif fi.id==4: # res = detect_class(message) # fi.passenger_class=res # fi.id=fi.id+1 # elif fi.id==5: # res = detect_adult_child_infant(message,fi) # fi.id=fi.id+1 # elif fi.id==6: # res = detect_up_down(message) # fi.id=fi.id+1 # if res!=-1: # fi.up_down=res return unicode(string)
def getCurrentWeather(): if myID.has_currently() is True: currently = FIOCurrently.FIOCurrently(myID) print('Currently') for item in currently.get().keys(): time = "" if item == 'time': time = datetime.utcfromtimestamp( currently.get()[item]).strftime('%H:%M:%S %d-%m-%Y ') extra = "" + time print(" " + item + ' : ' + unicode(currently.get()[item]) + " " + extra) # Or access attributes directly print(currently.temperature) print(currently.humidity) else: print('No Currently data')
def strip_accents(text): """ Strip accents from input String. :param text: The input string. :type text: String. :returns: The processed String. :rtype: String. """ try: text = unicode(text, 'utf-8') except (TypeError, NameError): # unicode is a default on python 3 pass text = unicodedata.normalize('NFD', text) text = text.encode('ascii', 'ignore') text = text.decode("utf-8") return str(text)
def get_files(root_path, cur_path, allow_types=[]): files = [] items = os.listdir(cur_path) for item in items: item = unicode(item) item_fullname = os.path.join(root_path, cur_path, item).replace("\\", "/") if os.path.isdir(item_fullname): files.extend(get_files(root_path, item_fullname, allow_types)) else: ext = os.path.splitext(item_fullname)[1] is_allow_list = (len(allow_types) == 0) or (ext in allow_types) if is_allow_list: files.append({ "url": urljoin(USettings.gSettings.MEDIA_URL, os.path.join(os.path.relpath(cur_path, root_path), item).replace("\\", "/")), "mtime": os.path.getmtime(item_fullname) }) return files
def sms_msg_cut(str, dstlen): # 2017.11. 9 - LSH # dstlen 길이로 나누어 list 에 담는다. text = "" strlist = [] for s in unicode(str): try: stemp = text + s strlen = len(stemp.decode('utf-8').encode('euc-kr')) if strlen > dstlen: strlist.append(text) text = s else: text = stemp except: continue # 마지막 김밥 꼬다리 저장. strlist.append(text) return strlist
def dump_json_file(json_data, json_file_abs_path): """ dump json data to file 转储json数据到文件 """ class PythonObjectEncoder(json.JSONEncoder): def default(self, obj): try: return super().default(self, obj) except TypeError: return str(obj) file_foder_path = os.path.dirname(json_file_abs_path) if not os.path.isdir(file_foder_path): os.makedirs(file_foder_path) try: with io.open(json_file_abs_path, 'w', encoding='utf-8') as outfile: if is_py2: outfile.write( unicode( json.dumps(json_data, indent=4, separators=(',', ':'), encoding="utf8", ensure_ascii=False, cls=PythonObjectEncoder))) else: json.dump(json_data, outfile, indent=4, separators=(',', ':'), ensure_ascii=False, cls=PythonObjectEncoder) msg = "dump file: {}".format(json_file_abs_path) logger.color_print(msg, "BLUE") except TypeError as ex: msg = "Failed to dump json file: {}\nReason: {}".format( json_file_abs_path, ex) logger.color_print(msg, "RED")
def command(self, command, arg=None): """ Initialize a socket connection, send a command (a json encoded dict) and receive the response (and decode it). """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect((self.host, self.port)) payload = {"command": command} if arg is not None: # Parameter must be converted to basestring (no int) payload.update({'parameter': unicode(arg)}) sock.send(json.dumps(payload).encode()) received = self._receive(sock) finally: sock.shutdown(socket.SHUT_RDWR) sock.close() return json.loads(received)