def parse_html(html):
soup = BeautifulSoup(html, 'lxml')
tables = soup.find('div', id='wrapper').find_all('table')
for table in tables:
img = table.find('img')['src']
name = table.select('.pl2')[0].a['title']
string = unicode(table.find('p').string)
lst = string.split(' / ')
author = lst[0].strip()
publisher = lst[-3].strip()
date = lst[-2].strip()
price = lst[-1].strip()
credit = unicode(table.select('.rating_nums')[0].string)
try:
desc = unicode(table.select('.inq')[0].string)
except IndexError:
desc = '无'
global id
id += 1
yield {
'id': id,
'img': img,
'name': name,
'author': author,
'publisher': publisher,
'date': date,
'price': price,
'credit': float(credit),
'desc': desc
}
def safe_unicode(obj, *args):
""" return the unicode representation of obj """
try:
return unicode(obj, *args)
except UnicodeDecodeError:
# obj is byte string
ascii_text = str(obj).encode('string_escape')
return unicode(ascii_text)
def _write_status(self):
b = game.get_board()
cbm = self._chess_camera.current_colored_board_mask()
cbm_board = pawn_board_from_colored_board_mask(cbm)
pgn = game.pgn()
formatted_datetime = time.strftime("%x %X")
text = unicode("\n\n").join(
[formatted_datetime,
unicode(cbm_board),
unicode(b), pgn])
with io.open(lib_path('status.txt'), 'w', encoding='utf8') as f:
f.write(text)
def __unicode__(self) -> str:
"""
Gets the unicode of the string representation of the dictionary.
:return: The unicode of the string representation of the dictionary.
:rtype: str
"""
return idna.unicode(repr(self.__range_key_dict))
def image_limiter():
with open('location.json', 'r') as file:
result = json.loads(file.read())
images_total = result[0]['Images total']
processed = {"pages": "?", "result": []}
percentual = {}
for zone in result[1:]:
for z in zone:
per = ((100 * zone[z][0]['count']) / images_total)
percentual[z] = f"{'%.2f'%(per)}%"
zone_limit = ceil(per * limit / 100)
processed['result'].append(zone[z][1:zone_limit])
processed['result'] = [j for i in processed['result'] for j in i]
with open('percentual.json', 'w+', encoding='utf8') as file:
result = json.dumps(percentual, indent=3, ensure_ascii=False)
file.write(unicode(result))
file.close()
with open('processed.json', 'w+', encoding='utf8') as file:
result = json.dumps(processed, indent=3)
file.write(result)
file.close()
download_images()
def check_light():
s = requests.Session()
r = s.get('https://dom.jch-online.com/ru').text
r = r[r.find('<meta name="csrf-token" content="') + 33:]
aut_token = r[:r.find('"')]
data = {
'utf8': '✓',
'authenticity_token': aut_token,
'username': '******',
'password': '******',
}
s.post('https://dom.jch-online.com/ru/login', data=data)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
'Cookie': '_jchbyt_session_v27=88e239f6df19e1abb7c8dea93766133f',
}
r = s.get('https://dom.jch-online.com/ru/accounts', headers=headers).text
r = r[r.find('<meta name="csrf-token" content="') + 33:]
xcsrf_token = r[:r.find('"')]
r = r[r.find('<li class="account selected" data-param="') + 41:]
access = r[:r.find('"')]
headers.update({'X-CSRF-Token': xcsrf_token, 'X-ACCESS-TOKEN': access})
r = s.post('https://dom.jch-online.com/ru/accounts/643486/saldo.json',
headers=headers)
r = unicode(r.content, 'utf-8')
status = r[r.find(':') + 2:r.find(',') - 1]
value = r[r.find('"sum":"') + 7:r.find('","ac')]
return status + ' ' + value
def format_str(content):
content = unicode(content,'utf-8')
content_str = ''
for i in content:
if is_chinese(i):
content_str = content_str+i
return content_str
def sendmail(subject, msg, toaddrs, fromaddr, smtpaddr, password):
'''''
@subject:邮件主题
@msg:邮件内容
@toaddrs:收信人的邮箱地址
@fromaddr:发信人的邮箱地址
@smtpaddr:smtp服务地址，可以在邮箱看，比如163邮箱为smtp.163.com
@password:发信人的邮箱密码
'''
mail_msg = MIMEMultipart()
if not isinstance(subject, unicode):
subject = unicode(subject, 'utf-8')
mail_msg['Subject'] = subject
mail_msg['From'] = fromaddr
mail_msg['To'] = ','.join(toaddrs)
mail_msg.attach(MIMEText(msg, 'html', 'utf-8'))
try:
s = smtplib.SMTP()
s.connect(smtpaddr) # 连接smtp服务器
s.login(fromaddr, password) # 登录邮箱
s.sendmail(fromaddr, toaddrs, mail_msg.as_string()) # 发送邮件
s.quit()
except Exception as e:
print("Error: unable to send email")
print(traceback.format_exc())
def safe_str(obj):
""" return the byte string representation of obj """
try:
return str(obj)
except UnicodeEncodeError:
# obj is unicode
return unicode(obj).encode('unicode_escape')
def export_as_csv(modeladmin, request, queryset):
"""
Generic csv export admin action.
based on http://djangosnippets.org/snippets/1697/
"""
opts = modeladmin.model._meta
field_names = set([field.name for field in opts.fields])
if fields:
fieldset = set(fields)
field_names = field_names & fieldset
elif exclude:
excludeset = set(exclude)
field_names = field_names - excludeset
response = HttpResponse(content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename=%s.csv' % unicode(
opts).replace('.', '_')
writer = csv.writer(response)
if header:
writer.writerow(list(field_names))
for obj in queryset:
writer.writerow([getattr(obj, field) for field in field_names])
return response
def get(self, response, format=None):
user = unicode(response.user.id)
final_expenses_v02 = make_a_list(response)
x_data, y_data = map(list, zip(*final_expenses_v02))
data = {
"labels": x_data,
"chartdata": y_data,
}
return Response(data)
def health_check(request, **kwargs):
if request.version == 'v1':
referer = request.META.get('HTTP_REFERER')
if referer:
print(referer)
else:
print("No referrer")
content = {
'user':
unicode(request.user), # django.contrib.auth.User` instance.
'auth': unicode(request.auth), # None
}
print(content)
return Response(data={'msg': 'Ok'}, status=status.HTTP_200_OK)
elif request.version == 'v2':
return Response(data={'msg': 'Not implemented yet'},
status=status.HTTP_404_NOT_FOUND)
def _run(self):
if not self._stop_event.isSet():
try:
self._fn()
except Exception:
logger = get_logger(unicode(self._fn))
logger.exception("Failed to execute PeriodicTask.")
finally:
gevent.spawn_later(self._period, self._run)
def facebook_handler(request, stage):
if stage == "login":
request.log_action("facebook.login")
if 'next' in request.GET:
request.session["fb_next"] = request.GET['next']
request.session.modified = True
elif "HTTP_REFERER" in request.META:
request.session["fb_next"] = request.META["HTTP_REFERER"]
request.session.modified = True
return redirect(login_url(fbview_url(request, "auth")))
elif stage == "auth":
request.log_action("facebook.auth")
return redirect(
auth_url(fbview_url(request, "done"),
["public_profile", "email", "user_friends"]))
elif stage == "done":
next_uri = "/"
if "fb_next" in request.session:
next_uri = request.session["fb_next"]
del request.session["fb_next"]
request.session.modified = True
code = request.GET.get("code")
if code:
try:
token = loads(url_read(token_url(request, code)))
data = loads(url_read(profile_url(token)))
except Exception as e:
request.log_action("facebook.url_read.exception",
{"message": unicode(e)})
raise
try:
profile = Profile.objects.get(fb_id=data.get("id"))
request_profile = request.get_user().profile
if request.user.is_authenticated():
request.log_action("facebook.merge",
{"id": data.get("id")}, profile)
profile.merge_from_other(request_profile)
user = profile.user
request.set_user(user)
except Profile.DoesNotExist:
user = request.get_user()
profile = user.profile
profile.update_from_fb_data(token, data)
request.log_action("facebook.connect", {"data": data}, profile)
return redirect(next_uri)
else:
request.log_action("facebook.error", {"params": request.GET})
return redirect("/fb_error")
def slugify(value):
"""
Normalizes string, converts to lowercase, removes non-alpha characters,
and converts spaces to hyphens.
From Django's "django/template/defaultfilters.py".
Copied from: https://gist.github.com/berlotto/6295018
"""
_slugify_strip_re = re.compile(r'[^\w\s-]')
_slugify_hyphenate_re = re.compile(r'[-\s]+')
if not isinstance(value, unicode):
value = unicode(value)
value = unicodedata.normalize('NFKD',
value).encode('ascii',
'ignore').decode('ascii')
value = unicode(_slugify_strip_re.sub('', value).strip().lower())
return _slugify_hyphenate_re.sub('-', value)
def post(self, request):
serializer = AuthenticationSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
user = serializer.validated_data['user']
token, created = Token.objects.get_or_create(user=user)
userObject = User.objects.get(email=user)
us = UserSerializer(userObject)
content = {'token': unicode(token.key), 'user': us.data}
return Response(content)
def bot_respond(message, fi):
message = message.lower()
if find_if_there(unflexible_with_sites, message) == True:
# string = GhurboFlight.detect_flight_query(message)
# if string!="":
# string=GhurboFlight.ask_info
# print (GhurboFlight.ask_info)
# return unicode(string)
#### string=GhurboFlight.ask_info
fi.id = 0
string = FlightConversationTemplate.ask_info.__getitem__(0)
return unicode(string)
print(fi.departure)
if fi.id == 0 and FlightConversationTemplate.detect_place(
message) != fi.departure and fi.departure != "":
string = FlightConversationTemplate.ask_info.__getitem__(2)
return string
if fi.id == 0 and FlightConversationTemplate.detect_place(
message) is not None:
string = FlightConversationTemplate.ask_info.__getitem__(1)
fi.departure = FlightConversationTemplate.detect_place(message)
return unicode(string)
if message in greeting_responses:
hour = datetime.datetime.now().hour
print(hour)
if (hour > 6 and hour < 12):
string = greeting_responses[message].__getitem__(0)
elif (hour >= 12 and hour < 18):
string = greeting_responses[message].__getitem__(1)
elif hour >= 18 and hour < 20:
string = greeting_responses[message].__getitem__(2)
else:
string = greeting_responses[message].__getitem__(3)
string = string + bot_welcome
reply = unicode(string)
return reply
string = GhurboFlight.detect_flight_query(message)
if (string != ""):
return unicode(string)
string = bot_sorry
return unicode(string)
def slugify(s):
slug = unicodedata.normalize('NFKD', unicode(s))
slug = slug.encode('ascii', 'ignore').lower()
slug = re.sub(r"[^a-z0-9]+", '_', slug.decode("utf-8")).strip('_')
# To avoid slug collision, append hash of raw string
m = md5()
m.update(s.encode('utf8'))
slug = '{0}_{1}'.format(slug, m.hexdigest())
return slug
def sub_emoji(text):
try:
from idna import unicode
text = unicode(text, "utf-8")
except TypeError as e:
pass
try:
highpoints = re.compile(u'[\U00010000-\U0010ffff]')
except re.error:
highpoints = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')
return highpoints.sub(u'', text)
def get_token(self, api_key, api_secret):
flickr = flickrapi.FlickrAPI(api_key, api_secret, store_token=False)
flickr.get_request_token(oauth_callback='oob')
authorize_url = flickr.auth_url(perms=u'write')
print(authorize_url)
verifier = unicode(raw_input('Verifier code: '))
flickr.get_access_token(verifier)
return flickr.token_cache.token.token, flickr.token_cache.token.token_secret
class TechCrunchArticleLoader(ItemLoader):
default_input_processor = MapCompose(lambda s: unicode(s, "utf-8"),
unicode.strip)
default_output_processor = Join()
title_in = MapCompose(unicode.strip, unicode.title)
title_out = Join()
text_in = MapCompose(unicode.strip)
text_out = Join()
tags_in = MapCompose(unicode.strip)
tags_out = Join(separator=u'; ')
def show_metadata(request, id):
file_name = request.POST.get("transformed_file", "")
if file_name == "" or not experiment.file_exists(id, file_name):
return render(request, '404.html')
file_content = unicode(experiment.read_file(id, file_name), "utf-8")
return render(
request, 'nix/metadata.html', {
'experiment_id': id,
'transformed_files': experiment.get_json_ld_files(id),
'selected_file': file_name,
'file_content': file_content
})
def get_id(self):
"""get user id from profile file, if not exist, it will
generate a uuid for the user.
"""
if self.username is not None:
try:
with open(PROFILE_FILE) as f:
user_profiles = json.load(f)
if self.username in user_profiles:
return user_profiles[self.username][1]
except IOError:
pass
except ValueError:
pass
return unicode(uuid.uuid4())
def converstation(message, fi):
message = message.lower()
# res=detect_place(message)
# if res is not None:
# string=ask_info.__getitem__(1)
# fi.departure=res
# return string
res = BotArchitecture.detect_date(message)
if res is not None:
fi.date = res
string = ask_info.__getitem__(2)
return string
res = detect_class(message)
if res is not None:
string = ask_info.__getitem__(3)
fi.passenger_class = res
return string
#
# if fi.id>6:
# return unicode("okay")
# string=ask_info.__getitem__(fi.id);
# if fi.id==1:
# res= detect_place(message)
# fi.departure=res
# fi.id=fi.id+1
# elif fi.id==2:
# res = detect_place(message)
# fi.destination=res
# fi.id=fi.id+1
# elif fi.id==3:
# res = BotArchitecture.detect_date(message)
# fi.date=res
# fi.id=fi.id+1
# elif fi.id==4:
# res = detect_class(message)
# fi.passenger_class=res
# fi.id=fi.id+1
# elif fi.id==5:
# res = detect_adult_child_infant(message,fi)
# fi.id=fi.id+1
# elif fi.id==6:
# res = detect_up_down(message)
# fi.id=fi.id+1
# if res!=-1:
# fi.up_down=res
return unicode(string)
def getCurrentWeather():
if myID.has_currently() is True:
currently = FIOCurrently.FIOCurrently(myID)
print('Currently')
for item in currently.get().keys():
time = ""
if item == 'time':
time = datetime.utcfromtimestamp(
currently.get()[item]).strftime('%H:%M:%S %d-%m-%Y ')
extra = "" + time
print(" " + item + ' : ' + unicode(currently.get()[item]) + " " +
extra)
# Or access attributes directly
print(currently.temperature)
print(currently.humidity)
else:
print('No Currently data')
def strip_accents(text):
"""
Strip accents from input String.
:param text: The input string.
:type text: String.
:returns: The processed String.
:rtype: String.
"""
try:
text = unicode(text, 'utf-8')
except (TypeError, NameError): # unicode is a default on python 3
pass
text = unicodedata.normalize('NFD', text)
text = text.encode('ascii', 'ignore')
text = text.decode("utf-8")
return str(text)
def get_files(root_path, cur_path, allow_types=[]):
files = []
items = os.listdir(cur_path)
for item in items:
item = unicode(item)
item_fullname = os.path.join(root_path, cur_path, item).replace("\\", "/")
if os.path.isdir(item_fullname):
files.extend(get_files(root_path, item_fullname, allow_types))
else:
ext = os.path.splitext(item_fullname)[1]
is_allow_list = (len(allow_types) == 0) or (ext in allow_types)
if is_allow_list:
files.append({
"url": urljoin(USettings.gSettings.MEDIA_URL, os.path.join(os.path.relpath(cur_path, root_path), item).replace("\\", "/")),
"mtime": os.path.getmtime(item_fullname)
})
return files
def sms_msg_cut(str, dstlen):
# 2017.11. 9 - LSH
# dstlen 길이로 나누어 list 에 담는다.
text = ""
strlist = []
for s in unicode(str):
try:
stemp = text + s
strlen = len(stemp.decode('utf-8').encode('euc-kr'))
if strlen > dstlen:
strlist.append(text)
text = s
else:
text = stemp
except:
continue
# 마지막 김밥 꼬다리 저장.
strlist.append(text)
return strlist
def dump_json_file(json_data, json_file_abs_path):
""" dump json data to file
转储json数据到文件
"""
class PythonObjectEncoder(json.JSONEncoder):
def default(self, obj):
try:
return super().default(self, obj)
except TypeError:
return str(obj)
file_foder_path = os.path.dirname(json_file_abs_path)
if not os.path.isdir(file_foder_path):
os.makedirs(file_foder_path)
try:
with io.open(json_file_abs_path, 'w', encoding='utf-8') as outfile:
if is_py2:
outfile.write(
unicode(
json.dumps(json_data,
indent=4,
separators=(',', ':'),
encoding="utf8",
ensure_ascii=False,
cls=PythonObjectEncoder)))
else:
json.dump(json_data,
outfile,
indent=4,
separators=(',', ':'),
ensure_ascii=False,
cls=PythonObjectEncoder)
msg = "dump file: {}".format(json_file_abs_path)
logger.color_print(msg, "BLUE")
except TypeError as ex:
msg = "Failed to dump json file: {}\nReason: {}".format(
json_file_abs_path, ex)
logger.color_print(msg, "RED")
def command(self, command, arg=None):
""" Initialize a socket connection,
send a command (a json encoded dict) and
receive the response (and decode it).
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.host, self.port))
payload = {"command": command}
if arg is not None:
# Parameter must be converted to basestring (no int)
payload.update({'parameter': unicode(arg)})
sock.send(json.dumps(payload).encode())
received = self._receive(sock)
finally:
sock.shutdown(socket.SHUT_RDWR)
sock.close()
return json.loads(received)
