def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):
"""pack up a function, args, and kwargs to be sent over the wire
Each element of args/kwargs will be canned for special treatment,
but inspection will not go any deeper than that.
Any object whose data is larger than `threshold` will not have their data copied
(only numpy arrays and bytes/buffers support zero-copy)
Message will be a list of bytes/buffers of the format:
[ cf, pinfo, <arg_bufs>, <kwarg_bufs> ]
With length at least two + len(args) + len(kwargs)
"""
arg_bufs = flatten(serialize_object(arg, buffer_threshold, item_threshold) for arg in args)
kw_keys = sorted(kwargs.keys())
kwarg_bufs = flatten(serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys)
info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys)
msg = [pickle.dumps(can(f), PICKLE_PROTOCOL)]
msg.append(pickle.dumps(info, PICKLE_PROTOCOL))
msg.extend(arg_bufs)
msg.extend(kwarg_bufs)
return msg
def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):
"""pack up a function, args, and kwargs to be sent over the wire
Each element of args/kwargs will be canned for special treatment,
but inspection will not go any deeper than that.
Any object whose data is larger than `threshold` will not have their data copied
(only numpy arrays and bytes/buffers support zero-copy)
Message will be a list of bytes/buffers of the format:
[ cf, pinfo, <arg_bufs>, <kwarg_bufs> ]
With length at least two + len(args) + len(kwargs)
"""
arg_bufs = flatten(serialize_object(arg, buffer_threshold, item_threshold) for arg in args)
kw_keys = sorted(kwargs.keys())
kwarg_bufs = flatten(serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys)
info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys)
msg = [pickle.dumps(can(f),-1)]
msg.append(pickle.dumps(info, -1))
msg.extend(arg_bufs)
msg.extend(kwarg_bufs)
return msg
def qw(words,flat=0,sep=None,maxsplit=-1):
"""Similar to Perl's qw() operator, but with some more options.
qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
words can also be a list itself, and with flat=1, the output will be
recursively flattened.
Examples:
>>> qw('1 2')
['1', '2']
>>> qw(['a b','1 2',['m n','p q']])
[['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
>>> qw(['a b','1 2',['m n','p q']],flat=1)
['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
"""
if isinstance(words, basestring):
return [word.strip() for word in words.split(sep,maxsplit)
if word and not word.isspace() ]
if flat:
return flatten(map(qw,words,[1]*len(words)))
return map(qw,words)
def qw(words, flat=0, sep=None, maxsplit=-1):
"""Similar to Perl's qw() operator, but with some more options.
qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
words can also be a list itself, and with flat=1, the output will be
recursively flattened.
Examples:
>>> qw('1 2')
['1', '2']
>>> qw(['a b','1 2',['m n','p q']])
[['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
>>> qw(['a b','1 2',['m n','p q']],flat=1)
['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
"""
if isinstance(words, basestring):
return [
word.strip() for word in words.split(sep, maxsplit)
if word and not word.isspace()
]
if flat:
return flatten(map(qw, words, [1] * len(words)))
return map(qw, words)
import numpy as np
import healpy as hp
from astropy.io import fits
import matplotlib.pyplot as plt
from IPython.utils.data import flatten
has=fits.open('/home/dweisberger/haslam_masked.fits')
ham=has[1].data.field(0)
flh=flatten(ham)
fha=np.array(flh)
Y = fha
free=fits.open('/home/dweisberger/freefree_mask.fits')
fre=free[1].data.field(0)
fra=flatten(fre)
ffa=np.array(fra)
Z = ffa
def crawl_recipe(self, recipe_num=None):
url_recipe = self._url_base.format(
recipe_num=recipe_num
)
driver = webdriver.PhantomJS(
'/Users/hongdonghyun/projects/team_project/wps-picky/django_app/utils/crawl/phantomjs')
driver.get(url_recipe)
html = driver.page_source
soup = BeautifulSoup(html)
# 전체 페이지에서 타이틀 부분 뽑아옴
title = soup.select_one('title').get_text()
title_list = title.split(':')
empty_list = []
for t in title_list:
t = t.strip(' ')
empty_list.append(t)
# 레시피의 이미지 뽑아옴
div_editor = soup.find("div", {"class": "edtitor_img_uio"})
recipe_img = div_editor.img['src']
# 레시피의 타이틀, 설명 뽑아옴
recipe_title = empty_list[0]
recipe_description = empty_list[1]
div_container_inner = soup.find("div", {"class": "__viewer_container_inner"})
div_na_summary_info = div_container_inner.find_all("div", {"class": "na_summary_info"})
# select함수는 리턴값이 리스트형식
_list = []
for before_list in div_na_summary_info:
a = before_list.select("ul > li > span")
_list.append(a)
# 위 과정을 거치면 이중 리스트 형식이됨
# 이중 리스트 형식을 일자화 시키는 함수 flatten
_list = flatten(_list)
# 일자화된 리스트에서 span을 제외한 text값 추출
# ingredient_list라는 변수에 할당
ingredient_list = []
for after_list in _list:
ingredient = after_list.get_text()
ingredient_list.append(ingredient)
# ,로 join
ingredient_list = ','.join(ingredient_list)
# 스탭 부분 이미지
div_tptype_5 = soup.find_all("div", {"class": "t_ptype5"})
# 스탭 부분 텍스트
div_p = soup.find_all("p", {"class": "t_txt"})
# 스탭 이미지 추출
img_list = []
for image in div_tptype_5:
img_list.append(image.img['src'])
# 스탭 설명 추출
text_list = []
for text in div_p:
text_list.append(text.get_text())
del text_list[0]
# 기존 텍스트에 \n,\t제거
i = ' '.join(text_list).replace('\n', '')
i = i.replace('\t', '')
# STEP기준으로 설명을 쪼갠다
# 이 기능에서 STEP삭제
# 삭제를 위한 정규표현식
pattern = re.compile(r'STEP \d+\s?')
# text_list에 다시 대입
# 공백제거, 만약 텍스트가 빈공백 요소일 경우 제거
text_list = [text.strip() for text in pattern.split(i) if text]
for text in text_list:
# 크롤링 도중 원하지 않는 부분이 있어 제거하기 위한 if문
if '(숫자)' in text:
del text_list[0]
# 임시 데이터의 File_name
file_name = '{}.jpg'.format(recipe_title)
# 무슨 기능을 하는지 확인필요 - hong 8/21
temp_file = NamedTemporaryFile(delete=True)
# response변수에 recipe_img 저장
response = requests.get(recipe_img)
# temp_file에 recipe_img의 content를 저장
# 동작 후 사라지는 임시파일
temp_file.write(response.content)
# 레시피 생성
_2bab_recipe = Recipe.objects.create(
title=recipe_title,
description=recipe_description,
ingredient=ingredient_list,
# 현재는 pickyuser 첫번째 유저를 가져오지만 최종 정리 후
# admin의 아이디를 가져와야함 - hong 8/21
user=PickyUser.objects.first()
)
# 위에서 저장한 임시파일을 _2bab_recipe의 이미지필드에 저장
_2bab_recipe.img_recipe.save(file_name, File(temp_file))
# zip으로 묶인 img_list 와 img에 대한 description_list
for x, y in zip(img_list, text_list):
# 위에서 만든 _2bab_recipe의 pk에 매핑
# 설명을 저장 후 생성
_2bab_recipe_step = RecipeStep.objects.create(description=y, recipe_id=_2bab_recipe.pk)
# 상단의 동일코드와 같음
# 설명생략
file_name = '{}.{}.jpg'.format(_2bab_recipe.pk, recipe_title)
temp_file = NamedTemporaryFile(delete=True)
response = requests.get(x)
temp_file.write(response.content)
_2bab_recipe_step.img_step.save(file_name, File(temp_file))
