def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS): """pack up a function, args, and kwargs to be sent over the wire Each element of args/kwargs will be canned for special treatment, but inspection will not go any deeper than that. Any object whose data is larger than `threshold` will not have their data copied (only numpy arrays and bytes/buffers support zero-copy) Message will be a list of bytes/buffers of the format: [ cf, pinfo, <arg_bufs>, <kwarg_bufs> ] With length at least two + len(args) + len(kwargs) """ arg_bufs = flatten(serialize_object(arg, buffer_threshold, item_threshold) for arg in args) kw_keys = sorted(kwargs.keys()) kwarg_bufs = flatten(serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys) info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys) msg = [pickle.dumps(can(f), PICKLE_PROTOCOL)] msg.append(pickle.dumps(info, PICKLE_PROTOCOL)) msg.extend(arg_bufs) msg.extend(kwarg_bufs) return msg
def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS): """pack up a function, args, and kwargs to be sent over the wire Each element of args/kwargs will be canned for special treatment, but inspection will not go any deeper than that. Any object whose data is larger than `threshold` will not have their data copied (only numpy arrays and bytes/buffers support zero-copy) Message will be a list of bytes/buffers of the format: [ cf, pinfo, <arg_bufs>, <kwarg_bufs> ] With length at least two + len(args) + len(kwargs) """ arg_bufs = flatten(serialize_object(arg, buffer_threshold, item_threshold) for arg in args) kw_keys = sorted(kwargs.keys()) kwarg_bufs = flatten(serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys) info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys) msg = [pickle.dumps(can(f),-1)] msg.append(pickle.dumps(info, -1)) msg.extend(arg_bufs) msg.extend(kwarg_bufs) return msg
def qw(words,flat=0,sep=None,maxsplit=-1): """Similar to Perl's qw() operator, but with some more options. qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit) words can also be a list itself, and with flat=1, the output will be recursively flattened. Examples: >>> qw('1 2') ['1', '2'] >>> qw(['a b','1 2',['m n','p q']]) [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]] >>> qw(['a b','1 2',['m n','p q']],flat=1) ['a', 'b', '1', '2', 'm', 'n', 'p', 'q'] """ if isinstance(words, basestring): return [word.strip() for word in words.split(sep,maxsplit) if word and not word.isspace() ] if flat: return flatten(map(qw,words,[1]*len(words))) return map(qw,words)
def qw(words, flat=0, sep=None, maxsplit=-1): """Similar to Perl's qw() operator, but with some more options. qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit) words can also be a list itself, and with flat=1, the output will be recursively flattened. Examples: >>> qw('1 2') ['1', '2'] >>> qw(['a b','1 2',['m n','p q']]) [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]] >>> qw(['a b','1 2',['m n','p q']],flat=1) ['a', 'b', '1', '2', 'm', 'n', 'p', 'q'] """ if isinstance(words, basestring): return [ word.strip() for word in words.split(sep, maxsplit) if word and not word.isspace() ] if flat: return flatten(map(qw, words, [1] * len(words))) return map(qw, words)
import numpy as np import healpy as hp from astropy.io import fits import matplotlib.pyplot as plt from IPython.utils.data import flatten has=fits.open('/home/dweisberger/haslam_masked.fits') ham=has[1].data.field(0) flh=flatten(ham) fha=np.array(flh) Y = fha free=fits.open('/home/dweisberger/freefree_mask.fits') fre=free[1].data.field(0) fra=flatten(fre) ffa=np.array(fra) Z = ffa
def crawl_recipe(self, recipe_num=None): url_recipe = self._url_base.format( recipe_num=recipe_num ) driver = webdriver.PhantomJS( '/Users/hongdonghyun/projects/team_project/wps-picky/django_app/utils/crawl/phantomjs') driver.get(url_recipe) html = driver.page_source soup = BeautifulSoup(html) # 전체 페이지에서 타이틀 부분 뽑아옴 title = soup.select_one('title').get_text() title_list = title.split(':') empty_list = [] for t in title_list: t = t.strip(' ') empty_list.append(t) # 레시피의 이미지 뽑아옴 div_editor = soup.find("div", {"class": "edtitor_img_uio"}) recipe_img = div_editor.img['src'] # 레시피의 타이틀, 설명 뽑아옴 recipe_title = empty_list[0] recipe_description = empty_list[1] div_container_inner = soup.find("div", {"class": "__viewer_container_inner"}) div_na_summary_info = div_container_inner.find_all("div", {"class": "na_summary_info"}) # select함수는 리턴값이 리스트형식 _list = [] for before_list in div_na_summary_info: a = before_list.select("ul > li > span") _list.append(a) # 위 과정을 거치면 이중 리스트 형식이됨 # 이중 리스트 형식을 일자화 시키는 함수 flatten _list = flatten(_list) # 일자화된 리스트에서 span을 제외한 text값 추출 # ingredient_list라는 변수에 할당 ingredient_list = [] for after_list in _list: ingredient = after_list.get_text() ingredient_list.append(ingredient) # ,로 join ingredient_list = ','.join(ingredient_list) # 스탭 부분 이미지 div_tptype_5 = soup.find_all("div", {"class": "t_ptype5"}) # 스탭 부분 텍스트 div_p = soup.find_all("p", {"class": "t_txt"}) # 스탭 이미지 추출 img_list = [] for image in div_tptype_5: img_list.append(image.img['src']) # 스탭 설명 추출 text_list = [] for text in div_p: text_list.append(text.get_text()) del text_list[0] # 기존 텍스트에 \n,\t제거 i = ' '.join(text_list).replace('\n', '') i = i.replace('\t', '') # STEP기준으로 설명을 쪼갠다 # 이 기능에서 STEP삭제 # 삭제를 위한 정규표현식 pattern = re.compile(r'STEP \d+\s?') # text_list에 다시 대입 # 공백제거, 만약 텍스트가 빈공백 요소일 경우 제거 text_list = [text.strip() for text in pattern.split(i) if text] for text in text_list: # 크롤링 도중 원하지 않는 부분이 있어 제거하기 위한 if문 if '(숫자)' in text: del text_list[0] # 임시 데이터의 File_name file_name = '{}.jpg'.format(recipe_title) # 무슨 기능을 하는지 확인필요 - hong 8/21 temp_file = NamedTemporaryFile(delete=True) # response변수에 recipe_img 저장 response = requests.get(recipe_img) # temp_file에 recipe_img의 content를 저장 # 동작 후 사라지는 임시파일 temp_file.write(response.content) # 레시피 생성 _2bab_recipe = Recipe.objects.create( title=recipe_title, description=recipe_description, ingredient=ingredient_list, # 현재는 pickyuser 첫번째 유저를 가져오지만 최종 정리 후 # admin의 아이디를 가져와야함 - hong 8/21 user=PickyUser.objects.first() ) # 위에서 저장한 임시파일을 _2bab_recipe의 이미지필드에 저장 _2bab_recipe.img_recipe.save(file_name, File(temp_file)) # zip으로 묶인 img_list 와 img에 대한 description_list for x, y in zip(img_list, text_list): # 위에서 만든 _2bab_recipe의 pk에 매핑 # 설명을 저장 후 생성 _2bab_recipe_step = RecipeStep.objects.create(description=y, recipe_id=_2bab_recipe.pk) # 상단의 동일코드와 같음 # 설명생략 file_name = '{}.{}.jpg'.format(_2bab_recipe.pk, recipe_title) temp_file = NamedTemporaryFile(delete=True) response = requests.get(x) temp_file.write(response.content) _2bab_recipe_step.img_step.save(file_name, File(temp_file))