Example #1
0
async def test_recognize_edge_case_urls(api: SoundcloudAPI):
    urls = ['https://soundcloud.com/nittigritti/lights-nitti-gritti-remix-1']
    for url in urls:
        track = await api.resolve(url)
        file = BytesIO()
        size = file.__sizeof__()
        await track.write_mp3_to(file)
        assert file.__sizeof__() > size
Example #2
0
async def test_track_accepts_correct_file_objects(api):

    track = await api.resolve(TEST_TRACK_URL)
    filename = os.path.realpath('faksjhflaksjfhlaksjfdhlkas.mp3')
    with open(filename, 'wb+') as mp3:
        await track.write_mp3_to(mp3)
    os.remove(filename)

    # Create file then open in rb+
    with open(filename, 'w'):
        pass

    with open(filename, 'rb+') as mp3:
        await track.write_mp3_to(mp3)
    os.remove(filename)

    try:
        with open(filename, 'w') as fp:
            await track.write_mp3_to(fp)
    except TypeError:
        pass

    try:
        with open(filename, 'wb') as fp:
            await track.write_mp3_to(fp)
    except ValueError:
        pass

    file = BytesIO()
    await track.write_mp3_to(file)
    assert file.__sizeof__() > 0
Example #3
0
def download_image(url_path):  # 下载图片,定义一个方法方便开启多线程,返回下载该图片的相对路径
    url = url_path[0]
    path = url_path[1]
    convert_status = url_path[2]
    global ERROR_PAGE_LIST  #全局变量
    try:
        comic_page = get(url, headers=public_headers)
        if comic_page.status_code != 200:
            # print('!= 200')
            ERROR_PAGE_LIST.append(url_path)
            pass
    except Exception:
        # print('Download Error')
        ERROR_PAGE_LIST.append(url_path)
        pass
    comic_name = url.split('/')[-1].split('?')[0]
    comic_local_position = path + '/' + comic_name
    image_bytes = BytesIO(comic_page.content)
    if image_bytes.__sizeof__() >= 1:  #防止下载的图片为0kb,玄学?!
        image_source = Image.open(image_bytes)
        image_source.save(comic_local_position)
    else:
        # print('content is lost')
        ERROR_PAGE_LIST.append(url_path)
        pass
    if convert_status:
        convertImg(comic_local_position)  # 对“无耻”的以修改图片的反爬虫机制进行反制!
    if url_path in ERROR_PAGE_LIST:  # 如果下载成功就再下载列表删除它
        ERROR_PAGE_LIST.remove(url_path)
Example #4
0
    def compress_image(self, my_image, my_width, my_height):
        img_temp = PILImage.open(my_image)
        x = my_width
        y = my_height
        if img_temp.mode != "RGB":
            img_temp.convert('RGB')

        # get current size of main photo
        width, height = img_temp.size
        ratio = width / height

        if width > x or height > y:
            if width >= height:
                y = int(x / ratio)
            elif height > width:
                x = int(y * ratio)
        else:
            x = width
            y = height

        img_temp.thumbnail((x, y), PILImage.ANTIALIAS)
        save_buff = BytesIO()
        img_temp.save(save_buff, format="JPEG", optimize=True, quality=70)
        save_buff.seek(0)
        my_image = InMemoryUploadedFile(
            save_buff, 'ImageField', "%s.jpg" % self.photo.name.split('.')[0],
            'image/jpeg', save_buff.__sizeof__(), None)
        return my_image
Example #5
0
    async def recognize(self, url: str, uuid: UUID):
        async with ClientSession() as session:
            raw_mp3 = BytesIO(await self._download_mp3(session, url))
            raw_flac = BytesIO()

            # convert audio file from mp3 to flac
            pydub.AudioSegment.from_mp3(raw_mp3).export(raw_flac,
                                                        format="flac")

            length = raw_flac.__sizeof__() / 25000

            if length < 59:
                # read content
                content = raw_flac.read()
                response = self._transcribe_content(content)
            else:
                self.logger.debug("Loading file to google storage")
                file_name = uuid.hex if uuid else GRecognizer.EMPTY_UUID.hex
                self._upload_audio(file_name, raw_flac)
                response = self._transcribe_content_async(
                    "gs://vrp-audio/{}".format(file_name))

            text = response.results[0].alternatives[0].transcript if len(
                response.results) else ""
            self.logger.debug("Parse: {}".format(text))
            return text
Example #6
0
async def do():
    api = SoundcloudAPI()
    urls = ['https://soundcloud.com/mt-marcy/cold-nights']
    for url in urls:
        track = await api.resolve(url)
        file = BytesIO()
        size = file.__sizeof__()
        await track.write_mp3_to(file)
Example #7
0
class Image(object):
    """
    An AsyncDagpi Image Returned
    This has special properties that can enhance the experience.

    .. container:: Attributes

        **Attributes**

        image: :class:`io.BytesIO`
            BytesIO object with Image
        format: :class:`str`
            String containing Image Format
            Either PNG or GIF
        process_time: :class:`str`
            String for Time Taken by API to process Request
        original_image: :class:`str`
            URL for the image passed to the API
    """
    def __init__(self, byt: bytes, image_format: str, process_time: str,
                 original_url: str):
        self.image = BytesIO(byt)
        self.format = image_format
        self.process_time = process_time
        self.original_image = original_url

    def size(self) -> int:
        """
        Returns the Size of the Image
        """
        return self.image.__sizeof__()

    def __repr__(self) -> str:
        """
        A description of the Image
        """
        return "AsyncDagpi.Image format={}".format(self.format)

    def read(self) -> bytes:
        """
        Get raw Image bytes
        :return: :class:`bytes`
        """
        return self.image.read()

    def write(self, name: str, **kwargs):
        """
        Writes the Image Object to file

        :param name: :class:`str`
            the name of the file to save
        kwargs: Optional To Pass in
            - path: path to write to can be relative/absolute. default is ./
        """
        path = kwargs.get("path", ".")
        with open("{}/{}.{}".format(path, name, self.format), 'wb') as file:
            file.write(self.read())
Example #8
0
 def get_temporary_image(self):
     io = BytesIO()
     size = (100, 100)
     color = (255, 0, 0)
     image = Image.new("RGB", size, color)
     image.save(io, format='PNG')
     image_file = InMemoryUploadedFile(io, None, 'foo.png', 'png', io.__sizeof__(), None)
     image_file.seek(0)
     return image_file
Example #9
0
 def to_base64(self, data):
     try:
         # we are discarding the data uri if present
         data = data.split(',')[1]
     except:
         pass
     file_ = BytesIO(b64decode(data))
     content_type = self.get_content_type(file_)
     extension = content_type.split('/')[1]
     return InMemoryUploadedFile(file_, None, extension, content_type,
                                 file_.__sizeof__(), None)
    def set_photo(self, response):

        image_url = response.url

        print('*' * 100)
        print(image_url)

        image_request_result = requests.get(image_url)

        if image_request_result.status_code != 200:
            self.crawler_log.writelines("통신 실패 : " + str(image_request_result.status_code))
            return

        file_bytes = BytesIO(image_request_result.content)

        # 이미지 데이터의 크기가 0 일 경우 실패 메세지를 띄우고 종료한다.
        if file_bytes.__sizeof__()==0 :
            print("download fail")
            return

        # 크기가 0이 아닌 경우, 사이즈를 확인한다.
        # 지나치게 큰 이미지의 경우는 조절해준다.
        pil_img =Image.open(BytesIO(image_request_result.content))
        width, height = pil_img.size
        max_size = [1024, 1024]
        if width > 1024 or height > 1024:
            pil_img.thumbnail(max_size)

        # 얼굴 검출을 위해서 opencv 이미지로 변환
        cv_img = np.array(pil_img)
        cv_img = cv_img[:, :, ::-1].copy()

        # 이미지를 흑백으로 변환한 뒤, 얼굴 검출 진행
        # 흑백인 이유는 얼굴 검출 시에 형태만 활용되므로 색상 정보가 불필요하기 때문
        gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
        faces = self.face_cascade.detectMultiScale(gray, 1.3, 5)

        # 얼굴이 검출되지 않을 경우 함수를 종료한다.
        if len(faces)==0:
            print("face fail")
            return

        # 얼굴이 포함된 이미지를 저장할 폴더명과 파일명을 설정한다.
        file_path = 'collected_image/naver/' + self.dirname + '/'
        full_path = file_path + 'naver_' + self.dirname + '_' + str(time.time()) + '.jpg'

        # 이미지를 저장할 폴더가 없을 경우 생성해준다.
        if not os.path.exists(file_path) :

            os.makedirs(file_path)
            
        print("face ??")
        cv2.imwrite(full_path, cv_img)
Example #11
0
 def generate_qrcode(self):
     qr = qrcode.QRCode(version=1, box_size=6, border=0)
     qr.add_data('http://' + ALLOWED_HOSTS[0] +
                 reverse('activate_product') + self.code)
     qr.make(fit=True)
     img = qr.make_image()
     buffer = BytesIO()
     img.save(buffer)
     filename = 'qr-%s.png' % (self.id)
     filebuffer = InMemoryUploadedFile(buffer, None, filename, 'image/png',
                                       buffer.__sizeof__(), None)
     self.qrcode.save(filename, filebuffer, False)
Example #12
0
    def set_photo(self, url, dirname):
        # 전달받은 url을 요청하여 이미지 데이터를 다운받는다.
        image_request_result = requests.get(url)

        # HTTP 통신이 성공했는지 여부를 확인하며, 실패시 종료한다.
        if image_request_result.status_code != 200:
            print("Network Fail")
            return

        file_bytes = BytesIO(image_request_result.content)

        # 이미지 데이터의 크기가 0 일 경우 종료한다.
        if file_bytes.__sizeof__() == 0:
            print("Download Fail")
            return

        # 크기가 0이 아닌 경우, 사이즈를 확인한다.
        # 지나치게 가로 세로 크가기 큰 이미지는 조절해준다. (1024x1024 이상)
        # 크기 조절의 이유를 추가할 것
        pil_img = Image.open(BytesIO(image_request_result.content))
        width, height = pil_img.size
        max_size = [1024, 1024]
        if width > 1024 or height > 1024:
            pil_img.thumbnail(max_size)

        try:
            # 얼굴 검출을 위해서 opencv 이미지로 변환
            cv_img = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR)

        except:
            print("변환 실패")
            return

        # 이미지를 흑백으로 변환한 뒤, 얼굴 검출 진행
        # 흑백인 이유는 얼굴 검출 시에 형태만 활용되므로 색상 정보가 불필요하기 때문
        gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
        faces = self.face_cascade.detectMultiScale(gray, 1.3, 5)

        # 얼굴이 검출되지 않을 경우 함수를 종료한다.
        if len(faces) == 0:
            return

        # 얼굴이 포함된 이미지를 저장할 폴더명과 파일명을 설정한다.
        file_path = self.root_dir + dirname + '/'
        full_path = file_path + 'insta_' + dirname + '_' + str(time.time()) + '.jpg'

        # 이미지를 저장할 폴더가 없을 경우 생성해준다.
        if not os.path.exists(file_path):
            os.mkdir(file_path)

        # 이미지를 지정된 경로에 저장한다.
        cv2.imwrite(full_path, cv_img)
    def set_photo(self, url, dirname):

        # 전달받은 url을 요청하여 이미지 데이터를 다운받는다.
        image_request_result = requests.get(url)

        # HTTP 통신이 성공했는지 여부를 확인하며, 실패시 상태 코드와 url을 로그로 남기고 종료한다.
        if image_request_result.status_code != 200:
            self.crawler_log.writelines("통신 실패 : " + str(image_request_result.status_code)+" , "+url)
            return

        file_bytes=BytesIO(image_request_result.content)

        # 이미지 데이터의 크기가 0 일 경우 url을 로그로 남기고 종료한다.
        if file_bytes.__sizeof__()==0 :
            self.crawler_log.writelines("이미지 다운 실패 : " + url)
            return

        # 크기가 0이 아닌 경우, 사이즈를 확인한다.
        # 지나치게 가로 세로 크가기 큰 이미지는 조절해준다. (1024x1024 이상)
        # 크기 조절의 이유를 추가할 것
        pil_img =Image.open(BytesIO(image_request_result.content))
        width, height = pil_img.size
        max_size = [1024, 1024]
        if width > 1024 or height > 1024:
            pil_img.thumbnail(max_size)

        # 얼굴 검출을 위해서 opencv 이미지로 변환
        # cv_img = np.array(pil_img)
        # cv_img = cv_img[:, :, ::-1].copy()

        cv_img = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR)

        # 이미지를 흑백으로 변환한 뒤, 얼굴 검출 진행
        # 흑백인 이유는 얼굴 검출 시에 형태만 활용되므로 색상 정보가 불필요하기 때문
        gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
        faces = self.face_cascade.detectMultiScale(gray, 1.3, 5)

        # 얼굴이 검출되지 않을 경우 함수를 종료한다.
        if len(faces)==0:
            return

        # 얼굴이 포함된 이미지를 저장할 폴더명과 파일명을 설정한다.
        file_path = self.root_dir + dirname + '/'
        full_path = file_path + 'google_' + dirname + '_' + str(time.time()) + '.jpg'

        # 이미지를 저장할 폴더가 없을 경우 생성해준다.
        if not os.path.exists(file_path) :
            os.makedirs(file_path)

        # 이미지를 지정된 경로에 저장한다.
        cv2.imwrite(full_path, cv_img)
Example #14
0
 def save(self, *args, **kwargs):
     """
     method to make thumnail of uploaded file
     in the memory itself
     """
     if self.profile_pic:
         im = Img.open(self.profile_pic)
         im.thumbnail((200,200), Img.ANTIALIAS)
         output = BytesIO()
         im.save(output, format='JPEG')
         output.seek(0)
         self.profile_pic= InMemoryUploadedFile(output,'ImageField', "%s.jpg"\
             %self.profile_pic.name, 'profile_pic/jpeg', output.__sizeof__(), None)
     super(Profile, self).save(*args, **kwargs)
Example #15
0
	def tile(self, x, y, z):
		nw = self.deg(x, y, z)
		se = self.deg(x + 1, y + 1, z)
		xmin = nw[0]
		ymin = se[1]
		xmax = se[0]
		ymax = nw[1]

		m = mapnik.Map(256, 256)
		m.srs = "+proj=merc +lon_0=0 +k=1 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs"

		m.append_style("pointstyle", self.style)
		layer = mapnik.Layer("pointlayer")
		layer.datasource = self.datasource
		layer.styles.append("pointstyle")
		m.layers.append(layer)

		bbox = mapnik.Box2d(xmin, ymin, xmax, ymax)
		merc = mapnik.Projection(m.srs)
		longlat = mapnik.Projection("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
		transform = mapnik.ProjTransform(longlat, merc)
		merc_bbox = transform.forward(bbox)

		m.zoom_to_box(merc_bbox)
		m.buffer_size = 10

		im = mapnik.Image(256, 256)
		mapnik.render(m, im)
		ims = im.tostring()
		pim = PIL.Image.frombuffer("RGBA", (256, 256), ims, "raw", "RGBA", 0, 1)
		buf = BytesIO()
		pim.save(buf, "png")
		self.db.tiles.remove({
			"id": self.id,
			"xyz": "%s_%s_%s" % (x, y, z)
		})
		if buf.__sizeof__() == self.blanksize:
			self.db.tiles.insert_one({
				"id": self.id,
				"xyz": "%s_%s_%s" % (x, y, z),
				"blank": True
			})
		else:
			self.db.tiles.insert_one({
				"id": self.id,
				"xyz": "%s_%s_%s" % (x, y, z),
				"tile": Binary(buf.getvalue())
			})
Example #16
0
    def tile(self, x, y, z):
        nw = self.deg(x, y, z)
        se = self.deg(x + 1, y + 1, z)
        xmin = nw[0]
        ymin = se[1]
        xmax = se[0]
        ymax = nw[1]

        m = mapnik.Map(256, 256)
        m.srs = "+proj=merc +lon_0=0 +k=1 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs"

        m.append_style("pointstyle", self.style)
        layer = mapnik.Layer("pointlayer")
        layer.datasource = self.datasource
        layer.styles.append("pointstyle")
        m.layers.append(layer)

        bbox = mapnik.Box2d(xmin, ymin, xmax, ymax)
        merc = mapnik.Projection(m.srs)
        longlat = mapnik.Projection(
            "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
        transform = mapnik.ProjTransform(longlat, merc)
        merc_bbox = transform.forward(bbox)

        m.zoom_to_box(merc_bbox)
        m.buffer_size = 10

        im = mapnik.Image(256, 256)
        mapnik.render(m, im)
        ims = im.tostring()
        pim = PIL.Image.frombuffer("RGBA", (256, 256), ims, "raw", "RGBA", 0,
                                   1)
        buf = BytesIO()
        pim.save(buf, "png")
        self.db.tiles.remove({"id": self.id, "xyz": "%s_%s_%s" % (x, y, z)})
        if buf.__sizeof__() == self.blanksize:
            self.db.tiles.insert_one({
                "id": self.id,
                "xyz": "%s_%s_%s" % (x, y, z),
                "blank": True
            })
        else:
            self.db.tiles.insert_one({
                "id": self.id,
                "xyz": "%s_%s_%s" % (x, y, z),
                "tile": Binary(buf.getvalue())
            })
    def save_photo(self, response):
        
        image_request_result = requests.get(response.url)
        if image_request_result.status_code != 200:
            self.crawler_log.writelines("통신 실패 : " + str(image_request_result.status_code))
            return

        # global file_bytes
        file_bytes = BytesIO(image_request_result.content)
        # print(">> file_bytes = "+str(file_bytes.__sizeof__()))
        # 이미지 데이터의 크기가 0 일 경우 실패 메세지를 띄우고 종료한다.
        if file_bytes.__sizeof__()==0 :
        # print(">> file_bytes = 0 ")
            print("download fail")
            return

        # 크기가 0이 아닌 경우, 사이즈를 확인한다.
        # 지나치게 큰 이미지의 경우는 조절해준다.
        pil_img =Image.open(BytesIO(image_request_result.content))
        width, height = pil_img.size
        max_size = [1024, 1024]
        if width > 1024 or height > 1024:
            pil_img.thumbnail(max_size)

        # 얼굴 검출을 위해서 PIL 이미지 객체를 opencv 이미지로 변환
        pil_img2 = pil_img.convert('RGB') # opencv이미지는 BGR이라서 맞춰줘야함
        cv_img = np.array(pil_img2)
        cv_img = cv_img[:, :, ::-1].copy()

        # 이미지를 흑백으로 변환한 뒤, 얼굴 검출 진행
        # 흑백인 이유는 얼굴 검출 시에 형태만 활용되므로 색상 정보가 불필요하기 때문
        gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
        faces = self.face_cascade.detectMultiScale(gray, 1.3, 5)

        # 얼굴이 검출되지 않을 경우 함수를 종료한다.
        if len(faces)==0:
            return
        
        file_path = self.root_dir + self.dirname + '/'
        full_path = file_path + 'tumblr_' + self.dirname + '_' + str(time.time()) + '.jpg'

        # 이미지를 저장할 폴더가 없을 경우 생성해준다.
        if not os.path.exists(file_path) :
            os.makedirs(file_path)

        cv2.imwrite(full_path, cv_img)
Example #18
0
	def __init__(self, id, config):
		self.id = id
		client = pymongo.MongoClient()
		self.db = client.tiles
		self.db.tiles.ensure_index([
			("id", pymongo.ASCENDING),
			("xyz", pymongo.ASCENDING)
		])

		buf = BytesIO()
		im = PIL.Image.new("RGBA", (256, 256))
		im.save(buf, "png")
		self.blanksize = buf.__sizeof__()
		print "Blank size: %i" % (self.blanksize)
		self.db.blank.drop()
		self.db.blank.insert_one({ "tile": Binary(buf.getvalue()) })

		self.datasource = mapnik.PostGIS(
			host = config.host,
			user = config.user,
			password = config.password,
			dbname = "obis",
			table = "(select geom from explore.points where species_id = %s) points" % (self.id),
			geometry_field = "geom",
			srid = "4326",
			extent = "%s, %s, %s, %s" % (-180, -90, 180, 90),
			connext_timeout = 10
		)

		self.style = mapnik.Style()
		rule = mapnik.Rule()
		symbolizer = mapnik.MarkersSymbolizer()
		symbolizer.width = 5.0
		symbolizer.stroke_width = 0.0
		symbolizer.fill = mapnik.Color("#ff0000")
		symbolizer.opacity = 1.0
		symbolizer.allow_overlap = mapnik.Expression("True")
		rule.symbols.append(symbolizer)
		self.style.rules.append(rule)
Example #19
0
    def do_GET(self):
        print("get: " + self.path)
        if self.path.endswith('.mjpg'):
            self.send_response(200)
            self.send_header(
                'Content-type',
                'multipart/x-mixed-replace; boundary=--jpgboundary')
            self.end_headers()

            print("before loop")
            while True:
                try:
                    rc, img = capture.read()
                    if not rc:
                        continue
                    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
                    jpg = Image.fromarray(imgRGB)
                    tmpFile = BytesIO()
                    jpg.save(tmpFile, 'JPEG')
                    s = "--jpgboundary"

                    self.wfile.write(s.encode("utf-8"))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', tmpFile.__sizeof__())
                    self.end_headers()
                    jpg.save(self.wfile, 'JPEG')
                    time.sleep(0.02)
                except KeyboardInterrupt:
                    break
            return

        if self.path.endswith('.html'):
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            self.wfile.write('<html><head></head><body>')
            self.wfile.write('<img src="http://127.0.0.1:8080/cam.mjpg" />')
            self.wfile.write('</body></html>')
            return
Example #20
0
	def on_post(self, req, res):
		#gathering file from SPA
		contact_id = req.get_param('id')
		imported_filename = req.get_param('file').filename
		file = req.get_param('file').file
		salt = ''.join(chr(random.randint(97, 122)) for i in range(20))
		filename = salt + '-' + imported_filename
		filename_thumb = salt + '-thumb-' + imported_filename

		#uploading normal sized image
		client.upload_fileobj(file, 'contacts-cloud-images', filename)	

		#save urls to database
		contact = Contact.get(Contact.id == contact_id)
		contact.image_url = image_url = BASE_AWS_URL + filename
		contact.image_thumb_url = thumb_image_url = BASE_AWS_URL + filename_thumb
		contact.save()

		# res.body = (contact.image_url)
		res.body = (json.dumps([image_url, thumb_image_url]))

		# pull down image again and resize
		img = Image.open(requests.get(image_url, stream=True).raw)
		img.thumbnail((50,50))
		print(img.format, img.size)

		#save it to BytesIO container
		file_content = BytesIO()
		img.save(file_content, img.format)

		print(dir(file_content))
		print(file_content.__sizeof__())

		file_content.seek(0)
		print('about to save thumb to s3')

		#upload value of BytesIO container
		client.upload_fileobj(file_content, 'contacts-cloud-images', filename_thumb)
Example #21
0
    def __init__(self, id, config):
        self.id = id
        client = pymongo.MongoClient()
        self.db = client.tiles
        self.db.tiles.ensure_index([("id", pymongo.ASCENDING),
                                    ("xyz", pymongo.ASCENDING)])

        buf = BytesIO()
        im = PIL.Image.new("RGBA", (256, 256))
        im.save(buf, "png")
        self.blanksize = buf.__sizeof__()
        print "Blank size: %i" % (self.blanksize)
        self.db.blank.drop()
        self.db.blank.insert_one({"tile": Binary(buf.getvalue())})

        self.datasource = mapnik.PostGIS(
            host=config.host,
            user=config.user,
            password=config.password,
            dbname="obis",
            table=
            "(select geom from explore.points where species_id = %s) points" %
            (self.id),
            geometry_field="geom",
            srid="4326",
            extent="%s, %s, %s, %s" % (-180, -90, 180, 90),
            connext_timeout=10)

        self.style = mapnik.Style()
        rule = mapnik.Rule()
        symbolizer = mapnik.MarkersSymbolizer()
        symbolizer.width = 5.0
        symbolizer.stroke_width = 0.0
        symbolizer.fill = mapnik.Color("#ff0000")
        symbolizer.opacity = 1.0
        symbolizer.allow_overlap = mapnik.Expression("True")
        rule.symbols.append(symbolizer)
        self.style.rules.append(rule)
Example #22
0
def download_image(url_path,
                   timeout=(5, 20)):  # 下载图片,定义一个方法方便开启多线程,返回下载该图片的相对路径
    #semaphore.acquire()  #执行中的线程计数器+1。已被ThreadPoolExecutor取代
    url = url_path[0]
    path = url_path[1]
    convert_status = url_path[2]
    comic_name = url.split('/')[-1].split('?')[0]
    comic_local_position = path + '/' + comic_name
    global ERROR_PAGE_LIST  #全局变量
    global WARNING_PAGE_LIST
    try:
        if not url_path in ERROR_PAGE_LIST:
            ERROR_PAGE_LIST.append(url_path)  # 先把网页加入错误列表,以防网络错误、I/O错误引发中断造成遗漏
        #注意:如果上一次没有解决的错误网页,会再次重复记录。所以重试下载时需要去重。
        comic_page = get(url, headers=public_headers,
                         timeout=timeout)  #可从传入timeout参数防止网络问题阻塞
        # if comic_page.status_code != 200:
        # print('!= 200')
        image_bytes = BytesIO(comic_page.content)
        if image_bytes.__sizeof__() >= 1:  #防止下载的图片为0kb,玄学?!
            image_source = Image.open(image_bytes)
            image_source.save(comic_local_position)
        else:
            # print('content is lost')
            # raise Exception("0字节图片")  # 后来发现没必要处理0k图片。是服务器的问题,重试也没用,过几天服务器就好了
            WARNING_PAGE_LIST.append(url_path)  #额外记录,后续不处理
        if convert_status:
            convertImg(comic_local_position)  # 对“无耻”的以修改图片的反爬虫机制进行反制!
        if url_path in ERROR_PAGE_LIST:  # 如果下载成功就再下载列表删除它
            ERROR_PAGE_LIST.remove(url_path)
            # semaphore.release()  #执行中的线程计数器-1。已被ThreadPoolExecutor取代
            # print ("【下载完成】 ", url_path[0])
            return url_path  #下载完成后返回url地址,完成的地址记录在进程池中,用于标记下载进度,或可取代ERROR_PAGE_LIST的记录动作
    except Exception:
        # print('Download Error, File', url_path)
        # semaphore.release()  #执行中的线程计数器-1。已被ThreadPoolExecutor取代
        pass
Example #23
0
 def save(self, *args, **kwargs):
     if self.theme_image:
         img = Image.open(BytesIO(self.theme_image.read()))
         if img.mode != 'RGB':
             img = img.convert('RGB')
     output = BytesIO()
     img.save(output, format='JPEG', quality=40)
     output.seek(0)
     self.theme_image = InMemoryUploadedFile(output, 'ImageField', '{}.jpg'.format(self.theme_image.name.split('.')[0]), 'image/jpg', output.__sizeof__(), None)
     return super(Album, self).save(*args, **kwargs)
Example #24
0
    }

    s.headers = headers

    if idSearch := re.search(r"/c/([A-Za-z0-9]+)key.html", url):
        imgUrl = imgUrlTemp.format(idSearch.group(1))
    else:
        print("getWeatherImage: No valid URL found for image")
        status = False

    # pull image
    r = s.get(imgUrl, stream=True)

    out = BytesIO(r.content)

    if r.status_code != 200 and out.__sizeof__() < byteSizeThreshold:
        print("getWeatherImage: Image failed to download!")
        status = False

    return *outputImage(out, status), imgUrl  # I hope the * unpacks correctly


def outputImage(imgBytes: BytesIO, status: bool):
    if debug and status:
        f = open("testImage.png", "wb")
        f.write(imgBytes.read())
        f.close()
    return status, imgBytes


if debug:
 print 'Verifying and fetching file ' + file_key.name
 # Setting encoding to None returns just bytes
 for retry in xrange(3):
     contents = file_key.get_contents_as_string(encoding=None)
     m = hashlib.md5()
     m.update(contents)
     if file_key.md5 == m.hexdigest():
         break
     else:
         print "The md5 of file %s does not match"%filename
         if retry == 2:
             print "Exceeded the number of md5 file check retries"
             raise ValueError("Could not verify md5 of file " + file_key.name)
 try:
     bio = BytesIO(contents)
     if bio.__sizeof__() > maxBuffCreated:
         maxBuffCreated = bio.__sizeof__()
     if fetchCount == 0:
         if cleanPath:
             # TODO: Added a function that checks if a path exists
             try:
                 client.list(hdfsPath)
             except HdfsError as e:
                 if e.message.lower().find("does not exist") != -1:
                     print "Hdfs path %s does not exists"%hdfsPath
             else:
                 for hdfsFile in client.list(hdfsPath):
                     if hdfsFile.startswith(fnPrefix):
                         print 'Removing HDFS path ' + hdfsPath + hdfsFile
                         client.delete(hdfsPath + hdfsFile)
                         rmFileCount += 1
Example #26
0
from sclib import SoundcloudAPI, Track
from io import BytesIO

api = SoundcloudAPI()
urls = ['https://soundcloud.com/nittigritti/lights-nitti-gritti-remix-1']
for url in urls:
    track = api.resolve(url)
    file = BytesIO()
    size = file.__sizeof__()
    track.write_mp3_to(file)
Example #27
0
def make_thumbnail(size, image, name):
    f = BytesIO()
    image.thumbnail(size, Image.ANTIALIAS)
    image.format = image.format or 'JPEG'
    image.save(f, format=image.format)
    f.seek(0)
    return InMemoryUploadedFile(f, 'ImageField', name, f'image/{image.format.lower()}', f.__sizeof__(), None)