def _save(self, name, content):
"""保存文件的时候使用"""
# name ：上传文件的名称
# content： 包含上传文件内容的File对象
# {
# 'Group name': group_name,
# 'Remote file_id': remote_file_id,
# 'Status': 'Upload successed.',
# 'Local file name': '',
# 'Uploaded size': upload_size,
# 'Storage IP': storage_ip
# }
# 上传文件到fdfs系统中
client = Fdfs_client(self.client_conf)
# 获取上传文件的内容
content = content.read()
# 调用upload_by_buffer()
res = client.upload_by_buffer(content)
# 判断是否上传成功
if res['Status'] != 'Upload successed.':
raise Exception('上传文件到fdfs失败')
# 获取文件的id
file_id= res['Remote file_id']
# 返回文件的id
return file_id
def update(self, instance, validated_data):
# image就是校验过后的"图片对象"
image = validated_data.pop('image')
# 1、获得fdfs链接对象
conn = Fdfs_client(settings.FDFS_CONFIG_PATH)
# 2、上传文件
content = image.read() # bytes
result = conn.upload_by_buffer(content)
# 3、判断是否上传成功
if not result.get("Status") == "Upload successed.":
raise serializers.ValidationError("上传失败")
# 4、如果成功，获得fdfs文件标示
url = result.get("Remote file_id")
validated_data['image'] = url
return super().update(instance, validated_data)
def update(self, instance, validated_data):
file = validated_data.pop("image")
content = file.read() # content: 上传来的文件"数据" byte:字节对象
# 1.2 获得fdfs链接对象
# conn = Fdfs_client('./meiduo_mall/client.conf')
conn = Fdfs_client(settings.FASTDFS_PATH)
# 1.3 根据文件数据上传
res = conn.upload_by_buffer(content) # 传入数据也是字节对象
if res['Status'] != 'Upload successed.':
# 上传失败
raise serializers.ValidationError("上传失败！")
instance.image = res['Remote file_id']
instance.save()
print(instance)
# 更新商品默认显示的图片
SKU.objects.filter(id=instance.id).update(default_image=instance.image)
return instance
def _save(self, name, content):
"""
name: 用户选择上传文件的名称 1.txt
content: 包含用户上传文件内容的一个File对象，可以通过content.read()获取上传文件内容
"""
# 创建Fdfs_client对象，参数传入fdfs_client客户端配置文件路径
client = Fdfs_client(self.client_conf)
# 上传文件到fastdfs文件存储系统
res = client.upload_by_buffer(content.read())
if res.get('Status') != "Upload successed.":
raise Exception("上传文件到FastDFS文件系统出错")
# 获取文件id
file_id = res.get('Remote file_id')
# Django会将该方法的返回值保存到数据库中对应的文件字段
return file_id
def _save(self, name, content):
"""上传（保存）文件"""
# 指定配置文件
# tracker_path = get_tracker_conf('./utils/fdfs/client.conf')
tracker_path = get_tracker_conf(self.conf_path)
client = Fdfs_client(tracker_path)
# 上传文件到fdfs服务器，并接受一个fdfs服务器返回的File对象（字典型）
receive = client.upload_by_buffer(content.read())
# {'Group name': 'group1',
# 'Remote file_id': 'group1/M00/00/00/wKgrY2A4Ty2AVr7cAAPYbeRSiPo554.jpg',
# 'Status': 'Upload successed.',
# 'Local file name': '/home/image/Desktop/miku1.jpg',
# 'Uploaded size': '246.00KB',
# 'Storage IP': '192.168.43.99'}
if receive.get('Status') != 'Upload successed.':
raise Exception('文件上传fdfs服务器失败')
# 修改返回对象：返回文件ID而不是File对象，浏览器接受时，会根据IP和文件ID渲染该地址的图片
fileid = receive.get('Remote file_id').decode()
return fileid
def save(self, name, content, max_length=None):
client = Fdfs_client(self.client_conf)
res = client.upload_by_buffer(content.read())
'''
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : '',
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
if res.get('Status') != 'Upload successed.':
raise Exception('upload file to fast dfs failed.')
file_name = res.get('Remote file_id')
return file_name
def save(self, name, content, max_length=None):
'''重写上传文件的函数'''
# 我们需要将文件上传到 FastDFS 上面.
# 创建客户端对象:
client = Fdfs_client(settings.FDFS_CLIENT_CONF)
# 调用上传函数, 进行上传:
# 我们这里调用的是上面说过的, 根据文件内容上传方法.
result = client.upload_by_buffer(content.read())
# 判断是否上传成功:
if result.get('Status') != 'Upload successed.':
raise Exception('上传文件到FDFS系统失败')
# 上传成功: 返回 file_id:
file_id = result.get('Remote file_id')
# 这个位置返回以后, django 自动会给我们保存到表字段里.
return file_id
def _save(self, name, content):
'''保存文件时使用'''
# name 选择文件的名字
# content 包含上传文件内容的File对象
#创建一个Fdfs_client对象
client = Fdfs_client(self.client_conf)
#上传文件到fast dfs系统中
res = client.upload_by_buffer(content.read())
if res.get('Status') != 'Upload successed.':
#上传失败
raise Exception('上传文件到FASTDFS失败')
filename = res.get('Remote file_id')
return filename
def _save(self, name, content):
f_client = Fdfs_client(self.client_conf)
res = f_client.upload_by_buffer(content.read())
# res 返回值
# dict
# {
# 'Group name': group_name,
# 'Remote file_id': remote_file_id,
# 'Status': 'Upload successed.',
# 'Local file name': '',
# 'Uploaded size': upload_size,
# 'Storage IP': storage_ip
# }
if res.get('Status') != 'Upload successed.':
raise Exception('上传失败')
filename = res.get('Remote file_id')
print(filename)
return self.tracker_server + filename.replace('\\', '/')
def _save(self, name, content):
"""
保存文件
:param name: 前端传递的文件名
:param context: 文件数据
:return: 存储到数据库中的文件名
"""
# 保存到FastDFS
client = Fdfs_client(self.client_conf)
try:
ret = client.upload_by_buffer(content.read())
except Exception as e:
print(e)
if ret.get("Status") != "Upload successed.":
raise Exception("upload file failed")
file_name = ret.get("Remote file_id")
return file_name
def _save(self, name, content):
"""
上传文件时会调用此方法,让文件上传到远程FastDFS服务器中
:param name: 要上传的文件名
:param content: 要上传的文件对象 将来需要content.read() 文件二进制读取出并上传
:return: file_id
"""
# 创建fastDFS客户端对象 指定客户端配置文件
client = Fdfs_client(self.client_conf)
# 上传文件
ret = client.upload_by_buffer(
content.read()) # 如果要上传的是文件数据二进制数据流,可以用此方法上传文件,并且上传后,此文件没有后缀
# 判断文件是否上传成功
if ret.get('Status') != 'Upload successed.':
raise Exception('Upload file failed')
file_id = ret.get('Remote file_id') # 获取字典中的file_id
return file_id
def update(self, instance, validated_data):
sid = transaction.savepoint()
try:
image = self.context['request'].FILES.get('image')
if not image:
return Response(status=status.HTTP_404_NOT_FOUND)
client = Fdfs_client(BASE_CONFIG)
result = client.upload_by_buffer(image.read())
if result['Status'] != "Upload successed.":
return Response(status=status.HTTP_406_NOT_ACCEPTABLE)
image = result['Remote file_id']
validated_data['image'] = image
SKUImage.objects.filter(id=instance.id).update(**validated_data)
except Exception as E:
transaction.savepoint_rollback(sid)
raise serializers.ValidationError('上传图片失败')
else:
transaction.savepoint_commit(sid)
return instance
def _save(self, name, content):
'''保存文件时使用'''
# name: 上传文件的名字
# content: 包含上传文件内容的File对象
# 创建Fdfs_client对象
# client = Fdfs_client('./utils/fastdfs/client.conf')
client = Fdfs_client(self.client_conf)
# 上传文件到 fdfs系统总
# result = client.upload_by_filename('test')
result = client.upload_by_buffer(content.read())
# 判断上传是否成功
if result.get('Status') != 'Upload successed.': # 上传失败
raise Exception("上传文件到fdfs系统失败,请重新上传...")
# 获取上传成功后返回的文件ID
file_id = result.get('Remote file_id')
return file_id
def _save(self, name, content):
"""
name: 用户选择上传文件的名称 1.txt
content: 包含用户上传文件内容的一个File对象，可以通过content.read()获取上传文件内容
"""
# client = Fdfs_client('客户端配置文件路径')
client = Fdfs_client(self.client_conf)
# 上传文件到FDFS文件存储系统
res = client.upload_by_buffer(content.read())
if res.get('Status') != 'Upload successed.':
# 上传失败
raise Exception('上传文件到FDFS系统失败')
# 获取文件ID
file_id = res.get('Remote file_id')
return file_id
def upload_image(image_data):
"""
上传图片
:param image_data: 图片的二进制数据
:return: 返回图片的访问路径
"""
# 创建Fdfs_client对象
client = Fdfs_client(constants.FDFS_CLIENT_CONFIG_PATH)
# 上传文件的流
res = client.upload_by_buffer(image_data)
if res.get('Status') != 'Upload successed.':
# 上传失败
raise Exception('上传文件到fast dfs失败')
# 获取返回文件的ID
filename = res.get("Remote file_id")
return filename
def _save(self, filename, content):
"""
在dfs中保存文件
:param filename:
:param content:
:return:
"""
client = Fdfs_client(self.client_conf)
result = client.upload_by_buffer(content.read())
"""
client.upliad_by_buffer 接收参数是文件bytes字节数据，所以要read（）
"""
if result.get('Status') != 'Upload successed.':
raise Exception("Upload file error when upload file %s" % filename)
file_name = result.get("Remote file_id")
if not file_name:
raise Exception(
"Can not get remote file name when upload file %s" % filename)
return file_name
def _save(self, name, content):
""" 保存文件时使用 """
# name: 你选择上传文件你的名字
# content： 包含你上传文件内容的File对象
# 创建一个Fdfs_client对象
client = Fdfs_client(self.client_conf)
# 上传文件到FastDFS系统中
res = client.upload_by_buffer(content.read())
if res.get('Status') != "Upload successed.":
# 上传失败
raise Exception(res)
# 获取返回的文件ID
filename = res.get("Remote file_id")
return filename
def save(self, name, content, max_length=None):
'''重写上传文件的函数'''
# 我们需要将文件上传到 FastDFS 上面.
# 创建客户端对象:
client = Fdfs_client(settings.FDFS_CLIENT_CONF)
# 调用上传函数, 进行上传:
# 我们这里调用的是上面说过的, 根据文件内容上传方法.
result = client.upload_by_buffer(content.read())
# 判断是否上传成功:
if result.get('Status') != 'Upload successed.':
raise Exception('上传文件到FDFS系统失败')
# 上传成功: 返回 file_id:
file_id = result.get('Remote file_id')
# 这个位置返回以后, django 自动会给我们保存到表字段里.
return file_id
def _save(self, name, content):
# path = super()._save(name, content)
client = Fdfs_client('utils/fdfs/client.conf')
try:
datas = content.read()
dict_data = client.upload_by_buffer(datas)
status = dict_data.get('Status')
if status == 'Upload successed.':
# 上传图片成功
path = dict_data.get('Remote file_id')
else:
raise Exception('上传图片失败：%s' % status)
except Exception as e:
print(e)
raise e
return path
def _save(self, name, content):
"""
name:上传文件的名称:1.jpg
content:包含上传文件内容FILE对象,可以通过content.read()获取上传文件的内容
"""
# 将文件上传到FDFS系统
# client = Fdfs_client("客户端配置文件路径")
# client = Fdfs_client(settings.FDFS_CLIENT_CONF)
client = Fdfs_client(self.client_conf)
res = client.upload_by_buffer(content.read())
# 判断上传文件是否成功
if res.get('Status') != 'Upload successed.':
raise Exception("上传文件到FDFS失败")
# 获取文件id
file_id = res.get('Remote file_id')
return file_id
def _save(self, name, content):
"""
name: 上传文件的名称 1.jpg
content: 包含上传文件内容的File对象，可以通过content.read()获取上传文件的内容
"""
# 创建Fdfs_client对象
# client = Fdfs_client(settings.FDFS_CLIENT_CONF)
client = Fdfs_client(self.client_conf)
# 将文件上传到FDFS系统
ret = client.upload_by_buffer(content.read())
if ret.get('Status') != 'Upload successed.':
# 上传文件失败
raise Exception('上传文件到FDFS失败')
# 获取文件id
file_id = ret.get('Remote file_id')
return file_id
def _save(self, name, content):
"""
文件上传时会调用此方法
:param name: 要上传的文件名
:param content: 要上传的文件对象
:return: file_id
"""
# 创建对象
client = Fdfs_client(self.client_conf)
# 上传文件到FDFS系统
res = client.upload_by_buffer(content.read())
if res.get('Status') != 'Upload successed.':
raise Exception('上传文件到FDFS系统失败')
# 获取file_id
file_id = res.get('Remote file_id')
return file_id
def save(self, name, content, max_length=None):
"""
重写上传文件的函数
:param name:
:param content:
:param max_length:
:return:
"""
# 创建客户端对象
client = Fdfs_client(settings.FDFS_CLIENT_CONF)
# 调用函数进行上传
result = client.upload_by_buffer(content.read())
# 判断是否上传成功
if result.get('Status') != 'Upload successed.':
raise Exception('上传文件到FDFS系统失败')
# 返回file_id
file_id = result.get('Remote file_id')
return file_id
def _save(self, name, content):
"""保存文件"""
# 创建一个客户端对象
client = Fdfs_client(self.client_conf)
ret = client.upload_by_buffer(content.read())
# @return dict
# {
# 'Group name': group_name,
# 'Remote file_id': remote_file_id,
# 'Status': 'Upload successed.',
# 'Local file name': '',
# 'Uploaded size': upload_size,
# 'Storage IP': storage_ip
# }
if ret.get('Status') != 'Upload successed.':
raise Exception('upload file failed')
filename = ret.get('Remote file_id')
return filename
def create(self, request, *args, **kwargs):
# sku, 图片
# 1. 接收数据
# POST
sku_id = request.data.get('sku')
# 2. 获取图片数据
photo = request.FILES.get('image')
# 3. 创建Fdfs客户端
from fdfs_client.client import Fdfs_client
client = Fdfs_client('meiduo_mall/utils/fastdfs/client.conf')
# 4. 上传图片二进制
# photo 是文件
# photo.read() 读取文件的二进制数据
result = client.upload_by_buffer(photo.read())
"""
{
'Group name': 'group1',
'Remote file_id': 'group1/M00/00/00/wKhnnlxw_gmAcoWmAAEXU5wmjPs35.jpeg',
'Status': 'Upload successed.',
'Local file name': '/Users/meihao/Desktop/demo.jpeg',
'Uploaded size': '69.00KB',
'Storage IP': '172.16.238.128'
}
"""
# 5. 获取上传结果，根据上传结果进行file_id的保存
if result['Status'] == 'Upload successed.':
file_id = result['Remote file_id']
# 6. 保存SKUImage
new_skuimage = SKUImage.objects.create(sku_id=sku_id,
image=file_id)
# 前端就是根据 201 进行判断的
return Response(
{
'id': new_skuimage.id,
'sku': sku_id,
'image': new_skuimage.image.url
},
status=201)
# 7. 返回响应
return Response({'msg': 'error'})
def post(self):
file = self.request.files
#用时间生成随机字符串作为文件名，以保证文件名的唯一
filename = time.strftime('%Y%m%d%H%M%S') + '.jpg'
file_img = file.get('file')
if file_img:
# 获取图片
img = file_img[0]['body']
#创建一个路径存储图片
with open(
os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(__file__))),
'bufferimage/' + filename), 'wb') as f:
f.write(img)
#配置文件路径
client = Fdfs_client(
os.path.join(
os.path.dirname(os.path.dirname(
os.path.dirname(__file__))), 'fastdfs/client.conf'))
# 把图片上传到服务器
ret = client.upload_by_filename(
os.path.join(
os.path.dirname(os.path.dirname(
os.path.dirname(__file__))),
'bufferimage/' + filename))
ip = ret['Storage IP']
file_id = ret['Remote file_id']
file_path = 'http://' + ip + '/' + file_id
print(file_path)
#保存图片路径
mobile = self.get_secure_cookie('mobile').decode('utf8')
cursor = self.db.cursor()
sql = "update user set file_id='" + file_path + "' where mobile='" + mobile + "'"
self.db.ping(reconnect=True)
cursor.execute(sql)
self.db.commit()
self.db.close()
self.write({'info': 0})
def create(self, request, *args, **kwargs):
# 接收 sku_id
sku_id = request.data.get('sku')
# 接收图片数据
image_file = request.data.get('image')
# 数据校验
if not all([sku_id, image_file]):
raise serializers.ValidationError('数据不完整')
try:
sku = SKU.objects.get(id=sku_id)
except:
raise serializers.ValidationError('修改的商品不存在')
# 数据处理
# 实例化 Fdfs_client
client = Fdfs_client(settings.FDFS_CLIENT_CONF)
# 通过二进制流上传图片
ret = client.upload_appender_by_buffer(image_file.read())
# 判断状态
if ret.get('Status') != 'Upload successed.':
return Response('图片上传失败')
# 获取上传图片的 url
image_url = ret.get('Remote file_id')
# 保存到数据库
# 判断商品以前是否有图片及是否有默认图片, 没有就将上传的图片作为默认图片
with transaction.atomic():
save_point = transaction.savepoint()
try:
if not sku.images.count() or sku.default_image == '':
# 没有就修改 sku 表中商品的默认图片
sku.default_image = image_url
sku.save()
sku_image = SKUImage.objects.create(sku=sku, image=image_url)
except:
transaction.savepoint_rollback(save_point)
raise serializers.ValidationError()
else:
transaction.savepoint_commit(save_point)
# 生成新的静态页面
generate_detail_html.delay(sku_id)
# 返回响应
serializer = self.get_serializer(sku_image)
return Response(serializer.data, status=201)
def _save(self, name, content):
"""
文件存储的时候调用此方法，但是默认是存储到本地的，需要重写，上传到FastDFS服务器上
:param name: 要上传的文件名
:param content: 以rb模式打开的文件对象，将来通过content.read(),就可以读取到文件的二进制数据
:return: file_id
"""
# 创建FastDFS客户端
client = Fdfs_client(self.client_path)
# 通过客户端调用上传文件的方法上传文件到FastDFS服务器
# client.upload_by_filename('要上传的文件的绝对路径') 只能通过文件的绝对路径上传，上传的文件有后缀
# upload_by_buffer可以通过文件二进制数据上传，，上传的文件没有后缀
ret = client.upload_by_buffer(content.read())
# 判断文件是否上传成功
if ret.get('Status') != 'Upload successed.':
raise Exception('Upload file failed')
# 获取file_id
file_id = ret.get('Remote file_id')
# 返回file_id
return file_id
def _save(self, name, context):
# C:\Users\lzj\Desktop\df\utils\fdfs\client.conf
trackers = get_tracker_conf(self.client_conf)
client = Fdfs_client(trackers)
res = client.upload_appender_by_buffer(context.read())
# dict
# {
# 'Group name': group_name,
# 'Remote file_id': remote_file_id,
# 'Status': 'Upload successed.',
# 'Local file name': local_file_name,
# 'Uploaded size': upload_size,
# 'Storage IP': storage_ip
# }
if res.get('Status') != 'Upload successed.':
raise Exception('上传失败')
filename = res.get('Remote file_id')
# return filename.encode()
return filename
def images(self, request):
image = request.data.get('image')
if image is None:
return Response(status=500)
# 链接fastDSF
client = Fdfs_client('/home/python/Desktop/meiduo_46/meiduo_demo/utils/fastdfs/client.conf')
# 上传图片
ret = client.upload_by_buffer(image.read())
# 判断是否成功
if ret['Status'] != 'Upload successed.':
return Response({'error': '上传失败'}, status=501)
# 提取图片链接地址
path = ret.get('Remote file_id')
# 返回结果
return Response({
'img_url': 'http://192.168.131.138:8888/' + path
})
def _save(self, name, content):
print(1)
conf = get_tracker_conf(self.client_conf)
client = Fdfs_client(conf)
result = client.upload_by_buffer(content.read())
# dict
# {
# 'Group name': group_name,
# 'Remote file_id': remote_file_id,
# 'Status': 'Upload successed.',
# 'Local file name': '',
# 'Uploaded size': upload_size,
# 'Storage IP': storage_ip
# }
if result.get('Status') != 'Upload successed.':
raise Exception('上传文件到FastDFS失败')
filename = result.get('Remote file_id')
return filename.decode()
def _save(self, name, content):
# content自己必须是一个File对象
client = Fdfs_client(conf_path=self.conf_client)
ret = client.upload_by_buffer(content.read())
"""
dict
{
'Group name': group_name,
'Remote file_id': remote_file_id,
'Status': 'Upload successed.',
'Local file name': '',
'Uploaded size': upload_size,
'Storage IP': storage_ip
} if success else None
"""
# 如果上传失败则抛出异常
if not ret['Status'] == 'Upload successed.':
raise Exception('FastDFS Upload Fail')
# 返回被保存文件的真实名称
return ret.get('Remote file_id')
def __init__(
self, item_id, num_id, crawl_path, server_path=None, org_server_path=None, statshost=None, statsport=0
):
self.item_id = item_id
self.results = {}
self.headers = {"Referer": "http://item.taobao.com/item.htm?id=%s" % num_id, "User-Agent": DEFAULT_UA}
self.success = False
self.crawl_path = crawl_path
self.server_path = server_path
self.org_server_path = org_server_path
self.image_set = set()
self.fdfs_client = Fdfs_client("/etc/fdfs/client.conf")
self.statshost = statshost
self.statsport = statsport
def _save(self, name, content):
"""保存 context是用户上传过来的文件对象"""
# 从文件对象中读取文件数据
file_data = content.read()
# 创建用户存储fastdfs的客户端
client = Fdfs_client(self.client_conf)
# 上传文件内容到fastdfs中
try:
ret = client.upload_by_buffer(file_data)
# ret的返回值格式 {'Group name':'group1','Status':'Upload successed.', 'Remote file_id':'group1/M00/00/00/
# wKjzh0_xaR63RExnAAAaDqbNk5E1398.py','Uploaded size':'6.0KB','Local file name':'test'
# , 'Storage IP':'192.168.243.133'}
except Exception as e:
print(e)
raise
if ret.get("Status") == "Upload successed.":
# 表示上传成功
file_id = ret.get("Remote file_id")
# 返回file_id， 也就是文件名，django会保存到数据库中
return file_id
else:
raise Exception("上传到FastDFS失败")
def __init__(self, item_id, num_id, crawl_path, server_path=None, org_server_path=None, statshost=None, statsport=0):
self.item_id = item_id
self.results = {}
self.headers = {
'Referer': "http://item.taobao.com/item.htm?id=%s" % num_id,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3',
'User-Agent': DEFAULT_UA
}
self.success = False
self.crawl_path = crawl_path
self.server_path = server_path
self.org_server_path = org_server_path
self.image_set = set()
self.fdfs_client = Fdfs_client('/etc/fdfs/client.conf')
self.statshost = statshost
self.statsport = statsport
class ItemCrawler:
def __init__(
self, item_id, num_id, crawl_path, server_path=None, org_server_path=None, statshost=None, statsport=0
):
self.item_id = item_id
self.results = {}
self.headers = {"Referer": "http://item.taobao.com/item.htm?id=%s" % num_id, "User-Agent": DEFAULT_UA}
self.success = False
self.crawl_path = crawl_path
self.server_path = server_path
self.org_server_path = org_server_path
self.image_set = set()
self.fdfs_client = Fdfs_client("/etc/fdfs/client.conf")
self.statshost = statshost
self.statsport = statsport
def crawl(self, image_rows, thumb_sizes, is_commit=False, conn=None, is_remove=False):
self.prepare_path(thumb_sizes)
for item in image_rows:
self.crawl_image(item, thumb_sizes, conn)
self.check_success(thumb_sizes)
if is_commit:
if self.success:
self.commit(conn)
else:
self.commit_fail(conn)
if is_remove:
try:
shutil.rmtree(self.crawl_tmp_path)
except:
pass
def prepare_path(self, thumb_sizes):
self.crawl_tmp_path = "%s/%s" % (self.crawl_path, self.item_id)
self.static_url = "static/%s" % self.item_id
try:
shutil.rmtree(self.crawl_tmp_path)
except:
pass
try:
os.makedirs(self.crawl_tmp_path)
except:
pass
for width, height in thumb_sizes:
try:
os.makedirs("%s/%s/%sx%s" % (self.crawl_path, self.item_id, width, height))
except:
pass
def check_success(self, thumb_sizes):
self.success_type1_count = 0
self.success_count = 0
self.type1_count = 0
self.count = 0
has_type1_pos1 = False
for pos, img in self.results.items():
self.count += 1
if img.image_type == 1:
self.type1_count += 1
if img.pos == 1:
has_type1_pos1 = True
if not img.disabled and len(img.thumbs) == len(thumb_sizes):
self.success_count += 1
if img.image_type == 1:
self.success_type1_count += 1
if self.success_type1_count > 0 and has_type1_pos1:
self.success = True
else:
self.success = False
self.summary = {
"suc1": self.success_type1_count,
"count1": self.type1_count,
"suc": self.success_count,
"count": self.count,
}
def commit_fail(self, conn):
conn.execute(
'update crawl_html set is_image_crawled=0,last_modified=now(),crawl_image_info="%s" where item_id=%s'
% (self.summary, self.item_id)
)
# conn.execute("update item set crawl_status=1 where id=%s" % self.item_id)
def commit(self, conn):
try:
trans = conn.begin()
conn.execute("delete from item_images where item_id=%s;" % self.item_id)
for pos, img in self.results.items():
if img.disabled or not img.url or not img.fullpath:
continue
if len(img.thumbs) == 0:
continue
if len(img.thumbs) > 1:
raise Exception("Thumbs size list len MUST BE ONE")
for thumbsize, thumb_filename in img.thumbs.items():
# uploading to fastdfs
ret = self.fdfs_client.upload_by_filename(thumb_filename)
if ret["Status"] == "Upload successed.":
group_name = ret["Group name"]
remote_file = ret["Remote file_id"]
# update database
conn.execute(
"insert into item_images (item_id,url,local_url,type,width,height,pos,is_downloaded,has_thumbnail,uniq_url, fastdfs_group, fastdfs_filename) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
self.item_id,
img.url,
img.get_server_path(),
img.image_type,
img.width,
img.height,
img.pos,
1,
1,
img.uniq_url,
group_name,
remote_file,
)
logger.debug("upload to fastdfs %s -> %s", thumb_filename, remote_file)
else:
logger.warn("upload to fastdfs failed, reason: %s", ret)
raise Exception("upload failed %s" % ret)
conn.execute(
'update crawl_html set is_image_crawled=1,crawl_image_info="%s" where item_id=%s'
% (self.summary, self.item_id)
)
conn.execute("update item set crawl_status=2 where id=%s" % self.item_id)
trans.commit()
except Exception, e:
logger.info("commit %s failed error: %s", self.item_id, traceback.format_exc())
trans.rollback()
raise
def save_fastfs(self):
client = Fdfs_client('/etc/fdfs/client.conf')
rv = client.upload_by_filename(self.tmp_path)
return rv
class FilesrvHandler(object):
def __init__(self):
self.secret_code = "sW$^dffGad3e3h8fh7gc~t7 `7 t"
self.root = "/data0/androidapk/"
self.client = Fdfs_client('/etc/fdfs/client.conf')
def test(self, words):
return words
def save(self, fileobj, meta):
try:
result = {}
hexs = sha1(meta.appid)
#hexs.update(str(meta.version_code))
hexs.update(self.secret_code)
hexs.update(str(meta.seq))
hex_code = hexs.hexdigest()
dirs = os.path.join(meta.file_type, *[hex_code[(i-1)*2:i*2] for i in range(1, 5)])
filename = "{}.{}".format(hex_code[8:], meta.ext)
filepath = os.path.join(dirs, filename)
fullpath = os.path.join(self.root, filepath)
try:
os.makedirs(os.path.join(self.root, dirs))
except:
pass
with open(fullpath, 'w') as f:
f.write(fileobj)
result["md5"] = md5(fileobj).hexdigest()
result["path"] = filepath
except Exception as e:
result = {"error": str(e)}
return json.dumps(result)
def save2fdfs(self, filebuff, meta):
meta_dict = {
"appid": meta.appid,
"version_code": meta.version_code,
"version_name": meta.version_name,
"file_type": meta.file_type,
"ext": meta.ext,
"seq": meta.seq
}
rs = self.client.upload_by_buffer(filebuff, meta.ext, meta_dict=meta_dict)
result = {
"md5": md5(filebuff).hexdigest(),
"path": rs["Remote file_id"],
"size": rs["Uploaded size"]
}
return json.dumps(result)
def save_media2fdfs(self, filebuff, ext):
rs = self.client.upload_by_buffer(filebuff, ext)
result = {
"md5": md5(filebuff).hexdigest(),
"path": rs["Remote file_id"],
"size": rs["Uploaded size"]
}
return json.dumps(result)
def get(self, fileid):
try:
meta = self.client.get_meta_data(fileid)
return Meta(
appid=meta['appid'],
version_code=int(meta['version_code']),
version_name=meta['version_name'],
file_type=meta['file_type'],
ext=meta['ext'],
seq=int(meta['seq'])
)
except Exception as e:
return str(e)
def remove(self, fileid):
os.remove(os.path.join(self.root, fileid))
return json.dumps({'appid': fileid})
def __init__(self):
self.secret_code = "sW$^dffGad3e3h8fh7gc~t7 `7 t"
self.root = "/data0/androidapk/"
self.client = Fdfs_client('/etc/fdfs/client.conf')
def post(self):
uri = self.request.uri
avatar_sha1=self.get_argument("f_file.sha1",None)
if avatar_sha1:
user_key = self.get_secure_cookie("user_key")
res = {}
upload_file_name=self.get_argument("f_file.name",None)
ext = upload_file_name.split('.')[1]
img_path_t=os.path.join("/tmp/upload_temp_dir/",str(avatar_sha1))
img_path = "%s.%s" % (img_path_t,ext)
upload_file_path=self.get_argument("f_file.path",None)
os .rename(upload_file_path,img_path)
client = Fdfs_client(globalSetting['fdfs_client_conf'])
ret = client.upload_by_filename(img_path)
remote_file_id = ret['Remote file_id']
storage_ip = ret['Storage IP']
pic_url = "http://%s:%d/%s" % (storage_ip,globalSetting['fdfsPort'],remote_file_id)
print pic_url
from models.avatar import Avatar
avatar = Avatar()
avatar.key = user_key
avatar.data['url'] = pic_url
avatar.post()
res['avatar'] = pic_url
res['error'] = ''
return self.dumpJson(res)
upload_file_sha1=self.get_argument("ifile.sha1",None)
#upload_file_size=self.get_argument("ifile.size",None)
upload_file_path=self.get_argument("ifile.path",None)
upload_file_name=self.get_argument("ifile.name",None)
print upload_file_name
ext = upload_file_name.split('.')[1]
img_path_t=os.path.join("/tmp/upload_temp_dir/",str(upload_file_sha1))
img_path = "%s.%s" % (img_path_t,ext)
img_path_170x170 = "%s_170x170.%s" % (img_path_t,ext)
try:
i=Img()
i.open('/usr/bin')
i.convert_resize(input_file=upload_file_path,output_file=img_path,output_size="500x")
i.convert_thumbnail(input_file=img_path,output_file=img_path_170x170,output_size="170x170")
except Exception as what:
#data=file(upload_file_path,'rb').close()
print what
pass
client = Fdfs_client(globalSetting['fdfs_client_conf'])
ret = client.upload_by_filename(img_path)
remote_file_id = ret['Remote file_id']
storage_ip = ret['Storage IP']
pic_url = "http://%s:%d/%s" % (storage_ip,globalSetting['fdfsPort'],remote_file_id)
ret = client.upload_by_filename(img_path_170x170)
remote_file_id = ret['Remote file_id']
storage_ip = ret['Storage IP']
thumb_url = "http://%s:%d/%s" % (storage_ip,globalSetting['fdfsPort'],remote_file_id)
user_key = self.get_secure_cookie("user_key")
board = Board()
query = {}
query['q'] = "user:%s" % user_key
_data = board.solr(query)
self.render('upload_form.html',boards = _data,pic_url=pic_url,thumb_url=thumb_url)
