def upload_photo():
"""上传照片到七牛，并返回私有链接地址"""
from qiniu import Auth
from qiniu import put_file
global config
progress_handler = lambda progress, total: progress
photo_path = http_get("http://127.0.0.1:9876/photo/shot").content
# Upload to qiniu
mime_type = "image/jpeg"
auth = Auth(str(config["qiniu"]["api_key"]), str(config["qiniu"]["secret"]))
print auth
filename = os.path.basename(photo_path)
print "filename: ", filename, type(filename)
token = auth.upload_token(str(config["qiniu"]["bucket"]))
print token
ret, info = put_file(token, filename, photo_path, {}, mime_type, progress_handler=progress_handler)
print "uploaded: ", ret, info
try:
os.remove(photo_path)
except Exception:
pass
# Return URL
base_url = "{}/{}".format(str(config["qiniu"]["domain"]), filename)
return auth.private_download_url(base_url, expires=3600)
class QiniuPro():
def __init__(self):
self.auth = Auth(ACCESS_KEY, SECRET_KEY)
def download_private_url(self, key, expires=7200):
base_url = '%s/%s' % (DOMAIN, key)
private_url = self.auth.private_download_url(base_url, expires)
return private_url
def get_download_file_url(file_name):
"""获取文件url"""
cache_key = '_'.join((settings.CACHE_PREFIX, file_name))
private_url = cache.get(cache_key)
if private_url:
return private_url
q = Auth(settings.QINIU_KEY, settings.QINIU_TOKEN)
base_url = 'http://%s/%s' % (settings.QINIU_HOST, file_name)
private_url = q.private_download_url(base_url, expires=3600*24)
cache.set(cache_key, private_url, 3600*24)
return private_url
def GetQiniuDlToken(key):
if key is '' or key is None:
return False,'文件名为空'
#开发者账号授权
q = Auth(access_key, secret_key)
private_url = ''
#生成私有的基本URL
base_url = 'http://%s/%s' % (bucket_url, key)
try:
#生成私密url
private_url = q.private_download_url(base_url, 3600)
return True,private_url
except:
print '第三方获取上传凭证函数异常'
return False,private_url
def test_fetch():
q = Auth(access_key, secret_key)
bm = BucketManager(q)
ret, info = bm.stat(bucket, mykey)
print(info)
print(ret)
bucket_domain = '7xpb8s.com1.z0.glb.clouddn.com'
base_url = 'http://%s/%s' % (bucket_domain, mykey)
private_url = q.private_download_url(base_url, expires=3600)
print(private_url)
r = requests.get(private_url)
assert r.status_code == 200
ret, info = bm.fetch(private_url, bucket, mykey)
print(info)
print(ret)
def list_file(username, path): # ok
user = User.objects.filter(username=username)
if bool(user):
user = user[0]
else :
return False
q = Auth(settings.QINIU_ACCESS_KEY, settings.QINIU_SECRET_KEY) # 授权
files = FileInfo.objects.filter(owner=user,file_path=path) #
filelist = {}
for f in files:
if f.file_type == 'dir': # 文件夹不需要下载 ，但可以设置另外的子目录获取的url，待定
filelist[f.file_name] = ""
continue
if path == '/':
path = ""
base_url = ''.join([settings.QINIU_DOMAIN,username, path, '/', f.file_name])
filelist[f.file_name] = q.private_download_url(base_url, expires=3600)
return json.dumps(filelist,ensure_ascii=False,encoding="utf-8",sort_keys=True)
def qiniu_private(base_url):
"""
get private resource
"""
cache_key = 'st:qiniu:' + base_url
cache_value = cache.get(cache_key)
if cache_value:
return cache_value
q = Auth(get_qiniu_config('QINIU_ACCESS_KEY'),
get_qiniu_config('QINIU_SECRET_KEY'))
expire = 3600 if not hasattr(settings,
'QINIU_PREVIEW_EXPIRE') else settings.QINIU_PREVIEW_EXPIRE
private_url = q.private_download_url(base_url, expires=expire)
cache.set(cache_key, private_url, timeout=max(10, expire - 10))
return private_url
def content_process(self, content):
# render content included gist
content = re.sub(r'http(s)?:\/\/gist.github.com\/(\d+)(.js)?', r'<script src="http://gist.github.com/\2.js"></script>', content)
# render sinaimg pictures
content = re.sub(r'(http:\/\/\w+.sinaimg.cn\/.*?\.(jpg|gif|png))', r'<img src="\1" />', content)
# render qiniu private image link
qiniu_re = re.compile(r'(%s/.*?\.(?:jpg|jpeg|gif|png))' % bucket_domain)
image_urls = qiniu_re.findall(content)
q = Auth(access_key, secret_key)
private_url = "jk"
for image_url in image_urls:
private_url = q.private_download_url(image_url, expires=10)
#private_url = re.sub(r'&', '&', private_url)
content = re.sub(image_url, r'<img src="%s" />' % private_url, content)
# render @ mention links
content = re.sub(r'@(\w+)(\s|)', r'@<a href="/u/\1">\1</a> ', content)
# render youku videos
content = re.sub(r'http://v.youku.com/v_show/id_(\w+).html', r'<iframe height=498 width=510 src="http://player.youku.com/embed/\1" frameborder=0 allowfullscreen style="width:100%;max-width:510px;"></iframe>', content)
return content
def GetQiniuDlToken(key,url='7xouyw.com1.z0.glb.clouddn.com',commonFlag=False):
if key is '' or key is None:
return False,'文件名为空'
if isSrc(key):
return True,key
#公共空间，不需要生成下载凭证
if commonFlag:
return True,'http://%s/%s' % (url, key)
#开发者账号授权
q = Auth(access_key, secret_key)
private_url = ''
#生成私有的基本URL
base_url = 'http://%s/%s' % (url, key)
try:
#生成私密url
private_url = q.private_download_url(base_url, 3600)
return True,private_url
except:
debug( '第三方获取上传凭证函数异常')
return False,private_url
class QiniuStorage(object):
'''
七牛云的文件上传、显示、删除
@auth:ZWJ
'''
def __init__(self, access_key, secret_key, bucket_name, bucket_domain):
"""
@para:
access_key:公钥
secret_key:私钥
bucket_name: 要上传的空间
bucket_domain:获取文件url路径时对应的私有域名
"""
self.auth = Auth(access_key, secret_key)
self.bucket_name = bucket_name
self.bucket_domain = bucket_domain
self.bucket_manager = BucketManager(self.auth)
def put_data(self, name, data):
"""
@def:put_data
@def_fun: 文件流上传
空间里的文件名不能重复，所以用_newname生成新文件名
@para:
name: 文件名
data: 上传二进制流
@ret:上传后的url路径
"""
#上传到七牛后保存的文件名
key = self._newname(name)
#生成上传 Token，可以指定过期时间等
token = self.auth.upload_token(self.bucket_name, key)
if hasattr(data, 'chunks'):
data = b''.join(c for c in data.chunks())
ret, info = put_data(token, key, data) #上传文件流到七牛
if ret is None or ret['key'] != key:
raise QiniuError(info)
return self.get_url(key)
def _newname(self, name):
'''加上6位日期和6位时间标识 PG.jpg --> PG_170211_044217.jpg '''
root, ext = splitext(basename(name))
time = datetime.now().strftime('_%y%m%d_%H%M%S')
return '{}{}{}'.format(root, time, ext)
def get_url(self, key):
'''
@def:get_url
@def_fun: 返回七牛云上文件名为key的文件对应的url地址
如果是公有空间，该地址可以直接访问文件；私有空间则需用private_download_url
@para:
key: 七牛云上的文件名
@ret:域名加文件名生成的url路径
'''
url = 'http://{}/{}'.format(self.bucket_domain, key)
return url
def private_download_url(self, url, expires=7200):
"""
@def:private_download_url
@def_fun: 生成私有资源下载链接
@para:
url: 私有空间资源的原始URL
expires: 下载凭证有效期，默认为7200s
@ret:私有资源的下载链接
"""
return self.auth.private_download_url(url, expires)
def put_file(self, filePath):
"""
@def:put_file
@def_fun: 本地文件上传
空间里的文件名不能重复，所以用_newname生成新文件名
@para:
filePath: 待上传文件在磁盘中的绝对路径
@ret:上传后的url路径
"""
key = self._newname(filePath)
token = self.auth.upload_token(self.bucket_name, key)
ret, info = put_file(token, key, filePath)
if ret is None or ret['key'] != key:
raise QiniuError(info)
return self.get_url(key)
def exists(self, key):
'''检测七牛云上是否有文件名为key的文件'''
bucket = self.bucket_manager
ret, info = bucket.stat(self.bucket_name, key.split('/')[-1])
return ret is not None
def delete(self, key):
'''删除七牛云上文件名为key的文件'''
if not self.exists(key):
return '{} not exist in qiniu_cloud'.format(key)
bm = self.bucket_manager
ret, info = bm.delete(self.bucket_name, key.split('/')[-1])
if ret == {}:
return 'success to delete {} in qiniu_cloud'.format(key)
else:
return info
def ls_files(self, prefix="", limit=None):
"""
@def:ls_file
@def_fun: 显示七牛云上的文件名
具体规格参考:
http://developer.qiniu.com/docs/v6/api/reference/rs/list.html
@para:
prefix: 列举前缀
limit: 单次列举个数限制
@ret: 文件名组成的set()集合
"""
files = set()
dirlist = bucket_lister(self.bucket_manager, self.bucket_name, prefix,
limit)
for item in dirlist:
files.add(item['key'])
return files
def get_download_file_url(file_name):
"""获取文件url"""
q = Auth(settings.QINIU_KEY, settings.QINIU_TOKEN)
base_url = 'http://%s/%s' % (settings.QINIU_HOST, file_name)
private_url = q.private_download_url(base_url, expires=3600 * 24)
return private_url
class EntryService(object):
"""EntryService."""
def __init__(self):
self.entries = {}
self.pages = {}
self.urls = []
self.by_tags = {}
self.by_categories = {}
self.by_months = {}
self.models = Models()
self.types = self.models.types()
self.params = self.models.params()
self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
self.blog_bucket = BucketManager(self.qiniu)
self.blog_prefix = "raw/"
self._init_blog()
def _init_blog(self):
"""
Initialize blog
- all entries in entry_dir
- all pages in page_dir
- others
"""
cloud_list = {}
ret, _, info = self.blog_bucket.list(config.qiniu_bucket, prefix = "raw/")
file_list = eval(info.text_body)["items"]
for f in file_list:
if f["key"].startswith(self.blog_prefix) and f["mimeType"] == "text/markdown":
file_name = f["key"][len(self.blog_prefix):]
print "Find qiniu file %s" % file_name
cloud_list[file_name] = (f["key"], f["fsize"])
print cloud_list
for root, _, files in os.walk(config.entry_dir):
for f in files:
if f in cloud_list:
print "Find local file %s/%s" % (root, f)
del cloud_list[f]
self.add_entry(False, root + '/' + f)
for f in cloud_list:
info = cloud_list[f]
self.download_file(f, info[0])
print "Download file %s/%s" % (root,f)
self.add_entry(False, root + '/' + f)
for root, _, files in os.walk(config.page_dir):
for f in files:
self._add_page(root + '/' + f)
self._init_miscellaneous(self.types.add, self.entries.values())
def download_file(self, f, key):
base_url = 'http://%s/%s' % (config.qiniu_bucket_domain, key)
private_url = self.qiniu.private_download_url(base_url, expires=3600)
urllib.urlretrieve(private_url, "%s/%s" % (config.entry_dir, f))
print(private_url)
def add_entry(self, inotified, path):
"""
Add entry
"""
entry = self._init_entry(self.types.entry, path)
if entry is not None:
self.entries[entry.url] = entry
if inotified:
self._init_miscellaneous(self.types.add, [entry])
def delete_entry(self, path):
"""
Delete entry
"""
for entry in self.entries.values():
if path == os.path.abspath(entry.path):
print "delete success!"
self.entries.pop(entry.url)
self._init_miscellaneous(self.types.delete, [entry])
def _add_page(self, path):
"""
Add page
"""
page = self._init_entry(self.types.page, path)
if page is not None:
self.pages[page.url] = page
def _init_entry(self, entry_type, path):
"""
initialize single entry
"""
url, raw_url, name, date, time, content = self._init_file(path, entry_type)
if url is not None:
entry = self.models.entry(entry_type)
entry.path = path
entry.name = name
entry.url = url
entry.raw_url = raw_url
entry.date = date
entry.time = time
header, title, categories, tags = extract.parse(entry)
entry.content = content
content = content.replace(header, '')
entry.html = markdown.markdown(content)
# FIXME How to extract the excerpt of an entry
entry.excerpt = content[:200] + ' ... ...'
entry.categories = categories
entry.tags = tags
return entry
return None
def _init_file(self, file_path, entry_type):
"""
Initialize single file
"""
# FIXME: how to determine the publish time of an entry
content, nones = None, [None for _ in xrange(6)]
try:
content = codecs.open(file_path, mode='r', encoding='utf-8').read()
except:
return nones
if content is None or len(content.strip()) == 0:
return nones
date, mtime = None, None
name, _ = os.path.splitext(os.path.basename(file_path))
chars = ['_', '-', '~']
pattern = r'\d{4}-\d{1,2}-\d{1,2}'
match = re.search(pattern, name)
if match:
y, m, d = match.group().split('-')
try:
date = datetime.date(int(y), int(m), int(d))
except:
print traceback.format_exc()
print file_path
name = name[len(match.group()):]
for c in chars:
if name.startswith(c):
name = name[1:]
stat = os.stat(file_path)
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
if date is None:
date = mtime
prefix, url_prefix, raw_prefix = date.strftime(config.url_date_fmt), '', ''
if entry_type == self.types.entry:
url_prefix = config.entry_url + '/' + prefix + '/'
raw_prefix = config.raw_url + '/' + prefix + '/'
elif entry_type == self.types.page:
url_prefix = '/'
raw_prefix = config.raw_url + '/'
date = date.strftime(config.date_fmt)
time = date + mtime.strftime(config.time_fmt)[len('yyyy-mm-dd'):]
url = url_prefix + name + config.url_suffix
raw_url = raw_prefix + name + config.raw_suffix
for c in chars:
name = name.replace(c, ' ')
return url, raw_url, name, date, time, content
def _init_miscellaneous(self, init_type, entries):
"""
Initialize miscellaneous
- tags
- categories
- archives
- all urls
"""
for entry in entries:
self._init_tag(init_type, entry.url, entry.tags)
self._init_category(init_type, entry.url, entry.categories)
self._init_monthly_archive(init_type, entry.url)
self.urls = sorted(self.entries.keys(), reverse=True)
self._init_params()
def _init_subscribe(self):
"""
Initialize subscriptions
"""
if not self.urls:
time = datetime.datetime.now().strftime(config.time_fmt)
else:
time = self.entries[self.urls[0]].time
return self.models.subscribe(time)
def _init_tag(self, init_type, url, tags):
"""
Initialize tags
"""
for tag in tags:
if tag not in self.by_tags:
if init_type == self.types.add:
self.by_tags[tag] = self.models.tag(tag, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_tags[tag].urls.insert(0, url)
self.by_tags[tag].count += 1
if init_type == self.types.delete:
self.by_tags[tag].count -= 1
self.by_tags[tag].urls.remove(url)
if self.by_tags[tag].count == 0:
self.by_tags.pop(tag)
def _init_category(self, init_type, url, categories):
"""
Initialize categories
"""
for category in categories:
if category not in self.by_categories:
if init_type == self.types.add:
self.by_categories[category] = \
self.models.category(category, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_categories[category].urls.insert(0, url)
self.by_categories[category].count += 1
if init_type == self.types.delete:
self.by_categories[category].count -= 1
self.by_categories[category].urls.remove(url)
if self.by_categories[category].count == 0:
self.by_categories.pop(category)
def _init_monthly_archive(self, init_type, url):
"""
Initialize archives
"""
start = len(config.entry_url) + 1
end = start + len('/yyyy/mm')
month = url[start:end]
if month not in self.by_months:
if init_type == self.types.add:
self.by_months[month] = \
self.models.monthly_archive(self.types.entry, month, url)
if init_type == self.types.delete:
pass
else:
if init_type == self.types.add:
self.by_months[month].urls.insert(0, url)
self.by_months[month].count += 1
else:
self.by_months[month].count -= 1
self.by_months[month].urls.remove(url)
if self.by_months[month].count == 0:
self.by_months.pop(month)
def _init_params(self):
"""
Initialize global params
:return:
"""
self.params.subscribe = self._init_subscribe()
self.params.primary.tags = self._init_tags_widget()
self.params.primary.recently_entries = self._init_recently_entries_widget()
self.params.secondary.categories = self._init_categories_widget()
self.params.secondary.calendar = self._init_calendar_widget()
self.params.secondary.archive = self._init_archive_widget()
def _init_related_entries(self, url):
"""
Initialize related entries
"""
# FIXME: related entries
try:
index = self.urls.index(url)
except:
print traceback.format_exc()
return None
urls = self.urls[:index]
urls.extend(self.urls[index + 1:])
urls = random.sample(urls, min(len(urls), 10))
return [self.entries.get(url) for url in sorted(urls, reverse=True)]
def _init_abouts_widget(self, about_types=None, url=None):
"""
Initialize abouts widget
:param about_types:
:param url:
:return:
"""
about_types = about_types or []
abouts = []
for about_type in about_types:
about = self.models.about(about_type)
if about_type == self.types.entry and url is not None:
try:
i = self.urls.index(url)
p, n = i + 1, i - 1
except:
print traceback.format_exc()
p, n = 999999999, -1
if p < len(self.urls):
url = self.urls[p]
about.prev_url = url
about.prev_name = self.entries[url].name
if n >= 0:
url = self.urls[n]
about.next_url = url
about.next_name = self.entries[url].name
if about_type == self.types.archive:
about.prev_url = '/'
about.prev_name = 'main index'
elif about_type == self.types.blog:
about.prev_url = '/'
about.prev_name = 'main index'
about.next_url = config.archive_url
about.next_name = 'archives'
abouts.append(about)
return abouts
def _init_tags_widget(self):
"""
Initialize tags widget
"""
# FIXME: calculate tags' rank
tags = sorted(self.by_tags.values(), key=lambda v: v.count, reverse=True)
ranks = config.ranks
div, mod = divmod(len(tags), ranks)
if div == 0:
ranks, div = mod, 1
for r in range(ranks):
s, e = r * div, (r + 1) * div
for tag in tags[s:e]:
tag.rank = r + 1
return tags
def _init_recently_entries_widget(self):
"""
Initialize recently entries widget
:return:
"""
return [self.entries[url] for url in self.urls[:config.recently]]
def _init_calendar_widget(self):
"""
Initialize calender widget
:return:
"""
date = datetime.datetime.today().strftime(config.date_fmt)
if len(self.urls) > 0:
date = self.entries[self.urls[0]].date
calendar = self.models.calendar(date)
y, m = calendar.month.split('-')
for url in self.urls:
_, _, _, _, d, _ = url.split('/')
prefix = config.entry_url + '/' + y + '/' + m + '/' + d
d = int(d)
if url.startswith(prefix):
calendar.counts[d] += 1
if calendar.counts[d] > 1:
start = len(config.entry_url)
end = start + len('/yyyy/mm/dd')
calendar.urls[d] = config.archive_url + url[start:end]
else:
calendar.urls[d] = url
else:
break
return calendar
def _init_categories_widget(self):
"""
Initialize categories widget
:return:
"""
return sorted(self.by_categories.values(), key=lambda c: c.name)
def _init_archive_widget(self):
"""
Initialize archive widget
:return:
"""
return sorted(self.by_months.values(), key=lambda m: m.url, reverse=True)
def _find_by_query(self, query, start, limit):
"""
Find by query
:param query:
:param start:
:param limit:
:return:
"""
# FIXME: how to search in the content of entries
queries = [q.lower() for q in query.split(' ')]
urls = []
for query in queries:
for entry in self.entries.values():
try:
entry.content.index(query)
urls.append(entry.url)
except:
print
return self._find_by_page(sorted(urls), start, limit)
def _find_by_page(self, urls, start, limit):
"""
Find by page
:param urls:
:param start:
:param limit:
:return:
"""
if urls is None or start < 0 or limit <= 0:
return [], 0
total = len(urls)
urls = sorted(urls, reverse=True)
s, e = (start - 1) * limit, start * limit
if s > total or s < 0:
return [], 0
return [self.entries[url] for url in urls[s:e]], total
def _paginate(self, pager_type, value, total, start, limit):
"""
Pagination
:param pager_type:
:param value:
:param total:
:param start:
:param limit:
:return:
"""
if limit <= 0:
return self.models.pager(pager_type, value, total, 0, start, limit)
pages, mod = divmod(total, limit)
if mod > 0:
pages += 1
return self.models.pager(pager_type, value, total, pages, start, limit)
def find_by_url(self, entry_type, url):
"""
Find content by url
:param entry_type:
:param url:
:return:
"""
entry, abouts = None, [self.types.blog]
if entry_type == self.types.entry:
entry = self.entries.get(url)
abouts.insert(0, self.types.entry)
elif entry_type == self.types.page:
entry = self.pages.get(url)
self.params.entry = entry
self.params.entries = self._init_related_entries(url)
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget(abouts, url)
return self.params
def find_raw(self, raw_url):
"""
Find the raw content by raw_url
:param raw_url:
:return:
"""
page_url = raw_url.replace(config.raw_url, '').replace(config.raw_suffix, config.url_suffix)
page = self.find_by_url(self.types.page, page_url).entry
if page is not None and page.raw_url == raw_url:
return page.content
entry_url = raw_url.replace(config.raw_url, config.entry_url).replace(config.raw_suffix, config.url_suffix)
entry = self.find_by_url(self.types.entry, entry_url).entry
if entry is not None and entry.raw_url == raw_url:
return entry.content
return None
def archive(self, archive_type, url, start=1, limit=999999999):
"""
Archives
:param archive_type:
:param url:
:param start:
:param limit:
:return:
"""
self.params.error = self.models.error(url=url)
if archive_type == self.types.raw:
url = url.replace(config.raw_url, config.archive_url)
entries, count, = [], 0
archive_url = url.replace(config.archive_url, '').strip('/')
prefix = url.replace(config.archive_url, config.entry_url)
pattern = r'\d{4}/\d{2}/\d{2}|\d{4}/\d{2}|\d{4}'
match = re.search(pattern, archive_url)
if match and match.group() == archive_url or archive_url == '':
urls = [url for url in self.urls if url.startswith(prefix)]
entries, _ = self._find_by_page(urls, start, limit)
count = len(entries)
else:
entries = None
if archive_url == '':
archive_url = self.types.all
self.params.entries = entries
self.params.archive = self.models.archive(archive_type, url, archive_url, url, count)
self.params.primary.abouts = self._init_abouts_widget([self.types.archive])
return self.params
def search(self, search_type, url, value='', start=config.start, limit=config.limit):
"""
Search the site
:param search_type:
:param url:
:param value:
:param start:
:param limit:
:return:
"""
entries, total, abouts = None, 0, [self.types.blog]
if search_type == self.types.query:
entries, total = self._find_by_query(value, start, limit)
elif search_type == self.types.tag:
if self.by_tags.get(value) is None:
entries = None
else:
entries, total = self._find_by_page(self.by_tags.get(value).urls, start, limit)
elif search_type == self.types.category:
if self.by_categories.get(value) is None:
entries = None
else:
entries, total = self._find_by_page(self.by_categories.get(value).urls, start, limit)
elif search_type == self.types.index:
entries, total = self._find_by_page(self.urls, start, limit)
abouts = []
self.params.error = self.models.error(url=url)
self.params.entries = entries
self.params.search = self.models.search(search_type, value, total)
self.params.pager = self._paginate(search_type, value, total, start, limit)
self.params.primary.abouts = self._init_abouts_widget(abouts)
return self.params
def error(self, url):
"""
Error params
:param url:
:return:
"""
self.params.error = self.models.error(url=url)
self.params.primary.abouts = self._init_abouts_widget([self.types.blog])
return self.params
path = os.path.join(path, r'dest_photo')
for root, dirs, files in os.walk(path):
print(files)
fileLength = len(files)
# filename = os.path.split(path)[1]
for i in range(fileLength):
localfile = path + "\%s" % files[i] # 照片路径
# file_name= os.path.splitext(files[i]) #文件名
img_url = url + files[i] + para
# 可以设置token过期时间
private_url = q.private_download_url(img_url, expires=3600)
r = requests.get(private_url)
img_name = os.getcwd()
img_name = os.path.join(img_name, r'restore_photo')
if not os.path.isdir(img_name):
os.mkdir(img_name)
img_name =img_name + "\%s" % files[i]
print(img_name)
with open(img_name, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
def qiniu_download(address, file_path):
access_key = 'ZSC-X2p4HG5uvEtfmn5fsTZ5nqB3h54oKjHt0tU6'
secret_key = 'Ya8qYwIDXZn6jSJDMz_ottWWOZqlbV8bDTNfCGO0'
q = Auth(access_key, secret_key)
if address.startswith('qiniu:'):
address = address.replace('qiniu:', '')
private_url = q.private_download_url(address, expires=3600)
if os.path.isdir(file_path):
file_path = os.path.join(file_path, str(uuid.uuid4()))
try:
fpath, _ = urllib.request.urlretrieve(private_url, file_path)
statinfo = os.stat(fpath)
size = statinfo.st_size
if size == 0:
logger.error('couldnt download data')
return None
return file_path
except:
logger.error('couldnt download data')
return None
# import tarfile
# import subprocess
#
# def _recursive_tar(root_path, path, tar, ignore=None):
# if path.split('/')[-1][0] == '.':
# return
#
# if os.path.isdir(path):
# for sub_path in os.listdir(path):
# _recursive_tar(root_path, os.path.join(path, sub_path), tar)
# else:
# if ignore is not None:
# if path.split('/')[-1] == ignore:
# return
# arcname = os.path.relpath(path, root_path)
# tar.add(path, arcname=arcname)
#
# random_code_package_name = str(uuid.uuid4())
# code_tar_path = os.path.join('/Users/Jian/Downloads/aaa', '%s_code.tar.gz' % random_code_package_name)
# tar = tarfile.open(code_tar_path, 'w:gz')
# for sub_path in os.listdir('/Users/Jian/Downloads/aaa'):
# _recursive_tar('/Users/Jian/Downloads/aaa',
# os.path.join('/Users/Jian/Downloads/aaa', sub_path),
# tar,
# ignore='%s_code.tar.gz' % random_code_package_name)
# tar.close()
#
# crypto_code = str(uuid.uuid4())
#
# crypto_shell = 'openssl enc -e -aes256 -in %s -out %s -k %s' % (
# '%s_code.tar.gz' % random_code_package_name,
# '%s_code_ssl.tar.gz' % random_code_package_name,
# crypto_code)
# subprocess.call(crypto_shell, shell=True, cwd='/Users/Jian/Downloads/aaa')
#
# # 解密
# decrypto_shell = 'openssl enc -d -aes256 -in %s -out %s -k %s'%('%s_code_ssl.tar.gz' % random_code_package_name,
# '%s_code.tar.gz' % random_code_package_name,
# crypto_code)
# subprocess.call(decrypto_shell, shell=True, cwd='/Users/Jian/Downloads/aaa')
# # qiniu_address = qiniu_upload(code_tar_path, bucket='mltalker', max_size=10)
# # print(qiniu_address)
#
# qiniu_download('http://pbzz7zw0y.bkt.clouddn.com/14697433-5bd9-4b3c-a392-ba72172c666e_code.tar.gz', '/Users/Jian/Downloads/zj')
def get_download_file_url(file_name):
"""获取文件url"""
q = Auth(settings.QINIU_KEY, settings.QINIU_TOKEN)
base_url = 'http://%s/%s' % (settings.QINIU_HOST, file_name)
private_url = q.private_download_url(base_url, expires=3600*24)
return private_url
def get_private_url(username, path,filename):
q = Auth(settings.QINIU_ACCESS_KEY, settings.QINIU_SECRET_KEY) # 授权
base_url = ''.join([settings.QINIU_DOMAIN,username, path, '/', filename])
return q.private_download_url(base_url, expires=3600)
def schedule_download_files(self,
order_id,
code_address,
code_address_code=None):
workspace = os.path.join(self.workspace, order_id)
if not os.path.exists(workspace):
os.makedirs(workspace)
if code_address is None or code_address == '':
with open(os.path.join(workspace, 'download.success'), 'w'):
pass
return
if code_address.startswith('ipfs:'):
# download from ipfs
response = ipfs_download(code_address.replace('ipfs:'), workspace)
if response:
self.log.info(
'IPFS_DONWLOAD_SUCCESS: download dependent files for order %s'
% order_id)
try:
with tarfile.open(os.path.join(workspace, 'code.tar.gz'),
'r:gz') as tar:
tar.extractall(workspace)
with open(os.path.join(workspace, 'download.success'),
'w'):
pass
except:
# clear incomplete data
# if os.path.exists(os.path.join(workspace, 'code')):
# shutil.rmtree(os.path.join(workspace, 'code'))
if os.path.exists(os.path.join(workspace, 'code.tar.gz')):
os.remove(os.path.join(workspace, 'code.tar.gz'))
self.log.error('IPFS_DOWNLOAD_ERROR: for order %s' %
order_id)
with open(os.path.join(workspace, 'download.fail'), 'w'):
pass
else:
# clear incomplete data
if os.path.exists(os.path.join(workspace, 'code.tar.gz')):
os.remove(os.path.join(workspace, 'code.tar.gz'))
self.log.error('IPFS_DOWNLOAD_ERROR: for order %s' % order_id)
with open(os.path.join(workspace, 'download.fail'), 'w'):
pass
elif code_address.startswith('qiniu:'):
access_key = 'ZSC-X2p4HG5uvEtfmn5fsTZ5nqB3h54oKjHt0tU6'
secret_key = 'Ya8qYwIDXZn6jSJDMz_ottWWOZqlbV8bDTNfCGO0'
q = Auth(access_key, secret_key)
base_url = code_address.replace('qiniu:', '')
private_url = q.private_download_url(base_url, expires=3600)
key = base_url.split('/')[-1] # code.tar.gz
try:
fpath, _ = urllib.request.urlretrieve(
private_url, os.path.join(workspace, key))
statinfo = os.stat(fpath)
size = statinfo.st_size
if size == 0:
self.log.error(
'FILE_DOWNLOAD_ERROR: empty file for order %s' %
order_id)
else:
try:
if code_address_code is not None:
decrypto_shell = 'openssl enc -d -aes256 -in %s -out %s -k %s' % (
key, key.replace('_ssl',
''), code_address_code)
subprocess.call(decrypto_shell,
shell=True,
cwd=workspace)
with tarfile.open(
os.path.join(workspace,
key.replace('_ssl', '')),
'r:gz') as tar:
tar.extractall(workspace)
if os.path.getsize(
os.path.join(workspace,
key.replace('_ssl', ''))) == 0:
self.log.error(
'FILE_DOWNLOAD_ERROR: for order %s' % order_id)
with open(os.path.join(workspace, 'download.fail'),
'w'):
pass
return
# clear tar file
os.remove(
os.path.join(workspace, key.replace('_ssl', '')))
os.remove(os.path.join(workspace, key))
self.log.info(
'FILE_DONWLOAD_SUCCESS: download dependent files for order %s'
% order_id)
with open(os.path.join(workspace, 'download.success'),
'w'):
pass
except:
# clear incomplete data
# if os.path.exists(os.path.join(workspace, 'code')):
# shutil.rmtree(os.path.join(workspace, 'code'))
# if os.path.exists(os.path.join(workspace, 'code.tar.gz')):
# os.remove(os.path.join(workspace, 'code.tar.gz'))
self.log.error('FILE_DOWNLOAD_ERROR: for order %s' %
order_id)
with open(os.path.join(workspace, 'download.fail'),
'w'):
pass
except:
self.log.error('FILE_DOWNLOAD_ERROR: for order %s' % order_id)
with open(os.path.join(workspace, 'download.fail'), 'w'):
pass
class qiniu():
def __init__(self):
self.access_key = "AK"
self.secret_key = "SK"
self.mdomain = "YourDomain"
self.mauth = Auth(self.access_key, self.secret_key)
def upload(self, upload_dir, upload_bucket):
if os.path.isdir(upload_dir):
if os.path.isdir(upload_dir):
up_file_list = []
for root, dirs, files in os.walk(upload_dir,topdown=True):
for v_file in files:
up_file_list.append(os.path.join(root, v_file))
for file_name in up_file_list:
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
# print(info)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
elif os.path.isfile(upload_dir):
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
elif os.path.isfile(upload_dir):
file_name = upload_dir
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
def list(self, bucket_name, prefix, limit=5):
bucket = BucketManager(self.mauth)
delimiter = None
marker = None
ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter)
for item in ret.get('items'):
print "{Name: %s Size:%s Hash:%s}" % (item['key'], item['fsize'], item['hash'])
def download(self, filename, output_dir):
base_url = 'http://%s/%s' % (self.mdomain, filename)
private_url = self.mauth.private_download_url(base_url)
print private_url
if os.path.exists(output_dir):
os.chdir(output_dir)
else:
os.makedirs(output_dir)
os.chdir(output_dir)
res = requests.get(private_url, stream=True)
assert res.status_code == 200
print res.status_code
with closing(res) as r:
accepts = 0
chunk_size = 512
with open(os.path.basename(filename), "wb") as code:
for chunk in r.iter_content(chunk_size=chunk_size):
#downlaod big data optimization；old mathed is code.write(r.content)
if chunk: # filter out keep-alive new chunks
code.write(chunk)
accepts += len(chunk)
# print accepts, int(r.headers['Content-Length'])
progress = round(float(accepts) / int(r.headers['Content-Length']), 4) * 100
sys.stdout.write('\r' + 'Now downlaod ' + str(progress) + '%')
sys.stdout.flush()
print '\n'
def delete(self, bucket_name, del_list):
mbucket = BucketManager(self.mauth)
del_item = build_batch_delete(bucket_name, del_list)
ret, info = mbucket.batch(del_item)
print info
def sort_out_list(request, data):
# 构建鉴权对象
q = Auth(configs.get('qiniu').get('AK'), configs.get('qiniu').get('SK'))
new_data = []
for obj in data:
# 获取商品图片链接，整理成数组
imgs = []
private_url = q.private_download_url(obj.get('goods_img1'),
expires=3600)
imgs.append(private_url)
if obj.get('goods_img2'):
private_url = q.private_download_url(obj.get('goods_img2'),
expires=3600)
imgs.append(private_url)
if obj.get('goods_img3'):
private_url = q.private_download_url(obj.get('goods_img3'),
expires=3600)
imgs.append(private_url)
if obj.get('goods_img4'):
private_url = q.private_download_url(obj.get('goods_img4'),
expires=3600)
imgs.append(private_url)
# 获取商品用户头像
product_username = obj.get('username')
try:
product_user = User.objects.get(username__exact=product_username)
except User.DoesNotExist:
product_user = None
if product_user.user_image_url:
product_avatar_url = q.private_download_url(
product_user.user_image_url, expires=3600)
else:
product_avatar_url = None
# 获取用户的收藏状态
try:
c = Collection.objects.get(username=str(request.user),
product_id=obj.get('id'))
except Collection.DoesNotExist:
c = None
if c:
collect_state = True
else:
collect_state = False
# 对时间字符串进行整理
date = obj.get('c_time').split(".")[0]
year = date.split("T")[0]
time = date.split("T")[1]
new_obj = {
'product_id': obj.get('id'),
'user_id': product_user.id,
'username': obj.get('username'),
'avatar_url': product_avatar_url,
'goods_price': obj.get('goods_price'),
'goods_img_url': imgs,
'collect_state': collect_state,
'title': obj.get('title'),
'description': obj.get('description'),
'time': year + " " + time
}
new_data.append(new_obj)
msg = {'stateCode': 200, 'list': new_data}
return msg
# -*- coding: utf-8 -*-
# flake8: noqa
import os
from qiniu import Auth
# 生成私有 url
# 需要填写你的 Access Key 和 Secret Key 要上传的空间
access_key = os.getenv('QINIU_ACCESS_KEY')
secret_key = os.getenv('QINIU_SECRET_KEY')
# 构建鉴权对象
q = Auth(access_key, secret_key)
# 生成私有 url
url = 'http://crawler-private.iamlj.com/1.png'
time = 3600
# 打印
private_url = q.private_download_url(url, time)
print private_url
class qiniu():
def __init__(self):
self.access_key = "AK"
self.secret_key = "SK"
self.mdomain = "YourDomain"
self.mauth = Auth(self.access_key, self.secret_key)
def upload(self, upload_dir, upload_bucket):
if os.path.isdir(upload_dir):
if os.path.isdir(upload_dir):
up_file_list = []
for root, dirs, files in os.walk(upload_dir, topdown=True):
for v_file in files:
up_file_list.append(os.path.join(root, v_file))
for file_name in up_file_list:
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
# print(info)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
elif os.path.isfile(upload_dir):
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
elif os.path.isfile(upload_dir):
file_name = upload_dir
token = self.mauth.upload_token(upload_bucket, file_name)
ret, info = put_file(token, file_name, file_name)
assert ret['key'] == file_name
assert ret['hash'] == etag(file_name)
print ret
def list(self, bucket_name, prefix, limit=5):
bucket = BucketManager(self.mauth)
delimiter = None
marker = None
ret, eof, info = bucket.list(bucket_name, prefix, marker, limit,
delimiter)
for item in ret.get('items'):
print "{Name: %s Size:%s Hash:%s}" % (item['key'], item['fsize'],
item['hash'])
def download(self, filename, output_dir):
base_url = 'http://%s/%s' % (self.mdomain, filename)
private_url = self.mauth.private_download_url(base_url)
print private_url
if os.path.exists(output_dir):
os.chdir(output_dir)
else:
os.makedirs(output_dir)
os.chdir(output_dir)
res = requests.get(private_url, stream=True)
assert res.status_code == 200
print res.status_code
with closing(res) as r:
accepts = 0
chunk_size = 512
with open(os.path.basename(filename), "wb") as code:
for chunk in r.iter_content(chunk_size=chunk_size):
#downlaod big data optimization；old mathed is code.write(r.content)
if chunk: # filter out keep-alive new chunks
code.write(chunk)
accepts += len(chunk)
# print accepts, int(r.headers['Content-Length'])
progress = round(
float(accepts) / int(r.headers['Content-Length']),
4) * 100
sys.stdout.write('\r' + 'Now downlaod ' +
str(progress) + '%')
sys.stdout.flush()
print '\n'
def delete(self, bucket_name, del_list):
mbucket = BucketManager(self.mauth)
del_item = build_batch_delete(bucket_name, del_list)
ret, info = mbucket.batch(del_item)
print info
class Logs():
def __init__(self):
#初始化Auth状态
self.au=Auth(access_key, secrek_ley)
self.bucket=BucketManager(self.au)
def time_format(self,date):
'''
校验日期格式
:param date:
:return:
'''
# domain_YYYY-MM-DD.gz
# prefix=domain_YYYY-MM-DD
# 正则校验日期格式
reg=re.compile('[0-9]{4}-(0|1)[0-9]{1}-[0-3]{1}[0-9]{1}')
if reg.match(date):
return str(date)
else:
return False
def log_list(self,prefix):
'''
根据前缀查询日志列表
:param prefix: 查询前缀
:return:返回查询结果
'''
res = self.bucket.list(bucket,prefix=prefix)
return res
def get_log_url(self,domain,date):
'''
生成日志下载链接
:param domain: 需要下载日志的域名
:param date: 时间：YYYY-MM-DD
:return:返回日志下载链接，有效期3600
'''
#生成查询前缀
if domain is None :
raise ValueError("Please Import Doamin!!!")
prefix=""
if self.time_format(str(date)):
prefix = domain + "_" + date
else:
raise ValueError("Date Format：YYYY-MM-DD")
#生成日志下载url
urls=[]
try:
res = self.log_list(prefix)
items=res[0]["items"]
for item in items:
key = item["key"]
base_url = 'http://%s/%s' % (bucket_domain, key)
private_url = self.au.private_download_url(base_url,3600)
urls.append(private_url)
return urls
except:
print "ERROR:"+res
class QiniuStorageAdapter(StorageAbstractAdapter):
__slots__ = ('__access_key', '__secret_key', '__auth', '__bucket',
'__file_info_cache', '__bucket_manager', '__domain')
def __init__(self, config, *args, **kw):
assert isinstance(config, dict)
assert isinstance(config.get("bucket"), str)
assert isinstance(config.get("access_key"), str)
assert isinstance(config.get("secret_key"), str)
assert isinstance(config.get("domain"), str)
self.__access_key = config.get("access_key")
self.__secret_key = config.get("secret_key")
self.__bucket = config.get("bucket")
self.__domain = config.get("domain")
self.__auth = Auth(self.__access_key, self.__secret_key)
self.__bucket_manager = BucketManager(self.__auth)
self.__file_info_cache = {}
def __gen_upload_token(self, file_name, expire=3600, policy=None):
assert isinstance(file_name, str)
assert isinstance(expire, int)
assert isinstance(policy, (str, dict, type(None)))
if policy:
token = self.__auth.upload_token(
self.__bucket, file_name, expire, policy)
else:
token = self.__auth.upload_token(self.__bucket, file_name, expire)
return token
def token(self, file_name, **kw):
expire = kw.get("expire", 3600)
policy = kw.get("policy")
return self.__gen_upload_token(file_name, expire, policy)
def put(self, file_name, local_file, **kw):
token = kw.get('token')
assert token, "token can't be empty"
ret, info = put_file(token, file_name, local_file)
assert ret['key'] == file_name
assert ret['hash'] == etag(local_file)
def __gen_download_url(self, bucket_domain, file_name, expire=3600):
assert isinstance(bucket_domain, str)
assert isinstance(file_name, str)
assert isinstance(expire, int)
base_url = 'http://%s/%s' % (bucket_domain, file_name)
private_url = self.__auth.private_download_url(
base_url, expires=expire)
return private_url
def get(self, file_name, **kw):
pass
def get_url(self, file_name, **kw):
bucket_domain = self.__domain
expire = kw.get("expire", 3600)
return self.__gen_download_url(bucket_domain, file_name, expire)
def move(self, src, dest):
src_bucket, src_key = src.split("<:>")
dest_bucket, dest_key = dest.split("<:>")
ret, info = self.__bucket_manager.move(
src_bucket, src_key, dest_bucket, dest_key)
if ret != {}:
return False
return True
def copy(self, src, dest):
src_bucket, src_key = src.split("<:>")
dest_bucket, dest_key = dest.split("<:>")
ret, info = self.__bucket_manager.copy(
src_bucket, src_key, dest_bucket, dest_key)
if ret != {}:
return False
return True
def delete(self, file_name):
ret, info = self.__bucket_manager.delete(self.__bucket, file_name)
if ret != {}:
return False
return True
def __cache_file_info(self, file_name, content):
self.__file_info_cache[self.__bucket + ':' + file_name] = content
def __get_file_info_from_cache(self, file_name, item=None):
if item:
return self.__file_info_cache[self.__bucket + ':' + file_name].get(item)
else:
return self.__file_info_cache[self.__bucket + ':' + file_name]
def __check_file_info_cache(self, file_name):
if self.__bucket + ':' + file_name in self.__file_info_cache:
return True
return False
def __get_file_info_item(self, file_name, item):
if self.__check_file_info_cache(file_name):
return self.__get_file_info_from_cache(file_name, item)
ret, info = self.__bucket_manager.stat(self.__bucket, file_name)
if ret:
self.__cache_file_info(file_name, ret)
return self.__get_file_info_from_cache(file_name, item)
return None
def file_info(self, file_name):
return self.__get_file_info_item(file_name, None)
def file_size(self, file_name):
return self.__get_file_info_item(file_name, 'fsize')
def file_hash(self, file_name):
return self.__get_file_info_item(file_name, 'hash')
def file_mime(self, file_name):
return self.__get_file_info_item(file_name, 'mimeType')
def file_create_time(self, file_name):
return self.__get_file_info_item(file_name, 'putTime')
# 前缀
prefix = None
# 列举条目
limit = 1000
# 列举出除'/'的所有文件以及以'/'为分隔的所有前缀
delimiter = None
# 标记
marker = None
with open('wyeth_qiniu_filepath_new.txt', mode='w+', encoding='utf-8') as f:
while True:
ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter)
text_body = json.loads(info.text_body)
marker = text_body.get('marker', '')
print('marker=' + marker)
for i in ret.get('items', []):
path = i['key']
if path.startswith('/'):
path = path[1:]
if not path:
continue
base_url = 'http://wyeth-course.nibaguai.com/' + path
base_url = 'http://wyeth-course.nibaguai.com/' + path
f.write('%s\n' % base_url)
# 如果空间有时间戳防盗链或是私有空间，可以调用该方法生成私有链接
private_url = q.private_download_url(base_url, expires=100)
# print('private_url' + private_url)
if eof:
break
print('success')
class WebConnection():
def __init__(self,init_path):
self.path = os.path.realpath(init_path)
self.AK = '72UZoe-AJao4469EfG0kSZ1B4CBzmn_WO6wd2fdQ'
self.SK = 'Bn6vHxfwuX5IKiruRJsBLNQCmJwzLWr_9CCN89-2'
self.BN = 'xiaoyphoto'
self.BD = '7xpg1h.com1.z0.glb.clouddn.com'
self.wait_time = 3600
self.pre = ''
self.user = ''
self.passwd = ''
self.inflock = thread.allocate_lock()
self.ordlock = thread.allocate_lock()
self.datlock = thread.allocate_lock()
self.lislock = thread.allocate_lock()
self.order = []
self.log_state = 0
self.filelist = []
self.clouds = []
self.static_file = '..\\map\\log.txt'
self.inf = dict()
self.__connect()
thread.start_new_thread(self.__run,())
def stop(self):
self.log_state = 4
def login(self,username,passwd):
self.user = username
self.passwd = passwd
marker = None
self.pre = username + '&&' +passwd + '&&'
ret, eof, info = self.bucket.list(self.BN,\
prefix=self.pre, marker=marker,\
limit=1)
if len(ret['items']) == 0:
self.log_state = 2
else:
self.log_state = 1
self.clouds = []
print 'Success!'
def signin(self,username,passwd):
self.user = username
self.passwd = passwd
marker = None
self.pre = username + '&&' +passwd + '&&'
ret, eof, info = self.bucket.list(self.BN,\
prefix=self.user, marker=marker,\
limit=1)
if len(ret['items']) > 0:
self.log_state = 0
else:
self.log_state = 1
self.clouds = []
#up static_file
localfile = self.static_file
key = self.pre + localfile.split('\\')[-1]
token = self.q.upload_token(self.BN, key)
mime_type = self.__get_mime_type(localfile)
params = {'x:a': 'a'}
progress_handler = lambda progress, total: progress
ret, info = qiniu.put_file(token, key, localfile, params,\
mime_type, progress_handler=progress_handler)
def logout(self):
self.user = ''
self.passwd = ''
self.pre = ''
self.log_state = 0
def getlist(self):
return self.filelist
def setinf(self,ak,sk,bn,bd):
self.inflock.acquire()
self.AK = ak
self.SK = sk
self.BN = bn
self.BD = bd
self.inflock.release()
self.__connect()
def setpath(self,path):
self.inflock.acquire()
self.path = path
self.inflock.release()
#order must be a tuple such as ('up','...')
def exeord(self,str):
self.ordlock.acquire()
self.order.append(str)
self.ordlock.release()
def __connect(self):
self.datlock.acquire()
self.q = Auth(self.AK,self.SK)
self.bucket = BucketManager(self.q)
self.datlock.release()
def __run(self):
while True:
if self.log_state == 4:
break
elif self.log_state != 1 or len(self.order)==0:
time.sleep(0.1)
else:
self.ordlock.acquire()
tord = self.order[0]
self.ordlock.release()
print tord
if tord[0] == 'up':
self.__upload_file(tord[1])
elif tord[0] == 'down':
self.__download_file(tord[1])
elif tord[0] == 'dir':
self.__list_all()
elif tord[0] == 'del':
self.__delete_file(tord[1])
elif tord[0] == 'upd':
self.__update_file(tord[1])
self.ordlock.acquire()
del self.order[0]
self.ordlock.release()
def __get_mime_type(self,__path):
mime_type = "text/plain"
return mime_type
def __list_all(self):
marker = None
limit = None
eof = False
self.lislock.acquire()
self.filelist = []
self.lislock.release()
temp = []
while eof is False:
ret, eof, info = self.bucket.list(self.BN, prefix=self.pre, marker=marker, limit=limit)
marker = ret.get('marker', None)
for item in ret['items']:
if item['key'][-4:] == '.txt':
continue
if item['key'][-4:] == '.inf':
info = item['key'].split('&&')
filename = info[0]+'&&'+info[1]+'&&'+info[2]
mood = int(info[3])
pl = info[4]
self.inf[filename] = [mood,pl]
continue
temp.append(item['key'].split(self.pre)[1])
self.lislock.acquire()
self.filelist = temp
self.lislock.release()
def __delete_file(self,key):
ret ,info = self.bucket.delete(self.BN,self.pre+key)
def __update_file(self,localphoto):
localfile = localphoto.getpath()
key = self.pre + localfile.split('\\')[-1]
inf_key = key + '&&' + str(localphoto.getmood())\
+'&&'+localphoto.getcomment()+'&&.inf'
self.inf[key] = [localphoto.getmood(),localphoto.getcomment]
token1 = self.q.upload_token(self.BN,inf_key)
mime_type = self.__get_mime_type(localfile)
params = {'x:a': 'a'}
progress_handler = lambda progress, total: progress
ret1, info1 = qiniu.put_file(token1, inf_key, self.static_file,params,\
mime_type, progress_handler=progress_handler)
#now it need a photo object
def __upload_file(self,localphoto):
localfile = localphoto.getpath()
key = self.pre + localfile.split('\\')[-1]
inf_key = key + '&&' + str(localphoto.getmood())\
+'&&'+localphoto.getcomment()+'&&.inf'
self.inf[key] = [localphoto.getmood(),localphoto.getcomment]
token1 = self.q.upload_token(self.BN,inf_key)
token = self.q.upload_token(self.BN, key)
mime_type = self.__get_mime_type(localfile)
params = {'x:a': 'a'}
progress_handler = lambda progress, total: progress
ret1, info1 = qiniu.put_file(token1, inf_key, self.static_file,params,\
mime_type, progress_handler=progress_handler)
ret, info = qiniu.put_file(token, key, localfile, params,\
mime_type, progress_handler=progress_handler)
def __download_file(self,key):
filename = self.pre + key
base_url = 'http://%s/%s' % (self.BD,filename)
private_url = self.q.private_download_url(base_url,expires=3600)
r = requests.get(private_url)
with open(self.path+'\\'+key,'wb') as code:
code.write(r.content)
p = photo(self.path + '\\' +key)
if filename in self.inf:
p.setmood(self.inf[filename][0])
p.setcomment(self.inf[filename][1])
p.setcloud(1)
self.clouds.append(p)
from django.test import TestCase # Create your tests here. import requests from qiniu import Auth from swiper import config qn = Auth(config.QN_ACCESS_KEY, config.QN_SECRET_KEY) #有两种方式构造base_url的形式 #或者直接输入url的方式下载 base_url = 'http://qaezcmqwq.bkt.clouddn.com/Avatar-2.jpg' #可以设置token过期时间 private_url = qn.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) print(r) assert r.status_code == 200
class QiniuStorage(Storage):
"""
Qiniu Storage Service
"""
location = ""
def __init__(
self,
access_key=QINIU_ACCESS_KEY,
secret_key=QINIU_SECRET_KEY,
bucket_name=QINIU_BUCKET_NAME,
bucket_domain=QINIU_BUCKET_DOMAIN,
secure_url=QINIU_SECURE_URL,
private_url=QINIU_PRIVATE_URL,
private_url_expires=QINIU_PRIVATE_URL_EXPIRES):
self.auth = Auth(access_key, secret_key)
self.bucket_name = bucket_name
self.bucket_domain = bucket_domain
self.bucket_manager = BucketManager(self.auth)
self.secure_url = secure_url
self.private_url = private_url
self.private_url_expires = private_url_expires
def _clean_name(self, name):
"""
Cleans the name so that Windows style paths work
"""
# Normalize Windows style paths
clean_name = posixpath.normpath(name).replace('\\', '/')
# os.path.normpath() can strip trailing slashes so we implement
# a workaround here.
if name.endswith('/') and not clean_name.endswith('/'):
# Add a trailing slash as it was stripped.
return clean_name + '/'
else:
return clean_name
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../foo.txt
work. We check to make sure that the path pointed to is not outside
the directory specified by the LOCATION setting.
"""
base_path = force_text(self.location)
base_path = base_path.rstrip('/')
final_path = urljoin(base_path.rstrip('/') + "/", name)
base_path_len = len(base_path)
if (not final_path.startswith(base_path) or
final_path[base_path_len:base_path_len + 1] not in ('', '/')):
raise SuspiciousOperation("Attempted access to '%s' denied." %
name)
return final_path.lstrip('/')
def _open(self, name, mode='rb'):
return QiniuFile(name, self, mode)
def _save(self, name, content):
cleaned_name = self._clean_name(name)
name = self._normalize_name(cleaned_name)
if hasattr(content, 'open'):
# Since Django 1.6, content should be a instance
# of `django.core.files.File`
content.open()
if hasattr(content, 'chunks'):
content_str = b''.join(chunk for chunk in content.chunks())
else:
content_str = content.read()
self._put_file(name, content_str)
content.close()
return cleaned_name
def _put_file(self, name, content):
token = self.auth.upload_token(self.bucket_name)
ret, info = put_data(token, name, content)
if ret is None or ret['key'] != name:
raise QiniuError(info)
def _read(self, name):
return requests.get(self.url(name)).content
def delete(self, name):
name = self._normalize_name(self._clean_name(name))
if six.PY2:
name = name.encode('utf-8')
ret, info = self.bucket_manager.delete(self.bucket_name, name)
if ret is None or info.status_code == 612:
raise QiniuError(info)
def _file_stat(self, name, silent=False):
name = self._normalize_name(self._clean_name(name))
if six.PY2:
name = name.encode('utf-8')
ret, info = self.bucket_manager.stat(self.bucket_name, name)
if ret is None and not silent:
raise QiniuError(info)
return ret
def exists(self, name):
stats = self._file_stat(name, silent=True)
return True if stats else False
def size(self, name):
stats = self._file_stat(name)
return stats['fsize']
def modified_time(self, name):
stats = self._file_stat(name)
time_stamp = float(stats['putTime']) / 10000000
return datetime.datetime.fromtimestamp(time_stamp)
def listdir(self, name):
name = self._normalize_name(self._clean_name(name))
if name and not name.endswith('/'):
name += '/'
dirlist = bucket_lister(self.bucket_manager, self.bucket_name,
prefix=name)
files = []
dirs = set()
base_parts = name.split("/")[:-1]
for item in dirlist:
parts = item['key'].split("/")
parts = parts[len(base_parts):]
if len(parts) == 1:
# File
files.append(parts[0])
elif len(parts) > 1:
# Directory
dirs.add(parts[0])
return list(dirs), files
def url(self, name):
name = self._normalize_name(self._clean_name(name))
name = filepath_to_uri(name)
protocol = u'https://' if self.secure_url else u'http://'
url = urljoin(protocol + self.bucket_domain, name)
if self.private_url:
return self.auth.private_download_url(url, self.private_url_expires)
else:
return url
class QiniuStorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
accesskeyid = kwargs['accesskeyid']
accesskeysecret = kwargs['accesskeysecret']
self._bucket = kwargs['bucket']
self._auth = Auth(accesskeyid, accesskeysecret)
self._domain = kwargs['domain_url']
self._qiniu_api = BucketManager(self._auth)
def download(self, cos_path, local_path):
if isinstance(local_path, str):
local_path = local_path.decode('utf-8')
if cos_path.startswith('/'):
cos_path = cos_path[1:]
if isinstance(cos_path, unicode):
cos_path = cos_path.encode('utf-8')
from urllib import quote
cos_path = quote(cos_path)
base_url = 'http://%s/%s' % (self._domain, cos_path)
# print base_url
private_url = self._auth.private_download_url(base_url, expires=3600)
# print private_url
logger.debug("private url: " + private_url)
ret = requests.get(private_url)
if ret.status_code != 200:
raise SystemError("download file from qiniu failed")
# print local_path.encode('utf-8')
with open(local_path.encode('utf-8'), 'wb') as fd:
for chunk in ret.iter_content(1024):
fd.write(chunk)
def upload(self, cos_path, local_path):
raise NotImplementedError
def list(self):
prefix = None
limit = 100
delimiter = None
marker = None
eof = False
while not eof:
try:
ret, eof, info = self._qiniu_api.list(self._bucket, prefix, marker, limit, delimiter)
if ret is None:
logger.warn("ret is None")
if info.error == 'bad token':
raise TokenException
else:
logger.warn(info.text_body)
raise IOError(info.error)
for i in ret['items']:
logger.info("yield new object: {}".format(i['key']))
yield i['key']
if eof is True:
logger.info("eof is {}".format(eof))
continue
if not eof and 'marker' in ret:
marker = ret['marker']
else:
eof = True
except TokenException as e:
eof = True
logger.warn("Your accessid/accesskey is incorrect, Please double check your configures")
except Exception as e:
logger.exception("list exception: " + str(e))
def exists(self, _path):
raise NotImplementedError
class QiniuStorage(object):
storage_type = 'qiniu'
def __init__(self, user_id, dbQiniuStorage, use_system=False):
if use_system is True:
self.access_key = 'lnjEMrmedpqOhO2IAw4WtyGNbNVQrFPGEVLQuyFp'
self.secret_key = 'Nw_4Cn0L0Z3LLOL-t-OvGOVVV2kSxY3_my85UJwG'
self.bucket_name = 'czlib-system'
self.download_url = '7xjrbh.com1.z0.glb.clouddn.com'
self.q = Auth(self.access_key, self.secret_key)
else:
self.access_key = dbQiniuStorage.access_key.encode('utf-8') # unicode to str
self.secret_key = dbQiniuStorage.secret_key.encode('utf-8')
self.bucket_name = dbQiniuStorage.bucket_name.encode('utf-8')
self.download_url = dbQiniuStorage.download_url.encode('utf-8')
self.q = Auth(self.access_key, self.secret_key)
# print('QiniuStorage - __init__: user_id: %s' % user_id)
# print('QiniuStorage - __init__: ak:%s, sk:%s, b_name:%s, d_url:%s' % (self.access_key, self.secret_key, self.bucket_name, self.download_url))
def upload_stream(self, key, input_stream, data_size):
# print('upload_stream: key=%s, data_size=%s' % (key, data_size))
# key = ''
# input_stream=self.fileobj.stream
# print('upload_stream: 1')
token = self.q.upload_token(self.bucket_name, key)
# print('upload_stream: 2')
ret, info = put_stream(token, key, input_stream, data_size)
# print(info)
assert ret['key'] == key
return self.get_download_url(key)
# 直接上传二进制流
def upload_binary_stream(self, key, data):
# key = ''
# data = 'hello bubby!'
token = self.q.upload_token(self.bucket_name, key)
ret, info = put_data(token, key, data, mime_type="application/octet-stream", check_crc=True)
# print(info)
assert ret['key'] == key
return self.get_download_url(key)
# 上传本地文件
def upload_file(self, key, localfile):
# key = 'home/carlo/test_file.py'
mime_type = "text/plain"
params = {'x:a': 'a'}
token = self.q.upload_token(self.bucket_name, key)
ret, info = put_file(token, key, localfile, mime_type=mime_type, check_crc=True)
# print(info)
assert ret['key'] == key
assert ret['hash'] == etag(localfile)
#断点续上传、分块并行上传
def upload_big_file(self, key, localfile):
mime_type = "text/plain"
params = {'x:a': 'a'}
# key = 'big'
token = self.q.upload_token(self.bucket_name, key)
progress_handler = lambda progress, total: progress
ret, info = put_file(token, key, localfile, params, mime_type, progress_handler=progress_handler)
# print(info)
assert ret['key'] == key
def single_delete(self, key):
bucket = BucketManager(self.q)
ret, info = bucket.delete(self.bucket_name, key)
# print(info)
assert info.status_code == 200 or info.status_code == 612
# 批量删除文件
def multi_delete(self, key_list):
# key_list: [key1, key2, key3, ...]
from qiniu import build_batch_delete
bucket = BucketManager(self.q)
ops = build_batch_delete(self.bucket_name, key_list)
ret, info = bucket.batch(ops)
# print('QiniuStorage - multi_delete: %s' % info)
json_info = json.loads(info.text_body)
for m_info in json_info:
# "code":612,"data":{"error":"no such file or directory"
assert m_info[u'code'] == 200 or m_info[u'code'] == 612
def get_private_download_url(self, key):
# bucket is private, http://<domain>/<key>?e=<deadline>&token=<dntoken>
base_url = 'http://%s/%s' % (self.bucket_name + '.qiniudn.com', key)
download_url = self.q.private_download_url(base_url, expires=3600)
assert isinstance(download_url, object)
return download_url
def get_download_url(self, key):
# bucket is public, http://<domain>/<key>
download_url = 'http://%s/%s' % (self.download_url, key)
return download_url
class QiniuStorage(object):
def __init__(self): #instanciate a Auth object with AK,SK
self.q = Auth(settings.QINIU_ACCESS_KEY, settings.QINIU_SECRET_KEY)
self.bucket = settings.QINIU_PRIVATE_BUCKET_NAME
self.domain = settings.QINIU_PRIVATE_DOMAIN
self.key = None
def upload_token(
self,
ie=False
): #got a token coding with policy and SK,when updating with policy ,it must get a corresponding token
key = str(uuid.uuid4()).replace('-', '')
key = 'ie_' + key if ie else key
return key, self.q.upload_token(self.bucket, key=key)
def upload_data(self,
data,
mime_type='application/octet-stream',
ie=True): #using it to upload a data
key, token = self.upload_token(ie)
ret, info = put_data(token, key, data, mime_type=mime_type)
self.key = key
if ret['key'] != key: # key is the key of the updated data
raise Code.file_update_fail
return key
def view_url(self, key, style=None):
if style:
base_url = '{}{}!{}'.format(self.domain, key, style)
else:
base_url = '{}{}'.format(self.domain, key)
# return self.q.private_download_url(base_url)
return base_url
def down_url(self, key, file_name=None):
file_name = "?attname=" + urllib.quote(
file_name.encode('utf8')) if file_name else ''
url = '%s%s' % ("?{}".format(key), "?{}".format(file_name)
) if file_name else "/{}".format(key)
base_url = 'http://' + self.domain + url
return self.q.private_download_url(base_url)
def download(self, key):
url = self.down_url(key)
r = requests.get(url)
if r.status_code != 200:
raise Code.file_update_fail
return r.content
def verify_callback(self, request):
auth = request.META.get('HTTP_AUTHORIZATION', None)
return self.q.verify_callback(auth, settings.QINIU_CALLBACK,
request.body)
def persistent_with_pfop(
self, perstr, fops
): #persistenet operations with fop sentence.the result save as 'origin key+perstr'
auth = self.q
bucket_name = self.bucket
saveas_key = self.key + perstr
entryuri = settings.QINIU_PRIVATE_BUCKET_NAME + ':' + saveas_key
entryuri = urlsafe_base64_encode(entryuri)
fops = fops + '|saveas/{}'.format(entryuri)
pfop = PersistentFop(auth,
bucket_name,
pipeline=settings.QINIU_PIPELINE)
ret, info = pfop.execute(self.key, [fops], 1)
return [ret, info]
def persistent_with_pfop_nfop(
self, perstr,
fops): #persistent fop operation with nums of fop execute sentence
auth = self.q
bucket_name = self.bucket
foplist = []
for i in range(len(fops)):
saveas_key = self.key + perstr[i]
entryuri = settings.QINIU_PRIVATE_BUCKET_NAME + ':' + saveas_key
entryuri = urlsafe_base64_encode(entryuri)
fop = fops[i] + '|saveas/{}'.format(entryuri)
foplist.append(fop)
pfop = PersistentFop(auth,
bucket_name,
pipeline=settings.QINIU_PIPELINE)
ret, info = pfop.execute(self.key, foplist, 1)
return [ret, info]
def get_key(self, key):
self.key = key
class qiniuHelper:
def __init__(self,access_key="HNkFJ3P5_5UpP5BOaXDFWTjZ6ZK9NIAEhafchRMh",secret_key="uwxfBCn-4Rc4M6B1awZx-084Kt-bMXYJbufZNKLk"):
self.access_key = access_key
self.secret_key = secret_key
def __GetBucket(self):
self.q = Auth(self.access_key, self.secret_key)
bucket = BucketManager(self.q)
if not bucket:
raise(NameError,"连接七牛失败")
else:
return bucket
"""
获取七牛文件列表
"""
def getListFile(self, bucket_name="bimuptest", prefix=None):
# 列举条目
limit = 100
# 列举出除'/'的所有文件以及以'/'为分隔的所有前缀
delimiter = None
# 标记
marker = None
bucket = self.__GetBucket()
listfile = []
while True:
ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter)
for item in ret["items"]:
listfile.append(item)
if "marker" in ret.keys():
marker = ret["marker"]
else:
return listfile
"""
修改文件存储类型 1表示低频存储，0是标准存储
"""
def changeSaveType(self,bucket_name="bimuptest",key="",saveType=1):
bucket = self.__GetBucket()
ret, info = bucket.change_type(bucket_name, key, saveType) # 1表示低频存储，0是标准存储
print(info)
return info
"""
获取七牛下载文件地址
"""
def getDownUrl(self,url, expires=3600):
self.q = Auth(self.access_key, self.secret_key)
downUrl=self.q.private_download_url(url, expires)
print(downUrl)
return downUrl
"""
上传文件
"""
def upLoadFile(self,bucket_name="bimuptest",keyName="ExamCode/SafeCode/QuestionID.anst",filePath=""):
self.q = Auth(self.access_key, self.secret_key)
# 生成上传 Token，可以指定过期时间等
token = self.q.upload_token(bucket_name, keyName, 3600)
ret, info = put_file(token, keyName, filePath)
print(info)
assert ret['key'] == keyName
assert ret['hash'] == etag(filePath)
class Logs():
def __init__(self):
#初始化Auth状态
self.au = Auth(access_key, secrek_ley)
self.bucket = BucketManager(self.au)
def time_format(self, date):
'''
校验日期格式
:param date:
:return:
'''
# domain_YYYY-MM-DD.gz
# prefix=domain_YYYY-MM-DD
# 正则校验日期格式
reg = re.compile('[0-9]{4}-(0|1)[0-9]{1}-[0-3]{1}[0-9]{1}')
if reg.match(date):
return str(date)
else:
return False
def log_list(self, prefix):
'''
根据前缀查询日志列表
:param prefix: 查询前缀
:return:返回查询结果
'''
res = self.bucket.list(bucket, prefix=prefix)
return res
def get_log_url(self, domain, date):
'''
生成日志下载链接
:param domain: 需要下载日志的域名
:param date: 时间：YYYY-MM-DD
:return:返回日志下载链接，有效期3600
'''
#生成查询前缀
if domain is None:
raise ValueError("Please Import Doamin!!!")
prefix = ""
if self.time_format(str(date)):
prefix = domain + "_" + date
else:
raise ValueError("Date Format：YYYY-MM-DD")
#生成日志下载url
urls = []
try:
res = self.log_list(prefix)
items = res[0]["items"]
for item in items:
key = item["key"]
base_url = 'http://%s/%s' % (bucket_domain, key)
private_url = self.au.private_download_url(base_url, 3600)
urls.append(private_url)
return urls
except:
print "ERROR:" + res
def private_url(base_url: str, duration_sec: int = URL_DURATION):
q = Auth(Facade.config["qiniu"]["access_key"],
Facade.config["qiniu"]["secret_key"])
return q.private_download_url(base_url, expires=duration_sec)
class qiniuBackup:
__qiniu = None
__bucket_name = None
__bucket_domain = None
__error_msg = "ERROR: 无法连接到七牛云服务器，请检查[AK/SK/存储空间]设置是否正确!"
def __init__(self):
#获取七牛秘钥
fp = open('data/qiniuAs.conf', 'r')
if not fp:
print 'ERROR: 请检查qiniuAs.conf文件中是否有七牛Key相关信息!'
exit()
keys = fp.read().split('|')
if len(keys) < 4:
print 'ERROR: 请检查qiniuAs.conf文件中的七牛Key信息是否完整!'
exit()
self.__bucket_name = keys[2]
self.__bucket_domain = keys[3]
#构建鉴权对象
self.__qiniu = Auth(keys[0], keys[1])
#上传文件
def upload_file(self, filename):
try:
from qiniu import put_file, etag, urlsafe_base64_encode
#上传到七牛后保存的文件名
key = filename.split('/')[-1]
#生成上传 Token，可以指定过期时间等
token = self.__qiniu.upload_token(self.__bucket_name, key,
3600 * 2)
result = put_file(token, key, filename)
return result[0]
except:
print self.__error_msg
return None
#取回文件信息
def get_files(self, filename):
try:
from qiniu import BucketManager
bucket = BucketManager(self.__qiniu)
result = bucket.stat(self.__bucket_name, filename)
return result[0]
except:
print self.__error_msg
return None
#取回文件列表
def get_list(self):
try:
from qiniu import BucketManager
bucket = BucketManager(self.__qiniu)
result = bucket.list(self.__bucket_name)
if not len(result[0]['items']):
return [{
"mimeType": "application/test",
"fsize": 0,
"hash": "",
"key": "没有文件",
"putTime": 14845314157209192
}]
return result[0]['items']
except:
print self.__error_msg
return None
#下载文件
def download_file(self, filename):
try:
base_url = 'http://%s/%s' % (self.__bucket_domain, filename)
private_url = self.__qiniu.private_download_url(base_url,
expires=3600)
return private_url
except:
print self.__error_msg
return None
#删除文件
def delete_file(self, filename):
try:
from qiniu import BucketManager
bucket = BucketManager(self.__qiniu)
result = bucket.delete(self.__bucket_name, filename)
return result[0]
except:
print self.__error_msg
return None
#备份网站
def backupSite(self, name, count):
sql = db.Sql()
path = sql.table('sites').where('name=?', (name, )).getField('path')
startTime = time.time()
if not path:
endDate = time.strftime('%Y/%m/%d %X', time.localtime())
log = "网站[" + name + "]不存在!"
print "★[" + endDate + "] " + log
print "----------------------------------------------------------------------------"
return
backup_path = sql.table('config').where(
"id=?", (1, )).getField('backup_path') + '/site'
if not os.path.exists(backup_path):
public.ExecShell("mkdir -p " + backup_path)
filename = backup_path + "/Web_" + name + "_" + time.strftime(
'%Y%m%d_%H%M%S', time.localtime()) + '.tar.gz'
public.ExecShell("cd " + os.path.dirname(path) + " && tar zcvf '" +
filename + "' '" + os.path.basename(path) +
"' > /dev/null")
endDate = time.strftime('%Y/%m/%d %X', time.localtime())
if not os.path.exists(filename):
log = "网站[" + name + "]备份失败!"
print "★[" + endDate + "] " + log
print "----------------------------------------------------------------------------"
return
#上传到七牛
self.upload_file(filename)
outTime = time.time() - startTime
pid = sql.table('sites').where('name=?', (name, )).getField('id')
sql.table('backup').add('type,name,pid,filename,addtime,size',
('0', os.path.basename(filename), pid, 'qiniu',
endDate, os.path.getsize(filename)))
log = "网站[" + name + "]已成功备份到七牛云,用时[" + str(round(outTime, 2)) + "]秒"
public.WriteLog('计划任务', log)
print "★[" + endDate + "] " + log
print "|---保留最新的[" + count + "]份备份"
print "|---文件名:" + os.path.basename(filename)
#清理本地文件
public.ExecShell("rm -f " + filename)
#清理多余备份
backups = sql.table('backup').where(
'type=? and pid=?', ('0', pid)).field('id,name,filename').select()
num = len(backups) - int(count)
if num > 0:
for backup in backups:
if os.path.exists(backup['filename']):
public.ExecShell("rm -f " + backup['filename'])
self.delete_file(backup['name'])
sql.table('backup').where('id=?', (backup['id'], )).delete()
num -= 1
print "|---已清理过期备份文件：" + backup['name']
if num < 1: break
return None
#备份数据库
def backupDatabase(self, name, count):
sql = db.Sql()
path = sql.table('databases').where('name=?',
(name, )).getField('path')
startTime = time.time()
if not path:
endDate = time.strftime('%Y/%m/%d %X', time.localtime())
log = "数据库[" + name + "]不存在!"
print "★[" + endDate + "] " + log
print "----------------------------------------------------------------------------"
return
backup_path = sql.table('config').where(
"id=?", (1, )).getField('backup_path') + '/database'
if not os.path.exists(backup_path):
public.ExecShell("mkdir -p " + backup_path)
filename = backup_path + "/Db_" + name + "_" + time.strftime(
'%Y%m%d_%H%M%S', time.localtime()) + ".sql.gz"
import re
mysql_root = sql.table('config').where("id=?",
(1, )).getField('mysql_root')
mycnf = public.readFile('/etc/my.cnf')
rep = "\[mysqldump\]\nuser=root"
sea = '[mysqldump]\n'
subStr = sea + "user=root\npassword="******"\n"
mycnf = mycnf.replace(sea, subStr)
if len(mycnf) > 100:
public.writeFile('/etc/my.cnf', mycnf)
public.ExecShell(
"/www/server/mysql/bin/mysqldump --opt --default-character-set=utf8 "
+ name + " | gzip > " + filename)
if not os.path.exists(filename):
endDate = time.strftime('%Y/%m/%d %X', time.localtime())
log = "数据库[" + name + "]备份失败!"
print "★[" + endDate + "] " + log
print "----------------------------------------------------------------------------"
return
mycnf = public.readFile('/etc/my.cnf')
mycnf = mycnf.replace(subStr, sea)
if len(mycnf) > 100:
public.writeFile('/etc/my.cnf', mycnf)
#上传到七牛
self.upload_file(filename)
endDate = time.strftime('%Y/%m/%d %X', time.localtime())
outTime = time.time() - startTime
pid = sql.table('databases').where('name=?', (name, )).getField('id')
sql.table('backup').add('type,name,pid,filename,addtime,size',
(1, os.path.basename(filename), pid, 'qiniu',
endDate, os.path.getsize(filename)))
log = "数据库[" + name + "]已成功备份到七牛云,用时[" + str(round(outTime, 2)) + "]秒"
public.WriteLog('计划任务', log)
print "★[" + endDate + "] " + log
print "|---保留最新的[" + count + "]份备份"
print "|---文件名:" + os.path.basename(filename)
#清理本地文件
public.ExecShell("rm -f " + filename)
#清理多余备份
backups = sql.table('backup').where(
'type=? and pid=?', ('1', pid)).field('id,name,filename').select()
num = len(backups) - int(count)
if num > 0:
for backup in backups:
if os.path.exists(backup['filename']):
public.ExecShell("rm -f " + backup['filename'])
self.delete_file(backup['name'])
sql.table('backup').where('id=?', (backup['id'], )).delete()
num -= 1
print "|---已清理过期备份文件：" + backup['name']
if num < 1: break
return None
token = q.upload_token('wm-test', None, 7200, {'callbackUrl':"http://298074.cicp.net:59295/upload/callback/", 'callbackBody':"name=$(key)&fname=$(fname)&hash=$(etag)&size=$(imageInfo.width)x$(imageInfo.height)", \
'saveKey':'aaa$(year)$(mon)$(day)$(hour)$(min)$(sec).jpg',
'mimeType':'image/jpg'})
return token
def upload():
pass
# def get_dnurl(key):
# base_url = qiniu.rs.make_base_url('wm-test.u.qiniudn.com', key)
if __name__ == '__main__':
# u_t = upload_token()
# print u_t
# key = '1.jpg'
# ret, info = put_file(u_t, None, '/home/liuxue/Downloads/mahua-logo.jpg', check_crc=True)
# print(info)
# print ret
import requests
q = Auth(ACCESS_KEY, SECRET_KEY)
bucket = 'wm-test'
key = '1.jpg'
base_url = 'http://%s/%s?imageView2/1/w/200/h/200/interlace/1' % (bucket + '.u.qiniudn.com', key)
private_url = q.private_download_url(base_url, expires=3600)
print(private_url)
r = requests.get(private_url)
print r.status_code
class FileService(BaseService):
table = 'filehub'
fields = 'id, filename, size, qiniu_id, owner, mime, hash, type, pid, lord, form, dir'
def __init__(self, ak, sk):
super().__init__()
self.qiniu = Auth(access_key=ak, secret_key=sk)
@run_on_executor
def upload_token(self, key):
saveas = '{bucket}:{key}'.format(bucket=settings['qiniu_file_bucket'],
key=key)
saveas_key = urlsafe_base64_encode(saveas)
policy = QINIU_POLICY.copy()
policy['persistentOps'] = QINIU_THUMB + '|saveas/' + saveas_key
token = self.qiniu.upload_token(
bucket=settings['qiniu_file_bucket'],
expires=settings['qiniu_token_timeout'],
policy=policy)
return token
@run_on_executor
def private_download_url(self, qiniu_id):
url = settings['qiniu_file_bucket_url'] + '/' + qiniu_id
expires = settings['qiniu_token_timeout']
download_url = self.qiniu.private_download_url(url=url,
expires=expires)
return download_url
@coroutine
def check_file_exist(self, hash):
sql = """
SELECT filename, size, qiniu_id, mime
FROM {table}
WHERE hash=%s and filename <> ''
ORDER BY update_time LIMIT 1
""".format(table=self.table)
cur = yield self.db.execute(sql, [hash])
return cur.fetchone()
@coroutine
def batch_upload(self, params):
arg = {
'filename': params['filename'],
'size': 0,
'qiniu_id': '',
'owner': params['owner'],
'mime': '',
'hash': params['hash'],
'type': 0,
'pid': params['pid'],
'lord': params['lord'],
'form': params['form']
}
resp = {'file_status': 0, 'token': '', 'file_id': ''}
data = yield self.check_file_exist(params['hash'])
if data and arg['filename'] == data['filename']:
resp['file_status'] = 1
arg['size'] = data['size']
arg['qiniu_id'] = data['qiniu_id']
arg['mime'] = data['mime']
else:
resp['token'] = yield self.upload_token(params['hash'])
add_result = yield self.add(arg)
# 获取父节点的绝对路径,用来生成新增文件的完整路径
pdata = yield self.select(conds={'id': params['pid']}, one=True)
pdir = (pdata.get('dir') if pdata else '/0') + '/' + str(
add_result['id'])
yield self.update(sets={'dir': pdir}, conds={'id': add_result['id']})
resp['file_id'] = add_result['id']
return resp
@coroutine
def seg_page(self, params):
sql = """
SELECT f.id, f.filename, f.size, f.qiniu_id, u.name, f.mime, f.hash, f.type, f.pid, f.dir,
CONCAT('{uri}', f.qiniu_id) as url, CONCAT('{uri}', f.hash) as thumb,
DATE_FORMAT(f.create_time, %s) as create_time, DATE_FORMAT(f.update_time, %s) as update_time
FROM {filehub} as f, {user} as u
WHERE f.pid = %s AND f.form = %s AND f.lord = %s AND f.owner = u.id
ORDER BY f.create_time DESC
""".format(filehub=self.table,
user='******',
uri=DISK_DOWNLOAD_URL)
arg = [
FULL_DATE_FORMAT, FULL_DATE_FORMAT, params['file_id'],
params['form'], params['lord']
]
cur = yield self.db.execute(sql, arg)
data = cur.fetchall()
return data
@coroutine
def total_pages(self, params):
sql = "SELECT count(*) as number FROM {table} WHERE pid = %s AND form = %s AND lord=%s".format(
table=self.table)
cur = yield self.db.execute(
sql, [params['pid'], params['form'], params['lord']])
data = cur.fetchone()
return data['number']
class __ImageTouken:
def __init__(self):
self.__acckey = ""
self.__secretkey = ""
self.__domain = ""
self.__bucket = ""
self.__callbackUrl = ""
self.__expires = 3600
self.__auth = None
def loadConfig(self):
filePath = os.path.split(__file__)[0]
configPath = os.path.join(filePath, "../../staticdata/utils/imagetoken.xml")
xmlDoc = xmlParser.parse(configPath)
xmlRoot = xmlDoc.getroot()
for child in xmlRoot:
if child.tag == "acckey":
self.__acckey = child.text
elif child.tag == "secretkey":
self.__secretkey = child.text
elif child.tag == "domain":
self.__domain = child.text
elif child.tag == "bucket":
self.__bucket = child.text
elif child.tag == "callbackUrl":
self.__callbackUrl = child.text
elif child.tag == "expires":
self.__expires = int(child.text)
self.__createAuth()
def __createAuth(self):
self.__auth = Auth(self.__acckey, self.__secretkey)
def getExpires(self):
return self.__expires
def uploadToken(self, key):
policy = None
if self.__callbackUrl:
policy = dict(callbackUrl=self.__callbackUrl,
callbackBody="key=$(key)&hash=$(etag)",
returnBody="key=$(key)&hash=$(etag)&name=$(fname)"
)
return self.__auth.upload_token(self.__bucket, key, self.__expires, policy)
def downloadToken(self, key, imageFormat=""):
bauseUrl = "http://{}/{}".format(self.__domain, key)
if imageFormat:
bauseUrl = "{}?{}".format(bauseUrl, imageFormat)
privateUrl = self.__auth.private_download_url(bauseUrl, self.__expires)
return privateUrl
class QiNiuFile(File):
def __init__(self,
name,
bucket_name,
domain,
access_key,
secret_key,
mode='rb'):
self.name = name
self.mode = mode
self.bucket_name = bucket_name
self.domain = domain
self.access_key = access_key
self.secret_key = secret_key
self.auth = Auth(self.access_key, self.secret_key)
self.file = None
if 'w' in mode or 'a' in mode:
raise ValueError('qi niu file can only read.')
if 'r' in mode and ('w' in mode or 'a' in mode):
raise ValueError(
'qi niu file cannot read and write in the same time.')
def _get_file(self):
if self._file is None:
data = self._read()
self._file = io.BytesIO(data)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def _read(self):
url = urljoin(self.domain, self.name)
private_url = self.auth.private_download_url(url)
r = requests.get(private_url)
if r.status_code == 200:
return r.content
return b''
def __len__(self):
bucket = BucketManager(self.auth)
ret, info = bucket.stat(self.bucket_name, self.name)
if ret is None:
raise QiNiuError(info)
return ret.get('fsize', 0)
@cached_property
def size(self):
bucket = BucketManager(self.auth)
ret, info = bucket.stat(self.bucket_name, self.name)
if ret is None:
raise QiNiuError(info)
return ret.get('fsize', 0)
def read(self, num_bytes=None):
return self.file.read(num_bytes)
def close(self):
self.file.close()
