def sync_loop_local_filesystem(root_path,
app_name,
check_md5=True,
extra_should_sync_func=None):
root_path = same_slash(root_path)
if not os.path.isdir(root_path): # 根目录不存在，不处理
return []
file_paths = []
for parent, folders, files in os.walk(root_path):
if is_a_hidden_path(parent):
continue
elif not is_real(parent): # link类型的不处理
continue
for fs in [files, folders]:
for filename in fs:
filepath = join(parent, filename)
# 看是否已经在本地数据库了
if not should_sync(
filepath,
root_path,
app_name,
check_md5,
extra_should_sync_func=extra_should_sync_func):
continue
file_paths.append(filepath)
return file_paths
def should_sync_file_for_sync_folder_simply(relative_path, extra_func=None):
# must return True or False, else will checked by`should_sync` func in detact.py
relative_path_without_dot = relative_path.lstrip('.')
ext = os.path.splitext(relative_path)[-1].lower().strip('.')
if ext in ['py', 'pyc']:
return False
if is_a_hidden_path(relative_path):
return False
if relative_path_without_dot in ['template', 'configs',]:
return False
if relative_path_without_dot.startswith('template/') or relative_path_without_dot.startswith('configs/'):
return False
if relative_path.startswith('_data/visits/'): # 原来 Bitcron 下记录每日访问的数据
return False
elif relative_path.startswith('_data/email_status/'): # 原 Bitcron 邮件发送的状态记录
return False
elif relative_path.startswith('_cache/'): # 原来 Bitcron site 的缓存数据
return False
if relative_path in ['_cache']:
return False
if '_sync_ignore_' in relative_path:
return False
# at last
if extra_func:
return extra_func(relative_path)
def get_template_info(template_dir):
info = {}
template_dir = template_dir.strip().rstrip('/')
if not os.path.isdir(template_dir):
return info # ignore
filepaths = get_all_sub_files(template_dir,
accept_func=os.path.isfile,
max_tried_times=1000)
for filepath in filepaths:
relative_path = get_relative_path(
filepath, root=template_dir).lower() # lower case
if not os.path.isfile(filepath):
continue
if not is_real(filepath) or is_a_hidden_path(relative_path):
continue
if relative_path.startswith('readme.') and is_a_markdown_file(
relative_path): # 模板 readme 上的信息
with open(filepath, 'rb') as f:
raw_markdown_content = smart_unicode(f.read())
compiled_markdown_content = compile_markdown(raw_markdown_content)
compiled_markdown_content_meta = compiled_markdown_content.metadata
readme_info = dict(content=compiled_markdown_content,
metadata=compiled_markdown_content_meta
) # raw_content=raw_markdown_content,
info['_readme'] = readme_info
else:
path_without_ext, ext = os.path.splitext(relative_path)
ext = ext.strip('.').lower()
if ext not in allowed_exts:
continue
with open(filepath, 'rb') as f:
raw_content = f.read()
raw_content = smart_unicode(raw_content) # to unicode
info[relative_path] = raw_content
matched_compiler = template_resource_compilers.get(ext)
if matched_compiler:
new_ext, compile_func = matched_compiler
try:
compiled_content = compile_func(raw_content)
new_key = path_without_ext + '.' + new_ext.strip('.')
info[new_key] = compiled_content
except Exception as e:
error_message = getattr(e, 'message', None)
if error_message:
try:
print('%s error: %s' %
(relative_path, error_message))
except:
pass
info['_route'] = get_templates_route_info(info)
return info
def upload_static_files_to_cdn(static_files_root,
cdn_static_prefix,
secret_id,
secret_key,
bucket,
region,
force_update=False):
sub_filepaths = get_all_sub_files(static_files_root)
for filepath in sub_filepaths:
if is_a_hidden_path(filepath):
continue
ext = os.path.splitext(filepath)[-1]
if ext in [".py", ".pyc", ".pyd"]:
continue
#print(filepath)
relative_path = get_relative_path(filepath, static_files_root)
cnd_path = "/%s/%s" % (cdn_static_prefix.strip("/"),
relative_path.strip("/"))
if not force_update:
if has_file_on_qcloud(cnd_path,
secret_id=secret_id,
secret_key=secret_key,
bucket=bucket,
region=region):
qcloud_file_meta = get_file_meta_on_qcloud(
cnd_path,
secret_id=secret_id,
secret_key=secret_key,
bucket=bucket,
region=region)
if qcloud_file_meta and isinstance(qcloud_file_meta, dict):
q_version = qcloud_file_meta.get("ETag",
"").strip("'").strip('"')
if q_version and get_md5_for_file(filepath) == q_version:
continue
with open(filepath, "rb") as f:
upload_file_obj_to_qcloud(file_obj=f,
url_path=cnd_path,
secret_id=secret_id,
secret_key=secret_key,
bucket=bucket,
region=region,
content_type=guess_type(filepath))
print(filepath)
def should_sync(filepath,
root,
app_name,
check_md5=True,
extra_should_sync_func=None):
if not os.path.exists(filepath):
return False
elif is_a_hidden_path(filepath):
return False
elif not is_real(filepath):
return False
if check_md5:
sync_data = get_sync_data(filepath, root, app_name)
if sync_data:
if sync_data.get('md5') == md5_for_file(
filepath): # has been synced
return False
# 比如 Bitcron, 不允许超过 100 mb 的文件上传
# elif os.path.getsize(filepath) > 100*1024*1024: # 100Mb+ is not supported
#return False
if extra_should_sync_func:
try:
result = extra_should_sync_func(filepath, root)
if isinstance(result, bool):
return result
except:
try:
relative_path = get_relative_path(filepath, root=root)
result = extra_should_sync_func(relative_path)
if isinstance(result, bool):
return result
except:
pass
return True