def __init__(self):
# assert hasattr(g, 'user_id'), '找不到用户信息'
# assert hasattr(g, 'tanent_id'), '找不到用户企业信息'
self.user_id = g.user_id if hasattr(g, 'user_id') else 0
self.tenant_id = g.tenant_id if hasattr(g, 'tenant_id') else 0
self.task_dao = TaskDAO()
def start_thread_save_db(self, msg_list):
taskdao = TaskDAO()
# 多线程将缓存保存到数据库
def start_thread(x):
batch = Utils.get_batch(x)
thread_key = taskdao.thread_key.format(batch)
if not self.aios_redis.get(thread_key):
threading.Thread(target=taskdao.save_to_db,
args=(g.user_id, g.tenant_id, batch),
daemon=True).start()
self.aios_print('启动定时调度线程', batch)
self.aios_redis.set(thread_key, 'Running',
taskdao.THREAD_KEY_TIMEOUT)
if self.aios_redis.ttl(thread_key) < 10:
threading.Thread(target=taskdao.save_to_db,
args=(g.user_id, g.tenant_id, batch),
daemon=True).start()
self.aios_print('线程已存在,失效时间快到了,重新启动定时调度线程', batch)
self.aios_redis.set(thread_key, 'Running',
taskdao.THREAD_KEY_TIMEOUT)
else:
self.aios_print('线程已存在,不需要开启', batch)
# 启动多个线程
_.chain(msg_list). \
map_(lambda x: Utils.get_host(x.get('dir_path'))). \
uniq(). \
for_each(start_thread). \
value()
class TaskBLL():
def __init__(self):
# assert hasattr(g, 'user_id'), '找不到用户信息'
# assert hasattr(g, 'tanent_id'), '找不到用户企业信息'
self.user_id = g.user_id if hasattr(g, 'user_id') else 0
self.tenant_id = g.tenant_id if hasattr(g, 'tenant_id') else 0
self.task_dao = TaskDAO()
def get_task_from_cache(self, task_id):
return self.task_dao.get_task_from_cache(self.tenant_id, task_id)
def single_file_handler(self, single_chunk_files):
'''单分片文件处理方法
'''
succ_list = []
err_list = []
wait_add_tasks = []
wait_update_tasks = []
taskdao = TaskDAO()
tenant_id = g.tenant_id if hasattr(g, 'tenant_id') else 0
batches = _.uniq(
[Utils.get_batch(i.get('dir_path')) for i in single_chunk_files])
task_ids = [i.get('file_key') for i in single_chunk_files]
exist_tasks = taskdao.get_tasks(tenant_id, batches, task_ids)
exist_tasks_map = {}
for exist_task in exist_tasks:
exist_tasks_map[exist_task.task_id] = exist_task.id
for single_chunk_file in single_chunk_files:
try:
file_key = single_chunk_file.get('file_key')
dir_path = single_chunk_file.get('dir_path')
file_name = single_chunk_file.get('file_name')
tenant_id = single_chunk_file.get('tenant_id')
user_id = single_chunk_file.get('user_id')
merged_file = os.path.join(dir_path, file_name)
final_merged_file = os.path.join(dir_path, f'{file_key}.1')
shutil.move(final_merged_file, merged_file)
task_json = {
'created_by': user_id,
'tenant_id': tenant_id,
'task_id': file_key,
'chunks': '1',
'status': TASK_STATUS_MERGED,
'size': os.path.getsize(merged_file),
'link': {
'host': Utils.get_host(dir_path)
},
'batch': Utils.get_batch(dir_path)
}
if exist_tasks_map.get(file_key):
task_json['id'] = exist_tasks_map.get(file_key)
wait_update_tasks.append(task_json)
else:
wait_add_tasks.append(task_json)
succ_list.append({'file_key': file_key, 'curr_chunk': 1})
except Exception as err:
err_list.append({'file_key': file_key, 'curr_chunk': 1})
if len(wait_add_tasks):
taskdao.bulk_add(wait_add_tasks)
if len(wait_update_tasks):
taskdao.bulk_update(wait_update_tasks)
return succ_list, err_list
def notify_thread_stop(self, msg_list):
taskdao = TaskDAO()
def stop_thread(x):
batch = Utils.get_batch(x)
thread_key = taskdao.thread_key.format(batch)
self.aios_redis.set(thread_key, 'Stop')
_.chain(msg_list). \
map_(lambda x: Utils.get_host(x.get('dir_path'))). \
uniq(). \
for_each(stop_thread). \
value()
def get_task(self, tenant_id, task_id):
return TaskDAO().get_task(tenant_id, task_id)
def multi_file_handler(self, msg):
'''子进程处理单个文件的写入
'''
try:
# 上传进程内部加载sqlalchemy时需要控制每个进程的数据库连接池大小
os.environ['SQLALCHEMY_POOL_SIZE'] = '1'
from run import app as inner_app
from app import aios_redis
with inner_app.app_context():
self.aios_redis = aios_redis
file_key = msg.get('file_key')
dir_path = msg.get('dir_path')
file_name = msg.get('file_name')
curr_chunk = msg.get('curr_chunk')
total_chunks = msg.get('total_chunks')
tenant_id = msg.get('tenant_id')
user_id = msg.get('user_id')
cache_expired_time = msg.get('cache_expired_time')
wait_lock = f'plus_uploader:lock:{file_key}'
while not self.aios_redis.setnx(wait_lock,
f'lock.{curr_chunk}'):
# self.aios_print(file_key, curr_chunk, 'task waiting...')
time.sleep(0.001)
else:
self.lock = f'{wait_lock} by {curr_chunk}'
self.aios_print(file_key, curr_chunk,
f'current lock: lock.{curr_chunk}')
# 锁独占状态
# wait_lock超时时间 = 已存在但未合并的分片个数 * 单个分片预估的失效时间
parts = len(
_.filter_(
os.listdir(dir_path),
lambda x: '-' not in x and '.deleted' not in x))
count = max(parts, 1)
self.aios_print(f'文件存放目录{dir_path}, 实时已存在的分片数: {parts}')
self.aios_redis.expire(wait_lock,
cache_expired_time * count)
# self.aios_print(f'修正的lock超时时间, {cache_expired_time * count}')
# 更新任务状态
args = {
'task_id': file_key,
'tenant_id': tenant_id,
'created_by': user_id,
'updated_by': user_id,
'batch': Utils.get_batch(dir_path),
'status': TASK_STATUS_NONE,
'chunks': '',
'size': 0,
'link': {
'host': Utils.get_host(dir_path)
}
}
taskdao = TaskDAO()
# task_json = taskdao.get_task_from_cache(tenant_id, file_key)
task_json = taskdao.get_task(tenant_id,
file_key,
json=True)
if task_json is None:
task_json = taskdao.add(args)
if task_json['status'] != TASK_STATUS_MERGED:
task_json['status'] = TASK_STATUS_MERGING
# taskdao.update(task_json)
TaskModel.bulk_update_mappings([task_json])
# 合并分片
# 合并后的完整文件路径
merged_file = os.path.join(dir_path, file_name)
merge_process = self.partation_merge(
dir_path, file_key, total_chunks)
merge_process = _.map_(
merge_process,
lambda x: x.replace(f'{file_key}.', ''))
except_complete_name = f'1-{total_chunks}' if total_chunks > 1 else '1'
# 检测是否完整 ['1-701'] except_complete_name 1-701
self.aios_print('检测是否完整', merge_process,
'except_complete_name',
except_complete_name)
if except_complete_name in merge_process:
for f in os.listdir(dir_path):
self.aios_print('比较文件名', f,
except_complete_name)
if f.startswith(f'{file_key}.') and (
not f.endswith(except_complete_name)):
Utils.try_remove(os.path.join(dir_path, f))
self.aios_print('删除残留异常文件', f)
# 修改合并后的文件名称
final_merged_file = os.path.join(
dir_path, f'{file_key}.{except_complete_name}')
shutil.move(final_merged_file, merged_file)
self.aios_print('文件改名', final_merged_file, '>>',
merged_file)
# 记录任务状态
task_json['chunks'] = ','.join(
[str(i + 1) for i in range(total_chunks)])
task_json['status'] = TASK_STATUS_MERGED
task_json['size'] = os.path.getsize(merged_file)
task_json['link'] = {
'host': Utils.get_host(dir_path)
}
# 合并完成
self.aios_print(f'{curr_chunk}.合并完成')
else:
# ['1', '3-5', '10']
covert_process = []
for section in merge_process:
if '-' in section:
[left, right] = section.split('-')
covert_process.extend([
str(i)
for i in range(int(left),
int(right) + 1)
])
else:
covert_process.append(section)
task_json['chunks'] = ','.join(covert_process)
# 不完整
self.aios_print(f'{curr_chunk}.不完整,继续等待')
# 保存到缓存
# taskdao.update(task_json)
self.aios_print(f'更新状态, {task_json}')
TaskModel.bulk_update_mappings([task_json])
else:
# 已经合并完成后,不需要再次合并,直接退出
self.aios_print(f'{curr_chunk}.已经合并完成后, 不需要再次合并')
# 释放锁
self.aios_redis.delete(wait_lock)
self.aios_print(f'结束.{curr_chunk}/{total_chunks}')
# 清理被标记的可删除分片
for file in os.listdir(dir_path):
if file.endswith('.deleted'):
self.aios_print('清理被标记的可删除分片', file)
Utils.try_remove(os.path.join(dir_path, file))
return {'file_key': file_key, 'curr_chunk': curr_chunk}, None
except Exception as err:
import traceback
traceback.print_exc()
print('multi_file_handler', err)
return None, {
'file_key': msg['file_key'],
'curr_chunk': msg['curr_chunk']
}