def gclone_execute(source, target):
#./gclone --config ./gclone.conf copy gc:{1Qs6xsVJF7TkMk00s6W28HjdZ8onx2C4O} gc:{1BhTY6WLPRUkqKukNtQTIDMyjLO_UKMzP} --drive-server-side-across-configs -vvv --progress --tpslimit 3 --transfers 3 --stats 1s
try:
data = {'type': 'success', 'msg': u'Target:%s 작업을 시작합니다.' % target}
socketio.emit("notify",
data,
namespace='/framework',
broadcast=True)
command = [
ModelSetting.get('gclone_path'), '--config',
ModelSetting.get('gclone_config_path'), 'copy', source, target
]
is_fclone = LogicGclone.is_fclone()
# fclone의 경우 log-level 강제설정
if is_fclone:
command += [
'--stats', '1s', '--log-level', 'NOTICE',
'--stats-log-level', 'NOTICE'
]
else:
command += ModelSetting.get_list('gclone_fix_option', ' ')
command += ModelSetting.get_list('gclone_user_option', ' ')
logger.debug(command)
if app.config['config']['is_py2']:
LogicGclone.current_process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
bufsize=1)
else:
LogicGclone.current_process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
LogicGclone.current_data['command'] = ' '.join(command)
LogicGclone.current_data['log'] = []
LogicGclone.current_data['files'] = []
LogicGclone.trans_callback('start')
if is_fclone:
LogicGclone.current_log_thread = threading.Thread(
target=LogicGclone.fclone_log_thread_fuction, args=())
else:
LogicGclone.current_log_thread = threading.Thread(
target=LogicGclone.log_thread_fuction, args=())
LogicGclone.current_log_thread.start()
logger.debug('normally process wait()')
ret = LogicGclone.current_process.wait()
LogicGclone.current_process = None
return ret
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
return 'fail'
def queue_append(queue_list):
try:
logger.debug(queue_list)
new_queue_list = []
for q in queue_list:
src, tar = q.split('|')
tmps = tar.split('/')
if len(tmps) > 1:
for i in range(1, len(tmps)):
tmps[i] = Util.change_text_for_use_filename(
tmps[i]).replace(' ', ' ').replace(
' ', ' ').rstrip('.').strip()
new_queue_list.append('%s|%s/%s' %
(src, tmps[0], '/'.join(tmps[1:])))
else:
new_queue_list.append(q)
logger.debug(new_queue_list)
tmp = ModelSetting.get('gclone_queue_list')
tmp += '\n' + '\n'.join(new_queue_list)
ModelSetting.set('gclone_queue_list', tmp)
socketio_callback('refresh_queue',
ModelSetting.get('gclone_queue_list'))
return LogicGclone.start()
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def process_ajax(sub, req):
try:
if sub == 'start':
ret = LogicGclone.start()
return jsonify(ret)
elif sub == 'stop':
LogicGclone.current_data['user_stop'] = True
ret = LogicGclone.kill()
return jsonify(ret)
elif sub == 'version':
command = [ModelSetting.get('gclone_path'), 'version']
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1)
ret = []
with process.stdout:
for line in iter(process.stdout.readline, b''):
ret.append(line)
process.wait() # wait for the subprocess to exit
return jsonify(ret)
elif sub == 'view_config':
from framework.common.util import read_file
data = read_file(ModelSetting.get('gclone_config_path'))
return jsonify({'ret':True, 'data':data})
elif sub == 'gen_config':
default = '''
[gc]
type = drive
scope = drive
service_account_file = {first_json}
service_account_file_path = {accounts_dir}/
'''
import glob
accounts_dir = ModelSetting.get('path_accounts')
sa_files = glob.glob(os.path.join(accounts_dir, '*.json'))
if len(sa_files) == 0:
ret = {'ret':False, 'log:': u'json 파일이 없습니다.'}
else:
first_json = os.path.join(accounts_dir, sa_files[0])
default = default.format(first_json=first_json, accounts_dir=accounts_dir)
logger.debug(default)
from framework.common.util import read_file, write_file
config_path = ModelSetting.get('gclone_config_path')
write_file(default, config_path)
ret = {'ret':True, 'data':read_file(config_path)}
return jsonify(ret)
elif sub == 'log_reset':
LogicGclone.current_data['log'] = []
return jsonify('')
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def get_user_copy_dest(category):
try:
if ModelSetting.get_bool('use_user_setting'):
rule_list = ModelSetting.get_list('user_copy_dest_rules', '\n')
for rule in rule_list:
orig, converted = rule.split('|')
if orig.endswith('*'): orig = orig.replace('*', '')
logger.debug('orig(%s), category(%s)', orig, category)
if category.startswith(orig): return converted
return category
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
return None
def get_first_json():
accounts_dir = ModelSetting.get('path_accounts')
for (path, dir, files) in os.walk(accounts_dir):
for fname in files:
if os.path.splitext(fname)[-1] == '.json':
return os.path.join(path, fname)
return None
def get_first_json():
import glob
accounts_dir = ModelSetting.get('path_accounts')
sa_files = glob.glob(os.path.join(accounts_dir, '*.json'))
if len(sa_files) == 0:
return None
first_json = os.path.join(accounts_dir, sa_files[0])
return os.path.join(accounts_dir, sa_files[0])
def gclone_copy(id):
try:
entity = ListModelItem.get(id)
if entity is None:
return {'ret': False, 'data': '유효한 아이템이 없습니다'}
category = LogicGSheet.get_user_copy_dest(entity.category)
logger.debug('category: %s -> %s', entity.category, category)
from gd_share_client.logic_user import LogicUser
#logger.debug(category)
my_remote = LogicUser.instance.get_my_copy_path('gsheet', category)
logger.debug('my_remote(%s)', my_remote)
if ModelSetting.get_bool('use_user_setting'):
dest_folder = entity.title2 if entity.title2 != u'' else entity.title
gcstring = 'gc:{%s}|%s/%s' % (entity.folder_id, my_remote,
dest_folder)
else:
dest_folder = entity.category + '/' + entity.title2 if entity.title2 != u'' else entity.title
gcstring = 'gc:{%s}|%s/%s' % (entity.folder_id, "gc:{}",
dest_folder)
tmp = ModelSetting.get('gclone_queue_list')
if tmp.find(gcstring) != -1:
return {'ret': True, 'data': '이미 큐에 존재합니다.'}
else:
LogicGclone.queue_append([gcstring])
entity.copied_time = datetime.now()
# 처음 복사하는 경우만 시트정보에 카운트 갱신
if entity.copy_count == 0:
wsentity = WSModelItem.get(entity.sheet_id)
if wsentity is not None:
wsentity.copy_count += 1
wsentity.save()
entity.copy_count += 1
entity.save()
return {'ret': True, 'data': '큐에 추가하였습니다.'}
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
return {'ret': False, 'data': 'Exception'}
def get_random_json():
import random
accounts_dir = ModelSetting.get('path_accounts')
sa_files = []
for (path, dir, files) in os.walk(accounts_dir):
for fname in files:
if os.path.splitext(fname)[-1] == '.json':
sa_files.append(fname)
if len(sa_files) == 0: return None
idx = int(random.random() * len(sa_files))
return os.path.join(accounts_dir, sa_files[idx])
def get_size(id):
try:
entity = ListModelItem.get(id)
if entity is None:
return {'ret': False, 'data': '유효한 아이템이 없습니다'}
command = [
ModelSetting.get('gclone_path'), '--config',
ModelSetting.get('gclone_config_path'), 'size',
'gc:{%s}' % entity.folder_id
]
data = SystemLogicCommand.execute_command_return(command)
if data.find('Failed') > 0:
logger.error('failed to get size! (%s)' % (''.join(data)))
return {'ret': False, 'data': ''.join(data)}
data = data.split('\n')
entity.obj_num = int(data[0].split(':')[1].strip())
entity.str_size = data[1].split(':')[1].split('(')[0].strip()
entity.byte_size = LogicGSheet.get_byte_size(entity.str_size)
entity.updated_time = datetime.now()
logger.debug('getsize: folder_id:%s obj_num: %d, size: %s',
entity.folder_id, entity.obj_num, entity.str_size)
entity.save()
info_str = '<br>파일수: {obj_num}<br>사이즈: {str_size}'.format(
obj_num=entity.obj_num, str_size=entity.str_size)
def func():
ret = LogicGSheet.update_size(entity.id)
thread = threading.Thread(target=func, args=())
thread.setDaemon(True)
thread.start()
return {'ret': True, 'data': info_str}
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
return {'ret': False, 'data': '{e}'.format(e=e)}
def process_api(sub, req):
try:
if sub == 'append':
ret = {}
cmd = req.form['cmd']
tmp = ModelSetting.get('gclone_queue_list')
if tmp.find(cmd) != -1:
ret['status'] = 'already_exist'
else:
ret['status'] = LogicGclone.current_data['status']
LogicGclone.queue_append([cmd])
logger.debug('process_api:%s', ret)
logger.debug('process_api:%s', cmd)
return jsonify(ret)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def func():
LogicGclone.current_data['status'] = 'is_running'
while True:
count = 0
job_list = ModelSetting.get_list(
'gclone_queue_list', '\n')
for job in job_list:
try:
if LogicGclone.current_data['user_stop']:
break
tmp = job.split('#')[0].split('|')
if len(tmp) == 2:
target = tmp[1].strip()
target = target.replace(
'{}', '{%s}' % ModelSetting.get(
'gclone_default_folderid'))
if target.find('{}') != -1:
continue
if target.find(':') == -1:
continue
return_code = LogicGclone.gclone_execute(
tmp[0].strip(), target)
# 0 정상
logger.debug('return_code:%s', return_code)
if return_code == 0:
tmp2 = ModelSetting.get(
'gclone_queue_list')
for t in tmp2.split('\n'):
if t.strip().startswith(
'%s|%s' %
(tmp[0], tmp[1])):
ModelSetting.set(
'gclone_queue_list',
tmp2.replace(t, ''))
socketio_callback(
'refresh_queue',
ModelSetting.get(
'gclone_queue_list'))
count += 1
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
if LogicGclone.current_data['user_stop']:
break
if count == 0:
break
LogicGclone.current_data['status'] = 'ready'
LogicGclone.current_data['user_stop'] = False
data = {'type': 'success', 'msg': u'gclone 작업을 완료하였습니다.'}
socketio.emit("notify",
data,
namespace='/framework',
broadcast=True)
def check_plex_condition(entity):
try:
if ModelSetting.get_int('plex_condition') == 0:
return False
rx_keyword = r'(?P<keyword>.*?)\s*?(\(|\[).*'
rx_year = r'\((?P<year>\d{4})\)'
title = entity.title2 if entity.title2 != '' else entity.title
return False
# TODO:
"""
from plex import LogicNormal as PlexLogic
return True
"""
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
return '-'
def search_gsheet(doc_id):
try:
ret = []
logger.debug('start to search_gsheet: %s', doc_id)
json_file = LogicGSheet.get_first_json()
if json_file is None:
logger.error(
'failed to get json file. please check json file in (%s)',
ModelSetting.get('path_accounts'))
return []
if doc_id.startswith(u'http'): doc_url = doc_id
else:
doc_url = 'https://docs.google.com/spreadsheets/d/{doc_id}'.format(
doc_id=doc_id)
logger.debug('url(%s)', doc_url)
try:
import gspread
except ImportError:
os.system("{} install gspread".format(
app.config['config']['pip']))
import gspread
gsp = gspread.authorize(LogicGSheet.credentials)
doc = gsp.open_by_url(doc_url)
for ws in doc.worksheets():
if LogicGSheet.validate_sheet(ws):
ret.append({
'doc_id': doc.id,
'doc_title': doc.title,
'doc_url': doc_url,
'ws_id': ws.id,
'ws_title': ws.title
})
return ret
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
return []
def search_gsheet(doc_id):
try:
ret = []
logger.debug('start to search_gsheet: %s', doc_id)
json_file = LogicGSheet.get_first_json()
if json_file is None:
logger.error(
'failed to get json file. please check json file in (%s)',
ModelSetting.get('path_accounts'))
return []
scope = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
if doc_id.startswith(u'http'): doc_url = doc_id
else:
doc_url = 'https://docs.google.com/spreadsheets/d/{doc_id}'.format(
doc_id=doc_id)
logger.debug('url(%s)', doc_url)
credentials = ServiceAccountCredentials.from_json_keyfile_name(
json_file, scope)
gsp = gspread.authorize(credentials)
doc = gsp.open_by_url(doc_url)
for ws in doc.worksheets():
if LogicGSheet.validate_sheet(ws):
ret.append({
'doc_id': doc.id,
'doc_title': doc.title,
'doc_url': doc_url,
'ws_id': ws.id,
'ws_title': ws.title
})
return ret
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
return []
def update_size(entity_id):
try:
entity = ListModelItem.get(entity_id)
wsentity = WSModelItem.get(entity.sheet_id)
doc_id = wsentity.doc_id
ws_id = wsentity.ws_id
logger.debug('start to get item from gsheet: %s, ws:%d', doc_id,
ws_id)
json_file = LogicGSheet.get_randon_json()
if json_file is None:
logger.error(
'failed to get json file. please check json file in (%s)',
ModelSetting.get('path_accounts'))
return ret
doc_url = wsentity.doc_url
try:
import gspread
except ImportError:
os.system("{} install gspread".format(
app.config['config']['pip']))
import gspread
gsp = gspread.authorize(LogicGSheet.credentials)
doc = gsp.open_by_url(doc_url)
ws = LogicGSheet.get_worksheet(doc, ws_id)
cols = ws.row_values(1)
index_size = cols.index(u'사이즈')
index_obj_num = cols.index(u'파일수')
cell = ws.find(entity.folder_id)
ws.update_cell(cell.row, index_size + 1, entity.str_size)
ws.update_cell(cell.row, index_obj_num + 1, entity.obj_num)
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
def is_fclone():
try:
command = [ModelSetting.get('gclone_path'), 'version']
if app.config['config']['is_py2']:
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
bufsize=1)
else:
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
ret = []
iter_arg = b'' if app.config['config']['is_py2'] else ''
with process.stdout:
for line in iter(process.stdout.readline, iter_arg):
if line.find('fclone') != -1: return True
return False
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def update_size(entity_id):
try:
entity = ListModelItem.get(entity_id)
wsentity = WSModelItem.get(entity.sheet_id)
doc_id = wsentity.doc_id
ws_id = wsentity.ws_id
logger.debug('start to get item from gsheet: %s, ws:%d', doc_id,
ws_id)
json_file = LogicGSheet.get_first_json()
if json_file is None:
logger.error(
'failed to get json file. please check json file in (%s)',
ModelSetting.get('path_accounts'))
return ret
scope = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
doc_url = wsentity.doc_url
credentials = ServiceAccountCredentials.from_json_keyfile_name(
json_file, scope)
gsp = gspread.authorize(credentials)
doc = gsp.open_by_url(doc_url)
ws = LogicGSheet.get_worksheet(doc, ws_id)
cols = ws.row_values(1)
index_size = cols.index(u'사이즈')
index_obj_num = cols.index(u'파일수')
cell = ws.find(entity.folder_id)
ws.update_cell(cell.row, index_size + 1, entity.str_size)
ws.update_cell(cell.row, index_obj_num + 1, entity.obj_num)
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
def scheduled_copy(sheet_id):
try:
wsentity = WSModelItem.get(sheet_id)
wsentity.is_running = True
wsentity.save()
# COPY items
succeed = 0
failed = 0
copy_mode = ModelSetting.get_int('copy_mode')
if copy_mode == 0: return
# keyword rule 적용: 미사용-너무느림
#rules = ModelSetting.get_list('keyword_rules', '|') if copy_mode == 1 else ModelSetting.get_list('except_keyword_rules', '|')
# TODO:Plex 연동저건에 따른 처리: 0 연동안함, 1: 있으면 skip, 2: 용량크면복사
#plex_condition = ModelSetting.get_int('plex_condition')
for entity in ListModelItem.get_schedule_target_items(sheet_id):
# keyword rule 적용: 파일타입의 경우만
if entity.mimetype == 1:
rules = ModelSetting.get_list(
'keyword_rules',
'|') if copy_mode == 1 else ModelSetting.get_list(
'except_keyword_rules', '|')
copy_flag = True
for rule in rules:
copy_flag = False if copy_mode == 1 else True
if entity.title.find(rule) != -1:
copy_flag = not copy_flag
logger.debug(
'keyword(%s) is matched, copy_flag(%s)', rule,
copy_flag)
break
if not copy_flag: continue
if entity.copy_count > 1: continue
# keyword rule 적용: 미사용-너무느림
"""
from gd_share_client.model import ModelSetting as gscModelSetting
remote_path = 'gc:{%s}' % entity.folder_id
ret = RcloneTool.lsjson(gscModelSetting.get('rclone_path'), gscModelSetting.get('rclone_config_path'), remote_path)
if len(ret) == 0: continue
for f in ret:
mtype = f['MimeType']
if not mtype.startswith('video/'):
continue
fname = f['Name']
# copy_mode: 1-whitelist, 2-blacklist
for rule in rules:
copy_flag = False if copy_mode == 1 else True
if fname.find(rule) != -1:
copy_flag = not copy_flag
logger.debug('keyword(%s) is matched, copy_flag(%s)', rule, copy_flag)
break
if not copy_flag: continue
"""
# TODO: Plex 연동 조건에 따른 복사 처리: 20.09.10
#if not LogicGSeet.check_plex_condition(entity):
#continue
logger.info(
'copy target: %s, %s, %s, %s',
entity.title2 if entity.title2 != u"" else entity.title,
entity.folder_id, entity.category, entity.str_size)
ret = LogicGSheet.gclone_copy(entity.id)
logger.info(ret['data'])
if ret['ret']: succeed += 1
else: failed += 1
total = succeed + failed
logger.info('scheduled_copy: total(%d), succeed(%d), failed(%d)',
total, succeed, failed)
ret = {
'ret':
True,
'data':
'copy result: total({total}), succeed({succeed}), failed({failed})'
.format(total=total, succeed=succeed, failed=failed)
}
except Exception as e:
logger.error('Exception %s', e)
logger.error(traceback.format_exc())
ret = {'ret': False, 'data': 'Exception'}
finally:
wsentity.is_running = False
wsentity.save()
return ret
def load_items(wsmodel_id):
try:
ret = []
wsentity = WSModelItem.get(wsmodel_id)
if wsentity is None:
return None
# 목록이 삭제된 경우 업데이트
if wsentity.total_count > 0:
wsentity.total_count = ListModelItem.get_total_count(
wsentity.id)
wsentity.copy_count = ListModelItem.get_copy_count(wsentity.id)
wsentity.save()
doc_id = wsentity.doc_id
ws_id = wsentity.ws_id
logger.debug('start to get items from gsheet: %s, ws:%d', doc_id,
ws_id)
json_file = LogicGSheet.get_first_json()
if json_file is None:
logger.error(
'failed to get json file. please check json file in (%s)',
ModelSetting.get('path_accounts'))
return ret
scope = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
doc_url = wsentity.doc_url
credentials = ServiceAccountCredentials.from_json_keyfile_name(
json_file, scope)
gsp = gspread.authorize(credentials)
doc = gsp.open_by_url(doc_url)
ws = LogicGSheet.get_worksheet(doc, ws_id)
count = 0
scount = 0
for r in ws.get_all_records(head=1):
try:
# 폴더ID, 분류가 없는 경우 제외
if r[u'분류'] == '' or r[u'폴더 ID'] == '':
scount += 1
continue
# 파일수, 사이즈가 없는 경우 예외처리
if r[u'파일수'] == '': obj_num = 0
else: obj_num = int(r[u'파일수'])
if r[u'사이즈'] == '': str_size = '-'
else: str_size = r[u'사이즈']
# 파일수0, 사이즈 0Bytes인경우 스킵
if obj_num == 0 and str_size == u'0 Bytes':
scount += 1
continue
info = {
'sheet_id': wsmodel_id,
'title': r[u'제목'],
'folder_id': r[u'폴더 ID'],
'category': r[u'분류'],
'title2': r[u'제목 매핑'],
'obj_num': obj_num,
'str_size': str_size
}
entity = ListModelItem.create(info)
if entity is None:
#logger.debug('already exist item(folder_id:%s)', info['folder_id'])
scount += 1
continue
count += 1
except KeyError:
logger.error('failed to get item info')
logger.error(r)
continue
wsentity.updated_time = datetime.now()
wsentity.total_count += count
wsentity.save()
logger.info('{count} 항목을 추가하였습니다(스킵: {scount}건)'.format(
count=count, scount=scount))
ret = {
'ret':
True,
'data':
'{count} 항목을 추가하였습니다(스킵: {scount}건)'.format(count=count,
scount=scount)
}
return ret
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())