def __init__(self, process_count=8):
"""初始化，通过process_count 参数指定最多使用的进程数量"""
self.process_count = process_count
self.disable_print = os.environ.get('disable_print', '0')
self.process_info = {}
#self.logger = create_logger('MultiProcessRunner')
self.logger = utils.getLogger('RES')
class ProgressLogger(Singleton):
_logger = utils.getLogger('job')
_width = 60
_error_list = []
_warning_list = []
def setLogName(self, name):
self._logger = utils.getLogger(name)
def setTotal(self, cnt):
self._total = cnt
self._count = 0
def __next__(self):
# return
self._count += 1
def debug(self, s):
self.log(s, 'debug')
def info(self, s):
self.log(s, 'info')
def warning(self, s):
self.log(s, 'warning')
def error(self, s):
self.log(s, 'error')
def log(self, s, level):
if level == 'error':
self._error_list.append(s)
if level == 'warning':
self._warning_list.append(s)
if hasattr(self, '_total') and self._total > 0:
sys.stdout.write(' ' * (self._width + 9) + '\r')
sys.stdout.flush()
getattr(self._logger, level)(s)
if not (hasattr(self, '_total') and self._total > 0):
return
progress = self._width * self._count / self._total
sys.stdout.write('{0:3}/{1:3}: '.format(self._count, self._total))
sys.stdout.write('#' * int(progress) +
'-' * int(self._width - progress) + '\r')
# if self._count == self._total:
# sys.stdout.write('\n')
sys.stdout.flush()
def export_error_logs(self):
for i in range(len(self._warning_list)):
self._logger.warning(self._warning_list[i])
for i in range(len(self._error_list)):
self._logger.error(self._error_list[i])
#coding=utf-8
import click
import sys
import os
import os.path
from rtool import utils
from rtool.pcutils import cache_tool
# from tkinter import *
# from tkinter.filedialog import askdirectory
logger = utils.getLogger('Demo')
def run(click_group):
from rtool.tasks import task
click_group.add_command(task.run_dev, name='task_dev')
click_group.add_command(task.run, name='task')
from rtool.tasks.Base import Base
from rtool.tasks.BasePostAction import output_to_temp
from rtool.tasks.BaseStatus import BaseStatus
from rtool import utils
from rtool.pcutils.md5sum_of_path import md5_for_str
from rtool.pcutils.md5sum_of_path import md5_for_file
from rtool.pcutils.md5sum_of_path import md5sum_of_path
from rtool.pcutils.md5sum_of_path import md5sum_of_dir
from rtool.pcutils.md5sum_of_path import uni_id_for_res
RC_ADDRESS = utils.getMainConfig().get("RC_ADDRESS", "")
RC_ROOT = utils.getMainConfig().get(
"RC_ROOT", "/data/home/user00/service/resource_center")
# SECRETS_FILE = "/etc/rsyncd.secrets"
# WORKER_USER = "******"
logger = utils.getLogger("BaseSync")
RSYNC_TEMPLATE = "rsync -avzP $src $dst "
RSYNC_TEMPLATE_WITH_EXCLUDE = "rsync -avzP --exclude $exclude $src $dst "
MAKE_RC_DIR_TMEPLATE = "ssh $rc_address 'mkdir -p $rc_dir'"
DB_CONFIG = {
"host": "172.16.153.49",
"port": 51000,
"user": "******",
"password": "******",
"db": "asset_vega"
}
DB_CONFIG = utils.getMainConfig().get("DB_CONFIG", DB_CONFIG)
DB_CONN = None
#coding=utf-8
import os
import os.path
import imp
import rtool.utils as mut
logger = mut.getLogger('BaseActionDispatch')
class BaseActionDispatch(object):
"""docstring for BaseActionDispatch"""
def __init__(self):
super(BaseActionDispatch, self).__init__()
self.plugindir=[]
def set_plugin_dir(self,plugindir):
self.plugindir = plugindir
def invokeAction(self,module,configs,tp=None,plugindir=[]):
dirlist=[]
if len(plugindir)>0:
dirlist = plugindir
else:
dirlist = self.plugindir
if not len(dirlist)>0:
logger.warning(u"插件目录未设置")
return
file,pathname,dec = imp.find_module(module,dirlist)
taskModule = imp.load_module(module,file,pathname,dec)
taskModule.run_with_configs(configs,tp)
pass
#coding=utf-8
from rtool import utils
from rtool.taskplugin.plugin.SampleAction import SampleAction
import json
logger = utils.getLogger('CustomAction')
def run_with_configs(configs, tp=None):
logger.debug("Executing CustomAction")
apaction = CustomPluginDemo()
apaction.go(configs)
pass
def clean_output(configs):
default_output_path = configs["output-dir"]
class CustomPluginDemo(SampleAction):
"""docstring for CustomPluginDemo"""
def go(self, config):
SampleAction.go(self, config)
logger.warning("CustomPluginDemo ")
logger.warning(json.dumps(config, indent=2))
def __init__(self):
self.logger = utils.getLogger(__name__)
#coding=utf-8
# 最简示例，用于展示action插件的编写方法
import yaml
import os
import os.path
import shutil
import json
import subprocess
import sys
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import MultiProcessRunner
import rtool.utils as utils
logger = utils.getLogger('SampleAction')
def run():
logger.debug("SampleAction")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing SampleAction")
apaction = SampleAction()
apaction.go(configs)
pass
def clean_output(configs):
default_output_path = configs["output-root"]
#coding=utf-8
# 使用CCSUIExporter将csd转化为csb
import yaml
import os
import os.path
import shutil
import json
import subprocess
import sys
import re
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner
import rtool.utils as utils
logger = utils.getLogger('MakeCSBAction')
def run():
logger.debug("MakeCSBAction")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing MakeCSBAction")
apaction = MakeCSBAction()
apaction.go(configs)
pass
def clean_output(configs):
default_output_path = configs["output-root"]
from rtool import utils
import click
import rtool.utils as mut
import os
import os.path
import rtool.pcutils.md5sum_of_path as mput
import sys
import json
import rtool.res.TPCache2 as TPCache2
import rtool.res.TPTask as TPTask
from rtool.tasks.task import AbstractTask
from rtool.tasks import BasePreAction
from rtool.tasks.BaseParseConfigs import BaseParseConfigs
from rtool.tasks.BasePostAction import BasePostAction
logger = utils.getLogger('DemosTask')
def getInstance(task_context):
return DemosTask(task_context)
class DemosTask(AbstractTask):
"""DemosTask"""
action_recorder = None
def __init__(self, ctx):
super(DemosTask, self).__init__(ctx)
pass
def rumCommand(self, cmd, options):
#coding=utf-8
import os
import os.path
import xlrd
import json
import shutil
import yaml
from rtool import utils
from rtool.tasks import BasePreAction
from rtool.projects.demos.extraCfg import genExtraConfigs as GEC
from rtool.tasks import parse_config_files as pcfg
from rtool.tasks import action_dispatch as ad
logger = utils.getLogger('Task_Pre_Action_Config_Parse')
dec_options = {}
class PresActionConfigParse(BasePreAction.BasePreAction):
"""PresActionConfigParse封装了在框架中Actions执行之前需要做的操作，主要负责处理额外的配置解析"""
def __init__(self, options):
super(PresActionConfigParse, self).__init__(options)
dec_options = options
# @BasePreAction.preProcess(options = dec_options)
def go(self):
# 有额外的配置信息需要添加至extraCfg中使用以下代码
if not pcfg.task_settings.has_key('extraCfg'):
pcfg.task_settings['extraCfg'] = {}
# GEC.genExtraCfgs(pcfg.task_settings['extraCfg'])
# 框架自身需要的前处理操作
BasePreAction.pre_process(self.options)
import time
import rtool.tasks.BaseActionDispatch as BaseActionDispatch
from rtool.tasks.BaseLocalChange import BaseLocalChange
from rtool.tasks import BaseSync as BS
from time import time
global_cfg_dict = {}
task_settings = {}
white_dict = {}
# img_in_csd_list = []
# img_in_csd_dict = {}
old_global_cfg_dict = {}
ncfg_dict = {}
run_dict = {}
logger = mut.getLogger('Task_parse_config')
def modifyPathWithPattern(pattern, path_part, path):
# print "modifyPathWithPattern" + pattern+ " "+path_part
# print path
if pattern in path:
return path.replace(pattern, path_part)
return path
class BaseParseConfigs(object):
"""docstring for BaseParseConfigs"""
def __init__(self, options):
super(BaseParseConfigs, self).__init__()
self._executeActionFunc = None
#coding=utf-8
import sys
import requests
import hashlib
from rtool import utils
from rtool.tasks.Base import Base
logger = utils.getLogger("BaseStatus")
class BaseStatus(Base):
"""docstring for BaseStatus"""
def __init__(self,options):
super(BaseStatus, self).__init__(options)
self.options = options
self.key = ""
self.url = ""
def doAPIRequest(self,tid,status):
game = self.options['game']
token_str = tid+status+game+self.key
md5 = hashlib.md5()
md5.update(token_str.encode('utf-8'))
token = md5.hexdigest()
payload={}
payload['id'] = tid
payload['status'] = status
payload['game'] = game
payload['token'] = token
r = requests.get(self.url,params=payload)
print(r.status_code)
print(r.content)
import subprocess
import sys
import tempfile
import plistlib
import zlib
import struct
from time import sleep
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner
import rtool.utils as utils
from rtool.taskplugin.plugin.TPTask import TPTask
import rtool.taskplugin.plugin.TPCache2 as TPCache2
logger = utils.getLogger('EncryptImage')
def run_with_configs(configs, tp=None):
logger.debug("Executing Image Compress Action")
apaction = EncryptImage()
apaction.go(configs, tp)
pass
def clean_output(configs):
pass
class EncryptImage:
#coding:utf-8
import os
import os.path
import xlrd
import json
import shutil
from . import extraCfg
from rtool import utils
logger = utils.getLogger('GenAnimIdleList')
class GenAnimIdleList(extraCfg.ExtraCfg):
"""docstring for GenAnimIdleList"""
def __init__(self, arg):
super(GenAnimIdleList, self).__init__(arg)
self.idle_name_list = []
def getIdleNameList(self, input_file_name):
xls_file = xlrd.open_workbook(input_file_name)
input_table = xls_file.sheets()[0]
for i in range(input_table.ncols):
value = input_table.cell_value(1, i)
if value == "pinyinID[string]":
for j in range(2, input_table.nrows):
self.idle_name_list.append(
str(input_table.cell_value(j, i)))
pass
def getAllIdleName(self, item_config_path=''):
import sys
import json
import copy
import shutil
import plistlib as pl
import rtool.pcutils.md5sum_of_path as mput
from xml.etree import ElementTree
import plistlib
from rtool.tasks import BaseSync as BS
import rtool.tasks.BaseLocalChange as BLC
import rtool.tasks.BaseDecorate as BD
import rtool.tasks.BaseParseConfigs as BPC
output_to_temp = {}
logger = utils.getLogger('Task_post_action')
#Function：缓存记录上次运行结果的md5
#param：上次配置， global_cfg_dict,
#output：
def calMD5forOutputPathAndPersisted(res_root, global_cfg_dict, target,
action_recorder):
if global_cfg_dict:
for action_type in global_cfg_dict[target]:
configs = global_cfg_dict[target][action_type]
for config in configs:
# print config['output-dir']
md5sum = mput.md5sum_of_path(config['input'])
if md5sum:
#coding=utf-8
import yaml
import os
import os.path
import shutil
import json
import subprocess
import sys
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner
import rtool.utils as utils
logger = utils.getLogger('CopyRes')
def run():
logger.debug("CopyRes")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing CopyRes")
apaction = CopyResAction()
apaction.go(configs)
pass
def safeRemoveDir(dir_path):
if os.path.exists(dir_path):
shutil.rmtree(dir_path)
pass
import sys
import tempfile
import plistlib
import zlib
import struct
from time import sleep
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner
import rtool.utils as utils
import rtool.taskplugin.plugin.TPtemplates as TPtemplates
from rtool.taskplugin.plugin.TPTask import TPTask
from rtool.taskplugin.plugin.MaliTask import MaliTask
import rtool.taskplugin.plugin.TPCache2 as TPCache2
logger = utils.getLogger('Compresser')
def run():
logger.debug("Executing Image Compress Action")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing Image Compress Action")
apaction = CompressAction()
apaction.go(configs, tp)
pass
def safeRemove(path):
def setLogName(self, name):
self._logger = utils.getLogger(name)
#coding=utf-8
#coding=utf-8
import yaml
import os
import os.path
import shutil
import json
import subprocess
import sys
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import MultiProcessRunner
import rtool.utils as utils
logger = utils.getLogger('SyncFilesAction')
def run():
logger.debug("SyncFilesAction")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing SyncFilesAction")
apaction = SyncFilesAction()
apaction.go(configs)
pass
def safeRemoveDir(dir_path):
if os.path.exists(dir_path):
shutil.rmtree(dir_path)
import rtool.utils as mut
import os
import os.path
import sys
import json
import imp
import rtool.pcutils.md5sum_of_path as mput
import rtool.res.TPCache2 as TPCache2
import rtool.res.TPTask as TPTask
import rtool.tasks.BaseSync as BS
import rtool.tasks.BaseStatus as BStatus
import rtool.taskplugin.plugin.TPDirFormater as TPDF
logger = mut.getLogger('Task')
class TaskContext(object):
def __init__(self):
self.taskName = ""
# self.logger = mut.getLogger(self.taskName)
self.settings = None
self.options = None
pass
def getLogger(self):
return logger
def getEnv(self, name):
# get environment variable with `name`
#coding=utf-8
import os
import json
import plistlib
from multiprocessing import Process, Manager, Lock
import shutil
import sys
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.utils as utils
import rtool.res.action_binaryJson as AJ
logger = utils.getLogger('Xml2Mclib')
def run():
logger.debug("Xml2Mclib")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing Xml2Mclib")
apaction = Xml2Mclib()
apaction.go(configs)
pass
def safeRemoveDir(dir_path):
if os.path.exists(dir_path):
shutil.rmtree(dir_path)
pass
#coding=utf-8
import yaml
import os
import os.path
import shutil
import json
import subprocess
import sys
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner
import rtool.utils as utils
logger = utils.getLogger('CSD2CSB')
def run():
logger.debug("Executing CsdPublish")
pass
def run_with_configs(configs, tp=None):
logger.debug("Executing CsdPublish")
# os.system('mtl ccs')
apaction = CsdPublishAction()
apaction.pack(configs)
pass
def clean_output(configs):
default_output_path = configs["output-root"]
import os.path
import shutil
import json
import subprocess
import tempfile
import sys
import plistlib
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import rtool.taskplugin.plugin.MultiProcessRunner
import rtool.utils as utils
import rtool.taskplugin.plugin.TPCache2 as TPCache2
from rtool.taskplugin.plugin.TPTask import TPTask
from rtool.taskplugin.plugin.pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json
from rtool.taskplugin.plugin.pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json_then_rename
logger = utils.getLogger('AnimPacker')
logger.setLevel("DEBUG")
def relativePath(base_path,path):
if not base_path == path:
return path.replace(base_path+os.path.sep,'')
else:
return ""
pass
def run():
print("AnimPacker")
pass
def run_with_configs(configs,tp=None):
logger.debug("Executing NAnimPacker")
import os
import os.path
import shutil
import json
import subprocess
import tempfile
import sys
import plistlib
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
import MultiProcessRunner
import rtool.utils as utils
import TPCache2
from TPTask import TPTask
from pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json
logger = utils.getLogger('TexPacker')
def relativePath(base_path, path):
if not base_path == path:
return path.replace(base_path + os.path.sep, '')
else:
return ""
pass
def run():
print "TexPacker"
pass
