def getPkgOpen(subpkgs):
"""获取open 模块
模块地址 http://ufo.sogou-inc.com/git/open.git
"""
targetfolder = os.path.join( conf.getConfig()['path'] , 'static' , 'js' )
utils.createfolder(TEMP_FOLDER)
if not os.path.exists( targetfolder ):
utils.createfolder(targetfolder)
subpkgs = subpkgs or conf.getConfig()['PKG_OPEN']
subpkgs.insert(0 , 'common')
os.system('git clone http://ufo.sogou-inc.com/git/open.git ' + os.path.join(TEMP_FOLDER , 'open'))
successpkg = []
for pkg in subpkgs:
source = os.path.join( TEMP_FOLDER , 'open' , pkg )
if not os.path.exists(source):
log.warn('Sub package ' + pkg + ' not exist in Open.')
continue
utils.copyfiles( source , os.path.join( targetfolder , 'open' , pkg ) )
successpkg.append( pkg )
utils.removefolder(TEMP_FOLDER)
log.success( 'Adding Open package include ' + ','.join(successpkg) + ' success!' )
def consume_events():
createfolder(pathLocalUp)
createfolder(pathLocalDl)
ftp = GestorFTP(cred=FTP_PM_CRED,
pLu=pathLocalUp,
pRu=pathRemotoUp,
pld=pathLocalDl,
pRd=pathRemotoDl)
# print(f"ftp[{ftp}]", flush=True)
telnet = ConectorTelnet(dat=CRED_U2020)
# print(f"telnet[{telnet}]", flush=True)
# print("Previous m in consumer", flush=True)
for m in consumer:
# print("After m in consumer", flush=True)
# print(m.value) # this works as expected
# show_message(message=m.value)
mml_event = get_mml_event(message=m.value)
# print('-' * 20)
if mml_event:
# print(f"mml_event[{mml_event}]")
print('-' * 20)
# process_mml_event(mml_event=mml_event)
if mml_event.some_valid():
# Debo generar archivo de comandos
path = mml_event.generate_mml_file(dir=pathLocalUp)
print(f"Se generó path[{path}]")
# En base al archivo de comandos grabar en la BD
db_entry(file_path=os.path.join(pathLocalUp, path),
client_id=mml_event.client_id,
script_id=mml_event.script_id)
# Vía ftp depositar archivo en U2020 folder
ftp.enviar(path)
# Enviar telnet a U2020
telnet.conecta()
telnet.ejecuta_scritp(path)
telnet.desconecta()
# Esperar aparición de archivo de resultados
# Traer archivo de resultados
band, path = ftp.extraer(path[:-4])
print(f"Se obtuvo path[{path}]")
# print('Extrayendo el archivo:', path[:-4])
# Actualizar BD
db_entry(file_path=os.path.join(pathLocalDl, path),
client_id=mml_event.client_id,
script_id=mml_event.script_id)
# Responder vía Kafka a cliente
# print("Previous producer.send()", flush=True)
producer.send(RST_TOPIC, value=m.value)
# print("After producer.send()", flush=True)
# Preguntas:
# - qué hago con los archivos generados ?
# Sugerencias:
# - generar log de proceso
else:
# Responder vía Kafka a cliente
pass
def compileHTML(needCompress=False, needHtml=False):
"""为所有tpl文件加上时间戳
"""
base = os.path.join(PATH, 'build', 'template')
tplfiles = []
for dirpath, dirnames, filenames in os.walk(base):
tplfiles.extend([
os.path.join(dirpath, f) for f in filenames if f.endswith('.tpl')
])
if COMPILE_FOLDER:
for dirpath, dirnames, filenames in os.walk(
os.path.join(PATH, 'build', COMPILE_FOLDER)):
tplfiles.extend([os.path.join(dirpath, f) for f in filenames])
for tpl in tplfiles:
f = parser.compileHTML(tpl, needCompress)
utils.writefile(tpl, f)
if needHtml:
log.log('Render html file.\nIt will under build folder.')
files = os.listdir(os.path.join(base))
tplfiles = []
for dirpath, dirnames, filenames in os.walk(base):
tplfiles.extend([
os.path.join(dirpath, f) for f in filenames
if f.endswith('.tpl')
])
for fname in tplfiles:
token = fname.replace(base + '/', '').replace('.tpl', '')
try: #有强行编译的需求
html = parser.parseTpl(token, isbuild=True)
if token.find('/') != -1:
subfolder = os.path.join(PATH, 'build', 'html',
token.split('/')[0])
if not os.path.exists(subfolder):
utils.createfolder(subfolder)
utils.writefile(
os.path.join(PATH, 'build', 'html', token + '.html'), html)
except Exception as e:
log.error(str(e))
if not conf.getConfig().get('html_force_output'):
raise
log.success('Render html success')
def compileHTML( needCompress = False , needHtml = False ):
"""为所有tpl文件加上时间戳
"""
base = os.path.join(PATH , 'build' , 'template')
tplfiles = []
for dirpath , dirnames , filenames in os.walk(base):
tplfiles.extend([ os.path.join( dirpath , f ) for f in filenames if f.endswith('.tpl') ])
if COMPILE_FOLDER:
for dirpath , dirnames , filenames in os.walk(os.path.join(PATH , 'build' , COMPILE_FOLDER)):
tplfiles.extend([ os.path.join( dirpath , f ) for f in filenames ])
for tpl in tplfiles:
f = parser.compileHTML(tpl , needCompress)
utils.writefile(tpl , f)
if needHtml:
log.log('Render html file.\nIt will under build folder.')
files = os.listdir( os.path.join( base ) )
tplfiles = []
for dirpath , dirnames, filenames in os.walk(base):
tplfiles.extend( [ os.path.join(dirpath , f) for f in filenames if f.endswith('.tpl') ] )
for fname in tplfiles:
token = fname.replace( base + '/' , '' ).replace('.tpl' , '')
try:#有强行编译的需求
html = parser.parseTpl(token , isbuild=True)
if token.find('/') != -1:
subfolder = os.path.join(PATH , 'build' , 'html' , token.split('/')[0])
if not os.path.exists(subfolder):
utils.createfolder(subfolder)
utils.writefile( os.path.join(PATH , 'build' , 'html' , token + '.html' ) , html )
except Exception as e:
log.error(str(e))
if not conf.getConfig().get('html_force_output'):
raise
log.success('Render html success');
def run(params , options):
"""
"""
tmbegin = time.time()
buildtype = None
if params and len(params):
if conf.getConfig().get(params[0]):
buildtype = params[0]
else:
log.error('No such build type:' + params[0])
sys.exit(1)
utils.removefolder(BUILD_DIR);
utils.createfolder(BUILD_DIR);
#直接在build目录操作哦
utils.copyfiles( 'template' , os.path.join(BUILD_DIR , 'template') )
utils.copyfiles( 'static' , os.path.join(BUILD_DIR , 'static') )
if COMPILE_FOLDER:
utils.copyfiles( COMPILE_FOLDER , os.path.join(BUILD_DIR , COMPILE_FOLDER) )
#模块分为CSS和js，之前使用require_modules代表js，建议现在使用require_js_modules；
#而require_css_modules代表css。
if conf.getConfig().get('require_modules'):
log.warn("'require_modules' is deprecated,you should use 'require_js_modules' instead!")
require_modules = conf.getConfig().get('require_js_modules') or conf.getConfig().get('require_modules') or ['main']
require_js_modules=require_modules#[email protected]:历史遗留
require_css_modules = conf.getConfig().get('require_css_modules') or ['main']
#@deprecated
#maincss = os.path.join( PATH , BUILD_DIR , 'static' , 'css' , conf.getConfig().get('css_folder') or '' , 'main.css' )
try:
log.log( 'Combine css&js with r.js' )
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR , 'static' , 'js' , conf.getConfig().get('js_folder') or '' , module + '.js' )
subprocess.call( 'node ' + RJSPATH +' -o name=' + module + ' out='+ js + ' optimize=none baseUrl=' + os.path.join(PATH , BUILD_DIR , 'static' , 'js' , conf.getConfig().get('js_folder') or '') , shell=True)#使用YUICompressor压缩，这里仅合并
#[email protected]:合并css集合
for module in require_css_modules:
css=os.path.join( PATH , BUILD_DIR , 'static' , 'css' , conf.getConfig().get('css_folder') or '' , module + '.css' )
subprocess.call( 'node ' + RJSPATH + ' -o cssIn=' + css + ' out=' + css , shell=True)#cssIn参数带.css后缀
log.success( 'Combine css&js with r.js success.' )
except:
log.error('Please insure you have installed r.js on your computer')
raise
log.log('Begin to add Timestamps...' , True)
#@todo:HTML内部的css或许也需要时间戳
compileCss();
log.success('Success!')
if options.get('html'):
utils.createfolder( os.path.join( BUILD_DIR , 'html'))
log.log('Begin to compile tpls...' )
compileHTML(options.get('compress') , options.get('html'))
log.success('Success!')
log.log('Begin to replace all token...', True)
compileCommon(buildtype)
log.success('Success!')
if options.get('compress'):
log.log('Begin to compile Js...' , True)
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR , 'static' , 'js', conf.getConfig().get('js_folder') or '' , module + '.js' )
subprocess.call( 'java -jar ' + YCPATH + ' --type js --charset ' + conf.getConfig()['encoding'] + ' ' + js + ' -o ' + js , shell=True );
log.success('Success!')
log.log('Begin to compile Css...' , True)
for module in require_css_modules:#[email protected]:编辑CSS集合
css = os.path.join(PATH, BUILD_DIR , 'static' , 'css', conf.getConfig().get('css_folder') or '' , module + '.css' )
subprocess.call( 'java -jar ' + YCPATH + ' --type css --charset ' + conf.getConfig()['encoding'] + ' ' + css + ' -o ' + css , shell=True );
# subprocess.call( 'java -jar ' + YCPATH + ' --type css --charset ' + conf.getConfig()['encoding'] + ' ' + maincss + ' -o ' + maincss , shell=True);
log.success('Success!')
log.success('Compiled successfully.')
log.success('Time cost %s s.' % (time.time()-tmbegin) )
def run(params, options):
"""
"""
tmbegin = time.time()
buildtype = None
if params and len(params):
if conf.getConfig().get(params[0]):
buildtype = params[0]
else:
log.error("No such build type:" + params[0])
sys.exit(1)
utils.removefolder(BUILD_DIR)
utils.createfolder(BUILD_DIR)
# 直接在build目录操作哦
utils.copyfiles("template", os.path.join(BUILD_DIR, "template"))
utils.copyfiles("static", os.path.join(BUILD_DIR, "static"))
if COMPILE_FOLDER:
utils.copyfiles(COMPILE_FOLDER, os.path.join(BUILD_DIR, COMPILE_FOLDER))
# 模块分为CSS和js，之前使用require_modules代表js，建议现在使用require_js_modules；
# 而require_css_modules代表css。
if conf.getConfig().get("require_modules"):
log.warn("'require_modules' is deprecated,you should use 'require_js_modules' instead!")
require_modules = conf.getConfig().get("require_js_modules") or conf.getConfig().get("require_modules") or ["main"]
require_js_modules = require_modules # [email protected]:历史遗留
require_css_modules = conf.getConfig().get("require_css_modules") or ["main"]
# @deprecated
# maincss = os.path.join( PATH , BUILD_DIR , 'static' , 'css' , conf.getConfig().get('css_folder') or '' , 'main.css' )
try:
log.log("Combine css&js with r.js")
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR, "static", "js", conf.getConfig().get("js_folder") or "", module + ".js")
p = subprocess.Popen(
"r.js -o name="
+ module
+ " out="
+ js
+ " optimize=none baseUrl="
+ os.path.join(PATH, BUILD_DIR, "static", "js", conf.getConfig().get("js_folder") or ""),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
) # 使用YUICompressor压缩，这里仅合并
(o, e) = p.communicate()
print o
print e
if not p.returncode == 0:
raise IOError("Combine js failed")
# [email protected]:合并css集合
for module in require_css_modules:
css = os.path.join(
PATH, BUILD_DIR, "static", "css", conf.getConfig().get("css_folder") or "", module + ".css"
)
p = subprocess.Popen(
"r.js -o cssIn=" + css + " out=" + css, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
) # cssIn参数带.css后缀
(o, e) = p.communicate()
print o
print e
if not p.returncode == 0:
raise IOError("Combine css failed")
log.success("Combine css&js with r.js success.")
except:
log.error("Please insure you have installed r.js on your computer")
raise
log.log("Begin to add Timestamps...", True)
# @todo:HTML内部的css或许也需要时间戳
compileCss()
log.success("Success!")
if options.get("html"):
utils.createfolder(os.path.join(BUILD_DIR, "html"))
log.log("Begin to compile tpls...")
compileHTML(options.get("compress"), options.get("html"))
log.success("Success!")
log.log("Begin to replace all token...", True)
compileCommon(buildtype)
log.success("Success!")
if options.get("compress"):
log.log("Begin to compile Js...", True)
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR, "static", "js", conf.getConfig().get("js_folder") or "", module + ".js")
subprocess.call(
"java -jar " + YCPATH + " --type js --charset " + conf.getConfig()["encoding"] + " " + js + " -o " + js,
shell=True,
)
log.success("Success!")
log.log("Begin to compile Css...", True)
for module in require_css_modules: # [email protected]:编辑CSS集合
css = os.path.join(
PATH, BUILD_DIR, "static", "css", conf.getConfig().get("css_folder") or "", module + ".css"
)
subprocess.call(
"java -jar "
+ YCPATH
+ " --type css --charset "
+ conf.getConfig()["encoding"]
+ " "
+ css
+ " -o "
+ css,
shell=True,
)
# subprocess.call( 'java -jar ' + YCPATH + ' --type css --charset ' + conf.getConfig()['encoding'] + ' ' + maincss + ' -o ' + maincss , shell=True);
log.success("Success!")
log.success("Compiled successfully.")
log.success("Time cost %s s." % (time.time() - tmbegin))
def run(params, options):
"""
"""
tmbegin = time.time()
buildtype = None
if params and len(params):
if conf.getConfig().get(params[0]):
buildtype = params[0]
else:
log.error('No such build type:' + params[0])
sys.exit(1)
utils.removefolder(BUILD_DIR)
utils.createfolder(BUILD_DIR)
#直接在build目录操作哦
utils.copyfiles('template', os.path.join(BUILD_DIR, 'template'))
utils.copyfiles('static', os.path.join(BUILD_DIR, 'static'))
if COMPILE_FOLDER:
utils.copyfiles(COMPILE_FOLDER, os.path.join(BUILD_DIR,
COMPILE_FOLDER))
#模块分为CSS和js，之前使用require_modules代表js，建议现在使用require_js_modules；
#而require_css_modules代表css。
if conf.getConfig().get('require_modules'):
log.warn(
"'require_modules' is deprecated,you should use 'require_js_modules' instead!"
)
require_modules = conf.getConfig().get(
'require_js_modules') or conf.getConfig().get('require_modules') or [
'main'
]
require_js_modules = require_modules #[email protected]:历史遗留
require_css_modules = conf.getConfig().get('require_css_modules') or [
'main'
]
#@deprecated
#maincss = os.path.join( PATH , BUILD_DIR , 'static' , 'css' , conf.getConfig().get('css_folder') or '' , 'main.css' )
try:
log.log('Combine css&js with r.js')
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR, 'static', 'js',
conf.getConfig().get('js_folder') or '',
module + '.js')
p = subprocess.Popen(
'r.js -o name=' + module + ' out=' + js +
' optimize=none baseUrl=' +
os.path.join(PATH, BUILD_DIR, 'static', 'js',
conf.getConfig().get('js_folder') or ''),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True) #使用YUICompressor压缩，这里仅合并
(o, e) = p.communicate()
print o
print e
if not p.returncode == 0:
raise IOError('Combine js failed')
#[email protected]:合并css集合
for module in require_css_modules:
css = os.path.join(PATH, BUILD_DIR, 'static', 'css',
conf.getConfig().get('css_folder') or '',
module + '.css')
p = subprocess.Popen('r.js -o cssIn=' + css + ' out=' + css,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True) #cssIn参数带.css后缀
(o, e) = p.communicate()
print o
print e
if not p.returncode == 0:
raise IOError('Combine css failed')
log.success('Combine css&js with r.js success.')
except:
log.error('Please insure you have installed r.js on your computer')
raise
log.log('Begin to add Timestamps...', True)
#@todo:HTML内部的css或许也需要时间戳
compileCss()
log.success('Success!')
if options.get('html'):
utils.createfolder(os.path.join(BUILD_DIR, 'html'))
log.log('Begin to compile tpls...')
compileHTML(options.get('compress'), options.get('html'))
log.success('Success!')
log.log('Begin to replace all token...', True)
compileCommon(buildtype)
log.success('Success!')
if options.get('compress'):
log.log('Begin to compile Js...', True)
for module in require_js_modules:
js = os.path.join(PATH, BUILD_DIR, 'static', 'js',
conf.getConfig().get('js_folder') or '',
module + '.js')
subprocess.call('java -jar ' + YCPATH + ' --type js --charset ' +
conf.getConfig()['encoding'] + ' ' + js + ' -o ' +
js,
shell=True)
log.success('Success!')
log.log('Begin to compile Css...', True)
for module in require_css_modules: #[email protected]:编辑CSS集合
css = os.path.join(PATH, BUILD_DIR, 'static', 'css',
conf.getConfig().get('css_folder') or '',
module + '.css')
subprocess.call('java -jar ' + YCPATH + ' --type css --charset ' +
conf.getConfig()['encoding'] + ' ' + css + ' -o ' +
css,
shell=True)
# subprocess.call( 'java -jar ' + YCPATH + ' --type css --charset ' + conf.getConfig()['encoding'] + ' ' + maincss + ' -o ' + maincss , shell=True);
log.success('Success!')
log.success('Compiled successfully.')
log.success('Time cost %s s.' % (time.time() - tmbegin))