def compileHTML(filepath, needCompress):
"""编译html文件
Arguments:
- `filepath`:
"""
if not os.path.exists(filepath):
return False
tpl = utils.readfile(filepath)
log.log("Compile for " + filepath + ".")
LINK_TOKEN = "<link.* href=['\"](?!http|https)(.*?\.css)['\"]" # 不包含以http/https开头的资源
SCRIPT_TOKEN = "<script.* src=['\"](?!http|https)(.*?\.js)['\"]"
iters = re.finditer(LINK_TOKEN, tpl)
for i in reversed(list(iters)):
path = i.group(1)
path = compileCommon(path, "local", True) # 内部可能有替换的变量
if not conf.getConfig().get("disableAutoTimestamp"):
tpl = tpl[0 : i.start(1)] + i.group(1) + "?t=" + getFileTimeStamp(path, filepath) + tpl[i.end(1) :]
iters = re.finditer(SCRIPT_TOKEN, tpl)
for i in reversed(list(iters)):
path = i.group(1)
path = compileCommon(path, "local", True) # 内部可能有替换的变量
if not conf.getConfig().get("disableAutoTimestamp"):
tpl = tpl[0 : i.start(1)] + i.group(1) + "?t=" + getFileTimeStamp(path, filepath) + tpl[i.end(1) :]
return tpl
def exportFrontendConfig(with_config=None, with_secrets=None):
import json
from conf import DEBUG, SERVER_HOST, UUID, ANNEX_DIR, API_PORT, getConfig, SHA1_INDEX
server_message_port = None
try:
server_message_port = getConfig('server_message_port')
except:
pass
server_user = None
try:
server_user = getConfig('server_user')
except:
with settings(warn_only=True):
server_user = local('whoami', capture=True)
config = {
'server_host' : SERVER_HOST,
'server_port' : API_PORT,
'annex_remote' : ANNEX_DIR,
'uv_uuid' : UUID,
'annex_remote_port' : 22,
'server_use_ssl' : False,
'gdrive_auth_no_ask' : True,
'server_message_port' : (API_PORT + 1) if server_message_port is None else server_message_port,
'server_user' : server_user,
'index.sha1' : SHA1_INDEX
}
if with_config is not None:
if type(with_config) is not list:
with_config = [with_config]
for c in with_config:
try:
config[c] = getConfig(c)
except exception as e:
if DEBUG: print e
if with_secrets is not None:
from conf import getSecrets
if type(with_secrets) is not list:
with_secrets = [with_secrets]
for s in with_secrets:
try:
config[s] = getSecrets(s)
except exception as e:
if DEBUG: print e
for key in [k for k in config.keys() if config[k] is None]:
del config[key]
print "***********************************************"
print "THE FOLLOWING LINES MAKE FOR YOUR FRONTEND CONFIG\n\n"
print json.dumps(config)
print "***********************************************"
return config
def createNewUser(username, password, as_admin=False):
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
user_data = copy.deepcopy(USER_CREDENTIAL_PACK)
user_data['username'] = username
if as_admin:
user_data['admin'] = True
user_data['annex_key_sent'] = False
if DEBUG: print "creating %s as admin!" % username
user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT)
if os.path.exists(os.path.join(USER_ROOT, user_root)):
if DEBUG: print "user already exists NOPE!"
return False
print user_data
with open(os.path.join(USER_ROOT, user_root), 'wb+') as user:
user.write(encryptUserData(user_data, password, p_salt=SALT, iv=IV))
return True
except Exception as e: print e
return False
def createNewUser(username, password, as_admin=False):
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
user_data = copy.deepcopy(USER_CREDENTIAL_PACK)
user_data['username'] = username
if as_admin:
user_data['admin'] = True
user_data['annex_key_sent'] = False
if DEBUG: print "creating %s as admin!" % username
user_root = "%s.txt" % generateMD5Hash(content=username,
salt=USER_SALT)
if os.path.exists(os.path.join(USER_ROOT, user_root)):
if DEBUG: print "user already exists NOPE!"
return False
print user_data
with open(os.path.join(USER_ROOT, user_root), 'wb+') as user:
user.write(encryptUserData(user_data, password, p_salt=SALT,
iv=IV))
return True
except Exception as e:
print e
return False
def smokeTest():
if conf.getConfig("AMI", "SmokeURL") and conf.getConfig("AMI", "SmokeFile") and int(launchindex) == 0:
smokeURL = conf.getConfig("AMI", "SmokeURL")
smokeFile = conf.getConfig("AMI", "SmokeFile")
import urllib
urllib.urlretrieve (smokeURL, 'smoke.tar.gz')
log = exe('tar xvf smoke.tar.gz', True)
os.chdir('/home/ubuntu/smoke')
log += "-----------------------------------------------------" + "\n"
log += "-------------------- SMOKE TESTS --------------------" + "\n"
log += "-----------------------------------------------------" + "\n"
log += "Retrieved: " + smokeURL + "\n"
log += "Executing: " + smokeFile + "\n"
log += "\n"
with open(smokeFile, 'r') as f:
log += f.read() + "\n"
log += "-----------------------------------------------------" + "\n"
log += exe('sudo chmod +x ' + smokeFile, True)
log += exe('./' + smokeFile, True)
log += "-----------------------------------------------------" + "\n"
log += "--------------------- END TESTS ---------------------" + "\n"
log += "-----------------------------------------------------" + "\n"
os.chdir('/home/ubuntu/')
emailReport('SMOKE-TESTS ::: ' + smokeFile + ' ::: ' + publichostname, log)
def getPkgOpen(subpkgs):
"""获取open 模块
模块地址 http://ufo.sogou-inc.com/git/open.git
"""
targetfolder = os.path.join( conf.getConfig()['path'] , 'static' , 'js' )
utils.createfolder(TEMP_FOLDER)
if not os.path.exists( targetfolder ):
utils.createfolder(targetfolder)
subpkgs = subpkgs or conf.getConfig()['PKG_OPEN']
subpkgs.insert(0 , 'common')
os.system('git clone http://ufo.sogou-inc.com/git/open.git ' + os.path.join(TEMP_FOLDER , 'open'))
successpkg = []
for pkg in subpkgs:
source = os.path.join( TEMP_FOLDER , 'open' , pkg )
if not os.path.exists(source):
log.warn('Sub package ' + pkg + ' not exist in Open.')
continue
utils.copyfiles( source , os.path.join( targetfolder , 'open' , pkg ) )
successpkg.append( pkg )
utils.removefolder(TEMP_FOLDER)
log.success( 'Adding Open package include ' + ','.join(successpkg) + ' success!' )
def compileHTML(filepath, needCompress):
"""编译html文件
Arguments:
- `filepath`:
"""
if not os.path.exists(filepath):
return False
tpl = utils.readfile(filepath)
log.log('Compile for ' + filepath + '.')
LINK_TOKEN = '<link.* href=[\'"](?!http|https)(.*?\.css)[\'"]' # 不包含以http/https开头的资源
SCRIPT_TOKEN = '<script.* src=[\'"](?!http|https)(.*?\.js)[\'"]'
iters = re.finditer(LINK_TOKEN, tpl)
for i in reversed(list(iters)):
path = i.group(1)
path = compileCommon(path, 'local', True) #内部可能有替换的变量
if not conf.getConfig().get('disableAutoTimestamp'):
tpl = tpl[0:i.start(1)] + i.group(1) + '?t=' + getFileTimeStamp(
path, filepath) + tpl[i.end(1):]
iters = re.finditer(SCRIPT_TOKEN, tpl)
for i in reversed(list(iters)):
path = i.group(1)
path = compileCommon(path, 'local', True) #内部可能有替换的变量
if not conf.getConfig().get('disableAutoTimestamp'):
tpl = tpl[0:i.start(1)] + i.group(1) + '?t=' + getFileTimeStamp(
path, filepath) + tpl[i.end(1):]
return tpl
def do_GET(self):
"""
Arguments:
- `self`:
"""
truncate_path = self.path.split('?')[0].split('#')[0]
path_items = self.path.split('?')
query = path_items[1] if len(path_items) > 1 else ''
response = 200
serverConfig = conf.getConfig()
isInServerConfig = False
if 'proxy' in serverConfig:
for reg, target in serverConfig['proxy'].items():
if re.search(re.sub("\{.+\}$", "", reg), truncate_path):
isInServerConfig = True
if target.startswith('http'): #http url
contentType, body = self.urlProxy(
target, query, reg, truncate_path)
elif target.startswith('plugins'): #local plugins
contentType, body = self.pluginsProxy(target, query)
if not isInServerConfig:
if truncate_path == '/': #default page
body = mgr.getIndex()
response, contentType, body = (200, 'text/html', body)
elif truncate_path.endswith('.ut'): #为模版文件
tplToken = truncate_path.replace('.ut', '')[1:]
body = parser.parseTpl(tplToken)
body = parser.compileCommon(body, 'local', True)
body = parser.compilePlugin(tplToken, body)
if conf.getConfig().get(
'type') and conf.getConfig()['type'] == 'mobile':
body = body.replace(
'http://p0.123.sogou.com/u/js/mursa.js',
'http://ufo.sogou-inc.com/cdn/js/mursa-debug.js'
) #mobile project
if len(body):
response, contentType = (200, 'text/html')
else:
response, contentType, body = (404, 'text/html',
'no template called ' +
tplToken)
elif truncate_path.endswith('.m'): #为模版管理
tplToken = truncate_path.replace('.m', '')[1:]
tpl = parser.parseTpl(tplToken)
if len(tpl):
body = mgr.getPage(tplToken)
response, contentType, body = (200, 'text/html', body)
else:
response, contentType, body = (404, 'text/html',
'Error finding tpl file.')
else:
response, contentType, body = self.server_static(truncate_path)
self.sendResponseWithOutput(response, contentType, body)
def copyfiles(ptype):
"""拷贝库文件到目标文件
"""
pfolder = 'project' if ptype == 'pc' else 'mproject'
base = os.path.join(conf.getConfig()['base'] , 'assets' , pfolder )
utils.copyfiles(base , conf.getConfig()['path'])
def amiErrorHandling():
if conf.getConfig("AMI", "Error"):
print
print conf.getConfig("AMI", "Error")
print
print "Please visit http://datastax.com/ami for this AMI's feature set."
print
sys.exit(1)
def copyfiles(ptype):
"""拷贝库文件到目标文件
"""
pfolder = 'project' if ptype == 'pc' else 'mproject'
base = os.path.join(conf.getConfig()['base'], 'assets', pfolder)
utils.copyfiles(base, conf.getConfig()['path'])
def processPDFMetadata(uv_task):
task_tag = "PDF METADATA EXTRACTION"
print "\n\n************** %s [START] ******************\n" % task_tag
print "extracting text from pdf at %s" % uv_task.doc_id
uv_task.setStatus(302)
from lib.Worker.Models.cp_pdf import CompassPDF
from conf import DEBUG
from vars import ASSET_TAGS
pdf = CompassPDF(_id=uv_task.doc_id)
if pdf is None:
print "PDF IS NONE"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
uv_task.fail()
return
import os
from conf import ANNEX_DIR, getConfig
from fabric.api import local, settings
with settings(warn_only=True):
peepdf_raw = local("%s %s -s %s" % (
getConfig('compass.peepdf.root'), os.path.join(ANNEX_DIR, pdf.file_name),
getConfig('compass.peepdf.batch')), capture=True)
if peepdf_raw is None:
print "METADATA COULD NOT BE GENERATED"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
uv_task.fail()
return
import re
peepdf = []
for line in peepdf_raw.splitlines():
if line != "":
peepdf.append(re.compile("\033\[[0-9;]+m").sub("", line))
# save to asset, next task: compile metadata
md_file = pdf.addAsset("\n".join(peepdf), "%s.peeped" % pdf.file_name)
if md_file is None or not pdf.addFile(md_file, None, sync=True):
print "METADATA COULD NOT BE ADDED"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
uv_task.fail()
return
pdf.addCompletedTask(uv_task.task_path)
uv_task.routeNext(inflate={
'md_file' : "%s.peeped" % pdf.file_name,
'md_namespace' : "PDF"
})
print "\n\n************** %s [END] ******************\n" % task_tag
uv_task.finish()
def do_GET(self):
"""
Arguments:
- `self`:
"""
truncate_path = self.path.split('?')[0].split('#')[0]
path_items = self.path.split('?')
query = path_items[1] if len(path_items) > 1 else ''
response = 200
serverConfig = conf.getConfig()
isInServerConfig = False
if 'proxy' in serverConfig:
for reg,target in serverConfig['proxy'].items():
if re.search(re.sub( "\{.+\}$" , "" , reg ) , truncate_path):
isInServerConfig = True
if target.startswith('http'):#http url
contentType,body =self.urlProxy(target , query, reg , truncate_path)
elif target.startswith('plugins'):#local plugins
contentType , body = self.pluginsProxy(target , query)
if not isInServerConfig:
if truncate_path == '/':#default page
body = mgr.getIndex()
response,contentType,body = (200 , 'text/html' , body)
elif truncate_path.endswith('.ut'):#为模版文件
tplToken = truncate_path.replace('.ut' , '') [1:]
body = parser.parseTpl(tplToken)
body = parser.compileCommon(body , 'local' , True)
body = parser.compilePlugin(tplToken,body)
if conf.getConfig().get('type') and conf.getConfig()['type'] == 'mobile':
body = body.replace('http://p0.123.sogou.com/u/js/mursa.js' , 'http://ufo.sogou-inc.com/cdn/js/mursa-debug.js') #mobile project
if len(body):
response,contentType = (200 , 'text/html')
else:
response,contentType,body = (404 , 'text/html' , 'no template called ' + tplToken)
elif truncate_path.endswith('.m'):#为模版管理
tplToken = truncate_path.replace('.m' , '')[1:]
tpl = parser.parseTpl(tplToken)
if len(tpl):
body = mgr.getPage(tplToken)
response,contentType,body = (200,'text/html' , body)
else:
response,contentType,body = (404,'text/html' , 'Error finding tpl file.')
else:
response, contentType, body = self.server_static(truncate_path)
self.sendResponseWithOutput(response , contentType , body)
def loginUser(self, username, password, handler):
try:
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
from base64 import b64encode
try:
from Utils.funcs import decryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData
try:
user_root = "%s.txt" % generateMD5Hash(content=username,
salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read(), password, p_salt=SALT)
if user_data is None: return None
try:
if user_data['admin']:
del user_data['admin']
handler.set_secure_cookie(UnveillanceCookie.ADMIN,
"true",
path="/",
expires_days=1)
if not self.do_get_drive_status():
self.initDriveClient()
if "annex_key_sent" not in user_data.keys(
) or not user_data['annex_key_sent']:
if self.drive_client.checkAnnexKeyStatus():
user_data['annex_key_sent'] = True
except KeyError as e:
if DEBUG: print e
pass
handler.set_secure_cookie(UnveillanceCookie.USER,
b64encode(json.dumps(user_data)),
path="/",
expires_days=1)
return user_data
except Exception as e:
if DEBUG: print e
return None
def logoutUser(self, credentials, handler):
handler.clear_cookie(UnveillanceCookie.USER)
handler.clear_cookie(UnveillanceCookie.ADMIN)
try:
username = credentials['username']
except KeyError as e:
return None
try:
password = credentials['password']
except KeyError as e:
return True
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
from Utils.funcs import decryptUserData, encryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData
user_root = "%s.txt" % generateMD5Hash(content=username,
salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read, password, p_salt=SALT)
if user_data is None: return None
new_data = copy.deepcopy(user_data)
new_data['saved_searches'] = credentials['save_data'][
'saved_searches']
try:
new_data['annex_key_sent'] = credentials['save_data'][
'annex_key_sent']
except KeyError as e:
pass
with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD:
UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT))
return True
return None
def startServer(self):
from fabric.api import local, settings
server_path = getConfig('nlp_server.path')
cmd = "python %s -S %s -p %d" % (
os.path.join(server_path, "corenlp", "corenlp.py"),
os.path.join(server_path, getConfig('nlp_server.pkg')),
getConfig('nlp_server.port'))
if DEBUG:
print "STARTING NLP SERVER:"
print cmd
with settings(warn_only=True):
start_cmd = local(cmd)
def server_static(self,file_path):
file_path = '.' + file_path
if not os.path.exists(file_path):
return (404, 'text/html', 'no such file.')
if os.path.isfile(file_path):
stat_result = os.stat(file_path)
mime_type, encoding = mimetypes.guess_type(file_path)
if not mime_type or (mime_type.find('image')==0 or mime_type.find('flash')!=-1 or mime_type.find('audio')!=-1):
f = open(file_path , 'rb')
fcontent = f.read()
else:
f = codecs.open(file_path, "rb" , conf.getConfig()['encoding'])
fcontent = parser.compileCommon(f.read() , 'local' , True)
try:
return (200, mime_type, fcontent)
finally:
f.close()
elif os.path.isdir(file_path):
if file_path.endswith('/'):
index_file = os.path.join(file_path, 'index.html')
if os.path.exists(index_file):
return (200, 'text/html', open(index_file).read())
else:
return (200 , 'text/html; charset=utf-8' , self.list_directory(os.path.abspath(file_path)).read().encode('utf-8')[150:])
else:
return (301, 'text/html', file_path + '/')
else:
pass
def iterate(self):
THIS_DIR = os.getcwd()
os.chdir(os.path.join(ANNEX_DIR, self.base_path))
try:
iter_num = len(self.getAssetsByTagName(ASSET_TAGS['DLXDD_DD']))
bc = BatCountry(os.path.join(getConfig('caffe_root'), "models", "bvlc_googlenet"))
img = bc.dream(np.float32(self.get_image(file_name="dream_%d.jpg" % iter_num)))
bc.cleanup()
os.chdir(THIS_DIR)
iter_num += 1
dream = Image.fromarray(np.uint8(img))
asset_path = self.addAsset(None, "dream_%d.jpg" % iter_num, \
tags=[ASSET_TAGS['DLXDD_DD']], description="deep dream iteration")
if asset_path is not None:
dream.save(os.path.join(ANNEX_DIR, asset_path))
return True
except Exception as e:
print "ERROR ON ITERATION:"
print e, type(e)
return False
def iterate(self):
THIS_DIR = os.getcwd()
os.chdir(os.path.join(ANNEX_DIR, self.base_path))
try:
iter_num = len(self.getAssetsByTagName(ASSET_TAGS['DLXDD_DD']))
bc = BatCountry(
os.path.join(getConfig('caffe_root'), "models",
"bvlc_googlenet"))
img = bc.dream(
np.float32(self.get_image(file_name="dream_%d.jpg" %
iter_num)))
bc.cleanup()
os.chdir(THIS_DIR)
iter_num += 1
dream = Image.fromarray(np.uint8(img))
asset_path = self.addAsset(None, "dream_%d.jpg" % iter_num, \
tags=[ASSET_TAGS['DLXDD_DD']], description="deep dream iteration")
if asset_path is not None:
dream.save(os.path.join(ANNEX_DIR, asset_path))
return True
except Exception as e:
print "ERROR ON ITERATION:"
print e, type(e)
return False
def server_static(self, file_path):
file_path = '.' + file_path
if not os.path.exists(file_path):
return (404, 'text/html', 'no such file.')
if os.path.isfile(file_path):
stat_result = os.stat(file_path)
mime_type, encoding = mimetypes.guess_type(file_path)
if not mime_type or (mime_type.find('image') == 0
or mime_type.find('flash') != -1
or mime_type.find('audio') != -1):
f = open(file_path, 'rb')
fcontent = f.read()
else:
f = codecs.open(file_path, "rb", conf.getConfig()['encoding'])
fcontent = parser.compileCommon(f.read(), 'local', True)
try:
return (200, mime_type, fcontent)
finally:
f.close()
elif os.path.isdir(file_path):
if file_path.endswith('/'):
index_file = os.path.join(file_path, 'index.html')
if os.path.exists(index_file):
return (200, 'text/html', open(index_file).read())
else:
return (200, 'text/html; charset=utf-8',
self.list_directory(os.path.abspath(
file_path)).read().encode('utf-8')[150:])
else:
return (301, 'text/html', file_path + '/')
else:
pass
def pullFromAnnex(uv_task):
task_tag = "PULL FROM ANNEX"
print "\n\n************** %s [START] ******************\n" % task_tag
print "pulling file from document %s from annex" % uv_task.doc_id
print uv_task.emit()
uv_task.setStatus(302)
from lib.Worker.Models.uv_document import UnveillanceDocument
from conf import DEBUG, BASE_DIR, getConfig
document = UnveillanceDocument(_id=uv_task.doc_id)
if document is None:
print "DOC IS NONE"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
uv_task.fail()
return
if not document.getFile(document.file_name):
print "NO FILE CONTENT"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
uv_task.fail()
return
if hasattr(uv_task, "atttempt_sync") and uv_task.attempt_sync:
print "SHOULD ATTEMPT SYNC AGAIN."
from fabric.api import settings, local
with settings(warn_only=True):
local("%s %s %s" % (getConfig('python_home'),
os.path.join(BASE_DIR, "sync_file.py"), document.file_name))
uv_task.finish()
print "\n\n************** %s [END] ******************\n" % task_tag
def compileCommon(filepath, token, force=False):
"""通用编译方法
编译 @tm:file_path@为6位时间戳
Arguments:
- `content`:
"""
if force:
content = filepath
else:
if not os.path.exists(filepath):
return False
ftype = filepath.split('.')[-1]
if not ftype in ['html', 'htm', 'css', 'js', 'tpl', 'jsp']:
return False
content = utils.readfile(filepath)
TM_TOKEN = '@tm:(.*?)@'
DATE_TOKEN = '@date@'
COMMON_TOKEN = '@(.*?)@'
iters = re.finditer(TM_TOKEN, content)
for i in reversed(list(iters)):
content = content[0:i.start(0)] + getFileTimeStamp(
i.group(1), filepath) + content[i.end(0):]
iters = re.finditer(DATE_TOKEN, content)
for i in reversed(list(iters)):
content = content[0:i.start(0)] + utils.getDate() + content[i.end(0):]
iters = re.finditer(COMMON_TOKEN, content)
for i in reversed(list(iters)):
config = conf.getConfig()
name = i.group(1)
value = (token and config[token].get(name)) or config.get(name)
if value:
if value.find('{num}') != -1:
num = (token and
config[token].get('num')) or config.get('num') or '10'
num = range(num + 1)
substr100 = content[i.end(0):i.end(0) + 100]
istimestamp = substr100.find('t=')
if istimestamp != -1: #has timestamp
try:
tm = int(substr100[istimestamp + 2:istimestamp + 3])
except ValueError:
continue
if tm >= len(num):
tm = tm - len(num)
value = value.replace('{num}', str(tm))
else:
global range_item
value = value.replace('{num}', str(num[range_item]))
range_item = range_item + 1
if range_item >= len(num):
range_item = 0
content = content[0:i.start(0)] + value + content[i.end(0):]
return content
def buildGensimDictionary(uv_task):
task_tag = "BUILDING GENSIM DICTIONARY!!!"
print "\n\n************** %s [START] ******************\n" % task_tag
uv_task.setStatus(302)
import os
from conf import DEBUG, getConfig
dictionary_dir = getConfig('compass.gensim.training_data')
wiki_bow_corpus = os.path.join(dictionary_dir, "wiki_en_bow.mm")
wiki_dict = os.path.join(dictionary_dir, "wiki_en_wordids.txt.bz2")
wiki_tfidf = os.path.join(dictionary_dir, "wiki_en_tfidf.mm.bz2")
for required in [wiki_dict, wiki_bow_corpus, wiki_tfidf]:
if not os.path.exists(required):
print "\n\n************** %s [WARNING] ******************\n" % task_tag
print "THIS COULD TAKE AWHILE (LIKE, 10+ HOURS)..."
from fabric.api import settings, local
os.chdir(dictionary_dir)
gensim_dict_raw = os.path.join(dictionary_dir, "enwiki-latest-pages-articles.xml.bz2")
if not os.path.exists(gensim_dict_raw):
with settings(warn_only=True):
local("wget http://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2")
with settings(warn_only=True):
local("python -m gensim.scripts.make_wiki %s %s" % (
os.path.join(dictionary_dir, "enwiki-latest-pages-articles.xml.bz2"),
os.path.join(dictionary_dir, "wiki_en")))
for b in ["wiki_en_tfidf.mm", "wiki_en_wordids.txt"]:
if os.path.exists(os.path.join(dictionary_dir, b)) and not os.path.exists(os.path.join(dictionary_dir, "%s.bz2" % b)):
with settings(warn_only=True):
local("bzip2 %s" % os.path.join(dictionary_dir, b))
import logging, bz2
from gensim import corpora, models
try:
wiki_corpus = corpora.MmCorpus(bz2.BZ2File(os.path.join(dictionary_dir, "wiki_en_tfidf.mm.bz2")))
wiki_tfidf = models.TfidfModel(wiki_corpus)
wiki_tfidf.save(os.path.join(dictionary_dir, "wiki_en_tfidf.tfidf_model"))
except Exception as e:
print "\n\n************** %s [ERROR] ******************\n" % task_tag
task.fail(message=e)
return
print "WIKI CORPUS SAVED AND SERIALIZED."
break
else:
print "Found required gensim asset %s" % required
uv_task.finish()
print "\n\n************** %s [END] ******************\n" % task_tag
def loginUser(self, username, password, handler):
try:
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
from base64 import b64encode
try:
from Utils.funcs import decryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData
try:
user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read(), password, p_salt=SALT)
if user_data is None: return None
try:
if user_data['admin']:
del user_data['admin']
handler.set_secure_cookie(UnveillanceCookie.ADMIN,
"true", path="/", expires_days=1)
if not self.do_get_drive_status():
self.initDriveClient()
if "annex_key_sent" not in user_data.keys() or not user_data['annex_key_sent']:
if self.drive_client.checkAnnexKeyStatus():
user_data['annex_key_sent'] = True
except KeyError as e:
if DEBUG: print e
pass
handler.set_secure_cookie(UnveillanceCookie.USER,
b64encode(json.dumps(user_data)), path="/", expires_days=1)
return user_data
except Exception as e:
if DEBUG: print e
return None
def writefile(filename , content):
try:
f = codecs.open(filename , 'w' , conf.getConfig()['encoding'])
f.write(content)
f.close()
except:
print filename,content
raise
def stopServer(self):
printAsLog("stopping NLP server")
stopDaemon(self.pid_file, extra_pids_port=getConfig('nlp_server.port'))
try:
del self.nlp_server
except Exception as e:
print "error stopping NLP server\n%s" % e
def callmail(self):
if self.item == "1":
code = "5"+str(self.apitype)+self.errcode+"000"
print code
policy = conf.getConfig("mobile", "policy")
templateId = conf.getConfig("mobile", "templateId")
postdata = {"code":code,"mobile":self.lis_mobile,"policy":policy,"templateId":templateId}
try:
request = Morequest(self.item,self.item,postdata)
print self.url
response = request.json_post()
print "call you--------------------------------!"
except:
print "callmail error"
def writefile(filename, content):
try:
f = codecs.open(filename, 'w', conf.getConfig()['encoding'])
f.write(content)
f.close()
except:
print filename, content
raise
def extractNEREntities(task):
task_tag = "NER ENTITY EXTRACTION"
print "\n\n************** %s [START] ******************\n" % task_tag
print "TOKENIZING TEXT DOCUMENT at %s" % task.doc_id
task.setStatus(302)
from lib.Worker.Models.uv_document import UnveillanceDocument
from conf import DEBUG
from vars import ASSET_TAGS
doc = UnveillanceDocument(_id=task.doc_id)
if doc is None:
print "DOC IS NONE"
print "\n\n************** %s [ERROR] ******************\n" % task_tag
task.fail()
return
from json import loads
try:
texts = loads(doc.loadAsset("doc_texts.json"))
except Exception as e:
print "ERROR GETTING DOC-TEXTS: %s" % e
print "\n\n************** %s [ERROR] ******************\n" % task_tag
task.fail()
return
import ner, os
from conf import getConfig
from lib.Core.Utils.funcs import cleanLine
st = ner.SocketNER(host='localhost', port=getConfig("nlp_server.port"))
entities = {}
for i, page in enumerate(texts):
if page is None: continue
lemmas = st.get_entities(cleanLine(page))
if len(lemmas.keys()) == 0: continue
for lemma_type in lemmas.keys():
entities = updateEntities(entities, lemmas[lemma_type], lemma_type, i)
#if DEBUG and i > 25: break
if len(entities.keys()) > 0:
ner_entity_path = doc.addAsset(entities, "stanford-ner_entities.json", as_literal=False,
description="Entities as per Stanford-NER Tagger (via NLTK)",
tags=[ASSET_TAGS['STANFORD_NER_ENTITIES'], ASSET_TAGS['CP_ENTITIES']])
if ner_entity_path is not None:
doc.addFile(ner_entity_path, None, sync=True)
doc.addCompletedTask(task.task_path)
task.routeNext()
print "\n\n************** %s [END] ******************\n" % task_tag
task.finish()
def giffify(self):
from fabric.api import settings, local
avi_path = None
gif_path = None
try:
avi_path = self.getAssetsByTagName(ASSET_TAGS['DLXDD_AVI'])[0]['file_name']
avi_path = os.path.join(ANNEX_DIR, self.base_path, avi_path)
except Exception as e:
pass
try:
gif_path = self.getAssetsByTagName(ASSET_TAGS['DLXDD_GIF'])[0]['file_name']
gif_path = os.path.join(ANNEX_DIR, self.base_path, gif_path)
except Exception as e:
pass
ffmpeg = getConfig('ffmpeg')
if avi_path is None:
avi_path = self.addAsset(None, "deepdream_gif.avi",
tags=[ASSET_TAGS['DLXDD_AVI']], description="avi of DeepDream gif")
avi_path = os.path.join(ANNEX_DIR, avi_path)
if gif_path is None:
gif_path = self.addAsset(None, "deepdream.gif",
tags=[ASSET_TAGS['DLXDD_GIF']], description="gif of DeepDream")
gif_path = os.path.join(ANNEX_DIR, gif_path)
avi_path_tmp = avi_path.replace(".avi", "_tmp.avi")
gif_cmds = [
"{0} -y -f image2 -i dream_%d.jpg {1}".format(ffmpeg, avi_path_tmp),
"%s -y -i %s -filter:v \"setpts=10.0*PTS\" %s" % (ffmpeg, avi_path_tmp, avi_path),
"%s -y -i %s -pix_fmt rgb24 %s" % (ffmpeg, avi_path, gif_path)
]
try:
THIS_DIR = os.getcwd()
os.chdir(os.path.join(ANNEX_DIR, self.base_path))
for cmd in gif_cmds:
with settings(warn_only=True):
c = local(cmd)
if c.return_code != 0:
print "Process failed with return code %d" % c.return_code
os.chdir(THIS_DIR)
return False
os.chdir(THIS_DIR)
return True
except Exception as e:
print "COULD NOT MAKE A GIF"
print e, type(e)
return False
def compileCommon(filepath, token, force=False):
"""通用编译方法
编译 @tm:file_path@为6位时间戳
Arguments:
- `content`:
"""
if force:
content = filepath
else:
if not os.path.exists(filepath):
return False
ftype = filepath.split(".")[-1]
if not ftype in ["html", "htm", "css", "js", "tpl", "jsp"]:
return False
content = utils.readfile(filepath)
TM_TOKEN = "@tm:(.*?)@"
DATE_TOKEN = "@date@"
COMMON_TOKEN = "@(.*?)@"
iters = re.finditer(TM_TOKEN, content)
for i in reversed(list(iters)):
content = content[0 : i.start(0)] + getFileTimeStamp(i.group(1), filepath) + content[i.end(0) :]
iters = re.finditer(DATE_TOKEN, content)
for i in reversed(list(iters)):
content = content[0 : i.start(0)] + utils.getDate() + content[i.end(0) :]
iters = re.finditer(COMMON_TOKEN, content)
for i in reversed(list(iters)):
config = conf.getConfig()
name = i.group(1)
value = (token and config[token].get(name)) or config.get(name)
if value:
if value.find("{num}") != -1:
num = (token and config[token].get("num")) or config.get("num") or "10"
num = range(num + 1)
substr100 = content[i.end(0) : i.end(0) + 100]
istimestamp = substr100.find("t=")
if istimestamp != -1: # has timestamp
try:
tm = int(substr100[istimestamp + 2 : istimestamp + 3])
except ValueError:
continue
if tm >= len(num):
tm = tm - len(num)
value = value.replace("{num}", str(tm))
else:
global range_item
value = value.replace("{num}", str(num[range_item]))
range_item = range_item + 1
if range_item >= len(num):
range_item = 0
content = content[0 : i.start(0)] + value + content[i.end(0) :]
return content
def getIndex():
base = conf.getConfig()['base']
indexTpl = os.path.join( base , 'assets' , 'mgr' , 'index.html' )
path = conf.getConfig()['path']
tpls = []
for dirpath , dirnames , filenames in os.walk( os.path.join( path , 'template' ) ):
for f in filenames:
if f.endswith('.tpl'):
tpldir = dirpath.replace( os.path.join( path , 'template' ) , '' )
url = '/'.join( [ tpldir , f.replace('.tpl' , '') ] )
tpls.extend([ url[1:] ])
body = parser.parseTpl( indexTpl , {
'tpls':tpls
} , True )
return body
def setCompassProject(self):
url = "projects.json"
projects = self.sendDCRequest(url, "get")
if projects is not None:
for project in projects['projects']:
if project['title'] == getConfig('documentcloud.proj_title'):
self.project_id = project['id']
return True
if not hasattr(self, "project_id"):
project = self.sendDCRequest(url, "post", data={
'title' : getConfig('documentcloud.proj_title')
})
if DEBUG: print "new project:\n%s\n" % project
return self.hasCompassProject()
return False
def logoutUser(self, credentials, handler):
handler.clear_cookie(UnveillanceCookie.USER)
handler.clear_cookie(UnveillanceCookie.ADMIN)
try:
username = credentials['username']
except KeyError as e: return None
try:
password = credentials['password']
except KeyError as e: return True
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
from Utils.funcs import decryptUserData, encryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData
user_root = "%s.txt" % generateMD5Hash(content=username,salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read, password, p_salt=SALT)
if user_data is None: return None
new_data = copy.deepcopy(user_data)
new_data['saved_searches'] = credentials['save_data']['saved_searches']
try:
new_data['annex_key_sent'] = credentials['save_data']['annex_key_sent']
except KeyError as e:
pass
with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD:
UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT))
return True
return None
def startServer(self):
from fabric.api import local, settings
this_dir = os.getcwd()
cmd = "java -mx1000m -cp stanford-ner.jar edu.stanford.nlp.ie.NERServer -loadClassifier classifiers/english.all.3class.distsim.crf.ser.gz -port %d -outputFormat inlineXML" % getConfig('nlp_server.port')
if DEBUG:
print "STARTING NLP SERVER:"
print cmd
os.chdir(getConfig('nlp_ner_base'))
with settings(warn_only=True):
local("kill $(lsof -t -i:%d)" % getConfig('nlp_server.port'))
start_cmd = local(cmd)
print start_cmd
startDaemon(self.log_file, self.pid_file)
while True: sleep(1)
def getStatus(self):
try:
server_port = getConfig('nlp_server.port')
r = requests.get("http://localhost:%d" % server_port)
if r.status_code == 501:
self.nlp_server = jsonrpclib.Server("http://localhost:%d" % server_port)
return True
except IOError as e: pass
return False
def setData(token , data):
"""
Arguments:
- `token`:
- `data`:
"""
fpath = BASE + token + SUFFIX
f = codecs.open(fpath , 'w' , conf.getConfig()['encoding'])
f.write( json.dumps(data , sort_keys = True , indent = 4, separators = ( ',' , ': ')) )
f.close()
def __init__(self):
self.host = conf.getConfig("database", "dbhost")
self.port = conf.getConfig("database", "dbport")
self.database = conf.getConfig("database", "dbname")
self.user = conf.getConfig("database", "dbuser")
self.passwd = conf.getConfig("database", "dbpassword")
self.charser = conf.getConfig("database", "dbcharset")
def notarize(self):
if hasattr(self, "notarized") and self.notarized:
print "ALREADY NOTARIZED: (%s)" % self.notarized
return False
if not self.get_provenance():
print "NO PROVENANCE YET"
return False
j3m = self.getAsset("j3m_raw.json", return_only="path")
if j3m is None:
print "NO RAW J3M TO NOTARIZE."
return False
from lib.Core.Utils.funcs import hashEntireFile
j3m_hash = hashEntireFile(j3m, hfunc="sha256")
if j3m_hash is None:
print "WHY DIDN'T HASH WORK?"
return False
import gnupg
from vars import ASSET_TAGS
from conf import getConfig, getSecrets
try:
gpg = gnupg.GPG(homedir=getConfig('gpg_homedir'))
gpg_pwd = getSecrets('gpg_pwd')
notarization_doc = gpg.sign(j3m_hash, \
default_key=getSecrets('org_fingerprint')[-8:], passphrase=gpg_pwd)
del gpg_pwd
if len(notarization_doc.data) == 0:
print "NO NOTARIZATION DOC"
return False
self.addAsset(notarization_doc.data, "j3m_notarized.asc", tags=[ASSET_TAGS['IC_NOTARIZE']],
description="Signed Notarization Doc, ready to publish.")
self.notarized = True
self.saveFields("notarized")
return True
except Exception as e:
print "PROBLEM SIGNING DOC:"
print e, type(e)
return False
def run(params , options):
ptype = 'mobile' if ( len(params) and params[0]=='mobile') else 'pc'
if os.listdir( conf.getConfig()['path'] ):
log.warn('Not an empty folder.\nContinue may change your exists files.\nStill Continue?')
iscontinue = utils.isyes(raw_input('(y/n):'))
if not iscontinue:
log.log('User cancel init.')
sys.exit(1)
log.log('Begin to init current folder')
copyfiles(ptype)
log.success('Init success.')
def getPage(token):
"""获取mgr的页面
Arguments:
- `token`:
"""
base = conf.getConfig()['base']
mgrTpl = os.path.join( base , 'assets' , 'mgr' , 'mgr.html')
body = parser.parseTpl(mgrTpl , {
'name':token,
'data': getRawData(token)
} , True)
return body
def compilePlugin(name, content):
plugins = conf.getConfig().get('serverplugins')
if not plugins:
return content
for plugin in plugins:
try:
pkg = __import__(plugin, globals(), locals(), ['main'], -1)
content = pkg.main(name, content)
except Exception as e:
print "Plugin " + plugin + " is invalid"
print e
content = content
return content
def compilePlugin(name, content):
plugins = conf.getConfig().get("serverplugins")
if not plugins:
return content
for plugin in plugins:
try:
pkg = __import__(plugin, globals(), locals(), ["main"], -1)
content = pkg.main(name, content)
except Exception as e:
print "Plugin " + plugin + " is invalid"
print e
content = content
return content
def readfile(filename, mode='r'):
"""
Arguments:
- `filename`:
- `encoding`:
"""
if 'b' in mode: #Binary file
f = open(filename, mode)
else:
f = codecs.open(filename, mode, conf.getConfig()['encoding'])
body = f.read()
f.close()
return body
def __init__(self,subject,content,type):
self.host=conf.getConfig("email","host")
self.port=conf.getConfig("email","port")
self.user=conf.getConfig("email","user")
self.password = conf.getConfig("email", "password")
self.sender = conf.getConfig("email","sender")
self.receivers = conf.getConfig("email","receivers").split(',')
self.subject = subject
self.content = content
self.type = type
def run(params, options):
ptype = 'mobile' if (len(params) and params[0] == 'mobile') else 'pc'
if os.listdir(conf.getConfig()['path']):
log.warn(
'Not an empty folder.\nContinue may change your exists files.\nStill Continue?'
)
iscontinue = utils.isyes(raw_input('(y/n):'))
if not iscontinue:
log.log('User cancel init.')
sys.exit(1)
log.log('Begin to init current folder')
copyfiles(ptype)
log.success('Init success.')
def compileHTML(needCompress=False, needHtml=False):
"""为所有tpl文件加上时间戳
"""
base = os.path.join(PATH, 'build', 'template')
tplfiles = []
for dirpath, dirnames, filenames in os.walk(base):
tplfiles.extend([
os.path.join(dirpath, f) for f in filenames if f.endswith('.tpl')
])
if COMPILE_FOLDER:
for dirpath, dirnames, filenames in os.walk(
os.path.join(PATH, 'build', COMPILE_FOLDER)):
tplfiles.extend([os.path.join(dirpath, f) for f in filenames])
for tpl in tplfiles:
f = parser.compileHTML(tpl, needCompress)
utils.writefile(tpl, f)
if needHtml:
log.log('Render html file.\nIt will under build folder.')
files = os.listdir(os.path.join(base))
tplfiles = []
for dirpath, dirnames, filenames in os.walk(base):
tplfiles.extend([
os.path.join(dirpath, f) for f in filenames
if f.endswith('.tpl')
])
for fname in tplfiles:
token = fname.replace(base + '/', '').replace('.tpl', '')
try: #有强行编译的需求
html = parser.parseTpl(token, isbuild=True)
if token.find('/') != -1:
subfolder = os.path.join(PATH, 'build', 'html',
token.split('/')[0])
if not os.path.exists(subfolder):
utils.createfolder(subfolder)
utils.writefile(
os.path.join(PATH, 'build', 'html', token + '.html'), html)
except Exception as e:
log.error(str(e))
if not conf.getConfig().get('html_force_output'):
raise
log.success('Render html success')
def compileCss(filepath):
if not os.path.exists(filepath):
return False
css = utils.readfile(filepath)
#@todo:正则有问题,识别url("data:image/png,base64...")之类带引号的有[email protected]
IMG_TOKEN = 'url\([\'"]?(?!data:image|about:blank|http|https)(.*?)[\'"]?\)' #[email protected]:忽略base64图片和about:blank
iters = re.finditer(IMG_TOKEN, css)
for i in reversed(list(iters)):
imgpath = i.group(1)
imgpath = compileCommon(imgpath, 'local', True) #内部可能有替换的变量
if not conf.getConfig().get('disableAutoTimestamp'
): #[email protected]:为什么http开头的不给加时间戳
css = css[0:i.end(0) - 1] + '?t=' + getFileTimeStamp(
imgpath,
filepath) + css[i.end(0) -
1:] #[email protected]:已经带?的做过识别么？
return css
def sendResponseWithOutput(self, response, contentType, body):
"""
handles both str and unicode types
"""
self.send_response(response)
self.send_header("Content-Type", contentType)
if response == 301:
self.send_header("Location", body)
if not contentType or (contentType.find('image') != 0
and contentType.find('flash') == -1
and contentType.find('audio') == -1):
try:
body = body.encode(conf.getConfig()['encoding'])
except UnicodeDecodeError:
body = body
self.send_header("Content-Length", len(body))
self.end_headers()
if response != 301:
self.wfile.write(body)
def getFileTimeStamp(fpath, parentpath=''):
"""为文件加上时间戳并返回
Arguments:
- `fpath`:
"""
if fpath.find('/') == 0:
fpath = fpath[1:]
fpath2 = os.path.join(conf.getConfig()['path'], 'build', fpath)
if not os.path.exists(fpath2) and parentpath:
parentpath = parentpath.split('/')
parentpath.pop()
fpath2 = '/'.join(parentpath) + '/' + fpath
if os.path.exists(fpath2):
f = utils.readfile(fpath2, 'rb')
m = hashlib.md5()
m.update(f)
md5 = utils.md5toInt(m.hexdigest())
return md5
return ''
def getdb():
global logger
global g_db
global g_db_lct
ct = time.time()
# 每5秒钟，重连一次DB
if g_db != None and ct - g_db_lct <= 5:
return g_db
try:
g_db = None
g_db = mydb.MyDB(conf.getConfig('db_net_assist'))
g_db_lct = ct
logger.debug('DB connect ok, [db_net_assist]')
return g_db
except:
logger.error('getdb[db_net_assist] faild, %s', str(sys.exc_info()))
g_db = None
g_db_lct = 0
return None
############################
### INITIALIZE WEB SERVER
############################
# get the current path (unix)
currentPath = os.path.dirname(os.path.abspath(__file__)) + '/'
global dbFileName
global apiUrl
global clanId
global playerId
global fileType
global apiToken
# read config file
dbFileName, apiUrl, clanId, playerId, fileType, apiToken = conf.getConfig()
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app.layout = buildHtmlDiv
app.title = 'Statistiques de clan'
############################
### CALLBACK FONCTIONS
############################
@app.callback(dash.dependencies.Output('donation-graphic', 'figure'),
[dash.dependencies.Input('member-filter', 'value')])
import json
import hashlib
import re
from jinja2 import Template, Environment, FileSystemLoader, TemplateNotFound, TemplateSyntaxError
import conf
import utils
import log
import mgr
range_item = 0
jinjaenv = Environment(loader=FileSystemLoader(
os.path.join(conf.getConfig()['path'],
conf.getConfig()['template_dir']),
conf.getConfig()['encoding']),
extensions=["jinja2.ext.do"],
autoescape=True)
build_jinjaenv = Environment(loader=FileSystemLoader(
os.path.join(conf.getConfig()['path'], 'build',
conf.getConfig()['template_dir']),
conf.getConfig()['encoding']))
def parseTpl(token, data={}, noGetTpl=False, isbuild=False):
"""
"""
if not noGetTpl:
tpl = token + '.tpl'
def __init__(self,item,apitype,errcode,lis_mobile):
self.item = item
self.errcode = errcode
self.apitype = apitype
self.url = conf.getConfig("mobile","url")
self.lis_mobile = lis_mobile
def giffify(self):
from fabric.api import settings, local
avi_path = None
gif_path = None
try:
avi_path = self.getAssetsByTagName(
ASSET_TAGS['DLXDD_AVI'])[0]['file_name']
avi_path = os.path.join(ANNEX_DIR, self.base_path, avi_path)
except Exception as e:
pass
try:
gif_path = self.getAssetsByTagName(
ASSET_TAGS['DLXDD_GIF'])[0]['file_name']
gif_path = os.path.join(ANNEX_DIR, self.base_path, gif_path)
except Exception as e:
pass
ffmpeg = getConfig('ffmpeg')
if avi_path is None:
avi_path = self.addAsset(None,
"deepdream_gif.avi",
tags=[ASSET_TAGS['DLXDD_AVI']],
description="avi of DeepDream gif")
avi_path = os.path.join(ANNEX_DIR, avi_path)
if gif_path is None:
gif_path = self.addAsset(None,
"deepdream.gif",
tags=[ASSET_TAGS['DLXDD_GIF']],
description="gif of DeepDream")
gif_path = os.path.join(ANNEX_DIR, gif_path)
avi_path_tmp = avi_path.replace(".avi", "_tmp.avi")
gif_cmds = [
"{0} -y -f image2 -i dream_%d.jpg {1}".format(
ffmpeg, avi_path_tmp),
"%s -y -i %s -filter:v \"setpts=10.0*PTS\" %s" %
(ffmpeg, avi_path_tmp, avi_path),
"%s -y -i %s -pix_fmt rgb24 %s" % (ffmpeg, avi_path, gif_path)
]
try:
THIS_DIR = os.getcwd()
os.chdir(os.path.join(ANNEX_DIR, self.base_path))
for cmd in gif_cmds:
with settings(warn_only=True):
c = local(cmd)
if c.return_code != 0:
print "Process failed with return code %d" % c.return_code
os.chdir(THIS_DIR)
return False
os.chdir(THIS_DIR)
return True
except Exception as e:
print "COULD NOT MAKE A GIF"
print e, type(e)
return False
def __init__(self):
self.__name = conf.getConfig("contacts","name")
self.__group = conf.getConfig("contacts","group")
self.__email = conf.getConfig("contacts","email")
self.mobile = conf.getConfig("contacts","mobile")
import sys
import os
import re
import codecs
import json
import urllib2
import urllib
import conf
import log
import uparser as parser
import utils
import mgr
import socket
sys.path.append(conf.getConfig()['path']) # add local plugins
class PrHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
"""
"""
def sendResponseWithOutput(self, response, contentType, body):
"""
handles both str and unicode types
"""
self.send_response(response)
self.send_header("Content-Type", contentType)
if response == 301:
self.send_header("Location", body)
if not contentType or (contentType.find('image') != 0
