def createSslRsaCert(self):
import subprocess
if os.path.isfile(
"%s/cert-rsa.pem" % config.data_dir) and os.path.isfile(
"%s/key-rsa.pem" % config.data_dir):
return True # Files already exits
proc = subprocess.Popen(
"%s req -x509 -newkey rsa:2048 -sha256 -batch -keyout %s -out %s -nodes -config %s"
% helper.shellquote(self.openssl_bin,
config.data_dir + "/key-rsa.pem",
config.data_dir + "/cert-rsa.pem",
self.openssl_env["OPENSSL_CONF"]),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip()
proc.wait()
logging.debug("Generating RSA cert and key PEM files...%s" % back)
if os.path.isfile(
"%s/cert-rsa.pem" % config.data_dir) and os.path.isfile(
"%s/key-rsa.pem" % config.data_dir):
return True
else:
logging.error(
"RSA ECC SSL cert generation failed, cert or key files not exist."
)
return False
def findCoffeescriptCompiler():
coffeescript_compiler = None
try:
import distutils.spawn
coffeescript_compiler = helper.shellquote(distutils.spawn.find_executable("coffee")) + " --no-header -p"
except:
pass
if coffeescript_compiler:
return coffeescript_compiler
else:
return False
def findCoffeescriptCompiler():
coffeescript_compiler = None
try:
import distutils.spawn
coffeescript_compiler = helper.shellquote(distutils.spawn.find_executable("coffee")) + " --no-header -p"
except:
pass
if coffeescript_compiler:
return coffeescript_compiler
else:
return False
def testShellquote(self):
assert helper.shellquote("hel'lo") == '"hel\'lo"' # Allow '
assert helper.shellquote('hel"lo') == '"hello"' # Remove "
assert helper.shellquote("hel'lo", 'hel"lo') == ('"hel\'lo"', '"hello"')
def createSslRsaCert(self):
casubjects = [
"/C=US/O=Amazon/OU=Server CA 1B/CN=Amazon",
"/C=US/O=Let's Encrypt/CN=Let's Encrypt Authority X3",
"/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 High Assurance Server CA",
"/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA"
]
fakedomains = [
"yahoo.com", "amazon.com", "live.com", "microsoft.com", "mail.ru",
"csdn.net", "bing.com", "amazon.co.jp", "office.com", "imdb.com",
"msn.com", "samsung.com", "huawei.com", "ztedevices.com",
"godaddy.com", "w3.org", "gravatar.com", "creativecommons.org",
"hatena.ne.jp", "adobe.com", "opera.com", "apache.org",
"rambler.ru", "one.com", "nationalgeographic.com",
"networksolutions.com", "php.net", "python.org", "phoca.cz",
"debian.org", "ubuntu.com", "nazwa.pl", "symantec.com"
]
self.openssl_env['CN'] = random.choice(fakedomains)
if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem):
return True # Files already exits
import subprocess
# Generate CAcert and CAkey
cmd = "%s req -new -newkey rsa:2048 -days 3650 -nodes -x509 -subj %s -keyout %s -out %s -batch -config %s" % helper.shellquote(
self.openssl_bin,
random.choice(casubjects),
self.cakey_pem,
self.cacert_pem,
self.openssl_env["OPENSSL_CONF"],
)
proc = subprocess.Popen(cmd.encode(sys.getfilesystemencoding()),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip()
proc.wait()
logging.debug("Generating RSA CAcert and CAkey PEM files...%s" % back)
if not (os.path.isfile(self.cacert_pem)
and os.path.isfile(self.cakey_pem)):
logging.error(
"RSA ECC SSL CAcert generation failed, CAcert or CAkey files not exist."
)
return False
# Generate certificate key and signing request
cmd = "%s req -new -newkey rsa:2048 -keyout %s -out %s -subj %s -sha256 -nodes -batch -config %s" % helper.shellquote(
self.openssl_bin,
self.key_pem,
self.cert_csr,
"/CN=" + self.openssl_env['CN'],
self.openssl_env["OPENSSL_CONF"],
)
proc = subprocess.Popen(cmd.encode(sys.getfilesystemencoding()),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip()
proc.wait()
logging.debug("Generating certificate key and signing request...%s" %
back)
# Sign request and generate certificate
cmd = "%s x509 -req -in %s -CA %s -CAkey %s -CAcreateserial -out %s -days 730 -sha256 -extensions x509_ext -extfile %s" % helper.shellquote(
self.openssl_bin,
self.cert_csr,
self.cacert_pem,
self.cakey_pem,
self.cert_pem,
self.openssl_env["OPENSSL_CONF"],
)
proc = subprocess.Popen(cmd.encode(sys.getfilesystemencoding()),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip()
proc.wait()
logging.debug("Generating RSA cert...%s" % back)
if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem):
return True
else:
logging.error(
"RSA ECC SSL cert generation failed, cert or key files not exist."
)
return False
def merge(merged_path):
merge_dir = os.path.dirname(merged_path)
s = time.time()
ext = merged_path.split(".")[-1]
if ext == "js": # If merging .js find .coffee too
find_ext = ["js", "coffee"]
else:
find_ext = [ext]
# If exist check the other files modification date
if os.path.isfile(merged_path):
merged_mtime = os.path.getmtime(merged_path)
else:
merged_mtime = 0
changed = {}
for file_path in findfiles(merge_dir, find_ext):
if os.path.getmtime(file_path) > merged_mtime + 1:
changed[file_path] = True
if not changed:
return # Assets not changed, nothing to do
if os.path.isfile(
merged_path): # Find old parts to avoid unncessary recompile
merged_old = open(merged_path, "rb").read().decode("utf8")
old_parts = {}
for match in re.findall("(/\* ---- (.*?) ---- \*/(.*?)(?=/\* ----|$))",
merged_old, re.DOTALL):
old_parts[match[1]] = match[2].strip("\n\r")
# Merge files
parts = []
s_total = time.time()
for file_path in findfiles(merge_dir, find_ext):
parts.append("\n\n/* ---- %s ---- */\n\n" %
file_path.replace(config.data_dir, ""))
if file_path.endswith(".coffee"): # Compile coffee script
if file_path in changed or file_path.replace(
config.data_dir, ""
) not in old_parts: # Only recompile if changed or its not compiled before
if config.coffeescript_compiler is None:
config.coffeescript_compiler = findCoffeescriptCompiler()
if not config.coffeescript_compiler:
logging.error(
"No coffeescript compiler defined, skipping compiling %s"
% merged_path)
return False # No coffeescript compiler, skip this file
# Replace / with os separators and escape it
file_path_escaped = helper.shellquote(
os.path.join(*file_path.split("/")))
if "%s" in config.coffeescript_compiler: # Replace %s with coffeescript file
command = config.coffeescript_compiler % file_path_escaped
else: # Put coffeescript file to end
command = config.coffeescript_compiler + " " + file_path_escaped
# Start compiling
s = time.time()
compiler = subprocess.Popen(command,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
out = compiler.stdout.read().decode("utf8")
compiler.wait()
logging.debug("Running: %s (Done in %.2fs)" %
(command, time.time() - s))
# Check errors
if out and out.startswith("("): # No error found
parts.append(out)
else: # Put error message in place of source code
error = out
logging.error("%s Compile error: %s" % (file_path, error))
parts.append("alert('%s compile error: %s');" %
(file_path, re.escape(error).replace(
"\n", "\\n").replace(r"\\n", r"\n")))
else: # Not changed use the old_part
parts.append(old_parts[file_path.replace(config.data_dir, "")])
else: # Add to parts
parts.append(open(file_path).read().decode("utf8"))
merged = u"\n".join(parts)
if ext == "css": # Vendor prefix css
from lib.cssvendor import cssvendor
merged = cssvendor.prefix(merged)
merged = merged.replace("\r", "")
open(merged_path, "wb").write(merged.encode("utf8"))
logging.debug("Merged %s (%.2fs)" % (merged_path, time.time() - s_total))
def createSslRsaCert(self):
if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir):
return True # Files already exits
import subprocess
proc = subprocess.Popen(
"%s req -x509 -newkey rsa:2048 -sha256 -batch -keyout %s -out %s -nodes -config %s" % helper.shellquote(
self.openssl_bin,
config.data_dir+"/key-rsa.pem",
config.data_dir+"/cert-rsa.pem",
self.openssl_env["OPENSSL_CONF"]
),
shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=self.openssl_env
)
back = proc.stdout.read().strip()
proc.wait()
logging.debug("Generating RSA cert and key PEM files...%s" % back)
if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir):
return True
else:
logging.error("RSA ECC SSL cert generation failed, cert or key files not exist.")
return False
def merge(merged_path):
merge_dir = os.path.dirname(merged_path)
s = time.time()
ext = merged_path.split(".")[-1]
if ext == "js": # If merging .js find .coffee too
find_ext = ["js", "coffee"]
else:
find_ext = [ext]
# If exist check the other files modification date
if os.path.isfile(merged_path):
merged_mtime = os.path.getmtime(merged_path)
else:
merged_mtime = 0
changed = {}
for file_path in findfiles(merge_dir, find_ext):
if os.path.getmtime(file_path) > merged_mtime:
changed[file_path] = True
if not changed:
return # Assets not changed, nothing to do
if os.path.isfile(merged_path): # Find old parts to avoid unncessary recompile
merged_old = open(merged_path, "rb").read().decode("utf8")
old_parts = {}
for match in re.findall("(/\* ---- (.*?) ---- \*/(.*?)(?=/\* ----|$))", merged_old, re.DOTALL):
old_parts[match[1]] = match[2].strip("\n\r")
# Merge files
parts = []
s_total = time.time()
for file_path in findfiles(merge_dir, find_ext):
parts.append("\n\n/* ---- %s ---- */\n\n" % file_path)
if file_path.endswith(".coffee"): # Compile coffee script
if file_path in changed or file_path not in old_parts: # Only recompile if changed or its not compiled before
if config.coffeescript_compiler is None:
config.coffeescript_compiler = findCoffeescriptCompiler()
if not config.coffeescript_compiler:
logging.error("No coffeescript compiler definied, skipping compiling %s" % merged_path)
return False # No coffeescript compiler, skip this file
# Replace / with os separators and escape it
file_path_escaped = helper.shellquote(os.path.join(*file_path.split("/")))
if "%s" in config.coffeescript_compiler: # Replace %s with coffeescript file
command = config.coffeescript_compiler % file_path_escaped
else: # Put coffeescript file to end
command = config.coffeescript_compiler + " " + file_path_escaped
# Start compiling
s = time.time()
compiler = subprocess.Popen(command, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
out = compiler.stdout.read().decode("utf8")
compiler.wait()
logging.debug("Running: %s (Done in %.2fs)" % (command, time.time() - s))
# Check errors
if out and out.startswith("("): # No error found
parts.append(out)
else: # Put error message in place of source code
error = out
logging.error("%s Compile error: %s" % (file_path, error))
parts.append(
"alert('%s compile error: %s');" %
(file_path, re.escape(error).replace("\n", "\\n").replace(r"\\n", r"\n"))
)
else: # Not changed use the old_part
parts.append(old_parts[file_path])
else: # Add to parts
parts.append(open(file_path).read().decode("utf8"))
merged = u"\n".join(parts)
if ext == "css": # Vendor prefix css
from lib.cssvendor import cssvendor
merged = cssvendor.prefix(merged)
merged = merged.replace("\r", "")
open(merged_path, "wb").write(merged.encode("utf8"))
logging.debug("Merged %s (%.2fs)" % (merged_path, time.time() - s_total))
def testShellquote(self):
assert helper.shellquote("hel'lo") == "\"hel'lo\"" # Allow '
assert helper.shellquote('hel"lo') == '"hello"' # Remove "
assert helper.shellquote("hel'lo",
'hel"lo') == ('"hel\'lo"', '"hello"')
def createSslRsaCert(self):
casubjects = [
"/C=US/O=Amazon/OU=Server CA 1B/CN=Amazon",
"/C=US/O=Let's Encrypt/CN=Let's Encrypt Authority X3",
"/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 High Assurance Server CA",
"/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA"
]
self.openssl_env['CN'] = random.choice(self.fakedomains)
if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem):
self.createSslContexts()
return True # Files already exits
import subprocess
# Replace variables in config template
conf_template = open(self.openssl_conf_template).read()
conf_template = conf_template.replace("$ENV::CN",
self.openssl_env['CN'])
open(self.openssl_conf, "w").write(conf_template)
# Generate CAcert and CAkey
cmd_params = helper.shellquote(self.openssl_bin, self.openssl_conf,
random.choice(casubjects),
self.cakey_pem, self.cacert_pem)
cmd = "%s req -new -newkey rsa:2048 -days 3650 -nodes -x509 -config %s -subj %s -keyout %s -out %s -batch" % cmd_params
self.log.debug("Generating RSA CAcert and CAkey PEM files...")
self.log.debug("Running: %s" % cmd)
proc = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip().decode(errors="replace").replace(
"\r", "")
proc.wait()
if not (os.path.isfile(self.cacert_pem)
and os.path.isfile(self.cakey_pem)):
self.log.error(
"RSA ECC SSL CAcert generation failed, CAcert or CAkey files not exist. (%s)"
% back)
return False
else:
self.log.debug("Result: %s" % back)
# Generate certificate key and signing request
cmd_params = helper.shellquote(
self.openssl_bin,
self.key_pem,
self.cert_csr,
"/CN=" + self.openssl_env['CN'],
self.openssl_conf,
)
cmd = "%s req -new -newkey rsa:2048 -keyout %s -out %s -subj %s -sha256 -nodes -batch -config %s" % cmd_params
self.log.debug("Generating certificate key and signing request...")
proc = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip().decode(errors="replace").replace(
"\r", "")
proc.wait()
self.log.debug("Running: %s\n%s" % (cmd, back))
# Sign request and generate certificate
cmd_params = helper.shellquote(self.openssl_bin, self.cert_csr,
self.cacert_pem, self.cakey_pem,
self.cert_pem, self.openssl_conf)
cmd = "%s x509 -req -in %s -CA %s -CAkey %s -set_serial 01 -out %s -days 730 -sha256 -extensions x509_ext -extfile %s" % cmd_params
self.log.debug("Generating RSA cert...")
proc = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
env=self.openssl_env)
back = proc.stdout.read().strip().decode(errors="replace").replace(
"\r", "")
proc.wait()
self.log.debug("Running: %s\n%s" % (cmd, back))
if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem):
self.createSslContexts()
# Remove no longer necessary files
os.unlink(self.openssl_conf)
os.unlink(self.cacert_pem)
os.unlink(self.cakey_pem)
os.unlink(self.cert_csr)
return True
else:
self.log.error(
"RSA ECC SSL cert generation failed, cert or key files not exist."
)