def test_read_file_displays_errors(self, mock_open_file,
mock_print_warning):
mock_open_file.side_effect = IOError()
fileutils.read_file(self.fake_path,
continue_on_error=True,
display_errors=True)
self.assertTrue(mock_print_warning.called)
def test_read_file_exits_code_2_on_unicodedecodeerror(
self, mock_open_file):
fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes',
0, 1, 'testing only')
mock_open_file.return_value.__enter__(
).read.side_effect = fake_decode_error
with self.assertRaises(SystemExit) as context:
fileutils.read_file(self.fake_path)
self.assertEqual(context.exception.code, 2)
def run_controller_in(controller, function, environment):
"""
Runs the controller.function() (for the app specified by
the current folder).
It tries pre-compiled controller_function.pyc first before compiling it.
"""
# if compiled should run compiled!
folder = environment["request"].folder
path = pjoin(folder, "compiled")
badc = "invalid controller (%s/%s)" % (controller, function)
badf = "invalid function (%s/%s)" % (controller, function)
if os.path.exists(path):
filename = pjoin(path, "controllers_%s_%s.pyc" % (controller, function))
if not os.path.exists(filename):
raise HTTP(404, rewrite.thread.routes.error_message % badf, web2py_error=badf)
restricted(read_pyc(filename), environment, layer=filename)
elif function == "_TEST":
# TESTING: adjust the path to include site packages
from settings import global_settings
from admin import abspath, add_path_first
paths = (global_settings.gluon_parent, abspath("site-packages", gluon=True), abspath("gluon", gluon=True), "")
[add_path_first(path) for path in paths]
# TESTING END
filename = pjoin(folder, "controllers/%s.py" % controller)
if not os.path.exists(filename):
raise HTTP(404, rewrite.thread.routes.error_message % badc, web2py_error=badc)
environment["__symbols__"] = environment.keys()
code = read_file(filename)
code += TEST_CODE
restricted(code, environment, layer=filename)
else:
filename = pjoin(folder, "controllers/%s.py" % controller)
if not os.path.exists(filename):
raise HTTP(404, rewrite.thread.routes.error_message % badc, web2py_error=badc)
code = read_file(filename)
exposed = regex_expose.findall(code)
if not function in exposed:
raise HTTP(404, rewrite.thread.routes.error_message % badf, web2py_error=badf)
code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
if is_gae:
layer = filename + ":" + function
code = getcfs(layer, filename, lambda: compile2(code, layer))
restricted(code, environment, filename)
response = environment["response"]
vars = response._vars
if response.postprocessing:
for p in response.postprocessing:
vars = p(vars)
if isinstance(vars, unicode):
vars = vars.encode("utf8")
elif hasattr(vars, "xml") and callable(vars.xml):
vars = vars.xml()
return vars
def map2alm(md, lmax, mmax=None, weights=None):
"""Determines (from whether pol_axis is set or not) whether or not to use
polarization if polarization=True. If polarization=False, treats each map
as an independent map.
"""
if mmax is None:
mmax = lmax
if weights is None:
#Try to find file based on data in md
weights = 'weight_ring_n%05d.fits' % md.nside
if isinstance(weights, str):
weights = fileutils.read_file(weights)
elif not isinstance(weights, np.ndarray):
raise TypeError("Weights must be either filename or numpy array")
if weights.shape != (3, 2*md.nside):
raise ValueError("Weights do not have the right shape")
computer = psht.PshtMmajorHealpix(nside=md.nside, lmax=lmax, mmax=mmax,
map=md.map,
alm_polarization=md.pol_axis,
alm_axis=md.pix_axis,
map_axis=md.pix_axis,
map_polarization=md.pol_axis,
weights=weights[0])
alm = computer.map2alm()
ad = almmod.AlmData(lmax, mmax=mmax, alms=alm, pol_axis=md.pol_axis,
pol_iter=md.pol_iter, ordering='m-major')
return ad
def close_index_html(index_file):
# wl_log.info('Will close %s' % index_file)
# TODO: add check to don't close a file twice
if not os.path.isfile(index_file):
fu.write_to_file(index_file, '') # create an empty file
index_src = fu.read_file(index_file)
if index_src.startswith('<html'):
wl_log.info('Index file %s already closed' % index_file)
return
scripts_src = """<script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery-1.9.1.min.js"></script>
<style type="text/css" title="currentStyle">
@import "../../js/css/demo_page.css";
@import "../../js/css/demo_table.css";
</style>
<script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery.dataTables.min.js"></script>
<script type="text/javascript" charset="utf-8">
$(document).ready(function() {
$('#results').dataTable( {
"aaSorting": [[ 2, "desc" ]]
} );
} );
</script>"""
html_str = "<html><head><meta http-equiv='Content-Type' content='text/html; charset=utf-8' />\
<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />" + scripts_src + "</head>\n<body><table id ='results'>\
\n<thead><tr><th>Rank</th><th>Domain</th><th>Fonts</th><th>OffsetWidth</th><th>OffsetHeight</th><th>FP found</th></tr></thead>" + index_src + '</table></body></html>'
fu.write_to_file(index_file, html_str)
def close_index_html(index_file):
# wl_log.info('Will close %s' % index_file)
# TODO: add check to don't close a file twice
if not os.path.isfile(index_file):
fu.write_to_file(index_file, '') # create an empty file
index_src = fu.read_file(index_file)
if index_src.startswith('<html'):
wl_log.info('Index file %s already closed' % index_file)
return
scripts_src = """<script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery-1.9.1.min.js"></script>
<style type="text/css" title="currentStyle">
@import "../../js/css/demo_page.css";
@import "../../js/css/demo_table.css";
</style>
<script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery.dataTables.min.js"></script>
<script type="text/javascript" charset="utf-8">
$(document).ready(function() {
$('#results').dataTable( {
"aaSorting": [[ 2, "desc" ]]
} );
} );
</script>"""
html_str = "<html><head><meta http-equiv='Content-Type' content='text/html; charset=utf-8' />\
<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />" + scripts_src + "</head>\n<body><table id ='results'>\
\n<thead><tr><th>Rank</th><th>Domain</th><th>Fonts</th><th>OffsetWidth</th><th>OffsetHeight</th><th>FP found</th></tr></thead>" + index_src + '</table></body></html>'
fu.write_to_file(index_file, html_str)
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment['request'].folder
c = environment['request'].controller
f = environment['request'].function
cpath = os.path.join(folder, 'compiled')
if os.path.exists(cpath):
for model in listdir(cpath, '^models_\w+\.pyc$', 0):
restricted(read_pyc(model), environment, layer=model)
path = os.path.join(cpath, 'models')
models = listdir(path, '^\w+\.pyc$',0,sort=False)
compiled=True
else:
path = os.path.join(folder, 'models')
models = listdir(path, '^\w+\.py$',0,sort=False)
compiled=False
paths = (path, os.path.join(path,c), os.path.join(path,c,f))
for model in models:
if not os.path.split(model)[0] in paths and c!='appadmin':
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model,
lambda: compile2(read_file(model), model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def getcfs(key, filename, filter=None):
"""
Caches the *filtered* file `filename` with `key` until the file is
modified.
:param key: the cache key
:param filename: the file to cache
:param filter: is the function used for filtering. Normally `filename` is a
.py file and `filter` is a function that bytecode compiles the file.
In this way the bytecode compiled file is cached. (Default = None)
This is used on Google App Engine since pyc files cannot be saved.
"""
t = os.stat(filename)[stat.ST_MTIME]
cfs_lock.acquire()
item = cfs.get(key, None)
cfs_lock.release()
if item and item[0] == t:
return item[1]
if not filter:
data = read_file(filename)
else:
data = filter()
cfs_lock.acquire()
cfs[key] = (t, data)
cfs_lock.release()
return data
def getcfs(key, filename, filter=None):
"""
Caches the *filtered* file `filename` with `key` until the file is
modified.
:param key: the cache key
:param filename: the file to cache
:param filter: is the function used for filtering. Normally `filename` is a
.py file and `filter` is a function that bytecode compiles the file.
In this way the bytecode compiled file is cached. (Default = None)
This is used on Google App Engine since pyc files cannot be saved.
"""
try:
t = os.stat(filename).st_mtime
except OSError:
return filter() if callable(filter) else ''
cfs_lock.acquire()
item = cfs.get(key, None)
cfs_lock.release()
if item and item[0] == t:
return item[1]
if not callable(filter):
data = read_file(filename)
else:
data = filter()
cfs_lock.acquire()
cfs[key] = (t, data)
cfs_lock.release()
return data
def run(
appname,
plain=False,
import_models=False,
startfile=None,
bpython=False,
python_code=False
):
"""
Start interactive shell or run Python script (startfile) in web2py
controller environment. appname is formatted like:
a web2py application name
a/c exec the controller c into the application environment
"""
(a, c, f) = parse_path_info(appname)
errmsg = 'invalid application name: %s' % appname
if not a:
die(errmsg)
adir = os.path.join('applications', a)
if not os.path.exists(adir):
if raw_input('application %s does not exist, create (y/n)?'
% a).lower() in ['y', 'yes']:
os.mkdir(adir)
w2p_unpack('welcome.w2p', adir)
for subfolder in ['models','views','controllers', 'databases',
'modules','cron','errors','sessions',
'languages','static','private','uploads']:
subpath = os.path.join(adir,subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(adir,'models/db.py')
if os.path.exists(db):
data = fileutils.read_file(db)
data = data.replace('<your secret key>','sha512:'+web2py_uuid())
fileutils.write_file(db, data)
if c:
import_models = True
_env = env(a, c=c, import_models=import_models)
if c:
cfile = os.path.join('applications', a, 'controllers', c + '.py')
if not os.path.isfile(cfile):
cfile = os.path.join('applications', a, 'compiled', "controllers_%s_%s.pyc" % (c,f))
if not os.path.isfile(cfile):
die(errmsg)
else:
exec read_pyc(cfile) in _env
else:
execfile(cfile, _env)
if f:
exec ('print %s()' % f, _env)
elif startfile:
exec_pythonrc()
try:
execfile(startfile, _env)
except RestrictedError, e:
print e.traceback
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment['request'].folder
c = environment['request'].controller
f = environment['request'].function
cpath = os.path.join(folder, 'compiled')
if os.path.exists(cpath):
for model in listdir(cpath, '^models_\w+\.pyc$', 0):
restricted(read_pyc(model), environment, layer=model)
path = os.path.join(cpath, 'models')
models = listdir(path, '^\w+\.pyc$', 0, sort=False)
compiled = True
else:
path = os.path.join(folder, 'models')
models = listdir(path, '^\w+\.py$', 0, sort=False)
compiled = False
paths = (path, os.path.join(path, c), os.path.join(path, c, f))
for model in models:
if not os.path.split(model)[0] in paths and c != 'appadmin':
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model,
lambda: compile2(read_file(model), model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def from_credentials_file(cls, filename):
"""Generates Credentials from a stored Credentials file.
The same file will be used to save the credentials when the access token is
refreshed.
Args:
filename: String, the name of a file containing JSON credentials to load.
The same filename will be used to save credentials back to disk.
Returns:
The credentials loaded from disk.
Raises:
InvalidCredentialsFileError: When the credentials file cannot be opened.
EmptyCredentialsFileError: When the provided file contains no credentials.
"""
file_content = fileutils.read_file(filename,
continue_on_error=True,
display_errors=False)
if file_content is None:
raise InvalidCredentialsFileError(
f'File {filename} could not be opened')
info = json.loads(file_content)
if not info:
raise EmptyCredentialsFileError(
f'File {filename} contains no credential data')
try:
# We read the existing data from the passed in file, but we also want to
# save future data/tokens in the same place.
return cls.from_authorized_user_info(info, filename=filename)
except ValueError as e:
raise InvalidCredentialsFileError(str(e))
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment["request"].folder
c = environment["request"].controller
f = environment["request"].function
cpath = pjoin(folder, "compiled")
if os.path.exists(cpath):
for model in listdir(cpath, "^models_\w+\.pyc$", 0):
restricted(read_pyc(model), environment, layer=model)
path = pjoin(cpath, "models")
models = listdir(path, "^\w+\.pyc$", 0, sort=False)
compiled = True
else:
path = pjoin(folder, "models")
models = listdir(path, "^\w+\.py$", 0, sort=False)
compiled = False
n = len(path) + 1
for model in models:
regex = environment["response"].models_to_run
if isinstance(regex, list):
regex = re_compile("|".join(regex))
file = model[n:].replace(os.path.sep, "/").replace(".pyc", ".py")
if not regex.search(file) and c != "appadmin":
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model, lambda: compile2(read_file(model), model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def test_add_index_html_line(self):
self.new_temp_file('index.html')
di = lp.DomainInfo()
di.log_filename = '/tmp/as.log'
lp.add_index_html_line(di)
ind_file = lp.get_index_filename_for_domain_info(di)
ind_src = fu.read_file(ind_file)
self.assertTrue('tr' in ind_src, "Cannot find tr in index.html")
def test_read_file_default_params(self, mock_open_file):
fake_content = 'some fake content'
mock_open_file.return_value.__enter__(
).read.return_value = fake_content
self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
self.assertEqual(mock_open_file.call_args[0][1], 'r')
self.assertIsNone(mock_open_file.call_args[1]['newline'])
def test_read_file_continues_on_errors_without_displaying(
self, mock_open_file, mock_print_warning):
mock_open_file.side_effect = IOError()
contents = fileutils.read_file(self.fake_path,
continue_on_error=True,
display_errors=False)
self.assertIsNone(contents)
self.assertFalse(mock_print_warning.called)
def test_add_index_html_line(self):
self.new_temp_file('index.html')
di = lp.DomainInfo()
di.log_filename = '/tmp/as.log'
lp.add_index_html_line(di)
ind_file = lp.get_index_filename_for_domain_info(di)
ind_src = fu.read_file(ind_file)
self.assertTrue('tr' in ind_src, "Cannot find tr in index.html")
def assert_all_patterns_not_in_file(self, filename, pats):
"""Assert if none of the patterns passed are included in the file."""
if type(pats) == str:
pats = (pats,)
txt = fu.read_file(filename)
for pat in pats:
res = re.findall(pat, txt)
if res:
self.fail("Should not find pattern %s in %s" % (pat, filename))
def assert_all_patterns_not_in_file(self, filename, pats):
"""Assert if none of the patterns passed are included in the file."""
if type(pats) == str:
pats = (pats, )
txt = fu.read_file(filename)
for pat in pats:
res = re.findall(pat, txt)
if res:
self.fail("Should not find pattern %s in %s" % (pat, filename))
def test_job_folder_should_be_writable(self):
out_dir = self.create_job_folder()
self.assert_(os.path.isdir(out_dir), 'Cannot create job folder')
out_file = os.path.join(out_dir, 'some.log')
file_content = '123456789'
fu.write_to_file(out_file, file_content)
self.assert_(os.path.isfile(out_file), 'Cannot create file in job folder')
self.assert_(file_content == fu.read_file(out_file), 'Cannot create file in job folder')
def test_a2m2a():
try:
ad = fileutils.read_file('eirikalms_l95.fits')
md = utils.alm2map(ad, nside=32)
except:
raise AssertionError()
try:
md = fileutils.read_file('eirikmap_n32_new.fits')
weights = fileutils.read_file('weight_ring_n00032.fits')
ad = utils.map2alm(md, lmax = 95, mmax = 95, weights=weights)
except:
raise AssertionError()
try:
#Should support giving filenames for the weights
md = fileutils.read_file('eirikmap_n32_new.fits')
weights = 'weight_ring_n00032.fits'
ad = utils.map2alm(md, lmax=95, mmax=95, weights=weights)
except:
raise AssertionError()
def read_pyc(filename):
"""
Read the code inside a bytecode compiled file if the MAGIC number is
compatible
:returns: a code object
"""
data = read_file(filename, 'rb')
if not is_gae and data[:4] != imp.get_magic():
raise SystemError('compiled code is incompatible')
return marshal.loads(data[8:])
def read_pyc(filename):
"""
Read the code inside a bytecode compiled file if the MAGIC number is
compatible
:returns: a code object
"""
data = read_file(filename, 'rb')
if not is_gae and data[:4] != imp.get_magic():
raise SystemError, 'compiled code is incompatible'
return marshal.loads(data[8:])
def app_create(app, request, force=False, key=None, info=False):
"""
Create a copy of welcome.w2p (scaffolding) app
Parameters
----------
app:
application name
request:
the global request object
"""
path = apath(app, request)
if not os.path.exists(path):
try:
os.mkdir(path)
except:
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
elif not force:
if info:
return False, "Application exists"
else:
return False
try:
w2p_unpack('welcome.w2p', path)
for subfolder in [
'models', 'views', 'controllers', 'databases', 'modules',
'cron', 'errors', 'sessions', 'cache', 'languages', 'static',
'private', 'uploads'
]:
subpath = os.path.join(path, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(path, 'models', 'db.py')
if os.path.exists(db):
data = read_file(db)
data = data.replace('<your secret key>',
'sha512:' + (key or web2py_uuid()))
write_file(db, data)
if info:
return True, None
else:
return True
except:
rmtree(path)
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
def compile_models(folder):
"""
Compiles all the models in the application specified by `folder`
"""
path = pjoin(folder, 'models')
for file in listdir(path, '.+\.py$'):
data = read_file(pjoin(path, file))
filename = pjoin(folder, 'compiled', 'models', file)
mktree(filename)
write_file(filename, data)
save_pyc(filename)
os.unlink(filename)
def compile_models(folder):
"""
Compiles all the models in the application specified by `folder`
"""
path = pjoin(folder, 'models')
for file in listdir(path, '.+\.py$'):
data = read_file(pjoin(path, file))
filename = pjoin(folder, 'compiled','models',file)
mktree(filename)
write_file(filename, data)
save_pyc(filename)
os.unlink(filename)
def app_create(app, request, force=False, key=None, info=False):
"""
Create a copy of welcome.w2p (scaffolding) app
Parameters
----------
app:
application name
request:
the global request object
"""
path = apath(app, request)
if not os.path.exists(path):
try:
os.mkdir(path)
except:
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
elif not force:
if info:
return False, "Application exists"
else:
return False
try:
w2p_unpack('welcome.w2p', path)
for subfolder in [
'models', 'views', 'controllers', 'databases',
'modules', 'cron', 'errors', 'sessions', 'cache',
'languages', 'static', 'private', 'uploads']:
subpath = os.path.join(path, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(path, 'models', 'db.py')
if os.path.exists(db):
data = read_file(db)
data = data.replace('<your secret key>',
'sha512:' + (key or web2py_uuid()))
write_file(db, data)
if info:
return True, None
else:
return True
except:
rmtree(path)
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
def test_close_index_html(self):
index_filename = 'files/html/results/index.html'
index_filename = self.abs_test_file_name(index_filename)
# self.new_temp_file(index_filename) # to remove it after test finishes
table_rows = """<tr><td>1</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/1-google-com.html">http://google.com/</a></td><td>10</td><td>1</td></tr>
<tr><td>118</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/118-google-com-ar.html">http://google.com.ar/</a></td><td>3</td><td>51</td></tr>
<tr><td>27</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/27-google-co-uk.html">http://google.co.uk/</a></td><td>1</td><td>11</td></tr>"""
fu.write_to_file(index_filename, table_rows)
lp.close_index_html(index_filename)
index_src = fu.read_file(index_filename)
self.assertTrue('<table' in index_src, 'No table in index.html')
self.assertTrue('<thead' in index_src, 'No thead in index.html')
self.assertTrue('</html>' in index_src, 'No closing html tag index.html')
def test_close_index_html(self):
index_filename = 'files/html/results/index.html'
index_filename = self.abs_test_file_name(index_filename)
# self.new_temp_file(index_filename) # to remove it after test finishes
table_rows = """<tr><td>1</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/1-google-com.html">http://google.com/</a></td><td>10</td><td>1</td></tr>
<tr><td>118</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/118-google-com-ar.html">http://google.com.ar/</a></td><td>3</td><td>51</td></tr>
<tr><td>27</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/27-google-co-uk.html">http://google.co.uk/</a></td><td>1</td><td>11</td></tr>"""
fu.write_to_file(index_filename, table_rows)
lp.close_index_html(index_filename)
index_src = fu.read_file(index_filename)
self.assertTrue('<table' in index_src, 'No table in index.html')
self.assertTrue('<thead' in index_src, 'No thead in index.html')
self.assertTrue('</html>' in index_src,
'No closing html tag index.html')
def compile_controllers(folder):
"""
Compiles all the controllers in the application specified by `folder`
"""
path = pjoin(folder, "controllers")
for file in listdir(path, ".+\.py$"):
### why is this here? save_pyc(pjoin(path, file))
data = read_file(pjoin(path, file))
exposed = regex_expose.findall(data)
for function in exposed:
command = data + "\nresponse._vars=response._caller(%s)\n" % function
filename = pjoin(
folder, "compiled", ("controllers/" + file[:-3]).replace("/", "_") + "_" + function + ".py"
)
write_file(filename, command)
save_pyc(filename)
os.unlink(filename)
def compile_controllers(folder):
"""
Compiles all the controllers in the application specified by `folder`
"""
path = pjoin(folder, 'controllers')
for file in listdir(path, '.+\.py$'):
### why is this here? save_pyc(pjoin(path, file))
data = read_file(pjoin(path, file))
exposed = regex_expose.findall(data)
for function in exposed:
command = data + "\nresponse._vars=response._caller(%s)\n" % \
function
filename = pjoin(folder, 'compiled', ('controllers/'
+ file[:-3]).replace('/', '_')
+ '_' + function + '.py')
write_file(filename, command)
save_pyc(filename)
os.unlink(filename)
def compile_controllers(folder):
"""
Compiles all the controllers in the application specified by `folder`
"""
path = pjoin(folder, 'controllers')
for file in listdir(path, '.+\.py$'):
### why is this here? save_pyc(pjoin(path, file))
data = read_file(pjoin(path,file))
exposed = regex_expose.findall(data)
for function in exposed:
command = data + "\nresponse._vars=response._caller(%s)\n" % \
function
filename = pjoin(folder, 'compiled', ('controllers/'
+ file[:-3]).replace('/', '_')
+ '_' + function + '.py')
write_file(filename, command)
save_pyc(filename)
os.unlink(filename)
def load(routes='routes.py', app=None, data=None, rdict=None):
"""
load: read (if file) and parse routes
store results in params
(called from main.py at web2py initialization time)
If data is present, it's used instead of the routes.py contents.
If rdict is present, it must be a dict to be used for routers (unit test)
"""
global params
global routers
if app is None:
# reinitialize
global params_apps
params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters
routers = None
if isinstance(rdict, dict):
symbols = dict(routers=rdict)
path = 'rdict'
else:
if data is not None:
path = 'routes'
else:
if app is None:
path = abspath(routes)
else:
path = abspath('applications', app, routes)
if not os.path.exists(path):
return
data = read_file(path).replace('\r\n','\n')
symbols = {}
try:
exec (data + '\n') in symbols
except SyntaxError, e:
logger.error(
'%s has a syntax error and will not be loaded\n' % path
+ traceback.format_exc())
raise e
def load(routes='routes.py', app=None, data=None, rdict=None):
"""
load: read (if file) and parse routes
store results in params
(called from main.py at web2py initialization time)
If data is present, it's used instead of the routes.py contents.
If rdict is present, it must be a dict to be used for routers (unit test)
"""
global params
global routers
if app is None:
# reinitialize
global params_apps
params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters
routers = None
if isinstance(rdict, dict):
symbols = dict(routers=rdict)
path = 'rdict'
else:
if data is not None:
path = 'routes'
else:
if app is None:
path = abspath(routes)
else:
path = abspath('applications', app, routes)
if not os.path.exists(path):
return
data = read_file(path).replace('\r\n','\n')
symbols = {}
try:
exec (data + '\n') in symbols
except SyntaxError, e:
logger.error(
'%s has a syntax error and will not be loaded\n' % path
+ traceback.format_exc())
raise e
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment['request'].folder
c = environment['request'].controller
f = environment['request'].function
cpath = pjoin(folder, 'compiled')
if os.path.exists(cpath):
for model in listdir(cpath, '^models_\w+\.pyc$', 0):
restricted(read_pyc(model), environment, layer=model)
path = pjoin(cpath, 'models')
models = listdir(path, '^\w+\.pyc$',0,sort=False)
compiled=True
else:
path = pjoin(folder, 'models')
models = listdir(path, '^\w+\.py$',0,sort=False)
compiled=False
n = len(path) + 1
for model in models:
regex = environment['response'].models_to_run
if isinstance(regex, list):
regex = re_compile('|'.join(regex))
file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py')
if not regex.search(file) and c!= 'appadmin':
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model,
lambda: compile2(read_file(model), model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def run_models_in(environment):
"""
Runs all models (in the app specified by the current folder)
It tries pre-compiled models first before compiling them.
"""
folder = environment['request'].folder
c = environment['request'].controller
f = environment['request'].function
cpath = pjoin(folder, 'compiled')
if os.path.exists(cpath):
for model in listdir(cpath, '^models_\w+\.pyc$', 0):
restricted(read_pyc(model), environment, layer=model)
path = pjoin(cpath, 'models')
models = listdir(path, '^\w+\.pyc$', 0)
compiled = True
else:
path = pjoin(folder, 'models')
models = listdir(path, '^\w+\.py$', 0)
compiled = False
n = len(path) + 1
for model in models:
regex = environment['response'].models_to_run
if isinstance(regex, list):
regex = re_compile('|'.join(regex))
file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py')
if not regex.search(file) and c != 'appadmin':
continue
elif compiled:
code = read_pyc(model)
elif is_gae:
code = getcfs(model, model,
lambda: compile2(read_file(model), model))
else:
code = getcfs(model, model, None)
restricted(code, environment, layer=model)
def run_controller_in(controller, function, environment):
"""
Runs the controller.function() (for the app specified by
the current folder).
It tries pre-compiled controller_function.pyc first before compiling it.
"""
# if compiled should run compiled!
folder = environment['request'].folder
path = pjoin(folder, 'compiled')
badc = 'invalid controller (%s/%s)' % (controller, function)
badf = 'invalid function (%s/%s)' % (controller, function)
if os.path.exists(path):
filename = pjoin(path, 'controllers_%s_%s.pyc'
% (controller, function))
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf)
restricted(read_pyc(filename), environment, layer=filename)
elif function == '_TEST':
# TESTING: adjust the path to include site packages
from settings import global_settings
from admin import abspath, add_path_first
paths = (global_settings.gluon_parent, abspath('site-packages', gluon=True), abspath('gluon', gluon=True), '')
[add_path_first(path) for path in paths]
# TESTING END
filename = pjoin(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc)
environment['__symbols__'] = environment.keys()
code = read_file(filename)
code += TEST_CODE
restricted(code, environment, layer=filename)
else:
filename = pjoin(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc)
code = read_file(filename)
exposed = regex_expose.findall(code)
if not function in exposed:
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf)
code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
if is_gae:
layer = filename + ':' + function
code = getcfs(layer, filename, lambda: compile2(code,layer))
restricted(code, environment, filename)
response = environment['response']
vars=response._vars
if response.postprocessing:
vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
if isinstance(vars,unicode):
vars = vars.encode('utf8')
elif hasattr(vars,'xml') and callable(vars.xml):
vars = vars.xml()
return vars
def test_write_to_file(self):
filename = self.new_temp_file('write_test.txt')
random_str = ut.rand_str(100)
fu.write_to_file(filename, random_str)
self.assertEqual(random_str, fu.read_file(filename))
def test_read_file(self):
file_content = fu.read_file(os.path.realpath(__file__))
if not 'whatever I write here' in file_content:
self.fail('Cannot read itself')
try:
import Tkinter, tkMessageBox
import contrib.taskbar_widget
from winservice import web2py_windows_service_handler
except:
pass
try:
BaseException
except NameError:
BaseException = Exception
ProgramName = 'web2py Web Framework'
ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str(datetime.datetime.now().year)
ProgramVersion = read_file('VERSION').strip()
ProgramInfo = '''%s
%s
%s''' % (ProgramName, ProgramAuthor, ProgramVersion)
if not sys.version[:3] in ['2.4', '2.5', '2.6', '2.7']:
msg = 'Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s'
msg = msg % sys.version
sys.stderr.write(msg)
logger = logging.getLogger("web2py")
def run_system_tests():
major_version = sys.version_info[0]
minor_version = sys.version_info[1]
def parse_crawl_log(filename, dump_fun=None, crawl_id=0, url=""):
"""Populate domain info object by parsing crawl log file of a site.
Call dump function to output dump log.
Logs to be parsed with this function are generated by setting env. variable FC_DEBUG=1 to 1 or logs from the browser.
See, fontconfig library for details.
"""
origins_to_fonts = {} # will keep origin to loaded fonts mapping
domaInfo = DomainInfo()
file_content = fu.read_file(filename)
wl_log.info('Parsing log for %s %s' % (url, filename))
fonts_by_fc_debug = re.findall(
r"Sort Pattern.*$\W+family: \"([^\"]*)", file_content, re.MULTILINE
) # match family field of font request (not the matched one)
domaInfo.num_offsetWidth_calls = len(
re.findall(r"Element::offsetWidth",
file_content)) # offset width attempts
domaInfo.num_offsetHeight_calls = len(
re.findall(r"Element::offsetHeight",
file_content)) # offset height attempts
# TODO add getBoundingClientRect
font_and_urls = re.findall(r"CSSFontSelector::getFontData:? (.*) ([^\s]*)",
file_content) # output from modified browser
#print 'font_and_urls', font_and_urls
font_face_pairs = re.findall(r"CSSFontFace::getFontData (.*)->(.*)",
file_content) # output from modified browser
#print 'font_and_urls', font_and_urls
domaInfo.log_complete = int(
bool(re.findall(r"Finished all steps!",
file_content))) # output from modified browser
#print 'domaInfo.log_complete', domaInfo.log_complete
js_log_prefix = ">>>FPLOG"
fpd_logs = re.findall(r'%s.*' % js_log_prefix,
file_content) # output from modified browser
domaInfo.fpd_logs = [
call[len(js_log_prefix) + 1:] for call in set(fpd_logs)
]
for font_name, font_url in font_and_urls:
if font_url.startswith('http') and len(
font_name) > 1 and not font_name[:5] in ('data:', 'http:',
'https'):
#font_name = font_name.rsplit(' ', 1)[0] if font_name.endswith(' onURL:') else font_name # TODO: unify chrome source code to log as Phantom do. then remove this line
font_name = font_name.lower().strip()
# origin = pub_suffix.get_public_suffix(font_url)\
origin = font_url
if origin in origins_to_fonts:
origins_to_fonts[origin].add(font_name)
#print 'added', font_name, 'to', origin, origins_to_fonts[origin]
else:
origins_to_fonts[origin] = set([
font_name,
])
for font, face in font_face_pairs:
font = font.lower().strip()
face = face.lower().strip()
# replace all occurrences of this font-family name with the face
for fonts_by_origin in origins_to_fonts.itervalues():
try:
fonts_by_origin.remove(font)
except: # we cannot find this font in this origin's list
pass
else:
fonts_by_origin.add(face)
# print 'removed', font, 'added', face
for origin, fonts in origins_to_fonts.iteritems():
domaInfo.fonts_by_origins[origin] = list(fonts)
domaInfo.fonts_loaded += domaInfo.fonts_by_origins[origin]
domaInfo.fc_dbg_font_loads = list(set([font.lower() for font in fonts_by_fc_debug \
if not font[:5] in ('data:', 'http:', 'https')])) # filter out the data urls and web fonts
domaInfo.fonts_loaded = list(set([font.lower() for font in domaInfo.fonts_loaded \
if not font[:5] in ('data:', 'http:', 'https')])) # filter out the data urls and web fonts
requests = re.findall(r"^requested: (http.*)", file_content, re.MULTILINE)
if not requests and filename.endswith(MITM_LOG_EXTENSION):
requests = re.findall(r"(http.*)", file_content, re.MULTILINE)
responses = ''
# populate domain info obj
domaInfo.num_font_loads = len(domaInfo.fonts_loaded)
domaInfo.requests = list(set(requests))
domaInfo.responses = list(set(responses))
domaInfo.fp_detected = get_fp_from_reqs(requests)
domaInfo.url = url
domaInfo.rank = get_rank_domain_from_filename(
filename
)[0] # !!! rank may not be right. It's only true if we make top Alexa crawl.
domaInfo.log_filename = filename
domaInfo.crawl_id = crawl_id
# Read canvas events and print them to log in canvas
urls_read_from_canvas = Set()
urls_wrote_to_canvas = Set()
canvas_log = os.path.join(cm.BASE_FP_LOGS_FOLDER,
str(crawl_id) + "canvas.log")
read = wrote = False
for read_event in cm.CANVAS_READ_EVENTS:
if read_event in file_content:
read = True
break
for write_event in cm.CANVAS_WRITE_EVENTS:
if write_event in file_content:
wrote = True
break
if read and wrote:
wl_log.info(
'Found both canvas read and write events in log %s, registering in : %s'
% (filename, canvas_log))
with open(canvas_log, "a+") as f:
f.write(" ".join([str(domaInfo.rank), domaInfo.url]) + "\n")
if dump_fun: # call dump function
try:
dump_fun(domaInfo)
except KeyboardInterrupt:
raise
except Exception as exc:
wl_log.exception("Exception while dumping %s: %s" %
(domaInfo.url, exc))
def run(appname,
plain=False,
import_models=False,
startfile=None,
bpython=False):
"""
Start interactive shell or run Python script (startfile) in web2py
controller environment. appname is formatted like:
a web2py application name
a/c exec the controller c into the application environment
"""
(a, c, f) = parse_path_info(appname)
errmsg = 'invalid application name: %s' % appname
if not a:
die(errmsg)
adir = os.path.join('applications', a)
if not os.path.exists(adir):
if raw_input('application %s does not exist, create (y/n)?' %
a).lower() in ['y', 'yes']:
os.mkdir(adir)
w2p_unpack('welcome.w2p', adir)
for subfolder in [
'models', 'views', 'controllers', 'databases', 'modules',
'cron', 'errors', 'sessions', 'languages', 'static',
'private', 'uploads'
]:
subpath = os.path.join(adir, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(adir, 'models/db.py')
if os.path.exists(db):
data = fileutils.read_file(db)
data = data.replace('<your secret key>',
'sha512:' + web2py_uuid())
fileutils.write_file(db, data)
if c:
import_models = True
_env = env(a, c=c, import_models=import_models)
if c:
cfile = os.path.join('applications', a, 'controllers', c + '.py')
if not os.path.isfile(cfile):
cfile = os.path.join('applications', a, 'compiled',
"controllers_%s_%s.pyc" % (c, f))
if not os.path.isfile(cfile):
die(errmsg)
else:
exec read_pyc(cfile) in _env
else:
execfile(cfile, _env)
if f:
exec('print %s()' % f, _env)
elif startfile:
exec_pythonrc()
try:
execfile(startfile, _env)
except RestrictedError, e:
print e.traceback
try:
import Tkinter, tkMessageBox
import contrib.taskbar_widget
from winservice import web2py_windows_service_handler
except:
pass
try:
BaseException
except NameError:
BaseException = Exception
ProgramName = "web2py Web Framework"
ProgramAuthor = "Created by Massimo Di Pierro, Copyright 2007-2011"
ProgramVersion = read_file("VERSION").strip()
ProgramInfo = """%s
%s
%s""" % (
ProgramName,
ProgramAuthor,
ProgramVersion,
)
if not sys.version[:3] in ["2.4", "2.5", "2.6", "2.7"]:
msg = "Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s"
msg = msg % sys.version
sys.stderr.write(msg)
logger = logging.getLogger("web2py")
def app_create(app, request, force=False, key=None, info=False):
"""
Create a copy of welcome.w2p (scaffolding) app
Parameters
----------
app:
application name
request:
the global request object
"""
path = apath(app, request)
if not os.path.exists(path):
try:
os.mkdir(path)
except:
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
elif not force:
if info:
return False, "Application exists"
else:
return False
try:
w2p_unpack("welcome.w2p", path)
for subfolder in [
"models",
"views",
"controllers",
"databases",
"modules",
"cron",
"errors",
"sessions",
"cache",
"languages",
"static",
"private",
"uploads",
]:
subpath = os.path.join(path, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(path, "models", "db.py")
if os.path.exists(db):
data = read_file(db)
data = data.replace("<your secret key>", "sha512:" + (key or web2py_uuid()))
write_file(db, data)
if info:
return True, None
else:
return True
except:
rmtree(path)
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False
def parse_crawl_log(filename, dump_fun=None, crawl_id=0, url=""):
"""Populate domain info object by parsing crawl log file of a site.
Call dump function to output dump log.
Logs to be parsed with this function are generated by setting env. variable FC_DEBUG=1 to 1 or logs from the browser.
See, fontconfig library for details.
"""
origins_to_fonts = {} # will keep origin to loaded fonts mapping
domaInfo = DomainInfo()
file_content = fu.read_file(filename)
wl_log.info("Parsing log for %s %s" % (url, filename))
fonts_by_fc_debug = re.findall(
r"Sort Pattern.*$\W+family: \"([^\"]*)", file_content, re.MULTILINE
) # match family field of font request (not the matched one)
domaInfo.num_offsetWidth_calls = len(re.findall(r"Element::offsetWidth", file_content)) # offset width attempts
domaInfo.num_offsetHeight_calls = len(re.findall(r"Element::offsetHeight", file_content)) # offset height attempts
# TODO add getBoundingClientRect
font_and_urls = re.findall(
r"CSSFontSelector::getFontData:? (.*) ([^\s]*)", file_content
) # output from modified browser
# print 'font_and_urls', font_and_urls
font_face_pairs = re.findall(r"CSSFontFace::getFontData (.*)->(.*)", file_content) # output from modified browser
# print 'font_and_urls', font_and_urls
domaInfo.log_complete = int(bool(re.findall(r"Finished all steps!", file_content))) # output from modified browser
# print 'domaInfo.log_complete', domaInfo.log_complete
js_log_prefix = ">>>FPLOG"
fpd_logs = re.findall(r"%s.*" % js_log_prefix, file_content) # output from modified browser
domaInfo.fpd_logs = [call[len(js_log_prefix) + 1 :] for call in set(fpd_logs)]
for font_name, font_url in font_and_urls:
if font_url.startswith("http") and len(font_name) > 1 and not font_name[:5] in ("data:", "http:", "https"):
# font_name = font_name.rsplit(' ', 1)[0] if font_name.endswith(' onURL:') else font_name # TODO: unify chrome source code to log as Phantom do. then remove this line
font_name = font_name.lower().strip()
# origin = pub_suffix.get_public_suffix(font_url)\
origin = font_url
if origin in origins_to_fonts:
origins_to_fonts[origin].add(font_name)
# print 'added', font_name, 'to', origin, origins_to_fonts[origin]
else:
origins_to_fonts[origin] = set([font_name])
for font, face in font_face_pairs:
font = font.lower().strip()
face = face.lower().strip()
# replace all occurrences of this font-family name with the face
for fonts_by_origin in origins_to_fonts.itervalues():
try:
fonts_by_origin.remove(font)
except: # we cannot find this font in this origin's list
pass
else:
fonts_by_origin.add(face)
# print 'removed', font, 'added', face
for origin, fonts in origins_to_fonts.iteritems():
domaInfo.fonts_by_origins[origin] = list(fonts)
domaInfo.fonts_loaded += domaInfo.fonts_by_origins[origin]
domaInfo.fc_dbg_font_loads = list(
set([font.lower() for font in fonts_by_fc_debug if not font[:5] in ("data:", "http:", "https")])
) # filter out the data urls and web fonts
domaInfo.fonts_loaded = list(
set([font.lower() for font in domaInfo.fonts_loaded if not font[:5] in ("data:", "http:", "https")])
) # filter out the data urls and web fonts
requests = re.findall(r"^requested: (http.*)", file_content, re.MULTILINE)
if not requests and filename.endswith(MITM_LOG_EXTENSION):
requests = re.findall(r"(http.*)", file_content, re.MULTILINE)
responses = ""
# populate domain info obj
domaInfo.num_font_loads = len(domaInfo.fonts_loaded)
domaInfo.requests = list(set(requests))
domaInfo.responses = list(set(responses))
domaInfo.fp_detected = get_fp_from_reqs(requests)
domaInfo.url = url
domaInfo.rank = get_rank_domain_from_filename(filename)[
0
] # !!! rank may not be right. It's only true if we make top Alexa crawl.
domaInfo.log_filename = filename
domaInfo.crawl_id = crawl_id
# Read canvas events and print them to log in canvas
urls_read_from_canvas = Set()
urls_wrote_to_canvas = Set()
canvas_log = os.path.join(cm.BASE_FP_LOGS_FOLDER, str(crawl_id) + "canvas.log")
read = wrote = False
for read_event in cm.CANVAS_READ_EVENTS:
if read_event in file_content:
read = True
break
for write_event in cm.CANVAS_WRITE_EVENTS:
if write_event in file_content:
wrote = True
break
if read and wrote:
wl_log.info("Found both canvas read and write events in log %s, registering in : %s" % (filename, canvas_log))
with open(canvas_log, "a+") as f:
f.write(" ".join([str(domaInfo.rank), domaInfo.url]) + "\n")
if dump_fun: # call dump function
try:
dump_fun(domaInfo)
except KeyboardInterrupt:
raise
except Exception as exc:
wl_log.exception("Exception while dumping %s: %s" % (domaInfo.url, exc))
def test_read_file_from_stdin(self, mock_stdin):
mock_stdin.read.return_value = 'some stdin content'
self.assertEqual(fileutils.read_file('-'),
mock_stdin.read.return_value)
[2451.456787, 1006.207645, 596.2362145, 238.8109685, 126.4624943])
invN = np.array([0.1667824, 0.1453026, 0.169609, 0.1212046, 0.0884299])
mixmat_base = np.array([[1.0, 23.0 / 94.0], [33.0 / 23.0, 33.0 / 94.0],
[41.0 / 23.0, 41.0 / 94.0], [61.0 / 23.0, 61.0 / 94.0],
[94.0 / 23.0, 1.0]])
fnames = []
nfnames = []
for i in range(5):
fnames.append('signalmaps_%02d.fits' % (i + 1))
nfnames.append('invN_%02d.fits' % (i + 1))
frequencies = np.array([23.0, 33.0, 41.0, 61.0, 94.0])
mds = []
nmds = []
for idx, fname in enumerate(fnames):
mds.append(fileutils.read_file(fnames[idx]))
nmds.append(fileutils.read_file(nfnames[idx]))
#data = []
#err = []
data = dat
err = 1 / np.sqrt(invN)
#for md, nmd in zip(mds, nmds):
# data.append(md.map[pix])
# err.append(1/np.sqrt(nmd.map[pix]))
data = data * a2t
plt.scatter(frequencies, data)
plt.errorbar(frequencies, data, err, fmt=None)
plt.plot(frequencies, a2t * 1445.2627 * mixmat_base[:, 0]**-3.033717)
plt.plot(frequencies, a2t * 66.74626193 * mixmat_base[:, 1]**-1.90380762)
plt.plot(
def test_write_to_file(self):
filename = self.new_temp_file('write_test.txt')
random_str = ut.rand_str(100)
fu.write_to_file(filename, random_str)
self.assertEqual(random_str, fu.read_file(filename))
def run(
appname,
plain=False,
import_models=False,
startfile=None,
bpython=False,
python_code=False,
cronjob=False):
"""
Start interactive shell or run Python script (startfile) in web2py
controller environment. appname is formatted like:
a web2py application name
a/c exec the controller c into the application environment
"""
(a, c, f, args, vars) = parse_path_info(appname, av=True)
errmsg = 'invalid application name: %s' % appname
if not a:
die(errmsg)
adir = os.path.join('applications', a)
if not os.path.exists(adir):
if sys.stdin and not sys.stdin.name == '/dev/null':
confirm = raw_input(
'application %s does not exist, create (y/n)?' % a)
else:
logging.warn('application does not exist and will not be created')
return
if confirm.lower() in ['y', 'yes']:
os.mkdir(adir)
w2p_unpack('welcome.w2p', adir)
for subfolder in ['models', 'views', 'controllers', 'databases',
'modules', 'cron', 'errors', 'sessions',
'languages', 'static', 'private', 'uploads']:
subpath = os.path.join(adir, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(adir, 'models/db.py')
if os.path.exists(db):
data = fileutils.read_file(db)
data = data.replace(
'<your secret key>', 'sha512:' + web2py_uuid())
fileutils.write_file(db, data)
if c:
import_models = True
extra_request = {}
if args:
extra_request['args'] = args
if vars:
extra_request['vars'] = vars
_env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request)
if c:
pyfile = os.path.join('applications', a, 'controllers', c + '.py')
pycfile = os.path.join('applications', a, 'compiled',
"controllers_%s_%s.pyc" % (c, f))
if ((cronjob and os.path.isfile(pycfile))
or not os.path.isfile(pyfile)):
exec read_pyc(pycfile) in _env
elif os.path.isfile(pyfile):
execfile(pyfile, _env)
else:
die(errmsg)
if f:
exec ('print %s()' % f, _env)
return
_env.update(exec_pythonrc())
if startfile:
try:
ccode = None
if startfile.endswith('.pyc'):
ccode = read_pyc(startfile)
exec ccode in _env
else:
execfile(startfile, _env)
if import_models:
BaseAdapter.close_all_instances('commit')
except Exception, e:
print traceback.format_exc()
if import_models:
BaseAdapter.close_all_instances('rollback')
import tkMessageBox
import contrib.taskbar_widget
from winservice import register_service_handler, Web2pyService
have_winservice = True
except:
have_winservice = False
try:
BaseException
except NameError:
BaseException = Exception
ProgramName = 'web2py Web Framework'
ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str(
datetime.datetime.now().year)
ProgramVersion = read_file('VERSION').strip()
ProgramInfo = '''%s
%s
%s''' % (ProgramName, ProgramAuthor, ProgramVersion)
if not sys.version[:3] in ['2.4', '2.5', '2.6', '2.7']:
msg = 'Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s'
msg = msg % sys.version
sys.stderr.write(msg)
logger = logging.getLogger("web2py")
def run_system_tests(options):
"""
def from_client_secrets_file(cls,
client_secrets_file,
scopes,
access_type='offline',
login_hint=None,
credentials_file=None,
use_console_flow=False):
"""Runs an OAuth Flow from secrets stored on disk to generate credentials.
Args:
client_secrets_file: String, path to a file containing a client ID and
secret.
scopes: Sequence[str], A list of scopes to include in the credentials.
access_type: String, 'offline' or 'online'. Indicates whether your
application can refresh access tokens when the user is not present at
the browser. Valid parameter values are online, which is the default
value, and offline. Set the value to offline if your application needs
to refresh access tokens when the user is not present at the browser.
This is the method of refreshing access tokens described later in this
document. This value instructs the Google authorization server to return
a refresh token and an access token the first time that your application
exchanges an authorization code for tokens.
login_hint: String, The email address that will be displayed on the Google
login page as a hint for the user to login to the correct account.
credentials_file: String, the path to a file to use to save the
credentials.
use_console_flow: Boolean, True if the authentication flow should be run
strictly from a console; False to launch a browser for authentication.
Raises:
InvalidClientSecretsFileError: If the client secrets file cannot be
opened.
InvalidClientSecretsFileFormatError: If the client secrets file does not
contain the required data or the data is malformed.
Returns:
Credentials
"""
cs_data = fileutils.read_file(client_secrets_file,
continue_on_error=True,
display_errors=False)
if not cs_data:
raise InvalidClientSecretsFileError(
f'File {client_secrets_file} could not be opened')
try:
cs_json = json.loads(cs_data)
client_id = cs_json['installed']['client_id']
# Chop off .apps.googleusercontent.com suffix as it's not needed
# and we need to keep things short for the Auth URL.
client_id = re.sub(r'\.apps\.googleusercontent\.com$', '',
client_id)
client_secret = cs_json['installed']['client_secret']
except (ValueError, IndexError, KeyError):
raise InvalidClientSecretsFileFormatError(
f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
)
return cls.from_client_secrets(client_id,
client_secret,
scopes,
access_type=access_type,
login_hint=login_hint,
filename=credentials_file,
use_console_flow=use_console_flow)
pix = 9000
a2t = np.array([1.013743, 1.028457, 1.044197, 1.0999, 1.25])
dat = np.array([2451.456787, 1006.207645, 596.2362145, 238.8109685, 126.4624943])
invN = np.array([0.1667824, 0.1453026, 0.169609, 0.1212046, 0.0884299])
mixmat_base = np.array([[1.0, 23.0 / 94.0], [33.0 / 23.0, 33.0 / 94.0], [41.0 / 23.0, 41.0 / 94.0], [61.0 / 23.0, 61.0 / 94.0], [94.0 / 23.0, 1.0]])
fnames = []
nfnames = []
for i in range(5):
fnames.append('signalmaps_%02d.fits' % (i + 1))
nfnames.append('invN_%02d.fits' % (i + 1))
frequencies = np.array([23.0, 33.0, 41.0, 61.0, 94.0])
mds = []
nmds = []
for idx, fname in enumerate(fnames):
mds.append(fileutils.read_file(fnames[idx]))
nmds.append(fileutils.read_file(nfnames[idx]))
#data = []
#err = []
data = dat
err = 1 / np.sqrt(invN)
#for md, nmd in zip(mds, nmds):
# data.append(md.map[pix])
# err.append(1/np.sqrt(nmd.map[pix]))
data = data * a2t
plt.scatter(frequencies, data)
plt.errorbar(frequencies, data, err, fmt=None)
plt.plot(frequencies, a2t * 1445.2627 * mixmat_base[:, 0] ** -3.033717)
plt.plot(frequencies, a2t * 66.74626193 * mixmat_base[:, 1] ** -1.90380762)
plt.plot(frequencies, a2t * (1445.2627 * mixmat_base[:, 0] ** -3.033717 + 66.74626193 * mixmat_base[:, 1] ** -1.90380762))
def run(appname, plain=False, import_models=False, startfile=None, bpython=False, python_code=False):
"""
Start interactive shell or run Python script (startfile) in web2py
controller environment. appname is formatted like:
a web2py application name
a/c exec the controller c into the application environment
"""
(a, c, f) = parse_path_info(appname)
errmsg = "invalid application name: %s" % appname
if not a:
die(errmsg)
adir = os.path.join("applications", a)
if not os.path.exists(adir):
if raw_input("application %s does not exist, create (y/n)?" % a).lower() in ["y", "yes"]:
os.mkdir(adir)
w2p_unpack("welcome.w2p", adir)
for subfolder in [
"models",
"views",
"controllers",
"databases",
"modules",
"cron",
"errors",
"sessions",
"languages",
"static",
"private",
"uploads",
]:
subpath = os.path.join(adir, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(adir, "models/db.py")
if os.path.exists(db):
data = fileutils.read_file(db)
data = data.replace("<your secret key>", "sha512:" + web2py_uuid())
fileutils.write_file(db, data)
if c:
import_models = True
_env = env(a, c=c, f=f, import_models=import_models)
if c:
cfile = os.path.join("applications", a, "controllers", c + ".py")
if not os.path.isfile(cfile):
cfile = os.path.join("applications", a, "compiled", "controllers_%s_%s.pyc" % (c, f))
if not os.path.isfile(cfile):
die(errmsg)
else:
exec read_pyc(cfile) in _env
else:
execfile(cfile, _env)
if f:
exec ("print %s()" % f, _env)
return
# "woodoo magic" workaround: reinitialize main.py
g = {}
exec "import main" in g
del g
_env.update(exec_pythonrc())
if startfile:
try:
execfile(startfile, _env)
if import_models:
BaseAdapter.close_all_instances("commit")
except Exception, e:
print traceback.format_exc()
if import_models:
BaseAdapter.close_all_instances("rollback")
def test_read_file_exits_code_6_when_continue_on_error_is_false(
self, mock_open_file):
mock_open_file.side_effect = IOError()
with self.assertRaises(SystemExit) as context:
fileutils.read_file(self.fake_path, continue_on_error=False)
self.assertEqual(context.exception.code, 6)
def run_controller_in(controller, function, environment):
"""
Runs the controller.function() (for the app specified by
the current folder).
It tries pre-compiled controller_function.pyc first before compiling it.
"""
# if compiled should run compiled!
folder = environment['request'].folder
path = pjoin(folder, 'compiled')
badc = 'invalid controller (%s/%s)' % (controller, function)
badf = 'invalid function (%s/%s)' % (controller, function)
if os.path.exists(path):
filename = pjoin(path, 'controllers_%s_%s.pyc'
% (controller, function))
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf)
restricted(read_pyc(filename), environment, layer=filename)
elif function == '_TEST':
# TESTING: adjust the path to include site packages
from settings import global_settings
from admin import abspath, add_path_first
paths = (global_settings.gluon_parent, abspath(
'site-packages', gluon=True), abspath('gluon', gluon=True), '')
[add_path_first(path) for path in paths]
# TESTING END
filename = pjoin(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc)
environment['__symbols__'] = environment.keys()
code = read_file(filename)
code += TEST_CODE
restricted(code, environment, layer=filename)
else:
filename = pjoin(folder, 'controllers/%s.py'
% controller)
if not os.path.exists(filename):
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc)
code = read_file(filename)
exposed = regex_expose.findall(code)
if not function in exposed:
raise HTTP(404,
rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf)
code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
if is_gae:
layer = filename + ':' + function
code = getcfs(layer, filename, lambda: compile2(code, layer))
restricted(code, environment, filename)
response = environment['response']
vars = response._vars
if response.postprocessing:
vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
if isinstance(vars, unicode):
vars = vars.encode('utf8')
elif hasattr(vars, 'xml') and callable(vars.xml):
vars = vars.xml()
return vars
def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
mock_open_file.return_value.__enter__(
).read.side_effect = UnicodeError()
with self.assertRaises(SystemExit) as context:
fileutils.read_file(self.fake_path)
self.assertEqual(context.exception.code, 2)
def app_create(db, app, request, force=False, key=None, info=False):
"""
Create a copy of welcome.w2p (scaffolding) app
Parameters
----------
app:
application name
request:
the global request object
"""
path = apath(app, request)
if not os.path.exists(path):
try:
os.mkdir(path)
except:
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False, None
elif not force:
if info:
return False, "Application exists"
else:
return False, None
try:
w2p_unpack('welcome.w2p', path)
for subfolder in [
'models', 'views', 'controllers', 'databases',
'modules', 'cron', 'errors', 'sessions', 'cache',
'languages', 'static', 'private', 'uploads']:
subpath = os.path.join(path, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
dbt = os.path.join(path, 'models', 'db.py')
if os.path.exists(dbt):
data = read_file(dbt)
data = data.replace('<your secret key>',
'sha512:' + (key or web2py_uuid()))
write_file(dbt, data)
parms = db(db.parametros.id==1).select()[0]
templates = os.path.join('\\\\'
, '127.0.0.1'
, 'c$'
, parms.web2py
, 'applications'
, parms.soag
, 'Template'
, 'web2py')
for subfolder in ['controllers', 'languages', 'models', 'modules', 'static', 'views']:
template = os.path.join(templates, subfolder)
subpath = os.path.join(path, subfolder)
shutil.rmtree(subpath)
shutil.copytree(template, subpath)
# shutil.copyfile(os.path.join(templates, 'routes.py'), os.path.join(path, 'routes.py'))
if info:
return True, None
else:
return True, None
except:
shutil.rmtree(path)
if info:
return False, traceback.format_exc(sys.exc_info)
else:
return False, None
def run(appname,
plain=False,
import_models=False,
startfile=None,
bpython=False,
python_code=False):
"""
Start interactive shell or run Python script (startfile) in web2py
controller environment. appname is formatted like:
a web2py application name
a/c exec the controller c into the application environment
"""
(a, c, f) = parse_path_info(appname)
errmsg = 'invalid application name: %s' % appname
if not a:
die(errmsg)
adir = os.path.join('applications', a)
if not os.path.exists(adir):
if sys.stdin and not sys.stdin.name == '/dev/null':
confirm = raw_input(
'application %s does not exist, create (y/n)?' % a)
else:
logging.warn('application does not exist and will not be created')
return
if confirm.lower() in ['y', 'yes']:
os.mkdir(adir)
w2p_unpack('welcome.w2p', adir)
for subfolder in [
'models', 'views', 'controllers', 'databases', 'modules',
'cron', 'errors', 'sessions', 'languages', 'static',
'private', 'uploads'
]:
subpath = os.path.join(adir, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
db = os.path.join(adir, 'models/db.py')
if os.path.exists(db):
data = fileutils.read_file(db)
data = data.replace('<your secret key>',
'sha512:' + web2py_uuid())
fileutils.write_file(db, data)
if c:
import_models = True
_env = env(a, c=c, f=f, import_models=import_models)
if c:
cfile = os.path.join('applications', a, 'controllers', c + '.py')
if not os.path.isfile(cfile):
cfile = os.path.join('applications', a, 'compiled',
"controllers_%s_%s.pyc" % (c, f))
if not os.path.isfile(cfile):
die(errmsg)
else:
exec read_pyc(cfile) in _env
else:
execfile(cfile, _env)
if f:
exec('print %s()' % f, _env)
return
_env.update(exec_pythonrc())
if startfile:
try:
execfile(startfile, _env)
if import_models:
BaseAdapter.close_all_instances('commit')
except Exception, e:
print traceback.format_exc()
if import_models:
BaseAdapter.close_all_instances('rollback')
def test_read_file(self):
file_content = fu.read_file(os.path.realpath(__file__))
if not 'whatever I write here' in file_content:
self.fail('Cannot read itself')
