def test_read_file_displays_errors(self, mock_open_file, mock_print_warning): mock_open_file.side_effect = IOError() fileutils.read_file(self.fake_path, continue_on_error=True, display_errors=True) self.assertTrue(mock_print_warning.called)
def test_read_file_exits_code_2_on_unicodedecodeerror( self, mock_open_file): fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0, 1, 'testing only') mock_open_file.return_value.__enter__( ).read.side_effect = fake_decode_error with self.assertRaises(SystemExit) as context: fileutils.read_file(self.fake_path) self.assertEqual(context.exception.code, 2)
def run_controller_in(controller, function, environment): """ Runs the controller.function() (for the app specified by the current folder). It tries pre-compiled controller_function.pyc first before compiling it. """ # if compiled should run compiled! folder = environment["request"].folder path = pjoin(folder, "compiled") badc = "invalid controller (%s/%s)" % (controller, function) badf = "invalid function (%s/%s)" % (controller, function) if os.path.exists(path): filename = pjoin(path, "controllers_%s_%s.pyc" % (controller, function)) if not os.path.exists(filename): raise HTTP(404, rewrite.thread.routes.error_message % badf, web2py_error=badf) restricted(read_pyc(filename), environment, layer=filename) elif function == "_TEST": # TESTING: adjust the path to include site packages from settings import global_settings from admin import abspath, add_path_first paths = (global_settings.gluon_parent, abspath("site-packages", gluon=True), abspath("gluon", gluon=True), "") [add_path_first(path) for path in paths] # TESTING END filename = pjoin(folder, "controllers/%s.py" % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.thread.routes.error_message % badc, web2py_error=badc) environment["__symbols__"] = environment.keys() code = read_file(filename) code += TEST_CODE restricted(code, environment, layer=filename) else: filename = pjoin(folder, "controllers/%s.py" % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.thread.routes.error_message % badc, web2py_error=badc) code = read_file(filename) exposed = regex_expose.findall(code) if not function in exposed: raise HTTP(404, rewrite.thread.routes.error_message % badf, web2py_error=badf) code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) if is_gae: layer = filename + ":" + function code = getcfs(layer, filename, lambda: compile2(code, layer)) restricted(code, environment, filename) response = environment["response"] vars = response._vars if response.postprocessing: for p in response.postprocessing: vars = p(vars) if isinstance(vars, unicode): vars = vars.encode("utf8") elif hasattr(vars, "xml") and callable(vars.xml): vars = vars.xml() return vars
def map2alm(md, lmax, mmax=None, weights=None): """Determines (from whether pol_axis is set or not) whether or not to use polarization if polarization=True. If polarization=False, treats each map as an independent map. """ if mmax is None: mmax = lmax if weights is None: #Try to find file based on data in md weights = 'weight_ring_n%05d.fits' % md.nside if isinstance(weights, str): weights = fileutils.read_file(weights) elif not isinstance(weights, np.ndarray): raise TypeError("Weights must be either filename or numpy array") if weights.shape != (3, 2*md.nside): raise ValueError("Weights do not have the right shape") computer = psht.PshtMmajorHealpix(nside=md.nside, lmax=lmax, mmax=mmax, map=md.map, alm_polarization=md.pol_axis, alm_axis=md.pix_axis, map_axis=md.pix_axis, map_polarization=md.pol_axis, weights=weights[0]) alm = computer.map2alm() ad = almmod.AlmData(lmax, mmax=mmax, alms=alm, pol_axis=md.pol_axis, pol_iter=md.pol_iter, ordering='m-major') return ad
def close_index_html(index_file): # wl_log.info('Will close %s' % index_file) # TODO: add check to don't close a file twice if not os.path.isfile(index_file): fu.write_to_file(index_file, '') # create an empty file index_src = fu.read_file(index_file) if index_src.startswith('<html'): wl_log.info('Index file %s already closed' % index_file) return scripts_src = """<script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery-1.9.1.min.js"></script> <style type="text/css" title="currentStyle"> @import "../../js/css/demo_page.css"; @import "../../js/css/demo_table.css"; </style> <script type="text/javascript" language="javascript" src="http://homes.esat.kuleuven.be/~gacar/jscss/jquery.dataTables.min.js"></script> <script type="text/javascript" charset="utf-8"> $(document).ready(function() { $('#results').dataTable( { "aaSorting": [[ 2, "desc" ]] } ); } ); </script>""" html_str = "<html><head><meta http-equiv='Content-Type' content='text/html; charset=utf-8' />\ <meta http-equiv='Content-Type' content='text/html; charset=utf-8' />" + scripts_src + "</head>\n<body><table id ='results'>\ \n<thead><tr><th>Rank</th><th>Domain</th><th>Fonts</th><th>OffsetWidth</th><th>OffsetHeight</th><th>FP found</th></tr></thead>" + index_src + '</table></body></html>' fu.write_to_file(index_file, html_str)
def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ folder = environment['request'].folder c = environment['request'].controller f = environment['request'].function cpath = os.path.join(folder, 'compiled') if os.path.exists(cpath): for model in listdir(cpath, '^models_\w+\.pyc$', 0): restricted(read_pyc(model), environment, layer=model) path = os.path.join(cpath, 'models') models = listdir(path, '^\w+\.pyc$',0,sort=False) compiled=True else: path = os.path.join(folder, 'models') models = listdir(path, '^\w+\.py$',0,sort=False) compiled=False paths = (path, os.path.join(path,c), os.path.join(path,c,f)) for model in models: if not os.path.split(model)[0] in paths and c!='appadmin': continue elif compiled: code = read_pyc(model) elif is_gae: code = getcfs(model, model, lambda: compile2(read_file(model), model)) else: code = getcfs(model, model, None) restricted(code, environment, layer=model)
def getcfs(key, filename, filter=None): """ Caches the *filtered* file `filename` with `key` until the file is modified. :param key: the cache key :param filename: the file to cache :param filter: is the function used for filtering. Normally `filename` is a .py file and `filter` is a function that bytecode compiles the file. In this way the bytecode compiled file is cached. (Default = None) This is used on Google App Engine since pyc files cannot be saved. """ t = os.stat(filename)[stat.ST_MTIME] cfs_lock.acquire() item = cfs.get(key, None) cfs_lock.release() if item and item[0] == t: return item[1] if not filter: data = read_file(filename) else: data = filter() cfs_lock.acquire() cfs[key] = (t, data) cfs_lock.release() return data
def getcfs(key, filename, filter=None): """ Caches the *filtered* file `filename` with `key` until the file is modified. :param key: the cache key :param filename: the file to cache :param filter: is the function used for filtering. Normally `filename` is a .py file and `filter` is a function that bytecode compiles the file. In this way the bytecode compiled file is cached. (Default = None) This is used on Google App Engine since pyc files cannot be saved. """ try: t = os.stat(filename).st_mtime except OSError: return filter() if callable(filter) else '' cfs_lock.acquire() item = cfs.get(key, None) cfs_lock.release() if item and item[0] == t: return item[1] if not callable(filter): data = read_file(filename) else: data = filter() cfs_lock.acquire() cfs[key] = (t, data) cfs_lock.release() return data
def run( appname, plain=False, import_models=False, startfile=None, bpython=False, python_code=False ): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: a web2py application name a/c exec the controller c into the application environment """ (a, c, f) = parse_path_info(appname) errmsg = 'invalid application name: %s' % appname if not a: die(errmsg) adir = os.path.join('applications', a) if not os.path.exists(adir): if raw_input('application %s does not exist, create (y/n)?' % a).lower() in ['y', 'yes']: os.mkdir(adir) w2p_unpack('welcome.w2p', adir) for subfolder in ['models','views','controllers', 'databases', 'modules','cron','errors','sessions', 'languages','static','private','uploads']: subpath = os.path.join(adir,subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(adir,'models/db.py') if os.path.exists(db): data = fileutils.read_file(db) data = data.replace('<your secret key>','sha512:'+web2py_uuid()) fileutils.write_file(db, data) if c: import_models = True _env = env(a, c=c, import_models=import_models) if c: cfile = os.path.join('applications', a, 'controllers', c + '.py') if not os.path.isfile(cfile): cfile = os.path.join('applications', a, 'compiled', "controllers_%s_%s.pyc" % (c,f)) if not os.path.isfile(cfile): die(errmsg) else: exec read_pyc(cfile) in _env else: execfile(cfile, _env) if f: exec ('print %s()' % f, _env) elif startfile: exec_pythonrc() try: execfile(startfile, _env) except RestrictedError, e: print e.traceback
def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ folder = environment['request'].folder c = environment['request'].controller f = environment['request'].function cpath = os.path.join(folder, 'compiled') if os.path.exists(cpath): for model in listdir(cpath, '^models_\w+\.pyc$', 0): restricted(read_pyc(model), environment, layer=model) path = os.path.join(cpath, 'models') models = listdir(path, '^\w+\.pyc$', 0, sort=False) compiled = True else: path = os.path.join(folder, 'models') models = listdir(path, '^\w+\.py$', 0, sort=False) compiled = False paths = (path, os.path.join(path, c), os.path.join(path, c, f)) for model in models: if not os.path.split(model)[0] in paths and c != 'appadmin': continue elif compiled: code = read_pyc(model) elif is_gae: code = getcfs(model, model, lambda: compile2(read_file(model), model)) else: code = getcfs(model, model, None) restricted(code, environment, layer=model)
def from_credentials_file(cls, filename): """Generates Credentials from a stored Credentials file. The same file will be used to save the credentials when the access token is refreshed. Args: filename: String, the name of a file containing JSON credentials to load. The same filename will be used to save credentials back to disk. Returns: The credentials loaded from disk. Raises: InvalidCredentialsFileError: When the credentials file cannot be opened. EmptyCredentialsFileError: When the provided file contains no credentials. """ file_content = fileutils.read_file(filename, continue_on_error=True, display_errors=False) if file_content is None: raise InvalidCredentialsFileError( f'File {filename} could not be opened') info = json.loads(file_content) if not info: raise EmptyCredentialsFileError( f'File {filename} contains no credential data') try: # We read the existing data from the passed in file, but we also want to # save future data/tokens in the same place. return cls.from_authorized_user_info(info, filename=filename) except ValueError as e: raise InvalidCredentialsFileError(str(e))
def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ folder = environment["request"].folder c = environment["request"].controller f = environment["request"].function cpath = pjoin(folder, "compiled") if os.path.exists(cpath): for model in listdir(cpath, "^models_\w+\.pyc$", 0): restricted(read_pyc(model), environment, layer=model) path = pjoin(cpath, "models") models = listdir(path, "^\w+\.pyc$", 0, sort=False) compiled = True else: path = pjoin(folder, "models") models = listdir(path, "^\w+\.py$", 0, sort=False) compiled = False n = len(path) + 1 for model in models: regex = environment["response"].models_to_run if isinstance(regex, list): regex = re_compile("|".join(regex)) file = model[n:].replace(os.path.sep, "/").replace(".pyc", ".py") if not regex.search(file) and c != "appadmin": continue elif compiled: code = read_pyc(model) elif is_gae: code = getcfs(model, model, lambda: compile2(read_file(model), model)) else: code = getcfs(model, model, None) restricted(code, environment, layer=model)
def test_add_index_html_line(self): self.new_temp_file('index.html') di = lp.DomainInfo() di.log_filename = '/tmp/as.log' lp.add_index_html_line(di) ind_file = lp.get_index_filename_for_domain_info(di) ind_src = fu.read_file(ind_file) self.assertTrue('tr' in ind_src, "Cannot find tr in index.html")
def test_read_file_default_params(self, mock_open_file): fake_content = 'some fake content' mock_open_file.return_value.__enter__( ).read.return_value = fake_content self.assertEqual(fileutils.read_file(self.fake_path), fake_content) self.assertEqual(mock_open_file.call_args[0][0], self.fake_path) self.assertEqual(mock_open_file.call_args[0][1], 'r') self.assertIsNone(mock_open_file.call_args[1]['newline'])
def test_read_file_continues_on_errors_without_displaying( self, mock_open_file, mock_print_warning): mock_open_file.side_effect = IOError() contents = fileutils.read_file(self.fake_path, continue_on_error=True, display_errors=False) self.assertIsNone(contents) self.assertFalse(mock_print_warning.called)
def assert_all_patterns_not_in_file(self, filename, pats): """Assert if none of the patterns passed are included in the file.""" if type(pats) == str: pats = (pats,) txt = fu.read_file(filename) for pat in pats: res = re.findall(pat, txt) if res: self.fail("Should not find pattern %s in %s" % (pat, filename))
def assert_all_patterns_not_in_file(self, filename, pats): """Assert if none of the patterns passed are included in the file.""" if type(pats) == str: pats = (pats, ) txt = fu.read_file(filename) for pat in pats: res = re.findall(pat, txt) if res: self.fail("Should not find pattern %s in %s" % (pat, filename))
def test_job_folder_should_be_writable(self): out_dir = self.create_job_folder() self.assert_(os.path.isdir(out_dir), 'Cannot create job folder') out_file = os.path.join(out_dir, 'some.log') file_content = '123456789' fu.write_to_file(out_file, file_content) self.assert_(os.path.isfile(out_file), 'Cannot create file in job folder') self.assert_(file_content == fu.read_file(out_file), 'Cannot create file in job folder')
def test_a2m2a(): try: ad = fileutils.read_file('eirikalms_l95.fits') md = utils.alm2map(ad, nside=32) except: raise AssertionError() try: md = fileutils.read_file('eirikmap_n32_new.fits') weights = fileutils.read_file('weight_ring_n00032.fits') ad = utils.map2alm(md, lmax = 95, mmax = 95, weights=weights) except: raise AssertionError() try: #Should support giving filenames for the weights md = fileutils.read_file('eirikmap_n32_new.fits') weights = 'weight_ring_n00032.fits' ad = utils.map2alm(md, lmax=95, mmax=95, weights=weights) except: raise AssertionError()
def read_pyc(filename): """ Read the code inside a bytecode compiled file if the MAGIC number is compatible :returns: a code object """ data = read_file(filename, 'rb') if not is_gae and data[:4] != imp.get_magic(): raise SystemError('compiled code is incompatible') return marshal.loads(data[8:])
def read_pyc(filename): """ Read the code inside a bytecode compiled file if the MAGIC number is compatible :returns: a code object """ data = read_file(filename, 'rb') if not is_gae and data[:4] != imp.get_magic(): raise SystemError, 'compiled code is incompatible' return marshal.loads(data[8:])
def app_create(app, request, force=False, key=None, info=False): """ Create a copy of welcome.w2p (scaffolding) app Parameters ---------- app: application name request: the global request object """ path = apath(app, request) if not os.path.exists(path): try: os.mkdir(path) except: if info: return False, traceback.format_exc(sys.exc_info) else: return False elif not force: if info: return False, "Application exists" else: return False try: w2p_unpack('welcome.w2p', path) for subfolder in [ 'models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'cache', 'languages', 'static', 'private', 'uploads' ]: subpath = os.path.join(path, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(path, 'models', 'db.py') if os.path.exists(db): data = read_file(db) data = data.replace('<your secret key>', 'sha512:' + (key or web2py_uuid())) write_file(db, data) if info: return True, None else: return True except: rmtree(path) if info: return False, traceback.format_exc(sys.exc_info) else: return False
def compile_models(folder): """ Compiles all the models in the application specified by `folder` """ path = pjoin(folder, 'models') for file in listdir(path, '.+\.py$'): data = read_file(pjoin(path, file)) filename = pjoin(folder, 'compiled', 'models', file) mktree(filename) write_file(filename, data) save_pyc(filename) os.unlink(filename)
def compile_models(folder): """ Compiles all the models in the application specified by `folder` """ path = pjoin(folder, 'models') for file in listdir(path, '.+\.py$'): data = read_file(pjoin(path, file)) filename = pjoin(folder, 'compiled','models',file) mktree(filename) write_file(filename, data) save_pyc(filename) os.unlink(filename)
def app_create(app, request, force=False, key=None, info=False): """ Create a copy of welcome.w2p (scaffolding) app Parameters ---------- app: application name request: the global request object """ path = apath(app, request) if not os.path.exists(path): try: os.mkdir(path) except: if info: return False, traceback.format_exc(sys.exc_info) else: return False elif not force: if info: return False, "Application exists" else: return False try: w2p_unpack('welcome.w2p', path) for subfolder in [ 'models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'cache', 'languages', 'static', 'private', 'uploads']: subpath = os.path.join(path, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(path, 'models', 'db.py') if os.path.exists(db): data = read_file(db) data = data.replace('<your secret key>', 'sha512:' + (key or web2py_uuid())) write_file(db, data) if info: return True, None else: return True except: rmtree(path) if info: return False, traceback.format_exc(sys.exc_info) else: return False
def test_close_index_html(self): index_filename = 'files/html/results/index.html' index_filename = self.abs_test_file_name(index_filename) # self.new_temp_file(index_filename) # to remove it after test finishes table_rows = """<tr><td>1</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/1-google-com.html">http://google.com/</a></td><td>10</td><td>1</td></tr> <tr><td>118</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/118-google-com-ar.html">http://google.com.ar/</a></td><td>3</td><td>51</td></tr> <tr><td>27</td><td><a href="/home/user/fpbase/run/jobs/20130420-010404/27-google-co-uk.html">http://google.co.uk/</a></td><td>1</td><td>11</td></tr>""" fu.write_to_file(index_filename, table_rows) lp.close_index_html(index_filename) index_src = fu.read_file(index_filename) self.assertTrue('<table' in index_src, 'No table in index.html') self.assertTrue('<thead' in index_src, 'No thead in index.html') self.assertTrue('</html>' in index_src, 'No closing html tag index.html')
def compile_controllers(folder): """ Compiles all the controllers in the application specified by `folder` """ path = pjoin(folder, "controllers") for file in listdir(path, ".+\.py$"): ### why is this here? save_pyc(pjoin(path, file)) data = read_file(pjoin(path, file)) exposed = regex_expose.findall(data) for function in exposed: command = data + "\nresponse._vars=response._caller(%s)\n" % function filename = pjoin( folder, "compiled", ("controllers/" + file[:-3]).replace("/", "_") + "_" + function + ".py" ) write_file(filename, command) save_pyc(filename) os.unlink(filename)
def compile_controllers(folder): """ Compiles all the controllers in the application specified by `folder` """ path = pjoin(folder, 'controllers') for file in listdir(path, '.+\.py$'): ### why is this here? save_pyc(pjoin(path, file)) data = read_file(pjoin(path, file)) exposed = regex_expose.findall(data) for function in exposed: command = data + "\nresponse._vars=response._caller(%s)\n" % \ function filename = pjoin(folder, 'compiled', ('controllers/' + file[:-3]).replace('/', '_') + '_' + function + '.py') write_file(filename, command) save_pyc(filename) os.unlink(filename)
def compile_controllers(folder): """ Compiles all the controllers in the application specified by `folder` """ path = pjoin(folder, 'controllers') for file in listdir(path, '.+\.py$'): ### why is this here? save_pyc(pjoin(path, file)) data = read_file(pjoin(path,file)) exposed = regex_expose.findall(data) for function in exposed: command = data + "\nresponse._vars=response._caller(%s)\n" % \ function filename = pjoin(folder, 'compiled', ('controllers/' + file[:-3]).replace('/', '_') + '_' + function + '.py') write_file(filename, command) save_pyc(filename) os.unlink(filename)
def load(routes='routes.py', app=None, data=None, rdict=None): """ load: read (if file) and parse routes store results in params (called from main.py at web2py initialization time) If data is present, it's used instead of the routes.py contents. If rdict is present, it must be a dict to be used for routers (unit test) """ global params global routers if app is None: # reinitialize global params_apps params_apps = dict() params = _params_default(app=None) # regex rewrite parameters thread.routes = params # default to base regex rewrite parameters routers = None if isinstance(rdict, dict): symbols = dict(routers=rdict) path = 'rdict' else: if data is not None: path = 'routes' else: if app is None: path = abspath(routes) else: path = abspath('applications', app, routes) if not os.path.exists(path): return data = read_file(path).replace('\r\n','\n') symbols = {} try: exec (data + '\n') in symbols except SyntaxError, e: logger.error( '%s has a syntax error and will not be loaded\n' % path + traceback.format_exc()) raise e
def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ folder = environment['request'].folder c = environment['request'].controller f = environment['request'].function cpath = pjoin(folder, 'compiled') if os.path.exists(cpath): for model in listdir(cpath, '^models_\w+\.pyc$', 0): restricted(read_pyc(model), environment, layer=model) path = pjoin(cpath, 'models') models = listdir(path, '^\w+\.pyc$',0,sort=False) compiled=True else: path = pjoin(folder, 'models') models = listdir(path, '^\w+\.py$',0,sort=False) compiled=False n = len(path) + 1 for model in models: regex = environment['response'].models_to_run if isinstance(regex, list): regex = re_compile('|'.join(regex)) file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py') if not regex.search(file) and c!= 'appadmin': continue elif compiled: code = read_pyc(model) elif is_gae: code = getcfs(model, model, lambda: compile2(read_file(model), model)) else: code = getcfs(model, model, None) restricted(code, environment, layer=model)
def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ folder = environment['request'].folder c = environment['request'].controller f = environment['request'].function cpath = pjoin(folder, 'compiled') if os.path.exists(cpath): for model in listdir(cpath, '^models_\w+\.pyc$', 0): restricted(read_pyc(model), environment, layer=model) path = pjoin(cpath, 'models') models = listdir(path, '^\w+\.pyc$', 0) compiled = True else: path = pjoin(folder, 'models') models = listdir(path, '^\w+\.py$', 0) compiled = False n = len(path) + 1 for model in models: regex = environment['response'].models_to_run if isinstance(regex, list): regex = re_compile('|'.join(regex)) file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py') if not regex.search(file) and c != 'appadmin': continue elif compiled: code = read_pyc(model) elif is_gae: code = getcfs(model, model, lambda: compile2(read_file(model), model)) else: code = getcfs(model, model, None) restricted(code, environment, layer=model)
def run_controller_in(controller, function, environment): """ Runs the controller.function() (for the app specified by the current folder). It tries pre-compiled controller_function.pyc first before compiling it. """ # if compiled should run compiled! folder = environment['request'].folder path = pjoin(folder, 'compiled') badc = 'invalid controller (%s/%s)' % (controller, function) badf = 'invalid function (%s/%s)' % (controller, function) if os.path.exists(path): filename = pjoin(path, 'controllers_%s_%s.pyc' % (controller, function)) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) restricted(read_pyc(filename), environment, layer=filename) elif function == '_TEST': # TESTING: adjust the path to include site packages from settings import global_settings from admin import abspath, add_path_first paths = (global_settings.gluon_parent, abspath('site-packages', gluon=True), abspath('gluon', gluon=True), '') [add_path_first(path) for path in paths] # TESTING END filename = pjoin(folder, 'controllers/%s.py' % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) environment['__symbols__'] = environment.keys() code = read_file(filename) code += TEST_CODE restricted(code, environment, layer=filename) else: filename = pjoin(folder, 'controllers/%s.py' % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) code = read_file(filename) exposed = regex_expose.findall(code) if not function in exposed: raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) if is_gae: layer = filename + ':' + function code = getcfs(layer, filename, lambda: compile2(code,layer)) restricted(code, environment, filename) response = environment['response'] vars=response._vars if response.postprocessing: vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars) if isinstance(vars,unicode): vars = vars.encode('utf8') elif hasattr(vars,'xml') and callable(vars.xml): vars = vars.xml() return vars
def test_write_to_file(self): filename = self.new_temp_file('write_test.txt') random_str = ut.rand_str(100) fu.write_to_file(filename, random_str) self.assertEqual(random_str, fu.read_file(filename))
def test_read_file(self): file_content = fu.read_file(os.path.realpath(__file__)) if not 'whatever I write here' in file_content: self.fail('Cannot read itself')
try: import Tkinter, tkMessageBox import contrib.taskbar_widget from winservice import web2py_windows_service_handler except: pass try: BaseException except NameError: BaseException = Exception ProgramName = 'web2py Web Framework' ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str(datetime.datetime.now().year) ProgramVersion = read_file('VERSION').strip() ProgramInfo = '''%s %s %s''' % (ProgramName, ProgramAuthor, ProgramVersion) if not sys.version[:3] in ['2.4', '2.5', '2.6', '2.7']: msg = 'Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s' msg = msg % sys.version sys.stderr.write(msg) logger = logging.getLogger("web2py") def run_system_tests(): major_version = sys.version_info[0] minor_version = sys.version_info[1]
def parse_crawl_log(filename, dump_fun=None, crawl_id=0, url=""): """Populate domain info object by parsing crawl log file of a site. Call dump function to output dump log. Logs to be parsed with this function are generated by setting env. variable FC_DEBUG=1 to 1 or logs from the browser. See, fontconfig library for details. """ origins_to_fonts = {} # will keep origin to loaded fonts mapping domaInfo = DomainInfo() file_content = fu.read_file(filename) wl_log.info('Parsing log for %s %s' % (url, filename)) fonts_by_fc_debug = re.findall( r"Sort Pattern.*$\W+family: \"([^\"]*)", file_content, re.MULTILINE ) # match family field of font request (not the matched one) domaInfo.num_offsetWidth_calls = len( re.findall(r"Element::offsetWidth", file_content)) # offset width attempts domaInfo.num_offsetHeight_calls = len( re.findall(r"Element::offsetHeight", file_content)) # offset height attempts # TODO add getBoundingClientRect font_and_urls = re.findall(r"CSSFontSelector::getFontData:? (.*) ([^\s]*)", file_content) # output from modified browser #print 'font_and_urls', font_and_urls font_face_pairs = re.findall(r"CSSFontFace::getFontData (.*)->(.*)", file_content) # output from modified browser #print 'font_and_urls', font_and_urls domaInfo.log_complete = int( bool(re.findall(r"Finished all steps!", file_content))) # output from modified browser #print 'domaInfo.log_complete', domaInfo.log_complete js_log_prefix = ">>>FPLOG" fpd_logs = re.findall(r'%s.*' % js_log_prefix, file_content) # output from modified browser domaInfo.fpd_logs = [ call[len(js_log_prefix) + 1:] for call in set(fpd_logs) ] for font_name, font_url in font_and_urls: if font_url.startswith('http') and len( font_name) > 1 and not font_name[:5] in ('data:', 'http:', 'https'): #font_name = font_name.rsplit(' ', 1)[0] if font_name.endswith(' onURL:') else font_name # TODO: unify chrome source code to log as Phantom do. then remove this line font_name = font_name.lower().strip() # origin = pub_suffix.get_public_suffix(font_url)\ origin = font_url if origin in origins_to_fonts: origins_to_fonts[origin].add(font_name) #print 'added', font_name, 'to', origin, origins_to_fonts[origin] else: origins_to_fonts[origin] = set([ font_name, ]) for font, face in font_face_pairs: font = font.lower().strip() face = face.lower().strip() # replace all occurrences of this font-family name with the face for fonts_by_origin in origins_to_fonts.itervalues(): try: fonts_by_origin.remove(font) except: # we cannot find this font in this origin's list pass else: fonts_by_origin.add(face) # print 'removed', font, 'added', face for origin, fonts in origins_to_fonts.iteritems(): domaInfo.fonts_by_origins[origin] = list(fonts) domaInfo.fonts_loaded += domaInfo.fonts_by_origins[origin] domaInfo.fc_dbg_font_loads = list(set([font.lower() for font in fonts_by_fc_debug \ if not font[:5] in ('data:', 'http:', 'https')])) # filter out the data urls and web fonts domaInfo.fonts_loaded = list(set([font.lower() for font in domaInfo.fonts_loaded \ if not font[:5] in ('data:', 'http:', 'https')])) # filter out the data urls and web fonts requests = re.findall(r"^requested: (http.*)", file_content, re.MULTILINE) if not requests and filename.endswith(MITM_LOG_EXTENSION): requests = re.findall(r"(http.*)", file_content, re.MULTILINE) responses = '' # populate domain info obj domaInfo.num_font_loads = len(domaInfo.fonts_loaded) domaInfo.requests = list(set(requests)) domaInfo.responses = list(set(responses)) domaInfo.fp_detected = get_fp_from_reqs(requests) domaInfo.url = url domaInfo.rank = get_rank_domain_from_filename( filename )[0] # !!! rank may not be right. It's only true if we make top Alexa crawl. domaInfo.log_filename = filename domaInfo.crawl_id = crawl_id # Read canvas events and print them to log in canvas urls_read_from_canvas = Set() urls_wrote_to_canvas = Set() canvas_log = os.path.join(cm.BASE_FP_LOGS_FOLDER, str(crawl_id) + "canvas.log") read = wrote = False for read_event in cm.CANVAS_READ_EVENTS: if read_event in file_content: read = True break for write_event in cm.CANVAS_WRITE_EVENTS: if write_event in file_content: wrote = True break if read and wrote: wl_log.info( 'Found both canvas read and write events in log %s, registering in : %s' % (filename, canvas_log)) with open(canvas_log, "a+") as f: f.write(" ".join([str(domaInfo.rank), domaInfo.url]) + "\n") if dump_fun: # call dump function try: dump_fun(domaInfo) except KeyboardInterrupt: raise except Exception as exc: wl_log.exception("Exception while dumping %s: %s" % (domaInfo.url, exc))
def run(appname, plain=False, import_models=False, startfile=None, bpython=False): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: a web2py application name a/c exec the controller c into the application environment """ (a, c, f) = parse_path_info(appname) errmsg = 'invalid application name: %s' % appname if not a: die(errmsg) adir = os.path.join('applications', a) if not os.path.exists(adir): if raw_input('application %s does not exist, create (y/n)?' % a).lower() in ['y', 'yes']: os.mkdir(adir) w2p_unpack('welcome.w2p', adir) for subfolder in [ 'models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'languages', 'static', 'private', 'uploads' ]: subpath = os.path.join(adir, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(adir, 'models/db.py') if os.path.exists(db): data = fileutils.read_file(db) data = data.replace('<your secret key>', 'sha512:' + web2py_uuid()) fileutils.write_file(db, data) if c: import_models = True _env = env(a, c=c, import_models=import_models) if c: cfile = os.path.join('applications', a, 'controllers', c + '.py') if not os.path.isfile(cfile): cfile = os.path.join('applications', a, 'compiled', "controllers_%s_%s.pyc" % (c, f)) if not os.path.isfile(cfile): die(errmsg) else: exec read_pyc(cfile) in _env else: execfile(cfile, _env) if f: exec('print %s()' % f, _env) elif startfile: exec_pythonrc() try: execfile(startfile, _env) except RestrictedError, e: print e.traceback
try: import Tkinter, tkMessageBox import contrib.taskbar_widget from winservice import web2py_windows_service_handler except: pass try: BaseException except NameError: BaseException = Exception ProgramName = "web2py Web Framework" ProgramAuthor = "Created by Massimo Di Pierro, Copyright 2007-2011" ProgramVersion = read_file("VERSION").strip() ProgramInfo = """%s %s %s""" % ( ProgramName, ProgramAuthor, ProgramVersion, ) if not sys.version[:3] in ["2.4", "2.5", "2.6", "2.7"]: msg = "Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s" msg = msg % sys.version sys.stderr.write(msg) logger = logging.getLogger("web2py")
def app_create(app, request, force=False, key=None, info=False): """ Create a copy of welcome.w2p (scaffolding) app Parameters ---------- app: application name request: the global request object """ path = apath(app, request) if not os.path.exists(path): try: os.mkdir(path) except: if info: return False, traceback.format_exc(sys.exc_info) else: return False elif not force: if info: return False, "Application exists" else: return False try: w2p_unpack("welcome.w2p", path) for subfolder in [ "models", "views", "controllers", "databases", "modules", "cron", "errors", "sessions", "cache", "languages", "static", "private", "uploads", ]: subpath = os.path.join(path, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(path, "models", "db.py") if os.path.exists(db): data = read_file(db) data = data.replace("<your secret key>", "sha512:" + (key or web2py_uuid())) write_file(db, data) if info: return True, None else: return True except: rmtree(path) if info: return False, traceback.format_exc(sys.exc_info) else: return False
def parse_crawl_log(filename, dump_fun=None, crawl_id=0, url=""): """Populate domain info object by parsing crawl log file of a site. Call dump function to output dump log. Logs to be parsed with this function are generated by setting env. variable FC_DEBUG=1 to 1 or logs from the browser. See, fontconfig library for details. """ origins_to_fonts = {} # will keep origin to loaded fonts mapping domaInfo = DomainInfo() file_content = fu.read_file(filename) wl_log.info("Parsing log for %s %s" % (url, filename)) fonts_by_fc_debug = re.findall( r"Sort Pattern.*$\W+family: \"([^\"]*)", file_content, re.MULTILINE ) # match family field of font request (not the matched one) domaInfo.num_offsetWidth_calls = len(re.findall(r"Element::offsetWidth", file_content)) # offset width attempts domaInfo.num_offsetHeight_calls = len(re.findall(r"Element::offsetHeight", file_content)) # offset height attempts # TODO add getBoundingClientRect font_and_urls = re.findall( r"CSSFontSelector::getFontData:? (.*) ([^\s]*)", file_content ) # output from modified browser # print 'font_and_urls', font_and_urls font_face_pairs = re.findall(r"CSSFontFace::getFontData (.*)->(.*)", file_content) # output from modified browser # print 'font_and_urls', font_and_urls domaInfo.log_complete = int(bool(re.findall(r"Finished all steps!", file_content))) # output from modified browser # print 'domaInfo.log_complete', domaInfo.log_complete js_log_prefix = ">>>FPLOG" fpd_logs = re.findall(r"%s.*" % js_log_prefix, file_content) # output from modified browser domaInfo.fpd_logs = [call[len(js_log_prefix) + 1 :] for call in set(fpd_logs)] for font_name, font_url in font_and_urls: if font_url.startswith("http") and len(font_name) > 1 and not font_name[:5] in ("data:", "http:", "https"): # font_name = font_name.rsplit(' ', 1)[0] if font_name.endswith(' onURL:') else font_name # TODO: unify chrome source code to log as Phantom do. then remove this line font_name = font_name.lower().strip() # origin = pub_suffix.get_public_suffix(font_url)\ origin = font_url if origin in origins_to_fonts: origins_to_fonts[origin].add(font_name) # print 'added', font_name, 'to', origin, origins_to_fonts[origin] else: origins_to_fonts[origin] = set([font_name]) for font, face in font_face_pairs: font = font.lower().strip() face = face.lower().strip() # replace all occurrences of this font-family name with the face for fonts_by_origin in origins_to_fonts.itervalues(): try: fonts_by_origin.remove(font) except: # we cannot find this font in this origin's list pass else: fonts_by_origin.add(face) # print 'removed', font, 'added', face for origin, fonts in origins_to_fonts.iteritems(): domaInfo.fonts_by_origins[origin] = list(fonts) domaInfo.fonts_loaded += domaInfo.fonts_by_origins[origin] domaInfo.fc_dbg_font_loads = list( set([font.lower() for font in fonts_by_fc_debug if not font[:5] in ("data:", "http:", "https")]) ) # filter out the data urls and web fonts domaInfo.fonts_loaded = list( set([font.lower() for font in domaInfo.fonts_loaded if not font[:5] in ("data:", "http:", "https")]) ) # filter out the data urls and web fonts requests = re.findall(r"^requested: (http.*)", file_content, re.MULTILINE) if not requests and filename.endswith(MITM_LOG_EXTENSION): requests = re.findall(r"(http.*)", file_content, re.MULTILINE) responses = "" # populate domain info obj domaInfo.num_font_loads = len(domaInfo.fonts_loaded) domaInfo.requests = list(set(requests)) domaInfo.responses = list(set(responses)) domaInfo.fp_detected = get_fp_from_reqs(requests) domaInfo.url = url domaInfo.rank = get_rank_domain_from_filename(filename)[ 0 ] # !!! rank may not be right. It's only true if we make top Alexa crawl. domaInfo.log_filename = filename domaInfo.crawl_id = crawl_id # Read canvas events and print them to log in canvas urls_read_from_canvas = Set() urls_wrote_to_canvas = Set() canvas_log = os.path.join(cm.BASE_FP_LOGS_FOLDER, str(crawl_id) + "canvas.log") read = wrote = False for read_event in cm.CANVAS_READ_EVENTS: if read_event in file_content: read = True break for write_event in cm.CANVAS_WRITE_EVENTS: if write_event in file_content: wrote = True break if read and wrote: wl_log.info("Found both canvas read and write events in log %s, registering in : %s" % (filename, canvas_log)) with open(canvas_log, "a+") as f: f.write(" ".join([str(domaInfo.rank), domaInfo.url]) + "\n") if dump_fun: # call dump function try: dump_fun(domaInfo) except KeyboardInterrupt: raise except Exception as exc: wl_log.exception("Exception while dumping %s: %s" % (domaInfo.url, exc))
def test_read_file_from_stdin(self, mock_stdin): mock_stdin.read.return_value = 'some stdin content' self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
[2451.456787, 1006.207645, 596.2362145, 238.8109685, 126.4624943]) invN = np.array([0.1667824, 0.1453026, 0.169609, 0.1212046, 0.0884299]) mixmat_base = np.array([[1.0, 23.0 / 94.0], [33.0 / 23.0, 33.0 / 94.0], [41.0 / 23.0, 41.0 / 94.0], [61.0 / 23.0, 61.0 / 94.0], [94.0 / 23.0, 1.0]]) fnames = [] nfnames = [] for i in range(5): fnames.append('signalmaps_%02d.fits' % (i + 1)) nfnames.append('invN_%02d.fits' % (i + 1)) frequencies = np.array([23.0, 33.0, 41.0, 61.0, 94.0]) mds = [] nmds = [] for idx, fname in enumerate(fnames): mds.append(fileutils.read_file(fnames[idx])) nmds.append(fileutils.read_file(nfnames[idx])) #data = [] #err = [] data = dat err = 1 / np.sqrt(invN) #for md, nmd in zip(mds, nmds): # data.append(md.map[pix]) # err.append(1/np.sqrt(nmd.map[pix])) data = data * a2t plt.scatter(frequencies, data) plt.errorbar(frequencies, data, err, fmt=None) plt.plot(frequencies, a2t * 1445.2627 * mixmat_base[:, 0]**-3.033717) plt.plot(frequencies, a2t * 66.74626193 * mixmat_base[:, 1]**-1.90380762) plt.plot(
def run( appname, plain=False, import_models=False, startfile=None, bpython=False, python_code=False, cronjob=False): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: a web2py application name a/c exec the controller c into the application environment """ (a, c, f, args, vars) = parse_path_info(appname, av=True) errmsg = 'invalid application name: %s' % appname if not a: die(errmsg) adir = os.path.join('applications', a) if not os.path.exists(adir): if sys.stdin and not sys.stdin.name == '/dev/null': confirm = raw_input( 'application %s does not exist, create (y/n)?' % a) else: logging.warn('application does not exist and will not be created') return if confirm.lower() in ['y', 'yes']: os.mkdir(adir) w2p_unpack('welcome.w2p', adir) for subfolder in ['models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'languages', 'static', 'private', 'uploads']: subpath = os.path.join(adir, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(adir, 'models/db.py') if os.path.exists(db): data = fileutils.read_file(db) data = data.replace( '<your secret key>', 'sha512:' + web2py_uuid()) fileutils.write_file(db, data) if c: import_models = True extra_request = {} if args: extra_request['args'] = args if vars: extra_request['vars'] = vars _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request) if c: pyfile = os.path.join('applications', a, 'controllers', c + '.py') pycfile = os.path.join('applications', a, 'compiled', "controllers_%s_%s.pyc" % (c, f)) if ((cronjob and os.path.isfile(pycfile)) or not os.path.isfile(pyfile)): exec read_pyc(pycfile) in _env elif os.path.isfile(pyfile): execfile(pyfile, _env) else: die(errmsg) if f: exec ('print %s()' % f, _env) return _env.update(exec_pythonrc()) if startfile: try: ccode = None if startfile.endswith('.pyc'): ccode = read_pyc(startfile) exec ccode in _env else: execfile(startfile, _env) if import_models: BaseAdapter.close_all_instances('commit') except Exception, e: print traceback.format_exc() if import_models: BaseAdapter.close_all_instances('rollback')
import tkMessageBox import contrib.taskbar_widget from winservice import register_service_handler, Web2pyService have_winservice = True except: have_winservice = False try: BaseException except NameError: BaseException = Exception ProgramName = 'web2py Web Framework' ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str( datetime.datetime.now().year) ProgramVersion = read_file('VERSION').strip() ProgramInfo = '''%s %s %s''' % (ProgramName, ProgramAuthor, ProgramVersion) if not sys.version[:3] in ['2.4', '2.5', '2.6', '2.7']: msg = 'Warning: web2py requires Python 2.4, 2.5 (recommended), 2.6 or 2.7 but you are running:\n%s' msg = msg % sys.version sys.stderr.write(msg) logger = logging.getLogger("web2py") def run_system_tests(options): """
def from_client_secrets_file(cls, client_secrets_file, scopes, access_type='offline', login_hint=None, credentials_file=None, use_console_flow=False): """Runs an OAuth Flow from secrets stored on disk to generate credentials. Args: client_secrets_file: String, path to a file containing a client ID and secret. scopes: Sequence[str], A list of scopes to include in the credentials. access_type: String, 'offline' or 'online'. Indicates whether your application can refresh access tokens when the user is not present at the browser. Valid parameter values are online, which is the default value, and offline. Set the value to offline if your application needs to refresh access tokens when the user is not present at the browser. This is the method of refreshing access tokens described later in this document. This value instructs the Google authorization server to return a refresh token and an access token the first time that your application exchanges an authorization code for tokens. login_hint: String, The email address that will be displayed on the Google login page as a hint for the user to login to the correct account. credentials_file: String, the path to a file to use to save the credentials. use_console_flow: Boolean, True if the authentication flow should be run strictly from a console; False to launch a browser for authentication. Raises: InvalidClientSecretsFileError: If the client secrets file cannot be opened. InvalidClientSecretsFileFormatError: If the client secrets file does not contain the required data or the data is malformed. Returns: Credentials """ cs_data = fileutils.read_file(client_secrets_file, continue_on_error=True, display_errors=False) if not cs_data: raise InvalidClientSecretsFileError( f'File {client_secrets_file} could not be opened') try: cs_json = json.loads(cs_data) client_id = cs_json['installed']['client_id'] # Chop off .apps.googleusercontent.com suffix as it's not needed # and we need to keep things short for the Auth URL. client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id) client_secret = cs_json['installed']['client_secret'] except (ValueError, IndexError, KeyError): raise InvalidClientSecretsFileFormatError( f'Could not extract Client ID or Client Secret from file {client_secrets_file}' ) return cls.from_client_secrets(client_id, client_secret, scopes, access_type=access_type, login_hint=login_hint, filename=credentials_file, use_console_flow=use_console_flow)
pix = 9000 a2t = np.array([1.013743, 1.028457, 1.044197, 1.0999, 1.25]) dat = np.array([2451.456787, 1006.207645, 596.2362145, 238.8109685, 126.4624943]) invN = np.array([0.1667824, 0.1453026, 0.169609, 0.1212046, 0.0884299]) mixmat_base = np.array([[1.0, 23.0 / 94.0], [33.0 / 23.0, 33.0 / 94.0], [41.0 / 23.0, 41.0 / 94.0], [61.0 / 23.0, 61.0 / 94.0], [94.0 / 23.0, 1.0]]) fnames = [] nfnames = [] for i in range(5): fnames.append('signalmaps_%02d.fits' % (i + 1)) nfnames.append('invN_%02d.fits' % (i + 1)) frequencies = np.array([23.0, 33.0, 41.0, 61.0, 94.0]) mds = [] nmds = [] for idx, fname in enumerate(fnames): mds.append(fileutils.read_file(fnames[idx])) nmds.append(fileutils.read_file(nfnames[idx])) #data = [] #err = [] data = dat err = 1 / np.sqrt(invN) #for md, nmd in zip(mds, nmds): # data.append(md.map[pix]) # err.append(1/np.sqrt(nmd.map[pix])) data = data * a2t plt.scatter(frequencies, data) plt.errorbar(frequencies, data, err, fmt=None) plt.plot(frequencies, a2t * 1445.2627 * mixmat_base[:, 0] ** -3.033717) plt.plot(frequencies, a2t * 66.74626193 * mixmat_base[:, 1] ** -1.90380762) plt.plot(frequencies, a2t * (1445.2627 * mixmat_base[:, 0] ** -3.033717 + 66.74626193 * mixmat_base[:, 1] ** -1.90380762))
def run(appname, plain=False, import_models=False, startfile=None, bpython=False, python_code=False): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: a web2py application name a/c exec the controller c into the application environment """ (a, c, f) = parse_path_info(appname) errmsg = "invalid application name: %s" % appname if not a: die(errmsg) adir = os.path.join("applications", a) if not os.path.exists(adir): if raw_input("application %s does not exist, create (y/n)?" % a).lower() in ["y", "yes"]: os.mkdir(adir) w2p_unpack("welcome.w2p", adir) for subfolder in [ "models", "views", "controllers", "databases", "modules", "cron", "errors", "sessions", "languages", "static", "private", "uploads", ]: subpath = os.path.join(adir, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(adir, "models/db.py") if os.path.exists(db): data = fileutils.read_file(db) data = data.replace("<your secret key>", "sha512:" + web2py_uuid()) fileutils.write_file(db, data) if c: import_models = True _env = env(a, c=c, f=f, import_models=import_models) if c: cfile = os.path.join("applications", a, "controllers", c + ".py") if not os.path.isfile(cfile): cfile = os.path.join("applications", a, "compiled", "controllers_%s_%s.pyc" % (c, f)) if not os.path.isfile(cfile): die(errmsg) else: exec read_pyc(cfile) in _env else: execfile(cfile, _env) if f: exec ("print %s()" % f, _env) return # "woodoo magic" workaround: reinitialize main.py g = {} exec "import main" in g del g _env.update(exec_pythonrc()) if startfile: try: execfile(startfile, _env) if import_models: BaseAdapter.close_all_instances("commit") except Exception, e: print traceback.format_exc() if import_models: BaseAdapter.close_all_instances("rollback")
def test_read_file_exits_code_6_when_continue_on_error_is_false( self, mock_open_file): mock_open_file.side_effect = IOError() with self.assertRaises(SystemExit) as context: fileutils.read_file(self.fake_path, continue_on_error=False) self.assertEqual(context.exception.code, 6)
def run_controller_in(controller, function, environment): """ Runs the controller.function() (for the app specified by the current folder). It tries pre-compiled controller_function.pyc first before compiling it. """ # if compiled should run compiled! folder = environment['request'].folder path = pjoin(folder, 'compiled') badc = 'invalid controller (%s/%s)' % (controller, function) badf = 'invalid function (%s/%s)' % (controller, function) if os.path.exists(path): filename = pjoin(path, 'controllers_%s_%s.pyc' % (controller, function)) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) restricted(read_pyc(filename), environment, layer=filename) elif function == '_TEST': # TESTING: adjust the path to include site packages from settings import global_settings from admin import abspath, add_path_first paths = (global_settings.gluon_parent, abspath( 'site-packages', gluon=True), abspath('gluon', gluon=True), '') [add_path_first(path) for path in paths] # TESTING END filename = pjoin(folder, 'controllers/%s.py' % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) environment['__symbols__'] = environment.keys() code = read_file(filename) code += TEST_CODE restricted(code, environment, layer=filename) else: filename = pjoin(folder, 'controllers/%s.py' % controller) if not os.path.exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) code = read_file(filename) exposed = regex_expose.findall(code) if not function in exposed: raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) if is_gae: layer = filename + ':' + function code = getcfs(layer, filename, lambda: compile2(code, layer)) restricted(code, environment, filename) response = environment['response'] vars = response._vars if response.postprocessing: vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars) if isinstance(vars, unicode): vars = vars.encode('utf8') elif hasattr(vars, 'xml') and callable(vars.xml): vars = vars.xml() return vars
def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file): mock_open_file.return_value.__enter__( ).read.side_effect = UnicodeError() with self.assertRaises(SystemExit) as context: fileutils.read_file(self.fake_path) self.assertEqual(context.exception.code, 2)
def app_create(db, app, request, force=False, key=None, info=False): """ Create a copy of welcome.w2p (scaffolding) app Parameters ---------- app: application name request: the global request object """ path = apath(app, request) if not os.path.exists(path): try: os.mkdir(path) except: if info: return False, traceback.format_exc(sys.exc_info) else: return False, None elif not force: if info: return False, "Application exists" else: return False, None try: w2p_unpack('welcome.w2p', path) for subfolder in [ 'models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'cache', 'languages', 'static', 'private', 'uploads']: subpath = os.path.join(path, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) dbt = os.path.join(path, 'models', 'db.py') if os.path.exists(dbt): data = read_file(dbt) data = data.replace('<your secret key>', 'sha512:' + (key or web2py_uuid())) write_file(dbt, data) parms = db(db.parametros.id==1).select()[0] templates = os.path.join('\\\\' , '127.0.0.1' , 'c$' , parms.web2py , 'applications' , parms.soag , 'Template' , 'web2py') for subfolder in ['controllers', 'languages', 'models', 'modules', 'static', 'views']: template = os.path.join(templates, subfolder) subpath = os.path.join(path, subfolder) shutil.rmtree(subpath) shutil.copytree(template, subpath) # shutil.copyfile(os.path.join(templates, 'routes.py'), os.path.join(path, 'routes.py')) if info: return True, None else: return True, None except: shutil.rmtree(path) if info: return False, traceback.format_exc(sys.exc_info) else: return False, None
def run(appname, plain=False, import_models=False, startfile=None, bpython=False, python_code=False): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: a web2py application name a/c exec the controller c into the application environment """ (a, c, f) = parse_path_info(appname) errmsg = 'invalid application name: %s' % appname if not a: die(errmsg) adir = os.path.join('applications', a) if not os.path.exists(adir): if sys.stdin and not sys.stdin.name == '/dev/null': confirm = raw_input( 'application %s does not exist, create (y/n)?' % a) else: logging.warn('application does not exist and will not be created') return if confirm.lower() in ['y', 'yes']: os.mkdir(adir) w2p_unpack('welcome.w2p', adir) for subfolder in [ 'models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'languages', 'static', 'private', 'uploads' ]: subpath = os.path.join(adir, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) db = os.path.join(adir, 'models/db.py') if os.path.exists(db): data = fileutils.read_file(db) data = data.replace('<your secret key>', 'sha512:' + web2py_uuid()) fileutils.write_file(db, data) if c: import_models = True _env = env(a, c=c, f=f, import_models=import_models) if c: cfile = os.path.join('applications', a, 'controllers', c + '.py') if not os.path.isfile(cfile): cfile = os.path.join('applications', a, 'compiled', "controllers_%s_%s.pyc" % (c, f)) if not os.path.isfile(cfile): die(errmsg) else: exec read_pyc(cfile) in _env else: execfile(cfile, _env) if f: exec('print %s()' % f, _env) return _env.update(exec_pythonrc()) if startfile: try: execfile(startfile, _env) if import_models: BaseAdapter.close_all_instances('commit') except Exception, e: print traceback.format_exc() if import_models: BaseAdapter.close_all_instances('rollback')