def main():
parser = argparse.ArgumentParser(
description="This script will rename all the theme elements to removes duplicated elements."
)
parser.add_argument(
'-i', '--iniconfig',
default='production.ini',
help='project .ini config file',
)
parser.add_argument(
'-n', '--app-name',
default="app",
help='The application name (optional, default is "app")',
)
options = parser.parse_args()
# read the configuration
get_app(options.iniconfig, options.app_name)
from c2cgeoportal.models import DBSession, LayerV1, LayerWMS, LayerWMTS, LayerGroup, Theme
for class_ in [LayerV1, LayerWMS, LayerWMTS, LayerGroup, Theme]:
names = []
for item in DBSession.query(class_).all():
if item.name in names:
i = 2
while "{}-{}".format(item.name, i) in names:
i += 1
item.name = "{}-{}".format(item.name, i)
names.append(item.name)
transaction.commit()
def main():
parser = argparse.ArgumentParser(description="Create and populate the database tables.")
parser.add_argument("-i", "--iniconfig", default="production.ini", help="project .ini config file")
parser.add_argument("-n", "--app-name", default="app", help='The application name (optional, default is "app")')
options = parser.parse_args()
# read the configuration
get_app(options.iniconfig, options.app_name)
from c2cgeoportal.models import DBSession, Interface, OGCServer, Theme, LayerGroup, LayerWMS
session = DBSession()
interfaces = session.query(Interface).all()
ogc_server = session.query(OGCServer).filter(OGCServer.name == u"source for image/png").one()
layer_borders = LayerWMS(u"Borders", u"borders")
layer_borders.interfaces = interfaces
layer_borders.ogc_server = ogc_server
layer_density = LayerWMS(u"Density", u"density")
layer_density.interfaces = interfaces
layer_density.ogc_server = ogc_server
group = LayerGroup(u"Demo")
group.children = [layer_borders, layer_density]
theme = Theme(u"Demo")
theme.children = [group]
theme.interfaces = interfaces
transaction.commit()
def run(self):
# Check if the pyramid config file is set, if so use paster to manage
# imports
conf_file = self.pyramid_conf()
if conf_file:
paster.get_app(conf_file)
else:
# instantiate a pyramid configurator
config = Configurator()
module_str = self.arguments[0]
# import the module
module = import_module(module_str)
# check if the module has an `includeme` method and call it
# because the base route must be added
if hasattr(module, 'includeme'):
module.includeme(config)
config.commit()
# scan the module for services
scanner = venusian.Scanner(config=config)
scanner.scan(module)
rendered = []
# fetch all services
services = get_services()
# if the services option is set render only the named services
names = self.options.get('services')
if names:
services = [s for s in services if s.baseRouteName in names]
for service in services:
service_id = "service_%d" % self.serialno('service')
rendered.append(ServiceDirective.render(service, service_id))
return rendered
def export_talk_preferences(**kw):
parser = ArgumentParser(description='Export talk preferences to CSV')
parser.add_argument('-c', '--config', type=str, help='app configuration',
default='development.ini')
args = vars(parser.parse_args(**kw))
get_app(args['config']).registry.settings
from .models import TalkPreference
for preference in TalkPreference.query:
print ','.join([str(talk_id) for talk_id in preference.talk_ids])
def add_user(**kw):
parser = ArgumentParser(description="Create user account")
parser.add_argument("-c", "--config", type=str, default="production.ini", help="app configuration file")
parser.add_argument("email", type=str, help="email of the user")
parser.add_argument("-f", "--firstname", type=str, help="first name of the user")
parser.add_argument("-l", "--lastname", type=str, help="last name of the user")
parser.add_argument("-p", "--password", type=str, help="password of the user (will be encrypted)")
parser.add_argument("-a", "--active", type=bool, default=True, help="is the user active")
parser.add_argument("-r", "--global_roles", nargs="*", type=str, help="one or more global roles")
data = vars(parser.parse_args(**kw))
get_app(abspath(data.pop("config"))) # setup application
Principal(**data)
commit()
def main(): # pragma: no cover
parser = ArgumentParser(
prog=sys.argv[0], add_help=True,
description="Tool used to migrate your old layers from the old structure to the new one.",
)
parser.add_argument(
"-i", "--app-config",
default="production.ini",
dest="app_config",
help="the application .ini config file (optional, default is 'production.ini')"
)
parser.add_argument(
"-n", "--app-name",
default="app",
dest="app_name",
help="the application name (optional, default is 'app')"
)
options = parser.parse_args()
app_config = options.app_config
app_name = options.app_name
if app_name is None and "#" in app_config:
app_config, app_name = app_config.split("#", 1)
get_app(app_config, name=app_name)
# must be done only once we have loaded the project config
from c2cgeoportal.models import DBSession, \
ServerOGC, LayerWMS, LayerWMTS, LayerV1
session = DBSession()
table_list = [LayerWMTS, LayerWMS, ServerOGC]
for table in table_list:
print "Emptying table %s." % str(table.__table__)
# must be done exactly this way othewise the cascade config in the
# models are not used
for t in session.query(table).all():
session.delete(t)
# list and create all distinct server_ogc
server_ogc(session)
print "Converting layerv1."
for layer in session.query(LayerV1).all():
layer_v1tov2(session, layer)
transaction.commit()
def main():
if len(sys.argv) != 3:
sys.exit("Usage: python -m test4.scripts.create_db INI_FILE OUT_FILE")
ini_file = sys.argv[1]
out_path = sys.argv[2]
if os.path.exists(out_path):
raise Exception("%s already exists" % out_path)
logging.config.fileConfig(ini_file)
log = logging.getLogger(__name__)
app = get_app(ini_file, "PersonalGallery")
settings = app.registry.settings
config.config = settings
s = models.DBSession()
if not out_path.endswith(".tar.bz2"):
raise Exception("Only .tar.bz2 archives supported")
out_name = os.path.basename(out_path)
out_name = out_name[:-len(".tar.bz2")]
tmpdir = tempfile.mkdtemp()
try:
q = s.query(Album).filter_by(id = 1)
a = q.one()
backup_album(tmpdir, s, a)
t = tarfile.open(out_path, "w:bz2")
t.add(tmpdir, out_name, recursive = True)
t.close()
finally:
shutil.rmtree(tmpdir)
def main():
parser = argparse.ArgumentParser(
description="Create Elasticsearch mapping on deployment", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('config_uri', help="path to configfile")
parser.add_argument('--app-name', help="Pyramid app name in configfile")
args = parser.parse_args()
app = get_app(args.config_uri, args.app_name)
# Loading app will have configured from config file. Reconfigure here:
set_logging(in_prod=app.registry.settings.get('production'), level=logging.DEBUG)
# set_logging(app.registry.settings.get('elasticsearch.server'), app.registry.settings.get('production'), level=logging.DEBUG)
# check if staging
try:
data_env = whodaman()
env = app.registry.settings.get('env.name')
if 'webprod' in env:
if data_env != env:
log.info("looks like we are on staging, run create mapping without check first")
run_create_mapping(app, check_first=False)
return
# handle mastertest ... by blowing away all data first
if 'mastertest' in env:
log.info("looks like we are on mastertest, run create mapping without check first")
run_create_mapping(app, check_first=False, purge_queue=True)
return
log.info("looks like we are NOT on staging or mastertest so run create mapping with check first")
except Exception:
import traceback
log.warning("error checking whodaman: %s " % traceback.format_exc())
log.warning("couldn't get wodaman, so assuming NOT staging")
log.info("... using default create mapping case")
run_create_mapping(app, check_first=True, purge_queue=True)
def main():
from pyramid.paster import get_app
from pyramid.scripting import get_root
from ..resources import BlogEntry
parser = OptionParser(description=__doc__, usage='usage: %prog [options]')
parser.add_option('-c', '--config', dest='config',
help='Specify a paster config file.')
parser.add_option('-n', '--name', dest='name', default='main',
help='The paster config file section indicating the app.')
parser.add_option('-i', '--num', dest='num', default='10000',
help='Specify the number of blog entries to add.')
options, args = parser.parse_args()
config = options.config
name = options.name
num = int(options.num)
if config is None:
raise ValueError('must supply config file name')
config = os.path.abspath(os.path.normpath(config))
app = get_app(config, name)
root, closer = get_root(app)
for n in range(0, num):
print ("adding", n)
entry = BlogEntry('title %s' % n, 'entry %s' % n,
'html', datetime.date.today())
id = 'blogentry_%s' % n
root[id] = entry
if n % 10000 == 0:
print ('committing')
transaction.commit()
print ('committing')
transaction.commit()
root._p_jar._db.close()
closer()
def start_applications(self, debug=False):
settings = get_appsettings(self.config_path)
not_found_application = settings.local_conf.pop(
'not_found_application',
settings.global_conf.get('not_found_application'))
if not_found_application:
not_found_application = loadapp(
not_found_application,
global_conf=settings.global_conf)
else:
not_found_application = not_found_api_application(
settings.global_conf, **settings.local_conf)
self.not_found_application = not_found_application
self.validate_config_seconds = maybe_integer(
settings.local_conf.pop('validate_config_seconds', None))
for path, app_name in settings.local_conf.items():
path = parse_path_expression(path)
self[path] = get_app(self.config_path, app_name)
if debug:
print (
'Application %s reloaded on pid %s'
% (app_name, getpid()))
def setUp(self):
from pyramid.paster import get_app
app = get_app('test.ini')
self.theapp = app
from webtest import TestApp
self.testapp = TestApp(app)
self.jobs_lib = jobslib.get_jobs_lib()
def setUp(self):
self.session = _initTestingDB()
self.config = testing.setUp()
from pyramid.paster import get_app
app = get_app('development.ini')
from webtest import TestApp
self.testapp = TestApp(app)
def setUp(self):
"""Setup Tests"""
from pyramid.paster import get_app
app = get_app('test.ini', 'main')
from webtest import TestApp
self.testapp = TestApp(app)
testing.setUp()
def setUp(self):
self.config = testing.setUp()
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (
Base,
Locale,
User,
Passhash,
Client
)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id = 1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
def setUpModule():
global browser, server
inipath = os.path.abspath(
os.environ.get('TEST_INI', 'testing.ini'))
if not os.path.isfile(inipath):
raise RuntimeError(
'Cannot find INI file to setup selenium tests. '
'Please specify the path via the TEST_INI environment variable '
'or by adding a testing.ini file to the current directory.')
parser = ConfigParser()
parser.read(inipath)
config.update(parser.items('testconfig'))
config['test_providers'] = splitlines(config['test_providers'])
app = get_app(inipath)
port = int(config['app_port'])
server = StopableWSGIServer.create(app, port=port)
driver = config.get('selenium.driver', 'firefox')
browser = {
'firefox': webdriver.Firefox,
'chrome': webdriver.Chrome,
'ie': webdriver.Ie,
}[driver]()
def main():
''' Indexes app data loaded to elasticsearch '''
import argparse
parser = argparse.ArgumentParser(
description="Index data in Elastic Search", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--item-type', action='append', help="Item type")
parser.add_argument('--record', default=False, action='store_true', help="Record the xmin in ES meta")
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('config_uri', help="path to configfile")
args = parser.parse_args()
logging.basicConfig()
options = {
'embed_cache.capacity': '5000',
'indexer': 'true',
'mpindexer': 'true',
}
app = get_app(args.config_uri, args.app_name, options)
# Loading app will have configured from config file. Reconfigure here:
logging.getLogger('encoded').setLevel(logging.DEBUG)
return run(app, args.item_type, args.record)
def setUp(self):
"""Launch app using webtest with test settings"""
self.appconf = get_app(test_ini)
self.app = TestApp(self.appconf)
#All auth via BasicAuth - never return the session cookie.
self.app.cookiejar.set_policy(DefaultCookiePolicy(allowed_domains=[]))
# This sets global var "engine" - in the case of SQLite this is a fresh RAM
# DB each time. If we only did this on class instantiation the database would
# be dirty and one test could influence another.
# TODO - add a test that tests this.
server.choose_engine("SQLite")
# Punch in new user account with direct server call
# This will implicitly generate the tables.
user_id = self.create_user("testuser")
#Here is what the user should look like when inspected
self.user_json = { "name" : "testuser testuser",
"handle" : "*****@*****.**",
"id" : 1,
"credits" : 0,
"username": "******"}
#print("user_id is %s" % str(user_id))
#print("user_from_db_is %s" % server.get_user_id_from_name("testuser"))
server.touch_to_add_password(user_id, "asdf")
# And log in as this user for all tests (via BasicAuth)
# FIXME - switch to token auth to speed up the tests.
self.app.authorization = ('Basic', ('testuser', 'asdf'))
def setUp(self):
'''Create the app'''
test_path = os.path.abspath(os.path.dirname(__file__))
testpath_command = "setglobal test_path " + test_path
twill.execute_string(testpath_command)
fixtures = os.path.join(test_path, 'fixtures')
for to_delete in [fname for fname in os.listdir(fixtures)
if fname.startswith('Data.fs') or fname in ['blobs']]:
_rm(os.path.join(fixtures, to_delete))
os.mkdir(os.path.join(fixtures, 'blobs'))
wsgi_app = get_app(os.path.join(test_path, 'fixtures', 'karl.ini'),
'main')
def build_app():
return wsgi_app
twill.add_wsgi_intercept('localhost', 6543, build_app)
# XXX How do we suppress the annoying "AT LINE: " output?
twill.set_output(open('/dev/null', 'wb'))
twill.execute_string("extend_with karl.twillcommands")
# mostly the same as karl3.conf without extending with flunc
# and few other adjustments.
twill.execute_string("runfile '" +
os.path.abspath(os.path.dirname(__file__)) +
"/test_twill_wsgi_karl3.conf'")
def main():
import argparse
parser = argparse.ArgumentParser(
description="Move attachment blobs to S3", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('--abort', action='store_true', help="Rollback transaction")
parser.add_argument('config_uri', help="path to configfile")
args = parser.parse_args()
logging.basicConfig()
app = get_app(args.config_uri, args.app_name)
# Loading app will have configured from config file. Reconfigure here:
logging.getLogger('encoded').setLevel(logging.DEBUG)
raised = False
try:
run(app)
except:
raised = True
raise
finally:
if raised or args.abort:
transaction.abort()
logger.info('Rolled back.')
else:
transaction.commit()
def main(port):
ini_path = './production.ini'
setup_logging(ini_path)
app = get_app(ini_path, 'main')
server = WSGIServer(('', port), app)
server.backlog = 256
server.serve_forever()
def main():
if len(sys.argv) != 2:
sys.exit("Usage: python -m test4.scripts.create_db INI_FILE")
ini_file = sys.argv[1]
logging.config.fileConfig(ini_file)
log = logging.getLogger(__name__)
app = get_app(ini_file, "PersonalGallery")
settings = app.registry.settings
config.config = settings
# Abort if any tables exist to prevent accidental overwriting
for table in models.Base.metadata.sorted_tables:
log.debug("checking if table '%s' exists", table.name)
if table.exists():
raise RuntimeError("database table '%s' exists" % table.name)
# Create the tables
models.Base.metadata.create_all()
sess = models.DBSession()
sess.commit()
album = models.Album(name = "root")
sess.add(album)
sess.commit()
def setUp(self):
"""Launch pserve using webtest with test settings"""
self.appconf = get_app(test_ini)
self.app = TestApp(self.appconf)
#For speed, allow cookie setting.
# self.app.cookiejar.set_policy(DefaultCookiePolicy(allowed_domains=[]))
# This sets global var "engine" - in the case of SQLite this is a fresh RAM
# DB each time. If we only did this on class instantiation the database would
# be dirty and one test could influence another.
# TODO - add a test that tests this.
server.choose_engine("SQLite")
# Punch in new administrator account with direct server call
# This will implicitly generate the tables.
user_id = server.create_user("administrators", "administrator", "administrator", "administrator")
#server.touch_to_add_user_group("administrator", "administrators")
server.touch_to_add_password(user_id, "adminpass")
self.app.authorization = ('Basic', ('administrator', 'adminpass'))
# This sorts out the auth token cookie.
self.app.get('/users')
self.app.authorization = None
def setUp(self):
from pyramid.paster import get_app
from webtest import TestApp
app = get_app("testing.ini")
self.testapp = TestApp(app)
self.session = _initDB()
def main():
import argparse
parser = argparse.ArgumentParser(
description="Migrate files to AWS", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('--abort', action='store_true', help="Rollback transaction")
parser.add_argument('files_processed', type=argparse.FileType('rb'), help="path to json file")
parser.add_argument('config_uri', help="path to configfile")
args = parser.parse_args()
logging.basicConfig()
app = get_app(args.config_uri, args.app_name)
# Loading app will have configured from config file. Reconfigure here:
logging.getLogger('encoded').setLevel(logging.DEBUG)
files_processed = json.load(args.files_processed)
good_files = {v['uuid']: v for v in files_processed
if 'errors' not in v and 'blacklisted' not in v}
raised = False
try:
run(app, good_files)
except:
raised = True
raise
finally:
if raised or args.abort:
transaction.abort()
logger.info('Rolled back.')
else:
transaction.commit()
def fetch_talks(**kw):
parser = ArgumentParser(description='Fetch talks from frab')
parser.add_argument('-c', '--config', type=str, help='app configuration',
default='development.ini')
args = vars(parser.parse_args(**kw))
settings = get_app(args['config']).registry.settings
sess = requests.Session()
new_session_page = sess.get(settings['talks_new_session_url'])
tree = etree.HTML(new_session_page.text)
auth_token = tree.xpath("//meta[@name='csrf-token']")[0].get("content")
login_data = dict()
login_data['user[email]'] = settings['talks_user']
login_data['user[password]'] = settings['talks_password']
login_data['user[remember_me]'] = 1
login_data['authenticity_token'] = auth_token
sess.post(settings['talks_login_url'], login_data, verify=False)
talks_json = sess.get(settings['talks_url'], verify=False, stream=True)
talks_full = ''
with open(settings['talks_full'], 'wb') as fd:
for chunk in talks_json.iter_content(1024):
fd.write(chunk)
talks_full += chunk
talks_full = json.loads(talks_full)
talks_filtered = [{ key: x[key] for key in [ 'event_id', 'track_id', 'room_id', 'duration', 'start_time', 'title', 'abstract', 'language', 'speaker_names', 'language' ] } for x in talks_full ]
with open(settings['talks_local'], 'wb') as fd:
json.dump(talks_filtered, fd)
def testapp(request):
import transaction
from pyramid.paster import get_app
app = get_app(TESTS_INI)
maker = app.registry['db_sessionmaker']
engine = maker.kw['bind']
connection = engine.connect()
# Start a transaction outside of the transaction manager
outer_transaction = connection.begin()
# Rebind the connection directly instead of the engine
maker.configure(bind=connection)
@request.addfinalizer
def cleanup():
# Abort the transaction under the manager, from the test
transaction.abort()
# Rollback the outermost transaction and clean up
outer_transaction.rollback()
connection.close()
from webtest import TestApp
return TestApp(app)
def setUp(self):
from pyramid.paster import get_app
from bookie.tests import BOOKIE_TEST_INI
app = get_app(BOOKIE_TEST_INI, 'main')
from webtest import TestApp
self.testapp = TestApp(app)
testing.setUp()
def setUp(self):
from pyramid.paster import get_app
app = get_app('development.ini')
self.testapp = TestApp(app)
self.grids = {
'21781': getTileGrid(21781),
'2056': getTileGrid(2056)
}
def customization_app(request, db_session):
"""
Use this fixture to retreive a TestApp object which can be used as self.app
in the test functions.
"""
request.cls.app = TestApp(get_app(LO_CUSTOMIZATION_TESTS_INI))
request.cls.db_session = db_session
return request
def main():
parser = argparse.ArgumentParser(
description="Create and populate the database tables."
)
parser.add_argument(
'-i', '--iniconfig',
default='geoportal/production.ini',
help='project .ini config file'
)
parser.add_argument(
'-n', '--app-name',
default="app",
help='The application name (optional, default is "app")'
)
options = parser.parse_args()
# read the configuration
fileConfig(options.iniconfig, defaults=os.environ)
get_app(options.iniconfig, options.app_name, options=os.environ)
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Interface, OGCServer, Theme, LayerGroup, LayerWMS
session = DBSession()
interfaces = session.query(Interface).all()
ogc_server = session.query(OGCServer).filter(OGCServer.name == "source for image/png").one()
layer_borders = LayerWMS("Borders", "borders")
layer_borders.interfaces = interfaces
layer_borders.ogc_server = ogc_server
layer_density = LayerWMS("Density", "density")
layer_density.interfaces = interfaces
layer_density.ogc_server = ogc_server
group = LayerGroup("Demo")
group.children = [layer_borders, layer_density]
theme = Theme("Demo")
theme.children = [group]
theme.interfaces = interfaces
session.add(theme)
transaction.commit()
def setUp(self):
from pyramid.paster import get_app
app = get_app('development.ini')
from webtest import TestApp
self.testapp = TestApp(app)
def main():
"""
Emergency user create and password reset script
exemple, reset toto password to foobar:
.build/venv/bin/manage_users -p foobar toto
exemple, create user foo with password bar and role admin:
.build/venv/bin/manage_users -c -r role_admin -p bar foo
to get the options list, do:
.build/venv/bin/manage_users -h
"""
usage = """Usage: %prog [options] USERNAME
Reset a user password.
The username is used as password if the password is not provided with the corresponding option.
User can be created if it doesn't exist yet."""
parser = OptionParser(usage)
parser.add_option(
"-i",
"--app-config",
default="production.ini",
dest="app_config",
help="The application .ini config file (optional, default is "
"'production.ini')")
parser.add_option("-n",
"--app-name",
default="app",
dest="app_name",
help="The application name (optional, default is 'app')")
parser.add_option(
"-p",
"--password",
help="Set password (if not set, username is used as password")
parser.add_option("-c",
"--create",
action="store_true",
default=False,
help="Create user if it doesn't already exist")
parser.add_option("-r",
"--rolename",
default="role_admin",
help="The role name which must exist in the database")
parser.add_option("-e", "--email", default=None, help="The user email")
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("You must specify a username")
username = args[0]
app_config = options.app_config
app_name = options.app_name
if app_name is None and "#" in app_config:
app_config, app_name = app_config.split("#", 1)
if not os.path.isfile(app_config):
parser.error("Can't find config file: %s" % app_config)
# loading schema name from config and setting its value to the
# corresponding global variable from c2cgeoportal
# Ignores pyramid deprecation warnings
warnings.simplefilter("ignore", DeprecationWarning)
get_app(app_config, name=app_name)
# must be done only once we have loaded the project config
from c2cgeoportal import models
print("\n")
# check that User and Role exist in model
model_list = ["User", "Role"]
for model in model_list:
try:
getattr(models, model)
except AttributeError:
print("models.%s not found" % model)
# check that user exists
sess = models.DBSession()
query = sess.query(models.User).filter_by(username=u"%s" % username)
result = query.count()
if result == 0:
if not options.create:
# if doesn"t exist and no -c option, throw error
raise Exception("User %s doesn't exist in database" % username)
else:
print("User %s doesn't exist in database, creating" % username)
# if doesn't exist and -c option, create user
password = options.password if options.password is not None else username
email = options.email if options.email is not None else username
# get roles
query_role = sess.query(models.Role).filter(
models.Role.name == u"%s" % options.rolename)
if query_role.count() == 0:
# role not found in db?
raise Exception("Role matching %s doesn't exist in database" %
options.rolename)
role = query_role.first()
user = models.User(username=u"%s" % username,
password=u"%s" % password,
email=u"%s" % email,
role=role)
sess.add(user)
transaction.commit()
print("User %s created with password %s and role %s" %
(username, password, options.rolename))
else:
# if user exists (assuming username are unique)
user = query.first()
if options.password is not None:
print("Password set to: %s" % options.password)
user.password = u"%s" % options.password
if options.email is not None:
user.email = options.email
sess.add(user)
transaction.commit()
print("Password resetted for user %s" % username)
from pyramid.paster import get_app
application = get_app(
'/var/www/argomenti.in/aranciulla/webkeywords/production.ini', 'main')
def setUp(self):
from pyramid.paster import get_app
app = get_app(BOOKIE_TEST_INI, 'bookie')
from webtest import TestApp
self.testapp = TestApp(app)
testing.setUp()
def main():
parser = ArgumentParser(
prog=sys.argv[0],
add_help=True,
description=
"Tool used to migrate your old layers from the old structure to the new one.",
)
parser.add_argument(
"-i",
"--app-config",
default="geoportal/production.ini",
dest="app_config",
help=
"the application .ini config file (optional, default is 'production.ini')"
)
parser.add_argument(
"-n",
"--app-name",
default="app",
dest="app_name",
help="the application name (optional, default is 'app')")
parser.add_argument("--no-layers",
dest="layers",
action="store_false",
help="do not import the layers")
parser.add_argument("--no-groups",
dest="groups",
action="store_false",
help="do not import the groups")
parser.add_argument("--no-themes",
dest="themes",
action="store_false",
help="do not import the themes")
options = parser.parse_args()
app_config = options.app_config
app_name = options.app_name
if app_name is None and "#" in app_config:
app_config, app_name = app_config.split("#", 1)
fileConfig(app_config, defaults=os.environ)
app = get_app(app_config, app_name, options=os.environ)
# must be done only once we have loaded the project config
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import OGCServer, Theme, LayerWMS, LayerWMTS, LayerV1, LayerGroup
session = DBSession()
if options.layers:
table_list = [LayerWMTS, LayerWMS, OGCServer]
for table in table_list:
print(("Emptying table {0!s}.".format(table.__table__)))
# must be done exactly this way othewise the cascade config in the
# models are not used
for t in session.query(table).all():
session.delete(t)
# list and create all distinct ogc_server
ogc_server(session, app.registry.settings)
print("Converting layerv1.")
for layer in session.query(LayerV1).all():
layer_v1tov2(session, layer)
if options.groups:
print("Converting layer groups.")
for group in session.query(LayerGroup).all():
layergroup_v1tov2(session, group)
if options.themes:
print("Converting themes.")
for theme in session.query(Theme).all():
theme_v1tov2(session, theme)
transaction.commit()
def lux_get_app(app_config, app_name):
environ = escape_variables(os.environ)
return get_app(app_config, app_name, options=environ)
def main():
args = parse_args()
app = get_app(args.config_uri, args.app_name)
with transaction.manager:
run(app, args.infile, args.update, args.batch_size)
def main():
import argparse
parser = argparse.ArgumentParser(
description="Run development servers",
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('config_uri', help="path to configfile")
parser.add_argument('--clear',
action="store_true",
help="Clear existing data")
parser.add_argument('--init', action="store_true", help="Init database")
parser.add_argument('--load', action="store_true", help="Load test set")
parser.add_argument('--datadir',
default='/tmp/snovault',
help="path to datadir")
args = parser.parse_args()
from snovault.tests import elasticsearch_fixture, postgresql_fixture
from snovault.elasticsearch import create_mapping
datadir = os.path.abspath(args.datadir)
pgdata = os.path.join(datadir, 'pgdata')
esdata = os.path.join(datadir, 'esdata')
if args.clear:
for dirname in [pgdata, esdata]:
if os.path.exists(dirname):
shutil.rmtree(dirname)
if args.init:
postgresql_fixture.initdb(pgdata, echo=True)
postgres = postgresql_fixture.server_process(pgdata, echo=True)
elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True)
nginx = nginx_server_process(echo=True)
processes = [postgres, elasticsearch, nginx]
@atexit.register
def cleanup_process():
for process in processes:
if process.poll() is None:
process.terminate()
for process in processes:
try:
for line in process.stdout:
sys.stdout.write(line.decode('utf-8'))
except IOError:
pass
process.wait()
if args.init:
app = get_app(args.config_uri, args.app_name)
create_mapping.run(app)
if args.load:
from pyramid.path import DottedNameResolver
load_test_data = app.registry.settings.get('snovault.load_test_data')
load_test_data = DottedNameResolver().resolve(load_test_data)
load_test_data(app)
print('Started. ^C to exit.')
stdouts = [p.stdout for p in processes]
# Ugly should probably use threads instead
while True:
readable, writable, err = select.select(stdouts, [], stdouts, 5)
for stdout in readable:
for line in iter(stdout.readline, b''):
sys.stdout.write(line.decode('utf-8'))
if err:
for stdout in err:
for line in iter(stdout.readline, b''):
sys.stdout.write(line.decode('utf-8'))
break
do_nothing_script_name = 'gwebdemo_do_nothing'
import sys
import os
from imp import load_source
from copy import copy
# Grab a copy of the original sys.path
calling_sys_path = copy(sys.path)
# Work out the do_nothing_script_path
loader_py_path = os.path.realpath(__file__)
loader_py_path = os.path.abspath(loader_py_path)
app_dir = os.path.dirname(loader_py_path)
do_nothing_script_path = os.path.join(app_dir, 'bin', do_nothing_script_name)
# Load our script if it were a module, this will alter the sys.path to
# that of our egg.
load_source(do_nothing_script_name, do_nothing_script_path)
# Prepend the original calling sys.path so we ensure that
# we use GAE paths before ours.
sys.path = sys.path + calling_sys_path
# Load application
from pyramid.paster import get_app
application = get_app('gae.ini', 'main')
#!/usr/bin/env python
import os
from pyramid.config import Configurator
from pyramid.response import Response
from pyramid.paster import get_app
#app = config.make_wsgi_app()
path = '/'.join(os.path.abspath(__file__).split('/')[0:-1])
application = get_app(path + '/development.ini', 'main')
#
# Below for testing only
#
if __name__ == '__main__':
from wsgiref.simple_server import make_server
httpd = make_server('0.0.0.0', 6543, application)
#httpd = make_server('0.0.0.0', 3000, application)
httpd.serve_forever()
from pyramid.paster import get_app,setup_logging init_path = "production.ini" setup_logging(init_path) app = get_app(init_path, 'main')
from pyramid.paster import get_app, setup_logging import os os.environ['PYTHON_EGG_CACHE'] = '/tmp/.python-eggs' __here__ = os.path.dirname(os.path.abspath(__file__)) __parent__ = os.path.dirname(__here__) ini_path = '%s/development.ini' % (__parent__) setup_logging(ini_path) application = get_app(ini_path, 'main')
def start_simple_wsgi_server(self):
from pyramid.paster import get_app
application = get_app('development.ini', 'main')
from webtest import TestApp
self.app = TestApp(application)
def setUp(self):
from pyramid.paster import get_app
app = get_app('test.ini', 'main')
from webtest import TestApp
self.testapp = TestApp(app)
testing.setUp()
def main():
parser = OptionParser("Create and populate the database tables.")
parser.add_option('-i',
'--iniconfig',
default='production.ini',
help='project .ini config file')
parser.add_option('-n',
'--app-name',
default="app",
help='The application name (optional, default is "app")')
parser.add_option('-d',
'--drop',
action="store_true",
default=False,
help='drop the table if already exists')
parser.add_option('-p',
'--populate',
action="store_true",
default=False,
help='populate the database')
(options, args) = parser.parse_args()
logging.config.fileConfig(options.iniconfig)
log = logging.getLogger(__name__)
# read the configuration
app = get_app(options.iniconfig, options.app_name)
# should be import after the configuration is read.
from demo import models
from c2cgeoportal import models as c2cmodels
from c2cgeoportal import schema
if options.drop:
print "Dropping tables"
for table in reversed(c2cmodels.Base.metadata.sorted_tables):
if table.schema == schema:
print "Dropping table %s" % table.name
table.drop(bind=c2cmodels.DBSession.bind, checkfirst=True)
print "Creating tables"
for table in c2cmodels.Base.metadata.sorted_tables:
if table.schema == schema:
print "Creating table %s" % table.name
table.create(bind=c2cmodels.DBSession.bind, checkfirst=True)
sess = c2cmodels.DBSession()
admin = c2cmodels.User(
username=u'admin',
password=u'admin',
)
roleadmin = c2cmodels.Role(name=u'role_admin')
admin.role = roleadmin
sess.add_all([admin, roleadmin])
if options.populate:
print "Populate the Database"
# add the objects creation there
sess.add_all([]) # add the oblect that we want to commit in the array
transaction.commit()
print "Commited successfully"
from socketio.server import SocketIOServer
from pyramid.paster import get_app
from gevent import monkey
monkey.patch_all()
if __name__ == '__main__':
app = get_app('development.ini')
print 'Listening on port http://0.0.0.0:8080 and on port 10843 (flash policy server)'
SocketIOServer(('0.0.0.0', 8080),
app,
resource="socket.io",
policy_server=True,
policy_listener=('0.0.0.0', 10843)).serve_forever()
def main():
import argparse
parser = argparse.ArgumentParser(
description="Run development servers", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('config_uri', help="path to configfile")
parser.add_argument('--clear', action="store_true", help="Clear existing data")
parser.add_argument('--init', action="store_true", help="Init database")
parser.add_argument('--load', action="store_true", help="Load test set")
parser.add_argument('--datadir', default='/tmp/encoded', help="path to datadir")
args = parser.parse_args()
logging.basicConfig()
# Loading app will have configured from config file. Reconfigure here:
logging.getLogger('encoded').setLevel(logging.DEBUG)
from encoded.tests import elasticsearch_fixture, postgresql_fixture
from encoded.commands import create_mapping
datadir = os.path.abspath(args.datadir)
pgdata = os.path.join(datadir, 'pgdata')
esdata = os.path.join(datadir, 'esdata')
if args.clear:
for dirname in [pgdata, esdata]:
if os.path.exists(dirname):
shutil.rmtree(dirname)
if args.init:
postgresql_fixture.initdb(pgdata, echo=True)
postgres = postgresql_fixture.server_process(pgdata, echo=True)
elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True)
processes = [postgres, elasticsearch]
@atexit.register
def cleanup_process():
for process in processes:
if process.poll() is None:
process.terminate()
for process in processes:
try:
for line in process.stdout:
sys.stdout.write(line)
except IOError:
pass
process.wait()
if args.init:
app = get_app(args.config_uri, args.app_name)
create_mapping.run(app)
if args.load:
from webtest import TestApp
environ = {
'HTTP_ACCEPT': 'application/json',
'REMOTE_USER': '******',
}
testapp = TestApp(app, environ)
from encoded.loadxl import load_all
from pkg_resources import resource_filename
inserts = resource_filename('encoded', 'tests/data/inserts/')
docsdir = [resource_filename('encoded', 'tests/data/documents/')]
load_all(testapp, inserts, docsdir)
print('Started. ^C to exit.')
stdouts = [p.stdout for p in processes]
# Ugly should probably use threads instead
while True:
readable, writable, err = select.select(stdouts, [], stdouts, 5)
for stdout in readable:
for line in iter(stdout.readline, ''):
sys.stdout.write(line)
if err:
for stdout in err:
for line in iter(stdout.readline, ''):
sys.stdout.write(line)
break
# -*- coding: utf-8 -*-
'''Serves MootiroForm using the high-performance CherryPy WSGI server.
(I don't think the Paste http server is very good,
it has been printing mysterious error messages on the console.)
Before running this script, install CherryPy 3.2 or later:
easy_install -UZ cherrypy
'''
from __future__ import unicode_literals # unicode by default
from cherrypy.wsgiserver import CherryPyWSGIServer
from pyramid.paster import get_app
app = get_app('development.ini', 'mootiro_form')
port = 6545
server = CherryPyWSGIServer(
('0.0.0.0', port),
app,
numthreads=10,
max=10,
request_queue_size=200,
timeout=120,
)
if __name__ == '__main__':
print('Running MootiroForm under CherryPy on http://localhost:{}/' \
.format(port))
try:
server.start()
def setUp(self, config_file=None):
self.wsgi_app = get_app(default_config_file)
self.app = TestApp(self.wsgi_app)
from waitress import serve
from pyramid.paster import get_app, setup_logging
import os, argparse
# custom `app.py` due to os.environs...
parser = argparse.ArgumentParser()
parser.add_argument('--d', action='store_true', help='run in dev mode')
if parser.parse_args().d:
print('*' * 10)
print('RUNNING IN DEV MODE')
print('*' * 10)
configfile = 'development.ini'
else:
configfile = 'production.ini'
setup_logging(configfile)
app = get_app(configfile, 'main', options={'SQL_URL': os.environ['SQL_URL']})
serve(app, host='0.0.0.0', port=8088, threads=50)
def _callFUT(self, config_file, section_name, loadapp):
from pyramid.paster import get_app
return get_app(config_file, section_name, loadapp)
def create_configuration_files_and_restart_apps(config):
app = get_app(abspath(config), name='senic_hub_backend')
create_configuration_files_and_restart_apps_(app.registry.settings)
def setUp(self):
print 'a'
app = paster.get_app('development.ini')
from webtest import TestApp
self.testapp = TestApp(app)
import os.path from pyramid.paster import get_app here = os.path.dirname(__file__) application = get_app(os.path.join(here, 'production.ini'), 'main')
from pyramid.paster import get_app from waitress import serve appIniPath = 'development.ini' app = get_app(appIniPath) serve(app, host="0.0.0.0", port=8080)
print(f'Environment variable {ev}: present')
elif environmental[ev] is None:
print(f'Environment variable {ev}: skipping')
else:
print(f'Environment variable {ev}: defaulting to {environmental[ev]}')
os.environ[ev] = environmental[ev]
# ======================================================================================================================
# custom `app.py` due to os.environs...
parser = argparse.ArgumentParser()
parser.add_argument('--d', action='store_true', help='run in dev mode')
if parser.parse_args().d:
print('*' * 10)
print('RUNNING IN DEV MODE')
print('*' * 10)
configfile = 'development.ini'
else:
configfile = 'production.ini'
# ======================================================================================================================
setup_logging(configfile)
app = get_app(configfile, 'main',
options={'SQL_URL':
os.environ['SQL_URL']}) #pyramid.router.Router
# ======================================================================================================================
if __name__ == '__main__':
serve(app, host='0.0.0.0', port=8088, threads=50)
def run(search_url, username='', password=''):
url = urlparse(search_url)
if url.scheme not in ('http', 'https') or '/search/?type=' not in search_url:
raise Exception('Invalid URL supplied.')
# Loading app will have configured from config file. Reconfigure here:
logging.getLogger('encoded').setLevel(logging.INFO)
logging.getLogger('wsgi').setLevel(logging.WARNING)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
auth_to_use = (username, password) if username and password else None
common_headers = {
'Accept' : "application/json",
'Content-Type' : "application/json"
}
app = paster.get_app('development.ini', 'app')
search_url_to_request = search_url + '&field=@id'
logger.info('Fetching Item @IDs via "' + search_url_to_request + '"')
search_response = requests.get(search_url_to_request, auth=auth_to_use, headers=common_headers)
search_results = search_response.json().get('@graph', [])
search_results_len = len(search_results)
counter = { 'count' : 0 }
def worker(search_result):
def perform_request(uri, attempt = 1):
if attempt > 5:
raise Exception('Failed 5x to get response for ' + uri)
try:
object_response = requests.get(item_uri, auth=auth_to_use, headers=common_headers)
except (requests.exceptions.ConnectionError, gaierror) as e:
logger.info('Encountered connection error during request to "' + item_uri + '"')
time.sleep(3 * attempt)
return perform_request(uri, attempt + 1)
try:
item = object_response.json()
except JSONDecodeError:
logger.info('Encountered JSONDecodeError during request to "' + item_uri + '"')
time.sleep(0.5 * attempt)
return perform_request(uri, attempt + 1)
return item
item_uri = url.scheme + '://' + url.netloc + search_result['@id'] + '@@object'
item = perform_request(item_uri)
item_collection = app.registry['collections'][ search_result['@id'].split('/')[1] ]
item_schema_properties = item_collection.type_info.schema['properties']
item_keys = list(item.keys())
for item_key in item_keys:
if item_key not in item_schema_properties or item_schema_properties[item_key].get('calculatedProperty', False) is True:
del item[item_key]
elif not item[item_key] or item_key == 'schema_version':
del item[item_key]
counter['count'] += 1
logger.info('Got insert for ' + search_result['@id'] + ' - ' + str(counter['count']) + ' / ' + str(search_results_len) )
return item
pool = ThreadPool(processes=6)
out_list = pool.map(worker, search_results)
pool.close()
pool.join()
return out_list
def setUp(self):
from pyramid.paster import get_app
fname = pkg_resources.resource_filename("reportmob", "../development.ini")
app = get_app(fname, 'main')
from webtest import TestApp
self.app = TestApp(app)
#!/usr/bin/env python import os import sys here = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.join(here, 'demo')) config = os.path.join(here, 'production.ini') from pyramid.paster import get_app application = get_app(config, 'main')
def for_file(settings_file):
"""Return WSGI application using settings file."""
setup_logging(settings_file)
return get_app(settings_file, 'main')
