def main(): parser = OptionParser(description=__doc__) parser.add_option('-s', '--source', dest='source', action="store", default='/', metavar='ZODB-PATH', help="The ZODB source path to dump (e.g. /foo/bar or /)") parser.add_option('-d', '--dest', dest='dest', action="store", default='dump', metavar='FILESYSTEM-PATH', help="The destination filesystem path to dump to.") options, args = parser.parse_args() if args: config_uri = args[0] else: parser.error("Requires a config_uri as an argument") source = options.source dest = os.path.expanduser(os.path.normpath(options.dest)) setup_logging(config_uri) env = bootstrap(config_uri) root = env['root'] source = traverse(root, source)['context'] dump(source, dest)
def init_app(self, args): """ Initialises Pyramid application. Loads config settings. Initialises SQLAlchemy. """ self._args = args setup_logging(self._args.config) settings = get_appsettings(self._args.config) if 'environment' not in settings: raise KeyError('Missing key "environment" in config. Specify ' 'environment in paster INI file.') rc = Rc(environment=settings['environment'], root_dir=os.path.abspath( os.path.join(os.path.dirname(__file__), '..') ) ) rc.load() settings.update(rc.data) settings['rc'] = rc pysite.models.init(settings, 'db.pysite.sa.') self._rc = rc self._settings = settings pysite._init_vmail(rc)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: model = MyModel(name='one', value=1) DBSession.add(model) message1 = Message(msgid = 'c1', content = 'yoloswag1', timestamp = '5.11.2015') message2 = Message(msgid = 'c2', content = 'yoloswag2', timestamp = '5.11.2015') DBSession.add(message1) DBSession.add(message2) delmessage = DelMessage(msgid = 'c1', timestamp = '6.11.2015') DBSession.add(delmessage) DBSession.delete(model) DBSession.delete(message1) DBSession.delete(message2) DBSession.delete(delmessage)
def install_start(request): page={'editor': False, 'allerts': '', 'banners': [['/static/images/lerni.png', 'Lerni Logo']]} page['page_title'] = "Lerni - instalacja" #ZSO nr 15 w Sosnowcu page['internet_on'] = internet_on() try: page['celery'] = 8 == task.delay(4,4).wait(timeout=2, propagate=True, interval=0.5) except celery.exceptions.TimeoutError: page['celery'] = False except socket_error as serr: if serr.args[0][0] != errno.ECONNREFUSED: raise serr try: gnupg.GPG(gnupghome='GPG') page['gpg'] = True except ValueError: page['gpg'] = False page['update_available'] = False #time.sleep(1) #task2 = AsyncResult(task1) #print task2.result if False: config_uri = "development.ini" setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) return page
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) mkdir(settings['static_files']) # Create Ziggurat tables alembic_ini_file = 'alembic.ini' if not os.path.exists(alembic_ini_file): alembic_ini = ALEMBIC_CONF.replace('{{db_url}}', settings['sqlalchemy.url']) f = open(alembic_ini_file, 'w') f.write(alembic_ini) f.close() bin_path = os.path.split(sys.executable)[0] alembic_bin = os.path.join(bin_path, 'alembic') command = '%s upgrade head' % alembic_bin os.system(command) os.remove(alembic_ini_file) # Insert data engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) init_model() create_schemas(engine) Base.metadata.create_all(engine) initial_data.insert() transaction.commit()
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) init_model(settings)
def main(argv=sys.argv): # Usage and configuration if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) config = Configurator(settings=settings) config.include('pyramid_sqlalchemy') # Make the database with schema and default data with transaction.manager: metadata.create_all() root = RootFolder(name='', title='Moonbase Demo', __acl__=[ ['Allow', ['paul'], 'view'] ] ) Session.add(root) f1 = root['f1'] = Folder( title='Folder 1', __acl__=[ ['Allow', ['shane'], 'view'] ] ) f1['da'] = Document(title='Document 1A')
def main(argv=sys.argv): config_uri = argv[1] argv = argv[2:] parser = optparse.OptionParser(usage=USAGE, description=DESCRIPTION) parser.add_option('-t', '--title', dest='title', type='string') parser.add_option('-s', '--slug', dest='slug', type='string') options, args = parser.parse_args(argv) if options.title is None: parser.error('You must provide at least --title') slug = options.slug if slug is None: slug = options.title.lower().replace(' ', '_') setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) with transaction.manager: board = Board(title=options.title, slug=slug) DBSession.add(board) DBSession.flush() print(("Successfully added %s (slug: %s)" % (board.title, board.slug)))
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) #with transaction.manager: # model = MyModel(name='one', value=1) # DBSession.add(model) with transaction.manager: with open('agencies.csv','r') as f: agencies = f.read().split('\n') for agency in agencies: if agency.strip() != '': # agencyid, shortname, longname, type, description, websiteurl parts = agency.split('\t') agency_type = AgencyTypes.get_from_code(DBSession, parts[3]) a = Agencies( agency_code = parts[1], agency_name = parts[2], type_id = agency_type.id, description = parts[4], website = parts[5], ) DBSession.add(a) transaction.commit()
def main(argv=sys.argv): if len(argv) < 3: usage(argv) config_uri = argv[1] json_path = argv[2] options = parse_vars(argv[3:]) setup_logging(config_uri) # Configure the application, so we can access the registry. env = bootstrap(config_uri, options=options) # Generate a DBSession using the sessionmaker: DBSession = env['registry']['db_sessionmaker']() # The SQLAlchemy engine is accessible as the session's bind. engine = DBSession.bind Base.metadata.create_all(engine) json_data = json.load(open(json_path)) with transaction.manager: for kitten_data in json_data: kitten = Kitten(source_url=kitten_data['source_url'], credit=kitten_data['credit']) r = requests.get(kitten_data['download_url']) if r.headers['content-type'] == 'image/jpeg': kitten.file_extension = '.jpeg' elif r.headers['content-type'] == 'image/png': kitten.file_extension = '.png' kitten.file_data = r.content DBSession.add(kitten) # Not strictly necessary, as everything gets unwound when main returns anyway. # But it's a good habit to keep. env['closer']()
def bootstrap_script(argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) return settings
def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ config = Configurator(settings=settings) setup_logging(global_config['__file__']) config.add_renderer('jsonp', JSONP(param_name='callback')) config.add_renderer('ujson', factory='app.renderers.UjsonRenderer') config.add_static_view('static', 'static', cache_max_age=3600) # initialise a connection to mongo on startup and store the client # in the registry which will be injected into each request conf = appConfig(config.registry.settings.get('app.config')) config.registry.app_config = conf.app_config config.registry.app_config['mongodb']['client'] = init_mongodb_connection(config.registry.app_config['mongodb']) config.add_route('home', '/') config.add_route('health-check', '/health-check') config.add_route('network-stats', '/stats/{code}/{explore}') config.add_route('network-build', '/network/{code}/{explore}') config.add_route('network-build-status', '/network/{code}/{explore}/status') config.add_route('entity-data', '/entity/{code}/data') config.add_route('entity-build', '/entity/{code}/{id}') config.add_route('entity-build-status', '/entity/{code}/{id}/status') config.add_route('convert-graph', '/convert/{code}') config.scan() return config.make_wsgi_app()
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with transaction.manager: model2 = Genre(id = 0,name='HipHop', info='Nigger ma f****r',year_of_found = 1990) DBSession.add(model2) model = Genre(id = 1,name='GlitchHop', info='Nu vaaassheee',year_of_found = 2014, parent_genre = model2) DBSession.add(model) model3 = Genre(id = 2,name='Grime', info='Zhostkiy rap',year_of_found = 2006, parent_genre = model2) DBSession.add(model3) model4 = Genre(id = 3,name='Dark Grime', info='Tyomniy Zhostkiy rap',year_of_found = 2009, parent_genre = model3) DBSession.add(model4) music1 = Music(author = 'Simplex_Method', title = 'Of Literal Data', year = 2015,genre = model3) DBSession.add(music1)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: model = RasPi() model.uuid = '11:22:33:44:55:66' model.description = "Testing Pi" model.url = "http://www.facebook.com" model.orientation = 0 model.browser = True model.lastseen = datetime.now() DBSession.add(model) tag = Tags() tag.uuid = '11:22:33:44:55:66' tag.tag = 'test' DBSession.add(tag) User = UserModel() User.email = '*****@*****.**' User.AccessLevel = 2 DBSession.add(User) DBSession.flush()
def main(argv=sys.argv): global logger, settings, ca, ri, aws_access, aws_secret, thekey if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) # TODO: need to add settings to define where the files will be logger = logging.getLogger('scripts') # make configurable? site_blacklist = set(['MedPass', 'Bridge']) print "# Stack to site info generated by build_ini_info script" stackmap = tools.parse_sites() for stack, sites in stackmap.items(): clean = set(sites) - site_blacklist print "stacks.%s = %s" % (stack, ', '.join(clean)) print "# RDS to site info generated by build_ini_info script" rdsmap = tools.parse_rds() # dbs.a4.gentivalink = a4-css-sup-cleanup-rds for stack, sites in rdsmap.items(): for site, rdses in sites.items(): for rds in rdses: print "dbs.%s.%s = %s" % (stack, site, rds)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with transaction.manager: user = User(username='******', password='******', email=settings.get('admin_mail'), activated=True, groups='admin') DBSession.add(user) DBSession.flush() for k, v in Settings.settings.iteritems(): if type(v) == dict: DBSession.add(Settings(key=unicode(k), value=unicode(v['value']))) else: if isinstance(v, basestring) and v.startswith('app_config.'): v = settings.get(v[len('app_config.'):]) DBSession.add(Settings(key=unicode(k), value=unicode(v))) DBSession.flush()
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) #Delete old data Base.metadata.drop_all(engine) #Create data Base.metadata.create_all(engine) with transaction.manager: nb_sensors = 5 nb_readings = 500 minutes_between_readings = 10; for i in range(nb_sensors): sensor = Sensor(id=i+1,type='Moisture') DBSession.add(sensor) first_dttm = datetime.datetime.now() for j in range(nb_readings,0,-1): reading = Reading( sensor_id=sensor.id, timestamp=first_dttm - datetime.timedelta(minutes=minutes_between_readings*j), value=random.randrange(1023) ) DBSession.add(reading)
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) cache.cache = cache.configure_cache(settings) config = Configurator( settings=settings, root_factory=RootFactory, locale_negotiator=locale_neg ) config.add_route_predicate('vhost', VHostPredicate) config.add_view_predicate('vhost', VHostPredicate) mmgr = config.registry.getUtility(IModuleManager) mmgr.load('core') mmgr.load_enabled() rts = rt.configure(mmgr, config.registry) app = rts.app() rt.run(rts, app)
def update(argv=sys.argv): """ Download the GeoIP database from the URL provided under the config key `geoip.city.source` and save it under the path provided by the config key `geoip.city.destination`. """ if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) source = settings['geoip.city.source'] output = settings['geoip.city.destination'] log.info("Downloading %s...", source) response = requests.get(source) log.info("Downloading done.") compressed = gzip.GzipFile(fileobj=StringIO(response.content)) with open(output, "wb") as f: log.info("Writting to %s...", output) f.write(compressed.read()) log.info("Writting done.")
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) application_pdf_doc_type = DocumentTypes.add( name="Adobe PDF", description="Adobe PDF file", mime_type="application/pdf", ) system_owner = Users.add( first="SYSTEM", last="USERS", email="system@localhost", password="******", ) default_scraper = Scrapers.add( name="Default Scraper", description="CivicDocs.IO loads with a single, defualt scraper.", owner_id=system_owner.id, ) print("DEFAULT SCRAPER ID:\r\n{0}\r\n".format(default_scraper.id))
def run(self, argv): if len(argv) < 3: self.usage(argv) config_uri = argv[1] fromDate = argv[2] options = parse_vars(argv[3:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: url = 'https://servicesenligne2.ville.montreal.qc.ca/api/infoneige/InfoneigeWebService?wsdl' client = Client(url) planification_request = client.factory.create('getPlanificationsForDate') planification_request.fromDate = fromDate planification_request.tokenString = 'ug33-b81ab488-c335-4021-9c52-26d6b8523301-e7aa002b-0d9d-4b5c-81ef-b012979cdafb-dab06588-1962-4b16-9942-a18054094f60-a4186179-d555-4fed-b35f-ec0c74da97a3-aa3b3766-4d26-42f0-888a-a6569a1dd745' response = client.service.GetPlanificationsForDate(planification_request) if response['responseStatus'] == 0: log.info('%s plannings returned', response['planifications']['count']) cartodb_client = CartoDBOAuth(settings['cartodb.key'], settings['cartodb.secret'], settings['cartodb.user'], settings['cartodb.password'], settings['cartodb.domain']) for result in response['planifications']['planification']: ''' street_side_status = StreetSideHistory( municipality_id = result['munid'], street_side_id = result['coteRueId'], state = result['etatDeneig'], observed_on = result['dateMaj'], ) DBSession.merge(street_side_status) ''' if any(val in result for val in ['dateDebutPlanif', 'dateFinPlanif', 'dateDebutReplanif', 'dateFinReplanif']): try: result['dateDebutReplanif'] except AttributeError: result['dateDebutReplanif'] = None try: result['dateFinReplanif'] except AttributeError: result['dateFinReplanif'] = None ''' print result planning = Planning( municipality_id = result['munid'], street_side_id = result['coteRueId'], planned_start_date = result['dateDebutPlanif'], planned_end_date = result['dateFinPlanif'], replanned_start_date = result['dateDebutReplanif'], replanned_end_date = result['dateFinReplanif'], modified_on = result['dateMaj'], ) DBSession.merge(planning) ''' #transaction.manager.commit() cartodb_client.sql('UPDATE cote SET etat = %(etat)s WHERE cote_rue_id = %(cote_rue_id)d' % {"etat": result['etatDeneig'], "cote_rue_id": result['coteRueId']}) else: log.info('Status %s: %s', response['responseStatus'], response['responseDesc'])
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) connect_string = settings['sqlalchemy.url']\ .replace('DBUser', os.environ['DBUSER'])\ .replace('DBPassword', os.environ['DBPASSWORD']) settings['sqlalchemy.url'] = connect_string engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: store_data = json.loads(open('emporium/scripts/store_data.json'). read()) suppliers = {} for item in store_data['suppliers']: supplier = Supplier(name=item['name'], tax_id=item['tax_id']) suppliers[supplier.name] = supplier DBSession.add(supplier) for item in store_data['bargains']: bargain = Bargain( sku=item['sku'], price=item['price'], supplier=suppliers[item['supplier_name']] ) bargain.info = item['info'] DBSession.add(bargain)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, name="testscaffold", options=options) engine = get_engine(settings) session_factory = get_session_factory(engine) dbsession = get_tm_session(session_factory, transaction.manager) with transaction.manager: user = User(user_name="admin", email="foo@localhost") UserService.set_password(user, "admin") admin_object = Group(group_name="Administrators") group_permission = GroupPermission(perm_name="root_administration") dbsession.add(admin_object) admin_object.permissions.append(group_permission) admin_object.users.append(user) test_group = Group(group_name="Other group") dbsession.add(test_group) for x in range(1, 25): user = User(user_name="test{}".format(x), email="foo{}@localhost".format(x)) UserService.set_password(user, "test") test_group.users.append(user)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.', echo=False) DBSession.configure(bind=engine) Base.metadata.create_all(engine) # populate databases with transaction.manager: print('\ninit_setpoint_interpolations:\n') init_setpoint_interpolations(DBSession) print('\ninit_parameter_types:\n') init_parameter_types(DBSession) print('\ninit_device_types:\n') init_device_types(DBSession) print('\ninit_periphery_controllers:\n') DBSession.flush() init_periphery_controllers(DBSession) DBSession.flush() print('\ninit_parameters:\n') init_parameters(DBSession) print('\ninit_devices:\n') init_devices(DBSession) print('\ninit_field_settings:\n') init_field_settings(DBSession) print('\ninit_regulators:\n') DBSession.flush() init_regulators(DBSession) init_log_diagrams(DBSession)
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, "sqlalchemy.") DBSession.configure(bind=engine) Base.metadata.create_all(engine) user = User() user.set_password("wolfvo8491") user.user_name = "wolfv" user.email = "*****@*****.**" user.status = 1 # Not verified yet DBSession.add(user) with transaction.manager: user = User() user.user_name = "wolf" user.email = "*****@*****.**" user.set_password("wolf") user.status = 1 DBSession.add(user) user = User() user.user_name = "peter" user.email = "*****@*****.**" user.set_password("p") user.status = 1 DBSession.add(user)
def listuserinfo(): arguments = listuserparser.parse_args() if not arguments.config or not arguments.username: listuserparser.print_usage() else: config_uri = arguments.config setup_logging(config_uri) try: settings = get_appsettings(config_uri, 'factored') except LookupError: settings = get_appsettings(config_uri, 'main') engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) session = DBSession() users = session.query(User).filter_by( username=arguments.username).all() if len(users) > 0: user = users[0] print 'username:%s, secret: %s' % ( user.username, user.secret) print 'bar code url:', get_barcode_image(user.username, user.secret, settings['appname']) else: print '"%s" user not found' % arguments.username
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] if config_uri not in ["jenkins.ini"]: print "You are not initialising using a testing URI" print "Edit init_testingdb.py if you really want to do this" sys.exit(0) setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, "sqlalchemy.") meta.Session.configure(bind=engine) Base.metadata.bind = engine # Create Everythin print "Dropping Tables" Base.metadata.drop_all(engine) print "Creating Tables" Base.metadata.create_all(engine) # We also want any alembic scripts to be executed alembic_cfg = Config(config_uri) # TODO: WARNING RELATIVE PATH command.stamp(alembic_cfg, "head") DBSession = meta.Session() # DBSession.configure(bind=engine) populateData.init_data(DBSession) populateUser() populatedata()
def __call__(self, argv=sys.argv): self.configure_parser() args = self.parser.parse_args(argv[1:]) if self.requires_config: self.config_uri = args.config_uri setup_logging(self.config_uri) self.run(args)
def main(argv=sys.argv): #pragma NOCOVER if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: password = hashlib.md5('secret'.encode()).hexdigest() user = User(name='distractionbike', password_md5=password) DBSession.add(user) user2 = User(name='otherguy', password_md5=password) DBSession.add(user2) journal = Journal(name='distractionbike') DBSession.add(journal) post = Post(journal_name='distractionbike', title='First Post', lede='Why do we love first posts so much?', text='This is the <i>very first post</i>.') DBSession.add(post) post.add_comment(text='Yay!', user_id='ezra')
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: user = User(email='*****@*****.**', password='******', admin=True) DBSession.add(user) site = Site(name='asd', key='80d621df066348e5938a469730ae0cab') DBSession.add(site) site.api_keys.append(SiteAPIKey(key='GIKfxIcIHPbM0uX9PrQ1To29Pb2on0pa')) site.users.append(user) aspect_ratio_1_1 = SiteAspectRatio(width=1, height=1) aspect_ratio_3_1 = SiteAspectRatio(width=3, height=1) site.aspect_ratios.append(aspect_ratio_1_1) site.aspect_ratios.append(aspect_ratio_3_1) from alembic.config import Config from alembic import command alembic_cfg = Config("alembic.ini") command.stamp(alembic_cfg, "head")
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) if os.environ.get('DATABASE_URL', ''): settings["sqlalchemy.url"] = os.environ["DATABASE_URL"] engine = get_engine(settings) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) session_factory = get_session_factory(engine) with transaction.manager: dbsession = get_tm_session(session_factory, transaction.manager) size_models = [] distance_models = [] absolute_magnitude_models = [] orbit_models = [] for item in SUPER_DICT: if item['orbit_body'] == 'Earth': if '2016' in item['approach_date'] or '2017' in item['approach_date'] or '2018' in item['approach_date']: if float("{0:.2f}".format(float(item['miss_lunar']))) <= 14.00: new_size = Size( date=item['approach_date'], neo_id=item["neo_id"], name=item["name"], url=item["rock_url"], kilometers=item["diakm"], meters=item["diam"], miles=item["diamiles"], feet=item["diafeet"], ) size_models.append(new_size) new_distance = Distance( date=item['approach_date'], neo_id=item["neo_id"], name=item["name"], url=item["rock_url"], astronomical=item["miss_astronomical"], lunar=item["miss_lunar"], kilometers=item["miss_km"], miles=item["miss_miles"], ) distance_models.append(new_distance) new_absmag = AbsoluteMag( date=item['approach_date'], neo_id=item["neo_id"], name=item["name"], url=item["rock_url"], absolutemag=item["absmag"], velocity_kps=item["velocity_kps"], velocity_kph=item["velocity_kph"], ) absolute_magnitude_models.append(new_absmag) new_orbit = Orbit( date=item['approach_date'], neo_id=item["neo_id"], name=item["name"], url=item["rock_url"], orbit_period=item["orbit_period"], perihelion_dist=item["perihelion_dist"], aphelion_dist=item["aphelion_dist"], eccentricity=item["orbit_eccentricity"], perihelion_time=item["perihelion_time"] ) orbit_models.append(new_orbit) dbsession.add_all(size_models) dbsession.add_all(distance_models) dbsession.add_all(absolute_magnitude_models) dbsession.add_all(orbit_models)
"""Pyramid bootstrap environment. """ from alembic import context from pyramid.paster import get_appsettings, setup_logging from sqlalchemy import engine_from_config from {{ cookiecutter.repo_name }}.models.meta import Base config = context.config setup_logging(config.config_file_name) settings = get_appsettings(config.config_file_name) target_metadata = Base.metadata def run_migrations_offline(): """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation we don't even need a DBAPI to be available. Calls to context.execute() here emit the given string to the script output. """ context.configure(url=settings['sqlalchemy.url']) with context.begin_transaction(): context.run_migrations()
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, "h", [ "config-file=", "scenario-id=", "scenario-name=", "scenario-description=", "topology=", "attacker=", "affected-area=", "target=", "attack-type=", "all-paths", "number-of-shortest-paths=" ]) except getopt.GetoptError: print( '\n' 'ERROR\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' ) sys.exit(2) config_file = scenario_id = scenario_name = scenario_description = topology = attacker = \ affected_area = target = attack_type = number_of_shortest_paths = '' for opt, arg in opts: if opt == '-h': print( '\n' 'HELP\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' ) sys.exit() if opt == '--config-file': config_file = arg elif opt == '--scenario-id': scenario_id = arg elif opt == '--scenario-name': scenario_name = arg elif opt == '--scenario-description': scenario_description = arg elif opt == '--topology': topology = arg elif opt == '--attacker': attacker = arg elif opt == '--affected-area': affected_area = arg elif opt == '--target': target = arg elif opt == '--attack-type': attack_type = arg elif opt == '--all-paths': number_of_shortest_paths = '0' elif opt == '--number-of-shortest-paths': number_of_shortest_paths = arg if (config_file and scenario_name and topology and attacker and affected_area and target and attack_type and number_of_shortest_paths) \ or (config_file and scenario_id): args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) try: with env['request'].tm: dbsession = env['request'].dbsession aa = AttackScenario(dbsession, scenario_id, scenario_name, scenario_description, topology, attacker, affected_area, target, attack_type, number_of_shortest_paths) attacker_list, affected_area_list, target_list, scenario_attack_type = aa.attack_scenario( ) # scenario_item / path / path_item if attacker_list and affected_area_list and target_list: if scenario_attack_type == 'attraction': with env['request'].tm: aa.attraction_attack_type() elif scenario_attack_type == 'interception': with env['request'].tm: aa.interception_attack_type() else: print('attack type unknown') return with env['request'].tm: if scenario_id: clear_database(dbsession, scenario_id) except OperationalError: print('Database error') else: print( '\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' )
def main(global_config, **settings): """ Return a Pyramid WSGI application. """ settings = dict(global_config, **settings) settings['config_uri'] = global_config['__file__'] # here we create the engine and bind it to the (not really a) session # factory settings = set_config(settings) if not session_maker_is_initialized(): configure_engine(settings) if settings.get('idealoom_debug_signal', False): from assembl.lib import signals signals.listen() import os if 'UWSGI_ORIGINAL_PROC_NAME' in os.environ: # uwsgi does not load logging properly from pyramid.paster import setup_logging setup_logging(global_config['__file__']) config = Configurator(registry=getGlobalSiteManager()) config.include('.lib.logging') from .views.traversal import root_factory config.setup_registry(settings=settings, root_factory=root_factory) config.add_translation_dirs('assembl:locale/') config.set_locale_negotiator(my_locale_negotiator) config.add_tween('assembl.tweens.logging.logging_tween_factory', over="pyramid_tm.tm_tween_factory") config.include('pyramid_retry') config.include('.auth') config.include('.models') bake_lazy_loaders() # Tasks first, because it includes ZCA registration (for now) config.include('.tasks') config.include('pyramid_dogpile_cache') config.include('.lib.zmqlib') session_factory = session_factory_from_settings(settings) config.set_session_factory(session_factory) if not settings.get('nosecurity', False): # import after session to delay loading of BaseOps from .auth.util import authentication_callback auth_policy_name = settings.get( "auth_policy_class", "assembl.auth.util.UpgradingSessionAuthenticationPolicy") auth_policy = resolver.resolve(auth_policy_name)( callback=authentication_callback) config.set_authentication_policy(auth_policy) config.set_authorization_policy(ACLAuthorizationPolicy()) # ensure default roles and permissions at startup if not settings.get('in_migration', False): from .lib.migration import bootstrap_db, bootstrap_db_data db = bootstrap_db(settings['config_uri']) bootstrap_db_data(db, settings['config_uri'] != "testing.ini") config.add_static_view('static', 'static', cache_max_age=3600) config.add_static_view('widget', 'widget', cache_max_age=3600) config.include('cornice') # REST services library. # config.include('.lib.alembic') # config.include('.lib.email') config.include('.lib') config.include('.views') # jinja2 config.include('pyramid_jinja2') config.add_jinja2_extension('jinja2.ext.i18n') # Mailer if 'pyramid_mailer.debug' not in settings.get('pyramid.includes', ()): config.include('pyramid_mailer') config.include('.view_def') wsgi_app = config.make_wsgi_app() if asbool(settings.get('sqltap', False)): import sqltap.wsgi wsgi_app = sqltap.wsgi.SQLTapMiddleware(wsgi_app) return wsgi_app
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) print("Beginning update.") PopulatedSystem.__table__.drop(engine) Listing.__table__.drop(engine) Station.__table__.drop(engine) Faction.__table__.drop(engine) Body.__table__.drop(engine) Faction.__table__.create(engine) PopulatedSystem.__table__.create(engine) Body.__table__.create(engine) Station.__table__.create(engine) Listing.__table__.create(engine) mark_changed(DBSession()) transaction.commit() # # Factions # print("Updating factions...") print("Downloading factions.jsonl from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/factions.jsonl", stream=True) with open('factions.json', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved factions.json. Updating...") url = str(engine.url) + "::" + Faction.__tablename__ ds = dshape("var *{ id: ?int64, name: ?string, updated_at: ?int64, government_id: ?int64, " "government: ?string, allegiance_id: ?int64, allegiance: ?string, " "state_id: ?int64, state: ?string, home_system_id: ?int64, " "is_player_faction: ?bool }") t = odo('jsonlines://factions.json', url, dshape=ds) print("Done! Creating index...") DBSession.execute("CREATE INDEX factions_idx ON factions(id)") mark_changed(DBSession()) transaction.commit() print("Completed processing factions.") # # Systems # print("Downloading systems_recently.csv from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/systems_recently.csv", stream=True) with open('systems_recently.csv', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved systems_recently.csv. Creating temporary table and importing...") DBSession.execute("CREATE TEMP TABLE systems_tmp (LIKE systems)") url = str(engine.url) + "::systems_tmp" ds = dshape("var *{ id: ?int64, edsm_id: ?int64, name: ?string, x: ?float64, y: ?float64, " "z: ?float64, population: ?int64, is_populated: ?bool, government_id: ?int64, " "government: ?string, allegiance_id: ?int64, allegiance: ?string, " "state_id: ?int64, state: ?string, security_id: ?float64, security: ?string, " "primary_economy_id: ?float64, primary_economy: ?string, power: ?string, " "power_state: ?string, power_state_id: ?string, needs_permit: ?bool, " "updated_at: ?int64, simbad_ref: ?string, controlling_minor_faction_id: ?string, " "controlling_minor_faction: ?string, reserve_type_id: ?float64, reserve_type: ?string }") t = odo('systems_recently.csv', url, dshape=ds) print("Updating systems...") DBSession.execute("INSERT INTO systems(id, edsm_id, name, x, y, z, population, is_populated, government_id, " "government, allegiance_id, allegiance, state_id, state, security_id, security, " "primary_economy_id, primary_economy, power, power_state, power_state_id, needs_permit, " "updated_at, simbad_ref, controlling_minor_faction_id, controlling_minor_faction, " "reserve_type_id, reserve_type) SELECT id, edsm_id, name, x, y, z, population, is_populated, " "government_id, government, allegiance_id, allegiance, state_id, state, security_id, security, " "primary_economy_id, primary_economy, power, power_state, power_state_id, needs_permit, " "updated_at, simbad_ref, controlling_minor_faction_id, controlling_minor_faction, " "reserve_type_id, reserve_type from systems_tmp ON CONFLICT DO UPDATE " "SET edsm_id = EXCLUDED.edsm_id, name = EXCLUDED.name, x = EXCLUDED.x, " "y = EXCLUDED.y, z = EXCLUDED.z, population = EXCLUDED.population, " "is_populated = EXCLUDED.population, government_id = EXCLUDED.government_id, " "government = EXCLUDED.government, allegiance_id = EXCLUDED.allegiance_id, " "allegiance = EXCLUDED.allegiance, state_id = EXCLUDED.state_id, " "state = EXCLUDED.state, security_id = EXCLUDED.security_id, security = EXCLUDED.security, " "primary_economy_id = EXCLUDED.primary_economy_id, primary_economy = EXCLUDED.primary_economy, " "power = EXCLUDED.power, power_state = EXCLUDED.power_state, power_state_id = " "EXCLUDED.power_state_id, needs_permit = EXCLUDED.needs_permit, updated_at = " "EXCLUDED.updated_at, simbad_ref = EXCLUDED.simbad_ref," "controlling_minor_faction_id = EXCLUDED.controlling_minor_faction_id, " "reserve_type_id = EXCLUDED.reserve_type_id, reserve_type = EXCLUDED.reserve_type") mark_changed(DBSession()) transaction.commit() print("Done!") # # Bodies # print("Downloading bodies.jsonl from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/bodies.jsonl", stream=True) with open('bodies.json', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved bodies.jsonl. Converting JSONL to SQL.") DBSession.execute("CREATE TEMP TABLE bodies_tmp (LIKE bodies)") url = str(engine.url) + "::bodies_tmp" ds = dshape("var *{ id: ?int64, created_at: ?int64, updated_at: ?int64, name: ?string, " "system_id: ?int64, group_id: ?int64, group_name: ?string, type_id: ?int64, " "type_name: ?string, distance_to_arrival: ?int64, full_spectral_class: ?string, " "spectral_class: ?string, spectral_sub_class: ?string, luminosity_class: ?string, " "luminosity_sub_class: ?string, surface_temperature: ?int64, is_main_star: ?bool, " "age: ?int64, solar_masses: ?float64, solar_radius: ?float64, catalogue_gliese_id : ?string, " "catalogue_hipp_id: ?string, catalogue_hd_id: ?string, volcanism_type_id: ?int64, " "volcanism_type_name: ?string, atmosphere_type_id: ?int64, atmosphere_type_name: ?string, " "terraforming_state_id: ?int64, terraforming_state_name: ?string, earth_masses: ?float64, " "radius: ?int64, gravity: ?float64, surface_pressure: ?int64, orbital_period: ?float64, " "semi_major_axis: ?float64, orbital_eccentricity: ?float64, orbital_inclination: ?float64, " "arg_of_periapsis: ?float64, rotational_period: ?float64, " "is_rotational_period_tidally_locked: ?bool, axis_tilt: ?float64, eg_id: ?int64, " "belt_moon_masses: ?float64, ring_type_id: ?int64, ring_type_name: ?string, " "ring_mass: ?int64, ring_inner_radius: ?float64, ring_outer_radius: ?float64, " "rings: ?json, atmosphere_composition: ?json, solid_composition: ?json, " "materials: ?json, is_landable: ?bool}") #url = str(engine.url) + "::" + Body.__tablename__ t = odo('jsonlines://bodies.json', url, dshape=ds) print("Creating indexes...") DBSession.execute("CREATE INDEX bodies_idx ON bodies(name text_pattern_ops)") mark_changed(DBSession()) transaction.commit() DBSession.execute("CREATE INDEX systemid_idx ON bodies(system_id)") mark_changed(DBSession()) transaction.commit() print("Done!") # # Populated systems # print("Downloading systems_populated.jsonl from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/systems_populated.jsonl", stream=True) with open('systems_populated.json', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved systems_populated.json. Updating...") url = str(engine.url) + "::" + PopulatedSystem.__tablename__ ds = dshape("var *{ id: ?int64, edsm_id: ?int64, name: ?string, x: ?float64, y: ?float64, " "z: ?float64, population: ?int64, is_populated: ?bool, government_id: ?int64, " "government: ?string, allegiance_id: ?int64, allegiance: ?string, " "state_id: ?int64, state: ?string, security_id: ?float64, security: ?string, " "primary_economy_id: ?float64, primary_economy: ?string, power: ?string, " "power_state: ?string, power_state_id: ?string, needs_permit: ?int64, " "updated_at: ?int64, simbad_ref: ?string, controlling_minor_faction_id: ?string, " "controlling_minor_faction: ?string, reserve_type_id: ?float64, reserve_type: ?string," "minor_faction_presences: ?json }") t = odo('jsonlines://systems_populated.json', url, dshape=ds) print("Done! Uppercasing system names...") DBSession.execute("UPDATE populated_systems SET name = UPPER(name)") mark_changed(DBSession()) transaction.commit() print("Creating indexes...") DBSession.execute("CREATE INDEX index_populated_system_names_trigram ON populated_systems " "USING GIN(name gin_trgm_ops)") mark_changed(DBSession()) transaction.commit() DBSession.execute("CREATE INDEX index_populated_system_names_btree ON populated_systems (name)") mark_changed(DBSession()) transaction.commit() print("Completed processing populated systems.") # # Stations # print("Downloading stations.jsonl from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/stations.jsonl", stream=True) with open('stations.json', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved stations.json. Updating...") DBSession.execute("CREATE TEMP TABLE stations_tmp (LIKE stations)") url = str(engine.url) + "::stations_tmp" #url = str(engine.url) + "::" + Station.__tablename__ ds = dshape("var *{ id: ?int64, name: ?string, system_id: ?int64, updated_at: ?int64, " "max_landing_pad_size: ?string, distance_to_star: ?int64, government_id: ?int64, " "government: ?string, allegiance_id: ?int64, allegiance: ?string, " "state_id: ?int64, state: ?string, type_id: ?int64, type: ?string, " "has_blackmarket: ?bool, has_market: ?bool, has_refuel: ?bool, " "has_repair: ?bool, has_rearm: ?bool, has_outfitting: ?bool, " "has_shipyard: ?bool, has_docking: ?bool, has_commodities: ?bool, " "import_commodities: ?json, export_commodities: ?json, prohibited_commodities: ?json, " "economies: ?json, shipyard_updated_at: ?int64, outfitting_updated_at: ?int64, " "market_updated_at: ?int64, is_planetary: ?bool, selling_ships: ?json, " "selling_modules: ?json, settlement_size_id: ?string, settlement_size: ?int64, " "settlement_security_id: ?int64, settlement_security: ?string, body_id: ?int64," "controlling_minor_faction_id: ?int64 }") t = odo('jsonlines://stations.json', url, dshape=ds) print("Done! Cleaning stations without body references...") DBSession.execute("DELETE FROM stations_tmp WHERE body_id NOT IN (SELECT b.id from bodies b)") mark_changed(DBSession()) transaction.commit() DBSession.execute("UPDATE stations SET id=t.id, name=t.name, system_id=t.system_id, updated_at=t.updated_at, " "max_landing_pad_size=t.max_landing_pad_size, ") DBSession.execute("CREATE INDEX index_stations_systemid_btree ON stations(system_id)") mark_changed(DBSession()) transaction.commit() DBSession.execute("CREATE INDEX index_stations_btree ON stations(id)") mark_changed(DBSession()) transaction.commit() print("Completed processing stations.") # # Listings # print("Downloading listings.csv from EDDB.io...") r = requests.get("https://eddb.io/archive/v5/listings.csv", stream=True) with open('listings.csv', 'wb') as f: for chunk in r.iter_content(chunk_size=4096): if chunk: f.write(chunk) print("Saved listings.csv. Updating...") url = str(engine.url) + "::" + Listing.__tablename__ ds = dshape("var *{ id: ?int64, station_id: ?int64, commodity: ?int64, supply: ?int64, " "buy_price: ?int64, sell_price: ?int64, demand: ?int64, collected_at: ?int64 }") t = odo('listings.csv', url, dshape=ds) print("Creating indexes...") DBSession.execute("CREATE INDEX index_listings_stationid_btree ON listings(station_id)") mark_changed(DBSession()) transaction.commit() print("Updates complete.")
import logging from sqlalchemy import engine_from_config from .helpers import ( get_lp, ) from .models import ( DBSession, ) from pyramid.paster import ( get_appsettings, setup_logging, ) setup_logging('%s.ini' % os.environ.get('ENV', 'development')) logger = logging.getLogger(__file__) def is_source(o): return inspect.isclass(o) and issubclass(o, SourcePlugin) class SourcePlugin(object): def __init__(self, lp=None, log=None): if not log: log = logger self.logger = log self.log = self.logger.info
def run(self): # pragma: no cover if self.options.stop_daemon: self._warn_daemon_deprecated() return self.stop_daemon() if not hasattr(self.options, 'set_user'): # Windows case: self.options.set_user = self.options.set_group = None # @@: Is this the right stage to set the user at? if self.options.set_user or self.options.set_group: self.change_user_group(self.options.set_user, self.options.set_group) if not self.args: self.out('You must give a config file') return 2 app_spec = self.args[0] if (len(self.args) > 1 and self.args[1] in self.possible_subcommands): cmd = self.args[1] else: cmd = None if self.options.reload: if (getattr(self.options, 'daemon', False) or cmd in ('start', 'stop', 'restart')): self.out( 'Error: Cannot use reloading while running as a dameon.') return 2 if os.environ.get(self._reloader_environ_key): if self.options.verbose > 1: self.out('Running reloading file monitor') install_reloader(int(self.options.reload_interval), [app_spec]) # if self.requires_config_file: # watch_file(self.args[0]) else: return self.restart_with_reloader() if cmd not in (None, 'start', 'stop', 'restart', 'status'): self.out('Error: must give start|stop|restart (not %s)' % cmd) return 2 if cmd == 'status' or self.options.show_status: self._warn_daemon_deprecated() return self.show_status() if cmd in ('restart', 'stop'): self._warn_daemon_deprecated() result = self.stop_daemon() if result: if cmd == 'restart': self.out("Could not stop daemon; aborting") else: self.out("Could not stop daemon") return result if cmd == 'stop': return result self.options.daemon = True if cmd == 'start': self.options.daemon = True app_name = self.options.app_name vars = self.get_options() if not self._scheme_re.search(app_spec): app_spec = 'config:' + app_spec server_name = self.options.server_name if self.options.server: server_spec = 'egg:pyramid' assert server_name is None server_name = self.options.server else: server_spec = app_spec base = os.getcwd() # warn before setting a default if self.options.pid_file or self.options.log_file: self._warn_daemon_deprecated() if getattr(self.options, 'daemon', False): if not self.options.pid_file: self.options.pid_file = 'pyramid.pid' if not self.options.log_file: self.options.log_file = 'pyramid.log' # Ensure the log file is writeable if self.options.log_file: try: writeable_log_file = open(self.options.log_file, 'a') except IOError as ioe: msg = 'Error: Unable to write to log file: %s' % ioe raise ValueError(msg) writeable_log_file.close() # Ensure the pid file is writeable if self.options.pid_file: try: writeable_pid_file = open(self.options.pid_file, 'a') except IOError as ioe: msg = 'Error: Unable to write to pid file: %s' % ioe raise ValueError(msg) writeable_pid_file.close() # warn before forking if (self.options.monitor_restart and not os.environ.get(self._monitor_environ_key)): self.out('''\ --monitor-restart has been deprecated in Pyramid 1.6. It will be removed in a future release per Pyramid's deprecation policy. Please consider using a real process manager for your processes like Systemd, Circus, or Supervisor. ''') if (getattr(self.options, 'daemon', False) and not os.environ.get(self._monitor_environ_key)): self._warn_daemon_deprecated() try: self.daemonize() except DaemonizeException as ex: if self.options.verbose > 0: self.out(str(ex)) return 2 if (not os.environ.get(self._monitor_environ_key) and self.options.pid_file): self.record_pid(self.options.pid_file) if (self.options.monitor_restart and not os.environ.get(self._monitor_environ_key)): return self.restart_with_monitor() if self.options.log_file: stdout_log = LazyWriter(self.options.log_file, 'a') sys.stdout = stdout_log sys.stderr = stdout_log logging.basicConfig(stream=stdout_log) log_fn = app_spec if log_fn.startswith('config:'): log_fn = app_spec[len('config:'):] elif log_fn.startswith('egg:'): log_fn = None if log_fn: log_fn = os.path.join(base, log_fn) setup_logging(log_fn) server = self.loadserver(server_spec, name=server_name, relative_to=base, global_conf=vars) app = self.loadapp(app_spec, name=app_name, relative_to=base, global_conf=vars) if self.options.verbose > 0: if hasattr(os, 'getpid'): msg = 'Starting server in PID %i.' % os.getpid() else: msg = 'Starting server.' self.out(msg) def serve(): try: server(app) except (SystemExit, KeyboardInterrupt) as e: if self.options.verbose > 1: raise if str(e): msg = ' ' + str(e) else: msg = '' self.out('Exiting%s (-v to see traceback)' % msg) if self.options.browser: def open_browser(): context = loadcontext(SERVER, app_spec, name=server_name, relative_to=base, global_conf=vars) url = 'http://127.0.0.1:{port}/'.format(**context.config()) time.sleep(1) webbrowser.open(url) t = threading.Thread(target=open_browser) t.setDaemon(True) t.start() serve()
def database(): """The :func:`~ess_test.conftest.database` fixture initialises the database specified in the "testing.ini", removes any existing data, creates the standard permissions, and four test users: * admin - user with full administrative permissions * developer - user with full experiment development permissions * content - user with full editing permissions * general - user with no permissions """ global dbsession_initialised # Load settings settings = get_appsettings('testing.ini') setup_logging('testing.ini') # Init the DB engine = engine_from_config(settings, 'sqlalchemy.') if not dbsession_initialised: DBSession.configure(bind=engine) dbsession_initialised = True Base.metadata.drop_all(engine) Base.metadata.create_all(engine) dbsession = DBSession() # Create Test Users with transaction.manager: admin_user = User(email='*****@*****.**', display_name='Admin', password='******') developer_user = User(email='*****@*****.**', display_name='Developer', password='******') content_user = User(email='*****@*****.**', display_name='Content', password='******') general_user = User(email='*****@*****.**', display_name='General', password='******') dbsession.add(general_user) group = PermissionGroup(title='Site administrator') group.permissions.append(Permission(name='admin.users', title='Administer the users')) group.permissions.append(Permission(name='admin.groups', title='Administer the permission groups')) group.permissions.append(Permission(name='admin.question_types', title='Administer the question types')) admin_user.permission_groups.append(group) group = PermissionGroup(title='Developer') group.permissions.append(Permission(name='survey.new', title='Create new experiments')) admin_user.permission_groups.append(group) developer_user.permission_groups.append(group) group = PermissionGroup(title='Content administrator') group.permissions.append(Permission(name='survey.view-all', title='View all experiments')) group.permissions.append(Permission(name='survey.edit-all', title='Edit all experiments')) group.permissions.append(Permission(name='survey.delete-all', title='Delete all experiments')) content_user.permission_groups.append(group) dbsession.add(admin_user) dbsession.add(developer_user) dbsession.add(content_user) question_types = QuestionTypeIOSchema(include_schemas=(QuestionTypeGroupIOSchema,), many=True).\ loads(resource_string('ess','scripts/templates/default_question_types.json')) dbsession.add_all(question_types.data) # Alembic Stamp alembic_config = config.Config('testing.ini', ini_section='app:main') alembic_config.set_section_option('app:main', 'script_location', 'ess:migrations') command.stamp(alembic_config, "head") dbsession.close() DBSession.close_all() yield DBSession
def main(argv=sys.argv): if len(argv) < 3: usage(argv) config_uri = argv[1] run_script = int(argv[2]) options = parse_vars(argv[3:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = get_engine(settings) Base.metadata.create_all(engine) session_factory = get_session_factory(engine) files = [ 'States', 'Cities', 'Projects', 'SubProjects', 'FinSources', 'Parties' ] import csv import psycopg2 # conn = psycopg2.connect("host='localhost' port='5432' dbname='smartcities'") # curr = conn.cursor() __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) if run_script == 1: print(__location__) for file in files: with open(__location__ + '\\' + file + '.csv', 'rt') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='|') for row in spamreader: if row: with transaction.manager: dbsession = get_tm_session(session_factory, transaction.manager) if file == 'States': dbsession.add(State(name=row[0])) elif file == 'Cities': name = row[0] state = dbsession.query(State).filter_by( name=row[1]).first() dbsession.add( City(name=name, state_id=state.id)) elif file == 'Projects': dbsession.add(ProjectCategory(name=row[0])) elif file == 'SubProjects': name = row[0] projectcategory = dbsession.query( ProjectCategory).filter_by( name=row[1]).first() dbsession.add( ProjectSubCategory( name=name, category_id=projectcategory.id)) elif file == 'FinSources': dbsession.add(FinanceSource(name=row[0])) elif file == 'Parties': dbsession.add(Party(name=row[0])) if run_script == 2: # Project data upload with open(__location__ + '\\' + 'ProjectsData.csv', 'rt') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='|') next(spamreader, None) # Skip the headers for row in spamreader: if row: with transaction.manager: dbsession = get_tm_session(session_factory, transaction.manager) category = dbsession.query(ProjectCategory).filter_by( name=row[6]).first() subcategory = dbsession.query(ProjectSubCategory) \ .filter_by(name=row[7]) \ .filter_by(category_id=category.id) \ .first() city = dbsession.query(City).filter_by( name=row[1]).first() it_es_text = row[9] ites_val = None if it_es_text == 'Yes': ites_val = True elif it_es_text == 'No': ites_val = False dbsession.add( Project(name=row[8], rank=int(row[0]), round=row[3], IT_ITES=ites_val, amount_total=float(row[5]) * 10000000, city_id=city.id, category_id=category.id, subcategory_id=subcategory.id))
def configure_logging(*args, **kwargs): setup_logging(ini_file)
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, "h", ["config-file=", "topology=", "new-topology-name="]) except getopt.GetoptError: print('\n' 'Usage: MiniSecBGP_duplicate_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--topology=3 the topology ID to be deleted\n' '--new-topology-name=<new topology name> the name to be used in new topology\n') sys.exit(2) config_file = old_id_topology = new_topology_name = '' for opt, arg in opts: if opt == '-h': print('\n' 'Usage: MiniSecBGP_delete_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--topology=3 the topology ID to be deleted\n' '--new-topology-name=<new topology name> the name to be used in new topology\n') sys.exit() elif opt == '--config-file': config_file = arg elif opt == '--topology': old_id_topology = arg elif opt == '--new-topology-name': new_topology_name = arg if config_file and old_id_topology and new_topology_name: args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) dt = DuplicateTopology() with env['request'].tm: dbsession = env['request'].dbsession downloading = 1 dt.downloading(dbsession, downloading) with env['request'].tm: dbsession = env['request'].dbsession new_id_topology = dt.duplicate_topology(dbsession, old_id_topology, new_topology_name) with env['request'].tm: dbsession = env['request'].dbsession dt.duplicate_region_data(dbsession, old_id_topology, new_id_topology) with env['request'].tm: dbsession = env['request'].dbsession dt.duplicate_autonomous_system_data(dbsession, old_id_topology, new_id_topology) with env['request'].tm: dbsession = env['request'].dbsession dt.duplicate_another_data(dbsession, old_id_topology, new_id_topology) with env['request'].tm: dbsession = env['request'].dbsession downloading = 0 dt.downloading(dbsession, downloading) else: print('\n' 'Usage: MiniSecBGP_delete_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--topology=3 the topology ID to be deleted\n' '--new-topology-name=<new topology name> the name to be used in new topology\n')
def main(): parser = argparse.ArgumentParser( description= 'Launch the pyramid-scheduler job execution process.' ' This process is only required for pyramid-scheduler' ' configurations that are not running in "combined" mode' ' (i.e. all-in-one-process).' ) parser.add_argument( '--app-name', metavar='NAME', dest='appName', default='main', help='Load the named application (default: %(default)s)') parser.add_argument( '-q', '--queue', metavar='NAME', dest='queues', action='append', default=[], help='Restrict queues that this scheduler will handle - if not' ' specified, all queues will be handled (can be specified' ' multiple times)') parser.add_argument( '--reload', action='store_true', help='Use auto-restart file monitor') parser.add_argument( '--reload-interval', metavar='SECONDS', dest='reload_interval', default=1, type=int, help='Seconds between checking files (default: %(default)i)') parser.add_argument( '--restart', action='store_true', help='Automatically restart on unexpected exit') parser.add_argument( '--restart-interval', metavar='SECONDS', dest='restart_interval', default=5, type=int, help='Seconds to wait after an unexpected exit' ' (default: %(default)i)') parser.add_argument( '--message', metavar='TEXT', help='Message to send to the current consumer (no check is' ' made to ensure that it is currently running)') parser.add_argument( 'configUri', metavar='CONFIG-URI', help='The configuration URI or filename') options = parser.parse_args() setup_logging(options.configUri) global log log = logging.getLogger(__name__) if options.reload: if os.environ.get(RELOADERENV) == 'true': log.info('installing reloading file monitor') # todo: the problem with pserve's implementation of the # restarter is that it uses `os._exit()` which is # a "rude" exit and does not allow APS's atexit() # registrations to kick in... use a better # implementation! install_reloader(options.reload_interval, [options.configUri]) else: return restart_with_reloader(options) try: log.debug('loading application from "%s"', options.configUri) from pyramid_scheduler.scheduler import Scheduler app = get_app(options.configUri, name=options.appName) if not hasattr(app.registry, 'scheduler'): log.error('application did not load a scheduler (try' ' "config.include(\'pyramid_scheduler\')") - aborting') return 10 if options.message: with transaction.manager: app.registry.scheduler.broker.send(options.message) return 0 if app.registry.scheduler.conf.combined: log.error('application is configured for combined (i.e. single-process) operation - aborting') return 11 log.debug('starting consumer process on queues %r', options.queues) app.registry.scheduler.startConsumer(daemon=True, queues=options.queues) register_for_graceful_shutdown(app) wait_for_exit() except (KeyboardInterrupt, SystemExit): return 0 except Exception: log.exception('error while starting pscheduler') return 20
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """ This script pickles all updates/bugs/cves/comments and writes it out to disk in the format of bodhi-pickledb-YYYYMMDD.HHMM """ __requires__ = 'bodhi' import sys import cPickle as pickle from progressbar import ProgressBar, SimpleProgress, Percentage, Bar from pyramid.paster import setup_logging setup_logging('/etc/bodhi/production.ini') from sqlalchemy.orm import scoped_session, sessionmaker from zope.sqlalchemy import ZopeTransactionExtension from bodhi.util import get_critpath_pkgs import bodhi def load_sqlalchemy_db(): print "\nLoading pickled database %s" % sys.argv[2] db = file(sys.argv[2], 'r') data = pickle.load(db) import transaction from bodhi.models import Base
def run(self): # pragma: no cover if self.options.stop_daemon: return self.stop_daemon() if not hasattr(self.options, 'set_user'): # Windows case: self.options.set_user = self.options.set_group = None # @@: Is this the right stage to set the user at? self.change_user_group( self.options.set_user, self.options.set_group) if not self.args: self.out('You must give a config file') return 2 app_spec = self.args[0] if (len(self.args) > 1 and self.args[1] in self.possible_subcommands): cmd = self.args[1] restvars = self.args[2:] else: cmd = None restvars = self.args[1:] if self.options.reload: if os.environ.get(self._reloader_environ_key): if self.verbose > 1: self.out('Running reloading file monitor') install_reloader(int(self.options.reload_interval), [app_spec]) # if self.requires_config_file: # watch_file(self.args[0]) else: return self.restart_with_reloader() if cmd not in (None, 'start', 'stop', 'restart', 'status'): self.out( 'Error: must give start|stop|restart (not %s)' % cmd) return 2 if cmd == 'status' or self.options.show_status: return self.show_status() if cmd == 'restart' or cmd == 'stop': result = self.stop_daemon() if result: if cmd == 'restart': self.out("Could not stop daemon; aborting") else: self.out("Could not stop daemon") return result if cmd == 'stop': return result self.options.daemon = True if cmd == 'start': self.options.daemon = True app_name = self.options.app_name vars = self.parse_vars(restvars) if not self._scheme_re.search(app_spec): app_spec = 'config:' + app_spec server_name = self.options.server_name if self.options.server: server_spec = 'egg:pyramid' assert server_name is None server_name = self.options.server else: server_spec = app_spec base = os.getcwd() if getattr(self.options, 'daemon', False): if not self.options.pid_file: self.options.pid_file = 'pyramid.pid' if not self.options.log_file: self.options.log_file = 'pyramid.log' # Ensure the log file is writeable if self.options.log_file: try: writeable_log_file = open(self.options.log_file, 'a') except IOError as ioe: msg = 'Error: Unable to write to log file: %s' % ioe raise ValueError(msg) writeable_log_file.close() # Ensure the pid file is writeable if self.options.pid_file: try: writeable_pid_file = open(self.options.pid_file, 'a') except IOError as ioe: msg = 'Error: Unable to write to pid file: %s' % ioe raise ValueError(msg) writeable_pid_file.close() if getattr(self.options, 'daemon', False): try: self.daemonize() except DaemonizeException as ex: if self.verbose > 0: self.out(str(ex)) return 2 if (self.options.monitor_restart and not os.environ.get(self._monitor_environ_key)): return self.restart_with_monitor() if self.options.pid_file: self.record_pid(self.options.pid_file) if self.options.log_file: stdout_log = LazyWriter(self.options.log_file, 'a') sys.stdout = stdout_log sys.stderr = stdout_log logging.basicConfig(stream=stdout_log) log_fn = app_spec if log_fn.startswith('config:'): log_fn = app_spec[len('config:'):] elif log_fn.startswith('egg:'): log_fn = None if log_fn: log_fn = os.path.join(base, log_fn) setup_logging(log_fn) server = self.loadserver(server_spec, name=server_name, relative_to=base, global_conf=vars) app = self.loadapp(app_spec, name=app_name, relative_to=base, global_conf=vars) if self.verbose > 0: if hasattr(os, 'getpid'): msg = 'Starting server in PID %i.' % os.getpid() else: msg = 'Starting server.' self.out(msg) def serve(): try: server(app) except (SystemExit, KeyboardInterrupt) as e: if self.verbose > 1: raise if str(e): msg = ' ' + str(e) else: msg = '' self.out('Exiting%s (-v to see traceback)' % msg) serve()
def get_engine(config_file): setup_logging(config_file) settings = get_appsettings_(config_file) engine = engine_from_config(settings, 'sqlalchemy.') setup_db_session(engine) return engine
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[3:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, prefix='sqlalchemy.') Base.metadata.create_all(engine) maker = sessionmaker() maker.configure(bind=engine) session = get_session(maker, transaction.manager) all_data = Get_data(session=session, project_id=int(argv[2])) cost_list_parameter = [] cost_list_param = all_data.proposal.delicate_budget.split( unichr(171)) if all_data.proposal.cost is not None else [] cost_list_param.pop() for i in cost_list_param: cost_list_parameter.append(i.split(unichr(172))) if all_data.type == u'competitive': Gen_Doc_compet( doc_name='FRA241PROJECT/static/Gened_DOC/' + 'Competitive_' + str(all_data.id) + '.docx', project_name_th=all_data.title, date_cap=u'17 มกราคม – 21 มีนาคม 2559', where=all_data.proposal.activity_location, rational=all_data.proposal.Reason, purpose_list=all_data.proposal.objective.split(unichr(171)), profit=all_data.proposal.profit.split(unichr(171)), owner_list=all_data.proposal.owner_for_proposal.split(unichr(171)), advisor_list=all_data.proposal.advisor_for_proposal, member_list=all_data.proposal.member_for_proposal.split( unichr(171)), activity_place=all_data.proposal.activity_location, type_of_activity=all_data.proposal.type_of_activity, cost_list=cost_list_parameter, success_criteria=all_data.proposal.success_criteria.split( unichr(171))) # os.startfile('C:\Users\PHURINPAT\Documents\GitHub\FRA241Group6\FRA241PROJECT\static\Gened_DOC\Competitive_2.docx') elif all_data.type == u'camp': Gen_Doc_camp( camp_name_th=all_data.title, year=all_data.proposal.year, date_gap=u'', where=all_data.proposal.activity_location, rational=all_data.proposal.Reason, purpose_list=all_data.proposal.objective.split(unichr(171)), hours_compare=u'', owner_list=all_data.proposal.owner_for_proposal.split(unichr(171)), durations=all_data.proposal.duration, member_list=all_data.proposal.member_for_proposal.split( unichr(171)), evaluation_index=all_data.proposal.evaluation_index, profit=all_data.proposal.profit.split(unichr(171)), cost_list=all_data.proposal.cost.split(unichr(171)), cost_list_detail=cost_list_parameter, activity_list=all_data.proposal.schedule.split(unichr(171))) os.startfile( 'C:\Users\PHURINPAT\Documents\GitHub\FRA241Group6\sdf.docx')
from pyramid.paster import get_app, setup_logging import os repo_folder = os.environ.get('OPENSHIFT_REPO_DIR', os.path.dirname(__file__)) ini_path = os.path.join(repo_folder, 'production.ini') setup_logging(ini_path) application = get_app(ini_path, 'main')
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 2: usage(argv) proxy = None serverurl = None config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = get_engine(settings) session_factory = get_session_factory(engine) session = get_tm_session(session_factory, transaction.manager) if 'xml_proxy' in settings: serverurl = settings['xml_proxy'] proxy = ServerProxy(serverurl) context = zmq.Context() subscriber = context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, b"") subscriber.setsockopt(zmq.RCVTIMEO, __timeoutEDDN) starttime = time.time() lasthourly = time.time( ) - 3700 # Ensure we start by running the hourly once. messages = 0 syscount = 0 starcount = 0 stationcount = 0 failstar = 0 failstation = 0 totmsg = 0 hmessages = 0 if proxy: try: proxy.command("botserv", "Absolver", "say #rattech [SAPI]: EDDN client has started.") except ProtocolError as e: print(f"Failed to send start message to XMLRPC. {e.errmsg}") except TimeoutError as e: print(f"Failed to send start message to XMLRPC. {e.strerror}") while True: try: subscriber.connect(__relayEDDN) while True: __message = subscriber.recv() if not __message: subscriber.disconnect(__relayEDDN) break __message = zlib.decompress(__message) __json = simplejson.loads(__message) totmsg = totmsg + 1 print( f"EDDN Client running. Messages: {messages:10} Stars: {starcount:10} Systems: {syscount:10} " f" Stations: {stationcount:5} Missing systems: {failstar+failstation:10}\r", end='') if validsoftware(__json['header']['softwareName'], __json['header']['softwareVersion']) \ and __json['$schemaRef'] in __allowedSchema: hmessages = hmessages + 1 if proxy: if time.time() > (starttime + 3600 * 24): try: startot = session.query(func.count( Star.id64)).scalar() systot = session.query(func.count( System.id64)).scalar() proxy.command( "botserv", "Absolver", f"say #ratchat [\x0315SAPI\x03] Daily report: " f"{'{:,}'.format(messages)} messages processed" f", {'{:,}'.format(syscount)} new systems," f" {'{:,}'.format(starcount)} new stars." f" DB contains {'{:,}'.format(startot)} stars " f"and {'{:,}'.format(systot)} systems.") messages = 0 syscount = 0 starcount = 0 failstar = 0 stationcount = 0 failstation = 0 starttime = time.time() except TimeoutError: print( "XMLRPC call failed due to timeout, retrying in 320 seconds." ) starttime = starttime + 320 except ProtocolError as e: print( f"XMLRPC call failed, skipping this update. {e.errmsg}" ) starttime = time.time() if time.time() > (lasthourly + 3600): # print("Running stats update...") loop = asyncio.get_event_loop() future = asyncio.Future() asyncio.ensure_future(update_stats( session, future)) future.add_done_callback(update_complete) try: loop.run_until_complete(future) proxy.command( f"botserv", "Absolver", f"say #announcerdev [\x0315SAPI\x03] " f"Hourly report: {hmessages} messages, " f"{totmsg - hmessages} ignored.") lasthourly = time.time() hmessages = 0 totmsg = 0 except TimeoutError: print( "XMLRPC call failed due to timeout, retrying in one hour." ) lasthourly = time.time() + 3600 except ProtocolError as e: print( f"XMLRPC call failed, skipping this update. {e.errmsg}" ) lasthourly = time.time() data = __json['message'] messages = messages + 1 if 'event' in data: if data['event'] in {'Docked', 'CarrierJump'}: if 'StationType' in data and data[ 'StationType'] == 'FleetCarrier': try: oldcarrier = session.query(Carrier).filter( Carrier.callsign == data['StationName']) # Consistency?! What's that? Bah. if oldcarrier: oldcarrier.marketId = data['MarketID'] oldcarrier.systemName = data[ 'StarSystem'] oldcarrier.systemId64 = data[ 'SystemAddress'] oldcarrier.haveShipyard = True if 'shipyard' in data['StationServices'] \ else False oldcarrier.haveOutfitting = True if 'outfitting' in data[ 'StationServices'] else False oldcarrier.haveMarket = True if 'commodities' in data['StationServices'] \ else False oldcarrier.updateTime = data[ 'timestamp'] elif 'StationType' in data and data[ 'StationType'] != 'FleetCarrier': newcarrier = Carrier( callsign=data['StationName'], marketId=data['MarketID'], name=data['StationName'], updateTime=data['timestamp'], systemName=data['StarSystem'], systemId64=data['SystemAddress'], haveShipyard=True if 'shipyard' in data['StationServices'] else False, haveOutfitting=True if 'outfitting' in data['StationServices'] else False, haveMarket=True if 'commodities' in data['StationServices'] else False) session.add(newcarrier) transaction.commit() except DataError as e: print( f"Failed to add a carrier! Invalid data passed: {e}" ) transaction.abort() except KeyError as e: print(f"Invalid key in carrier data: {e}") print(data) print( f"Software: {__json['header']['softwareName']} {__json['header']['softwareVersion']}" ) transaction.abort() else: try: #print(f"Got a station: {data}") # Station data, check if exists. oldstation = session.query(Station).filter(Station.name == data['StationName']).\ filter(Station.systemName == data['StarSystem']) if oldstation: continue else: # New station, add it! newstation = Station( id64=data['MarketID'], name=data['StationName'], distanceToArrival=data[ 'DistFromStarLS'], government=data[ 'StationGovernment'], economy=data['StationEconomy'], haveMarket=True if 'commodities' in data['StationServices'] else False, haveShipyard=True if 'shipyard' in data['StationServices'] else False, haveOutfitting=True if 'outfitting' in data['StationServices'] else False, otherServices=data[ 'StationServices'], updateTime=data['timestamp'], systemId64=data['SystemAddress'], systemName=data['StarSystem']) session.add(newstation) stationcount += 1 except DataError as e: print( f"Failed to add a station! Invalid data passed: {e}" ) transaction.abort() except KeyError as e: print(f"Invalid key in station data: {e}") except IntegrityError: failstation = failstation + 1 transaction.abort() # TODO: Handle other detail Carrier events, such as Stats. if data['event'] == 'FSDJump': id64 = data['SystemAddress'] res = session.query(System.id64).filter( System.id64 == id64).scalar() or False if not res: syscount = syscount + 1 newsys = System(id64=data['SystemAddress'], name=data['StarSystem'], coords={ 'x': data['StarPos'][0], 'y': data['StarPos'][1], 'z': data['StarPos'][2] }, date=data['timestamp']) try: session.add(newsys) transaction.commit() except DataError: print( "Failed to add a system! Invalid data passed" ) transaction.abort() except IntegrityError: transaction.abort() if data['event'] == 'Scan': bodyid = data['SystemAddress'] + ( data['BodyID'] << 55) if 'AbsoluteMagnitude' in data: res = session.query(Star.id64).filter( Star.id64 == bodyid).scalar() or False if not res: starcount = starcount + 1 newstar = Star( id64=bodyid, bodyId=data['BodyID'], name=data['BodyName'], age=data['Age_MY'], axialTilt=data['AxialTilt'], orbitalEccentricity=data['Eccentricity'] if 'Eccentricity' in data else None, orbitalInclination=data[ 'OrbitalInclination'] if 'OrbitalInclination' in data else None, orbitalPeriod=data['OrbitalPeriod'] if 'OrbitalPeriod' in data else None, parents=data['Parents'] if 'Parents' in data else None, argOfPeriapsis=data['Periapsis'] if 'Periapsis' in data else None, belts=data['Rings'] if 'Rings' in data else None, semiMajorAxis=data['SemiMajorAxis'] if 'SemiMajorAxis' in data else None, systemName=data['StarSystem'], distanceToArrival=data[ 'DistanceFromArrivalLS'], luminosity=data['Luminosity'], solarRadius=data['Radius'], rotationalPeriod=data[ 'RotationPeriod'], type=data['StarType'], solarMasses=data['StellarMass'], subType=data['Subclass'] if 'Subclass' in data else None, surfaceTemperature=data[ 'SurfaceTemperature'], isScoopable=True if data['StarType'] in __scoopable else False, isMainStar=True if data['BodyID'] == 0 else False, updateTime=data['timestamp'], systemId64=data['SystemAddress']) try: session.add(newstar) transaction.commit() except DataError: print( "Failed to add star - Data Error!") transaction.abort() except IntegrityError: try: r = requests.get( f"{__EDSM_url}/systems?systemName={data['SystemName']}&" f"showId=1&showCoordinates=1&showInformation=1" ).json()[0] if r: system = System( id64=r['id64'], name=r['name'], coords=r['coords']) session.add(system) transaction.commit() except IntegrityError: print( "Failed to add system during missing star handling. Bah. Give up." ) failstar = failstar + 1 transaction.abort() except KeyError: #print("No EDSM data found.") failstar += 1 transaction.abort() sys.stdout.flush() except zmq.ZMQError as e: print('ZMQSocketException: ' + str(e)) proxy.command( "botserv", "Absolver", f"say #rattech [\x0315SAPI\x03] EDDN error: " f"Exiting due to exception: {str(e)}") sys.stdout.flush() subscriber.disconnect(__relayEDDN) time.sleep(5)
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, 'h', ["config-file=", "execution-type=", "node-ip-address=", "username="******"password="]) except getopt.GetoptError: print('\n' 'Usage: MiniSecBGP_node_service [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--execution-type=[manual|scheduled] manual = when user execute this function in CLI\n' ' scheduled = when this function is executed by crontab\n' '--node-ip-address=[192.168.0.1|3232239375] cluster node IP address\n' '--username=ubuntu the username to use to configure the cluster node\n' '--password=ubuntu the user password to access the cluster node\n') sys.exit(2) config_file = execution_type = node_ip_address = username = password = '' for opt, arg in opts: if opt == '-h': print('\n' 'Usage: MiniSecBGP_node_service [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--execution-type=[manual|scheduled] manual = when user execute this function in CLI\n' ' scheduled = when this function is executed by crontab\n' '--node-ip-address=[192.168.0.1|3232239375] cluster node IP address\n' '--username=ubuntu the username to use to configure the cluster node\n' '--password=ubuntu the user password to access the cluster node\n') sys.exit() elif opt == '--config-file': config_file = arg elif opt == '--execution-type' and (arg == 'manual' or arg == 'scheduled'): execution_type = arg elif opt == '--node-ip-address': node_ip_address = arg elif opt == '--username': username = arg elif opt == '--password': password = arg if config_file and execution_type and node_ip_address: args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) try: with env['request'].tm: dbsession = env['request'].dbsession ccn = TestClusterNode(dbsession, execution_type, node_ip_address, username, password) ccn.test_ping() ccn.test_ssh() except OperationalError: print('Database error') else: print('\n' 'Usage: MiniSecBGP_node_service [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--execution-type=[manual|scheduled] manual = when user execute this function in CLI\n' ' scheduled = when this function is executed by crontab\n' '--node-ip-address=[192.168.0.1|3232239375] cluster node IP address\n' '--username=ubuntu the username to use to configure the cluster node\n' '--password=ubuntu the user password to access the cluster node\n')
def main(argv=sys.argv): # pragma: no cover from substanced.evolution import EvolutionManager def usage(e=None): if e is not None: _print(e) _print('') _print("""\ sd_evolve [--latest] [--dry-run] [--mark-finished=stepname] [--mark-unfinished=stepname] config_uri Evolves new database with changes from scripts in evolve packages - with no arguments, evolve displays finished and unfinished steps - with the --latest argument, evolve runs scripts as necessary - with the --dry-run argument, evolve runs scripts but does not issue any commits - with the --mark-finished argument, marks the stepname as finished - with the --mark-unfinished argument, marks the stepname as unfinished e.g. sd_evolve --latest etc/development.ini""") sys.exit(2) name, argv = argv[0], argv[1:] latest = False dry_run = False mark_finished = [] mark_unfinished = [] try: opts, args = getopt.getopt(argv, 'l?hdu:f:', [ 'latest', 'help', 'dry-run', 'mark-unfinished=', 'mark-finished=', ]) except getopt.GetoptError as e: usage(e) if args: config_uri = args[0] else: usage('Requires a config_uri as an argument') for k, v in opts: if k in ('-h', '-?', '--help'): usage() if k in ('-l', '--latest'): latest = True if k in ('-d', '--dry-run'): dry_run = True if k in ('-f', '--mark-finished'): mark_finished.append(v) if k in ('-u', '--mark-unfinished'): mark_unfinished.append(v) if latest and dry_run: usage('--latest and --dry-run cannot be used together') if (latest or dry_run) and (mark_finished or mark_unfinished): usage( '--latest/--dry-run cannot be used with --mark-finished/--mark-unfinished' ) setup_logging(config_uri) env = bootstrap(config_uri) root = env['root'] registry = env['registry'] manager = EvolutionManager(root, registry) if latest or dry_run: complete = manager.evolve(latest) if complete: if dry_run: _print('Evolution steps dry-run:') else: _print('Evolution steps executed:') for item in complete: _print(' %s' % item) else: if dry_run: _print('No evolution steps dry-run') else: _print('No evolution steps executed') elif mark_finished or mark_unfinished: t = transaction.get() for step in mark_finished: finished_steps = manager.get_finished_steps() unfinished_steps = dict(manager.get_unfinished_steps()) if step in finished_steps: _print('Step %s already marked as finished' % step) else: if step in unfinished_steps: manager.add_finished_step(step) _print('Step %s marked as finished' % step) t.note('Marked %s evolution step as finished' % step) else: _print('Unknown step %s, not marking as finished' % step) for step in mark_unfinished: finished_steps = manager.get_finished_steps() unfinished_steps = dict(manager.get_unfinished_steps()) if step in finished_steps: manager.remove_finished_step(step) _print('Step %s marked as unfinished' % step) t.note('Marked %s evolution step as unfinished' % step) else: if step in unfinished_steps: _print('Step %s already marked as unfinished' % step) else: _print('Unknown step %s, not marking as unfinished' % step) t.commit() else: _print('Finished steps:\n') for ts, stepname in manager.get_finished_steps_by_value(): tp = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') _print(' %s %s' % (tp, stepname)) _print('\nUnfinished steps:\n') for stepname, func in manager.get_unfinished_steps(): _print(' ' * 24 + stepname)
def main(argv=sys.argv): from fixtures.data import trees, geo from fixtures.styles_and_cultures import styles_and_cultures from fixtures.materials import materials from fixtures.eventtypes import eventtypes from fixtures.heritagetypes import heritagetypes from fixtures.periods import periods from fixtures.species import species if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') session_maker = sessionmaker(bind=engine, extension=ZopeTransactionExtension()) db_session = session_maker() with transaction.manager: import_provider( trees, ConceptScheme(id=1, uri='urn:x-skosprovider:trees', labels=[ Label('Verschillende soorten bomen', u'prefLabel', u'nl'), Label('Different types of trees', u'prefLabel', u'en') ]), db_session) import_provider( geo, ConceptScheme(id=2, uri='urn:x-skosprovider:geo', labels=[ Label('Geografie', u'prefLabel', u'nl'), Label('Geography', u'prefLabel', u'en') ]), db_session) import_provider( styles_and_cultures, ConceptScheme( id=3, uri='https://id.erfgoed.net/thesauri/stijlen_en_culturen', labels=[ Label('Stijlen en Culturen', u'prefLabel', u'nl'), Label('Styles and Cultures', u'prefLabel', u'en') ]), db_session) import_provider( materials, ConceptScheme(id=4, uri='https://id.erfgoed.net/thesauri/materialen', labels=[ Label('Materialen', u'prefLabel', u'nl'), Label('Materials', u'prefLabel', u'en') ]), db_session) import_provider( eventtypes, ConceptScheme( id=5, uri='https://id.erfgoed.net/thesauri/gebeurtenistypes', labels=[ Label('Gebeurtenistypes', u'prefLabel', u'nl'), Label('Event types', u'prefLabel', u'en') ]), db_session) import_provider( heritagetypes, ConceptScheme(id=6, uri='https://id.erfgoed.net/thesauri/erfgoedtypes', labels=[ Label('Erfgoedtypes', u'prefLabel', u'nl'), Label('Heritage types', u'prefLabel', u'en') ]), db_session) import_provider( periods, ConceptScheme(id=7, uri='https://id.erfgoed.net/thesauri/dateringen', labels=[ Label('Dateringen', u'prefLabel', u'nl'), Label('Periods', u'prefLabel', u'en') ]), db_session) import_provider( species, ConceptScheme(id=8, uri='https://id.erfgoed.net/thesauri/soorten', labels=[ Label('Soorten', u'prefLabel', u'nl'), Label('Species', u'prefLabel', u'en') ]), db_session) print('--atramhasis-db-initialized--')
from ..models import * from pyramid.paster import ( get_appsettings, setup_logging, ) from pyramid.config import Configurator from paste.deploy.loadwsgi import appconfig from sqlalchemy import engine_from_config from sqlalchemy.orm import sessionmaker import os ROOT_PATH = os.path.dirname(__file__) CONFIG_PATH = os.path.join(ROOT_PATH, '..', '..', 'config', 'test.ini') setup_logging(CONFIG_PATH) from dbbase import settings def setup(): engine = engine_from_config(settings, prefix='sqlalchemy.') config = Configurator(settings=settings) DBSession.configure(bind=engine) Base.metadata.create_all(engine) def teardown(): engine = engine_from_config(settings, prefix='sqlalchemy.') DBSession.configure(bind=engine)
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = get_engine(settings) Base.metadata.create_all(engine) session_factory = get_session_factory(engine) with transaction.manager: dbsession = get_tm_session(session_factory, transaction.manager) group1 = Group(name='Group 1') group2 = Group(name='Group 2') dbsession.add(group1) dbsession.add(group2) dbsession.flush() student=[] student.append(Student(name='Vovan', group_id=group1.id)) student.append(Student(name='Petya', group_id=group1.id)) student.append(Student(name='Serega', group_id=group1.id)) student.append(Student(name='Kolyan', group_id=group1.id)) student.append(Student(name='Harry', group_id=group1.id)) student.append(Student(name='Vasiliy', group_id=group2.id)) for s in student: dbsession.add(s) dbsession.flush() prof1 = Professor(name='Mr White') prof2 = Professor(name='Mr Black') prof3 = Professor(name='Mr Brown') prof4 = Professor(name='Mr Pink') dbsession.add(prof1) dbsession.add(prof2) dbsession.add(prof3) dbsession.add(prof4) sub1 = Subject(name='Math') sub2 = Subject(name='Physics') sub3 = Subject(name='Programming') sub4 = Subject(name='History') dbsession.add(sub1) dbsession.add(sub2) dbsession.add(sub3) dbsession.add(sub4) dbsession.flush() course=[] course.append(Course(group_id=group1.id, professor_id=prof1.id, subject_id=sub1.id)) course.append(Course(group_id=group1.id, professor_id=prof2.id, subject_id=sub2.id)) course.append(Course(group_id=group1.id, professor_id=prof3.id, subject_id=sub3.id)) course.append(Course(group_id=group1.id, professor_id=prof4.id, subject_id=sub4.id)) course.append(Course(group_id=group2.id, professor_id=prof2.id, subject_id=sub2.id)) course.append(Course(group_id=group2.id, professor_id=prof3.id, subject_id=sub3.id)) course.append(Course(group_id=group2.id, professor_id=prof4.id, subject_id=sub4.id)) course.append(Course(group_id=group2.id, professor_id=prof1.id, subject_id=sub1.id)) for c in course: dbsession.add(c) dbsession.flush() work=[] work.append(Work(course_id=course[0].id, max_point=25, name='Control Work 1')) work.append(Work(course_id=course[0].id, max_point=30, name='Control Work 2')) work.append(Work(course_id=course[1].id, max_point=30, name='Control Work 3')) work.append(Work(course_id=course[1].id, max_point=30, name='Control Work 4')) work.append(Work(course_id=course[2].id, max_point=30, name='Control Work 5')) work.append(Work(course_id=course[3].id, max_point=25, name='Control Work 6')) work.append(Work(course_id=course[4].id, max_point=30, name='Control Work 7')) work.append(Work(course_id=course[5].id, max_point=30, name='Control Work 8')) work.append(Work(course_id=course[6].id, max_point=30, name='Control Work 9')) work.append(Work(course_id=course[7].id, max_point=30, name='Control Work 10')) for w in work: dbsession.add(w) dbsession.flush() rating=[] rating.append(Rating(student_id=student[0].id, work_id=work[0].id, point=15)) rating.append(Rating(student_id=student[0].id, work_id=work[1].id, point=25)) rating.append(Rating(student_id=student[1].id, work_id=work[2].id, point=24)) rating.append(Rating(student_id=student[2].id, work_id=work[3].id, point=15)) rating.append(Rating(student_id=student[3].id, work_id=work[4].id, point=25)) rating.append(Rating(student_id=student[4].id, work_id=work[5].id, point=20)) rating.append(Rating(student_id=student[4].id, work_id=work[6].id, point=25)) for r in rating: dbsession.add(r) dbsession.flush() user=[] user.append(User(name='student', role='base')) user[0].set_password("123456") user.append(User(name='professor', role='editor')) user[1].set_password("123456") for u in user: dbsession.add(u)
def update(argv=sys.argv): """ Download the GeoIP database from the URL provided under the config key `geoip.city.source` and save it under the path provided by the config key `geoip.city.destination`. """ if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) source = settings['geoip.city.csv_source'] destination = settings['geoip.city.csv_destination'] engine = engine_from_config(settings, 'sqlalchemy.') db_session = scoped_session(sessionmaker()) db_session.configure(bind=engine) if source.startswith('http://'): log.info("Downloading %s...", source) response = requests.get(source) log.info("Downloading done.") compressed = zipfile.ZipFile(StringIO(response.content)) else: log.info("Opening %s...", source) compressed = zipfile.ZipFile(source) log.info("Writing to %s...", destination) compressed.extractall(path=destination) log.info("Writing done.") source_dir = os.path.join(destination, listdir(destination)[0]) final_path = os.path.join(source_dir, 'GeoLiteCity-Location.csv') log.info("Dropping the GeoRegion table") GeoRegion.__table__.drop(engine) log.info("Creating the GeoRegion table") GeoRegion.__table__.create(engine) countries = {} for country in db_session.query(Country).all(): countries[country.alpha2] = country db_session.commit() if six.PY3: infile = open(final_path, 'r', newline='', encoding='latin1') else: infile = open(final_path, 'rb') with infile as f: reader = csv_reader(f, delimiter=',') rows = list(reader) chunk_size = 5000 split_rows = chunks(rows[2:], chunk_size) thread_count = 10.0 row_len = len(split_rows) loops = math.ceil(row_len / thread_count) current_row = 0 total = 0 for i in range(int(loops)): threads = [] for j in range(int(thread_count)): if current_row < row_len: rows = split_rows[current_row] t = Thread(target=write_to_db, args=(db_session, rows, countries)) threads.append(t) current_row += 1 [x.start() for x in threads] for x in threads: x.join() total += int(thread_count) * chunk_size logging.info("Done adding %s locations in DB\n" % total) log.info('Finished, pushing to the database')
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, 'h', ["config-file=", "file="]) except getopt.GetoptError: print( '\n' 'Usage: MiniSecBGP_realistic_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--file=20191201.as-rel2.txt.bz2 CAIDA AS-Relationship compressed topology filename\n' ' * the file must be in /tmp\n' ) sys.exit(2) config_file = file = '' for opt, arg in opts: if opt == '-h': print( '\n' 'Usage: MiniSecBGP_realistic_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--file=20191201.as-rel2.txt.bz2 CAIDA AS-Relationship compressed topology filename\n' ' * the file must be in /tmp\n' ) sys.exit() elif opt == '--config-file': config_file = arg elif opt == '--file': file = arg if config_file and file: args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) try: t = RealisticTopology(file) with env['request'].tm: dbsession = env['request'].dbsession downloading = 1 t.downloading(dbsession, downloading) with env['request'].tm: pandas_unique_autonomous_systems, pandas_stub_autonomous_systems, df_from_file = t.as_relationship( ) with env['request'].tm: dbsession = env['request'].dbsession t.topology(dbsession) with env['request'].tm: dbsession = env['request'].dbsession id_topology = t.get_topology_id(dbsession) with env['request'].tm: dbsession = env['request'].dbsession t.region(dbsession, id_topology) with env['request'].tm: dbsession = env['request'].dbsession id_region = t.get_region_id(dbsession, id_topology) with env['request'].tm: dbsession = env['request'].dbsession t.autonomous_system(dbsession, id_topology, id_region, pandas_unique_autonomous_systems, pandas_stub_autonomous_systems) with env['request'].tm: dbsession = env['request'].dbsession t.automatic_router_id(dbsession, id_topology) with env['request'].tm: dbsession = env['request'].dbsession t.automatic_prefix(dbsession, id_topology) with env['request'].tm: dbsession = env['request'].dbsession t.automatic_link(dbsession, id_topology, df_from_file) with env['request'].tm: dbsession = env['request'].dbsession downloading = 0 t.downloading(dbsession, downloading) t.erase_file() except OperationalError: print('Database error') else: print( '\n' 'Usage: MiniSecBGP_realistic_topology [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file=minisecbgp.ini pyramid config filename [.ini]\n' '--file=20191201.as-rel2.txt.bz2 CAIDA AS-Relationship compressed topology filename\n' ' * the file must be in /tmp\n' )
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, "h", [ "config-file=", "scenario-id=", "scenario-name=", "scenario-description=", "topology=", "attacker=", "affected-area=", "target=", "attack-type=", "all-paths", "number-of-shortest-paths=" ]) except getopt.GetoptError: print( '\n' 'ERROR\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' ) sys.exit(2) config_file = scenario_id = scenario_name = scenario_description = topology = attacker = \ affected_area = target = attack_type = number_of_shortest_paths = '' for opt, arg in opts: if opt == '-h': print( '\n' 'HELP\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' ) sys.exit() if opt == '--config-file': config_file = arg elif opt == '--scenario-id': scenario_id = arg elif opt == '--scenario-name': scenario_name = arg elif opt == '--scenario-description': scenario_description = arg elif opt == '--topology': topology = arg elif opt == '--attacker': attacker = arg elif opt == '--affected-area': affected_area = arg elif opt == '--target': target = arg elif opt == '--attack-type': attack_type = arg elif opt == '--all-paths': number_of_shortest_paths = '0' elif opt == '--number-of-shortest-paths': number_of_shortest_paths = arg if (config_file and scenario_name and topology and attacker and affected_area and target and attack_type and number_of_shortest_paths) \ or (config_file and scenario_id): args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) try: with env['request'].tm: dbsession = env['request'].dbsession print('iniciando o objeto') aa = AttackScenario(dbsession, scenario_id, scenario_name, scenario_description, topology, attacker, affected_area, target, attack_type, number_of_shortest_paths) print('iniciando o attack_scenario') links, edge_array_length, count_asys = aa.attack_scenario() link = links link_filename = '/tmp/link_' + str(datetime.now()).replace( ' ', '').replace(':', '').replace('-', '').replace( '.', '') + '.MiniSecBGP' f = open(link_filename, "a") f.write(link) f.close() print('VOU CHAMAR O CÓDIGO C++') print(link_filename) print(edge_array_length) print(count_asys) #arguments = [link_filename, edge_array_length, count_asys] arguments = ['/tmp/bla.MiniSecBGP', str(5), str(5)] subprocess.Popen(['./venv/bin/asp'] + arguments) print('RETORNEI DO CÓDIGO C++') with env['request'].tm: if scenario_id: clear_database(dbsession, scenario_id) except OperationalError: print('Database error') else: print( '\n' 'Usage: MiniSecBGP_hijack_attack_scenario [options]\n' '\n' 'options (with examples):\n' '\n' '-h this help\n' '\n' '--config-file="minisecbgp.ini" pyramid config filename [.ini]\n' '--scenario-name="Test topology" the name that will be used to identify this scenario\n' '--scenario-description="date 20200729" the scenario description\n' '--topology=3 the topology used as the original base of the scenario\n' '--attacker=[65001,65002] define which AS(s) will be the attacker\n' '--affected-area=[65001,65003] define which these AS(s) will receive and accept the hijacked routes\n' '--target=[\'10.0.0.0/24\',\'20.0.0.0/24\'] define the prefix(s) and mask(s) that will be hijacked by the attacker(s)\n' '--attack-type=attraction|interception if the attack is an attraction attack or an interception attack\n' '--all-paths or --number-of-shortest-paths=[1..999] number of valid paths between the attacker AS, affected AS and target AS\n' '\n' 'or\n' '\n' '--scenario-id=16 scenario ID\n' )
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') db_session.configure(bind=engine) es_product_index = get_es_product_index(settings) es_client.indices.create( index=es_product_index.get_name(), body={ 'settings': { 'index': { 'number_of_shards': 4, 'number_of_replicas': 0, } } } ) es_product_index.put_mapping(ProductDoc) limit = 5 last_product_id = None indexed_count = 0 while True: query = ( db_session.query(Product.id, Product.name, Product.status) .order_by(Product.id) ) if last_product_id is not None: query = query.filter(Product.id > last_product_id) products = query.limit(limit).all() if not products: break docs = [ ProductDoc(_id=p.id, name=p.name, status=p.status) for p in products ] bulk_res = es_product_index.add(docs) indexed_count += len(docs) if bulk_res.errors: raise ElasticsearchBulkError( '\n' + '\n'.join( '\t{}'.format(it.error.reason) for it in bulk_res.items ) ) if len(products) < limit: break last_product_id = products[-1].id es_product_index.refresh() log.info('Indexed %s products', indexed_count) log.info( '%s products in the index', es_product_index.search_query().count() )
the database does not exists. """ import sys import psycopg2 import logging from datetime import datetime, timedelta import time from pyramid.paster import get_appsettings, setup_logging from sqlalchemy.engine.url import make_url config_uri = sys.argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) log = logging.getLogger(__name__) setting_url = settings['sqlalchemy.url'] url = make_url(setting_url) if url.drivername not in ('postgresql', 'postgresql+psycopg2'): log.warn('{0} is not a postgresql database. ignored'.format(setting_url)) sys.exit(0) params = { 'db': url.database, 'user': url.username, 'password': url.password, 'encoding': 'utf-8',
def main(argv=sys.argv[1:]): try: opts, args = getopt.getopt(argv, "h:", [ "config-file=", "topology=", "include-stub=", "cluster-list=", "topology-distribution-method=", "emulation-platform=", "router-platform=" ]) except getopt.GetoptError as error: print( 'MiniSecBGP_hijack_realistic_analysis ' '--config-file=<pyramid config file .ini> ' '--topology=<Topology ID> ' '--include-stub=<True|False> ' '--cluster-list=<hostname list of cluster nodes> ' '--topology-distribution-method=<Topology distribution method ID (CUSTOMER CONE|METIS|MANUAL|ROUND ROBIN)> ' '--emulation-platform=<Emulation platform ID (MININET|DOCKER)> ' '--router-platform=<Router platform ID (QUAGGA|BIRD)>') sys.exit(2) for opt, arg in opts: if opt == '-h': print( 'MiniSecBGP_hijack_realistic_analysis ' '--config-file=<pyramid config file .ini> ' '--topology=<Topology ID> ' '--include-stub=<True|False> ' '--cluster-list=<hostname list of cluster nodes> ' '--topology-distribution-method=<Topology distribution method ID (CUSTOMER CONE|METIS|MANUAL|ROUND ROBIN)> ' '--emulation-platform=<Emulation platform ID (MININET|DOCKER)> ' '--router-platform=<Router platform ID (QUAGGA|BIRD)>') sys.exit() elif opt == '--config-file': config_file = arg elif opt == '--topology': id_topology = arg elif opt == '--include-stub': include_stub = str2bool(arg) elif opt == '--cluster-list': cluster_list = arg elif opt == '--topology-distribution-method': topology_distribution_method = arg elif opt == '--emulation-platform': emulation_platform = arg elif opt == '--router-platform': router_platform = arg args = parse_args(config_file) setup_logging(args.config_uri) env = bootstrap(args.config_uri) try: with env['request'].tm: dbsession = env['request'].dbsession quagga = dbsession.query(models.RouterPlatform.id). \ filter(func.lower(models.RouterPlatform.router_platform) == 'quagga').first() ra = RealisticAnalysis(dbsession, id_topology, include_stub, cluster_list, topology_distribution_method, emulation_platform, router_platform) time_get_data = time.time() ra.dfs_from_database() ra.data_frames() time_get_data = time.time() - time_get_data save_to_database(dbsession, ['time_get_data'], [time_get_data], id_topology) time_autonomous_system_per_server = time.time() ra.autonomous_system_per_server() time_autonomous_system_per_server = time.time( ) - time_autonomous_system_per_server save_to_database(dbsession, ['time_autonomous_system_per_server'], [time_autonomous_system_per_server], id_topology) time_emulate_platform_commands = time.time() ra.emulation_commands() time_emulate_platform_commands = time.time( ) - time_emulate_platform_commands save_to_database(dbsession, ['time_emulate_platform_commands'], [time_emulate_platform_commands], id_topology) # Router platform time_router_platform_commands = time.time() if router_platform == str(quagga[0]): ra.quagga_commands() time_router_platform_commands = time.time( ) - time_router_platform_commands save_to_database(dbsession, ['time_router_platform_commands'], [time_router_platform_commands], id_topology) time_write_files = time.time() ra.write_to_file() time_write_files = time.time() - time_write_files save_to_database(dbsession, ['time_write_files'], [time_write_files], id_topology) time_copy_files = time.time() ra.copy_files_to_cluster_nodes() time_copy_files = time.time() - time_copy_files save_to_database(dbsession, ['time_copy_files'], [time_copy_files], id_topology) except OperationalError: print('Database error')
def main(): setup_logging('development.ini') settings = get_appsettings('development.ini') load_local_settings(settings) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) translation_manager.options.update({ 'locales': settings['available_languages'].split(), 'get_locale_fallback': True }) configure_mappers() postgis_version = DBSession.execute(func.postgis_version()).scalar() if not postgis_version.startswith('2.'): # With PostGIS 1.x the AddGeometryColumn and DropGeometryColumn # management functions should be used. Area.__table__.c.geometry.type.management = True Area.__table__.c.centroid.type.management = True Task.__table__.c.geometry.type.management = True Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with transaction.manager: geometry = '{"type":"Polygon","coordinates":[[[85.31038284301758,27.70731518595052],[85.31089782714842,27.698120147680104],[85.3242015838623,27.69842412827061],[85.323429107666,27.70731518595052],[85.31038284301758,27.70731518595052]]]}' # noqa geometry = geojson.loads(geometry, object_hook=geojson.GeoJSON.to_instance) geometry = shapely.geometry.asShape(geometry) geometry = shape.from_shape(geometry, 4326) area = Area(geometry) DBSession.add(area) license1 = License() license1.name = 'NextView' license1.description = "This data is licensed for use by the US Government (USG) under the NextView (NV) license and copyrighted by Digital Globe or GeoEye. The NV license allows the USG to share the imagery and Literal Imagery Derived Products (LIDP) with entities outside the USG when that entity is working directly with the USG, for the USG, or in a manner that is directly beneficial to the USG. The party receiving the data can only use the imagery or LIDP for the original purpose or only as otherwise agreed to by the USG. The party receiving the data cannot share the imagery or LIDP with a third party without express permission from the USG. At no time should this imagery or LIDP be used for other than USG-related purposes and must not be used for commercial gain. The copyright information should be maintained at all times. Your acceptance of these license terms is implied by your use." # noqa license1.plain_text = "In other words, you may only use NextView imagery linked from this site for digitizing OpenStreetMap data for humanitarian purposes." # noqa DBSession.add(license1) license2 = License() license2.name = 'Astrium/UNOSAT' license2.description = "UNOSAT allow any INTERNET USER to use the IMAGE to develop DERIVATIVE WORKS provided that the INTERNET USER includes the DERIVATIVE WORKS he/she created in the OpenStreetMap database under CC-BY-SA licence (http://creativecommons.org/licenses/by-sa/2.0/) and/or Open Database licence (ODbL: http://www.opendatacommons.org/licenses/odbl/), with the credit of the corresponding PRODUCT conspicuously displayed and written in full, in order to allow any OpenStreetMap database user to have access to and to use the DERIVATIVE WORKS. Except for the foregoing, the END USER and/or the INTERNET USER shall not be entitled to sell, distribute, assign, dispose of, lease, sublicense or transfer, directly or indirectly, any DERIVATIVE WORKS to any third party." # noqa license2.plain_text = "Astrium GEO-Information Services and UNOSAT are allowing access to this imagery for creating information in OpenStreetMap. Other uses are not allowed." # noqa DBSession.add(license2) project = Project('Kathmandu - Map all primary roads and buildings') project.area = area project.short_description = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." # noqa project.description = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." # noqa project.instructions = "**The detailed instructions**\n\nLorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat." # noqa project.entities_to_map = "primary roads, buildings" project.imagery = "tms[22]:http://hiu-maps.net/hot/1.0.0/kathmandu_flipped/{zoom}/{x}/{y}.png" # noqa project.license = license1 DBSession.add(project) with project.force_locale('fr'): project.name = "Kathmandu - Cartographier les routes et les bâtiments" # noqa project.auto_fill(17)