def feedConsumer(): """ 'feedConsumer()' will connect and continuously listen to the changes feed for any docs with a 'state' value of 'unsent'. Finding one; it will parse the data into email headers and body, send the email, and update the doc's 'state' value to either 'sent' or 'error'. If the latter, it will also add a errormsg field with the contents of the message for debugging purposes. """ global auth, data, sender, recipients, toaddr, ccs, bccs, fromaddr, toaddrs, headers, msgformat # TODO:FIXME """ change to couchdb:couchdb user:group combination""" auth = Session() auth.name = '' auth.password = '' s1 = Server('http://localhost:5984/', session=auth) db1 = s1['mailspool'] ch = db1.changes(feed='continuous',heartbeat='1000',include_docs=True,filter='spooler/unsent') for line in ch: """ reset all vars to preclude any pollution between lines """ data = '' sender = {} recipients = {} fromaddr = '' toaddrs = [] toaddr = [] ccs = [] bccs = [] headers = '' msgformat = '' """ now grab the document object included in the feed """ data = line['doc'] """ assign some vars to populate the actual email fields """ sender = data['sender'] recipients = data['recipients'] msgformat = data['msgformat'] message = data['message'] subject = data['subject'] """ begin parsing """ parseChange() sendEmail(toaddr, ccs, bccs, message, subject) updateState(data)
def __init__(self, tender_id, worker_defaults={}, auction_data={}): self.tender_id = tender_id self.auction_doc_id = tender_id self._end_auction_event = Event() self.tender_url = urljoin( worker_defaults["resource_api_server"], '/api/{0}/auctions/{1}'.format( worker_defaults["resource_api_version"], tender_id)) if auction_data: self.debug = True LOGGER.setLevel(logging.DEBUG) self._auction_data = auction_data else: self.debug = False self.bids_actions = BoundedSemaphore() self.session = RequestsSession() self.features = {} # bw self.worker_defaults = worker_defaults if self.worker_defaults.get('with_document_service', False): self.session_ds = RequestsSession() self._bids_data = {} self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]), session=Session(retry_delays=range(10))) self.audit = {} self.retries = 10 self.mapping = {} self._bids_data = defaultdict(list) LOGGER.info(self.debug) # auction phases controllers # Configuration for SealedBids phase self.has_critical_error = False if REQUEST_QUEUE_SIZE == -1: self.bids_queue = Queue() else: self.bids_queue = Queue(REQUEST_QUEUE_SIZE) self.bidders_data = []
def __init__(self, tender_id, worker_defaults={}, debug=False): super(Auction, self).__init__() self.tender_id = tender_id self.debug = debug self._end_auction_event = Event() self.bids_actions = BoundedSemaphore() self.worker_defaults = worker_defaults self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]), session=Session(retry_delays=range(10))) self.audit = {} self.retries = 10 self.bidders_count = 0 self.bidders_data = [] self.bids_mapping = {} gsm = getGlobalSiteManager() self.datasource = gsm.queryUtility(IDataSource) self.database = gsm.queryUtility(IDatabase) self.context = gsm.queryUtility(IContext) self.job_service = gsm.queryUtility(IJobService) self.context['end_auction_event'] = self._end_auction_event
def __init__(self, config, re_planning=False, debug=False): super(AuctionsDataBridge, self).__init__() self.config = config self.tenders_ids_list = [] self.tz = tzlocal() self.debug = debug self.mapper = components.qA(self, IAuctionsManager) self.re_planning = re_planning DEFAULT_RETRIEVERS_PARAMS.update( self.config.get('main').get('retrievers_params', {})) self.couch_url = urljoin( self.config_get('couch_url'), self.config_get('auctions_db') ) self.db = Database(self.couch_url, session=Session(retry_delays=range(10))) sync_design(self.db) self.feeder = ResourceFeeder( host=self.config_get('resource_api_server'), resource=self.config_get('resource_name'), version=self.config_get('resource_api_version'), key='', extra_params=API_EXTRA, retrievers_params=DEFAULT_RETRIEVERS_PARAMS )
def __init__(self, config): self._db = Database(str(config["COUCH_DATABASE"]), session=Session(retry_delays=range(10)))
#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright © 2012 blurrcat <*****@*****.**> """ A manage script for couchDB. Functions are: - Create new jobs; - Reset job status; """ from couchdb import Server, Session dbname = 'soup' session = Session() session.username = '******' session.password = '******' server = Server(session=session) db = server[dbname] def job(key, action): return { '_id': action[0]+key, 'key': key, 'type': 'Job', 'action': action, 'status': 'idle', 'next_page': 1, 'max_page': None }
def set_api_security(settings): # CouchDB connection db_name = os.environ.get("DB_NAME", settings["couchdb.db_name"]) server = Server(settings.get("couchdb.url"), session=Session(retry_delays=list(range(10)))) # removing provided credentials from url and create a new connection without any ?? if "couchdb.admin_url" not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server(extract_credentials(settings.get("couchdb.url"))[0]) # admin_url is provided and url contains credentials if "couchdb.admin_url" in settings and server.resource.credentials: # init admin connection aserver = Server(settings.get("couchdb.admin_url"), session=Session(retry_delays=list(range(10)))) # updating _users security for "_users" database ?? # in fact this drops security as "names" are empty at the moment users_db = aserver["_users"] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={"MESSAGE_ID": "update_users_security"}) users_db.security = SECURITY # non admin user credentials from couchdb.url username, password = server.resource.credentials # update non-admin user's password ?? user_doc = users_db.get("org.couchdb.user:{}".format(username), {"_id": "org.couchdb.user:{}".format(username)}) if not user_doc.get("derived_key", "") \ or build_derived_key(password, user_doc) != user_doc.get("derived_key", ""): user_doc.update({"name": username, "roles": [], "type": "user", "password": password}) LOGGER.info("Updating api db main user", extra={"MESSAGE_ID": "update_api_main_user"}) users_db.save(user_doc) # adding non-admin user to SECURITY["members"]["names"] ?? security_users = [username] # updating reader user password and adding it to SECURITY["members"]["names"] if "couchdb.reader_username" in settings and "couchdb.reader_password" in settings: reader_username = settings.get("couchdb.reader_username") reader_password = settings.get("couchdb.reader_password") reader = users_db.get( "org.couchdb.user:{}".format(reader_username), {"_id": "org.couchdb.user:{}".format(reader_username)} ) if not reader.get("derived_key", "") \ or build_derived_key(reader_password, reader) != reader.get("derived_key", ""): reader.update( { "name": reader_username, "roles": ["reader"], "type": "user", "password": reader_password, } ) LOGGER.info("Updating api db reader user", extra={"MESSAGE_ID": "update_api_reader_user"}) users_db.save(reader) security_users.append(reader_username) # ensure database exists if db_name not in aserver: aserver.create(db_name) # updating application database SECURITY db = aserver[db_name] SECURITY["members"]["names"] = security_users if SECURITY != db.security: LOGGER.info("Updating api db security", extra={"MESSAGE_ID": "update_api_security"}) db.security = SECURITY # updating validation document # VALIDATE_DOC_UPDATE: 1) forbids deleting documents with the tenderId field # 2) allows _design document updates for users with _admin role ( # Is this forbidden without this doc? No -_-, It works for `username` as well # 3) allows updates only for `username` (user from couchdb.url), any other user cannot update anything # seems application can perfectly work without this auth_doc = db.get(VALIDATE_DOC_ID, {"_id": VALIDATE_DOC_ID}) if auth_doc.get("validate_doc_update") != VALIDATE_DOC_UPDATE % username: auth_doc["validate_doc_update"] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating api db validate doc", extra={"MESSAGE_ID": "update_api_validate_doc"}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: # ensure database exists # in fact non admin user can't do this if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) aserver = None return aserver, server, db
def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.workers_config = {} self.bridge_id = uuid.uuid4().hex self.api_host = self.config_get('resources_api_server') self.api_version = self.config_get('resources_api_version') self.retrievers_params = self.config_get('retrievers_params') # Check up_wait_sleep up_wait_sleep = self.retrievers_params.get('up_wait_sleep') if up_wait_sleep is not None and up_wait_sleep < 30: raise DataBridgeConfigError('Invalid \'up_wait_sleep\' in ' '\'retrievers_params\'. Value must be ' 'grater than 30.') # Workers settings for key in WORKER_CONFIG: self.workers_config[key] = (self.config_get(key) or WORKER_CONFIG[key]) # Init config for key in DEFAULTS: setattr(self, key, self.config_get(key) or DEFAULTS[key]) # Pools self.workers_pool = gevent.pool.Pool(self.workers_max) self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max) self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count) # Queues if self.input_queue_size == -1: self.input_queue = Queue() else: self.input_queue = Queue(self.input_queue_size) if self.resource_items_queue_size == -1: self.resource_items_queue = Queue() else: self.resource_items_queue = Queue(self.resource_items_queue_size) self.api_clients_queue = Queue() # self.retry_api_clients_queue = Queue() if self.retry_resource_items_queue_size == -1: self.retry_resource_items_queue = Queue() else: self.retry_resource_items_queue = Queue( self.retry_resource_items_queue_size) self.process = psutil.Process(os.getpid()) if self.api_host != '' and self.api_host is not None: api_host = urlparse(self.api_host) if api_host.scheme == '' and api_host.netloc == '': raise DataBridgeConfigError( 'Invalid \'tenders_api_server\' url.') else: raise DataBridgeConfigError('In config dictionary empty or missing' ' \'tenders_api_server\'') self.db = prepare_couchdb(self.couch_url, self.db_name, logger) db_url = self.couch_url + '/' + self.db_name prepare_couchdb_views(db_url, self.workers_config['resource'], logger) self.server = Server(self.couch_url, session=Session(retry_delays=range(10))) self.view_path = '_design/{}/_view/by_dateModified'.format( self.workers_config['resource']) extra_params = { 'mode': self.retrieve_mode, 'limit': self.resource_items_limit } self.feeder = ResourceFeeder(host=self.api_host, version=self.api_version, key='', resource=self.workers_config['resource'], extra_params=extra_params, retrievers_params=self.retrievers_params, adaptive=True) self.api_clients_info = {}
#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright © 2012 blurrcat <*****@*****.**> """ A manage script for couchDB. Functions are: - Create new jobs; - Reset job status; """ from couchdb import Server, Session dbname = 'soup' session = Session() session.username = '******' session.password = '******' server = Server(session=session) db = server[dbname] def job(key, action): return { '_id': action[0] + key, 'key': key, 'type': 'Job', 'action': action, 'status': 'idle', 'next_page': 1, 'max_page': None }
def set_chronograph_security(settings): db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(60))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server(extract_credentials( settings.get('couchdb.url'))[0], session=Session(retry_delays=range(60))) if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info('Updating users db security', extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( 'org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if (not user_doc.get('derived_key', '') or PBKDF2(password, user_doc.get('salt', ''), user_doc.get('iterations', 10)).hexread( int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', '')): user_doc.update({ 'name': username, 'roles': [], 'type': 'user', 'password': password }) LOGGER.info('Updating chronograph db main user', extra={'MESSAGE_ID': 'update_chronograph_main_user'}) users_db.save(user_doc) security_users = [username, ] if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY['members']['names'] = security_users if SECURITY != db.security: LOGGER.info('Updating chronograph db security', extra={'MESSAGE_ID': 'update_chronograph_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if auth_doc.get('validate_doc_update') != VALIDATE_DOC_UPDATE % username: auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info('Updating chronograph db validate doc', extra={'MESSAGE_ID': 'update_chronograph_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) return server, db
def set_api_security(settings): # CouchDB connection db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(10))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server( extract_credentials(settings.get('couchdb.url'))[0]) if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( 'org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if (not user_doc.get('derived_key', '') or PBKDF2(password, user_doc.get('salt', ''), user_doc.get('iterations', 10)).hexread( int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', '')): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating api db main user", extra={'MESSAGE_ID': 'update_api_main_user'}) users_db.save(user_doc) security_users = [ username, ] if ('couchdb.reader_username' in settings and 'couchdb.reader_password' in settings): reader_username = settings.get('couchdb.reader_username') reader = users_db.get( 'org.couchdb.user:{}'.format(reader_username), {'_id': 'org.couchdb.user:{}'.format(reader_username)}) if (not reader.get('derived_key', '') or PBKDF2(settings.get('couchdb.reader_password'), reader.get('salt', ''), reader.get('iterations', 10)).hexread( int(len(reader.get('derived_key', '')) / 2)) != reader.get('derived_key', '')): reader.update({ "name": reader_username, "roles": ['reader'], "type": "user", "password": settings.get('couchdb.reader_password') }) LOGGER.info("Updating api db reader user", extra={'MESSAGE_ID': 'update_api_reader_user'}) users_db.save(reader) security_users.append(reader_username) if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u'members'][u'names'] = security_users if SECURITY != db.security: LOGGER.info("Updating api db security", extra={'MESSAGE_ID': 'update_api_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if (auth_doc.get('validate_doc_update') != VALIDATE_DOC_UPDATE % username): auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating api db validate doc", extra={'MESSAGE_ID': 'update_api_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) aserver = None return aserver, server, db
from couchdb import Server, Session from io import BytesIO auth = Session() auth.name = "rohan" auth.password = "******" couchserver = Server('http://localhost:5984/', session=auth) db=couchserver['sync'] ''' For creating a document from uuid import uuid4 doc_id = uuid4().hex db[doc_id] = {'type': 'person', 'name': 'John Doe'} ''' sync_folders = ["DroidA","DroidB"] ''' Make this a class ''' def store_file(f,folder): file_name=folder+"/"+f.filename doc={'name':file_name} # f=open(file_name,'r') for doc_id in db: document=db.get(doc_id,attachments=True) doc_name=document['name'] if doc_name == file_name: db.delete_attachment(document,filename=file_name) db.delete(document)
def make_auctions_app(global_conf, redis_url='redis://localhost:7777/0', external_couch_url='http://localhost:5000/auction', internal_couch_url='http://localhost:9000/', proxy_internal_couch_url='http://localhost:9000/', auctions_db='auctions', hash_secret_key='', timezone='Europe/Kiev', preferred_url_scheme='http', debug=False, auto_build=False, event_source_connection_limit=1000): """ [app:main] use = egg:openprocurement.auction#auctions_server redis_url = redis://:passwod@localhost:1111/0 external_couch_url = http://localhost:1111/auction internal_couch_url = http://localhost:9011/ auctions_db = auction timezone = Europe/Kiev """ auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection, max_size=20, backend="gevent") auctions_server.proxy_mappings = Memoizer({}) auctions_server.event_sources_pool = deque([]) auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme auctions_server.config['REDIS_URL'] = redis_url auctions_server.config['event_source_connection_limit'] = int( event_source_connection_limit) auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url, auctions_db) auctions_server.add_url_rule('/' + auctions_db + '/<path:path>', 'couch_server_proxy', couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '/', 'couch_server_proxy', couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '_secured/', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.config['INT_COUCH_URL'] = internal_couch_url auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url auctions_server.config['COUCH_DB'] = auctions_db auctions_server.config['TIMEZONE'] = tz(timezone) auctions_server.redis = Redis(auctions_server) auctions_server.couch_server = Server( auctions_server.config.get('INT_COUCH_URL'), session=Session(retry_delays=range(10))) if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server: auctions_server.couch_server.create(auctions_server.config['COUCH_DB']) auctions_server.db = auctions_server.couch_server[ auctions_server.config['COUCH_DB']] auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key sync_design(auctions_server.db) auctions_server.config['ASSETS_DEBUG'] = True if debug else False assets.auto_build = True if auto_build else False return auctions_server
def make_auctions_app(global_conf, redis_url='redis://localhost:9002/1', redis_password='', redis_database='', sentinel_cluster_name='', sentinels='', external_couch_url='http://localhost:5000/auction', internal_couch_url='http://localhost:9000/', proxy_internal_couch_url='http://localhost:9000/', auctions_db='database', hash_secret_key='', timezone='Europe/Kiev', preferred_url_scheme='http', debug=False, auto_build=False, event_source_connection_limit=1000, limit_replications_progress=99, limit_replications_func='any'): """ [app:main] use = egg:openprocurement.auction#auctions_server redis_url = redis://:passwod@localhost:1111/0 external_couch_url = http://localhost:1111/auction internal_couch_url = http://localhost:9011/ auctions_db = auction timezone = Europe/Kiev """ auctions_server = components.queryUtility(IAuctionsServer) auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection, max_size=20, backend='gevent') auctions_server.proxy_mappings = Memoizer({}) auctions_server.event_sources_pool = deque([]) auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme auctions_server.config['limit_replications_progress'] = float( limit_replications_progress) auctions_server.config['limit_replications_func'] = limit_replications_func auctions_server.config['REDIS'] = { 'redis': redis_url, 'redis_password': redis_password, 'redis_database': redis_database, 'sentinel_cluster_name': sentinel_cluster_name, 'sentinel': loads(sentinels) } auctions_server.config['event_source_connection_limit'] = int( event_source_connection_limit) auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url, auctions_db) auctions_server.add_url_rule('/' + auctions_db + '/<path:path>', 'couch_server_proxy', couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '/', 'couch_server_proxy', couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '_secured/', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.config['INT_COUCH_URL'] = internal_couch_url auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url auctions_server.config['COUCH_DB'] = auctions_db auctions_server.config['TIMEZONE'] = tz(timezone) auctions_server.couch_server = Server( auctions_server.config.get('INT_COUCH_URL'), session=Session(retry_delays=range(10))) if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server: auctions_server.couch_server.create(auctions_server.config['COUCH_DB']) auctions_server.db = auctions_server.\ couch_server[auctions_server.config['COUCH_DB']] auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key sync_design(auctions_server.db) for entry_point in iter_entry_points(PKG_NAMESPACE): plugin = entry_point.load() plugin(components) return auctions_server
def open_db(self): self.server = Server(self.db_url, session=Session(retry_delays=range(10))) self.db = self.server[self.db_name]
def test_save_tender_in_db(self): log_string = io.BytesIO() stream_handler = logging.StreamHandler(log_string) logger.addHandler(stream_handler) bridge = EdgeDataBridge(self.config) mock_tender = {'data': test_tender_data} bridge.client.get_tender = MagicMock(return_value=mock_tender) # Save tender tid = uuid.uuid4().hex t_date_modified = datetime.datetime.utcnow().isoformat() mock_tender['data']['dateModified'] = t_date_modified bridge.save_tender_in_db(tid, t_date_modified) x = log_string.getvalue().split('\n') self.assertEqual(x[1].strip(), 'Save tender ' + tid) tender_in_db = bridge.db.get(tid) self.assertEqual(tender_in_db.id, tid) # Tender exist in db and not modified result = bridge.save_tender_in_db(tid, t_date_modified) self.assertEqual(result, None) # Update tender t_date_modified = datetime.datetime.utcnow().isoformat() mock_tender['data']['dateModified'] = t_date_modified bridge.save_tender_in_db(tid, t_date_modified) x = log_string.getvalue().split('\n') self.assertEqual(x[2].strip(), 'Update tender ' + tid) updated_tender = bridge.db.get(tid) self.assertEqual(updated_tender['dateModified'], unicode(t_date_modified)) # Tender not found bridge.client.get_tender = MagicMock(return_value=test_tender_data) bridge.save_tender_in_db(tid, datetime.datetime.utcnow().isoformat()) x = log_string.getvalue().split('\n') self.assertEqual(x[3].strip(), 'Tender ' + tid + ' not found') bridge.db.delete(updated_tender) # Saving tender with exception bridge.client.get_tender = MagicMock(return_value=mock_tender) bridge.config['main']['couch_url'] = '' bridge.config['main']['public_db'] = '' bridge.db = Database('bridge.couch_url', session=Session(retry_delays=range(10))) new_mock_tender = mock_tender new_mock_tender['dateModified'] = datetime.datetime.utcnow().isoformat( ) new_mock_tender['_rev'] = '2-' + uuid.uuid4().hex bridge.db.get = MagicMock(return_value=new_mock_tender) tid = uuid.uuid4().hex bridge.save_tender_in_db(tid, datetime.datetime.utcnow().isoformat()) x = log_string.getvalue().split('\n') self.assertEqual( x[5].strip(), 'Saving tender ' + tid + ' fail with error (400, (u\'illegal_database_name\', u"Name: \'bridge.couch_url\'. Only lowercase characters (a-z), digits (0-9), and any of the characters _, $, (, ), +, -, and / are allowed. Must begin with a letter."))' ) logger.removeHandler(stream_handler) log_string.close()
def set_api_security(settings): # CouchDB connection db_name = os.environ.get("DB_NAME", settings["couchdb.db_name"]) server = Server(settings.get("couchdb.url"), session=Session(retry_delays=range(10))) if "couchdb.admin_url" not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server( extract_credentials(settings.get("couchdb.url"))[0]) if "couchdb.admin_url" in settings and server.resource.credentials: aserver = Server(settings.get("couchdb.admin_url"), session=Session(retry_delays=range(10))) users_db = aserver["_users"] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={"MESSAGE_ID": "update_users_security"}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( "org.couchdb.user:{}".format(username), {"_id": "org.couchdb.user:{}".format(username)}) if not user_doc.get( "derived_key", "") or PBKDF2(password, user_doc.get( "salt", ""), user_doc.get("iterations", 10)).hexread( int(len(user_doc.get("derived_key", "")) / 2)) != user_doc.get("derived_key", ""): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating api db main user", extra={"MESSAGE_ID": "update_api_main_user"}) users_db.save(user_doc) security_users = [username] if "couchdb.reader_username" in settings and "couchdb.reader_password" in settings: reader_username = settings.get("couchdb.reader_username") reader = users_db.get( "org.couchdb.user:{}".format(reader_username), {"_id": "org.couchdb.user:{}".format(reader_username)}) if not reader.get("derived_key", "") or PBKDF2( settings.get("couchdb.reader_password"), reader.get("salt", ""), reader.get( "iterations", 10)).hexread( int(len(reader.get("derived_key", "")) / 2)) != reader.get("derived_key", ""): reader.update({ "name": reader_username, "roles": ["reader"], "type": "user", "password": settings.get("couchdb.reader_password"), }) LOGGER.info("Updating api db reader user", extra={"MESSAGE_ID": "update_api_reader_user"}) users_db.save(reader) security_users.append(reader_username) if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u"members"][u"names"] = security_users if SECURITY != db.security: LOGGER.info("Updating api db security", extra={"MESSAGE_ID": "update_api_security"}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {"_id": VALIDATE_DOC_ID}) if auth_doc.get( "validate_doc_update") != VALIDATE_DOC_UPDATE % username: auth_doc["validate_doc_update"] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating api db validate doc", extra={"MESSAGE_ID": "update_api_validate_doc"}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) aserver = None return aserver, server, db
def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ config = Configurator(settings=settings) config.add_subscriber(add_logging_context, ContextFound) config.include('pyramid_exclog') config.add_route('home', '/') config.add_route('resync_all', '/resync_all') config.add_route('resync_back', '/resync_back') config.add_route('resync', '/resync/{auction_id}') config.add_route('recheck', '/recheck/{auction_id}') config.add_route('calendar', '/calendar') config.add_route('calendar_entry', '/calendar/{date}') config.add_route('streams', '/streams') config.scan(ignore='openprocurement.chronograph.tests') config.add_subscriber(start_scheduler, ApplicationCreated) config.registry.api_token = os.environ.get('API_TOKEN', settings.get('api.token')) db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(60))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server(extract_credentials( settings.get('couchdb.url'))[0], session=Session(retry_delays=range(60))) config.registry.couchdb_server = server if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( 'org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if not user_doc.get( 'derived_key', '') or PBKDF2(password, user_doc.get( 'salt', ''), user_doc.get('iterations', 10)).hexread( int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', ''): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating chronograph db main user", extra={'MESSAGE_ID': 'update_chronograph_main_user'}) users_db.save(user_doc) security_users = [ username, ] if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u'members'][u'names'] = security_users if SECURITY != db.security: LOGGER.info("Updating chronograph db security", extra={'MESSAGE_ID': 'update_chronograph_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if auth_doc.get( 'validate_doc_update') != VALIDATE_DOC_UPDATE % username: auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info( "Updating chronograph db validate doc", extra={'MESSAGE_ID': 'update_chronograph_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) config.registry.db = db jobstores = { #'default': CouchDBJobStore(database=db_name, client=server) } #executors = { #'default': ThreadPoolExecutor(5), #'processpool': ProcessPoolExecutor(5) #} job_defaults = {'coalesce': False, 'max_instances': 3} config.registry.api_url = settings.get('api.url') config.registry.callback_url = settings.get('callback.url') scheduler = Scheduler( jobstores=jobstores, #executors=executors, job_defaults=job_defaults, timezone=TZ) if 'jobstore_db' in settings: scheduler.add_jobstore('sqlalchemy', url=settings['jobstore_db']) config.registry.scheduler = scheduler # scheduler.remove_all_jobs() # scheduler.start() resync_all_job = scheduler.get_job('resync_all') now = datetime.now(TZ) if not resync_all_job or resync_all_job.next_run_time < now - timedelta( hours=1): if resync_all_job: args = resync_all_job.args else: args = [settings.get('callback.url') + 'resync_all', None] run_date = now + timedelta(seconds=60) scheduler.add_job(push, 'date', run_date=run_date, timezone=TZ, id='resync_all', args=args, replace_existing=True, misfire_grace_time=60 * 60) return config.make_wsgi_app()
def main(global_config, **settings): config = Configurator( autocommit=True, settings=settings, authentication_policy=AuthenticationPolicy(settings['auth.file'], __name__), authorization_policy=AuthorizationPolicy(), route_prefix=ROUTE_PREFIX, ) config.include('pyramid_exclog') config.include("cornice") config.add_forbidden_view(forbidden) config.add_request_method(request_params, 'params', reify=True) config.add_request_method(authenticated_role, reify=True) config.add_request_method(extract_tender, 'tender', reify=True) config.add_renderer('prettyjson', JSON(indent=4)) config.add_renderer('jsonp', JSONP(param_name='opt_jsonp')) config.add_renderer('prettyjsonp', JSONP(indent=4, param_name='opt_jsonp')) config.add_subscriber(add_logging_context, NewRequest) config.add_subscriber(set_logging_context, ContextFound) config.add_subscriber(set_renderer, NewRequest) config.add_subscriber(beforerender, BeforeRender) config.scan("openprocurement.api.views.spore") # tender procurementMethodType plugins support config.add_route_predicate('procurementMethodType', isTender) config.registry.tender_procurementMethodTypes = {} config.add_request_method(tender_from_data) config.add_directive('add_tender_procurementMethodType', register_tender_procurementMethodType) # search for plugins plugins = settings.get('plugins') and settings['plugins'].split(',') for entry_point in iter_entry_points('openprocurement.api.plugins'): if not plugins or entry_point.name in plugins: plugin = entry_point.load() plugin(config) # CouchDB connection db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(10))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server( extract_credentials(settings.get('couchdb.url'))[0]) config.registry.couchdb_server = server if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( 'org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if not user_doc.get( 'derived_key', '') or PBKDF2(password, user_doc.get( 'salt', ''), user_doc.get('iterations', 10)).hexread( int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', ''): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating api db main user", extra={'MESSAGE_ID': 'update_api_main_user'}) users_db.save(user_doc) security_users = [ username, ] if 'couchdb.reader_username' in settings and 'couchdb.reader_password' in settings: reader_username = settings.get('couchdb.reader_username') reader = users_db.get( 'org.couchdb.user:{}'.format(reader_username), {'_id': 'org.couchdb.user:{}'.format(reader_username)}) if not reader.get('derived_key', '') or PBKDF2( settings.get('couchdb.reader_password'), reader.get('salt', ''), reader.get( 'iterations', 10)).hexread( int(len(reader.get('derived_key', '')) / 2)) != reader.get('derived_key', ''): reader.update({ "name": reader_username, "roles": ['reader'], "type": "user", "password": settings.get('couchdb.reader_password') }) LOGGER.info("Updating api db reader user", extra={'MESSAGE_ID': 'update_api_reader_user'}) users_db.save(reader) security_users.append(reader_username) if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u'members'][u'names'] = security_users if SECURITY != db.security: LOGGER.info("Updating api db security", extra={'MESSAGE_ID': 'update_api_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if auth_doc.get( 'validate_doc_update') != VALIDATE_DOC_UPDATE % username: auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating api db validate doc", extra={'MESSAGE_ID': 'update_api_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) config.registry.db = db # migrate data migrate_data(config.registry.db) # S3 connection if 'aws.access_key' in settings and 'aws.secret_key' in settings and 'aws.s3_bucket' in settings: connection = S3Connection(settings['aws.access_key'], settings['aws.secret_key']) config.registry.s3_connection = connection bucket_name = settings['aws.s3_bucket'] if bucket_name not in [b.name for b in connection.get_all_buckets()]: connection.create_bucket(bucket_name, location=Location.EU) config.registry.bucket_name = bucket_name config.registry.server_id = settings.get('id', '') return config.make_wsgi_app()
def main(global_config, **settings): config = Configurator( autocommit=True, settings=settings, authentication_policy=AuthenticationPolicy(settings['auth.file'], __name__), authorization_policy=AuthorizationPolicy(), route_prefix=route_prefix(settings), ) config.include('pyramid_exclog') config.include("cornice") config.add_forbidden_view(forbidden) config.add_request_method(request_params, 'params', reify=True) config.add_request_method(authenticated_role, reify=True) config.add_request_method(extract_tender, 'tender', reify=True) config.add_request_method(check_accreditation) config.add_request_method(json_body, 'json_body', reify=True) config.add_renderer('json', JSON(serializer=simplejson.dumps)) config.add_renderer('prettyjson', JSON(indent=4, serializer=simplejson.dumps)) config.add_renderer('jsonp', JSONP(param_name='opt_jsonp', serializer=simplejson.dumps)) config.add_renderer('prettyjsonp', JSONP(indent=4, param_name='opt_jsonp', serializer=simplejson.dumps)) config.add_subscriber(add_logging_context, NewRequest) config.add_subscriber(set_logging_context, ContextFound) config.add_subscriber(set_renderer, NewRequest) config.add_subscriber(beforerender, BeforeRender) config.scan("openprocurement.api.views.spore") config.scan("openprocurement.api.views.health") # tender procurementMethodType plugins support config.add_route_predicate('procurementMethodType', isTender) config.registry.tender_procurementMethodTypes = {} config.add_request_method(tender_from_data) config.add_directive('add_tender_procurementMethodType', register_tender_procurementMethodType) # search for plugins plugins = settings.get('plugins') and settings['plugins'].split(',') for entry_point in iter_entry_points('openprocurement.api.plugins'): if not plugins or entry_point.name in plugins: plugin = entry_point.load() plugin(config) # CouchDB connection db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(10))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server(extract_credentials(settings.get('couchdb.url'))[0]) config.registry.couchdb_server = server if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) config.registry.admin_couchdb_server = aserver users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get('org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if not user_doc.get('derived_key', '') or PBKDF2(password, user_doc.get('salt', ''), user_doc.get('iterations', 10)).hexread(int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', ''): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating api db main user", extra={'MESSAGE_ID': 'update_api_main_user'}) users_db.save(user_doc) security_users = [username, ] if 'couchdb.reader_username' in settings and 'couchdb.reader_password' in settings: reader_username = settings.get('couchdb.reader_username') reader = users_db.get('org.couchdb.user:{}'.format(reader_username), {'_id': 'org.couchdb.user:{}'.format(reader_username)}) if not reader.get('derived_key', '') or PBKDF2(settings.get('couchdb.reader_password'), reader.get('salt', ''), reader.get('iterations', 10)).hexread(int(len(reader.get('derived_key', '')) / 2)) != reader.get('derived_key', ''): reader.update({ "name": reader_username, "roles": ['reader'], "type": "user", "password": settings.get('couchdb.reader_password') }) LOGGER.info("Updating api db reader user", extra={'MESSAGE_ID': 'update_api_reader_user'}) users_db.save(reader) security_users.append(reader_username) if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u'members'][u'names'] = security_users if SECURITY != db.security: LOGGER.info("Updating api db security", extra={'MESSAGE_ID': 'update_api_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if auth_doc.get('validate_doc_update') != VALIDATE_DOC_UPDATE % username: auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating api db validate doc", extra={'MESSAGE_ID': 'update_api_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) config.registry.db = db # readjust couchdb json decoder couchdb_json_decode() # Document Service key config.registry.docservice_url = settings.get('docservice_url') config.registry.docservice_username = settings.get('docservice_username') config.registry.docservice_password = settings.get('docservice_password') config.registry.docservice_upload_url = settings.get('docservice_upload_url') config.registry.docservice_key = dockey = Signer(settings.get('dockey', '').decode('hex')) config.registry.keyring = keyring = {} dockeys = settings.get('dockeys') if 'dockeys' in settings else dockey.hex_vk() for key in dockeys.split('\0'): keyring[key[:8]] = Verifier(key) # migrate data if not os.environ.get('MIGRATION_SKIP'): for entry_point in iter_entry_points('openprocurement.api.migrations'): plugin = entry_point.load() plugin(config.registry) config.registry.server_id = settings.get('id', '') config.registry.health_threshold = float(settings.get('health_threshold', 99)) config.registry.update_after = asbool(settings.get('update_after', True)) return config.make_wsgi_app()
def main(global_config, **settings): config = Configurator( autocommit=True, settings=settings, authentication_policy=AuthenticationPolicy(settings['auth.file'], __name__), authorization_policy=AuthorizationPolicy(), route_prefix=route_prefix(settings), ) config.include('pyramid_exclog') config.include("cornice") config.add_forbidden_view(forbidden) config.add_request_method(request_params, 'params', reify=True) config.add_request_method(authenticated_role, reify=True) config.add_request_method(extract_tender, 'tender', reify=True) config.add_request_method(check_accreditation) config.add_renderer('prettyjson', JSON(indent=4)) config.add_renderer('jsonp', JSONP(param_name='opt_jsonp')) config.add_renderer('prettyjsonp', JSONP(indent=4, param_name='opt_jsonp')) config.add_subscriber(add_logging_context, NewRequest) config.add_subscriber(set_logging_context, ContextFound) config.add_subscriber(set_renderer, NewRequest) config.add_subscriber(beforerender, BeforeRender) config.scan("openprocurement.edge.views.spore") config.scan("openprocurement.edge.views.health") config.scan("openprocurement.edge.views.tenders") if auctions_core: config.add_request_method(extract_auction, 'auction', reify=True) config.scan("openprocurement.edge.views.auctions") add_auction_design() if contracting: config.add_request_method(extract_contract, 'contract', reify=True) config.scan("openprocurement.edge.views.contracts") add_contract_design() if planning: config.add_request_method(extract_plan, 'plan', reify=True) config.scan("openprocurement.edge.views.plans") add_plan_design() # CouchDB connection db_name = os.environ.get('DB_NAME', settings['couchdb.db_name']) server = Server(settings.get('couchdb.url'), session=Session(retry_delays=range(10))) if 'couchdb.admin_url' not in settings and server.resource.credentials: try: server.version() except Unauthorized: server = Server( extract_credentials(settings.get('couchdb.url'))[0]) config.registry.couchdb_server = server if 'couchdb.admin_url' in settings and server.resource.credentials: aserver = Server(settings.get('couchdb.admin_url'), session=Session(retry_delays=range(10))) config.registry.admin_couchdb_server = aserver users_db = aserver['_users'] if SECURITY != users_db.security: LOGGER.info("Updating users db security", extra={'MESSAGE_ID': 'update_users_security'}) users_db.security = SECURITY username, password = server.resource.credentials user_doc = users_db.get( 'org.couchdb.user:{}'.format(username), {'_id': 'org.couchdb.user:{}'.format(username)}) if not user_doc.get( 'derived_key', '') or PBKDF2(password, user_doc.get( 'salt', ''), user_doc.get('iterations', 10)).hexread( int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', ''): user_doc.update({ "name": username, "roles": [], "type": "user", "password": password }) LOGGER.info("Updating edge db main user", extra={'MESSAGE_ID': 'update_edge_main_user'}) users_db.save(user_doc) security_users = [ username, ] if 'couchdb.reader_username' in settings and 'couchdb.reader_password' in settings: reader_username = settings.get('couchdb.reader_username') reader = users_db.get( 'org.couchdb.user:{}'.format(reader_username), {'_id': 'org.couchdb.user:{}'.format(reader_username)}) if not reader.get('derived_key', '') or PBKDF2( settings.get('couchdb.reader_password'), reader.get('salt', ''), reader.get( 'iterations', 10)).hexread( int(len(reader.get('derived_key', '')) / 2)) != reader.get('derived_key', ''): reader.update({ "name": reader_username, "roles": ['reader'], "type": "user", "password": settings.get('couchdb.reader_password') }) LOGGER.info("Updating edge db reader user", extra={'MESSAGE_ID': 'update_edge_reader_user'}) users_db.save(reader) security_users.append(reader_username) if db_name not in aserver: aserver.create(db_name) db = aserver[db_name] SECURITY[u'members'][u'names'] = security_users if SECURITY != db.security: LOGGER.info("Updating edge db security", extra={'MESSAGE_ID': 'update_edge_security'}) db.security = SECURITY auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID}) if auth_doc.get( 'validate_doc_update') != VALIDATE_DOC_UPDATE % username: auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username LOGGER.info("Updating edge db validate doc", extra={'MESSAGE_ID': 'update_edge_validate_doc'}) db.save(auth_doc) # sync couchdb views sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views sync_design(db) config.registry.db = db config.registry.server_id = settings.get('id', '') config.registry.health_threshold = float( settings.get('health_threshold', 99)) config.registry.update_after = asbool(settings.get('update_after', True)) return config.make_wsgi_app()