def __init__(self, url):
print "iniciando a verificação..."
Server.__init__(self, url)
act = self
test_and_create = lambda db: db in act and act[db] or act.create(db)
for attribute in _DOCBASES:
setattr(Activ, attribute, test_and_create(attribute))
def __init__(self, url=None, db='crafts'):
if url is None:
s = Server()
else:
s = Server(url)
self.collection = MetricCollection(s['crafts'], 'arts')
def __init__(self, mountpoint, uri=None, *args, **kw):
fuse.Fuse.__init__(self, *args, **kw)
self.fuse_args.mountpoint = mountpoint
if uri is not None:
self.server = Server(uri)
else:
self.server = Server()
def __init__(self, url):
print "iniciando a pesquisa..."
Server.__init__(self, url)
act = self
test_and_create = lambda doc: doc in act and act[doc] or act.create(doc)
for attribute in _DOCBASES:
setattr(Activ, attribute, test_and_create(attribute))
def loadCouchDB(db_name, view_name, excel_name):
tweet_dict = {}
couch_server = Server("http://115.146.94.12:5984")
couch_server.resource.credentials = ('admin', 'admin')
couch_server.config()
db = couch_server[db_name]
wb = Workbook()
ws = wb.active
ws.title = "range names"
rowid = 1
for row in db.view(view_name):
coordinate = re.sub(r"\[|\]", "", str(row.key))
# write coordinate
col = get_column_letter(1)
ws.cell('%s%s'%(col, rowid)).value = coordinate
#write polarity
col = get_column_letter(2)
ws.cell('%s%s'%(col, rowid)).value = getPolarity(row.value)
#write text
col = get_column_letter(3)
ws.cell('%s%s'%(col, rowid)).value = row.value
rowid += 1
ws = wb.create_sheet()
ws.title = 'coordinate'
wb.save(filename = excel_name)
return tweet_dict
def __init__(self, url=URL):
Server.__init__(self, url)
act = self
test_and_create = lambda doc: doc in act and act[doc] or act.create(doc
)
for attribute in _DOCBASES:
setattr(Activ, attribute, test_and_create(attribute))
def __init__(self,
user='******',
password='******',
url='http://127.0.0.1:5984',
db_name='py2store',
key_fields=('_id', ),
couchdb_client_kwargs=None):
if couchdb_client_kwargs is None:
couchdb_client_kwargs = {}
if user and password:
# put credentials in url if provided like https://username:[email protected]:5984/
if '//' in url: # if scheme present
url = f'{url.split("//")[0]}//{user}:{password}@{url.split("//")[1]}'
else:
url = f'http//{user}:{password}@{url}'
self._couchdb_server = Server(url=url, **couchdb_client_kwargs)
self._db_name = db_name
# if db not created
if db_name not in self._couchdb_server:
self._couchdb_server.create(db_name)
self._cdb = self._couchdb_server[db_name]
if isinstance(key_fields, str):
key_fields = (key_fields, )
self._key_fields = key_fields
def setUp(self):
user = self.config['main']['couch_url'].get('user', '')
password = self.config['main']['couch_url'].get('password', '')
if (user and password):
self.couch_url = "http://{user}:{password}@{host}:{port}".format(
**self.config['main']['couch_url'])
else:
self.couch_url = "http://{host}:{port}".format(
**self.config['main']['couch_url'])
server = Server(self.couch_url)
if self.config['main']['db_name'] in server:
self.db = server[self.config['main']['db_name']]
else:
self.db = server.create(self.config['main']['db_name'])
array_path = os.path.dirname(os.path.abspath(__file__)).split('/')
app_path = ""
for p in array_path[:-1]:
app_path += p + '/'
app_path += 'couch_views'
# for resource in ('/lots'):
# push_views(couchapp_path=app_path + resource,
# couch_url=couchdb_url)
validate_doc = {
'_id': VALIDATE_BULK_DOCS_ID,
'validate_doc_update': VALIDATE_BULK_DOCS_UPDATE
}
try:
self.db.save(validate_doc)
except Exception:
pass
def test_start(self):
service = self._start_service()
couch = Server("http://%s:%d" % (service.host, service.port))
database = couch.create("test-database")
posted = {"this": "value"}
identifier, revision = database.save(posted)
self.assertEqual(posted, database[identifier])
def database_connection():
try:
server = Server('http://localhost:5984/')
server.version()
return server
except Exception:
print('Cannot connect to the database')
def run(raw_url, db_name, config_doc):
db = Server(raw_url)[db_name]
config = db.get(config_doc)
if 'logger' in config:
log_stream = db.get_attachment(config_doc, config['logger'])
logging_conf = tempfile.NamedTemporaryFile(delete=False)
logging_conf.write(log_stream.read())
logging_conf.close()
logging.config.fileConfig(logging_conf.name)
os.unlink(logging_conf.name)
def get_component(name):
components = name.split('.')
mod = __import__('.'.join(components[:-1]))
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
mal_cls = get_component(config['maldriver'])
predictor_cls = get_component(config['predictor'])
planner_cls = get_component(config['planner'])
scaler_cls = get_component(config['scaler'])
with daemon.DaemonContext():
pass
def main(bulk_size, up_to, num_threads):
global timer
s = Server('http://localhost:5984')
if 'test' in s:
del s['test']
db = s.create('test')
stats_file = 'bulk-perf-%s-%s-%s.dat' % (bulk_size, up_to, num_threads)
title_file = stats_file + '.meta'
f = open(title_file, 'w')
f.write('Bulk size: %s, num threads: %s' % (bulk_size, num_threads))
f.close()
stats_file = open(stats_file, 'w')
stats_lock = Lock()
exit_event = Event()
chunks = Queue.Queue()
def process_chunks():
global count, timer, internal_counter
s = Server('http://localhost:5984')
db = s['test']
while not exit_event.isSet():
try:
chunk = list(chunks.get(timeout=5))
chunks.task_done()
db.update(chunk)
stats_lock.acquire()
try:
count += bulk_size
internal_counter += bulk_size
if internal_counter >= max(num_threads * bulk_size,
up_to / 1000):
end = time()
stats_file.write(
'%s %s\n' %
(count, internal_counter / float(end - timer)))
stats_file.flush()
timer = end
internal_counter = 0
print '%.1f%%' % (float(count) / up_to * 100)
finally:
stats_lock.release()
except Queue.Empty:
pass
except Exception, e:
print 'Exception: %r' % (e, )
chunks.put(chunk)
sleep(1)
def main(global_config, **settings):
config = Configurator(settings=settings)
config.set_authentication_policy(AuthenticationPolicy(None))
config.set_authorization_policy(AuthorizationPolicy())
config.add_renderer('prettyjson', JSON(indent=4))
config.add_renderer('jsonp', JSONP(param_name='opt_jsonp'))
config.add_renderer('prettyjsonp', JSONP(indent=4, param_name='opt_jsonp'))
config.add_subscriber(set_renderer, NewRequest)
config.include("cornice")
config.route_prefix = '/api/{}'.format(VERSION)
config.scan("openprocurement.api.views")
# CouchDB connection
server = Server(settings.get('couchdb.url'))
config.registry.couchdb_server = server
db_name = os.environ.get('DB_NAME', settings['couchdb.db_name'])
if db_name not in server:
server.create(db_name)
config.registry.db = server[db_name]
# sync couchdb views
sync_design(config.registry.db)
# migrate data
migrate_data(config.registry.db)
# S3 connection
if 'aws.access_key' in settings and 'aws.secret_key' in settings and 'aws.bucket' in settings:
connection = S3Connection(settings['aws.access_key'], settings['aws.secret_key'])
config.registry.s3_connection = connection
bucket_name = settings['aws.bucket']
if bucket_name not in [b.name for b in connection.get_all_buckets()]:
connection.create_bucket(bucket_name, location=Location.EU)
config.registry.bucket_name = bucket_name
return config.make_wsgi_app()
def init_db():
server = CouchDBServer(db_addr)
try:
server.create(db_name)
except PreconditionFailed:
del server[db_name]
server.create(db_name)
def create_test_users_db(name=TEST_USERS_DB, server=TEST_SERVER):
try:
couchdb = Server(server)
return couchdb.create(name)
except PreconditionFailed:
del couchdb[name]
return couchdb.create(name)
def __init__(self, convoy_conf):
LOGGER.info('Init Convoy...')
self.convoy_conf = convoy_conf
self.stop_transmitting = False
self.transmitter_timeout = self.convoy_conf.get(
'transmitter_timeout', 10)
self.documents_transfer_queue = Queue()
self.timeout = self.convoy_conf.get('timeout', 10)
self.api_client = APIClient(**self.convoy_conf['cdb'])
self.lots_client = LotsClient(**self.convoy_conf['lots_db'])
self.assets_client = AssetsClient(**self.convoy_conf['assets_db'])
user = self.convoy_conf['couchdb'].get('user', '')
password = self.convoy_conf['couchdb'].get('password', '')
if user and password:
self.db = Server(
"http://{user}:{password}@{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(
retry_delays=range(10)))[self.convoy_conf['couchdb']['db']]
else:
self.db = Server(
"http://{host}:{port}".format(**self.convoy_conf['couchdb']),
session=Session(
retry_delays=range(10)))[self.convoy_conf['couchdb']['db']]
push_filter_doc(self.db)
LOGGER.info('Added filters doc to db.')
def _prepare_couchdb(self):
server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
try:
if self.db_name not in server:
self.db = server.create(self.db_name)
else:
self.db = server[self.db_name]
except Exception as e:
LOGGER.error('Database error: {}'.format(repr(e)))
raise
by_date_modified_view = ViewDefinition(
self.resource, 'by_dateModified', '''function(doc) {
if (doc.doc_type == '%(resource)s') {
var fields=['%(doc_type)sID'], data={};
for (var i in fields) {
if (doc[fields[i]]) {
data[fields[i]] = doc[fields[i]]
}
}
emit(doc.dateModified, data);
}}''' % dict(resource=self.resource[:-1].title(),
doc_type=self.resource[:-1])
)
by_date_modified_view.sync(self.db)
validate_doc = self.db.get(VALIDATE_BULK_DOCS_ID,
{'_id': VALIDATE_BULK_DOCS_ID})
if validate_doc.get('validate_doc_update') != VALIDATE_BULK_DOCS_UPDATE:
validate_doc['validate_doc_update'] = VALIDATE_BULK_DOCS_UPDATE
self.db.save(validate_doc)
LOGGER.info('Validate document update view saved.')
else:
LOGGER.info('Validate document update view already exist.')
def OnAddTag( self, event ):
bl = Server(self.URL)[BLOG]
p = Post.load(bl, self.blogpost)
tags = [ str(x.key) for x in bl.view("all/tags", group = True) if str(x.key) not in map(str,p.tags) ]
if tags:
dialog = wx.SingleChoiceDialog(None, "Choose a Tag or press Cancel to type it", "Tags", tags)
tag = ""
if dialog.ShowModal() == wx.ID_OK:
tag = dialog.GetStringSelection()
else:
tag = wx.GetTextFromUser( "Type a Tag ", "Tag")
if tag:
tag = tag.upper()
dialog.Destroy()
if tag:
bl = Server(self.URL)[BLOG]
p = Post.load(bl, self.blogpost)
tagList = p.tags
tagList.append(tag)
tagList = list(set(tagList))
tagList.sort()
p.tags = tagList
p.store(bl)
#event = wx.CommandEvent( wx.wxEVT_COMMAND_LIST_ITEM_SELECTED, self.list.GetId())
#self.GetEventHandler().ProcessEvent( event )
self.OnLCtrl(None)
def __init__(self, config):
super(LogsCollector, self).__init__()
self.config = config
self.storage = self.config_get('storage')
if not self.storage:
raise LogsCollectorConfigError('Configuration Error: Missing logs'
' storage.')
if self.storage == 'couchdb':
self.couch_url = self.config_get('couch_url')
self.db_name = self.config_get('log_db')
if not self.couch_url:
raise LogsCollectorConfigError('Configuration Error: Missing '
'couch_url')
else:
couch_url = urlparse(self.couch_url)
if couch_url.scheme == '' or couch_url.netloc == '':
raise LogsCollectorConfigError('Configuration Error:'
'Invalid couch_url')
if not self.db_name:
raise LogsCollectorConfigError('ConnectionError: Missing '
'couchdb name')
server = Server(self.couch_url,
session=Session(retry_delays=range(10)),
full_commit=False)
try:
if self.db_name not in server:
self.db = server.create(self.db_name)
else:
self.db = server[self.db_name]
except error as e:
logger.error('Database error: {}'.format(e.message))
raise LogsCollectorConfigError(e.message)
def remove_db(database):
'''
Destroy given database.
'''
server = Server('http://localhost:5984/')
server.delete(database)
logger.info('[DB] Local database %s removed' % database)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
self.offset = int(options['offset'])
self.limit = int(options['limit'])
self.logger.info("Starting export!")
server = Server()
if 'maps-places' not in server:
self.db = server.create('maps-places')
else:
self.db = server['maps-places']
# places uri startup
places_uri = "{0}/maps/places".format(settings.OP_API_URI)
self.logger.debug("GET {0}".format(places_uri))
# get all places, following the next link
places_json = self.export_page(places_uri)
while places_json and places_json['next']:
places_json = self.export_page(places_json['next'])
def init_db():
server = CouchDBServer(db_addr)
try:
server.create(db_name)
except PreconditionFailed:
del server[db_name]
server.create(db_name)
def __init__(self, config):
"""
Check CouchDB availability and set _db attribute
"""
server, db = config.get("COUCH_DATABASE").rsplit('/', 1)
server = Server(server, session=Session(retry_delays=range(10)))
database = server[db] if db in server else server.create(db)
self._db = database
def initialize(self):
couch = Server()
if "stories" not in couch:
couch.create("stories")
self.db_stories = couch['stories']
if "posts" not in couch:
couch.create("posts")
self.db_posts = couch['posts']
def isuserp(self, user, pw):
db = Server("http://localhost:8888/")['unit_tasks']
if len(
list(
db.view('_design/users/isvalid',
key=[user, self.hash(pw)]))) == 1:
return "1"
return "0"
class Connection(object):
"""
Facet for CouchDB client module Server object.
https://pythonhosted.org/CouchDB/client.html#server
Most DDL operations required Database Admin or Server Admin
privilege. And these operations can be also done directly on
CouchDB Fauxton UI or through curl API calls. So minimal DDL
implementation is required here.
"""
def __init__(self, conn):
self.conn = conn
self.server = None
self.establish()
def establish(self):
try:
self.server = Server(url=self.conn)
except Unauthorized:
msg = 'Unauthorized error. Check connection string: {}'.format(
self.conn)
logging.error(msg)
except ConnectionRefusedError:
msg = 'Can not connect to {}'.format(self.conn)
logging.error(msg)
def spawn(self, name):
try:
new_db = self.server.create(name)
if new_db is not None:
msg = '[SPAWN DB] spawning new database --: {}'.format(name)
logging.info(msg)
return new_db
except PreconditionFailed:
msg = '[SPAWN DB] Existing database --: {}'.format(name)
logging.warning(msg)
return self.server[name]
except Unauthorized:
msg = 'Unauthorized error. You are not a server admin: {}'.format(
self.conn)
logging.error(msg)
def drop(self, name):
try:
self.server.delete(name)
msg = '[DROP DB] --: {}'.format(name)
logging.warning(msg)
return True
except ResourceNotFound:
msg = '[DROP DB] Database does not exist --: {}'.format(name)
logging.warning(msg)
return False
except Unauthorized:
msg = 'Unauthorized error. You are not a server admin: {}'.format(
self.conn)
logging.error(msg)
return False
def __init__(self, host='localhost', port=6379, db=0, profiling_level=0):
from couchdb import Server
self.server = Server()
self.db_name = "{}_{}".format(self._db_name_prefix, uuid.uuid4().hex)
self.db = self.server.create(self.db_name)
self.doc_index_id = {}
def setUp(self):
uri = os.environ.get('COUCHDB_URI', 'http://localhost:5984/')
self.cache_dir = tempfile.mkdtemp(prefix='couchdb')
self.server = Server(uri, cache=self.cache_dir)
try:
self.server.delete('python-tests')
except client.ResourceNotFound:
pass
self.db = self.server.create('python-tests')
def main(bulk_size, up_to, num_threads):
global timer
s = Server('http://localhost:5984')
if 'test' in s:
del s['test']
db = s.create('test')
stats_file = 'bulk-perf-%s-%s-%s.dat' % (bulk_size, up_to, num_threads)
title_file = stats_file + '.meta'
f = open(title_file, 'w')
f.write('Bulk size: %s, num threads: %s' % (bulk_size, num_threads))
f.close()
stats_file = open(stats_file, 'w')
stats_lock = Lock()
exit_event = Event()
chunks = Queue.Queue()
def process_chunks():
global count, timer, internal_counter
s = Server('http://localhost:5984')
db = s['test']
while not exit_event.isSet():
try:
chunk = list(chunks.get(timeout=5))
chunks.task_done()
db.update(chunk)
stats_lock.acquire()
try:
count += bulk_size
internal_counter += bulk_size
if internal_counter >= max(num_threads*bulk_size, up_to/1000):
end = time()
stats_file.write('%s %s\n' % (count, internal_counter/float(end-timer)))
stats_file.flush()
timer = end
internal_counter = 0
print '%.1f%%' % (float(count) / up_to * 100)
finally:
stats_lock.release()
except Queue.Empty:
pass
except Exception, e:
print 'Exception: %r' % (e,)
chunks.put(chunk)
sleep(1)
def __init__(self):
couchdb_host = getattr(settings, 'COUCHDB_HOST')
server = Server(couchdb_host)
try:
self.db = server['wiki']
except ResourceNotFound:
self.db = server.create('wiki')
Page.get_pages.sync(self.db)
self.wiki_form = curry(PageForm, db = self.db)
def couchdb_connection(config):
LOGGER = getLogger("BILLING")
LOGGER.info("Start database initialization")
# CouchDB connection
db_name = config.get('db').get('name')
db_host = config.get('db').get('host')
db_port = config.get('db').get('port')
admin_name = config.get('admin').get('username')
admin_pass = config.get('admin').get('password')
aserver = Server(
create_db_url(db_host, db_port, admin_name, admin_pass),
session=Session(retry_delays=range(10)))
users_db = aserver['_users']
username = config.get('user').get('username')
password = config.get('user').get('password')
user_doc = users_db.get(
'org.couchdb.user:{}'.format(username),
{'_id': 'org.couchdb.user:{}'.format(username)}
)
if not user_doc.get('derived_key', '') or PBKDF2(password, user_doc.get('salt', ''), user_doc.get('iterations', 10)).hexread(int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get('derived_key', ''):
user_doc.update({
"name": username,
"roles": [],
"type": "user",
"password": password
})
LOGGER.info(
"Updating api db main user",
extra={'MESSAGE_ID': 'update_api_main_user'}
)
users_db.save(user_doc)
if db_name not in aserver:
aserver.create(db_name)
db = aserver[db_name]
SECURITY[u'members'][u'names'] = [username, ]
if SECURITY != db.security:
LOGGER.info(
"Updating api db security",
extra={'MESSAGE_ID': 'update_api_security'}
)
db.security = SECURITY
auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID})
if auth_doc.get('validate_doc_update') != VALIDATE_DOC_UPDATE % username:
auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username
LOGGER.info(
"Updating api db validate doc",
extra={'MESSAGE_ID': 'update_api_validate_doc'}
)
db.save(auth_doc)
def remove_db(database):
'''
Destroy given database.
'''
server = Server('http://localhost:5984/')
try:
server.delete(database)
except http.ResourceNotFound:
logger.info('[DB] Local database %s already removed' % database)
logger.info('[DB] Local database %s removed' % database)
def get_db(self):
try:
server=Server("http://127.0.0.1:5984/")
if 'cia_factbook' not in server:
server.create('cia_factbook')
db = server['cia_factbook']
except:
print ("DB_ERROR")
os._exit(-1)
return(db)
def __init__(self, url):
print "conectando com o banco..."
Server.__init__(self, url)
#self.erase_tables(_DOCBASES_TO_REMOVE)
act = self
test_and_create = lambda doc: doc in act and act[doc] or act.create(doc)
for attribute in _DOCBASES:
setattr(Activ, attribute, test_and_create(attribute))
def get_couchdb(dbname, uri=None):
if uri:
server = Server(uri)
else:
server = Server()
if dbname in server:
db = server[dbname]
else:
db = server.create(dbname)
return db
def establish(self):
try:
self.server = Server(url=self.conn)
except Unauthorized:
msg = 'Unauthorized error. Check connection string: {}'.format(
self.conn)
logging.error(msg)
except ConnectionRefusedError:
msg = 'Can not connect to {}'.format(self.conn)
logging.error(msg)
def create_db(database):
server = Server('http://localhost:5984/')
try:
db = server.create(database)
logger.info('[DB] Database %s created' % database)
except PreconditionFailed:
db = server[database]
logger.info('[DB] Database %s already exists.' % database)
return db
def create_test_users_db(config=None):
if config is None:
config = load_test_config()
name = config['couchdb.users_database']
try:
couchdb = Server(config['couchdb.address'])
return couchdb.create(name)
except PreconditionFailed:
del couchdb[name]
return couchdb.create(name)
def prepare_couchdb(couch_url, db_name, logger):
server = Server(couch_url, session=Session(retry_delays=range(10)))
try:
if db_name not in server:
db = server.create(db_name)
else:
db = server[db_name]
except error as e:
logger.error('Database error: {}'.format(e.message))
raise ConfigError(e.strerror)
sync_design(db)
return db
def couchdb_connection(config):
LOGGER = getLogger("BILLING")
LOGGER.info("Start database initialization")
# CouchDB connection
db_name = config.get('db').get('name')
db_host = config.get('db').get('host')
db_port = config.get('db').get('port')
admin_name = config.get('admin').get('username')
admin_pass = config.get('admin').get('password')
aserver = Server(create_db_url(db_host, db_port, admin_name, admin_pass),
session=Session(retry_delays=range(10)))
users_db = aserver['_users']
username = config.get('user').get('username')
password = config.get('user').get('password')
user_doc = users_db.get('org.couchdb.user:{}'.format(username),
{'_id': 'org.couchdb.user:{}'.format(username)})
if not user_doc.get('derived_key', '') or PBKDF2(
password, user_doc.get('salt', ''),
user_doc.get('iterations', 10)).hexread(
int(len(user_doc.get('derived_key', '')) / 2)) != user_doc.get(
'derived_key', ''):
user_doc.update({
"name": username,
"roles": [],
"type": "user",
"password": password
})
LOGGER.info("Updating api db main user",
extra={'MESSAGE_ID': 'update_api_main_user'})
users_db.save(user_doc)
if db_name not in aserver:
aserver.create(db_name)
db = aserver[db_name]
SECURITY[u'members'][u'names'] = [
username,
]
if SECURITY != db.security:
LOGGER.info("Updating api db security",
extra={'MESSAGE_ID': 'update_api_security'})
db.security = SECURITY
auth_doc = db.get(VALIDATE_DOC_ID, {'_id': VALIDATE_DOC_ID})
if auth_doc.get('validate_doc_update') != VALIDATE_DOC_UPDATE % username:
auth_doc['validate_doc_update'] = VALIDATE_DOC_UPDATE % username
LOGGER.info("Updating api db validate doc",
extra={'MESSAGE_ID': 'update_api_validate_doc'})
db.save(auth_doc)
def create_db(name):
'''
Create a new name for given name.
'''
server = Server('http://localhost:5984/')
try:
db = server.create(name)
logger.info('[DB] Database %s created' % name)
except PreconditionFailed:
db = server[name]
logger.info('[DB] Database %s already exists.' % name)
return db
def main(couchdb_url=None,
couchdb_db='openprocurement',
seq_file=None,
dump_dir=None):
if JournalHandler:
params = {
'TAGS': 'python,concord',
}
LOGGER.addHandler(JournalHandler(**params))
if couchdb_url:
server = Server(url=couchdb_url,
session=Session(retry_delays=range(10)))
else:
server = Server(session=Session(retry_delays=range(10)))
for i in range(10):
try:
db = server[couchdb_db]
except:
sleep(i)
else:
break
else:
db = server[couchdb_db]
if dump_dir and not os.path.isdir(dump_dir):
os.mkdir(dump_dir)
if seq_file and os.path.isfile(seq_file):
with open(seq_file) as f:
fdata = f.read()
last_seq = int(fdata) if fdata.isdigit() else 0
else:
last_seq = 0
seq_block = last_seq / 100
while True:
cc = db.changes(timeout=55000,
since=last_seq,
feed='longpoll',
filter='_view',
view='conflicts/all',
include_docs=True,
conflicts=True)
#wait([
#spawn(conflicts_resolve, db, c, dump_dir)
#for c in cc[u'results']
#])
for c in cc[u'results']:
conflicts_resolve(db, c, dump_dir)
last_seq = cc[u'last_seq']
if seq_file and seq_block < last_seq / 100:
with open(seq_file, 'w+') as f:
f.write(str(last_seq))
seq_block = last_seq / 100
class CouchDBUploadBackend(AbstractUploadBackend):
"""Stores the file in a CouchDB Backend
Requirements: couchdb
Settings:
AJAXUPLOAD_COUCHDB_HOST: Specify the host of your MongoDB server.
Defaults to http://localhost:5984 if not specified.
Arguments:
db (required): Specify the database within CouchDB you wish to use
Context returned:
None
"""
def __init__(self, *args, **kwargs):
self.database = kwargs.pop('db')
self.connection = None
self._dest = None
super(CouchDBUploadBackend, self).__init__(*args, **kwargs)
def setup(self, filename):
self.connection = Server(
getattr(settings, 'AJAXUPLOAD_COUCHDB_HOST',
'http://localhost:5984'))[self.database]
self._dest = TemporaryFile()
def upload_chunk(self, chunk):
self._dest.write(chunk)
def upload_complete(self, request, filename):
self._dest.seek(0)
doc_id = uuid4().hex
# create doc by self defined uuid. We need the _rev for attachment
doc = self.connection[doc_id] = {
'_id': doc_id,
# append anything you like maybe from
# request
}
# mimetype is guessed by extension.
# We don't put the whole document back in the put_attachment request
self.connection.put_attachment({
'_id': doc['_id'],
'_rev': doc['_rev']
},
self._dest,
filename=filename)
self._dest.close()
def get_db():
from couchdb import Server
server = getenv("CDB_SERVER")
db = getenv("JOB_CURRENT")
db = str(db).strip().lower() + "_render_log"
try:
server = Server(server)
if db in server: db = server[db]
else: db = server.create(db)
except:
return 0
return db
def setUp(self):
user = self.config['main']['storage'].get('user', '')
password = self.config['main']['storage'].get('password', '')
if (user and password):
self.couch_url = "http://{user}:{password}@{host}:{port}".format(
**self.config['main']['storage'])
else:
self.couch_url = "http://{host}:{port}".format(
**self.config['main']['storage'])
server = Server(self.couch_url)
if self.config['main']['storage']['db_name'] in server:
self.db = server[self.config['main']['storage']['db_name']]
else:
self.db = server.create(self.config['main']['storage']['db_name'])
def get_replication_progress(name):
'''
Get the progress of all continuous replications
'''
server = Server('http://localhost:5984/')
tasks = server.tasks()
i = 0
for task in tasks:
if i == 0:
if task.get('continuous'):
print "Continuous r={}, w={}, p={}%".format(task.get('docs_read'), task.get('docs_written'), task.get('progress'))
else:
print "Initialize r={}, w={}, p={}%".format(task.get('docs_read'), task.get('docs_written'), task.get('progress'))
i = i + 1
return 0
def init():
log.info('Initialize database')
from ctx import config
from couchdb import Server
global db, server
server = Server(url=config.DB_URL)
log.info('db server: {!r}'.format(config.DB_URL))
try:
db = server[config.DB_NAME]
except couchdb.http.ResourceNotFound:
db = server.create(config.DB_NAME)
log.info('db {!r} created'.format(config.DB_NAME))
class GlitterHandler(logging.Handler):
def __init__(self, url='http://localhost:5984/', db='logs'):
from couchdb import Server
self.db = Server(url)[db]
super(GlitterHandler, self).__init__()
def emit(self, record):
try:
self.db.save(record.__dict__)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def load_database(entries):
'''
Loads entries data into database
'''
# Open connection
server = Server(DB_HOST)
# Create a clean database
if DB_NAME in server:
del server[DB_NAME]
db = server.create(DB_NAME)
keys = []
for entry in entries:
db.create(json.dumps(entry))
def load_database(entries):
'''
Loads entries data into database
'''
# Open connection
server = Server(DB_HOST)
# Create a clean database
if DB_NAME in server:
del server[DB_NAME]
db = server.create(DB_NAME)
keys = []
for entry in entries:
db.create(json.dumps(entry))
class PostTestCase(unittest.TestCase):
def setUp(self):
self.server = Server(settings.COUCHDB_SERVER)
try:
self.db = self.server.create('comfy_blog_test')
except:
self.db = self.server['comfy_blog_test']
self.post1 = Post(title=u"Hello, World!", slug=u"foo-bar", published=datetime(2008, 8, 8), author={'name': 'Myles Braithwaite', 'email': '*****@*****.**'})
self.post2 = Post(title=u"Hello, World!", published=datetime(2007, 7, 7))
self.post1.store()
self.post2.store()
def testURL(self):
self.assertEquals(self.post1.get_absolute_url(), '/blog/2008/8/8/foo-bar/')
self.assertEquals(self.post2.get_absolute_url(), '/blog/2007/7/7/hello-world/')
def testSlugify(self):
self.assertEquals(self.post2.slug, 'hello-world')
#def testAddComment(self):
# post = Post.load(self.db, self.post1.id)
# coment = post.comments()
# comment.author = {'name': u"Myles Braithwaite", 'email': "*****@*****.**", 'url': u"http://mylesbraithwaite.com/"}
# comment.comment = u"Hello, World!"
# comment.time = datetime.now()
# comment.user_agent = u"Python Unit Test"
# comment.ip_address = u"127.0.0.1"
# comment.is_spam = False
# post.store()
# # TODO Still working on doing something here to see if the test actually worked.
def tearDown(self):
del self.server['comfy_blog_test']
def tearDown(self):
self.config['resources_api_server'] = \
'https://lb.api-sandbox.openprocurement.org'
self.config['resources_api_version'] = "0"
self.config['public_resources_api_server'] = \
'https://lb.api-sandbox.openprocurement.org'
self.config['couch_url'] = 'http://localhost:5984'
self.config['db_name'] = 'test_db'
self.config['queue_size'] = 101
self.config['api_clients_count'] = 3
self.config['workers_count'] = 3
self.config['retry_workers_count'] = 2
self.config['filter_workers_count'] = 1
self.config['retry_workers_count'] = 2
self.config['retry_default_timeout'] = 5
self.config['worker_sleep'] = 0.1
self.config['watch_interval'] = 0.5
self.config['queue_timeout'] = 0.1
self.config['resource_items_queue_size'] = -1
self.config['retry_resource_items_queue_size'] = -1
self.config['bulk_query_limit'] = 1
self.config['bulk_query_interval'] = 0.5
try:
server = Server(self.config['main'].get('couch_url') or
'http://127.0.0.1:5984')
del server[self.config['main']['db_name']]
except:
pass
def OnScreenshot(self, event):
wx.MessageBox("You got 5 seconds to go", "Screenshot Warning")
time.sleep(5)
sfile = REGEXP.sub("","screenshot{0}".format(datetime.now()))
sfile = "{0}.png".format(sfile)
screenshot = Screenshot(filename = sfile)
try:
blog = Server(self.URL)[BLOG]
doc = blog[self.blogpost]
f = open(sfile,"rb")
blog.put_attachment(doc,f, sfile)
f.close()
self.OnLCtrl(None)
except:
pass
def setUp(self):
uri = os.environ.get('COUCHDB_URI', 'http://localhost:5984/')
self.cache_dir = tempfile.mkdtemp(prefix='couchdb')
self.server = Server(uri, cache=self.cache_dir)
if 'python-tests' in self.server:
del self.server['python-tests']
self.db = self.server.create('python-tests')
def test_collector_init(self):
test_config = {}
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
test_config = {'main': {}}
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
test_config['main'] = {
'storage': 'couchdb'
}
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
test_config['main']['couch_url'] = 'labuda'
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
test_config['main']['couch_url'] = 'http://127.0.0.1:5984'
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
test_config['main']['log_db'] = 'test_db'
try:
server = Server(test_config['main']['couch_url'])
del server[test_config['main']['log_db']]
except ResourceNotFound:
pass
with patch('openprocurement.edge.collector.Server.create') as mock_create:
mock_create.side_effect = error('test error')
with self.assertRaises(LogsCollectorConfigError):
LogsCollector(test_config)
def __init__(self, server=SERVER_HOST, database_name = DATABASE_NAME, *args, **kwargs):
self.url = server
self.server = Server(self.url)
try:
self.database = self.server[database_name]
except ResourceNotFound:
self.database = self.server.create(database_name)
class CouchDBUploadBackend(AbstractUploadBackend):
"""Stores the file in a CouchDB Backend
Requirements: couchdb
Settings:
AJAXUPLOAD_COUCHDB_HOST: Specify the host of your MongoDB server.
Defaults to http://localhost:5984 if not specified.
Arguments:
db (required): Specify the database within CouchDB you wish to use
Context returned:
None
"""
def __init__(self, *args, **kwargs):
self.database = kwargs.pop('db')
self.connection = None
self._dest = None
super(CouchDBUploadBackend, self).__init__(*args, **kwargs)
def setup(self, request, filename, *args, **kwargs):
self.connection = Server(getattr(settings,
'AJAXUPLOAD_COUCHDB_HOST',
'http://localhost:5984')
)[self.database]
self._dest = TemporaryFile()
def upload_chunk(self, chunk, *args, **kwargs):
self._dest.write(chunk)
def upload_complete(self, request, filename, *args, **kwargs):
self._dest.seek(0)
doc_id = uuid4().hex
# create doc by self defined uuid. We need the _rev for attachment
doc = self.connection[doc_id] = {'_id':doc_id,
# append anything you like maybe from
# request
}
# mimetype is guessed by extension.
# We don't put the whole document back in the put_attachment request
self.connection.put_attachment({'_id':doc['_id'], '_rev':doc['_rev']},
self._dest,
filename=filename)
self._dest.close()
def create_database():
server = Server('http://localhost:8888/')
db = server.create('unit_tasks')
#XXX: figure out how to run python view functions?
db.create({
"_id": "_design/users",
"views": {
"isvalid": """
function(doc) {
if (doc.type == 'User')
map([doc._id, doc.password], doc);
}"""
}
})
User(_id="llimllib", password=md5("tao").hexdigest()).store(db)
