def run(self):
crm = get_crm_module(self.settings)
server = Server(self.settings.couchdb_uri)
db = server.get_db(self.settings.couchdb_db)
invoice_cycle = InvoiceCycle(self.settings.invoice_nr)
clients = db.view(
self._cmd("client_is_billable"), include_docs=True
)
counter = 0
for client in clients:
if not self.settings.limit_client_id or client['id'] == self.settings.limit_client_id:
# print(client['doc']['name'])
invoice = Invoice(
self.settings, crm=crm,
client_doc=client['doc'],
invoice_cycle=invoice_cycle
)
invoice_start_date = min(
[d['start_date'] for d in invoice.doc['services'].itervalues()]
)
if invoice_start_date < invoice_cycle.doc['end_date']:
invoice.render_pdf()
invoice_cycle.add_invoice(invoice)
counter += 1
print(".", end="")
if self.settings.max != 0 and counter >= self.settings.max:
break
print("\n", "total", invoice_cycle.get_total())
class CouchDBServer(object):
def __init__(self):
self.__get_server_uri()
self.__authenticate()
self.__connect()
def __get_server_uri(self):
couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port
self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port)
def __authenticate(self):
user, passwd = config.couchdb.user, config.couchdb.password
if all((user, passwd)):
auth = restkit.BasicAuth(user, passwd)
self.__auth_resource = CouchdbResource(filters=[auth])
else:
self.__auth_resource = None
def __connect(self):
self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource)
def list_workspaces(self):
return filter(is_usable_workspace, self.__server.all_dbs())
def get_workspace_handler(self, ws_name):
return self.__server.get_db(ws_name)
def get_or_create_db(self, ws_name):
return self.__server.get_or_create_db(ws_name)
def __init__(self, settings, crm, client_doc, invoice_cycle):
self.invoice_template_dir = settings.invoice_template_dir
self.output_dir = settings.invoice_output_dir
self.tax = float(settings.invoice_tax)
self.jinja_env = Environment(
loader=FileSystemLoader(self.invoice_template_dir)
)
self.jinja_env.filters['nl2br'] = filters.nl2br
self.jinja_env.filters['format_date'] = filters.format_date
self.crm = crm
self.client_id = client_doc['_id']
try:
self.extcrm_id = client_doc['extcrm_id']
except KeyError:
print(client_doc)
raise KeyError
self.client_doc = client_doc
self.settings = settings
self.invoice_cycle = invoice_cycle
self.invoice_nr = invoice_cycle.current_nr
server = Server(self.settings.couchdb_uri)
self.db = server.get_db(self.settings.couchdb_db)
if not Invoice.service_definitons:
Invoice.load_service_definitions(self.db)
self.setup_invoice()
class CouchDBServer(object):
def __init__(self):
self.__get_server_uri()
self.__authenticate()
self.__connect()
def __get_server_uri(self):
couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port
self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol,
config.couchdb.host, couchdb_port)
def __authenticate(self):
user, passwd = config.couchdb.user, config.couchdb.password
if (all((user, passwd))):
auth = restkit.BasicAuth(user, passwd)
self.__auth_resource = CouchdbResource(filters=[auth])
else:
self.__auth_resource = None
def __connect(self):
self.__server = Server(uri=self.__couchdb_uri,
resource_instance=self.__auth_resource)
def list_workspaces(self):
return filter(is_usable_workspace, self.__server.all_dbs())
def get_workspace_handler(self, ws_name):
return self.__server.get_db(ws_name)
def __init__(self, settings, crm, client_doc, invoice_cycle):
self.invoice_template_dir = settings.invoice_template_dir
self.output_dir = settings.invoice_output_dir
self.tax = float(settings.invoice_tax)
self.home_country = settings.invoice_home_country if hasattr(settings, 'invoice_home_country', ) else None
self.jinja_env = Environment(
loader=FileSystemLoader(self.invoice_template_dir)
)
self.jinja_env.filters['nl2br'] = filters.nl2br
self.jinja_env.filters['format_date'] = filters.format_date
self.crm = crm
self.client_id = client_doc['_id']
try:
self.extcrm_id = client_doc['extcrm_id']
except KeyError:
extcrm_id = client_doc['extcrm_id'] if 'extcrm_id' in client_doc else "None"
print('\nNOT creating invoice missing extcrm_id:%s, %s' % (extcrm_id, client_doc['name']), client_doc)
self.client_doc = None
return
self.client_doc = client_doc
self.settings = settings
self.invoice_cycle = invoice_cycle
self.invoice_nr = invoice_cycle.current_nr
server = Server(self.settings.couchdb_uri)
self.db = server.get_db(self.settings.couchdb_db)
if not Invoice.service_definitons:
Invoice.load_service_definitions(self.db)
self.setup_invoice()
class SettingsCommandDb(SettingsCommand):
def set_settings(self, settings, hostname=os.uname()[1]):
super(SettingsCommandDb, self).set_settings(settings, hostname)
self.server = Server(uri=self.settings.couchdb_uri)
self.db = self.server.get_db(self.settings.couchdb_db)
def _cmd(self, cmd):
return "{}/{}".format(self.settings.couchdb_db, cmd)
def get_prod_db(source_uri):
"""
Get the production database object since we need to get some doc_ids from the prod database
"""
prod_db_name = source_uri.split('/')[-1]
prod_server = Server(uri=source_uri[:-len(prod_db_name)])
prod_db = prod_server.get_db(prod_db_name)
return prod_db
def get_detailed_status(options):
total_tests = 0
total_errors = 0
server = Server(options.node)
db = server.get_db(options.database)
doc = get_build_doc(db, options.build)
doc_content = db.open_doc(doc['_id'])
tests_list = doc_content['_attachments'].keys()
# TODO: Check for number of tests that ran
print "List of tests against %s are %s " % (options.build, tests_list)
failed_tests = []
# Data struct to store information per test_class (attachment)
# {test_name:{tests:<int>, errros:<int>, time:<float>}}
test_data = {}
for attachment, value in doc_content['_attachments'].items():
errors_count = 0
tests_count = 0
print "Fetching attachment %s " % attachment
file = db.fetch_attachment(doc, attachment)
file = file.encode('ascii', 'ignore')
xmldoc = etree.parse(StringIO.StringIO(file))
root = xmldoc.getroot()
for child in root:
attributes = child.attrib
for childish in child:
# To print the error
#print childish.text
failed_tests.append(attributes.get('name'))
attributes = root.attrib
# To get root element attributes
#print attributes
name = attributes.get('name')
errors_count = int(attributes.get('errors'))
tests_count = int(attributes.get('tests'))
total_tests += tests_count
total_errors += errors_count
print "Testname: %s, passed: %s, failed: %s" % (
name, tests_count - errors_count, errors_count)
test_data[name] = {}
test_data[name]['tests'] = tests_count
test_data[name]['errors'] = errors_count
test_data[name]['time'] = float(attributes.get('time'))
#print test_data
text = "Passed %s out of %s tests on %s build\n" % (
total_tests - total_errors, total_tests, options.build)
num_failed = len(failed_tests)
if num_failed > 0:
text += "\nList of Failed tests:\n"
for i in range(num_failed):
text += "%s: %s\n" % (i + 1, failed_tests[i])
print text
return test_data
def get_detailed_status(options):
total_tests = 0
total_errors = 0
server = Server(options.node)
db = server.get_db(options.database)
doc = get_build_doc(db, options.build)
doc_content = db.open_doc(doc['_id'])
tests_list = doc_content['_attachments'].keys()
# TODO: Check for number of tests that ran
print "List of tests against %s are %s " % (options.build, tests_list)
failed_tests = []
# Data struct to store information per test_class (attachment)
# {test_name:{tests:<int>, errros:<int>, time:<float>}}
test_data = {}
for attachment, value in doc_content['_attachments'].items():
errors_count = 0
tests_count = 0
print "Fetching attachment %s " % attachment
file = db.fetch_attachment(doc, attachment)
file = file.encode('ascii', 'ignore')
xmldoc = etree.parse(StringIO.StringIO(file))
root = xmldoc.getroot()
for child in root:
attributes = child.attrib
for childish in child:
# To print the error
#print childish.text
failed_tests.append(attributes.get('name'))
attributes = root.attrib
# To get root element attributes
#print attributes
name = attributes.get('name')
errors_count = int(attributes.get('errors'))
tests_count = int(attributes.get('tests'))
total_tests += tests_count
total_errors += errors_count
print "Testname: %s, passed: %s, failed: %s" % (name, tests_count-errors_count,
errors_count)
test_data[name] = {}
test_data[name]['tests'] = tests_count
test_data[name]['errors'] = errors_count
test_data[name]['time'] = float(attributes.get('time'))
#print test_data
text = "Passed %s out of %s tests on %s build\n" % (total_tests-total_errors, total_tests,
options.build)
num_failed = len(failed_tests)
if num_failed > 0:
text += "\nList of Failed tests:\n"
for i in range(num_failed):
text += "%s: %s\n" % (i+1, failed_tests[i])
print text
return test_data
def run(self):
server = Server(self.settings.couchdb_uri)
db = server.get_db(self.settings.couchdb_db)
def next_migration(version, doc_id, doc_type):
new_version = version + 1
next_migration_name = "to%04d" % new_version
if hasattr(migrations, next_migration_name):
do_migration = getattr(migrations, next_migration_name)
doc = db.get(doc_id)
print('updating to:', new_version, doc_type, doc_id)
do_migration(doc, doc_type, db)
next_migration(new_version, doc_id, doc_type)
for item in db.view(self._cmd("version")):
version, doc_id, doc_type = item['key'], item['id'], item['value']
next_migration(version, doc_id, doc_type)
def run(self):
server = Server(self.settings.couchdb_uri)
db = server.get_db(self.settings.couchdb_db)
provides = {}
if hasattr(self.settings, 'worker_dns'):
provides['domain'], worker_dns = [], self.settings.worker_dns
dns_servers = map(
lambda x: x.strip().split(':'), worker_dns.split(',')
)
for backend, name in dns_servers:
provides['domain'].append({'backend': backend, 'name': name})
worker_id = "worker-{}".format(self.hostname)
d = {
"_id": worker_id, "type": "worker", "hostname": self.hostname,
"provides": provides
}
db.save_doc(d)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
self.__serv = Server(uri = uri)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
proto, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
_, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" % aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db, feed="longpoll", since = last_seq, timeout = timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x,y: max(y['seq'], x) , changes, self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"), self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class ImportServices(object):
def __init__(self, settings):
self.service_tpl = {"type": "service", "state": "active"}
self.docs, self.clients_extcrm_ids = [], {}
self.settings = settings
self.nr_cols = settings.nr_cols
self.server = Server(self.settings.couchdb_uri)
self.db = self.server.get_db(self.settings.couchdb_db)
self.init_spreadsheet()
def init_spreadsheet(self):
spreadsheet = ezodf.opendoc(self.settings.src)
sheet = spreadsheet.sheets[0]
self.rows = sheet.rows()
self.header = [
c.value.lower() if hasattr(c.value, "lower") else c.value for c in self.rows.next()[: self.nr_cols]
]
def process_row(self, row):
for service_type in ["web", "email"]:
if row[service_type]:
try:
service_doc = self.service_tpl.copy()
service_doc["_id"] = "service-%s" % self.server.next_uuid()
if row["todoyu"] not in self.clients_extcrm_ids:
client_id = self.get_or_create_client(row)
self.clients_extcrm_ids[row["todoyu"]] = client_id
else:
client_id = self.clients_extcrm_ids[row["todoyu"]]
service_doc["service_type"] = service_type
service_doc["start_date"] = row["start_date"]
service_doc["client_id"] = client_id
if isinstance(row[service_type], dict):
service_doc.update(row[service_type])
else:
service_doc["package_type"] = row[service_type]
addon_key = "%s_addon_items" % service_type
if addon_key in row and row[addon_key]:
service_doc["addon_service_items"] = row[addon_key]
included_key = "%s_included_items" % service_type
if included_key in row and row[included_key]:
service_doc["included_service_items"] = row[included_key]
self.docs.append((service_doc["_id"], service_doc))
except:
print("!!! couldn't import service")
pass
def process_sheet(self):
services = []
i = 0
for row in self.rows:
i += 1
if any([cell.value for cell in row[: self.nr_cols]]):
cell_values = []
for cell in row[: self.nr_cols]:
cell_is_python = hasattr(cell.value, "startswith") and (
cell.value.startswith("{") or cell.value.startswith("[")
)
if cell_is_python:
value = ast.literal_eval(cell.value)
else:
value = cell.value
cell_values.append(value)
row_dict = dict(zip(self.header[: self.nr_cols], cell_values))
services.append(row_dict)
else:
break
return services
def _cmd(self, cmd):
return "{}/{}".format(self.settings.couchdb_db, cmd)
def get_or_create_client(self, row):
if row["org_name"]:
name = row["org_name"]
else:
name = "%s %s" % (row["n_given"], row["n_family"])
query_results = self.db.view(self._cmd("client_by_name"), key=name, include_docs=False)
if query_results.count() == 1:
client_doc = query_results.first()["value"]
elif query_results.count() == 0:
client_doc = {"type": "client"}
client_doc["_id"] = "client-%s" % self.server.next_uuid()
else:
raise
client_doc["is_billable"] = 1
client_doc["extcrm_id"] = row["todoyu"]
client_doc["name"] = name
self.docs.append((client_doc["_id"], client_doc))
return client_doc["_id"]
def create_docs(self):
self.rows = self.process_sheet()
self.todoyu = Todoyu(self.settings)
self.dest = "{}/services_json".format(os.path.dirname(self.settings.src))
for row in self.rows:
self.process_row(row)
def create_files(self):
self.create_docs()
if os.path.exists(self.dest):
shutil.rmtree(self.dest)
os.mkdir(self.dest)
json_files = FilesForCouch(self.docs, self.dest)
json_files.create()
if not self.settings.only_files:
couch_up = CouchdbUploader(
path=self.dest, couchdb_uri=self.settings.couchdb_uri, couchdb_db=self.settings.couchdb_db
)
for fname in os.listdir(self.dest):
couch_up.put(data="@{}".format(fname), doc_id=fname[:-5])
class QueryServices(object):
def __init__(self, settings):
self.clients_extcrm_ids = {}
self.settings = settings
self.server = Server(self.settings.couchdb_uri)
self.db = self.server.get_db(self.settings.couchdb_db)
self.crm = get_crm_module(self.settings)
def _cmd(self, cmd):
return "{}/{}".format(self.settings.couchdb_db, cmd)
def query(self):
services = []
startkey = [self.settings.service]
endkey = [self.settings.service]
only_billable = self.settings.only_billable
if self.settings.service_packages:
startkey.append(self.settings.service_packages)
endkey.append(self.settings.service_packages)
couchdb_view = 'service_package_addon'
else:
couchdb_view = 'service_addon'
if self.settings.service_addons:
startkey.append(self.settings.service_addons)
endkey.append(self.settings.service_addons)
else:
if self.settings.service_packages:
couchdb_view = 'service_package_addon'
endkey.append({})
else:
couchdb_view = 'service_type'
for item in self.db.view(
self._cmd(couchdb_view),
startkey=startkey, endkey=endkey, include_docs=True):
if 'extcrm_id' in item['doc']:
client_doc = item['doc']
extcrm_id = client_doc['extcrm_id']
service_name = '-'.join([part for part in item['key'] if part])
included_items = []
is_billable = client_doc['is_billable'] if 'is_billable' in client_doc else False
if 'value' in item and 'included_service_items' in item['value']:
included_items = [
included['itemid'] for included in item['value']['included_service_items']
]
included_items = ','.join(included_items)
address_id = client_doc['extcrm_id'] if 'extcrm_id' in client_doc else None
if 'extcrm_contact_id' in client_doc and client_doc['extcrm_contact_id']:
contact_id = client_doc['extcrm_contact_id']
elif 'p' in extcrm_id:
for part in extcrm_id.split('-'):
if part.startswith('p'):
contact_id = part
else:
contact_id = ''
if not only_billable or only_billable and is_billable:
if address_id and all([
self.crm.has_contact(cid.strip()) for cid in contact_id.split(',')
]):
contacts = [
self.crm.get_contact(cid.strip()) for cid in contact_id.split(',')
]
services.append([
','.join([contact.email for contact in contacts]),
','.join([contact.name for contact in contacts]),
service_name,
included_items,
extcrm_id, contact_id
])
else:
services.append([
"# No Email #", "# no contact#", service_name,
included_items,
extcrm_id, ''
])
else:
print("*** no extcrm_id", item)
for service in services:
print(';'.join(service))
print("total: %s" % len(services))
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
url = urlparse(uri)
print("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
print(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" %
aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument,
use_uuids=True,
force_update=True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName,
self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since=0, timeout=15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db,
feed="longpoll",
since=last_seq,
timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x, y: max(y['seq'], x), changes,
self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"),
self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class IspmanDnsLDIF(LDIFParser):
structure_map = {
'a': ['host', 'ip'],
'aaaa': ['host', 'ipv6'],
'cname': ['alias', 'host'],
'mx': ['host', 'priority'],
'txt': ['name', 'txt'],
}
def __init__(self, input, output, settings, clients_ldif=None, editor=None):
LDIFParser.__init__(self, input)
self.domains = {}
self.domains_lookup = (
clients_ldif.domains_lookup if clients_ldif else None
)
self.editor = editor
self.a_record_ips = set([])
self.a_record_hosts = {}
self.server = Server(settings.couchdb_uri)
self.db = self.server.get_db(settings.couchdb_db)
def handle(self, dn, entry):
if dn.startswith('relativeDomainName='):
domain = ".".join(
[dc.split('=')[1] for dc in dn.split(',')
if dc.startswith('dc=')]
)
domain = domain.decode("utf-8").encode("iso8859-1")
if domain not in self.domains:
self.add_domain(domain)
def cname(entry):
self.domains[domain]['cname'].append(
{'alias': entry['relativeDomainName'][0].strip(),
'host': entry['cNAMERecord'][0].strip()}
)
def a(entry):
host = entry['relativeDomainName'][0].strip()
ip = entry['aRecord'][0].strip()
self.domains[domain]['a'].append(
{'host': host,
'ip': ip}
)
self.a_record_ips.add(ip)
full_host = "%s.%s" % (host, domain)
if ip in self.a_record_hosts:
self.a_record_hosts[ip].append(full_host)
else:
self.a_record_hosts[ip] = [full_host]
def mx(entry):
entry = entry['mXRecord'][0].split(' ')
self.domains[domain]['mx'].append(
{'host': entry[1].strip(),
'priority': entry[0]}
)
def ns(entry):
nameserver = entry['nSRecord'][0]
self.domains[domain]['nameservers'].append(nameserver)
self.domains[domain]['nameservers'].sort()
if "cNAMERecord" in dn:
cname(entry)
elif "aRecord" in dn:
a(entry)
elif "mXRecord" in dn:
mx(entry)
elif "nSRecord" in dn:
ns(entry)
def add_domain(self, domain):
if domain not in self.domains:
next_uuid = self.server.next_uuid()
self.domains[domain] = {
'_id': 'domain-%s' % next_uuid,
'state': 'new',
'type': 'domain',
'domain': domain,
'a': [],
'cname': [],
'mx': [],
'nameservers': [],
}
if self.domains_lookup:
self.domains[domain]['client_id'] = (
self.domains_lookup[domain]
)
if self.editor:
self.domains[domain]['editor'] = self.editor
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
#setting the doc types to load from couch
def get_types(subclasses):
if len(subclasses):
head = subclasses[0]
tail = []
if len(subclasses[1:]):
tail = subclasses[1:]
return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail)
return []
self._model_object_types = get_types([ModelObject])
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self._getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self._getDb(aWorkspaceName)
self.incrementSeqNumber(aWorkspaceName)
self._getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
getLogger(self).debug("Saving document in remote workspace %s" % aWorkspaceName)
return self._getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
def _getDb(self, aWorkspaceName):
if not self.__dbs.has_key(aWorkspaceName):
self.__getDb(aWorkspaceName)
return self.__dbs.get(aWorkspaceName, None)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
getLogger(self).debug("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
getLogger(self).debug("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
getLogger(self).debug("Getting document for workspace [%s]" % aWorkspaceName)
return self._getDb(aWorkspaceName).get(documentId)
@trap_timeout
def getDeletedDocument(self, aWorkspaceName, documentId, documentRev):
return self._getDb(aWorkspaceName).get(documentId, rev=documentRev)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self._getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self._getDb(db_name)
with ChangesStream(db, feed="longpoll", since=last_seq, timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
self.setLastChangeSeq(db_name, change['seq'])
if not change['id'].startswith('_design'):
#fake doc type for deleted objects
doc = {'type': 'unknown', '_deleted': 'False', '_rev':[0]}
if not change.get('deleted'):
doc = self.getDocument(db_name, change['id'])
changes.append(change_factory.create(doc))
if len(changes):
getLogger(self).debug("Changes from another instance")
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(self.filterConditions, self._getDb(workspaceName).documents(include_docs=True))
def filterConditions(self, doc):
ret = True
ret = ret and not doc["id"].startswith("_")
ret = ret and doc['doc']["type"] in self._model_object_types
return ret
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self._getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
if not self.__seq_nums.has_key(workspaceName):
self.__seq_nums[workspaceName] = 0
self.__seq_nums[workspaceName] += 1
self.mutex.release()
import httplib2
import json
h = httplib2.Http()
resp, content = h.request('http://localhost:5984/jlblog','POST',json.dumps({'oi':1}),headers={'Content-Type':'application/json'})
resp, content = h.request('http://localhost:5984/jlblog/users','GET')
c = json.loads(content)
c.has_key('admin')
c.get('admin')
from couchdbkit import Server
s = Server()
db = s.get_db('jlblog')
db.save_doc(dict(oi=1))
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" %
(url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_"
) and x not in CONST_BLACKDBS
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization"
% dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error(
"Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it"
)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri, couch_exception_callback):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self._available = False
self.couch_exception_callback = couch_exception_callback
test_couch_thread = threading.Thread(target=self.continuosly_check_connection)
test_couch_thread.daemon = True
test_couch_thread.start()
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def continuosly_check_connection(self):
"""Intended to use on a separate thread. Call module-level
function testCouch every second to see if response to the server_uri
of the DB is still 200. Call the exception_callback if we can't access
the server three times in a row.
"""
tolerance = 0
server_uri = self.__uri
while True:
time.sleep(1)
test_was_successful = test_couch(server_uri)
if test_was_successful:
tolerance = 0
else:
tolerance += 1
if tolerance == 3:
self.couch_exception_callback()
return False # kill the thread if something went wrong
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
def _delete(self, name):
self.__serv.delete_db(name)
def _loadDbs(self):
def conditions(database):
begins_with_underscore = database.startswith("_")
is_blacklisted = database in CONST_BLACKDBS
return not begins_with_underscore and not is_blacklisted
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error("Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it")
return self.__uri + "/reports/_design/reports/index.html"
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class DocsProcessor(SettingsCommand):
allowed_template_type = None # needs to be set by subclass
map_id = None # needs to be set by subclass
replace_id = None # needs to be set by subclass
_map_cache = None
_map_cache_time = None
@classmethod
def map_cache(cls):
if cls._map_cache_time and time.time() > (cls._map_cache_time + 3600):
cls._map_cache = None
return cls._map_cache
@classmethod
def set_map_cache(cls, value):
cls._map_cache_time = time.time()
cls._map_cache = value
def __init__(self, settings, docs):
self.set_settings(settings)
template_ids, map_id = self.settings.template_ids, self.settings.map_id
template_ids = template_ids.split(',') if template_ids else []
self.server = Server(uri=self.settings.couchdb_uri)
self.db = self.server.get_db(self.settings.couchdb_db)
self.template_docs, self.map_keys = [], None
if not template_ids:
self.template_docs = self.get_all_templates()
else:
self.template_docs = self.get_templates(template_ids)
if self.map_id:
self.map_dict = self.get_map()
self.map_keys = self.map_dict.keys()
if self.replace_id:
self.replace_dict = self.get_replace()
self.replace_keys = self.replace_dict.keys()
self.docs = docs
def clean_template(self, template_doc):
attrs_to_delete = [
'_rev', 'type', 'template_type', 'template_autoload', 'name'
]
for attr in attrs_to_delete:
del template_doc[attr]
if (hasattr(self, 'postprocess_tpl')
and callable(self.postprocess_tpl)):
template_doc = self.postprocess_tpl(template_doc)
return template_doc
def get_map(self):
map_dict = self.map_cache()
if map_dict:
return map_dict
try:
map_dict = self.db.get(self.map_id)['map']
self.set_map_cache(map_dict)
except ResourceNotFound:
pass
return map_dict
def get_replace(self):
replace_dict = None
try:
replace_dict = self.db.get(self.replace_id)['replace']
except ResourceNotFound:
pass
return replace_dict
def get_templates(self, template_ids):
docs = []
for doc_id in template_ids:
template_doc = self.clean_template(self.db.get(doc_id))
docs.append(template_doc)
return docs
def get_all_templates(self):
docs = []
for t in self.db.view("%s/template" % (self.settings.couchdb_db)):
template_doc = self.db.get(t['id'])
if (template_doc['template_type'] == self.allowed_template_type
and 'template_autoload' in template_doc
and template_doc['template_autoload'] is True):
docs.append(self.clean_template(template_doc))
# assumption that templates with more keys will be checked first
docs.sort(key=len)
docs.reverse()
return docs
def is_child_of(self, parent, child):
def is_child_of_inner(parent, child):
for key, value in child.iteritems():
if key in parent and (parent[key] == value or key == '_id'):
yield True
else:
yield False
is_child = all(is_child_of_inner(parent, child))
return is_child
def replace_with_template(self, doc, template):
for key in template.keys():
if key != '_id':
del doc[key]
doc['template_id'] = template['_id']
def process_doc(self, doc):
if self.template_docs:
for template in self.template_docs:
if self.is_child_of(doc, template):
self.replace_with_template(doc, template)
break
if (hasattr(self, 'postprocess_doc')
and callable(self.postprocess_doc)):
doc = self.postprocess_doc(doc)
return doc
def process(self):
for doc in self.docs:
doc[1] = self.process_doc(doc[1])