class ImportServices(object): def __init__(self, settings): self.service_tpl = {"type": "service", "state": "active"} self.docs, self.clients_extcrm_ids = [], {} self.settings = settings self.nr_cols = settings.nr_cols self.server = Server(self.settings.couchdb_uri) self.db = self.server.get_db(self.settings.couchdb_db) self.init_spreadsheet() def init_spreadsheet(self): spreadsheet = ezodf.opendoc(self.settings.src) sheet = spreadsheet.sheets[0] self.rows = sheet.rows() self.header = [ c.value.lower() if hasattr(c.value, "lower") else c.value for c in self.rows.next()[: self.nr_cols] ] def process_row(self, row): for service_type in ["web", "email"]: if row[service_type]: try: service_doc = self.service_tpl.copy() service_doc["_id"] = "service-%s" % self.server.next_uuid() if row["todoyu"] not in self.clients_extcrm_ids: client_id = self.get_or_create_client(row) self.clients_extcrm_ids[row["todoyu"]] = client_id else: client_id = self.clients_extcrm_ids[row["todoyu"]] service_doc["service_type"] = service_type service_doc["start_date"] = row["start_date"] service_doc["client_id"] = client_id if isinstance(row[service_type], dict): service_doc.update(row[service_type]) else: service_doc["package_type"] = row[service_type] addon_key = "%s_addon_items" % service_type if addon_key in row and row[addon_key]: service_doc["addon_service_items"] = row[addon_key] included_key = "%s_included_items" % service_type if included_key in row and row[included_key]: service_doc["included_service_items"] = row[included_key] self.docs.append((service_doc["_id"], service_doc)) except: print("!!! couldn't import service") pass def process_sheet(self): services = [] i = 0 for row in self.rows: i += 1 if any([cell.value for cell in row[: self.nr_cols]]): cell_values = [] for cell in row[: self.nr_cols]: cell_is_python = hasattr(cell.value, "startswith") and ( cell.value.startswith("{") or cell.value.startswith("[") ) if cell_is_python: value = ast.literal_eval(cell.value) else: value = cell.value cell_values.append(value) row_dict = dict(zip(self.header[: self.nr_cols], cell_values)) services.append(row_dict) else: break return services def _cmd(self, cmd): return "{}/{}".format(self.settings.couchdb_db, cmd) def get_or_create_client(self, row): if row["org_name"]: name = row["org_name"] else: name = "%s %s" % (row["n_given"], row["n_family"]) query_results = self.db.view(self._cmd("client_by_name"), key=name, include_docs=False) if query_results.count() == 1: client_doc = query_results.first()["value"] elif query_results.count() == 0: client_doc = {"type": "client"} client_doc["_id"] = "client-%s" % self.server.next_uuid() else: raise client_doc["is_billable"] = 1 client_doc["extcrm_id"] = row["todoyu"] client_doc["name"] = name self.docs.append((client_doc["_id"], client_doc)) return client_doc["_id"] def create_docs(self): self.rows = self.process_sheet() self.todoyu = Todoyu(self.settings) self.dest = "{}/services_json".format(os.path.dirname(self.settings.src)) for row in self.rows: self.process_row(row) def create_files(self): self.create_docs() if os.path.exists(self.dest): shutil.rmtree(self.dest) os.mkdir(self.dest) json_files = FilesForCouch(self.docs, self.dest) json_files.create() if not self.settings.only_files: couch_up = CouchdbUploader( path=self.dest, couchdb_uri=self.settings.couchdb_uri, couchdb_db=self.settings.couchdb_db ) for fname in os.listdir(self.dest): couch_up.put(data="@{}".format(fname), doc_id=fname[:-5])
class IspmanDnsLDIF(LDIFParser): structure_map = { 'a': ['host', 'ip'], 'aaaa': ['host', 'ipv6'], 'cname': ['alias', 'host'], 'mx': ['host', 'priority'], 'txt': ['name', 'txt'], } def __init__(self, input, output, settings, clients_ldif=None, editor=None): LDIFParser.__init__(self, input) self.domains = {} self.domains_lookup = ( clients_ldif.domains_lookup if clients_ldif else None ) self.editor = editor self.a_record_ips = set([]) self.a_record_hosts = {} self.server = Server(settings.couchdb_uri) self.db = self.server.get_db(settings.couchdb_db) def handle(self, dn, entry): if dn.startswith('relativeDomainName='): domain = ".".join( [dc.split('=')[1] for dc in dn.split(',') if dc.startswith('dc=')] ) domain = domain.decode("utf-8").encode("iso8859-1") if domain not in self.domains: self.add_domain(domain) def cname(entry): self.domains[domain]['cname'].append( {'alias': entry['relativeDomainName'][0].strip(), 'host': entry['cNAMERecord'][0].strip()} ) def a(entry): host = entry['relativeDomainName'][0].strip() ip = entry['aRecord'][0].strip() self.domains[domain]['a'].append( {'host': host, 'ip': ip} ) self.a_record_ips.add(ip) full_host = "%s.%s" % (host, domain) if ip in self.a_record_hosts: self.a_record_hosts[ip].append(full_host) else: self.a_record_hosts[ip] = [full_host] def mx(entry): entry = entry['mXRecord'][0].split(' ') self.domains[domain]['mx'].append( {'host': entry[1].strip(), 'priority': entry[0]} ) def ns(entry): nameserver = entry['nSRecord'][0] self.domains[domain]['nameservers'].append(nameserver) self.domains[domain]['nameservers'].sort() if "cNAMERecord" in dn: cname(entry) elif "aRecord" in dn: a(entry) elif "mXRecord" in dn: mx(entry) elif "nSRecord" in dn: ns(entry) def add_domain(self, domain): if domain not in self.domains: next_uuid = self.server.next_uuid() self.domains[domain] = { '_id': 'domain-%s' % next_uuid, 'state': 'new', 'type': 'domain', 'domain': domain, 'a': [], 'cname': [], 'mx': [], 'nameservers': [], } if self.domains_lookup: self.domains[domain]['client_id'] = ( self.domains_lookup[domain] ) if self.editor: self.domains[domain]['editor'] = self.editor
try: del doc["_id"] del doc["_rev"] except: pass for num in numDocs: results = [] try: for ii in range(6): docs = [] # print 'getting uuid' time.sleep(2) for i in range(num): newdoc = copy.deepcopy(doc) newdoc["_id"] = s.next_uuid(num) # print newdoc['_id'] docs.append(newdoc) dumbdoc = db["run_ll20f003_011_kdatascript"] time.sleep(4) start = datetime.datetime.now() # print start db.bulk_save(docs) del docs stop = datetime.datetime.now() td = stop - start totalsec = (td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10.0 ** 6) / 10.0 ** 6 # print num, 'file puts in ', totalsec, ' seconds', float(num)/float(totalsec), 'docs per second' results.append(str(totalsec)) except:
class ClientServerTestCase(unittest.TestCase): def setUp(self): self.couchdb = CouchdbResource() self.Server = Server() def tearDown(self): try: del self.Server["couchdbkit_test"] del self.Server["couchdbkit/test"] except: pass def testGetInfo(self): info = self.Server.info() self.assertIn("version", info) def testCreateDb(self): res = self.Server.create_db("couchdbkit_test") self.assertIsInstance(res, Database) all_dbs = self.Server.all_dbs() self.assertIn("couchdbkit_test", all_dbs) del self.Server["couchdbkit_test"] res = self.Server.create_db("couchdbkit/test") self.assertIn("couchdbkit/test", self.Server.all_dbs()) del self.Server["couchdbkit/test"] def testGetOrCreateDb(self): # create the database gocdb = self.Server.get_or_create_db("get_or_create_db") self.assertEqual(gocdb.dbname, "get_or_create_db") self.assertIn("get_or_create_db", self.Server) self.Server.delete_db("get_or_create_db") # get the database (already created) self.assertNotIn("get_or_create_db", self.Server) db = self.Server.create_db("get_or_create_db") self.assertIn("get_or_create_db", self.Server) gocdb = self.Server.get_or_create_db("get_or_create_db") self.assertEqual(db.dbname, gocdb.dbname) self.Server.delete_db("get_or_create_db") def testCreateInvalidDbName(self): def create_invalid(): res = self.Server.create_db("123ab") self.assertRaises(ValueError, create_invalid) def testServerLen(self): res = self.Server.create_db("couchdbkit_test") self.assertGreaterEqual(len(self.Server), 1) self.assertTrue(self.Server) del self.Server["couchdbkit_test"] def testServerContain(self): res = self.Server.create_db("couchdbkit_test") self.assertIn("couchdbkit_test", self.Server) del self.Server["couchdbkit_test"] def testGetUUIDS(self): uuid = self.Server.next_uuid() self.assertIsInstance(uuid, basestring) self.assertEqual(len(self.Server._uuids), 999) uuid2 = self.Server.next_uuid() self.assertNotEqual(uuid, uuid2) self.assertEqual(len(self.Server._uuids), 998)