class ImportServices(object):
def __init__(self, settings):
self.service_tpl = {"type": "service", "state": "active"}
self.docs, self.clients_extcrm_ids = [], {}
self.settings = settings
self.nr_cols = settings.nr_cols
self.server = Server(self.settings.couchdb_uri)
self.db = self.server.get_db(self.settings.couchdb_db)
self.init_spreadsheet()
def init_spreadsheet(self):
spreadsheet = ezodf.opendoc(self.settings.src)
sheet = spreadsheet.sheets[0]
self.rows = sheet.rows()
self.header = [
c.value.lower() if hasattr(c.value, "lower") else c.value for c in self.rows.next()[: self.nr_cols]
]
def process_row(self, row):
for service_type in ["web", "email"]:
if row[service_type]:
try:
service_doc = self.service_tpl.copy()
service_doc["_id"] = "service-%s" % self.server.next_uuid()
if row["todoyu"] not in self.clients_extcrm_ids:
client_id = self.get_or_create_client(row)
self.clients_extcrm_ids[row["todoyu"]] = client_id
else:
client_id = self.clients_extcrm_ids[row["todoyu"]]
service_doc["service_type"] = service_type
service_doc["start_date"] = row["start_date"]
service_doc["client_id"] = client_id
if isinstance(row[service_type], dict):
service_doc.update(row[service_type])
else:
service_doc["package_type"] = row[service_type]
addon_key = "%s_addon_items" % service_type
if addon_key in row and row[addon_key]:
service_doc["addon_service_items"] = row[addon_key]
included_key = "%s_included_items" % service_type
if included_key in row and row[included_key]:
service_doc["included_service_items"] = row[included_key]
self.docs.append((service_doc["_id"], service_doc))
except:
print("!!! couldn't import service")
pass
def process_sheet(self):
services = []
i = 0
for row in self.rows:
i += 1
if any([cell.value for cell in row[: self.nr_cols]]):
cell_values = []
for cell in row[: self.nr_cols]:
cell_is_python = hasattr(cell.value, "startswith") and (
cell.value.startswith("{") or cell.value.startswith("[")
)
if cell_is_python:
value = ast.literal_eval(cell.value)
else:
value = cell.value
cell_values.append(value)
row_dict = dict(zip(self.header[: self.nr_cols], cell_values))
services.append(row_dict)
else:
break
return services
def _cmd(self, cmd):
return "{}/{}".format(self.settings.couchdb_db, cmd)
def get_or_create_client(self, row):
if row["org_name"]:
name = row["org_name"]
else:
name = "%s %s" % (row["n_given"], row["n_family"])
query_results = self.db.view(self._cmd("client_by_name"), key=name, include_docs=False)
if query_results.count() == 1:
client_doc = query_results.first()["value"]
elif query_results.count() == 0:
client_doc = {"type": "client"}
client_doc["_id"] = "client-%s" % self.server.next_uuid()
else:
raise
client_doc["is_billable"] = 1
client_doc["extcrm_id"] = row["todoyu"]
client_doc["name"] = name
self.docs.append((client_doc["_id"], client_doc))
return client_doc["_id"]
def create_docs(self):
self.rows = self.process_sheet()
self.todoyu = Todoyu(self.settings)
self.dest = "{}/services_json".format(os.path.dirname(self.settings.src))
for row in self.rows:
self.process_row(row)
def create_files(self):
self.create_docs()
if os.path.exists(self.dest):
shutil.rmtree(self.dest)
os.mkdir(self.dest)
json_files = FilesForCouch(self.docs, self.dest)
json_files.create()
if not self.settings.only_files:
couch_up = CouchdbUploader(
path=self.dest, couchdb_uri=self.settings.couchdb_uri, couchdb_db=self.settings.couchdb_db
)
for fname in os.listdir(self.dest):
couch_up.put(data="@{}".format(fname), doc_id=fname[:-5])
class IspmanDnsLDIF(LDIFParser):
structure_map = {
'a': ['host', 'ip'],
'aaaa': ['host', 'ipv6'],
'cname': ['alias', 'host'],
'mx': ['host', 'priority'],
'txt': ['name', 'txt'],
}
def __init__(self, input, output, settings, clients_ldif=None, editor=None):
LDIFParser.__init__(self, input)
self.domains = {}
self.domains_lookup = (
clients_ldif.domains_lookup if clients_ldif else None
)
self.editor = editor
self.a_record_ips = set([])
self.a_record_hosts = {}
self.server = Server(settings.couchdb_uri)
self.db = self.server.get_db(settings.couchdb_db)
def handle(self, dn, entry):
if dn.startswith('relativeDomainName='):
domain = ".".join(
[dc.split('=')[1] for dc in dn.split(',')
if dc.startswith('dc=')]
)
domain = domain.decode("utf-8").encode("iso8859-1")
if domain not in self.domains:
self.add_domain(domain)
def cname(entry):
self.domains[domain]['cname'].append(
{'alias': entry['relativeDomainName'][0].strip(),
'host': entry['cNAMERecord'][0].strip()}
)
def a(entry):
host = entry['relativeDomainName'][0].strip()
ip = entry['aRecord'][0].strip()
self.domains[domain]['a'].append(
{'host': host,
'ip': ip}
)
self.a_record_ips.add(ip)
full_host = "%s.%s" % (host, domain)
if ip in self.a_record_hosts:
self.a_record_hosts[ip].append(full_host)
else:
self.a_record_hosts[ip] = [full_host]
def mx(entry):
entry = entry['mXRecord'][0].split(' ')
self.domains[domain]['mx'].append(
{'host': entry[1].strip(),
'priority': entry[0]}
)
def ns(entry):
nameserver = entry['nSRecord'][0]
self.domains[domain]['nameservers'].append(nameserver)
self.domains[domain]['nameservers'].sort()
if "cNAMERecord" in dn:
cname(entry)
elif "aRecord" in dn:
a(entry)
elif "mXRecord" in dn:
mx(entry)
elif "nSRecord" in dn:
ns(entry)
def add_domain(self, domain):
if domain not in self.domains:
next_uuid = self.server.next_uuid()
self.domains[domain] = {
'_id': 'domain-%s' % next_uuid,
'state': 'new',
'type': 'domain',
'domain': domain,
'a': [],
'cname': [],
'mx': [],
'nameservers': [],
}
if self.domains_lookup:
self.domains[domain]['client_id'] = (
self.domains_lookup[domain]
)
if self.editor:
self.domains[domain]['editor'] = self.editor
try:
del doc["_id"]
del doc["_rev"]
except:
pass
for num in numDocs:
results = []
try:
for ii in range(6):
docs = []
# print 'getting uuid'
time.sleep(2)
for i in range(num):
newdoc = copy.deepcopy(doc)
newdoc["_id"] = s.next_uuid(num)
# print newdoc['_id']
docs.append(newdoc)
dumbdoc = db["run_ll20f003_011_kdatascript"]
time.sleep(4)
start = datetime.datetime.now()
# print start
db.bulk_save(docs)
del docs
stop = datetime.datetime.now()
td = stop - start
totalsec = (td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10.0 ** 6) / 10.0 ** 6
# print num, 'file puts in ', totalsec, ' seconds', float(num)/float(totalsec), 'docs per second'
results.append(str(totalsec))
except:
class ClientServerTestCase(unittest.TestCase):
def setUp(self):
self.couchdb = CouchdbResource()
self.Server = Server()
def tearDown(self):
try:
del self.Server["couchdbkit_test"]
del self.Server["couchdbkit/test"]
except:
pass
def testGetInfo(self):
info = self.Server.info()
self.assertIn("version", info)
def testCreateDb(self):
res = self.Server.create_db("couchdbkit_test")
self.assertIsInstance(res, Database)
all_dbs = self.Server.all_dbs()
self.assertIn("couchdbkit_test", all_dbs)
del self.Server["couchdbkit_test"]
res = self.Server.create_db("couchdbkit/test")
self.assertIn("couchdbkit/test", self.Server.all_dbs())
del self.Server["couchdbkit/test"]
def testGetOrCreateDb(self):
# create the database
gocdb = self.Server.get_or_create_db("get_or_create_db")
self.assertEqual(gocdb.dbname, "get_or_create_db")
self.assertIn("get_or_create_db", self.Server)
self.Server.delete_db("get_or_create_db")
# get the database (already created)
self.assertNotIn("get_or_create_db", self.Server)
db = self.Server.create_db("get_or_create_db")
self.assertIn("get_or_create_db", self.Server)
gocdb = self.Server.get_or_create_db("get_or_create_db")
self.assertEqual(db.dbname, gocdb.dbname)
self.Server.delete_db("get_or_create_db")
def testCreateInvalidDbName(self):
def create_invalid():
res = self.Server.create_db("123ab")
self.assertRaises(ValueError, create_invalid)
def testServerLen(self):
res = self.Server.create_db("couchdbkit_test")
self.assertGreaterEqual(len(self.Server), 1)
self.assertTrue(self.Server)
del self.Server["couchdbkit_test"]
def testServerContain(self):
res = self.Server.create_db("couchdbkit_test")
self.assertIn("couchdbkit_test", self.Server)
del self.Server["couchdbkit_test"]
def testGetUUIDS(self):
uuid = self.Server.next_uuid()
self.assertIsInstance(uuid, basestring)
self.assertEqual(len(self.Server._uuids), 999)
uuid2 = self.Server.next_uuid()
self.assertNotEqual(uuid, uuid2)
self.assertEqual(len(self.Server._uuids), 998)
