def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.http.add_credentials(username, password) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.loads(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.loads(payload) del doc['_rev'] print>>sys.stderr, 'Loading document %r' % docid try: db[docid] = doc except Exception, e: if not ignore_errors: raise print>>sys.stderr, 'Error: %s' % e
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = (username, password) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.decode(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.decode(payload) del doc['_rev'] print >> sys.stderr, 'Loading document %r' % docid try: db[docid] = doc except Exception as e: if not ignore_errors: raise print >> sys.stderr, 'Error: %s' % e
def load_multipart_file(fn): fh = file(fn, 'r') #codecs.open(fn, 'r', 'utf-8') for headers, is_multipart, payload in read_multipart(fh): doc = cdb_json.decode(payload) #print "Reading speech", doc.get('speech_id'), "..." speech = Speech(doc) yield speech #speeches.append() fh.close()
def test_unicode_headers(self): # http://code.google.com/p/couchdb-python/issues/detail?id=179 dump = u'''Content-Type: multipart/mixed; boundary="==123456789==" --==123456789== Content-ID: =?utf-8?b?5paH5qGj?= Content-Length: 63 Content-MD5: Cpw3iC3xPua8YzKeWLzwvw== Content-Type: application/json {"_rev": "3-bc27b6930ca514527d8954c7c43e6a09", "_id": "文档"} ''' parts = multipart.read_multipart(StringIO(dump.encode('utf-8'))) for headers, is_multipart, payload in parts: self.assertEqual(headers['content-id'], u'文档') break
def test_unicode_headers(self): # http://code.google.com/p/couchdb-python/issues/detail?id=179 dump = u'''Content-Type: multipart/mixed; boundary="==123456789==" --==123456789== Content-ID: =?utf-8?b?5paH5qGj?= Content-Length: 63 Content-MD5: Cpw3iC3xPua8YzKeWLzwvw== Content-Type: application/json {"_rev": "3-bc27b6930ca514527d8954c7c43e6a09", "_id": "文档"} ''' parts = multipart.read_multipart(StringIO(dump.encode('utf-8'))) for headers, is_multipart, payload in parts: self.assertEqual(headers['content-id'], u'文档') break
def test_flat(self): text = b'''\ Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: "1-4229094393" { "_id": "bar", "_rev": "1-4229094393" } --===============1946781859== Content-Type: application/json Content-ID: foo ETag: "1-2182689334" { "_id": "foo", "_rev": "1-2182689334", "something": "cool" } --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) if num == 0: self.assertEqual('bar', headers['content-id']) self.assertEqual('"1-4229094393"', headers['etag']) self.assertEqual( b'{\n "_id": "bar",\n ' b'"_rev": "1-4229094393"\n}', payload) elif num == 1: self.assertEqual('foo', headers['content-id']) self.assertEqual('"1-2182689334"', headers['etag']) self.assertEqual( b'{\n "_id": "foo",\n "_rev": "1-2182689334",' b'\n "something": "cool"\n}', payload) num += 1 self.assertEqual(num, 2)
def test_flat(self): text = b'''\ Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: "1-4229094393" { "_id": "bar", "_rev": "1-4229094393" } --===============1946781859== Content-Type: application/json Content-ID: foo ETag: "1-2182689334" { "_id": "foo", "_rev": "1-2182689334", "something": "cool" } --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) if num == 0: self.assertEqual('bar', headers['content-id']) self.assertEqual('"1-4229094393"', headers['etag']) self.assertEqual(b'{\n "_id": "bar",\n ' b'"_rev": "1-4229094393"\n}', payload) elif num == 1: self.assertEqual('foo', headers['content-id']) self.assertEqual('"1-2182689334"', headers['etag']) self.assertEqual(b'{\n "_id": "foo",\n "_rev": "1-2182689334",' b'\n "something": "cool"\n}', payload) num += 1 self.assertEqual(num, 2)
def test_flat(self): text = '''MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: 2235038212 { "_id": "bar", "_rev": "2235038212" } --===============1946781859== Content-Type: application/json Content-ID: foo ETag: 2779219239 { "_id": "foo", "_rev": "2779219239", "something": "cool" } --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) if num == 0: self.assertEqual('bar', headers['content-id']) self.assertEqual('2235038212', headers['etag']) self.assertEqual('{\n "_id": "bar", \n ' '"_rev": "2235038212"\n}', payload) elif num == 1: self.assertEqual('foo', headers['content-id']) self.assertEqual('2779219239', headers['etag']) self.assertEqual('{\n "_id": "foo", \n "_rev": "2779219239",' ' \n "something": "cool"\n}', payload) num += 1 self.assertEqual(num, 2)
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = (username, password) hostname = Is_Server(dburl, db.name) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] time.sleep(1) if '-db' in db.name: if 'credential' in docid or 'flow' in docid or 'setting' in docid or 'functions' in docid: docid = Rename_docid(docid, hostname) obj = db.get(docid) if obj == None: new_doc = {'_id': docid} db.save(new_doc) obj = db.get(docid) if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.decode(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload).decode('ascii'), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.decode(payload) doc['_rev'] = obj['_rev'] doc['_id'] = obj['_id'] print('Loading document %r' % docid, file=sys.stderr) try: db[docid] = doc except Exception as e: if not ignore_errors: raise print('Error: %s' % e, file=sys.stderr)
def test_nested(self): text = b'''\ Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: "1-4229094393" { "_id": "bar", "_rev": "1-4229094393" } --===============1946781859== Content-Type: multipart/mixed; boundary="===============0909101126==" Content-ID: foo ETag: "1-919589747" --===============0909101126== Content-Type: application/json { "_id": "foo", "_rev": "1-919589747", "something": "cool" } --===============0909101126== Content-Type: text/plain Content-ID: mail.txt Hello, friends. How are you doing? Regards, Chris --===============0909101126==-- --===============1946781859== Content-Type: application/json Content-ID: baz ETag: "1-3482142493" { "_id": "baz", "_rev": "1-3482142493" } --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: if num == 0: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) self.assertEqual('bar', headers['content-id']) self.assertEqual('"1-4229094393"', headers['etag']) self.assertEqual(b'{\n "_id": "bar", \n ' b'"_rev": "1-4229094393"\n}', payload) elif num == 1: self.assertEqual(is_multipart, True) self.assertEqual('foo', headers['content-id']) self.assertEqual('"1-919589747"', headers['etag']) partnum = 0 for headers, is_multipart, payload in payload: self.assertEqual(is_multipart, False) if partnum == 0: self.assertEqual('application/json', headers['content-type']) self.assertEqual(b'{\n "_id": "foo", \n "_rev": ' b'"1-919589747", \n "something": ' b'"cool"\n}', payload) elif partnum == 1: self.assertEqual('text/plain', headers['content-type']) self.assertEqual('mail.txt', headers['content-id']) self.assertEqual(b'Hello, friends.\nHow are you doing?' b'\n\nRegards, Chris', payload) partnum += 1 elif num == 2: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) self.assertEqual('baz', headers['content-id']) self.assertEqual('"1-3482142493"', headers['etag']) self.assertEqual(b'{\n "_id": "baz", \n ' b'"_rev": "1-3482142493"\n}', payload) num += 1 self.assertEqual(num, 3)
def test_nested(self): text = b'''\ Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: "1-4229094393" { "_id": "bar", "_rev": "1-4229094393" } --===============1946781859== Content-Type: multipart/mixed; boundary="===============0909101126==" Content-ID: foo ETag: "1-919589747" --===============0909101126== Content-Type: application/json { "_id": "foo", "_rev": "1-919589747", "something": "cool" } --===============0909101126== Content-Type: text/plain Content-ID: mail.txt Hello, friends. How are you doing? Regards, Chris --===============0909101126==-- --===============1946781859== Content-Type: application/json Content-ID: baz ETag: "1-3482142493" { "_id": "baz", "_rev": "1-3482142493" } --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: if num == 0: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) self.assertEqual('bar', headers['content-id']) self.assertEqual('"1-4229094393"', headers['etag']) self.assertEqual( b'{\n "_id": "bar", \n ' b'"_rev": "1-4229094393"\n}', payload) elif num == 1: self.assertEqual(is_multipart, True) self.assertEqual('foo', headers['content-id']) self.assertEqual('"1-919589747"', headers['etag']) partnum = 0 for headers, is_multipart, payload in payload: self.assertEqual(is_multipart, False) if partnum == 0: self.assertEqual('application/json', headers['content-type']) self.assertEqual( b'{\n "_id": "foo", \n "_rev": ' b'"1-919589747", \n "something": ' b'"cool"\n}', payload) elif partnum == 1: self.assertEqual('text/plain', headers['content-type']) self.assertEqual('mail.txt', headers['content-id']) self.assertEqual( b'Hello, friends.\nHow are you doing?' b'\n\nRegards, Chris', payload) partnum += 1 elif num == 2: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) self.assertEqual('baz', headers['content-id']) self.assertEqual('"1-3482142493"', headers['etag']) self.assertEqual( b'{\n "_id": "baz", \n ' b'"_rev": "1-3482142493"\n}', payload) num += 1 self.assertEqual(num, 3)
def test_nested(self): text = '''MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============1946781859==" --===============1946781859== Content-Type: application/json Content-ID: bar ETag: 2235038212 { "_id": "bar", "_rev": "2235038212" } --===============1946781859== Content-Type: multipart/mixed; boundary="===============0909101126==" Content-ID: foo ETag: 2779219239 --===============0909101126== Content-Type: application/json { "_id": "foo", "_rev": "2779219239", "something": "cool" } --===============0909101126== Content-Type: text/plain Content-ID: mail.txt Hello, friends. How are you doing? Regards, Chris --===============0909101126==-- --===============1946781859==-- ''' num = 0 parts = multipart.read_multipart(StringIO(text)) for headers, is_multipart, payload in parts: if num == 0: self.assertEqual(is_multipart, False) self.assertEqual('application/json', headers['content-type']) self.assertEqual('bar', headers['content-id']) self.assertEqual('2235038212', headers['etag']) self.assertEqual('{\n "_id": "bar", \n ' '"_rev": "2235038212"\n}', payload) elif num == 1: self.assertEqual(is_multipart, True) self.assertEqual('foo', headers['content-id']) self.assertEqual('2779219239', headers['etag']) partnum = 0 for headers, is_multipart, payload in payload: self.assertEqual(is_multipart, False) if partnum == 0: self.assertEqual('application/json', headers['content-type']) self.assertEqual('{\n "_id": "foo", \n "_rev": ' '"2779219239", \n "something": ' '"cool"\n}', payload) elif partnum == 1: self.assertEqual('text/plain', headers['content-type']) self.assertEqual('mail.txt', headers['content-id']) self.assertEqual('Hello, friends.\nHow are you doing?' '\n\nRegards, Chris', payload) partnum += 1 num += 1 self.assertEqual(num, 2)