def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.http.add_credentials(username, password)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.loads(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.loads(payload)
del doc['_rev']
print>>sys.stderr, 'Loading document %r' % docid
try:
db[docid] = doc
except Exception, e:
if not ignore_errors:
raise
print>>sys.stderr, 'Error: %s' % e
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.credentials = (username, password)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.decode(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.decode(payload)
del doc['_rev']
print >> sys.stderr, 'Loading document %r' % docid
try:
db[docid] = doc
except Exception as e:
if not ignore_errors:
raise
print >> sys.stderr, 'Error: %s' % e
def load_multipart_file(fn):
fh = file(fn, 'r') #codecs.open(fn, 'r', 'utf-8')
for headers, is_multipart, payload in read_multipart(fh):
doc = cdb_json.decode(payload)
#print "Reading speech", doc.get('speech_id'), "..."
speech = Speech(doc)
yield speech
#speeches.append()
fh.close()
def test_unicode_headers(self):
# http://code.google.com/p/couchdb-python/issues/detail?id=179
dump = u'''Content-Type: multipart/mixed; boundary="==123456789=="
--==123456789==
Content-ID: =?utf-8?b?5paH5qGj?=
Content-Length: 63
Content-MD5: Cpw3iC3xPua8YzKeWLzwvw==
Content-Type: application/json
{"_rev": "3-bc27b6930ca514527d8954c7c43e6a09", "_id": "文档"}
'''
parts = multipart.read_multipart(StringIO(dump.encode('utf-8')))
for headers, is_multipart, payload in parts:
self.assertEqual(headers['content-id'], u'文档')
break
def test_unicode_headers(self):
# http://code.google.com/p/couchdb-python/issues/detail?id=179
dump = u'''Content-Type: multipart/mixed; boundary="==123456789=="
--==123456789==
Content-ID: =?utf-8?b?5paH5qGj?=
Content-Length: 63
Content-MD5: Cpw3iC3xPua8YzKeWLzwvw==
Content-Type: application/json
{"_rev": "3-bc27b6930ca514527d8954c7c43e6a09", "_id": "文档"}
'''
parts = multipart.read_multipart(StringIO(dump.encode('utf-8')))
for headers, is_multipart, payload in parts:
self.assertEqual(headers['content-id'], u'文档')
break
def test_flat(self):
text = b'''\
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: "1-4229094393"
{
"_id": "bar",
"_rev": "1-4229094393"
}
--===============1946781859==
Content-Type: application/json
Content-ID: foo
ETag: "1-2182689334"
{
"_id": "foo",
"_rev": "1-2182689334",
"something": "cool"
}
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
if num == 0:
self.assertEqual('bar', headers['content-id'])
self.assertEqual('"1-4229094393"', headers['etag'])
self.assertEqual(
b'{\n "_id": "bar",\n '
b'"_rev": "1-4229094393"\n}', payload)
elif num == 1:
self.assertEqual('foo', headers['content-id'])
self.assertEqual('"1-2182689334"', headers['etag'])
self.assertEqual(
b'{\n "_id": "foo",\n "_rev": "1-2182689334",'
b'\n "something": "cool"\n}', payload)
num += 1
self.assertEqual(num, 2)
def test_flat(self):
text = b'''\
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: "1-4229094393"
{
"_id": "bar",
"_rev": "1-4229094393"
}
--===============1946781859==
Content-Type: application/json
Content-ID: foo
ETag: "1-2182689334"
{
"_id": "foo",
"_rev": "1-2182689334",
"something": "cool"
}
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
if num == 0:
self.assertEqual('bar', headers['content-id'])
self.assertEqual('"1-4229094393"', headers['etag'])
self.assertEqual(b'{\n "_id": "bar",\n '
b'"_rev": "1-4229094393"\n}', payload)
elif num == 1:
self.assertEqual('foo', headers['content-id'])
self.assertEqual('"1-2182689334"', headers['etag'])
self.assertEqual(b'{\n "_id": "foo",\n "_rev": "1-2182689334",'
b'\n "something": "cool"\n}', payload)
num += 1
self.assertEqual(num, 2)
def test_flat(self):
text = '''MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: 2235038212
{
"_id": "bar",
"_rev": "2235038212"
}
--===============1946781859==
Content-Type: application/json
Content-ID: foo
ETag: 2779219239
{
"_id": "foo",
"_rev": "2779219239",
"something": "cool"
}
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
if num == 0:
self.assertEqual('bar', headers['content-id'])
self.assertEqual('2235038212', headers['etag'])
self.assertEqual('{\n "_id": "bar", \n '
'"_rev": "2235038212"\n}', payload)
elif num == 1:
self.assertEqual('foo', headers['content-id'])
self.assertEqual('2779219239', headers['etag'])
self.assertEqual('{\n "_id": "foo", \n "_rev": "2779219239",'
' \n "something": "cool"\n}', payload)
num += 1
self.assertEqual(num, 2)
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.credentials = (username, password)
hostname = Is_Server(dburl, db.name)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
time.sleep(1)
if '-db' in db.name:
if 'credential' in docid or 'flow' in docid or 'setting' in docid or 'functions' in docid:
docid = Rename_docid(docid, hostname)
obj = db.get(docid)
if obj == None:
new_doc = {'_id': docid}
db.save(new_doc)
obj = db.get(docid)
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.decode(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload).decode('ascii'),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.decode(payload)
doc['_rev'] = obj['_rev']
doc['_id'] = obj['_id']
print('Loading document %r' % docid, file=sys.stderr)
try:
db[docid] = doc
except Exception as e:
if not ignore_errors:
raise
print('Error: %s' % e, file=sys.stderr)
def test_nested(self):
text = b'''\
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: "1-4229094393"
{
"_id": "bar",
"_rev": "1-4229094393"
}
--===============1946781859==
Content-Type: multipart/mixed; boundary="===============0909101126=="
Content-ID: foo
ETag: "1-919589747"
--===============0909101126==
Content-Type: application/json
{
"_id": "foo",
"_rev": "1-919589747",
"something": "cool"
}
--===============0909101126==
Content-Type: text/plain
Content-ID: mail.txt
Hello, friends.
How are you doing?
Regards, Chris
--===============0909101126==--
--===============1946781859==
Content-Type: application/json
Content-ID: baz
ETag: "1-3482142493"
{
"_id": "baz",
"_rev": "1-3482142493"
}
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
if num == 0:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
self.assertEqual('bar', headers['content-id'])
self.assertEqual('"1-4229094393"', headers['etag'])
self.assertEqual(b'{\n "_id": "bar", \n '
b'"_rev": "1-4229094393"\n}', payload)
elif num == 1:
self.assertEqual(is_multipart, True)
self.assertEqual('foo', headers['content-id'])
self.assertEqual('"1-919589747"', headers['etag'])
partnum = 0
for headers, is_multipart, payload in payload:
self.assertEqual(is_multipart, False)
if partnum == 0:
self.assertEqual('application/json',
headers['content-type'])
self.assertEqual(b'{\n "_id": "foo", \n "_rev": '
b'"1-919589747", \n "something": '
b'"cool"\n}', payload)
elif partnum == 1:
self.assertEqual('text/plain', headers['content-type'])
self.assertEqual('mail.txt', headers['content-id'])
self.assertEqual(b'Hello, friends.\nHow are you doing?'
b'\n\nRegards, Chris', payload)
partnum += 1
elif num == 2:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
self.assertEqual('baz', headers['content-id'])
self.assertEqual('"1-3482142493"', headers['etag'])
self.assertEqual(b'{\n "_id": "baz", \n '
b'"_rev": "1-3482142493"\n}', payload)
num += 1
self.assertEqual(num, 3)
def test_nested(self):
text = b'''\
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: "1-4229094393"
{
"_id": "bar",
"_rev": "1-4229094393"
}
--===============1946781859==
Content-Type: multipart/mixed; boundary="===============0909101126=="
Content-ID: foo
ETag: "1-919589747"
--===============0909101126==
Content-Type: application/json
{
"_id": "foo",
"_rev": "1-919589747",
"something": "cool"
}
--===============0909101126==
Content-Type: text/plain
Content-ID: mail.txt
Hello, friends.
How are you doing?
Regards, Chris
--===============0909101126==--
--===============1946781859==
Content-Type: application/json
Content-ID: baz
ETag: "1-3482142493"
{
"_id": "baz",
"_rev": "1-3482142493"
}
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
if num == 0:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
self.assertEqual('bar', headers['content-id'])
self.assertEqual('"1-4229094393"', headers['etag'])
self.assertEqual(
b'{\n "_id": "bar", \n '
b'"_rev": "1-4229094393"\n}', payload)
elif num == 1:
self.assertEqual(is_multipart, True)
self.assertEqual('foo', headers['content-id'])
self.assertEqual('"1-919589747"', headers['etag'])
partnum = 0
for headers, is_multipart, payload in payload:
self.assertEqual(is_multipart, False)
if partnum == 0:
self.assertEqual('application/json',
headers['content-type'])
self.assertEqual(
b'{\n "_id": "foo", \n "_rev": '
b'"1-919589747", \n "something": '
b'"cool"\n}', payload)
elif partnum == 1:
self.assertEqual('text/plain', headers['content-type'])
self.assertEqual('mail.txt', headers['content-id'])
self.assertEqual(
b'Hello, friends.\nHow are you doing?'
b'\n\nRegards, Chris', payload)
partnum += 1
elif num == 2:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
self.assertEqual('baz', headers['content-id'])
self.assertEqual('"1-3482142493"', headers['etag'])
self.assertEqual(
b'{\n "_id": "baz", \n '
b'"_rev": "1-3482142493"\n}', payload)
num += 1
self.assertEqual(num, 3)
def test_nested(self):
text = '''MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============1946781859=="
--===============1946781859==
Content-Type: application/json
Content-ID: bar
ETag: 2235038212
{
"_id": "bar",
"_rev": "2235038212"
}
--===============1946781859==
Content-Type: multipart/mixed; boundary="===============0909101126=="
Content-ID: foo
ETag: 2779219239
--===============0909101126==
Content-Type: application/json
{
"_id": "foo",
"_rev": "2779219239",
"something": "cool"
}
--===============0909101126==
Content-Type: text/plain
Content-ID: mail.txt
Hello, friends.
How are you doing?
Regards, Chris
--===============0909101126==--
--===============1946781859==--
'''
num = 0
parts = multipart.read_multipart(StringIO(text))
for headers, is_multipart, payload in parts:
if num == 0:
self.assertEqual(is_multipart, False)
self.assertEqual('application/json', headers['content-type'])
self.assertEqual('bar', headers['content-id'])
self.assertEqual('2235038212', headers['etag'])
self.assertEqual('{\n "_id": "bar", \n '
'"_rev": "2235038212"\n}', payload)
elif num == 1:
self.assertEqual(is_multipart, True)
self.assertEqual('foo', headers['content-id'])
self.assertEqual('2779219239', headers['etag'])
partnum = 0
for headers, is_multipart, payload in payload:
self.assertEqual(is_multipart, False)
if partnum == 0:
self.assertEqual('application/json',
headers['content-type'])
self.assertEqual('{\n "_id": "foo", \n "_rev": '
'"2779219239", \n "something": '
'"cool"\n}', payload)
elif partnum == 1:
self.assertEqual('text/plain', headers['content-type'])
self.assertEqual('mail.txt', headers['content-id'])
self.assertEqual('Hello, friends.\nHow are you doing?'
'\n\nRegards, Chris', payload)
partnum += 1
num += 1
self.assertEqual(num, 2)
