def post(self, strints=[]):
"""
Cleanup after migration
@param strints : List of tuples (tablename, fieldname) to convert from string to integer
"""
db = self.db
# @ToDo: Do prepops of new tables
# Restore data from backup
folder = "%s/databases/backup" % current.request.folder
db_bak = DAL("sqlite://backup.db", folder=folder, auto_import=True, migrate=False)
for tablename, fieldname in strints:
newtable = db[tablename]
newrows = db(newtable.id > 0).select(newtable.id)
oldtable = db_bak[tablename]
oldrows = db_bak(oldtable.id > 0).select(oldtable.id, oldtable[fieldname])
oldvals = oldrows.as_dict()
for row in newrows:
id = row.id
val = oldvals[id][fieldname]
if not val:
continue
try:
vars = {fieldname: int(val)}
except:
s3_debug("S3Migrate: Unable to convert %s to an integer - skipping" % val)
else:
db(newtable.id == id).update(**vars)
db.commit()
def post(self, strints=[], strbools=[]):
"""
Cleanup after migration
@param strints : List of tuples (tablename, fieldname) to convert from string to integer
@param strbools : List of tuples (tablename, fieldname) to convert from string/integer to bools
"""
db = self.db
# @ToDo: Do prepops of new tables
# Restore data from backup
folder = "%s/databases/backup" % current.request.folder
db_bak = DAL("sqlite://backup.db",
folder=folder,
auto_import=True,
migrate=False)
for tablename, fieldname in strints:
newtable = db[tablename]
newrows = db(newtable.id > 0).select(newtable.id)
oldtable = db_bak[tablename]
oldrows = db_bak(oldtable.id > 0).select(oldtable.id,
oldtable[fieldname])
oldvals = oldrows.as_dict()
for row in newrows:
id = row.id
val = oldvals[id][fieldname]
if not val:
continue
try:
vars = {fieldname: int(val)}
except:
s3_debug(
"S3Migrate: Unable to convert %s to an integer - skipping"
% val)
else:
db(newtable.id == id).update(**vars)
for tablename, fieldname in strbools:
to_bool = self.to_bool
newtable = db[tablename]
newrows = db(newtable.id > 0).select(newtable.id)
oldtable = db_bak[tablename]
oldrows = db_bak(oldtable.id > 0).select(oldtable.id,
oldtable[fieldname])
oldvals = oldrows.as_dict()
for row in newrows:
id = row.id
val = oldvals[id][fieldname]
if not val:
continue
val = to_bool(val)
if val:
vars = {fieldname: val}
db(newtable.id == id).update(**vars)
db.commit()
def document_create_index(document, user_id=None):
import os
from xlrd import open_workbook
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
import sunburnt
document = json.loads(document)
tablename = document["tablename"]
table = s3db[tablename]
id = document["id"]
name = document["name"]
filename = document["filename"]
filename = "%s/%s/uploads/%s" % (os.path.abspath("applications"), \
request.application, filename)
try:
si = sunburnt.SolrInterface(settings.get_base_solr_url())
except:
from s3.s3utils import s3_debug
s3_debug("Connection Error: Solr not connected")
return
extension = os.path.splitext(filename)[1][1:]
if extension == "pdf":
data = os.popen("pdf2txt.py " + filename).read()
elif extension == "doc":
data = os.popen("antiword " + filename).read()
elif extension == "xls":
wb = open_workbook(filename)
data=" "
for s in wb.sheets():
for row in range(s.nrows):
values = []
for col in range(s.ncols):
values.append(str(s.cell(row, col).value))
data = data + ",".join(values) + "\n"
elif extension == "rtf":
doct = Rtf15Reader.read(open(filename))
data = PlaintextWriter.write(doct).getvalue()
else:
data = os.popen("strings " + filename).read()
# The text needs to be in unicode or ascii, with no contol characters
data = str(unicode(data, errors="ignore"))
data = "".join(c if ord(c) >= 32 else " " for c in data)
# Put the data according to the Multiple Fields
# @ToDo: Also, would change this according to requirement of Eden
document = {
"id": str(id), # doc_document.id
"name": data, # the data of the file
"tablename": tablename, # name of the database table
"url": filename, # the encoded file name stored in uploads/
"filename": name, # the filename actually uploaded by the user
"filetype": extension # x.pdf -> pdf is the extension of the file
}
# To delete the index if the document with the same id is available
si.delete(id)
si.commit()
# Add and commit Indices
si.add(document)
si.commit()
# After Indexing, set the value for has_been_indexed to True in the database
db(table.id == id).update(has_been_indexed = True)
