def write_file(self, fullFileName):
"""10/24/08, called by ChapterListFormHandle.py."""
if self.chapterList == None or self.chapterList == {}:
#
# then we remove the chapter list file.
#
ryw.db_print_info_browser('ChapterList.write_file: ' + \
'chapterList is empty.', 41)
if ryw.cleanup_path(fullFileName,
'ChapterList.write_file'):
return True
ryw.give_bad_news('ChapterList.write_file:' +
'failed to remove file', logging.error)
return False
try:
su.pickdump(self.chapterList, fullFileName)
except:
ryw.give_bad_news('ChapterList.write_file: pickdump failed: ' +
fullFileName, logging.error)
return False
ryw.db_print_info_browser('ChapterList.write_file success: ' +
fullFileName, 41)
return True
def attempt_make_tmpdir():
"""attempt to make a temporary directory.
returns the directory name made."""
try:
resources = su.parseKeyValueFile(os.path.join(RepositoryRoot, "Resources.txt"))
resTmpin = resources["tmpin"]
except:
give_bad_news("fatal_error: failed to read Resources.txt", logging.critical)
return None
tmpdir = ryw_upload.attempt_make_tmpdir(resTmpin)
ryw.db_print_info_browser("attempt_make_tmpdir: " + repr(tmpdir), 27)
return tmpdir
def process_date_time_str(dateTimeStr):
ryw.db_print_info_browser('process_date_time_str: input: ' +
dateTimeStr, 99)
dateTimeStr = dateTimeStr.replace(' ', ',')
dateTimeStr = dateTimeStr.replace('-', ',')
dateTimeStr = dateTimeStr.replace(':', ',')
dateTimeStr = dateTimeStr + ',0'
#
# ugly hack to fix bug for chopping leading zeros.
#
dateTimeStr = dateTimeStr.replace(',,', ',0,')
dateTimeStr = 'datetime.datetime(' + dateTimeStr + ')'
ryw.db_print_info_browser('process_date_time_str: after replacement: ' +
dateTimeStr, 99)
try:
dateTime = eval(dateTimeStr)
#logging.debug('process_date_time: ' + repr(dateTime))
ryw.db_print_info_browser('process_date_time_str: eval success: ' +
repr(dateTime), 99)
return repr(dateTime)
except:
ryw.db_print_info_browser('process_date_time_str: '+
'eval failed!!!!!', 99)
ryw.give_bad_news('ryw_upload.process_date_time_str: eval failed: ' +
repr(dateTimeStr), logging.error)
return None
def create_aux_dir(tmpdir):
auxDir = tmpdir+'_AUXI'
try:
su.createdirpath(auxDir)
ryw.db_print_info_browser('create_aux_dir: ' + auxDir, 27)
for auxType in AUX_FIELD_TYPES:
auxTypeDir = os.path.join(auxDir, auxType)
su.createdirpath(auxTypeDir)
logging.debug('create_aux_dir: ' + auxTypeDir)
except:
ryw.give_bad_news('create_aux_dir failed: ' + auxDir, logging.critical)
return (False, None)
return (True, auxDir)
def write_meta(metaPath, meta, paths):
#
# if this fails, we need to restore the old meta file and old done flag.
#
try:
su.pickdump(meta, metaPath)
ryw.db_print_info_browser('write_meta: ' + repr(meta), 101)
except:
ryw.give_bad_news('write_meta: su.pickdump failed: ' + metaPath,
logging.critical)
restore_old_meta(paths, metaPath)
return False
logging.debug('write_meta: metadata successfully written: ' + metaPath)
return True
def output_text(outputFile, meta, firstLine=False):
"""mirrors ryw_view.get_auto_hindi_translation_string()."""
f = outputFile
count = 0
ryw.db_print_info_browser("\n", 89)
for type in ["title", "description"]:
if not meta.has_key(type):
continue
allText = meta[type]
textList = ryw.split_content(allText)
for text in textList:
md5sum = ryw.md5sum(text)
#
# if it's already in the dictionary, don't bother.
#
autoDict = google_english2hindi.GOOGLE_AUTO_DICT
if autoDict.has_key(md5sum):
ryw.db_print_info_browser("already found in dictionary: " + md5sum, 89)
continue
text = text.replace('"', '\\"')
# text = text.replace("^M", "")
outputStr = '"' + md5sum + '"' + " => " + '"' + text + '"'
try:
if not firstLine:
f.write(",")
f.write("\n")
f.write(outputStr)
count += 1
ryw.db_print2(outputStr, 89)
firstLine = False
except:
ryw.give_bad_news("output_text: failed to write line: " + outputStr, logging.error)
return (False, count)
return (True, count)
def dos2unix(fileName):
"""called during reformant_ntfs.py."""
commandPath = "/usr/bin/dos2unix"
if not os.path.exists(commandPath):
ryw.give_bad_news(
"ryw_linux.dos2unix: dos2unix doesn't exist. " + "should fix this: apt-get install tofrodos.", logging.error
)
return False
command = commandPath + " " + fileName
ret = subprocess.call(command, shell=True)
if ret != 0:
ryw.give_bad_news("ryw_linux.dos2unix: error in running dos2unix: " + fileName, logging.error)
return False
else:
ryw.db_print_info_browser("ryw_linux.dos2unix: dos2unix success: " + fileName, 69)
return True
def launch_explorer(path):
"""from explorer.launchExplorer()"""
#
# If I don't redirect stdout and stderr,
# this stuff gets sent to the browser window.
#
pair = stdio_logs_open()
if pair:
stdout = pair[0]
stderr = pair[1]
else:
stdout = None
stderr = None
if os.path.exists("/usr/local/bin/rox"):
ryw.db_print_info_browser("launch_explorer: found rox.", 91)
command = "/usr/local/bin/rox "
elif os.path.exists("/usr/bin/thunar"):
ryw.db_print_info_browser("launch_explorer: found thunar.", 91)
command = "/usr/bin/thunar --display :0.0 "
else:
ryw.give_bad_news("launch_explorer: can't find an explorer.", logging.error)
return False
command = command + path + " &"
ryw.db_print_info_browser("launch_explorer: command is: " + command, 91)
ret = subprocess.call(command, shell=True, stdout=stdout, stderr=stderr, env={"DISPLAY": ":0.0"})
stdio_logs_close(pair)
if ret != 0:
ryw.give_bad_news("launch_explorer: error in launching explorer.", logging.error)
return False
return True
def read_list_and_merge(self, fullFileName):
"""10/24/08: called by ChapterListForm.py."""
if not self.read_file(fullFileName):
ryw.give_bad_news('read_list_and_merge: read_file failed: ' +
fullFileName, logging.error)
return False
if self.chapterList == None or self.chapterList == {}:
return True
for objstr,chapterStr in self.chapterList.iteritems():
if not self.chapterDict.has_key(objstr):
ryw.db_print_info_browser(
'read_list_and_merge: objstr in chapter ' +
'list but not in meta list: ' + objstr, 41)
continue
self.chapterDict[objstr][2] = chapterStr
return True
def main():
name = ShowQueue.init_log('DisplayVersions')
ryw_view.print_header_logo()
print '<TITLE>Display All Versions</TITLE>'
form = cgi.FieldStorage()
objID = form.getfirst('objid', '')
if not objID:
ryw.give_bad_news(
'DisplayVersions: no objID given.', logging.error)
exit_now(1)
else:
ryw.db_print_info_browser('DisplayVersions: objID = ' + objID, 32)
success,searchFile,reverseLists = \
ReverseLists.open_searchfile_reverselists(
'DisplaySelection.main:')
if not success:
exit_now(1)
print_title()
success,allVersions = searchFile.get_all_version_meta(objID)
if not success:
ryw.give_bad_news(
'DisplayVersion: searchFile.get_all_version_meta failed: ' +
objID, logging.error)
exit_now(1)
numVers = len(allVersions)
if numVers < 1:
ryw.give_bad_news(
'DisplayVersions: no version found.', logging.error)
exit_now(1)
metaList = allVersions.values()
ryw.db_print_info_browser('DisplayVersions: ' + repr(metaList), 32)
metaList = ryw.sortmeta(metaList)
displayObject = ryw_view.DisplayObject(
RepositoryRoot,
calledByVillageSide = False,
missingFileFunc = Browse.reqDownloadFunc,
searchFile = searchFile,
reverseLists = reverseLists)
print "<BR><B><FONT SIZE=2>Displaying " + str(numVers) + \
" version(s) of the object.</FONT></B>"
displayObject.begin_print()
for meta in metaList:
displayObject.show_an_object_compact(meta)
displayObject.end_print()
ryw_view.print_footer()
reverseLists.done()
searchFile.done()
def main():
name = print_header()
form = cgi.FieldStorage()
WebUpload_ryw.setup_logging()
#
# get objstr.
#
success,objID,version = ryw.get_obj_str2(form)
if not success:
ryw.give_bad_news('CloneVersion: failed to get objstr.',
logging.critical)
ryw_upload.quick_exit(1)
message = 'CloneVersion: ' + objID + '#' + str(version)
logging.info(message)
ryw.db_print2("<BR>" + message + "<BR>", 23);
#
# open search file.
#
success,searchFile = ryw.open_search_file(
'CloneVerson',
os.path.join(RepositoryRoot, 'WWW', 'logs'),
'upload.log',
os.path.join(RepositoryRoot, 'SearchFile'),
False)
if not success:
if searchFile:
searchFile.done()
ryw.give_bad_news('CloneVersion: ' +
'open search file failed. ',
logging.critical)
ryw_upload.quick_exit(1)
else:
ryw.db_print2("search file opened." + "<BR>", 23);
#
# get meta and paths.
#
success,paths,meta = DisplaySelection.get_all_paths(
objID, version, skipLock=False, searchFile=searchFile,
allowNullSearchFile=False)
if success:
ryw.db_print_info_browser('CloneVersion: paths: ' + repr(paths), 24)
ryw.db_print_info_browser('CloneVersion: meta: ' + repr(meta), 29)
else:
ryw_upload.quick_exit(1)
if (searchFile):
searchFile.done()
#
# we do want to clone the data if it were a list.
#
isList = ryw_meta.isList(meta)
if isList:
dataPath = paths[0]
selName = DisplaySelection.get_sel_name(dataPath)
if not selName:
ryw.give_bad_news(
'CloneVersion: isList but failed to get selection name.',
logging.error)
ryw_upload.quick_exit(1)
selPath = os.path.join(dataPath, selName)
else:
selPath,selName = None,None
#
# change meta.
#
meta = change_meta(meta, name)
#
# deal with auxi dir.
#
originalAuxiDir = paths[2]
newAuxiDir = None
tmpdir = None
if os.path.exists(originalAuxiDir):
tmpdir = WebUpload_ryw.attempt_make_tmpdir()
if not tmpdir:
ryw_upload.quick_exit(1)
newAuxiDir = os.path.join(tmpdir, '_AUXI')
message = 'CloneVersion: shutil.copytree(): ' + \
originalAuxiDir + ' -> ' + newAuxiDir
try:
shutil.copytree(originalAuxiDir, newAuxiDir)
except:
ryw.give_bad_news('failed: ' + message, logging.critical)
ryw_upload.cleanup_and_exit(tmpdir, None, None, 1)
ryw.db_print_info_browser(message, 29)
#
# Now try to put a new object in the repository.
# note that the version number will be incremented.
#
# "selPath" used to be just None.
# when I added cloning list, I'm just using this to pass in the
# path name of the selection file.
#
if not WebUpload_ryw.try_upload_object(meta, selPath, newAuxiDir,
cloneVersion=True):
ryw_upload.cleanup_and_exit(tmpdir, None, None, 1)
searchFile = WebUpload_ryw.show_one_server_object(meta)
searchFile.done()
ryw_upload.cleanup_and_exit(tmpdir, None, None, 0,
successMessage = 'clone version completed.')
def uploadobject(values, data, auxDir, hasVersion = False,
cloneVersion=False):
"""update metadata file.
upload object into object store.
create view path."""
logging.debug('UploadObject.uploadobject entered.')
ryw.db_print2('uploadobject: values are: ' + repr(values), 57)
##################################################
# check attributes.
##################################################
success,resources = check_attributes(values)
if not success:
return False
objectid = values['id']
#
# not strictly necessary, but I'm doing this to hardwire all
# places of gettting objectstoreroot.
#
#objectstoreroot = values['objectstore']
objectstoreroot = ryw.hard_wired_objectstore_root()
logging.debug('uploadobject: passed check_attributes: '+
objectstoreroot + ' ' + objectid)
##################################################
# give metadata to the search server.
##################################################
success,version = add_to_search_file(values, hasVersion, cloneVersion)
if not success:
remove_from_search_file(objectid, version)
return False
else:
ryw.db_print_info_browser('uploadobject: ' + objectid + '#' +
str(version), 29)
##################################################
# put stuff into the object store.
##################################################
success,parent,datapath,metapath,auxpath,donepath = \
add_to_object_store(values, data, objectstoreroot, objectid, version,
auxDir, cloneVersion)
if not success:
remove_from_search_file(objectid, version)
remove_from_object_store(parent, datapath, metapath, auxpath, donepath)
return False
##################################################
# create view path.
##################################################
if not cloneVersion:
success,path = create_view_path(values, resources, objectid, version)
#ryw.give_news('uploadobject: introducing error...', logging.debug)
#success = False
if not success:
remove_from_search_file(objectid,version)
remove_from_object_store(parent, datapath, metapath,
auxpath, donepath)
remove_view_path(path)
return False
# logging.debug('just for testing...')
# remove_view_path(path)
# return False
return True
def add_to_object_store(values, data, objectstoreroot, objectid, version,
auxDir, cloneVersion=False):
"""put (DATA, META, DONE) in the object store."""
##################################################
# check free disk space.
##################################################
if not cloneVersion and \
not ryw_uploadObj.check_free_space(values, objectstoreroot):
return (False, None, None, None, None, None)
##################################################
# acquire prefix.
##################################################
try:
prefix = objectstore.nameversiontoprefix(objectstoreroot, objectid,
version)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: objectstore.nameversiontoprefix() failed.',
logging.critical)
return (False, None, None, None, None, None)
datapath = prefix + '_DATA'
metapath = prefix + '_META'
donepath = prefix + '_DONE'
auxpath = prefix + '_AUXI'
##################################################
# create parent directory.
##################################################
parent = None
try:
parent = su.createparentdirpath(prefix)
logging.debug('add_to_object_store: created parent dir: ' + parent)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: su.createparentdirpath() failed.',
logging.critical)
return (False, parent, None, None, None, None)
if os.path.exists(donepath):
ryw.give_news(
'add_to_object_store: destination object already exists: ' +
donepath, logging.warning)
return (True, parent, datapath, metapath, auxpath, donepath)
datapath = os.path.abspath(os.path.normpath(datapath))
if (not cloneVersion):
data = os.path.abspath(os.path.normpath(data))
logging.debug('add_to_object_store: ' + datapath + ' ' +
metapath + ' ' + donepath + '. source: ' + data)
auxpath = os.path.abspath(os.path.normpath(auxpath))
##################################################
# move data into the object store.
##################################################
if cloneVersion:
try:
os.mkdir(datapath)
ryw.db_print_info_browser('add_to_object_store: mkdir: ' +
datapath, 29)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: cloneVersion mkdir: ' +
datapath, logging.critical)
return (False, parent, datapath, None, None, None)
#
# now deal with cloning a list, if necessary.
#
isList = ryw_meta.isList(values)
if isList:
baseName = os.path.basename(data)
dirName = os.path.dirname(data)
ryw.db_print2('UploadObject.add_to_object_store: basename: ' +
baseName, 44)
ryw.db_print2('UploadObject.add_to_object_store: dirname: ' +
dirName, 44)
destPath = os.path.join(datapath, baseName)
srcChapterPath = os.path.join(dirName,
ChapterList.CHAPTER_LIST_NAME)
destChapterPath = os.path.join(datapath,
ChapterList.CHAPTER_LIST_NAME)
try:
shutil.copyfile(data, destPath)
ryw.db_print_info_browser(
'add_to_object_store: ' +
data + ' --> ' + destPath, 36)
if os.path.exists(srcChapterPath):
shutil.copyfile(srcChapterPath, destChapterPath)
ryw.db_print_info_browser(
'UploadObject.add_to_object_store: ' +
srcChapterPath + ' --> ' + destChapterPath, 44)
else:
ryw.db_print_info_browser(
'UploadObject.add_to_object_store: no chapter file.',
44)
except:
ryw.give_bad_news(
'add_to_object_store: failed to copy selection file.',
logging.critical)
return (False, parent, datapath, None, None, None)
elif os.path.isdir(data):
try:
mymove(data, datapath)
logging.debug('add_to_object_store: moved dir: ' + data + ' ' +
datapath)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: move dir failed: ' +
data + ' ' + datapath,
logging.critical)
return (False, parent, datapath, None, None, None)
else:
try:
os.mkdir(datapath)
mymove(data, os.path.join(datapath, os.path.basename(data)))
logging.debug('add_to_object_store: moved file: ' + data + ' ' +
os.path.join(datapath, os.path.basename(data)))
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: move file failed: ' +
data + ' ' + datapath,
logging.critical)
return (False, parent, datapath, None, None, None)
##################################################
# put metadata in the object store.
##################################################
try:
su.pickdump(values, metapath)
logging.debug('add_to_object_store: written metapath: ' + metapath)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: su.pickdump() failed.',
logging.critical)
return (False, parent, datapath, metapath, None, None)
##################################################
# put aux files in the object store.
##################################################
try:
if auxDir and os.path.exists(auxDir) and os.path.isdir(auxDir):
mymove(auxDir, auxpath)
logging.debug('add_to_object_store: moved auxpath: ' + auxpath)
except:
ryw.give_bad_news(
'add_to_object_store: moving aux directory failed: ' + auxDir +
' -> ' + auxpath, logging.critical)
return (False, parent, datapath, metapath, auxpath, None)
##################################################
# put DONE flag in the object store.
##################################################
try:
f = open(donepath, 'w')
f.close()
logging.debug('add_to_object_store: written donepath: ' + donepath)
except:
ryw.give_bad_news(
'fatal_error: add_to_object_store: write DONE file failed.',
logging.critical)
return (False, parent, datapath, metapath, auxpath, donepath)
# TODO: In case of ANY failure, upload request copied to a REJECTED folder which can be looked upon later.
return (True, parent, datapath, metapath, auxpath, donepath)