def main():
if sys.platform == "win32":
print "Warning: test me directly on the server!"
fRecreate = arsutils.fDetectRemoveCmdFlag("-recreate")
fCreate = arsutils.fDetectRemoveCmdFlag("-create")
if len(sys.argv) != 1:
usageAndExit()
try:
conn = getRootConnection()
fExists = FDbExists()
if fExists:
if fRecreate:
print "infoman database already exists."
recreateDb(conn)
else:
print "infoman database already exists. Use -recreate flag to force it's recreation"
else:
if fCreate:
print "infoman database doesn't exist."
createDb(conn)
else:
print "infoman database doesn't exist. Use -create flag to create it"
finally:
deinitDatabase()
def main():
if sys.platform == "win32":
print "Warning: test me directly on the server!"
fRecreate = arsutils.fDetectRemoveCmdFlag("-recreate")
fCreate = arsutils.fDetectRemoveCmdFlag("-create")
if len(sys.argv) != 1:
usageAndExit()
try:
conn = getRootConnection()
fExists = FDbExists()
if fExists:
if fRecreate:
print "infoman database already exists."
recreateDb(conn)
else:
print "infoman database already exists. Use -recreate flag to force it's recreation"
else:
if fCreate:
print "infoman database doesn't exist."
createDb(conn)
else:
print "infoman database doesn't exist. Use -create flag to create it"
finally:
deinitDatabase()
def main():
# _test()
# return
spider = fDetectRemoveCmdFlag("-spider")
reindex = fDetectRemoveCmdFlag("-reindex")
update = fDetectRemoveCmdFlag("-update")
force = fDetectRemoveCmdFlag("-force")
test_query = getRemoveCmdArg("-test")
# undocumented flags just for testing ;)
backup = fDetectRemoveCmdFlag("--backup-index")
use_old_index = fDetectRemoveCmdFlag("--update-old-index")
update_index = fDetectRemoveCmdFlag("--force-index-update")
bump_version = fDetectRemoveCmdFlag("--bump_version")
if test_query:
formats = FORMATS_ALL
if -1 != test_query.find(";"):
test_query, formats = [s.strip() for s in test_query.split(";", 1)]
type = SEARCH_ANY
if -1 != test_query.find(":"):
t, q = [s.strip() for s in test_query.split(":", 1)]
if t in SEARCH_TYPES:
type, test_query = t, q
for doc in _find_proxy(test_query, formats, type):
d = _doc_to_tuple(doc)
if 0 != len(d[1]):
print "%s, \"%s (%s)\"" % (d[2], d[0], d[1])
else:
print "%s, \"%s\"" % (d[2], d[0])
return
if spider and reindex:
update = True
if update:
update_all(force, force_index_update = update_index, update_old_index = use_old_index )
return
if spider:
spider_all(force)
return
if reindex:
reindex_all(use_old_index )
return
if backup:
_backup_index()
return
if bump_version:
_bump_version()
return
print """
usage: ebooks.py (-spider | -reindex | -update | -test "phrase") [-force]
Options:
-spider - performs only incremental spidering of all data
-reindex - only reindexes existing spidered data.
-update - performs spidering and reindexing (only if fresh data was
spidered).
-force - (only with -spider or -update) discards previously spidered data
prior to spidering.
-test - tests searching in the index.
"""
return 1
def main():
global g_dbName
dbs = iPediaServer.getIpediaDbList()
if 0==len(dbs):
print "No databases available"
dbs.sort()
fListDbs = arsutils.fDetectRemoveCmdFlag("-listdbs")
if fListDbs:
for dbName in dbs:
print dbName
sys.exit(0)
dbName=arsutils.getRemoveCmdArg("-db")
if dbName:
if dbName in dbs:
print "Using database '%s'" % dbName
else:
print "Database '%s' doesn't exist" % dbName
print "Available databases:"
for name in dbs:
print " %s" % name
sys.exit(0)
else:
dbName=dbs[-1] # use the latest database
print "Using database '%s'" % dbName
g_dbName = dbName
validateRedirects()
closeConn()
def main():
global g_dbName
dbs = iPediaServer.getIpediaDbList()
if 0 == len(dbs):
print "No databases available"
dbs.sort()
fListDbs = arsutils.fDetectRemoveCmdFlag("-listdbs")
if fListDbs:
for dbName in dbs:
print dbName
sys.exit(0)
dbName = arsutils.getRemoveCmdArg("-db")
if dbName:
if dbName in dbs:
print "Using database '%s'" % dbName
else:
print "Database '%s' doesn't exist" % dbName
print "Available databases:"
for name in dbs:
print " %s" % name
sys.exit(0)
else:
dbName = dbs[-1] # use the latest database
print "Using database '%s'" % dbName
g_dbName = dbName
validateRedirects()
closeConn()
def main():
fDumpCache = arsutils.fDetectRemoveCmdFlag("-dump-cache")
if not fDumpCache:
fDumpCache = arsutils.fDetectRemoveCmdFlag("--dump-cache")
if fDumpCache:
dumpCache()
sys.exit(0)
fileName = arsutils.getRemoveCmdArg("-file")
if None != fileName:
if 1 != len(sys.argv):
usage()
print sys.argv
sys.exit(0)
fo = open(fileName, "rb")
htmlTxt = fo.read()
fo.close()
(resultType, resultBody) = parseGas(htmlTxt, url=fileName, dbgLevel=1)
else:
if 2 != len(sys.argv):
usage()
sys.exit(0)
zipCode = sys.argv[1]
print "zipCode: %s" % zipCode
(resultType, resultBody) = getGasPricesForZip(cookielib.CookieJar(),
zipCode,
dbgLevel=1)
if RETRIEVE_FAILED == resultType:
print "retrieve failed"
if PARSING_FAILED == resultType:
htmlTxt = resultBody[1]
#print htmlTxt
print "parsing failed"
if NO_RESULTS == resultType:
print "no results"
if LOCATION_UNKNOWN == resultType:
print "location unknown"
if GAS_DATA == resultType:
print udfPrettyPrint(resultBody)
def main():
fDumpCache = arsutils.fDetectRemoveCmdFlag("-dump-cache")
if not fDumpCache:
fDumpCache = arsutils.fDetectRemoveCmdFlag("--dump-cache")
if fDumpCache:
dumpCache()
sys.exit(0)
fileName = arsutils.getRemoveCmdArg("-file")
if None != fileName:
if 1 != len(sys.argv):
usage()
print sys.argv
sys.exit(0)
fo = open(fileName, "rb")
htmlTxt = fo.read()
fo.close()
(resultType, resultBody) = parseGas(htmlTxt, url=fileName, dbgLevel=1)
else:
if 2 != len(sys.argv):
usage()
sys.exit(0)
zipCode = sys.argv[1]
print "zipCode: %s" % zipCode
(resultType, resultBody) = getGasPricesForZip(cookielib.CookieJar(), zipCode, dbgLevel=1)
if RETRIEVE_FAILED == resultType:
print "retrieve failed"
if PARSING_FAILED == resultType:
htmlTxt = resultBody[1]
#print htmlTxt
print "parsing failed"
if NO_RESULTS == resultType:
print "no results"
if LOCATION_UNKNOWN == resultType:
print "location unknown"
if GAS_DATA == resultType:
print udfPrettyPrint(resultBody)
if __name__ == "__main__":
serverToUse = None
serverToUse = arsutils.getRemoveCmdArg("-server")
if None == serverToUse:
detectAndSetServerToUse()
else:
if not g_serverList.has_key(serverToUse):
print "server %s is not known. Known servers:" % serverToUse
for serverName in g_serverList.keys():
print " %s" % serverName
sys.exit(0)
else:
g_serverToUse = serverToUse
if arsutils.fDetectRemoveCmdFlag("-raw"):
g_showRawOutput = True
printUsedServer()
unpickleState()
pos = None
for argName in argsInfo.keys():
pos = getIndexSafe(sys.argv, argName)
if None != pos:
break
if None == pos:
usageAndExit()
sys.exit(0)
numArgs = argsInfo[argName][0]
def dumpArticle(fileName,title):
for article in iterArticlesExactTitle(fileName,title):
if not article:
print "couldn't find the body of article %s" % title
return
title = article.getTitle().strip() + "\n"
txt = article.getTxt()
converted = articleconvert.convertArticle(title,txt)
print "TITLE: %s" % title
print "ORIGINAL: %s" % txt
print "CONVERTED: %s" % converted
return
if __name__=="__main__":
fRandom = arsutils.fDetectRemoveCmdFlag("-random")
fDump = arsutils.fDetectRemoveCmdFlag("-dump")
if fDump and fRandom:
print "Can't use -dump and -random at the same time"
usageAndExit()
title = arsutils.getRemoveCmdArg("-title")
fSave = arsutils.fDetectRemoveCmdFlag("-save")
fForceConvert = arsutils.fDetectRemoveCmdFlag("-forceconvert")
if title and fRandom:
print "Can't use -title and -random at the same time"
usageAndExit()
g_prevEmailUpdateTime = curTime
def emailImmediateServerStateChange(prevState, currState):
assert prevState != currState
curTime = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime() )
subject = "InfoMan watchdog for %s:%d, server change state notice on %s" % (SERVER_USER, SERVER_PORT, curTime)
body = "Time: %s\n" % curTime
body += "Server changed the state from %s to %s" % (stateName(prevState), stateName(currState))
sendEmail(subject, body)
if __name__ == '__main__':
state = currentServerState()
print stateName(state)
# -nodeamon option is for testing, so that we see problem in the script code
fNoDemon = arsutils.fDetectRemoveCmdFlag("-nodemon")
if not fNoDemon:
fNoDemon = arsutils.fDetectRemoveCmdFlag("-nodaemon")
if not fNoDemon:
arsutils.daemonize('/dev/null', '/dev/null')
g_serverState = currentServerState()
emailImmediateStartupState(g_serverState)
while True: # the only way to stop me is to kill me
time.sleep(SERVER_STATE_CHECK_FREQUENCY)
state = currentServerState()
if state != g_serverState:
emailImmediateServerStateChange(g_serverState, state)
g_serverState = state
assert prevState != currState
curTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
subject = "InfoMan watchdog for %s:%d, server change state notice on %s" % (
SERVER_USER, SERVER_PORT, curTime)
body = "Time: %s\n" % curTime
body += "Server changed the state from %s to %s" % (stateName(prevState),
stateName(currState))
sendEmail(subject, body)
if __name__ == '__main__':
state = currentServerState()
print stateName(state)
# -nodeamon option is for testing, so that we see problem in the script code
fNoDemon = arsutils.fDetectRemoveCmdFlag("-nodemon")
if not fNoDemon:
fNoDemon = arsutils.fDetectRemoveCmdFlag("-nodaemon")
if not fNoDemon:
arsutils.daemonize('/dev/null', '/dev/null')
g_serverState = currentServerState()
emailImmediateStartupState(g_serverState)
while True: # the only way to stop me is to kill me
time.sleep(SERVER_STATE_CHECK_FREQUENCY)
state = currentServerState()
if state != g_serverState:
emailImmediateServerStateChange(g_serverState, state)
g_serverState = state
def dumpArticle(fileName, title):
for article in iterArticlesExactTitle(fileName, title):
if not article:
print "couldn't find the body of article %s" % title
return
title = article.getTitle().strip() + "\n"
txt = article.getTxt()
converted = articleconvert.convertArticle(title, txt)
print "TITLE: %s" % title
print "ORIGINAL: %s" % txt
print "CONVERTED: %s" % converted
return
if __name__ == "__main__":
fRandom = arsutils.fDetectRemoveCmdFlag("-random")
fDump = arsutils.fDetectRemoveCmdFlag("-dump")
if fDump and fRandom:
print "Can't use -dump and -random at the same time"
usageAndExit()
title = arsutils.getRemoveCmdArg("-title")
fSave = arsutils.fDetectRemoveCmdFlag("-save")
fForceConvert = arsutils.fDetectRemoveCmdFlag("-forceconvert")
if title and fRandom:
print "Can't use -title and -random at the same time"
usageAndExit()
cur.execute(query)
cur.close()
def revLinksOnly(sqlDump):
try:
dbName = getDbNameFromFileName(sqlDump)
if dbName not in getDbList():
print "Database '%s' doesn't exist and we need it for -revlinksonly" % dbName
calcReverseLinks(sqlDump)
finally:
deinitDatabase()
if __name__=="__main__":
fNoPsyco = arsutils.fDetectRemoveCmdFlag("-nopsyco")
if g_fPsycoAvailable and not fNoPsyco:
print "using psyco"
psyco.full()
g_fVerbose = arsutils.fDetectRemoveCmdFlag("-verbose")
g_fShowDups = arsutils.fDetectRemoveCmdFlag("-showdups")
fRecreateDb = arsutils.fDetectRemoveCmdFlag("-recreatedb")
fRecreateDataDb = arsutils.fDetectRemoveCmdFlag("-recreatedatadb")
articleLimit = arsutils.getRemoveCmdArgInt("-limit")
fRevLinksOnly = arsutils.fDetectRemoveCmdFlag("-revlinksonly")
# we always need to try to create it
recreateDataDb(fRecreateDataDb)
if fRecreateDataDb and len(sys.argv) == 1:
# no arguments allowed if we only try to recreate data db
if __name__=="__main__":
serverToUse = None
serverToUse = arsutils.getRemoveCmdArg("-server")
if None == serverToUse:
detectAndSetServerToUse()
else:
if not g_serverList.has_key(serverToUse):
print "server %s is not known. Known servers:" % serverToUse
for serverName in g_serverList.keys():
print " %s" % serverName
sys.exit(0)
else:
g_serverToUse = serverToUse
if arsutils.fDetectRemoveCmdFlag("-raw"):
g_showRawOutput = True
printUsedServer()
unpickleState()
pos = None
for argName in argsInfo.keys():
pos = getIndexSafe(sys.argv, argName)
if None != pos:
break
if None == pos:
usageAndExit()
sys.exit(0)
numArgs = argsInfo[argName][0]
def revLinksOnly(sqlDump):
try:
dbName = getDbNameFromFileName(sqlDump)
if dbName not in getDbList():
print "Database '%s' doesn't exist and we need it for -revlinksonly" % dbName
calcReverseLinks(sqlDump)
finally:
deinitDatabase()
if __name__ == "__main__":
fNoPsyco = arsutils.fDetectRemoveCmdFlag("-nopsyco")
if g_fPsycoAvailable and not fNoPsyco:
print "using psyco"
psyco.full()
g_fVerbose = arsutils.fDetectRemoveCmdFlag("-verbose")
g_fShowDups = arsutils.fDetectRemoveCmdFlag("-showdups")
fRecreateDb = arsutils.fDetectRemoveCmdFlag("-recreatedb")
fRecreateDataDb = arsutils.fDetectRemoveCmdFlag("-recreatedatadb")
articleLimit = arsutils.getRemoveCmdArgInt("-limit")
fRevLinksOnly = arsutils.fDetectRemoveCmdFlag("-revlinksonly")
# we always need to try to create it
recreateDataDb(fRecreateDataDb)
if fRecreateDataDb and len(sys.argv) == 1:
# no arguments allowed if we only try to recreate data db