def Login():
token = GetConfig("Confidential", "Token")
if not token:
url = client.get_authorize_url()
print(url)
webbrowser.open_new(url)
code = input("Code?")
r = client.request_access_token(code)
token = r.access_token
expires = r.expires_in
SetConfig("Confidential", "Token", token)
SetConfig("Confidential", "Expires", str(r.expires_in))
SaveConfig()
else:
expires = int(GetConfig("Confidential", "Expires"))
logging.info("Token expires in %s .", datetime.fromtimestamp(expires))
client.set_access_token(token, expires)
def SelectRandomPage():
# Pages in certain categories will not be pushed.
EXCLUDED_CATEGORY_PARTS = ["翻译", "删除"]
# Only pages with more than x revisions will be pushed.
pageMinRevisions = int(GetConfig("Wiki", "PageMinimumRevisions", 5))
maxFetchedPages = int(GetConfig("Wiki", "MaxRandomPicks", 500))
ignorePushedTerms = bool(GetConfig("Wiki", "IgnorePushedTermsList", False))
forceWithImage = bool(GetConfig("Wiki", "ForceWithImage", True))
fetchedPages = 0
for page in pagegenerators.RandomPageGenerator(maxFetchedPages,
siteZh,
namespaces=(0, )):
fetchedPages += 1
if page.title() in pushedTerms:
logging.debug("Skipped: %s . Page has been pushed.", page)
continue
if page.revision_count() < pageMinRevisions:
logging.debug("Skipped: %s . Insufficient revisions.", page)
continue
# Check categories
cats = [cat.title() for cat in page.categories()]
if next(
(c for c in cats
if next((ec for ec in EXCLUDED_CATEGORY_PARTS if c in ec), None)),
None):
logging.debug("Skipped: %s . Excluded by category.", page)
continue
# Check cover image
if forceWithImage and not GetCoverImage(page)[1]:
logging.debug("Skipped: %s . No cover image found.", page)
continue
# The page might be suitable
logging.info("After %s/%s pages, choose: %s .", fetchedPages,
maxFetchedPages, page)
return page
# No page selected
logging.warn("After %s pages, no page meets the requirements.",
fetchedPages)
return None
def CreateDatabase():
dbParams = GetConfig("postgresql")
conn = psycopg2.connect(host=dbParams["host"],
user=dbParams["user"],
password=dbParams["password"])
conn.set_session(autocommit=True)
cur = conn.cursor()
dropStr = "DROP DATABASE IF EXISTS {}".format(dbParams["database"])
cur.execute(dropStr)
createStr = 'CREATE DATABASE {} WITH ENCODING "utf8" TEMPLATE template0'.format(
dbParams["database"])
cur.execute(createStr)
conn.close()
def LoadPushedTerms():
# The terms pushed {ttl} days before may be pushed again.
ttl = float(GetConfig("Wiki", "PushedTermsTTL", 180))
ttl = timedelta(days=ttl)
now = datetime.utcnow()
try:
with open("Confidential/PushedTerms.txt", "r") as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for entry in reader:
# Ignore trailing blank lines
if len(entry) < 2: continue
time = datetime.strptime(entry[1], '%Y-%m-%dT%H:%M:%S.%f')
if (time - now) < ttl:
pushedTerms[str(entry[0]).strip()] = time
logging.info("Loaded %i shared terms.", len(pushedTerms))
except FileNotFoundError:
logging.info("No shared terms recorded yet.")
def ParseWikiPagePushInfo(page: pywikibot.Page):
parsed_text = ParsePage(page)
# If the score of a trivia is higher than this,
# we'll try to show it only, without leading text.
triviaSignificance = float(GetConfig("Wiki", "PushedTermsTTL", 180))
# Distill text
bareTitle = BareDisambigTitle(page.title())
distilled = WikiPageDistiller.DistillHtml(parsed_text)
info = WikiPagePushInfo(page.title(), page.full_url())
if distilled.trivia != None:
# Trivia only
info.postText = distilled.trivia
# Leading + trivia
if distilled.triviaScore < triviaSignificance or not bareTitle in info.postText:
info.postText = distilled.introduction + info.postText
else:
# Leading
info.postText = distilled.introduction
#elif len(distilled.introduction) < 50 :
#info.post
# Choose cover image
info.postImageName, info.postImageUrl = GetCoverImage(page)
return info
def GetCur():
dbParams = GetConfig("postgresql")
conn = psycopg2.connect(**dbParams)
conn.set_session(autocommit=True)
cur = conn.cursor()
return conn, cur
def MakeBatch():
print('Enter Batch Directory Name for new batch.')
batch_dir = input('>> ')
print('Enter Batch prefix.')
print('Warning, all configuration files with the same prefix will'
+ ' be deleted and replaced with new ones upon export.')
batch_name = input('>> ')
config_dir = os.path.join(batch_dir, 'configs')
if not os.path.exists(batch_dir):
os.mkdir(batch_dir)
os.mkdir(config_dir)
elif not os.path.exists(config_dir):
os.mkdir(config_dir)
while True:
try:
print('Enter total number of configurations.')
total_configs = int(input('>> '))
print("Total configs = %d" % (total_configs))
break
except ValueError:
print("Error: input is not a valid number.")
continue
temp = TemporaryDirectory()
while True:
print("Enter filename of the base configuration,")
print('or enter "default" to use the default configuration.')
config_input = input('>> ')
if config_input == 'default':
Default(temp.name, False)
config_input = os.path.join(temp.name, 'defaultconfig.in')
break
elif os.path.exists(config_input):
break
else:
print('Error: File does not exist:')
continue
initial_config = GetConfig(config_input)
batch_config = [GetConfig(config_input) for i in range(total_configs)]
temp.cleanup()
cancel = False
while True:
print('Enter Section and key name, (seperated by whitespace)')
print('Enter "export" to write batch configuration files,')
print('Or enter "cancel" to quit without saving')
user_input = input('>> ')
if user_input == 'cancel':
cancel = True
break
elif user_input == 'export':
break
elif len(user_input.split()) == 2:
section, key = user_input.split()
print('Input Successful')
else:
print('Error: Input of %s is invalid.' % (user_input))
continue
if section in initial_config.sections():
if key in initial_config[section].keys():
print("Key: %s, \t Current Value: %s" % (key,
initial_config
[section][key]))
else:
print('%s is not a valid key.' % (key))
continue
else:
print('%s is not a valid section' % (section))
continue
while True:
print('Enter scale choice. (linear/logarithimic/manual)')
user_input = input('>> ')
if user_input.startswith('lin'):
print("Linear scale selected.")
scale = 'linear'
break
elif user_input.startswith('log'):
print("Logarithmic scale selected.")
scale = 'logarithmic'
break
elif user_input.startswith('man'):
print('Manual scale selected')
scale = 'manual'
break
else:
print("Error: input of %s is invalid" % (user_input))
continue
if scale == 'linear':
while True:
try:
print('Enter initial and final values. '
+ '(seperated by whitespace) ')
start, stop = input('>> ').split()
start = float(start)
stop = float(stop)
steps = np.linspace(start, stop, total_configs)
break
except ValueError:
print('Error: Input is invalid')
continue
elif scale == 'logarithmic':
while True:
try:
print("Scale = multiplier x base^(exponent) ")
print('Enter multiplier value.')
factor = input('>>')
print('Enter base value.')
base = input('>> ')
print('Enter initial and final exponents. '
+ '(seperated by whitespace) ')
start, stop = input('>> ').split()
factor = float(factor)
base = float(base)
start = float(start)
stop = float(stop)
steps = factor *\
np.logspace(start, stop, total_configs, base=base)
break
except ValueError:
print('Error: Input is invalid')
continue
elif scale == 'manual':
while True:
print('Enter %d values (seperated by whitespace)' %
(total_configs))
user_input = input(">> ")
steps = user_input.split()
if len(steps) == total_configs:
break
elif len(steps) < total_configs:
print('Error: User entered insufficient values')
elif len(steps) > total_configs:
print('Error: User entered too many values')
if scale == 'manual':
for i in range(total_configs):
batch_config[i][section][key] = '%s' % (steps[i])
else:
for i in range(total_configs):
batch_config[i][section][key] = '%8.3g' % (steps[i])
print("Values:")
print(steps)
continue
if not cancel:
for entry in os.scandir(batch_dir):
if os.path.split(entry.path)[1].startswith(batch_name):
os.remove(entry.path)
digits = int(np.ceil(np.log10(total_configs+1)))
for i in range(total_configs):
file_iter = str(i)
while len(file_iter) < digits:
file_iter = '0'+file_iter
fname = "%s%s.in" % (batch_name, file_iter)
save_name = os.path.join(config_dir, fname)
with open(save_name, 'w') as f:
batch_config[i].write(f)
f.close()
print("Written %d configurations to %s" % (total_configs, config_dir))
return batch_dir
else:
print("Batch aborted by user.")
return ''