def _runSqlFile(fSql, usr, pwd, db, host=None, port=None):
host = utils.getValue(host, 'MYSQL_SERVER')
port = utils.getValue(port, 'MYSQL_PORT', 3306)
mysql_exe = utils.getValue(None, 'MYSQL_EXE', 'mysql')
cmd="{mysql_exe} --user={usr} --password={pwd} --host={host} --port={port} {db} < {fSql}".format(**locals())
os.system(cmd)
def __init__(self, parameters):
self.parameters = parameters
self.version = utils.getValue(self.parameters, 'version')
if self.version == None:
self.version = '1.0.0'
self.request = utils.getValue(self.parameters, 'request')
def process(self):
"""
Processes the GetTile parameters and prepares the GEE tile URL.
"""
if utils.getValue(self.parameters, 'layer') == None:
return ExceptionReport('MissingParameterValue',
'LAYER is a required parameter.', 'LAYER')
utils.logger.debug('find the target layer')
layer = utils.getValue(self.parameters, 'layer')
self.initializeLayerInfo(layer)
if self.layerInfo == None:
return ExceptionReport('InvalidParameterValue', 'Layer not found.',
'LAYER')
self.z = utils.getValue(self.parameters, 'tilematrix')
if self.z == None:
return ExceptionReport('MissingParameterValue',
'TILEMATRIX is a required parameter.',
'TILEMATRIX')
self.z = int(self.z)
if self.z < 1 or self.z > 24:
return ExceptionReport('InvalidParameterValue',
'TILEMATRIX out of range.', 'TILEMATRIX')
self.y = utils.getValue(self.parameters, 'tilerow')
if self.y == None:
return ExceptionReport('MissingParameterValue',
'TILEROW is a required parameter.',
'TILEROW')
self.y = int(self.y)
if self.y < 0 or self.y >= (2**self.z):
return ExceptionReport('InvalidParameterValue',
'TILEROW out of range.', 'TILEROW')
self.x = utils.getValue(self.parameters, 'tilecol')
if self.x == None:
return ExceptionReport('MissingParameterValue',
'TILECOL is a required parameter.',
'TILECOL')
self.x = int(self.x)
if self.x < 0 or self.x >= (2**int(self.z)):
return ExceptionReport('InvalidParameterValue',
'TILECOL out of range.', 'TILECOL')
utils.logger.debug('generate google query url')
self.initializeTileURL(self.x, self.y, self.z)
return None
def initDB(dbOpts, year):
minYear = 0
maxYear = 3000
year = year.split(",")
if len(year) == 1:
minYear = int(year[0])
elif len(year) == 2:
minYear = int(year[0])
maxYear = int(year[1])
db = DB(dbOpts)
datas = []
for kd in getValue(minYear=minYear, maxYear=maxYear):
# for kd in getValue(minYear=2020, minNumber=4, maxNumber=9):
# print(kd)
datas.append(kd)
if len(datas) > 1000:
print("insert upto %s" % kd[0])
db.addCVEData(datas)
datas = []
db.addCVEData(datas)
def solve(self):
self.fillQ()
answer = utils.solve_with_qbsolv(self.Q)
value = utils.getValue(self.Q, answer)
#print("Value: ", value, "[", -self.V * (self.V + 1), "]")
print("Size of Q: ", self.z * self.E)
path = [-1] * (self.V + 1)
path[0] = 0
path[self.V] = 0
for edgeNr, (a, b) in enumerate(self.edges):
pos = 0
for z1 in range(self.z):
pos += self.zValues[z1] * answer[edgeNr * self.z + z1]
if pos != 0:
path[pos] = b
if value > -self.V * (self.V + 1):
return False
else:
print("The hamilton path is: ", path)
return True
def noportal(bot, filename, dump, mode, debug):
transform = link
firstPage = int(utils.getValue('first', 1))
lastPage = sys.maxint
nbColumns = 3
nbArticlesPerSection = 100
nbSectionsPerPage = 5
if mode == 1:
firstPage = 1
lastPage = 1
root = 'Projet:Articles sans portail'
subject = 'Articles sans portail'
elif mode == 2:
lastPage = 1
root = 'Projet:Articles sans portail/Album musical'
subject = 'Articles sans portail/Album musical'
elif mode == 3:
lastPage = 1
root = 'Projet:Articles sans portail/Acteur'
subject = 'Articles sans portail/Acteur'
elif mode == 4:
lastPage = 1
root = 'Projet:Articles sans portail/Homonymies'
subject = 'Articles sans portail/Homonymies'
elif mode == 5:
nbColumns = 1
nbSectionsPerPage = 6
nbArticlesPerSection = 50
lastPage = 2
root = 'Projet:Pages les moins modifiées'
subject = 'Pages les moins modifiées'
transform = contrib
elif mode == 6:
nbColumns = 1
nbSectionsPerPage = 6
nbArticlesPerSection = 50
lastPage = 1
root = 'Utilisateur:Arkbot/Pages les plus modifiées'
subject = 'Pages les plus modifiées'
transform = edits
elif mode == 7:
lastPage = 3
root = 'Projet:Articles sans infobox'
subject = 'Articles sans infobox'
elif mode == 8:
lastPage = 1
root = 'Projet:Articles sans infobox/Album musical'
subject = 'Articles sans infobox/Album musical'
elif mode == 9:
lastPage = 1
root = 'Projet:Articles sans infobox/Acteur'
subject = 'Articles sans infobox/Acteur'
elif mode == 10:
lastPage = 10
root = 'Projet:Articles non liés depuis Wikidata'
subject = 'Articles non liés depuis Wikidata'
transform = identity
elif mode == 11:
lastPage = 1
root = 'Utilisateur:Arkbot/Titres contenant une parenthèse non précédée d\'une espace'
subject = 'Titres contenant une parenthèse non précédée d\'une espace'
elif mode == 12:
lastPage = 45
root = 'Projet:Articles dont le nom est à vérifier'
subject = 'Articles dont le nom est à vérifier'
elif mode == 13:
lastPage = 1
root = 'Utilisateur:Arkbot/Homonymies à renommer'
subject = 'Homonymies à renommer'
elif mode == 14:
lastPage = 3
root = 'Utilisateur:Arkbot/Caractères spéciaux à vérifier'
subject = 'Caractères spéciaux à vérifier'
else:
assert False, 'Invalid mode'
summary = subject.split('/')[0]
introPath = '../' + 'intro'
nbArticlesPerSubSection = nbArticlesPerSection / nbColumns
nbArticlesPerSection = nbColumns * nbArticlesPerSubSection
nbArticlesPerPage = nbArticlesPerSection * nbSectionsPerPage
if nbColumns > 1:
columns = ' style="-moz-column-count:%i; column-count:%i;"' % (nbColumns, nbColumns)
else:
columns = ''
with open(filename) as articles:
page = ''
model = """{{Méta palette de navigation
| modèle = Palette %s
| titre = [[%s|%s]]
| styleliste = text-align:center
| liste1 = {{liste horizontale|""" % (subject, root, summary)
section = ''
subSection = ''
number = 0
startPage = 1
startSection = 1
startSubSection = 1
pageNumber = 1
for article in articles:
name = transform(article.rstrip())
if not name:
continue
subSection += '<li>%s</li>\n' % name
number += 1
if not number % nbArticlesPerSubSection:
section += '\n<!-- %i à %i -->\n\n%s' % (startSubSection, number, subSection)
subSection = ''
startSubSection = number + 1
if not number % nbArticlesPerSection:
page += '\n\n=== %i à %i ===\n\n<ol start="%i"%s>\n%s</ol>' % (startSection, number, startSection, columns, section)
section = ''
startSection = number + 1
if not number % nbArticlesPerPage:
publishPage(bot, dump, page, startPage, number, pageNumber, debug, firstPage, lastPage, root, summary, introPath)
model += '\n* [[%s/%i|%i à %i]]' % (root, pageNumber, startPage, number)
pageNumber += 1
page = ''
startPage = number + 1
if pageNumber > lastPage:
break
if number >= startPage and pageNumber <= lastPage:
if number >= startSection:
if number % nbArticlesPerSubSection:
section += '\n<!-- %i à %i -->\n\n%s' % (startSubSection, number, subSection)
page += '\n\n=== %i à %i ===\n\n<ol start="%i"%s>\n%s</ol>' % (startSection, number, startSection, columns, section)
publishPage(bot, dump, page, startPage, number, pageNumber, debug, firstPage, lastPage, root, summary, introPath)
model += '\n* [[%s/%i|%i à %i]]' % (root, pageNumber, startPage, number)
pageNumber += 1
model += """
}}
}}<noinclude>{{Documentation palette}}
[[Catégorie:Palette de navigation espace non encyclopédique|%s]]
</noinclude>""" % subject
if lastPage != sys.maxint:
for pageNumber in xrange(max(firstPage, pageNumber), lastPage + 1):
clearPage(bot, dump, pageNumber, debug, firstPage, lastPage, root, summary, introPath)
if mode in [1, 4, 7, 8, 9, 10, 11, 12, 13]:
editOrDebug(bot, 'Modèle:Palette %s' % subject, '%s au %s' % (summary, dump), model, debug, botEdit=True)
# TODO généraliser le script pour publier n'importe quelle liste d'articles
# - ./publishList.py -root 'Projet:Portails' -first 1 -last 127 -delay 10 -dump '3 novembre 2010' -subject 'Articles sans portail'
# - ./publishList.py -root 'Projet:Relance' -first 1 -last 5 -delay 10 -dump '3 novembre 2010' -subject 'Articles rarement modifiés' -transform 'contrib'
# + faire des scripts shell pour chacune des lignes de paramètres fréquemment utilisées
# TODO accepter plusieurs fichiers en entrée (pour différentes sous pages : musique, acteurs), les traiter à la suite et générer une palette commune
import datetime
import getpass
import logging
import sys
import time
import arkbot
import utils
_secondsBetweenEdits = int(utils.getValue('delay', '1'))
def identity(line):
return line
def link(line):
if line[0] == '/':
return '[[:%s]]' % line
return '[[%s]]' % line
def contrib(line):
parts = line.split(' || ')
if parts[4] == 'h**o':
return False
return '%s — {{a-court|%s}} ({{u\'|%s}})' % (parts[0], parts[2], parts[3])
def main(createDB=False, skipTestData=False, skipI18N=False, mysqlRootUsr=None, mysqlRootPwd=None, mysqlUsr=None, mysqlPwd=None, mysqlDb=None, mysqlHost=None, mysqlPort=None):
# Load config
config=json.load(open("env.json"))
# Get the values for db/usr/pwd
mysqlDb = utils.getValue(mysqlDb , 'MYSQL_DB' )
mysqlUsr = utils.getValue(mysqlUsr, 'MYSQL_USR', mysqlDb)
mysqlPwd = utils.getValue(mysqlPwd, 'MYSQL_PWD', mysqlDb)
# ------------------------------------------------------------ Create Database
if createDB:
createDBSql="""
DROP DATABASE IF EXISTS {mysqlDb};
CREATE DATABASE IF NOT EXISTS {mysqlDb};
GRANT ALL PRIVILEGES ON {mysqlDb}.* TO {mysqlUsr}@localhost IDENTIFIED BY '{mysqlPwd}';
""".format(**locals())
# Get the usr/pwd for root, so we can create the database
mysqlRootUsr = utils.getValue(mysqlRootUsr, 'MYSQLROOT_USR')
mysqlRootPwd = utils.getValue(mysqlRootPwd, 'MYSQLROOT_PWD')
print "Creating the database {mysqlDb} ...".format(**locals())
_runSqlCmd(createDBSql, mysqlRootUsr, mysqlRootPwd, 'mysql', mysqlHost, mysqlPort)
print "Database created!"
# ---------------------------------------------------------------- Drop Tables
print "Dropping tables into the database {mysqlDb} ...".format(**locals())
_runSqlFile("dropTables.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Tables removed!"
# -------------------------------------------------------------- Create Tables
print "Creating tables into the database {mysqlDb} ...".format(**locals())
_runSqlFile("createTables.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Tables created!"
# ---------------------------------------------------------- Create Permission
print "Inserting permissions into the database {mysqlDb} ...".format(**locals())
_runSqlFile("permissions.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Permissions inserted!"
# Insert Config Data
print "Inserting config data into the database {mysqlDb} ...".format(**locals())
_runSqlFile("insertCfgData.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Config data inserted!"
# ---------------------------------------------------------------- Insert I18N
if skipI18N:
print "*** Skip insert I18N data"
else:
print "Inserting I18N data into the database {mysqlDb}. This can take a while ...".format(**locals())
_runSqlFile("i18n.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "I18N data inserted!"
# ----------------------------------------------------------- Insert Test Data
if skipTestData:
print "*** Skip insert test data"
else:
print "Inserting test data into the database {mysqlDb} ...".format(**locals())
_runSqlFile("insertTestData.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Test data inserted!"
# --------------------------------------------------------- All Modules Config
print "Inserting all modules config into the database {mysqlDb} ...".format(**locals())
_runSqlFile("allModules.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
print "Module config data inserted!"
# http://github.com/Arkanosis/Wikipedia/Arkbot
# Ce script est mis à disposition sous licence MIT
# http://www.opensource.org/licenses/mit-license.php
import datetime
import getpass
import logging
import sys
import time
import arkbot
import utils
_dump = utils.getValue('dump')
_mode = int(utils.getValue('mode'))
_debug = utils.getOption('debug')
if _mode == 1:
_page = 'Projet:Pages en impasse/liste des pages en impasse'
_summary = 'Pages en impasse au %s' % _dump
_pages = 'en impasse'
elif _mode == 2:
_page = 'Projet:Pages vides/liste des pages vides'
_summary = 'Pages vides au %s' % _dump
_pages = 'vides'
elif _mode == 3:
_page = 'Utilisateur:Arkbot/Pages redirigeant hors de l\'espace de nom principal'
_summary = 'Pages redirigeant hors de l\'espace de nom principal au %s' % _dump
_pages = 'redirigeant hors de l\'espace de nom principal'
def run(epoch):
global kt, global_step
pbar = tqdm.tqdm(dataloader, desc='Epoch %02d' % epoch, dynamic_ncols=True)
pbar_info = tqdm.tqdm(None, bar_format='{bar}{postfix}')
avg_acc = 0
for it, sample in enumerate(pbar, start=iter_init):
global_step += 1
image, label, image_s = sample
image = Variable(image)
label = Variable(label)
image_s = Variable(image_s)
if FLAGS.cuda:
image = image.cuda()
label = label.cuda(async=True)
image_s = image_s.cuda()
# generator
outputs = netHg(image)
loss_hg_content = 0
for out in outputs: # TODO: speed up with multiprocessing map?
loss_hg_content += criterion(out, label)
# Discriminator
input_real = torch.cat([image_s, label], dim=1)
d_real = netD(input_real)
loss_d_real = criterion_D(d_real, label)
input_fake = torch.cat([image_s, outputs[-1]], dim=1)
d_fake = netD(input_fake)
loss_d_fake = criterion_D(d_fake, label)
loss_d = loss_d_real - kt * loss_d_fake
loss_hg = loss_hg_content + FLAGS.lambda_G * loss_d_fake.detach()
''' Backward seperatedly '''
optimD.zero_grad()
loss_d.backward(retain_graph=True)
optimD.step()
optimHg.zero_grad()
loss_hg.backward()
# optimD.step()
optimHg.step()
''' Backward all at once (slightly different ?) '''
# loss_total = loss_d + loss_hg
# optimD.zero_grad()
# optimHg.zero_grad()
# loss_total.backward()
# optimD.step()
# optimHg.step()
# update kt
loss_d_real_ = getValue(loss_d_real)
loss_d_fake_ = getValue(loss_d_fake)
balance = FLAGS.gamma * loss_d_real_ - loss_d_fake_ / FLAGS.lambda_G # dividing is Good?
kt = kt + FLAGS.lambda_kt * balance
kt = min(1, max(0, kt))
measure = loss_d_real_ + abs(balance)
accs = accuracy(outputs[-1].data.cpu(), label.data.cpu(),
dataset.accIdxs)
# TODO: summary, printing (maybe tqdm)
sumWriter.add_scalar('loss_d_real', loss_d_real_, global_step)
sumWriter.add_scalar('loss_d_fake', loss_d_fake_, global_step)
sumWriter.add_scalar('loss_d', loss_d, global_step)
sumWriter.add_scalar('loss_hg_content', loss_hg_content, global_step)
sumWriter.add_scalar('loss_hg', loss_hg, global_step)
sumWriter.add_scalar('kt', kt, global_step)
sumWriter.add_scalar('balance', balance, global_step)
sumWriter.add_scalar('meauser', measure, global_step)
sumWriter.add_scalar('acc', accs[0], global_step)
# TODO: learning rate scheduling
# sumWriter.add_scalar('lr', lr, global_step)
pbar_info.set_postfix({
'balance': balance,
'loss_hg': getValue(loss_hg),
'loss_d': getValue(loss_d),
'acc': accs[0],
})
pbar_info.update()
# pbar.write('hello %d\r' % it, end='')
avg_acc += accs[0] / len(dataloader)
pbar_info.set_postfix_str('avg_acc: {}'.format(avg_acc))
pbar.close()
pbar_info.close()
import logging
import sys
import time
import arkbot
import noportal
import post
import utils
if __name__ == '__main__':
print 'DumpWork 0.1'
print '(C) 2017 Arkanosis'
print '*****@*****.**'
print
dump = utils.getValue('dump')
debug = utils.getOption('debug')
if len(sys.argv) != 2:
print 'Usage: dumpWork.py <dump>'
sys.exit(1)
date = datetime.datetime.now()
logger = logging.getLogger('ArkbotLogger')
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', '%H:%M:%S')
logger.setLevel(logging.INFO)
bot = arkbot.Arkbot(arkbot._botName, arkbot._wiki, logger)
try:
if not debug:
bot.login(getpass.getpass('Bot password ? '))
def sunat():
ruc = request.args.get('ruc', '')
resp = getValue('ruc', ruc)
return jsonify(resp)
def reniec():
dni = request.args.get('dni', '')
resp = getValue('dni', dni)
return jsonify(resp)
# TODO généraliser le script pour publier n'importe quelle liste d'articles
# - ./publishList.py -root 'Projet:Portails' -first 1 -last 127 -delay 10 -dump '3 novembre 2010' -subject 'Articles sans portail'
# - ./publishList.py -root 'Projet:Relance' -first 1 -last 5 -delay 10 -dump '3 novembre 2010' -subject 'Articles rarement modifiés' -transform 'contrib'
# + faire des scripts shell pour chacune des lignes de paramètres fréquemment utilisées
# TODO accepter plusieurs fichiers en entrée (pour différentes sous pages : musique, acteurs), les traiter à la suite et générer une palette commune
import datetime
import getpass
import logging
import sys
import time
import arkbot
import utils
_dump = utils.getValue('dump')
_mode = int(utils.getValue('mode'))
_debug = utils.getOption('debug')
_secondsBetweenEdits = int(utils.getValue('delay', '5'))
_firstPage = int(utils.getValue('first', 1))
_lastPage = sys.maxint
_nbColumns = 3
_nbArticlesPerSection = 100
_nbSectionsPerPage = 5
_subPage = False
if _debug: