Example #1
0
def _runSqlFile(fSql, usr, pwd, db, host=None, port=None):
  host      = utils.getValue(host, 'MYSQL_SERVER')
  port      = utils.getValue(port, 'MYSQL_PORT', 3306)
  mysql_exe = utils.getValue(None, 'MYSQL_EXE', 'mysql')

  cmd="{mysql_exe} --user={usr} --password={pwd} --host={host} --port={port} {db} < {fSql}".format(**locals())
  os.system(cmd)   
Example #2
0
    def __init__(self, parameters):
        self.parameters = parameters

        self.version = utils.getValue(self.parameters, 'version')
        if self.version == None:
            self.version = '1.0.0'

        self.request = utils.getValue(self.parameters, 'request')
Example #3
0
    def process(self):
        """
    Processes the GetTile parameters and prepares the GEE tile URL.
    """
        if utils.getValue(self.parameters, 'layer') == None:
            return ExceptionReport('MissingParameterValue',
                                   'LAYER is a required parameter.', 'LAYER')

        utils.logger.debug('find the target layer')
        layer = utils.getValue(self.parameters, 'layer')
        self.initializeLayerInfo(layer)

        if self.layerInfo == None:
            return ExceptionReport('InvalidParameterValue', 'Layer not found.',
                                   'LAYER')

        self.z = utils.getValue(self.parameters, 'tilematrix')
        if self.z == None:
            return ExceptionReport('MissingParameterValue',
                                   'TILEMATRIX is a required parameter.',
                                   'TILEMATRIX')

        self.z = int(self.z)

        if self.z < 1 or self.z > 24:
            return ExceptionReport('InvalidParameterValue',
                                   'TILEMATRIX out of range.', 'TILEMATRIX')

        self.y = utils.getValue(self.parameters, 'tilerow')
        if self.y == None:
            return ExceptionReport('MissingParameterValue',
                                   'TILEROW is a required parameter.',
                                   'TILEROW')

        self.y = int(self.y)

        if self.y < 0 or self.y >= (2**self.z):
            return ExceptionReport('InvalidParameterValue',
                                   'TILEROW out of range.', 'TILEROW')

        self.x = utils.getValue(self.parameters, 'tilecol')
        if self.x == None:
            return ExceptionReport('MissingParameterValue',
                                   'TILECOL is a required parameter.',
                                   'TILECOL')

        self.x = int(self.x)

        if self.x < 0 or self.x >= (2**int(self.z)):
            return ExceptionReport('InvalidParameterValue',
                                   'TILECOL out of range.', 'TILECOL')

        utils.logger.debug('generate google query url')
        self.initializeTileURL(self.x, self.y, self.z)

        return None
Example #4
0
def initDB(dbOpts, year):
    minYear = 0
    maxYear = 3000
    year = year.split(",")
    if len(year) == 1:
        minYear = int(year[0])
    elif len(year) == 2:
        minYear = int(year[0])
        maxYear = int(year[1])
    db = DB(dbOpts)
    datas = []
    for kd in getValue(minYear=minYear, maxYear=maxYear):
        # for kd in getValue(minYear=2020, minNumber=4, maxNumber=9):
        # print(kd)
        datas.append(kd)
        if len(datas) > 1000:
            print("insert upto %s" % kd[0])
            db.addCVEData(datas)
            datas = []
    db.addCVEData(datas)
 def solve(self):
     self.fillQ()
     answer = utils.solve_with_qbsolv(self.Q)
     value = utils.getValue(self.Q, answer)
     #print("Value: ", value, "[", -self.V * (self.V + 1), "]")
     print("Size of Q: ", self.z * self.E)
     path = [-1] * (self.V + 1)
     path[0] = 0
     path[self.V] = 0
     for edgeNr, (a, b) in enumerate(self.edges):
         pos = 0
         for z1 in range(self.z):
             pos += self.zValues[z1] * answer[edgeNr * self.z + z1]
         if pos != 0:
             path[pos] = b
     if value > -self.V * (self.V + 1):
         return False
     else:
         print("The hamilton path is: ", path)
         return True
Example #6
0
def noportal(bot, filename, dump, mode, debug):
	transform = link

	firstPage = int(utils.getValue('first', 1))
	lastPage = sys.maxint

	nbColumns = 3
	nbArticlesPerSection = 100
	nbSectionsPerPage = 5

	if mode == 1:
		firstPage = 1
		lastPage = 1
		root = 'Projet:Articles sans portail'
		subject = 'Articles sans portail'
	elif mode == 2:
		lastPage = 1
		root = 'Projet:Articles sans portail/Album musical'
		subject = 'Articles sans portail/Album musical'
	elif mode == 3:
		lastPage = 1
		root = 'Projet:Articles sans portail/Acteur'
		subject = 'Articles sans portail/Acteur'
	elif mode == 4:
		lastPage = 1
		root = 'Projet:Articles sans portail/Homonymies'
		subject = 'Articles sans portail/Homonymies'
	elif mode == 5:
		nbColumns = 1
		nbSectionsPerPage = 6
		nbArticlesPerSection = 50
		lastPage = 2
		root = 'Projet:Pages les moins modifiées'
		subject = 'Pages les moins modifiées'
		transform = contrib
	elif mode == 6:
		nbColumns = 1
		nbSectionsPerPage = 6
		nbArticlesPerSection = 50
		lastPage = 1
		root = 'Utilisateur:Arkbot/Pages les plus modifiées'
		subject = 'Pages les plus modifiées'
		transform = edits
	elif mode == 7:
		lastPage = 3
		root = 'Projet:Articles sans infobox'
		subject = 'Articles sans infobox'
	elif mode == 8:
		lastPage = 1
		root = 'Projet:Articles sans infobox/Album musical'
		subject = 'Articles sans infobox/Album musical'
	elif mode == 9:
		lastPage = 1
		root = 'Projet:Articles sans infobox/Acteur'
		subject = 'Articles sans infobox/Acteur'
	elif mode == 10:
		lastPage = 10
		root = 'Projet:Articles non liés depuis Wikidata'
		subject = 'Articles non liés depuis Wikidata'
		transform = identity
	elif mode == 11:
		lastPage = 1
		root = 'Utilisateur:Arkbot/Titres contenant une parenthèse non précédée d\'une espace'
		subject = 'Titres contenant une parenthèse non précédée d\'une espace'
	elif mode == 12:
		lastPage = 45
		root = 'Projet:Articles dont le nom est à vérifier'
		subject = 'Articles dont le nom est à vérifier'
	elif mode == 13:
		lastPage = 1
		root = 'Utilisateur:Arkbot/Homonymies à renommer'
		subject = 'Homonymies à renommer'
	elif mode == 14:
		lastPage = 3
		root = 'Utilisateur:Arkbot/Caractères spéciaux à vérifier'
		subject = 'Caractères spéciaux à vérifier'
	else:
		assert False, 'Invalid mode'

	summary = subject.split('/')[0]
	introPath = '../' + 'intro'

	nbArticlesPerSubSection = nbArticlesPerSection / nbColumns
	nbArticlesPerSection = nbColumns * nbArticlesPerSubSection
	nbArticlesPerPage = nbArticlesPerSection * nbSectionsPerPage

	if nbColumns > 1:
		columns = ' style="-moz-column-count:%i; column-count:%i;"' % (nbColumns, nbColumns)
	else:
		columns = ''
        
	with open(filename) as articles:
		page = ''
		model = """{{Méta palette de navigation
 | modèle     = Palette %s
 | titre      = [[%s|%s]]
 | styleliste = text-align:center
 | liste1     = {{liste horizontale|""" % (subject, root, summary)
		section = ''
		subSection = ''
		number = 0
		startPage = 1
		startSection = 1
		startSubSection = 1
		pageNumber = 1
		for article in articles:
			name = transform(article.rstrip())
			if not name:
				continue
			subSection += '<li>%s</li>\n' % name
			number += 1
			if not number % nbArticlesPerSubSection:
				section += '\n<!-- %i à %i -->\n\n%s' % (startSubSection, number, subSection)
				subSection = ''
				startSubSection = number + 1
			if not number % nbArticlesPerSection:
				page += '\n\n=== %i à %i ===\n\n<ol start="%i"%s>\n%s</ol>' % (startSection, number, startSection, columns, section)
				section = ''
				startSection = number + 1
			if not number % nbArticlesPerPage:
				publishPage(bot, dump, page, startPage, number, pageNumber, debug, firstPage, lastPage, root, summary, introPath)
				model += '\n* [[%s/%i|%i à %i]]' % (root, pageNumber, startPage, number)
				pageNumber += 1
				page = ''
				startPage = number + 1
				if pageNumber > lastPage:
					break
		if number >= startPage and pageNumber <= lastPage:
			if number >= startSection:
				if number % nbArticlesPerSubSection:
					section += '\n<!-- %i à %i -->\n\n%s' % (startSubSection, number, subSection)
				page += '\n\n=== %i à %i ===\n\n<ol start="%i"%s>\n%s</ol>' % (startSection, number, startSection, columns, section)
			publishPage(bot, dump, page, startPage, number, pageNumber, debug, firstPage, lastPage, root, summary, introPath)
			model += '\n* [[%s/%i|%i à %i]]' % (root, pageNumber, startPage, number)
			pageNumber += 1
	model += """
  }}
}}<noinclude>{{Documentation palette}}
[[Catégorie:Palette de navigation espace non encyclopédique|%s]]
</noinclude>""" % subject

	if lastPage != sys.maxint:
		for pageNumber in xrange(max(firstPage, pageNumber), lastPage + 1):
			clearPage(bot, dump, pageNumber, debug, firstPage, lastPage, root, summary, introPath)

	if mode in [1, 4, 7, 8, 9, 10, 11, 12, 13]:
		editOrDebug(bot, 'Modèle:Palette %s' % subject, '%s au %s' % (summary, dump), model, debug, botEdit=True)
Example #7
0
# TODO généraliser le script pour publier n'importe quelle liste d'articles
#  - ./publishList.py -root 'Projet:Portails' -first 1 -last 127 -delay 10 -dump '3 novembre 2010' -subject 'Articles sans portail'
#  - ./publishList.py -root 'Projet:Relance' -first 1 -last 5 -delay 10 -dump '3 novembre 2010' -subject 'Articles rarement modifiés' -transform 'contrib'
# + faire des scripts shell pour chacune des lignes de paramètres fréquemment utilisées
# TODO accepter plusieurs fichiers en entrée (pour différentes sous pages : musique, acteurs), les traiter à la suite et générer une palette commune

import datetime
import getpass
import logging
import sys
import time

import arkbot
import utils

_secondsBetweenEdits = int(utils.getValue('delay', '1'))

def identity(line):
	return line

def link(line):
	if line[0] == '/':
		return '[[:%s]]' % line
	return '[[%s]]' % line

def contrib(line):
	parts = line.split(' || ')
	if parts[4] == 'h**o':
		return False
	return '%s — {{a-court|%s}} ({{u\'|%s}})' % (parts[0], parts[2], parts[3])
Example #8
0
def main(createDB=False, skipTestData=False, skipI18N=False, mysqlRootUsr=None, mysqlRootPwd=None, mysqlUsr=None, mysqlPwd=None, mysqlDb=None, mysqlHost=None, mysqlPort=None):
  
  # Load config
  config=json.load(open("env.json"))

  # Get the values for db/usr/pwd
  mysqlDb  = utils.getValue(mysqlDb , 'MYSQL_DB' )
  mysqlUsr = utils.getValue(mysqlUsr, 'MYSQL_USR', mysqlDb)
  mysqlPwd = utils.getValue(mysqlPwd, 'MYSQL_PWD', mysqlDb)

  # ------------------------------------------------------------ Create Database
  if createDB:
    createDBSql="""
DROP DATABASE IF EXISTS {mysqlDb};
CREATE DATABASE IF NOT EXISTS {mysqlDb};
GRANT ALL PRIVILEGES ON {mysqlDb}.* TO {mysqlUsr}@localhost IDENTIFIED BY '{mysqlPwd}';  
""".format(**locals())  

    # Get the usr/pwd for root, so we can create the database
    mysqlRootUsr = utils.getValue(mysqlRootUsr, 'MYSQLROOT_USR')
    mysqlRootPwd = utils.getValue(mysqlRootPwd, 'MYSQLROOT_PWD')

    print "Creating the database {mysqlDb} ...".format(**locals())
    _runSqlCmd(createDBSql, mysqlRootUsr, mysqlRootPwd, 'mysql', mysqlHost, mysqlPort)
    print "Database created!"

  # ---------------------------------------------------------------- Drop Tables
  print "Dropping tables into the database {mysqlDb} ...".format(**locals())
  _runSqlFile("dropTables.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
  print "Tables removed!"

  # -------------------------------------------------------------- Create Tables
  print "Creating tables into the database {mysqlDb} ...".format(**locals())
  _runSqlFile("createTables.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
  print "Tables created!"

  # ---------------------------------------------------------- Create Permission
  print "Inserting permissions into the database {mysqlDb} ...".format(**locals())
  _runSqlFile("permissions.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
  print "Permissions inserted!"

  # Insert Config Data
  print "Inserting config data into the database {mysqlDb} ...".format(**locals())
  _runSqlFile("insertCfgData.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
  print "Config data inserted!"

  # ---------------------------------------------------------------- Insert I18N
  if skipI18N:
    print "*** Skip insert I18N data"
  else:
    print "Inserting I18N data into the database {mysqlDb}. This can take a while ...".format(**locals())
    _runSqlFile("i18n.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
    print "I18N data inserted!"

  # ----------------------------------------------------------- Insert Test Data
  if skipTestData:
    print "*** Skip insert test data"
  else:
    print "Inserting test data into the database {mysqlDb} ...".format(**locals())
    _runSqlFile("insertTestData.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
    print "Test data inserted!"

  # --------------------------------------------------------- All Modules Config
  print "Inserting all modules config into the database {mysqlDb} ...".format(**locals())
  _runSqlFile("allModules.sql", mysqlUsr, mysqlPwd, mysqlDb, mysqlHost, mysqlPort)
  print "Module config data inserted!"
Example #9
0
# http://github.com/Arkanosis/Wikipedia/Arkbot

# Ce script est mis à disposition sous licence MIT
# http://www.opensource.org/licenses/mit-license.php

import datetime
import getpass
import logging
import sys
import time

import arkbot
import utils

_dump = utils.getValue('dump')
_mode = int(utils.getValue('mode'))
_debug = utils.getOption('debug')

if _mode == 1:
	_page = 'Projet:Pages en impasse/liste des pages en impasse'
	_summary = 'Pages en impasse au %s' % _dump
	_pages = 'en impasse'
elif _mode == 2:
	_page = 'Projet:Pages vides/liste des pages vides'
	_summary = 'Pages vides au %s' % _dump
	_pages = 'vides'
elif _mode == 3:
	_page = 'Utilisateur:Arkbot/Pages redirigeant hors de l\'espace de nom principal'
	_summary = 'Pages redirigeant hors de l\'espace de nom principal au %s' % _dump
	_pages = 'redirigeant hors de l\'espace de nom principal'
Example #10
0
def run(epoch):
    global kt, global_step
    pbar = tqdm.tqdm(dataloader, desc='Epoch %02d' % epoch, dynamic_ncols=True)
    pbar_info = tqdm.tqdm(None, bar_format='{bar}{postfix}')
    avg_acc = 0
    for it, sample in enumerate(pbar, start=iter_init):
        global_step += 1
        image, label, image_s = sample
        image = Variable(image)
        label = Variable(label)
        image_s = Variable(image_s)
        if FLAGS.cuda:
            image = image.cuda()
            label = label.cuda(async=True)
            image_s = image_s.cuda()

        # generator
        outputs = netHg(image)
        loss_hg_content = 0
        for out in outputs:  # TODO: speed up with multiprocessing map?
            loss_hg_content += criterion(out, label)

        # Discriminator
        input_real = torch.cat([image_s, label], dim=1)
        d_real = netD(input_real)
        loss_d_real = criterion_D(d_real, label)

        input_fake = torch.cat([image_s, outputs[-1]], dim=1)
        d_fake = netD(input_fake)
        loss_d_fake = criterion_D(d_fake, label)

        loss_d = loss_d_real - kt * loss_d_fake
        loss_hg = loss_hg_content + FLAGS.lambda_G * loss_d_fake.detach()
        ''' Backward seperatedly '''
        optimD.zero_grad()
        loss_d.backward(retain_graph=True)
        optimD.step()
        optimHg.zero_grad()
        loss_hg.backward()
        # optimD.step()
        optimHg.step()
        ''' Backward all at once (slightly different ?) '''
        # loss_total = loss_d + loss_hg
        # optimD.zero_grad()
        # optimHg.zero_grad()
        # loss_total.backward()
        # optimD.step()
        # optimHg.step()

        # update kt
        loss_d_real_ = getValue(loss_d_real)
        loss_d_fake_ = getValue(loss_d_fake)
        balance = FLAGS.gamma * loss_d_real_ - loss_d_fake_ / FLAGS.lambda_G  # dividing is Good?
        kt = kt + FLAGS.lambda_kt * balance
        kt = min(1, max(0, kt))
        measure = loss_d_real_ + abs(balance)

        accs = accuracy(outputs[-1].data.cpu(), label.data.cpu(),
                        dataset.accIdxs)

        # TODO: summary, printing (maybe tqdm)
        sumWriter.add_scalar('loss_d_real', loss_d_real_, global_step)
        sumWriter.add_scalar('loss_d_fake', loss_d_fake_, global_step)
        sumWriter.add_scalar('loss_d', loss_d, global_step)
        sumWriter.add_scalar('loss_hg_content', loss_hg_content, global_step)
        sumWriter.add_scalar('loss_hg', loss_hg, global_step)
        sumWriter.add_scalar('kt', kt, global_step)
        sumWriter.add_scalar('balance', balance, global_step)
        sumWriter.add_scalar('meauser', measure, global_step)
        sumWriter.add_scalar('acc', accs[0], global_step)
        # TODO: learning rate scheduling
        # sumWriter.add_scalar('lr', lr, global_step)

        pbar_info.set_postfix({
            'balance': balance,
            'loss_hg': getValue(loss_hg),
            'loss_d': getValue(loss_d),
            'acc': accs[0],
        })
        pbar_info.update()
        # pbar.write('hello %d\r' % it, end='')
        avg_acc += accs[0] / len(dataloader)
    pbar_info.set_postfix_str('avg_acc: {}'.format(avg_acc))
    pbar.close()
    pbar_info.close()
Example #11
0
import logging
import sys
import time

import arkbot
import noportal
import post
import utils

if __name__ == '__main__':
	print 'DumpWork 0.1'
	print '(C) 2017 Arkanosis'
	print '*****@*****.**'
	print

	dump = utils.getValue('dump')
	debug = utils.getOption('debug')

	if len(sys.argv) != 2:
		print 'Usage: dumpWork.py <dump>'
		sys.exit(1)

	date = datetime.datetime.now()
	logger = logging.getLogger('ArkbotLogger')
	formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', '%H:%M:%S')
	logger.setLevel(logging.INFO)

	bot = arkbot.Arkbot(arkbot._botName, arkbot._wiki, logger)
	try:
		if not debug:
			bot.login(getpass.getpass('Bot password ? '))
Example #12
0
def sunat():
    ruc = request.args.get('ruc', '')
    resp = getValue('ruc', ruc)
    return jsonify(resp)
Example #13
0
def reniec():
    dni = request.args.get('dni', '')
    resp = getValue('dni', dni)
    return jsonify(resp)
Example #14
0
# TODO généraliser le script pour publier n'importe quelle liste d'articles
#  - ./publishList.py -root 'Projet:Portails' -first 1 -last 127 -delay 10 -dump '3 novembre 2010' -subject 'Articles sans portail'
#  - ./publishList.py -root 'Projet:Relance' -first 1 -last 5 -delay 10 -dump '3 novembre 2010' -subject 'Articles rarement modifiés' -transform 'contrib'
# + faire des scripts shell pour chacune des lignes de paramètres fréquemment utilisées
# TODO accepter plusieurs fichiers en entrée (pour différentes sous pages : musique, acteurs), les traiter à la suite et générer une palette commune

import datetime
import getpass
import logging
import sys
import time

import arkbot
import utils

_dump = utils.getValue('dump')
_mode = int(utils.getValue('mode'))
_debug = utils.getOption('debug')

_secondsBetweenEdits = int(utils.getValue('delay', '5'))

_firstPage = int(utils.getValue('first', 1))
_lastPage = sys.maxint

_nbColumns = 3
_nbArticlesPerSection = 100
_nbSectionsPerPage = 5

_subPage = False

if _debug: