Beispiel #1
0
  def setUp(self):
    self.utils = UtilsForTest()

    self.curdir = os.path.dirname(os.path.realpath(__file__))
    self.examples = os.path.join(self.curdir,'bank') + '/'

    BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)

    '''
Beispiel #2
0
    def setUp(self):
        self.utils = UtilsForTest()

        self.curdir = os.path.dirname(os.path.realpath(__file__))
        self.examples = os.path.join(self.curdir, 'bank') + '/'

        BiomajConfig.load_config(self.utils.global_properties,
                                 allow_user_config=False)
        '''
Beispiel #3
0
  def setUp(self):
    self.utils = UtilsForTest()
    curdir = os.path.dirname(os.path.realpath(__file__))
    BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)

    #Delete all banks
    b = Bank('local')
    b.banks.remove({})

    self.config = BiomajConfig('local')
    data_dir = self.config.get('data.dir')
    lock_file = os.path.join(data_dir,'local.lock')
    if os.path.exists(lock_file):
      os.remove(lock_file)
Beispiel #4
0
    def setUp(self):
        self.utils = UtilsForTest()
        curdir = os.path.dirname(os.path.realpath(__file__))
        BiomajConfig.load_config(self.utils.global_properties,
                                 allow_user_config=False)

        #Delete all banks
        b = Bank('local')
        b.banks.remove({})

        self.config = BiomajConfig('local')
        data_dir = self.config.get('data.dir')
        lock_file = os.path.join(data_dir, 'local.lock')
        if os.path.exists(lock_file):
            os.remove(lock_file)
Beispiel #5
0
    def setUp(self):
        BmajIndex.es = None
        self.utils = UtilsForTest()
        curdir = os.path.dirname(os.path.realpath(__file__))
        BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
        if BmajIndex.do_index == False:
            self.skipTest("Skipping indexing tests due to elasticsearch not available")
        # Delete all banks
        b = Bank("local")
        b.banks.remove({})

        self.config = BiomajConfig("local")
        data_dir = self.config.get("data.dir")
        lock_file = os.path.join(data_dir, "local.lock")
        if os.path.exists(lock_file):
            os.remove(lock_file)
Beispiel #6
0
    def __init__(self):
        # If connector type not set, try to get it from the global.properties
        if not BiomajConfig.global_config:
            BiomajConfig.load_config()
        url = BiomajConfig.global_config.get('GENERAL', 'db.url')
        db = BiomajConfig.global_config.get('GENERAL', 'db.name')
        if url is None:
            raise Exception("No connection url set!")
        if db is None:
            raise Exception("No connection db set!")
        driver = split(url, ':')[0]
        if not driver:
            raise Exception("Can't determine database driver")

        Connector.url = url
        Connector.db = db
        Connector.driver = driver
Beispiel #7
0
    def setUp(self):
        BmajIndex.es = None
        self.utils = UtilsForTest()
        curdir = os.path.dirname(os.path.realpath(__file__))
        BiomajConfig.load_config(self.utils.global_properties,
                                 allow_user_config=False)
        if BmajIndex.do_index == False:
            self.skipTest(
                "Skipping indexing tests due to elasticsearch not available")
        # Delete all banks
        b = Bank('local')
        b.banks.remove({})
        BmajIndex.delete_all_bank('local')

        self.config = BiomajConfig('local')
        data_dir = self.config.get('data.dir')
        lock_file = os.path.join(data_dir, 'local.lock')
        if os.path.exists(lock_file):
            os.remove(lock_file)
Beispiel #8
0
database into a PostgreSQL data using Jsonb data typexs
"""
from __future__ import print_function
from pymongo import MongoClient
import psycopg2
from psycopg2 import OperationalError, DatabaseError, IntegrityError
import json
from biomaj.config import BiomajConfig
import os
import sys

__author__ = 'tuco'

if __name__ == '__main__':

    BiomajConfig.load_config()
    mongo_url = BiomajConfig.global_config.get('GENERAL', 'db.url')
    mongo_db = BiomajConfig.global_config.get('GENERAL', 'db.name')
    mc = MongoClient(mongo_url)
    m_bank = mc[mongo_db].banks
    banks = []
    dbname = 'biomaj'

    insert_query = "INSERT INTO bank(data) VALUES "
    for bank in m_bank.find({}, {'_id': 0}):
        insert_query += "('%s')," % json.dumps(bank)
    insert_query = insert_query.strip(',')

    # In case we empty the databble
    if len(sys.argv) > 1:
        dbname = sys.argv[1]
Beispiel #9
0
 def setUp(self):
     self.utils = UtilsForTest()
     self.curdir = os.path.dirname(os.path.realpath(__file__))
     BiomajConfig.load_config(self.utils.global_properties,
                              allow_user_config=False)
Beispiel #10
0
def main():

    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('-c', '--config', dest="config",help="Configuration file")
    parser.add_argument('--check', dest="check", help="Check bank property file", action="store_true", default=False)
    parser.add_argument('-u', '--update', dest="update", help="Update action", action="store_true", default=False)
    parser.add_argument('--fromscratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False)
    parser.add_argument('-z', '--from-scratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False)
    parser.add_argument('-p', '--publish', dest="publish", help="Publish", action="store_true", default=False)
    parser.add_argument('--unpublish', dest="unpublish", help="Unpublish", action="store_true", default=False)

    parser.add_argument('--release', dest="release", help="release of the bank")
    parser.add_argument('--from-task', dest="from_task", help="Start cycle at a specific task (init always executed)")
    parser.add_argument('--process', dest="process", help="Linked to from-task, optionally specify a block, meta or process name to start from")
    parser.add_argument('-l', '--log', dest="log", help="log level")
    parser.add_argument('-r', '--remove', dest="remove", help="Remove a bank release", action="store_true", default=False)
    parser.add_argument('--remove-all', dest="removeall", help="Remove all bank releases and database records", action="store_true", default=False)
    parser.add_argument('--remove-pending', dest="removepending", help="Remove pending release", action="store_true", default=False)
    parser.add_argument('-s', '--status', dest="status", help="Get status", action="store_true", default=False)
    parser.add_argument('-b', '--bank', dest="bank", help="bank name")
    parser.add_argument('--owner', dest="owner", help="change owner of the bank")
    parser.add_argument('--stop-before', dest="stop_before", help="Store workflow before task")
    parser.add_argument('--stop-after', dest="stop_after", help="Store workflow after task")
    parser.add_argument('--freeze', dest="freeze", help="Freeze a bank release", action="store_true", default=False)
    parser.add_argument('--unfreeze', dest="unfreeze", help="Unfreeze a bank release", action="store_true", default=False)
    parser.add_argument('-f', '--force', dest="force", help="Force action", action="store_true", default=False)
    parser.add_argument('-h', '--help', dest="help", help="Show usage", action="store_true", default=False)

    parser.add_argument('--search', dest="search", help="Search by format and types", action="store_true", default=False)
    parser.add_argument('--formats', dest="formats", help="List of formats to search, comma separated")
    parser.add_argument('--types', dest="types", help="List of types to search, comma separated")
    parser.add_argument('--query', dest="query", help="Lucene query syntax to search in index")

    parser.add_argument('--show', dest="show", help="Show format files for selected bank", action="store_true", default=False)

    parser.add_argument('-n', '--change-dbname', dest="newbank", help="Change old bank name to this new bank name")
    parser.add_argument('-e', '--move-production-directories', dest="newdir",help="Change bank production directories location to this new path, path must exists")
    parser.add_argument('--visibility', dest="visibility", help="visibility status of the bank")

    parser.add_argument('--maintenance', dest="maintenance", help="Maintenance mode (on/off/status)")

    parser.add_argument('--version', dest="version", help="Show version", action="store_true", default=False)
    parser.add_argument('--status-ko', dest="statusko", help="Get bank in KO status", action="store_true", default=False)


    options = Options()
    parser.parse_args(namespace=options)

    options.no_log = False

    if options.help:
        print('''
    --config: global.properties file path

    --status: list of banks with published release
        [OPTIONAL]
        --bank xx / bank: Get status details of bank

    --status-ko: list of banks in error status (last run)

    --log DEBUG|INFO|WARN|ERR  [OPTIONAL]: set log level in logs for this run, default is set in global.properties file

    --check: Check bank property file
        [MANDATORY]
        --bank xx: name of the bank to check (will check xx.properties)

    --owner yy: Change owner of the bank (user id)
        [MANDATORY]
        --bank xx: name of the bank

    --visibility public|private: change visibility public/private of a bank
        [MANDATORY]
        --bank xx: name of the bank

    --change-dbname yy: Change name of the bank to this new name
        [MANDATORY]
        --bank xx: current name of the bank

    --move-production-directories yy: Change bank production directories location to this new path, path must exists
        [MANDATORY]
        --bank xx: current name of the bank

    --update: Update bank
        [MANDATORY]
        --bank xx: name of the bank(s) to update, comma separated
        [OPTIONAL]
        --publish: after update set as *current* version
        --from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1)
        --stop-before xx: stop update cycle before the start of step xx
        --stop-after xx: stop update cycle after step xx has completed
        --from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx*
        --process xx: linked to from-task, optionally specify a block, meta or process name to start from
        --release xx: release to update

    --publish: Publish bank as current release to use
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to publish
    --unpublish: Unpublish bank (remove current)
        [MANDATORY]
        --bank xx: name of the bank to update

    --remove-all: Remove all bank releases and database records
        [MANDATORY]
        --bank xx: name of the bank to update
        [OPTIONAL]
        --force: remove freezed releases

    --remove-pending: Remove pending releases
        [MANDATORY]
        --bank xx: name of the bank to update

    --remove: Remove bank release (files and database release)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

        Release must not be the *current* version. If this is the case, publish a new release before.

    --freeze: Freeze bank release (cannot be removed)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

    --unfreeze: Unfreeze bank release (can be removed)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

    --search: basic search in bank production releases, return list of banks
       --formats xx,yy : list of comma separated format
      AND/OR
       --types xx,yy : list of comma separated type

       --query "LUCENE query syntax": search in index (if activated)

    --show: Show bank files per format
      [MANDATORY]
      --bank xx: name of the bank to show
      [OPTIONAL]
      --release xx: release of the bank to show

    --maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal

        ''')
        return

    if options.version:
        version = pkg_resources.require('biomaj')[0].version
        print('Version: '+str(version))
        return

    if options.stop_after or options.stop_before or options.from_task:
        available_steps = []
        for flow in UpdateWorkflow.FLOW:
            available_steps.append(flow['name'])
        for flow in RemoveWorkflow.FLOW:
            available_steps.append(flow['name'])
        if options.stop_after:
            if options.stop_after not in available_steps:
                print('Invalid step: '+options.stop_after)
                sys.exit(1)
        if options.stop_before:
            if options.stop_before not in available_steps:
                print('Invalid step: '+options.stop_before)
                sys.exit(1)
        if options.from_task:
            if options.from_task not in available_steps:
                print('Invalid step: '+options.from_task)
                sys.exit(1)

    bmaj = None
    try:
        if options.config is not None:
            BiomajConfig.load_config(options.config)
        else:
            BiomajConfig.load_config()
    except Exception as e:
        print(str(e))
        sys.exit(1)

    try:

        if options.maintenance:
            if options.maintenance not in ['on', 'off', 'status']:
                print("Wrong maintenance value [on,off,status]")
                sys.exit(1)
            data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir')
            if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'):
                lock_dir = BiomajConfig.global_config.get('GENERAL', 'lock.dir')
            else:
                lock_dir = data_dir
            maintenance_lock_file = os.path.join(lock_dir,'biomaj.lock')
            if options.maintenance == 'status':
                if os.path.exists(maintenance_lock_file):
                    print("Maintenance: On")
                else:
                    print("Maintenance: Off")
                sys.exit(0)
            if options.maintenance == 'on':
                f = open(maintenance_lock_file, 'w')
                f.write('1')
                f.close()
                print("Maintenance set to On")
                sys.exit(0)
            if options.maintenance == 'off':
                if os.path.exists(maintenance_lock_file):
                    os.remove(maintenance_lock_file)
                print("Maintenance set to Off")
                sys.exit(0)

        if options.owner:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            bank.set_owner(options.owner)
            sys.exit(0)

        if options.visibility:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            if options.visibility not in ['public', 'private']:
                print("Valid values are public|private")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            bank.set_visibility(options.visibility)
            print("Do not forget to update accordingly the visibility.default parameter in the configuration file")
            sys.exit(0)

        if options.newdir:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            if not os.path.exists(options.newdir):
                print("Destination directory does not exists")
            bank = Bank(options.bank, options=options, no_log=True)
            if not bank.bank['production']:
                print("Nothing to move, no production directory")
                sys.exit(0)
            bank.load_session(Workflow.FLOW, None)
            w = Workflow(bank)
            res = w.wf_init()
            if not res:
                sys.exit(1)
            for prod in bank.bank['production']:
                session = bank.get_session_from_release(prod['release'])
                bank.load_session(Workflow.FLOW, session)
                prod_path = bank.session.get_full_release_directory()
                if os.path.exists(prod_path):
                    shutil.move(prod_path, options.newdir)
                prod['data_dir'] = options.newdir
            bank.banks.update({'name': options.bank}, {'$set' : { 'production': bank.bank['production'] }})
            print("Bank production directories moved to " + options.newdir)
            print("WARNING: do not forget to update accordingly the data.dir and dir.version properties")
            w.wf_over()
            sys.exit(0)

        if options.newbank:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
            bank_prop_file = os.path.join(conf_dir,options.bank+'.properties')
            config_bank = configparser.SafeConfigParser()
            config_bank.read([os.path.join(conf_dir,options.bank+'.properties')])
            config_bank.set('GENERAL', 'db.name', options.newbank)
            newbank_prop_file = open(os.path.join(conf_dir,options.newbank+'.properties'),'w')
            config_bank.write(newbank_prop_file)
            newbank_prop_file.close()
            bank.banks.update({'name': options.bank}, {'$set' : { 'name': options.newbank }})
            os.remove(bank_prop_file)
            print("Bank "+options.bank+" renamed to "+options.newbank)
            sys.exit(0)

        if options.search:
            if options.query:
                res = Bank.searchindex(options.query)
                print("Query matches for :"+options.query)
                results = [["Release", "Format(s)", "Type(s)", "Files"]]
                for match in res:
                    results.append([match['_source']['release'],
                                    str(match['_source']['format']),
                                    str(match['_source']['types']),
                                    ','.join(match['_source']['files'])])
                print(tabulate(results, headers="firstrow", tablefmt="grid"))
            else:
                formats = []
                if options.formats:
                    formats = options.formats.split(',')
                types = []
                if options.types:
                    types = options.types.split(',')
                print("Search by formats="+str(formats)+", types="+str(types))
                res = Bank.search(formats, types, False)
                results = [["Name", "Release", "Format(s)", "Type(s)", 'Current']]
                for bank in sorted(res, key=lambda bank: (bank['name'])):
                    b = bank['name']
                    bank['production'].sort(key=lambda n: n['release'], reverse=True)
                    for prod in bank['production']:
                        iscurrent = ""
                        if prod['session'] == bank['current']:
                            iscurrent = "yes"
                        results.append([b if b else '', prod['release'], ','.join(prod['formats']),
                                        ','.join(prod['types']), iscurrent])
                        b = None
                print(tabulate(results, headers="firstrow", tablefmt="grid"))
                sys.exit(0)

        if options.show:
            if not options.bank:
                print("Bank option is required")
                sys.exit(1)

            bank = Bank(options.bank, no_log=True)
            results = [["Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"]]
            current = None
            fformat = None
            if 'current' in bank.bank and bank.bank['current']:
                current = bank.bank['current']
            for prod in bank.bank['production']:
                include = True
                release = prod['release']
                if current == prod['session']:
                    release += ' (current)'
                if options.release and (prod['release'] != options.release and prod['prod_dir'] != options.release):
                    include =False
                if include:
                    session = bank.get_session_from_release(prod['release'])
                    formats = session['formats']
                    afiles = []
                    atags = []
                    atypes = []
                    for fformat in list(formats.keys()):
                        for elt in formats[fformat]:
                            atypes.append(','.join(elt['types']))
                            for tag in list(elt['tags'].keys()):
                                atags.append(elt['tags'][tag])
                            for eltfile in elt['files']:
                                afiles.append(eltfile)
                    results.append([bank.bank['name'], release, fformat, ','.join(atypes),
                                ','.join(atags), ','.join(afiles)])
            print(tabulate(results, headers="firstrow", tablefmt="grid"))
            sys.exit(0)

        if options.check:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            print(options.bank+" check: "+str(bank.check())+"\n")
            sys.exit(0)

        if options.status:
            if options.bank:
                bank = Bank(options.bank, no_log=True)
                info = bank.get_bank_release_info(full=True)
                print(tabulate(info['info'], headers='firstrow', tablefmt='psql'))
                print(tabulate(info['prod'], headers='firstrow', tablefmt='psql'))
                # do we have some pending release(s)
                if 'pend' in info and len(info['pend']) > 1:
                    print(tabulate(info['pend'], headers='firstrow', tablefmt='psql'))
            else:
                banks = Bank.list()
                # Headers of output table
                banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
                for bank in sorted(banks, key=lambda k: k['name']):
                    bank = Bank(bank['name'], no_log=True)
                    banks_list.append(bank.get_bank_release_info()['info'])
                print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))
            sys.exit(0)

        if options.statusko:
            banks = Bank.list()
            banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
            for bank in sorted(banks, key=lambda k: k['name']):
                try:
                    bank = Bank(bank['name'], no_log=True)
                    bank.load_session(UpdateWorkflow.FLOW)
                    if bank.session is not None:
                        if bank.use_last_session and not bank.session.get_status(Workflow.FLOW_OVER):
                            wf_status = bank.session.get('workflow_status')
                            if wf_status is None or not wf_status:
                                banks_list.append(bank.get_bank_release_info()['info'])
                except Exception as e:
                    print(str(e))
            print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))

        if options.update:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            banks = options.bank.split(',')
            gres = True
            for bank in banks:
                options.bank = bank
                bmaj = Bank(bank, options)
                print('Log file: '+bmaj.config.log_file)
                check_status = bmaj.check()
                if not check_status:
                    print('Skip bank ' + options.bank + ': wrong config')
                    gres = False
                    continue
                res = bmaj.update(depends=True)
                if not res:
                    gres = False
                Notify.notifyBankAction(bmaj)
            if not gres:
                sys.exit(1)

        if options.freeze:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if not options.release:
                print("Bank release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options)
            res = bmaj.freeze(options.release)
            if not res:
                sys.exit(1)

        if options.unfreeze:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if not options.release:
                print("Bank release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options)
            res = bmaj.unfreeze(options.release)
            if not res:
                sys.exit(1)

        if options.remove or options.removeall:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if options.remove and not options.release:
                print("Bank release is missing")
                sys.exit(1)
            if options.removeall:
                bmaj = Bank(options.bank, options, no_log=True)
                print('Log file: '+bmaj.config.log_file)
                res = bmaj.removeAll(options.force)
            else:
                bmaj = Bank(options.bank, options)
                print('Log file: '+bmaj.config.log_file)
                res = bmaj.remove(options.release)
                Notify.notifyBankAction(bmaj)
            if not res:
                sys.exit(1)

        if options.removepending:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            print('Log file: '+bmaj.config.log_file)
            res = bmaj.remove_pending(options.release)
            if not res:
                sys.exit(1)

        if options.unpublish:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            bmaj.load_session()
            bmaj.unpublish()
            sys.exit(0)

        if options.publish:
            if not options.bank:
                print("Bank name or release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            print('Log file: '+bmaj.config.log_file)
            bmaj.load_session()
            bank = bmaj.bank
            session = None
            if options.get_option('release') is None:
                # Get latest prod release
                if len(bank['production'])>0:
                    prod = bank['production'][len(bank['production'])-1]
                    for s in bank['sessions']:
                        if s['id'] == prod['session']:
                            session = s
                            break
            else:
                # Search production release matching release
                for prod in bank['production']:
                    if prod['release'] == options.release or prod['prod_dir'] == options.release:
                        # Search session related to this production release
                        for s in bank['sessions']:
                            if s['id'] == prod['session']:
                                session = s
                                break
                        break
            if session is None:
                print("No production session could be found for this release")
                sys.exit(1)
            bmaj.session._session = session
            bmaj.publish()
    except Exception as e:
        print(str(e))
Beispiel #11
0
 def setUp(self):
   self.utils = UtilsForTest()
   BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
   self.config = BiomajConfig('testhttp')
Beispiel #12
0
def main():

    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('-c',
                        '--config',
                        dest="config",
                        help="Configuration file")
    parser.add_argument('--check',
                        dest="check",
                        help="Check bank property file",
                        action="store_true",
                        default=False)
    parser.add_argument('-u',
                        '--update',
                        dest="update",
                        help="Update action",
                        action="store_true",
                        default=False)
    parser.add_argument('--fromscratch',
                        dest="fromscratch",
                        help="Force a new cycle update",
                        action="store_true",
                        default=False)
    parser.add_argument('-z',
                        '--from-scratch',
                        dest="fromscratch",
                        help="Force a new cycle update",
                        action="store_true",
                        default=False)
    parser.add_argument('-p',
                        '--publish',
                        dest="publish",
                        help="Publish",
                        action="store_true",
                        default=False)
    parser.add_argument('--unpublish',
                        dest="unpublish",
                        help="Unpublish",
                        action="store_true",
                        default=False)

    parser.add_argument('--release',
                        dest="release",
                        help="release of the bank")
    parser.add_argument(
        '--from-task',
        dest="from_task",
        help="Start cycle at a specific task (init always executed)")
    parser.add_argument(
        '--process',
        dest="process",
        help=
        "Linked to from-task, optionally specify a block, meta or process name to start from"
    )
    parser.add_argument('-l', '--log', dest="log", help="log level")
    parser.add_argument('-r',
                        '--remove',
                        dest="remove",
                        help="Remove a bank release",
                        action="store_true",
                        default=False)
    parser.add_argument('--remove-all',
                        dest="removeall",
                        help="Remove all bank releases and database records",
                        action="store_true",
                        default=False)
    parser.add_argument('--remove-pending',
                        dest="removepending",
                        help="Remove pending release",
                        action="store_true",
                        default=False)
    parser.add_argument('-s',
                        '--status',
                        dest="status",
                        help="Get status",
                        action="store_true",
                        default=False)
    parser.add_argument('-b', '--bank', dest="bank", help="bank name")
    parser.add_argument('--owner',
                        dest="owner",
                        help="change owner of the bank")
    parser.add_argument('--stop-before',
                        dest="stop_before",
                        help="Store workflow before task")
    parser.add_argument('--stop-after',
                        dest="stop_after",
                        help="Store workflow after task")
    parser.add_argument('--freeze',
                        dest="freeze",
                        help="Freeze a bank release",
                        action="store_true",
                        default=False)
    parser.add_argument('--unfreeze',
                        dest="unfreeze",
                        help="Unfreeze a bank release",
                        action="store_true",
                        default=False)
    parser.add_argument('-f',
                        '--force',
                        dest="force",
                        help="Force action",
                        action="store_true",
                        default=False)
    parser.add_argument('-h',
                        '--help',
                        dest="help",
                        help="Show usage",
                        action="store_true",
                        default=False)

    parser.add_argument('--search',
                        dest="search",
                        help="Search by format and types",
                        action="store_true",
                        default=False)
    parser.add_argument('--formats',
                        dest="formats",
                        help="List of formats to search, comma separated")
    parser.add_argument('--types',
                        dest="types",
                        help="List of types to search, comma separated")
    parser.add_argument('--query',
                        dest="query",
                        help="Lucene query syntax to search in index")

    parser.add_argument('--show',
                        dest="show",
                        help="Show format files for selected bank",
                        action="store_true",
                        default=False)

    parser.add_argument('-n',
                        '--change-dbname',
                        dest="newbank",
                        help="Change old bank name to this new bank name")
    parser.add_argument(
        '-e',
        '--move-production-directories',
        dest="newdir",
        help=
        "Change bank production directories location to this new path, path must exists"
    )
    parser.add_argument('--visibility',
                        dest="visibility",
                        help="visibility status of the bank")

    parser.add_argument('--maintenance',
                        dest="maintenance",
                        help="Maintenance mode (on/off/status)")

    parser.add_argument('--version',
                        dest="version",
                        help="Show version",
                        action="store_true",
                        default=False)
    parser.add_argument('--status-ko',
                        dest="statusko",
                        help="Get bank in KO status",
                        action="store_true",
                        default=False)

    options = Options()
    parser.parse_args(namespace=options)

    options.no_log = False

    if options.help:
        print('''
    --config: global.properties file path

    --status: list of banks with published release
        [OPTIONAL]
        --bank xx / bank: Get status details of bank

    --status-ko: list of banks in error status (last run)

    --log DEBUG|INFO|WARN|ERR  [OPTIONAL]: set log level in logs for this run, default is set in global.properties file

    --check: Check bank property file
        [MANDATORY]
        --bank xx: name of the bank to check (will check xx.properties)

    --owner yy: Change owner of the bank (user id)
        [MANDATORY]
        --bank xx: name of the bank

    --visibility public|private: change visibility public/private of a bank
        [MANDATORY]
        --bank xx: name of the bank

    --change-dbname yy: Change name of the bank to this new name
        [MANDATORY]
        --bank xx: current name of the bank

    --move-production-directories yy: Change bank production directories location to this new path, path must exists
        [MANDATORY]
        --bank xx: current name of the bank

    --update: Update bank
        [MANDATORY]
        --bank xx: name of the bank(s) to update, comma separated
        [OPTIONAL]
        --publish: after update set as *current* version
        --from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1)
        --stop-before xx: stop update cycle before the start of step xx
        --stop-after xx: stop update cycle after step xx has completed
        --from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx*
        --process xx: linked to from-task, optionally specify a block, meta or process name to start from
        --release xx: release to update

    --publish: Publish bank as current release to use
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to publish
    --unpublish: Unpublish bank (remove current)
        [MANDATORY]
        --bank xx: name of the bank to update

    --remove-all: Remove all bank releases and database records
        [MANDATORY]
        --bank xx: name of the bank to update
        [OPTIONAL]
        --force: remove freezed releases

    --remove-pending: Remove pending releases
        [MANDATORY]
        --bank xx: name of the bank to update

    --remove: Remove bank release (files and database release)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

        Release must not be the *current* version. If this is the case, publish a new release before.

    --freeze: Freeze bank release (cannot be removed)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

    --unfreeze: Unfreeze bank release (can be removed)
        [MANDATORY]
        --bank xx: name of the bank to update
        --release xx: release of the bank to remove

    --search: basic search in bank production releases, return list of banks
       --formats xx,yy : list of comma separated format
      AND/OR
       --types xx,yy : list of comma separated type

       --query "LUCENE query syntax": search in index (if activated)

    --show: Show bank files per format
      [MANDATORY]
      --bank xx: name of the bank to show
      [OPTIONAL]
      --release xx: release of the bank to show

    --maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal

        ''')
        return

    if options.version:
        version = pkg_resources.require('biomaj')[0].version
        print('Version: ' + str(version))
        return

    if options.stop_after or options.stop_before or options.from_task:
        available_steps = []
        for flow in UpdateWorkflow.FLOW:
            available_steps.append(flow['name'])
        for flow in RemoveWorkflow.FLOW:
            available_steps.append(flow['name'])
        if options.stop_after:
            if options.stop_after not in available_steps:
                print('Invalid step: ' + options.stop_after)
                sys.exit(1)
        if options.stop_before:
            if options.stop_before not in available_steps:
                print('Invalid step: ' + options.stop_before)
                sys.exit(1)
        if options.from_task:
            if options.from_task not in available_steps:
                print('Invalid step: ' + options.from_task)
                sys.exit(1)

    bmaj = None
    try:
        if options.config is not None:
            BiomajConfig.load_config(options.config)
        else:
            BiomajConfig.load_config()
    except Exception as e:
        print(str(e))
        sys.exit(1)

    try:

        if options.maintenance:
            if options.maintenance not in ['on', 'off', 'status']:
                print("Wrong maintenance value [on,off,status]")
                sys.exit(1)
            data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir')
            if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'):
                lock_dir = BiomajConfig.global_config.get(
                    'GENERAL', 'lock.dir')
            else:
                lock_dir = data_dir
            maintenance_lock_file = os.path.join(lock_dir, 'biomaj.lock')
            if options.maintenance == 'status':
                if os.path.exists(maintenance_lock_file):
                    print("Maintenance: On")
                else:
                    print("Maintenance: Off")
                sys.exit(0)
            if options.maintenance == 'on':
                f = open(maintenance_lock_file, 'w')
                f.write('1')
                f.close()
                print("Maintenance set to On")
                sys.exit(0)
            if options.maintenance == 'off':
                if os.path.exists(maintenance_lock_file):
                    os.remove(maintenance_lock_file)
                print("Maintenance set to Off")
                sys.exit(0)

        if options.owner:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            bank.set_owner(options.owner)
            sys.exit(0)

        if options.visibility:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            if options.visibility not in ['public', 'private']:
                print("Valid values are public|private")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            bank.set_visibility(options.visibility)
            print(
                "Do not forget to update accordingly the visibility.default parameter in the configuration file"
            )
            sys.exit(0)

        if options.newdir:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            if not os.path.exists(options.newdir):
                print("Destination directory does not exists")
            bank = Bank(options.bank, options=options, no_log=True)
            if not bank.bank['production']:
                print("Nothing to move, no production directory")
                sys.exit(0)
            bank.load_session(Workflow.FLOW, None)
            w = Workflow(bank)
            res = w.wf_init()
            if not res:
                sys.exit(1)
            for prod in bank.bank['production']:
                session = bank.get_session_from_release(prod['release'])
                bank.load_session(Workflow.FLOW, session)
                prod_path = bank.session.get_full_release_directory()
                if os.path.exists(prod_path):
                    shutil.move(prod_path, options.newdir)
                prod['data_dir'] = options.newdir
            bank.banks.update(
                {'name': options.bank},
                {'$set': {
                    'production': bank.bank['production']
                }})
            print("Bank production directories moved to " + options.newdir)
            print(
                "WARNING: do not forget to update accordingly the data.dir and dir.version properties"
            )
            w.wf_over()
            sys.exit(0)

        if options.newbank:
            if not options.bank:
                print("Bank option is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
            bank_prop_file = os.path.join(conf_dir,
                                          options.bank + '.properties')
            config_bank = configparser.SafeConfigParser()
            config_bank.read(
                [os.path.join(conf_dir, options.bank + '.properties')])
            config_bank.set('GENERAL', 'db.name', options.newbank)
            newbank_prop_file = open(
                os.path.join(conf_dir, options.newbank + '.properties'), 'w')
            config_bank.write(newbank_prop_file)
            newbank_prop_file.close()
            bank.banks.update({'name': options.bank},
                              {'$set': {
                                  'name': options.newbank
                              }})
            os.remove(bank_prop_file)
            print("Bank " + options.bank + " renamed to " + options.newbank)
            sys.exit(0)

        if options.search:
            if options.query:
                res = Bank.searchindex(options.query)
                print("Query matches for :" + options.query)
                results = [["Release", "Format(s)", "Type(s)", "Files"]]
                for match in res:
                    results.append([
                        match['_source']['release'],
                        str(match['_source']['format']),
                        str(match['_source']['types']),
                        ','.join(match['_source']['files'])
                    ])
                print(tabulate(results, headers="firstrow", tablefmt="grid"))
            else:
                formats = []
                if options.formats:
                    formats = options.formats.split(',')
                types = []
                if options.types:
                    types = options.types.split(',')
                print("Search by formats=" + str(formats) + ", types=" +
                      str(types))
                res = Bank.search(formats, types, False)
                results = [[
                    "Name", "Release", "Format(s)", "Type(s)", 'Published'
                ]]
                for bank in sorted(res, key=lambda bank: (bank['name'])):
                    b = bank['name']
                    bank['production'].sort(key=lambda n: n['release'],
                                            reverse=True)
                    for prod in bank['production']:
                        iscurrent = ""
                        if prod['session'] == bank['current']:
                            iscurrent = "yes"
                        results.append([
                            b if b else '', prod['release'],
                            ','.join(prod['formats']), ','.join(prod['types']),
                            iscurrent
                        ])
                print(tabulate(results, headers="firstrow", tablefmt="grid"))
                sys.exit(0)

        if options.show:
            if not options.bank:
                print("Bank option is required")
                sys.exit(1)

            bank = Bank(options.bank, no_log=True)
            results = [[
                "Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"
            ]]
            current = None
            fformat = None
            if 'current' in bank.bank and bank.bank['current']:
                current = bank.bank['current']
            for prod in bank.bank['production']:
                include = True
                release = prod['release']
                if current == prod['session']:
                    release += ' (current)'
                if options.release and (prod['release'] != options.release and
                                        prod['prod_dir'] != options.release):
                    include = False
                if include:
                    session = bank.get_session_from_release(prod['release'])
                    formats = session['formats']
                    afiles = []
                    atags = []
                    atypes = []
                    for fformat in list(formats.keys()):
                        for elt in formats[fformat]:
                            atypes.append(','.join(elt['types']))
                            for tag in list(elt['tags'].keys()):
                                atags.append(elt['tags'][tag])
                            for eltfile in elt['files']:
                                afiles.append(eltfile)
                    results.append([
                        bank.bank['name'], release, fformat, ','.join(atypes),
                        ','.join(atags), ','.join(afiles)
                    ])
            print(tabulate(results, headers="firstrow", tablefmt="grid"))
            sys.exit(0)

        if options.check:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bank = Bank(options.bank, no_log=True)
            print(options.bank + " check: " + str(bank.check()) + "\n")
            sys.exit(0)

        if options.status:
            if options.bank:
                bank = Bank(options.bank, no_log=True)
                info = bank.get_bank_release_info(full=True)
                print(
                    tabulate(info['info'], headers='firstrow',
                             tablefmt='psql'))
                print(
                    tabulate(info['prod'], headers='firstrow',
                             tablefmt='psql'))
                # do we have some pending release(s)
                if 'pend' in info and len(info['pend']) > 1:
                    print(
                        tabulate(info['pend'],
                                 headers='firstrow',
                                 tablefmt='psql'))
            else:
                banks = Bank.list()
                # Headers of output table
                banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
                for bank in sorted(banks, key=lambda k: k['name']):
                    bank = Bank(bank['name'], no_log=True)
                    banks_list.append(bank.get_bank_release_info()['info'])
                print(tabulate(banks_list, headers="firstrow",
                               tablefmt="psql"))
            sys.exit(0)

        if options.statusko:
            banks = Bank.list()
            banks_list = [[
                "Name", "Type(s)", "Release", "Visibility", "Last run"
            ]]
            for bank in sorted(banks, key=lambda k: k['name']):
                try:
                    bank = Bank(bank['name'], no_log=True)
                    bank.load_session(UpdateWorkflow.FLOW)
                    if bank.session is not None:
                        if bank.use_last_session and not bank.session.get_status(
                                Workflow.FLOW_OVER):
                            wf_status = bank.session.get('workflow_status')
                            if wf_status is None or not wf_status:
                                banks_list.append(
                                    bank.get_bank_release_info()['info'])
                except Exception as e:
                    print(str(e))
            print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))

        if options.update:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            banks = options.bank.split(',')
            gres = True
            for bank in banks:
                options.bank = bank
                bmaj = Bank(bank, options)
                print('Log file: ' + bmaj.config.log_file)
                check_status = bmaj.check()
                if not check_status:
                    print('Skip bank ' + options.bank + ': wrong config')
                    gres = False
                    continue
                res = bmaj.update(depends=True)
                if not res:
                    gres = False
                Notify.notifyBankAction(bmaj)
            if not gres:
                sys.exit(1)

        if options.freeze:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if not options.release:
                print("Bank release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options)
            res = bmaj.freeze(options.release)
            if not res:
                sys.exit(1)

        if options.unfreeze:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if not options.release:
                print("Bank release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options)
            res = bmaj.unfreeze(options.release)
            if not res:
                sys.exit(1)

        if options.remove or options.removeall:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            if options.remove and not options.release:
                print("Bank release is missing")
                sys.exit(1)
            if options.removeall:
                bmaj = Bank(options.bank, options, no_log=True)
                print('Log file: ' + bmaj.config.log_file)
                res = bmaj.removeAll(options.force)
            else:
                bmaj = Bank(options.bank, options)
                print('Log file: ' + bmaj.config.log_file)
                res = bmaj.remove(options.release)
                Notify.notifyBankAction(bmaj)
            if not res:
                sys.exit(1)

        if options.removepending:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            res = bmaj.remove_pending(options.release)
            if not res:
                sys.exit(1)

        if options.unpublish:
            if not options.bank:
                print("Bank name is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            bmaj.load_session()
            bmaj.unpublish()
            sys.exit(0)

        if options.publish:
            if not options.bank:
                print("Bank name or release is missing")
                sys.exit(1)
            bmaj = Bank(options.bank, options, no_log=True)
            bmaj.load_session()
            bank = bmaj.bank
            session = None
            if options.get_option('release') is None:
                # Get latest prod release
                if len(bank['production']) > 0:
                    prod = bank['production'][len(bank['production']) - 1]
                    for s in bank['sessions']:
                        if s['id'] == prod['session']:
                            session = s
                            break
            else:
                # Search production release matching release
                for prod in bank['production']:
                    if prod['release'] == options.release or prod[
                            'prod_dir'] == options.release:
                        # Search session related to this production release
                        for s in bank['sessions']:
                            if s['id'] == prod['session']:
                                session = s
                                break
                        break
            if session is None:
                print("No production session could be found for this release")
                sys.exit(1)
            bmaj.session._session = session
            bmaj.publish()
    except Exception as e:
        print(str(e))
Beispiel #13
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-c',
                        '--config',
                        dest="config",
                        help="Biomaj3 Configuration file")
    parser.add_argument('-o',
                        '--oldconfig',
                        dest="oldconfig",
                        help="Old configuration file")
    parser.add_argument('-u',
                        '--user',
                        dest="user",
                        help="MySQL user to override global properties")
    parser.add_argument('-p',
                        '--password',
                        dest="password",
                        help="MySQL password to override global properties")
    parser.add_argument('-l',
                        '--host',
                        dest="host",
                        help="MySQL host to override global properties")
    parser.add_argument('-d',
                        '--database',
                        dest="database",
                        help="MySQL database to override global properties")
    parser.add_argument('-H',
                        '--keep_history',
                        dest="history",
                        action="store_true",
                        default=False,
                        help="Keep bank history, not only production")

    args = parser.parse_args()

    biomajconfig = {}
    banks = []
    with open(args.oldconfig, 'r') as old:
        for line in old:
            vals = line.split('=')
            if len(vals) > 1:
                biomajconfig[vals[0].strip()] = vals[1].strip()

    BiomajConfig.load_config(args.config, allow_user_config=False)
    db_properties_dir = os.path.dirname(args.oldconfig)
    if db_properties_dir == os.path.dirname(args.config):
        logging.error(
            "Bank properties use the same directory, please use a different conf.dir"
        )
        sys.exit(1)

    data_dir = biomajconfig['data.dir']
    if data_dir.endswith('/'):
        data_dir = data_dir[:-1]

    if not os.path.dirname(data_dir) == os.path.dirname(
            BiomajConfig.global_config.get('GENERAL', 'data.dir')):
        logging.error('Data dirs are different, please use the same data dirs')
        sys.exit(1)

    prop_files = []
    for root, dirnames, filenames in os.walk(db_properties_dir):
        for filename in fnmatch.filter(filenames, '*.properties'):
            if filename != 'global.properties':
                prop_files.append(os.path.join(root, filename))

    if not os.path.exists(BiomajConfig.global_config.get(
            'GENERAL', 'conf.dir')):
        os.makedirs(BiomajConfig.global_config.get('GENERAL', 'conf.dir'))
    for prop_file in prop_files:

        propbankconfig = {}
        with open(prop_file, 'r') as old:
            for line in old:
                vals = line.split('=')
                if len(vals) > 1:
                    propbankconfig[vals[0].strip()] = vals[1].strip()

        newpropfile = os.path.join(
            BiomajConfig.global_config.get('GENERAL', 'conf.dir'),
            os.path.basename(prop_file))
        newprop = open(newpropfile, 'w')
        # logging.warn("manage "+prop_file+" => "+newpropfile)
        newprop.write("[GENERAL]\n")
        with open(prop_file, 'r') as props:
            for line in props:
                if not (line.startswith('*') or line.startswith('/*')):
                    # Replace config variables with new syntax ${xx} => %(xx)s, not other env variables
                    pattern = re.compile("\$\{([a-zA-Z0-9-_.]+)\}")
                    varmatch = pattern.findall(line)
                    if varmatch:
                        for match in varmatch:
                            if match in biomajconfig or match in propbankconfig:
                                line = line.replace('${' + match + '}',
                                                    '%(' + match + ')s')
                newprop.write(
                    line.replace('\\\\', '\\').replace('db.source', 'depends'))
        newprop.close()
        b = Bank(os.path.basename(prop_file).replace('.properties', ''),
                 no_log=True)
        banks.append(b.name)

    # database.url=jdbc\:mysql\://genobdd.genouest.org/biomaj_log
    vals = biomajconfig['database.url'].split('/')
    urllen = len(vals)
    db_name = vals[urllen - 1]
    if args.database:
        db_name = args.database
    db_host = vals[urllen - 2]
    if args.host:
        db_host = args.host
    db_user = biomajconfig['database.login']
    if args.user:
        db_user = args.user
    db_password = biomajconfig['database.password']
    if args.password:
        db_password = args.password

    try:
        cnx = mysql.connector.connect(host=db_host,
                                      database=db_name,
                                      user=db_user,
                                      password=db_password)
        cur = cnx.cursor()
        cur.execute("SELECT name FROM bank")
        for row in cur.fetchall():
            migrate_bank(cur, row[0], history=args.history)
    except mysql.connector.Error as error:
        if error.errno == errorcode.ER_ACCESS_DENIED_ERROR:
            print("Wrong username or password: %s" % error.msg)
        elif error.errno == errorcode.ER_BAD_DB_ERROR:
            print("Database does not exist: %s" % error.msg)
        else:
            print("Unknown error: %s" % error)
    finally:
        cnx.close()
Beispiel #14
0
 def setUp(self):
     self.utils = UtilsForTest()
     BiomajConfig.load_config(self.utils.global_properties,
                              allow_user_config=False)
     self.config = BiomajConfig('testhttp')
Beispiel #15
0
def load_config(request):
  if BiomajConfig.global_config is None:
    settings = request.registry.settings
    global_properties = settings['global_properties']
    BiomajConfig.load_config(global_properties)
Beispiel #16
0
def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    #config = Configurator(settings=settings)
    global_properties = settings.get(
               'global_properties', '/etc/biomaj/global.properties')
    if not os.path.exists(global_properties):
      print 'global.properties configuration field is not set'
      sys.exit(1)

    BiomajConfig.load_config(global_properties)

    settings['global_properties'] = global_properties

    config = Configurator(settings=settings)
    config.include('pyramid_chameleon')

    config.add_subscriber(before_render, BeforeRender)

    authentication_policy = AuthTktAuthenticationPolicy('seekrit',
        callback=None, hashalg='sha512')
    authorization_policy = ACLAuthorizationPolicy()

    config.set_authentication_policy(authentication_policy)
    config.set_authorization_policy(authorization_policy)




    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_static_view('app', 'biomajwatcher:webapp/app')
    config.add_route('home', '/')

    config.add_route('user','/user')
    config.add_route('user_banks','/user/{id}/banks')

    config.add_route('bank', '/bank')
    config.add_route('bankdetails', '/bank/{id}')
    config.add_route('banklocked', '/bank/{id}/locked')
    config.add_route('bankstatus', '/bank/{id}/status')
    config.add_route('bankconfig', '/bank/{id}/config')
    config.add_route('bankreleaseremove', '/bank/{id}/{release}')
    config.add_route('sessionlog', '/bank/{id}/log/{session}')

    config.add_route('schedulebank','/schedule')
    config.add_route('updateschedulebank','/schedule/{name}')

    config.add_route('search', '/search')

    config.add_route('search_format', '/search/format/{format}')
    config.add_route('search_format_type', '/search/format/{format}/type/{type}')
    config.add_route('search_type', '/search/type/{type}')

    config.add_route('stat', '/stat')

    config.add_route('is_auth', '/auth')
    config.add_route('auth', '/auth/{id}')
    config.add_route('logout', '/logout')

    config.add_route('old_api', 'BmajWatcher/GET')

    config.scan()


    # automatically serialize bson ObjectId and datetime to Mongo extended JSON
    json_renderer = JSON()
    def pymongo_adapter(obj, request):
        return json_util.default(obj)
    json_renderer.add_adapter(ObjectId, pymongo_adapter)
    json_renderer.add_adapter(datetime.datetime, pymongo_adapter)

    config.add_renderer('json', json_renderer)


    return config.make_wsgi_app()
Beispiel #17
0
    def migrate_pendings():
        """
        Migrate database

        3.0.18: Check the actual BioMAJ version and if older than 3.0.17, do the 'pending' key migration
        """
        if BiomajConfig.global_config is None:
            try:
                BiomajConfig.load_config()
            except Exception as err:
                print("* SchemaVersion: Can't find config file")
                return None
        if MongoConnector.db is None:
            MongoConnector(
                BiomajConfig.global_config.get('GENERAL', 'db.url'),
                BiomajConfig.global_config.get('GENERAL', 'db.name'))

        schema = MongoConnector.db_schema
        banks = MongoConnector.banks

        schema_version = schema.find_one({'id': 1})
        installed_version = pkg_resources.get_distribution("biomaj").version
        if schema_version is None:
            schema_version = {'id': 1, 'version': '3.0.0'}
            schema.insert(schema_version)

        moderate = int(schema_version['version'].split('.')[1])
        minor = int(schema_version['version'].split('.')[2])

        if moderate == 0 and minor <= 17:
            print("Migrate from release: %s" % schema_version['version'])
            # Update pending releases
            bank_list = banks.find()
            updated = 0
            for bank in bank_list:
                if 'pending' in bank:
                    # Check we have an old pending type
                    if type(bank['pending']) == dict:
                        updated += 1
                        pendings = []
                        for release in sorted(
                                bank['pending'],
                                key=lambda r: bank['pending'][r]):
                            pendings.append({
                                'release': str(release),
                                'id': bank['pending'][str(release)]
                            })
                            if len(pendings) > 0:
                                banks.update({'name': bank['name']},
                                             {'$set': {
                                                 'pending': pendings
                                             }})
                    else:
                        # We remove old type for 'pending'
                        banks.update({'name': bank['name']},
                                     {'$unset': {
                                         'pending': ""
                                     }})

            print("Migration: %d bank(s) updated" % updated)
        schema.update_one({'id': 1}, {'$set': {'version': installed_version}})
Beispiel #18
0
 def setUp(self):
   self.utils = UtilsForTest()
   self.curdir = os.path.dirname(os.path.realpath(__file__))
   BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
Beispiel #19
0
import bcrypt

from biomaj.config import BiomajConfig

parser = argparse.ArgumentParser(description='Initialize database content.')
parser.add_argument('--config')
parser.add_argument('--user')
parser.add_argument('--pwd')
parser.add_argument('--email')
args = parser.parse_args()

if not args.config:
    print "config argument is missing"
    sys.exit(2)

BiomajConfig.load_config(args.config)

from biomaj.user import BmajUser
from hashlib import sha1

if not args.user:
    print 'user parameter is missing'
    sys.exit(1)

rootuser = BmajUser(args.user)

if args.pwd:
    pwd = args.pwd
else:
    pwd = sha1("%s" % randint(1, 1e99)).hexdigest()