def biomaj_move_production_directories(options):
'''
Change bank production directories
'''
if not options.bank:
return (False, "Bank option is missing")
if not os.path.exists(options.newdir):
return (False, "Destination directory does not exists")
bank = Bank(options.bank, options=options, no_log=True)
if not bank.bank['production']:
return (False, "Nothing to move, no production directory")
bank.load_session(Workflow.FLOW, None)
w = Workflow(bank)
res = w.wf_init()
if not res:
return (False, 'Bank initialization failure')
for prod in bank.bank['production']:
session = bank.get_session_from_release(prod['release'])
bank.load_session(Workflow.FLOW, session)
prod_path = bank.session.get_full_release_directory()
if os.path.exists(prod_path):
shutil.move(prod_path, options.newdir)
prod['data_dir'] = options.newdir
bank.banks.update({'name': options.bank}, {'$set': {'production': bank.bank['production']}})
w.wf_over()
return (True, "Bank production directories moved to " + options.newdir + "\nWARNING: do not forget to update accordingly the data.dir and dir.version properties")
def biomaj_status_ko(options, config):
'''
Get failed banks
'''
banks = Bank.list()
banks_list = []
headers = ["Name", "Type(s)", "Release", "Visibility", "Last run"]
if not options.json:
banks_list.append(headers)
for bank in sorted(banks, key=lambda k: k['name']):
try:
bank = Bank(bank['name'], options=options, no_log=True)
bank.load_session(UpdateWorkflow.FLOW)
if bank.session is not None:
if bank.use_last_session and not bank.session.get_status(
Workflow.FLOW_OVER):
wf_status = bank.session.get('workflow_status')
if wf_status is None or not wf_status:
banks_list.append(bank.get_bank_release_info()['info'])
except Exception as e:
return (False, str(e))
if options.json:
msg = {'headers': headers, 'banks': banks_list}
return (True, msg)
return (True, tabulate(banks_list, headers="firstrow", tablefmt="psql"))
def biomaj_publish(options):
'''
Publish a bank
'''
if not options.bank:
return (False, "Bank name or release is missing")
bmaj = Bank(options.bank, options=options, no_log=True)
bmaj.load_session()
bank = bmaj.bank
session = None
if options.get_option('release') is None:
# Get latest prod release
if len(bank['production']) > 0:
prod = bank['production'][len(bank['production']) - 1]
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
else:
# Search production release matching release
for prod in bank['production']:
if prod['release'] == options.release or prod['prod_dir'] == options.release:
# Search session related to this production release
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
break
if session is None:
return (False, "No production session could be found for this release")
bmaj.session._session = session
bmaj.publish()
return (True, None)
def test_new_session(self):
'''
Checks an empty session is created
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session['status'].keys():
self.assertFalse(b.session.get_status(key))
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt'))
def test_extract_release_from_file_content(self):
b = Bank("local")
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set("release.file", "test_100\.txt")
b.session.config.set("release.regexp", "Release\s*(\d+)")
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get("release") == "103")
def test_extract_release_from_file_content(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_100\.txt')
b.session.config.set('release.regexp', 'Release\s*(\d+)')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '103')
def test_new_session(self):
"""
Checks an empty session is created
"""
b = Bank("alu")
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session["status"].keys():
self.assertFalse(b.session.get_status(key))
def test_extract_release_from_file_content(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_100\.txt')
b.session.config.set('release.regexp', 'Release\s*(\d+)')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '103')
def test_new_session(self):
'''
Checks an empty session is created
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session['status'].keys():
self.assertFalse(b.session.get_status(key))
def test_remoterelease_check(self):
b = Bank('local')
b.load_session(ReleaseCheckWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
workflow = ReleaseCheckWorkflow(b)
res = workflow.start()
remoterelease = b.session.get('remoterelease')
self.assertTrue(remoterelease == '100')
def test_remoterelease_check(self):
b = Bank('local')
b.load_session(ReleaseCheckWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
workflow = ReleaseCheckWorkflow(b)
res = workflow.start()
remoterelease = b.session.get('remoterelease')
self.assertTrue(remoterelease == '100')
def test_get_release(self):
"""
Get release
"""
b = Bank("alu")
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get("update"))
self.assertTrue(res)
self.assertTrue(b.session._session["release"] is not None)
def test_get_release(self):
'''
Get release
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
def test_get_release(self):
'''
Get release
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
def biomaj_unpublish(options):
'''
Unpublish a bank
'''
if not options.bank:
return (False, "Bank name is missing")
bmaj = Bank(options.bank, options=options, no_log=True)
bmaj.load_session()
bmaj.unpublish()
return (True, None)
def test_session_reload_notover(self):
"""
Checks a session is used if present
"""
b = Bank("alu")
for i in range(1, 5):
s = Session("alu", self.config, UpdateWorkflow.FLOW)
s._session["status"][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank("alu")
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_notover(self):
'''
Checks a session is used if present
'''
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_notover(self):
'''
Checks a session is used if present
'''
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_fromscratch_update_with_release(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratch)
Use case with release defined in release file
"""
b = Bank("local")
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set("release.file", "test_(\d+)\.txt")
b.session.config.set("release.regexp", "")
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get("release") == "100")
os.makedirs(b.session.get_full_release_directory())
w = UpdateWorkflow(b)
# Reset release
b.session.set("release", None)
w.options.fromscratch = True
w.wf_release()
self.assertTrue(b.session.get("release") == "100__1")
def test_fromscratch_update_with_release(self):
'''
Try updating twice, at second time, bank should be updated (force with fromscratch)
Use case with release defined in release file
'''
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
os.makedirs(b.session.get_full_release_directory())
w = UpdateWorkflow(b)
# Reset release
b.session.set('release', None)
w.options.fromscratch = True
w.wf_release()
self.assertTrue(b.session.get('release') == '100__1')
def test_fromscratch_update_with_release(self):
'''
Try updating twice, at second time, bank should be updated (force with fromscratch)
Use case with release defined in release file
'''
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
os.makedirs(b.session.get_full_release_directory())
w = UpdateWorkflow(b)
# Reset release
b.session.set('release', None)
w.options.fromscratch = True
w.wf_release()
self.assertTrue(b.session.get('release') == '100__1')
def migrate_bank(cur, bank, history=False):
"""
Migrate old MySQL information to the new MongoDB
:param cur: MySQL cursor
:type cur: MySQL cursor
:param bank: Bank name
:type bank: String
:param history: Keep bank history
:type history: Boolean, default False
:return:
"""
query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile, s.status, r.protocol, "
query += "r.server, r.remoteDir FROM productionDirectory p "
query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank "
query += "LEFT JOIN session s ON s.idsession = u.idLastSession "
query += "JOIN configuration c ON c.idconfiguration = u.ref_idconfiguration JOIN remoteInfo r "
query += "ON r.idremoteInfo = c.ref_idremoteInfo "
query += "WHERE b.name='" + str(bank) + "' "
if not history:
query += "AND p.remove IS NULL "
query += "ORDER BY p.creation ASC"
cur.execute(query)
banks = []
not_prod = {}
for row in cur.fetchall():
banks.append({
'path': row[0],
'session': row[1],
'creation': row[2],
'remove': row[3],
'size': humanfriendly.parse_size(row[4].replace(',', '.')),
'release': row[5],
'remoterelease': row[5],
'logfile': row[6],
'status': row[7],
'protocol': row[8],
'server': row[9],
'remotedir': row[10]
})
# If we want to keep the history we will need to delete from 'production',
# session(s) which have been tagged as 'removed', so row[3] is a date
if row[3] and history:
sess = time.mktime(
datetime.datetime.strptime(str(row[2]),
"%Y-%m-%d %H:%M:%S").timetuple())
not_prod[sess] = True
for prod in banks:
pathelts = prod['path'].split('/')
release_dir = pathelts[len(pathelts) - 1]
prod['prod_dir'] = release_dir
pattern = re.compile(".*__(\d+)$")
relmatch = pattern.match(release_dir)
if relmatch:
prod['release'] = prod['release'] + '__' + relmatch.group(1)
# We create the session id from the productionDirectory.creation field
session_id = time.mktime(
datetime.datetime.strptime(str(prod['creation']),
"%Y-%m-%d %H:%M:%S").timetuple())
session_exists = False
b = Bank(bank, no_log=True)
# We check we did not already imported this session into the database
for s in b.bank['sessions']:
if s['id'] == session_id:
logging.warn('Session already imported: ' + b.name + ':' +
str(prod['creation']))
session_exists = True
break # No need to continue
if session_exists:
continue
for p in b.bank['production']:
if p['release'] == prod['release']:
logging.warn('Prod release already imported: ' + b.name + ":" +
p['release'])
continue
b.load_session(UpdateWorkflow.FLOW)
b.session.set('prod_dir', prod['prod_dir'])
b.session.set('action', 'update')
b.session.set('release', prod['release'])
b.session.set('remoterelease', prod['remoterelease'])
# Biomaj >= 3.0.14 introduce new field in sessions 'workflow_status'
# Set production size from productionDirectory.size field
b.session.set('workflow_status', True if prod['status'] else False)
b.session.set('fullsize', prod['size'])
b.session._session['status'][Workflow.FLOW_OVER] = True
b.session._session['update'] = True
# We set the session.id (timestamp) with creation field from productionDirectory table
b.session.set('id', session_id)
b.save_session()
# We need set update the field 'last_update_time' to the time the bank has been created
# because 'save_session' set this value to the time it is called
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.last_update_time': session_id
}})
# Keep trace of the logfile. We need to do a manual update
if prod['logfile'] and os.path.exists(prod['logfile']):
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.log_file': prod['logfile']
}})
# Downloaded and local files (created by 'bank.save_session')
# remove as created empty due to 'save_sessions' call
cache_dir = b.config.get('cache.dir')
download_file = os.path.join(cache_dir, 'files_' + str(session_id))
local_file = os.path.join(cache_dir, 'local_files_' + str(session_id))
if os.path.isfile(download_file):
os.unlink(download_file)
if os.path.isfile(local_file):
os.unlink(local_file)
# Due to the way save_session set also the production, to exclude last session
# from the production entries, we need to loop over each production entries
if history:
# If we want to keep history, we also need to keep trace of the time session/update has been deleted
# from the database/disk
if prod['remove']:
removed = time.mktime(
datetime.datetime.strptime(str(
prod['remove']), "%Y-%m-%d %H:%M:%S").timetuple())
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.deleted': removed
}})
for production in b.bank['production']:
if production['session'] in not_prod:
b.banks.update(
{
'name': b.name,
'production.session': production['session']
}, {
'$pull': {
'production': {
'session': production['session']
}
}
})
# Listing files ?
root_files = []
if os.path.exists(prod['path']):
root_files = os.listdir(prod['path'])
for root_file in root_files:
if root_file.startswith('listing.'):
fileName, fileExtension = os.path.splitext(root_file)
f = open(os.path.join(prod['path'], root_file), 'r')
listing = f.read()
f.close()
f = open(
os.path.join(prod['path'], 'listingv1' + fileExtension),
'w')
listing = "{\"files\": [], \"name\": \"" + fileExtension.replace(
'.', '') + "\"," + listing + "}"
f.write(listing)
f.close()
if root_file.startswith('flat') and session_id not in not_prod:
flat_dir = os.path.join(prod['path'], root_file)
if not os.path.exists(flat_dir):
print(
"[%s] [WARN] Can't list %s/flat directory. Skipping ..."
% (b.name, prod['path']))
continue
flat_files = os.listdir(flat_dir)
files_info = []
for flat_file in flat_files:
try:
file_info = os.stat(os.path.join(flat_dir, flat_file))
except IOError as err:
print(
"[%s][WARN] Failed to stat get info from %s: %s" %
(b.name, str(file_info), str(err)))
continue
ctime = datetime.datetime.fromtimestamp(file_info[9])
files_info.append({
'name':
flat_file,
'save_as':
flat_file,
'user':
file_info[4],
'group':
file_info[5],
'year':
ctime.year,
'day':
ctime.day,
'month':
ctime.month,
'root':
prod['remotedir'],
'url':
prod['protocol'] + '://' + prod['server'],
'size':
file_info[6]
})
if len(files_info) > 0:
# Saved files
local_file = open(local_file, 'w')
local_file.write(json.dumps(files_info))
local_file.close()
# Current link?
pathelts = prod['path'].split('/')
del pathelts[-1]
current_link = os.path.join('/'.join(pathelts), 'current')
if os.path.lexists(current_link):
b.bank['current'] = b.session._session['id']
b.banks.update({'name': b.name},
{'$set': {
'current': b.session._session['id']
}})
def main():
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('-c',
'--config',
dest="config",
help="Configuration file")
parser.add_argument('--check',
dest="check",
help="Check bank property file",
action="store_true",
default=False)
parser.add_argument('-u',
'--update',
dest="update",
help="Update action",
action="store_true",
default=False)
parser.add_argument('--fromscratch',
dest="fromscratch",
help="Force a new cycle update",
action="store_true",
default=False)
parser.add_argument('-z',
'--from-scratch',
dest="fromscratch",
help="Force a new cycle update",
action="store_true",
default=False)
parser.add_argument('-p',
'--publish',
dest="publish",
help="Publish",
action="store_true",
default=False)
parser.add_argument('--unpublish',
dest="unpublish",
help="Unpublish",
action="store_true",
default=False)
parser.add_argument('--release',
dest="release",
help="release of the bank")
parser.add_argument(
'--from-task',
dest="from_task",
help="Start cycle at a specific task (init always executed)")
parser.add_argument(
'--process',
dest="process",
help=
"Linked to from-task, optionally specify a block, meta or process name to start from"
)
parser.add_argument('-l', '--log', dest="log", help="log level")
parser.add_argument('-r',
'--remove',
dest="remove",
help="Remove a bank release",
action="store_true",
default=False)
parser.add_argument('--remove-all',
dest="removeall",
help="Remove all bank releases and database records",
action="store_true",
default=False)
parser.add_argument('--remove-pending',
dest="removepending",
help="Remove pending release",
action="store_true",
default=False)
parser.add_argument('-s',
'--status',
dest="status",
help="Get status",
action="store_true",
default=False)
parser.add_argument('-b', '--bank', dest="bank", help="bank name")
parser.add_argument('--owner',
dest="owner",
help="change owner of the bank")
parser.add_argument('--stop-before',
dest="stop_before",
help="Store workflow before task")
parser.add_argument('--stop-after',
dest="stop_after",
help="Store workflow after task")
parser.add_argument('--freeze',
dest="freeze",
help="Freeze a bank release",
action="store_true",
default=False)
parser.add_argument('--unfreeze',
dest="unfreeze",
help="Unfreeze a bank release",
action="store_true",
default=False)
parser.add_argument('-f',
'--force',
dest="force",
help="Force action",
action="store_true",
default=False)
parser.add_argument('-h',
'--help',
dest="help",
help="Show usage",
action="store_true",
default=False)
parser.add_argument('--search',
dest="search",
help="Search by format and types",
action="store_true",
default=False)
parser.add_argument('--formats',
dest="formats",
help="List of formats to search, comma separated")
parser.add_argument('--types',
dest="types",
help="List of types to search, comma separated")
parser.add_argument('--query',
dest="query",
help="Lucene query syntax to search in index")
parser.add_argument('--show',
dest="show",
help="Show format files for selected bank",
action="store_true",
default=False)
parser.add_argument('-n',
'--change-dbname',
dest="newbank",
help="Change old bank name to this new bank name")
parser.add_argument(
'-e',
'--move-production-directories',
dest="newdir",
help=
"Change bank production directories location to this new path, path must exists"
)
parser.add_argument('--visibility',
dest="visibility",
help="visibility status of the bank")
parser.add_argument('--maintenance',
dest="maintenance",
help="Maintenance mode (on/off/status)")
parser.add_argument('--version',
dest="version",
help="Show version",
action="store_true",
default=False)
parser.add_argument('--status-ko',
dest="statusko",
help="Get bank in KO status",
action="store_true",
default=False)
options = Options()
parser.parse_args(namespace=options)
options.no_log = False
if options.help:
print('''
--config: global.properties file path
--status: list of banks with published release
[OPTIONAL]
--bank xx / bank: Get status details of bank
--status-ko: list of banks in error status (last run)
--log DEBUG|INFO|WARN|ERR [OPTIONAL]: set log level in logs for this run, default is set in global.properties file
--check: Check bank property file
[MANDATORY]
--bank xx: name of the bank to check (will check xx.properties)
--owner yy: Change owner of the bank (user id)
[MANDATORY]
--bank xx: name of the bank
--visibility public|private: change visibility public/private of a bank
[MANDATORY]
--bank xx: name of the bank
--change-dbname yy: Change name of the bank to this new name
[MANDATORY]
--bank xx: current name of the bank
--move-production-directories yy: Change bank production directories location to this new path, path must exists
[MANDATORY]
--bank xx: current name of the bank
--update: Update bank
[MANDATORY]
--bank xx: name of the bank(s) to update, comma separated
[OPTIONAL]
--publish: after update set as *current* version
--from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1)
--stop-before xx: stop update cycle before the start of step xx
--stop-after xx: stop update cycle after step xx has completed
--from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx*
--process xx: linked to from-task, optionally specify a block, meta or process name to start from
--release xx: release to update
--publish: Publish bank as current release to use
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to publish
--unpublish: Unpublish bank (remove current)
[MANDATORY]
--bank xx: name of the bank to update
--remove-all: Remove all bank releases and database records
[MANDATORY]
--bank xx: name of the bank to update
[OPTIONAL]
--force: remove freezed releases
--remove-pending: Remove pending releases
[MANDATORY]
--bank xx: name of the bank to update
--remove: Remove bank release (files and database release)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
Release must not be the *current* version. If this is the case, publish a new release before.
--freeze: Freeze bank release (cannot be removed)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
--unfreeze: Unfreeze bank release (can be removed)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
--search: basic search in bank production releases, return list of banks
--formats xx,yy : list of comma separated format
AND/OR
--types xx,yy : list of comma separated type
--query "LUCENE query syntax": search in index (if activated)
--show: Show bank files per format
[MANDATORY]
--bank xx: name of the bank to show
[OPTIONAL]
--release xx: release of the bank to show
--maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal
''')
return
if options.version:
version = pkg_resources.require('biomaj')[0].version
print('Version: ' + str(version))
return
if options.stop_after or options.stop_before or options.from_task:
available_steps = []
for flow in UpdateWorkflow.FLOW:
available_steps.append(flow['name'])
for flow in RemoveWorkflow.FLOW:
available_steps.append(flow['name'])
if options.stop_after:
if options.stop_after not in available_steps:
print('Invalid step: ' + options.stop_after)
sys.exit(1)
if options.stop_before:
if options.stop_before not in available_steps:
print('Invalid step: ' + options.stop_before)
sys.exit(1)
if options.from_task:
if options.from_task not in available_steps:
print('Invalid step: ' + options.from_task)
sys.exit(1)
bmaj = None
try:
if options.config is not None:
BiomajConfig.load_config(options.config)
else:
BiomajConfig.load_config()
except Exception as e:
print(str(e))
sys.exit(1)
try:
if options.maintenance:
if options.maintenance not in ['on', 'off', 'status']:
print("Wrong maintenance value [on,off,status]")
sys.exit(1)
data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir')
if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'):
lock_dir = BiomajConfig.global_config.get(
'GENERAL', 'lock.dir')
else:
lock_dir = data_dir
maintenance_lock_file = os.path.join(lock_dir, 'biomaj.lock')
if options.maintenance == 'status':
if os.path.exists(maintenance_lock_file):
print("Maintenance: On")
else:
print("Maintenance: Off")
sys.exit(0)
if options.maintenance == 'on':
f = open(maintenance_lock_file, 'w')
f.write('1')
f.close()
print("Maintenance set to On")
sys.exit(0)
if options.maintenance == 'off':
if os.path.exists(maintenance_lock_file):
os.remove(maintenance_lock_file)
print("Maintenance set to Off")
sys.exit(0)
if options.owner:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
bank.set_owner(options.owner)
sys.exit(0)
if options.visibility:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
if options.visibility not in ['public', 'private']:
print("Valid values are public|private")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
bank.set_visibility(options.visibility)
print(
"Do not forget to update accordingly the visibility.default parameter in the configuration file"
)
sys.exit(0)
if options.newdir:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
if not os.path.exists(options.newdir):
print("Destination directory does not exists")
bank = Bank(options.bank, options=options, no_log=True)
if not bank.bank['production']:
print("Nothing to move, no production directory")
sys.exit(0)
bank.load_session(Workflow.FLOW, None)
w = Workflow(bank)
res = w.wf_init()
if not res:
sys.exit(1)
for prod in bank.bank['production']:
session = bank.get_session_from_release(prod['release'])
bank.load_session(Workflow.FLOW, session)
prod_path = bank.session.get_full_release_directory()
if os.path.exists(prod_path):
shutil.move(prod_path, options.newdir)
prod['data_dir'] = options.newdir
bank.banks.update(
{'name': options.bank},
{'$set': {
'production': bank.bank['production']
}})
print("Bank production directories moved to " + options.newdir)
print(
"WARNING: do not forget to update accordingly the data.dir and dir.version properties"
)
w.wf_over()
sys.exit(0)
if options.newbank:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
bank_prop_file = os.path.join(conf_dir,
options.bank + '.properties')
config_bank = configparser.SafeConfigParser()
config_bank.read(
[os.path.join(conf_dir, options.bank + '.properties')])
config_bank.set('GENERAL', 'db.name', options.newbank)
newbank_prop_file = open(
os.path.join(conf_dir, options.newbank + '.properties'), 'w')
config_bank.write(newbank_prop_file)
newbank_prop_file.close()
bank.banks.update({'name': options.bank},
{'$set': {
'name': options.newbank
}})
os.remove(bank_prop_file)
print("Bank " + options.bank + " renamed to " + options.newbank)
sys.exit(0)
if options.search:
if options.query:
res = Bank.searchindex(options.query)
print("Query matches for :" + options.query)
results = [["Release", "Format(s)", "Type(s)", "Files"]]
for match in res:
results.append([
match['_source']['release'],
str(match['_source']['format']),
str(match['_source']['types']),
','.join(match['_source']['files'])
])
print(tabulate(results, headers="firstrow", tablefmt="grid"))
else:
formats = []
if options.formats:
formats = options.formats.split(',')
types = []
if options.types:
types = options.types.split(',')
print("Search by formats=" + str(formats) + ", types=" +
str(types))
res = Bank.search(formats, types, False)
results = [[
"Name", "Release", "Format(s)", "Type(s)", 'Published'
]]
for bank in sorted(res, key=lambda bank: (bank['name'])):
b = bank['name']
bank['production'].sort(key=lambda n: n['release'],
reverse=True)
for prod in bank['production']:
iscurrent = ""
if prod['session'] == bank['current']:
iscurrent = "yes"
results.append([
b if b else '', prod['release'],
','.join(prod['formats']), ','.join(prod['types']),
iscurrent
])
print(tabulate(results, headers="firstrow", tablefmt="grid"))
sys.exit(0)
if options.show:
if not options.bank:
print("Bank option is required")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
results = [[
"Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"
]]
current = None
fformat = None
if 'current' in bank.bank and bank.bank['current']:
current = bank.bank['current']
for prod in bank.bank['production']:
include = True
release = prod['release']
if current == prod['session']:
release += ' (current)'
if options.release and (prod['release'] != options.release and
prod['prod_dir'] != options.release):
include = False
if include:
session = bank.get_session_from_release(prod['release'])
formats = session['formats']
afiles = []
atags = []
atypes = []
for fformat in list(formats.keys()):
for elt in formats[fformat]:
atypes.append(','.join(elt['types']))
for tag in list(elt['tags'].keys()):
atags.append(elt['tags'][tag])
for eltfile in elt['files']:
afiles.append(eltfile)
results.append([
bank.bank['name'], release, fformat, ','.join(atypes),
','.join(atags), ','.join(afiles)
])
print(tabulate(results, headers="firstrow", tablefmt="grid"))
sys.exit(0)
if options.check:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
print(options.bank + " check: " + str(bank.check()) + "\n")
sys.exit(0)
if options.status:
if options.bank:
bank = Bank(options.bank, no_log=True)
info = bank.get_bank_release_info(full=True)
print(
tabulate(info['info'], headers='firstrow',
tablefmt='psql'))
print(
tabulate(info['prod'], headers='firstrow',
tablefmt='psql'))
# do we have some pending release(s)
if 'pend' in info and len(info['pend']) > 1:
print(
tabulate(info['pend'],
headers='firstrow',
tablefmt='psql'))
else:
banks = Bank.list()
# Headers of output table
banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
for bank in sorted(banks, key=lambda k: k['name']):
bank = Bank(bank['name'], no_log=True)
banks_list.append(bank.get_bank_release_info()['info'])
print(tabulate(banks_list, headers="firstrow",
tablefmt="psql"))
sys.exit(0)
if options.statusko:
banks = Bank.list()
banks_list = [[
"Name", "Type(s)", "Release", "Visibility", "Last run"
]]
for bank in sorted(banks, key=lambda k: k['name']):
try:
bank = Bank(bank['name'], no_log=True)
bank.load_session(UpdateWorkflow.FLOW)
if bank.session is not None:
if bank.use_last_session and not bank.session.get_status(
Workflow.FLOW_OVER):
wf_status = bank.session.get('workflow_status')
if wf_status is None or not wf_status:
banks_list.append(
bank.get_bank_release_info()['info'])
except Exception as e:
print(str(e))
print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))
if options.update:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
banks = options.bank.split(',')
gres = True
for bank in banks:
options.bank = bank
bmaj = Bank(bank, options)
print('Log file: ' + bmaj.config.log_file)
check_status = bmaj.check()
if not check_status:
print('Skip bank ' + options.bank + ': wrong config')
gres = False
continue
res = bmaj.update(depends=True)
if not res:
gres = False
Notify.notifyBankAction(bmaj)
if not gres:
sys.exit(1)
if options.freeze:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if not options.release:
print("Bank release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options)
res = bmaj.freeze(options.release)
if not res:
sys.exit(1)
if options.unfreeze:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if not options.release:
print("Bank release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options)
res = bmaj.unfreeze(options.release)
if not res:
sys.exit(1)
if options.remove or options.removeall:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if options.remove and not options.release:
print("Bank release is missing")
sys.exit(1)
if options.removeall:
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: ' + bmaj.config.log_file)
res = bmaj.removeAll(options.force)
else:
bmaj = Bank(options.bank, options)
print('Log file: ' + bmaj.config.log_file)
res = bmaj.remove(options.release)
Notify.notifyBankAction(bmaj)
if not res:
sys.exit(1)
if options.removepending:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
res = bmaj.remove_pending(options.release)
if not res:
sys.exit(1)
if options.unpublish:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
bmaj.load_session()
bmaj.unpublish()
sys.exit(0)
if options.publish:
if not options.bank:
print("Bank name or release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
bmaj.load_session()
bank = bmaj.bank
session = None
if options.get_option('release') is None:
# Get latest prod release
if len(bank['production']) > 0:
prod = bank['production'][len(bank['production']) - 1]
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
else:
# Search production release matching release
for prod in bank['production']:
if prod['release'] == options.release or prod[
'prod_dir'] == options.release:
# Search session related to this production release
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
break
if session is None:
print("No production session could be found for this release")
sys.exit(1)
bmaj.session._session = session
bmaj.publish()
except Exception as e:
print(str(e))
def migrate_bank(cur, bank, history=False):
"""
Migrate old MySQL information to the new MongoDB
:param cur: MySQL cursor
:type cur: MySQL cursor
:param bank: Bank name
:type bank: String
:param history: Keep bank history
:type history: Boolean, default False
:return:
"""
query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile from productionDirectory p "
query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank "
query += "LEFT JOIN session s ON s.idsession = u.idLastSession "
query += "WHERE b.name='" + str(bank) + "' "
if not history:
query += "AND p.remove IS NULL "
query += "ORDER BY p.creation ASC"
cur.execute(query)
banks = []
not_prod = {}
for row in cur.fetchall():
banks.append({
'path': row[0],
'session': row[1],
'creation': row[2],
'remove': row[3],
'size': row[4],
'release': row[5],
'remoterelease': row[5],
'logfile': row[6]
})
# If we want to keep the history we will need to delete from 'production',
# session(s) which have been tagged as 'removed', so row[3] is a date
if row[3] and history:
sess = time.mktime(
datetime.datetime.strptime(str(row[2]),
"%Y-%m-%d %H:%M:%S").timetuple())
not_prod[sess] = True
for prod in banks:
pathelts = prod['path'].split('/')
release_dir = pathelts[len(pathelts) - 1]
prod['prod_dir'] = release_dir
pattern = re.compile(".*__(\d+)$")
relmatch = pattern.match(release_dir)
if relmatch:
prod['release'] = prod['release'] + '__' + relmatch.group(1)
# We create the session id from the productionDirectory.creation field
session_id = time.mktime(
datetime.datetime.strptime(str(prod['creation']),
"%Y-%m-%d %H:%M:%S").timetuple())
session_exists = False
b = Bank(bank, no_log=True)
# We check we did not already imported this session into the database
for s in b.bank['sessions']:
if s['id'] == session_id:
logging.warn('Session already imported: ' + b.name + ':' +
str(prod['creation']))
session_exists = True
break # No need to continue
if session_exists:
continue
for p in b.bank['production']:
if p['release'] == prod['release']:
logging.warn('Prod release already imported: ' + b.name + ":" +
p['release'])
continue
b.load_session(UpdateWorkflow.FLOW)
b.session.set('prod_dir', prod['prod_dir'])
b.session.set('action', 'update')
b.session.set('release', prod['release'])
b.session.set('remoterelease', prod['remoterelease'])
# Set production size from productionDirectory.size field
b.session.set('fullsize', prod['size'])
b.session._session['status'][Workflow.FLOW_OVER] = True
b.session._session['update'] = True
# We set the session.id (timestamp) with creation field from productionDirectory table
b.session.set('id', session_id)
b.save_session()
# Keep trace of the logfile. We need to do a manual update
if prod['logfile'] and os.path.exists(prod['logfile']):
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
"sessions.$.log_file": prod['logfile']
}})
# Due to the way save_session set also the production, to exclude last session
# from the production entries, we need to loop over each production entries
if history:
for production in b.bank['production']:
if production['session'] in not_prod:
b.banks.update(
{
'name': b.name,
'production.session': production['session']
}, {
'$pull': {
'production': {
'session': production['session']
}
}
})
# Listing files ?
root_files = []
if os.path.exists(prod['path']):
root_files = os.listdir(prod['path'])
for root_file in root_files:
if root_file.startswith('listing.'):
fileName, fileExtension = os.path.splitext(root_file)
f = open(os.path.join(prod['path'], root_file), 'r')
listing = f.read()
f.close()
f = open(
os.path.join(prod['path'], 'listingv1' + fileExtension),
'w')
listing = "{\"files\": [], \"name\": \"" + fileExtension.replace(
'.', '') + "\"," + listing + "}"
f.write(listing)
f.close()
# Current link?
pathelts = prod['path'].split('/')
del pathelts[-1]
current_link = os.path.join('/'.join(pathelts), 'current')
if os.path.lexists(current_link):
b.bank['current'] = b.session._session['id']
b.banks.update({'name': b.name},
{'$set': {
'current': b.session._session['id']
}})
def main():
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('-c', '--config', dest="config",help="Configuration file")
parser.add_argument('--check', dest="check", help="Check bank property file", action="store_true", default=False)
parser.add_argument('-u', '--update', dest="update", help="Update action", action="store_true", default=False)
parser.add_argument('--fromscratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False)
parser.add_argument('-z', '--from-scratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False)
parser.add_argument('-p', '--publish', dest="publish", help="Publish", action="store_true", default=False)
parser.add_argument('--unpublish', dest="unpublish", help="Unpublish", action="store_true", default=False)
parser.add_argument('--release', dest="release", help="release of the bank")
parser.add_argument('--from-task', dest="from_task", help="Start cycle at a specific task (init always executed)")
parser.add_argument('--process', dest="process", help="Linked to from-task, optionally specify a block, meta or process name to start from")
parser.add_argument('-l', '--log', dest="log", help="log level")
parser.add_argument('-r', '--remove', dest="remove", help="Remove a bank release", action="store_true", default=False)
parser.add_argument('--remove-all', dest="removeall", help="Remove all bank releases and database records", action="store_true", default=False)
parser.add_argument('--remove-pending', dest="removepending", help="Remove pending release", action="store_true", default=False)
parser.add_argument('-s', '--status', dest="status", help="Get status", action="store_true", default=False)
parser.add_argument('-b', '--bank', dest="bank", help="bank name")
parser.add_argument('--owner', dest="owner", help="change owner of the bank")
parser.add_argument('--stop-before', dest="stop_before", help="Store workflow before task")
parser.add_argument('--stop-after', dest="stop_after", help="Store workflow after task")
parser.add_argument('--freeze', dest="freeze", help="Freeze a bank release", action="store_true", default=False)
parser.add_argument('--unfreeze', dest="unfreeze", help="Unfreeze a bank release", action="store_true", default=False)
parser.add_argument('-f', '--force', dest="force", help="Force action", action="store_true", default=False)
parser.add_argument('-h', '--help', dest="help", help="Show usage", action="store_true", default=False)
parser.add_argument('--search', dest="search", help="Search by format and types", action="store_true", default=False)
parser.add_argument('--formats', dest="formats", help="List of formats to search, comma separated")
parser.add_argument('--types', dest="types", help="List of types to search, comma separated")
parser.add_argument('--query', dest="query", help="Lucene query syntax to search in index")
parser.add_argument('--show', dest="show", help="Show format files for selected bank", action="store_true", default=False)
parser.add_argument('-n', '--change-dbname', dest="newbank", help="Change old bank name to this new bank name")
parser.add_argument('-e', '--move-production-directories', dest="newdir",help="Change bank production directories location to this new path, path must exists")
parser.add_argument('--visibility', dest="visibility", help="visibility status of the bank")
parser.add_argument('--maintenance', dest="maintenance", help="Maintenance mode (on/off/status)")
parser.add_argument('--version', dest="version", help="Show version", action="store_true", default=False)
parser.add_argument('--status-ko', dest="statusko", help="Get bank in KO status", action="store_true", default=False)
options = Options()
parser.parse_args(namespace=options)
options.no_log = False
if options.help:
print('''
--config: global.properties file path
--status: list of banks with published release
[OPTIONAL]
--bank xx / bank: Get status details of bank
--status-ko: list of banks in error status (last run)
--log DEBUG|INFO|WARN|ERR [OPTIONAL]: set log level in logs for this run, default is set in global.properties file
--check: Check bank property file
[MANDATORY]
--bank xx: name of the bank to check (will check xx.properties)
--owner yy: Change owner of the bank (user id)
[MANDATORY]
--bank xx: name of the bank
--visibility public|private: change visibility public/private of a bank
[MANDATORY]
--bank xx: name of the bank
--change-dbname yy: Change name of the bank to this new name
[MANDATORY]
--bank xx: current name of the bank
--move-production-directories yy: Change bank production directories location to this new path, path must exists
[MANDATORY]
--bank xx: current name of the bank
--update: Update bank
[MANDATORY]
--bank xx: name of the bank(s) to update, comma separated
[OPTIONAL]
--publish: after update set as *current* version
--from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1)
--stop-before xx: stop update cycle before the start of step xx
--stop-after xx: stop update cycle after step xx has completed
--from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx*
--process xx: linked to from-task, optionally specify a block, meta or process name to start from
--release xx: release to update
--publish: Publish bank as current release to use
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to publish
--unpublish: Unpublish bank (remove current)
[MANDATORY]
--bank xx: name of the bank to update
--remove-all: Remove all bank releases and database records
[MANDATORY]
--bank xx: name of the bank to update
[OPTIONAL]
--force: remove freezed releases
--remove-pending: Remove pending releases
[MANDATORY]
--bank xx: name of the bank to update
--remove: Remove bank release (files and database release)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
Release must not be the *current* version. If this is the case, publish a new release before.
--freeze: Freeze bank release (cannot be removed)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
--unfreeze: Unfreeze bank release (can be removed)
[MANDATORY]
--bank xx: name of the bank to update
--release xx: release of the bank to remove
--search: basic search in bank production releases, return list of banks
--formats xx,yy : list of comma separated format
AND/OR
--types xx,yy : list of comma separated type
--query "LUCENE query syntax": search in index (if activated)
--show: Show bank files per format
[MANDATORY]
--bank xx: name of the bank to show
[OPTIONAL]
--release xx: release of the bank to show
--maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal
''')
return
if options.version:
version = pkg_resources.require('biomaj')[0].version
print('Version: '+str(version))
return
if options.stop_after or options.stop_before or options.from_task:
available_steps = []
for flow in UpdateWorkflow.FLOW:
available_steps.append(flow['name'])
for flow in RemoveWorkflow.FLOW:
available_steps.append(flow['name'])
if options.stop_after:
if options.stop_after not in available_steps:
print('Invalid step: '+options.stop_after)
sys.exit(1)
if options.stop_before:
if options.stop_before not in available_steps:
print('Invalid step: '+options.stop_before)
sys.exit(1)
if options.from_task:
if options.from_task not in available_steps:
print('Invalid step: '+options.from_task)
sys.exit(1)
bmaj = None
try:
if options.config is not None:
BiomajConfig.load_config(options.config)
else:
BiomajConfig.load_config()
except Exception as e:
print(str(e))
sys.exit(1)
try:
if options.maintenance:
if options.maintenance not in ['on', 'off', 'status']:
print("Wrong maintenance value [on,off,status]")
sys.exit(1)
data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir')
if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'):
lock_dir = BiomajConfig.global_config.get('GENERAL', 'lock.dir')
else:
lock_dir = data_dir
maintenance_lock_file = os.path.join(lock_dir,'biomaj.lock')
if options.maintenance == 'status':
if os.path.exists(maintenance_lock_file):
print("Maintenance: On")
else:
print("Maintenance: Off")
sys.exit(0)
if options.maintenance == 'on':
f = open(maintenance_lock_file, 'w')
f.write('1')
f.close()
print("Maintenance set to On")
sys.exit(0)
if options.maintenance == 'off':
if os.path.exists(maintenance_lock_file):
os.remove(maintenance_lock_file)
print("Maintenance set to Off")
sys.exit(0)
if options.owner:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
bank.set_owner(options.owner)
sys.exit(0)
if options.visibility:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
if options.visibility not in ['public', 'private']:
print("Valid values are public|private")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
bank.set_visibility(options.visibility)
print("Do not forget to update accordingly the visibility.default parameter in the configuration file")
sys.exit(0)
if options.newdir:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
if not os.path.exists(options.newdir):
print("Destination directory does not exists")
bank = Bank(options.bank, options=options, no_log=True)
if not bank.bank['production']:
print("Nothing to move, no production directory")
sys.exit(0)
bank.load_session(Workflow.FLOW, None)
w = Workflow(bank)
res = w.wf_init()
if not res:
sys.exit(1)
for prod in bank.bank['production']:
session = bank.get_session_from_release(prod['release'])
bank.load_session(Workflow.FLOW, session)
prod_path = bank.session.get_full_release_directory()
if os.path.exists(prod_path):
shutil.move(prod_path, options.newdir)
prod['data_dir'] = options.newdir
bank.banks.update({'name': options.bank}, {'$set' : { 'production': bank.bank['production'] }})
print("Bank production directories moved to " + options.newdir)
print("WARNING: do not forget to update accordingly the data.dir and dir.version properties")
w.wf_over()
sys.exit(0)
if options.newbank:
if not options.bank:
print("Bank option is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
bank_prop_file = os.path.join(conf_dir,options.bank+'.properties')
config_bank = configparser.SafeConfigParser()
config_bank.read([os.path.join(conf_dir,options.bank+'.properties')])
config_bank.set('GENERAL', 'db.name', options.newbank)
newbank_prop_file = open(os.path.join(conf_dir,options.newbank+'.properties'),'w')
config_bank.write(newbank_prop_file)
newbank_prop_file.close()
bank.banks.update({'name': options.bank}, {'$set' : { 'name': options.newbank }})
os.remove(bank_prop_file)
print("Bank "+options.bank+" renamed to "+options.newbank)
sys.exit(0)
if options.search:
if options.query:
res = Bank.searchindex(options.query)
print("Query matches for :"+options.query)
results = [["Release", "Format(s)", "Type(s)", "Files"]]
for match in res:
results.append([match['_source']['release'],
str(match['_source']['format']),
str(match['_source']['types']),
','.join(match['_source']['files'])])
print(tabulate(results, headers="firstrow", tablefmt="grid"))
else:
formats = []
if options.formats:
formats = options.formats.split(',')
types = []
if options.types:
types = options.types.split(',')
print("Search by formats="+str(formats)+", types="+str(types))
res = Bank.search(formats, types, False)
results = [["Name", "Release", "Format(s)", "Type(s)", 'Current']]
for bank in sorted(res, key=lambda bank: (bank['name'])):
b = bank['name']
bank['production'].sort(key=lambda n: n['release'], reverse=True)
for prod in bank['production']:
iscurrent = ""
if prod['session'] == bank['current']:
iscurrent = "yes"
results.append([b if b else '', prod['release'], ','.join(prod['formats']),
','.join(prod['types']), iscurrent])
b = None
print(tabulate(results, headers="firstrow", tablefmt="grid"))
sys.exit(0)
if options.show:
if not options.bank:
print("Bank option is required")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
results = [["Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"]]
current = None
fformat = None
if 'current' in bank.bank and bank.bank['current']:
current = bank.bank['current']
for prod in bank.bank['production']:
include = True
release = prod['release']
if current == prod['session']:
release += ' (current)'
if options.release and (prod['release'] != options.release and prod['prod_dir'] != options.release):
include =False
if include:
session = bank.get_session_from_release(prod['release'])
formats = session['formats']
afiles = []
atags = []
atypes = []
for fformat in list(formats.keys()):
for elt in formats[fformat]:
atypes.append(','.join(elt['types']))
for tag in list(elt['tags'].keys()):
atags.append(elt['tags'][tag])
for eltfile in elt['files']:
afiles.append(eltfile)
results.append([bank.bank['name'], release, fformat, ','.join(atypes),
','.join(atags), ','.join(afiles)])
print(tabulate(results, headers="firstrow", tablefmt="grid"))
sys.exit(0)
if options.check:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bank = Bank(options.bank, no_log=True)
print(options.bank+" check: "+str(bank.check())+"\n")
sys.exit(0)
if options.status:
if options.bank:
bank = Bank(options.bank, no_log=True)
info = bank.get_bank_release_info(full=True)
print(tabulate(info['info'], headers='firstrow', tablefmt='psql'))
print(tabulate(info['prod'], headers='firstrow', tablefmt='psql'))
# do we have some pending release(s)
if 'pend' in info and len(info['pend']) > 1:
print(tabulate(info['pend'], headers='firstrow', tablefmt='psql'))
else:
banks = Bank.list()
# Headers of output table
banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
for bank in sorted(banks, key=lambda k: k['name']):
bank = Bank(bank['name'], no_log=True)
banks_list.append(bank.get_bank_release_info()['info'])
print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))
sys.exit(0)
if options.statusko:
banks = Bank.list()
banks_list = [["Name", "Type(s)", "Release", "Visibility"]]
for bank in sorted(banks, key=lambda k: k['name']):
try:
bank = Bank(bank['name'], no_log=True)
bank.load_session(UpdateWorkflow.FLOW)
if bank.session is not None:
if bank.use_last_session and not bank.session.get_status(Workflow.FLOW_OVER):
wf_status = bank.session.get('workflow_status')
if wf_status is None or not wf_status:
banks_list.append(bank.get_bank_release_info()['info'])
except Exception as e:
print(str(e))
print(tabulate(banks_list, headers="firstrow", tablefmt="psql"))
if options.update:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
banks = options.bank.split(',')
gres = True
for bank in banks:
options.bank = bank
bmaj = Bank(bank, options)
print('Log file: '+bmaj.config.log_file)
check_status = bmaj.check()
if not check_status:
print('Skip bank ' + options.bank + ': wrong config')
gres = False
continue
res = bmaj.update(depends=True)
if not res:
gres = False
Notify.notifyBankAction(bmaj)
if not gres:
sys.exit(1)
if options.freeze:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if not options.release:
print("Bank release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options)
res = bmaj.freeze(options.release)
if not res:
sys.exit(1)
if options.unfreeze:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if not options.release:
print("Bank release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options)
res = bmaj.unfreeze(options.release)
if not res:
sys.exit(1)
if options.remove or options.removeall:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
if options.remove and not options.release:
print("Bank release is missing")
sys.exit(1)
if options.removeall:
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: '+bmaj.config.log_file)
res = bmaj.removeAll(options.force)
else:
bmaj = Bank(options.bank, options)
print('Log file: '+bmaj.config.log_file)
res = bmaj.remove(options.release)
Notify.notifyBankAction(bmaj)
if not res:
sys.exit(1)
if options.removepending:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: '+bmaj.config.log_file)
res = bmaj.remove_pending(options.release)
if not res:
sys.exit(1)
if options.unpublish:
if not options.bank:
print("Bank name is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
bmaj.load_session()
bmaj.unpublish()
sys.exit(0)
if options.publish:
if not options.bank:
print("Bank name or release is missing")
sys.exit(1)
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: '+bmaj.config.log_file)
bmaj.load_session()
bank = bmaj.bank
session = None
if options.get_option('release') is None:
# Get latest prod release
if len(bank['production'])>0:
prod = bank['production'][len(bank['production'])-1]
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
else:
# Search production release matching release
for prod in bank['production']:
if prod['release'] == options.release or prod['prod_dir'] == options.release:
# Search session related to this production release
for s in bank['sessions']:
if s['id'] == prod['session']:
session = s
break
break
if session is None:
print("No production session could be found for this release")
sys.exit(1)
bmaj.session._session = session
bmaj.publish()
except Exception as e:
print(str(e))
