def _make_links(self, links=None, hard=False):
"""
Try to create the links (symbolic or hard)
:param links: List of links to create
:type links: list
:param hard: Create hard link
:type hard: boole
:return: Number of created link(s)
:rtype: int
:raises SystemExit: If link(s) cannot be created
"""
if not links or not len(links):
return 0
for slink, tlink in links:
if not os.path.exists(tlink) and not os.path.islink(tlink):
if Manager.get_simulate() and Manager.get_verbose():
Utils.verbose("Linking %s -> %s" % (tlink, os.path.relpath(slink, start=self.target)))
else:
try:
if not Manager.get_simulate():
source_link = os.path.relpath(slink, start=self.target)
if hard:
os.link(source_link, tlink)
else:
os.symlink(source_link, tlink)
except OSError as err:
Utils.error("[%s] Can't create %slink %s: %s" %
(self.manager.bank.name, 'hard ' if hard else 'sym', tlink, str(err)))
self.add_link()
return self.created_links
def __init__(self, manager=None):
"""
Init class
:param manager: Manager instance
:type manager: :class:`biomajmanager.manager.Manager`
:raises SystemExit: If 'manager' not given
:raises SystemExit: If 'manager' arg not an instance of :class:`biomajmanager.manager.Manager`
:raises SystemExit: If current production dir can't be found for current bank
"""
self.source = None
self.target = None
if not manager:
Utils.error("A manager is required")
if not isinstance(manager, Manager):
Utils.error("A Manager instance is required")
self.manager = manager
self.bank_name = self.manager.bank.name
if not self.manager.config.has_option('GENERAL', 'data.dir'):
Utils.error("'data.dir' not defined in global.properties or bank.properties")
if not self.manager.config.has_option('MANAGER', 'production.dir'):
Utils.error("'production.dir' not defined in manager.properties.")
self.prod_dir = self.manager.config.get('MANAGER', 'production.dir')
current_release = self.manager.current_release()
if current_release is None:
Utils.error("Can't determine current release for bank %s" % self.bank_name)
# Get the 'current'
bank_data_dir = self.manager.get_current_link()
self.bank_data_dir = bank_data_dir
self.created_links = 0
def update_db_with_data(self, filter, data, collection=None):
"""
Method used to update the Bioweb mongo database
:param filter: Query that match the document
:type filter: dict
:param data: Original data to update
:type data: dict
:param collection: Collection name
:type collection: str
:return: Boolean
:rtype: bool
"""
if not collection:
Utils.error("A collection name is required")
if not Bioweb.CONNECTED:
self._init_db()
if (pymongo.version_tuple)[0] > 2:
res = self.getCollection(collection).update_one(filter, {'$set': data}, upsert=True)
else:
res = self.getCollection(collection).update(filter, {'$set': data}, upsert=True)
self._update_documents_counts(res)
self._print_updated_documents()
def __init__(self, news_dir=None, config=None, max_news=None):
"""
Initiate object building
:param news_dir: Path to directory containing templates
:type news_dir: str
:param config: Configuration object
:type config: :class:`configParser`
:param max_news: Number of news to get when displaying news (default :const:`News.MAX_NEWS`)
:type max_news: int
:raises SystemExit: If 'news_dir' is not a directory
:raises SystemExit: If 'NEWS' section is not defined in :py:data:`manager.properties`
:raises SystemExit: If 'news.dir' is not set in :py:data:`manager.properties`
"""
self.news_dir = None
self.max_news = News.MAX_NEWS
self.data = None
if max_news:
self.max_news = max_news
if news_dir is not None:
Utils.verbose("[news] 'news_dir' set from %s" % str(news_dir))
if not os.path.isdir(news_dir):
Utils.error("News dir %s is not a directory." % news_dir)
self.news_dir = news_dir
if config is not None:
if not config.has_section('NEWS'):
Utils.error("Configuration has no 'NEWS' section.")
elif not config.has_option('NEWS', 'news.dir'):
Utils.error("Configuration has no 'news.dir' key.")
else:
self.news_dir = config.get('NEWS', 'news.dir')
Utils.verbose("[news] 'news_dir' set to %s" % str(self.news_dir))
def _generate_dir_link(self, source=None, target=None, hard=False, fallback=None, requires=None, limit=0):
"""
Create a symbolic link between 'source' and 'target' for a directory
:param source: Source directory to link
:type source: str
:param target: Destination directory name (relative to config param 'production.dir')
:type target: str
:param hard: Create hard link instead of symlink
:type hard: bool (default False)
:param fallback: Alternative source if source does not exist
:type fallback: str
:param requires: A required directory
:type requires: str
:param limit: Limit deepest search to `limit` depth, default 0, no limit
:type limit: int
:return: Number of created link(s)
:rtype: int
"""
if not self._prepare_links(source=source, target=target, fallback=fallback,
requires=requires, get_deepest=True, limit=limit):
return 0
slink = os.path.join(self.source)
tlink = os.path.join(self.target, self.manager.bank.name)
self._make_links(links=[(slink, tlink)], hard=hard)
if Manager.get_simulate() and Manager.get_verbose():
Utils.verbose("%s -> %s directory link done" % (self.target, self.source))
return self.created_links
def update_bioweb(self):
"""
Update the Bioweb.catalog MongoDB collection
:return: Boolean
:rtype: boole
"""
history = self.manager.history()
if not self._update_mongodb(data=history):
Utils.error("Can't update bioweb.catalog")
return True
def __init__(self, rss_file=None, *args, **kwargs):
"""Initiate object building"""
super(RSS, self).__init__(*args, **kwargs)
self.rss_file = None
self.fh = None
if rss_file is not None:
self.rss_file = rss_file
if 'config' in kwargs:
self.config = kwargs['config']
if self.config.has_option('RSS', 'rss.file') and self.rss_file is None:
self.rss_file = self.config.get('RSS', 'rss.file')
if self.rss_file is None:
self.fh = sys.stdout
Utils.verbose("[rss] rss_file set to %s" % str(self.rss_file))
def do_links(self, dirs=None, files=None, clone_dirs=None):
"""
Create a list of links
:param dirs: Directory to symlink
:type dirs: dict {'source1': ['target1', 'target2', ...], 'source2': [], ...}
:param files: Files to symlink
:type files: dict {'source1': ['target1','target2', ...], 'source2': [], ...},
:param clone_dirs: Directory to clone
:type clone_dirs: dict
:return: Number of created links
:rtype: int
:raises SystemExit: If user noth allowed to create link, see :py:data:`global.properties:admin`
"""
props = self.manager.bank.get_properties()
admin = None
if 'owner' in props and props['owner']:
admin = props['owner']
if Utils.user() != admin:
Utils.error("%s is not allowed to create link(s)" % Utils.user())
# Our default internal use
if dirs is None:
dirs = Links.DIRS
# EXPERIMENTAL AS OF 12 May 2016, New Structure for BioMAJ Links
if clone_dirs is None:
clone_dirs = Links.CLONE_DIRS
if files is None:
files = {
'golden': [{'target': 'index/golden'}],
'blast2': [{'target': 'fasta'}, {'target': 'index/blast2'}],
'hmmer': [{'target': 'index/hmmer'}],
'fasta': [{'target': 'fasta', 'remove_ext': True}],
'bdb': [{'target': 'index/bdb', 'remove_ext': True}]
}
for target, sources in list(clone_dirs.items()):
for source in sources:
self._clone_structure(target=target, **source)
for source, targets in list(dirs.items()):
for target in targets:
self._generate_dir_link(source=source, **target)
for source, targets in list(files.items()):
for target in targets:
self._generate_files_link(source=source, **target)
return self.created_links
def _print_updated_documents(self):
"""
Print a small report of action(s) done duinrg the update
:return: Boolean
:rtype: bool
"""
if not self.get_manager().get_verbose():
return True
bank = ""
if self.manager.bank:
bank = "[%s] " % self.manager.bank.name
Utils.ok("%sDocument(s) modification(s):\n\tMatched %d\n\tUpdated %d\n\tInserted %d"
% (bank, self.doc_matched, self.doc_modified, self.doc_upserted))
self.doc_matched = self.doc_modified = self.doc_upserted = 0
return True
def getCollection(self, name):
"""
Get a collection (pymongo) object from the list loaded at connection.
If the collection does not exists it prints an error on STDERR and exit(1)
:param name: Collection name
:type name: str
:return: Collection object, or throws error
:rtype: :class:`pymongo.collection`
"""
if not name:
Utils.error("A collection name is required")
if not name in self.collections:
Utils.error("Collection %s not found" % str(name))
return self.collections[name]
def _update_mongodb(self, data=None, collection='catalog', params=None, upsert=True):
"""
Function that really update the Mongodb collection ('catalog')
It does an upsert to update the collection
:param data: Data to be updated
:type data: dict
:param collection: Collection name to update (Default 'catalog')
:type collection: str
:param params: Extra parameters to filter documents to update
:type params: dict
:param upsert: Perform upsert or not
:type upsert: bool
:return: Boolean
:rtype: bool
"""
if not self.manager.bank.name:
Utils.error("Can't update, bank name required")
if not data:
Utils.warn("[%s] No data to update bioweb catalog" % self.manager.bank.name)
return True
if not Bioweb.CONNECTED:
self._init_db()
# Find arguments
search_params = {'type': Bioweb.COLLECTION_TYPE, 'name': self.manager.bank.name}
# Extra serach parameters?
if params is not None:
search_params.update(params)
for item in data:
if '_id' in item:
search_params['_id'] = item['_id']
if (pymongo.version_tuple)[0] > 2:
res = self.getCollection(collection).update_one(search_params, {'$set': item},
upsert=upsert)
else:
res = self.getCollection(collection).update(search_params, {'$set': item},
upsert=upsert)
self._update_documents_counts(res)
self._print_updated_documents()
return True
def _check_source_target_parameters(self, source=None, target=None):
"""
Check all parameters are set and ok to prepare link building
:param source: Source path
:type source: str
:param target: Destination path
:type target: str
:return: True if all is ok, throws otherwise
:rtype: bool
:raises SystemExit: If 'source' or 'target' are None
:raises SystemExit: If 'data.dir' not set in :py:data:`global.properties`
:raises SystemExit: If 'production.dir' not set in :py:data:`manager.properties`
"""
if not source:
Utils.error("source required")
if not target:
Utils.error("target required")
return True
def _init_db(self):
"""Load and connect to Mongodb database"""
if not self.config:
Utils.error("No configuration object set")
try:
# Try to connect to MongoDB using args
mongo_host = self.config.get(self.get_name(), 'bioweb.mongo.host')
mongo_port = int(self.config.get(self.get_name(), 'bioweb.mongo.port'))
mongo_use_ssl = int(self.config.get(self.get_name(), 'bioweb.mongo.use_ssl'))
mongo_options = {}
if mongo_use_ssl:
mongo_options['ssl'] = True
mongo_options['ssl_cert_reqs'] = ssl.CERT_NONE
# Specific SSL agrs for bioweb-prod
self.mongo_client = pymongo.MongoClient(host=mongo_host, port=mongo_port, **mongo_options)
dbname = self.config.get(self.get_name(), 'bioweb.mongo.db')
self.dbname = self.mongo_client[dbname]
self.collections = {}
# Keep trace of updated documents
self.doc_matched = self.doc_modified = self.doc_upserted = 0
if not self.config.has_option(self.get_name(), 'bioweb.mongo.collections'):
Utils.error("No collection(s) set for bioweb database")
for collection in self.config.get(self.get_name(), 'bioweb.mongo.collections').strip().split(','):
self.collections[collection] = self.dbname[collection]
except pymongo.errors.ConnectionFailure as err:
raise Exception("[ConnectionFailure] Can't connect to Mongo database %s: %s" % (dbname, str(err)))
except pymongo.errors.InvalidName as err:
raise Exception("Error getting collection: %s" % str(err))
except pymongo.errors.InvalidURI as err:
raise Exception("[InvalidURI] Can't connect to Mongo database %s: %s" % (dbname, str(err)))
except pymongo.errors.OperationFailure as err:
raise Exception("Operation failed: %s" % str(err))
except pymongo.errors.InvalidName as err:
raise Exception("Error getting collection: %s" % str(err))
except Exception as err:
raise Exception("Error while setting Mongo configuration: %s" % str(err))
Bioweb.CONNECTED = True
def set_bank_update_news(self):
"""
Send a new to MongoDB to let bioweb know about bank update. Update MongoDB
:return: Boolean
:rtype: bool
"""
if not self.manager.bank:
Utils.error("A bank name is required")
if 'current' in self.manager.bank.bank:
data = {}
data['message'] = "Bank %s updated to version %s" % (self.manager.bank.name, str(self.manager.current_release()))
data['date'] = Utils.time2date(self.manager.bank.bank['current'])
data['operation'] = "databank update"
data['type'] = Bioweb.COLLECTION_TYPE
data['name'] = self.manager.bank.name
return self._update_mongodb(data=[data], collection='news')
Utils.warn("Can't set new %s bank version, not published yet" % self.manager.bank.name)
return False
def _check_user_granted(*args, **kwargs):
"""
Check the user has enough right to perform action(s).
If a bank is set, we first set the user as the owner of
the current bank. Otherwise we try to find it from the
config file, we search for 'admin' property
:return: Boolean
"""
self = args[0]
admin = self.config.get('GENERAL', 'admin')
if self.bank:
props = self.bank.get_properties()
if 'owner' in props and props['owner']:
admin = props['owner']
if not admin:
Utils.error("Could not find admin user either in config nor in bank")
user = self.get_current_user()
if admin != user:
Utils.error("[%s] User %s, permission denied" % (admin, user))
return func(*args, **kwargs)
def write(self, template=None, data=None):
"""
Print template 'data' to stdout using template file 'template'.
'data' arg can be left None, this way method can be used to render file
from scratch
:param template: Template file name
:type template: str
:param data: Template data
:type data: dict
:return: True, throws on error
:rtype: bool
:raises SystemExit: If 'template' is None
:raises SystemExit: If 'template' is not found
:raises SystemExit: If 'template' has a syntax error in it
:raises SystemExit: If 'output' file cannot be opened
"""
if template is None:
Utils.error("A template name is required")
try:
template = self.env.get_template(template)
except TemplateNotFound as err:
Utils.error("Template %s not found in %s" % (err, self.template_dir))
except TemplateSyntaxError as err:
Utils.error("Syntax error found in template '%s', line %d: %s" % (err.name, err.lineno, err.message))
if self.output is None:
ofile = sys.stdout
else:
try:
ofile = open(self.output, "w")
except IOError as err:
Utils.error("Can't open %s: %s" % (self.output, str(err)))
Utils.uprint(template.render(data), to=ofile)
return True
def __init__(self, template_dir=None, config=None, output=None):
"""
Create Writer object
:param template_dir: Root directory where to find templates
:type template_dir: str
:param config: Global configuration file from BiomajConfig
:type config: :class:`configparser`
:param output: Output file. Default STDOUT
:type output: str
:raises SystemExit: If 'template_dir' is not given
:raises SystemExit: If 'MANAGER' section not found in :py:data:`manager.properties`
:raises SystemExit: If 'template.dir' not set in :py:data:`manager.properties`
"""
self.env = None
self.output = None
self.template_dir = None
if template_dir is not None:
if not os.path.isdir(template_dir):
Utils.error("Template dir %s is not a directory" % template_dir)
self.template_dir = template_dir
elif config is not None:
if not config.has_section("MANAGER"):
Utils.error("Configuration has no 'MANAGER' section.")
elif not config.has_option("MANAGER", "template.dir"):
Utils.error("Configuration has no 'template.dir' key.")
else:
self.template_dir = config.get("MANAGER", "template.dir")
if self.template_dir is None:
Utils.error("'template.dir' not set")
self.env = Environment(
loader=FileSystemLoader(os.path.join(self.template_dir)),
trim_blocks=True,
lstrip_blocks=True,
extensions=["jinja2.ext.with_"],
)
self.output = output
def get_news(self, news_dir=None, reverse=True):
"""
Get the news to be displayed from the specific news.dir directory
:param news_dir: Path to news directory
:type news_dir: str
:param reverse: Reverse list of news files, default True
:type reverse: bool
:return: news_files, list of news files found into 'news' directory
:rtype: list
:raises SystemExit: If path 'news_dir' does not exist
:raises SystemExit: If 'news_dir' not set
"""
if news_dir is not None:
if not os.path.isdir(news_dir):
Utils.error("News dir %s is not a directory" % news_dir)
else:
self.news_dir = news_dir
if not self.news_dir:
Utils.error("Can't get news, no 'news.dir' defined.")
news_data = []
item = 0
# shamefully copied from
# http://stackoverflow.com/questions/168409/how-do-you-get-a-directory-listing-sorted-by-creation-date-in-python
# get all entries in the directory w/ stats
files = (os.path.join(self.news_dir, file) for file in os.listdir(self.news_dir))
#files = ((os.stat(path), path) for path in files)
#files = ((stat[ST_CTIME], path) for stat, path in files if S_ISREG(stat[ST_MODE]))
#for _, ifile in sorted(files):
for ifile in sorted(files):
with open(ifile) as new:
Utils.verbose("[news] Reading news file %s ..." % ifile)
(label, date, title) = new.readline().strip().split(':')
text = ''
for line in new.readlines():
text += line
news_data.append({'label': label, 'date': date, 'title': title, 'text': text, 'item': item})
item += 1
new.close()
if reverse:
news_data.reverse()
self.data = {'news': news_data}
return self.data
def _prepare_links(self, source=None, target=None, get_deepest=False, fallback=None, requires=None, limit=0):
"""
Prepare stuff to create links
:param source: Source path
:type source: str
:param target: Destination path
:type target: str
:param get_deepest: Try to find deepest directory(ies) from source
:type get_deepest: bool
:param fallback: Alternative source if source does not exist
:type fallback: str
:param requires: A required file or directory
:type requires: str
:param limit: Limit deepest search to `limit` depth, default 0, no limit
:type limit: int
:return: Boolean
:rtype: bool
:raises SystemExit: If 'source' or 'target' are None
:raises SystemExit: If 'data.dir' not set in :py:data:`global.properties`
:raises SystemExit: If 'production.dir' not set in :py:data:`manager.properties`
:raises SystemExit: If 'target' directory cannot be created
"""
self._check_source_target_parameters(source=source, target=target)
data_dir = self.bank_data_dir
source = os.path.join(data_dir, source)
target_dir = self.manager.config.get('MANAGER', 'production.dir')
bank_name = self.manager.bank.name
if requires is not None:
if not os.path.exists(os.path.join(data_dir, requires)):
Utils.warn("[%s] Can't create %s, requires param %s not here." % (bank_name, source, requires))
return False
if not os.path.isdir(source):
if fallback is None:
if self.manager.get_verbose():
Utils.warn("[%s] %s does not exist" % (bank_name, source))
return False
else:
if self.manager.get_verbose():
Utils.verbose("[%s] %s does not exist. Fallback to %s" % (bank_name, source, fallback))
source = os.path.join(data_dir, fallback)
if not os.path.isdir(source):
if self.manager.get_verbose():
Utils.warn("[%s] Fallback %s does not exist" % (bank_name, source))
return False
if get_deepest:
source = Utils.get_deepest_dir(source, full=get_deepest, limit=limit)
target = os.path.join(target_dir, target)
# Check destination directory where to create link(s)
if not os.path.exists(target) and not os.path.isdir(target):
if Manager.get_simulate() and Manager.get_verbose():
Utils.verbose("[_prepare_links] [%s] Creating directory %s" % (bank_name, target))
else:
try:
if not Manager.get_simulate():
os.makedirs(target)
except OSError as err:
Utils.error("[%s] Can't create %s dir: %s" % (bank_name, target, str(err)))
self.source = source
self.target = target
if Manager.get_verbose():
Utils.verbose("[prepare_links] source %s" % self.source)
Utils.verbose("[prepare_links] target %s" % self.target)
return True
def _clone_structure(self, source=None, target=None, remove_ext=False, limit=0):
"""
Create a directory structure from a source to a target point and link all files from source inside target
:param source: Source directory to clone
:type source: str
:param target: Destination directory to create if does not exist
:type target: str
:param remove_ext: Create another link of the file without the file name extension
:type remove_ext: bool
:param limit: Limit subtree seach to value, default 0, no limit
:tpye limit: int
:return: True if structure cloning build OK, throws otherwise
:rtype: bool
:raise SystemExit: If error occurred during directory structure building
"""
self._check_source_target_parameters(source=source, target=target)
# Check do_links.clone_dirs. As we want to recreate the same architecture as for the source,
# we need to recreate the target because Utils.get_subtree removes the source path which contains
# the target name
target = os.path.join(target, source)
source = os.path.join(self.bank_data_dir, source)
subtrees = Utils.get_subtree(path=source, limit=limit)
try:
for subtree in subtrees:
end_target = os.path.join(self.prod_dir, target, subtree)
if not os.path.exists(end_target) and not os.path.isdir(end_target):
if Manager.get_simulate() and Manager.get_verbose():
Utils.verbose("[_clone_structure] [%s] Creating directory %s" % (self.bank_name, end_target))
else:
if not Manager.get_simulate():
os.makedirs(end_target)
sub_files = Utils.get_files(path=os.path.join(source, subtree))
if len(sub_files) == 0:
continue
links = []
for ffile in sub_files:
# Source file link
slink = os.path.join(source, subtree, ffile)
tlink = os.path.join(end_target, ffile)
links.append((slink, tlink))
if Manager.get_verbose():
Utils.verbose("[_generate_files_link] append slink %s" % slink)
Utils.verbose("[_generate_files_link] append tlink %s" % tlink)
# If asked to create another symbolic link without extension name
if remove_ext:
new_file = os.path.splitext(os.path.basename(ffile))[0]
tlink = os.path.join(end_target, new_file)
links.append((slink, tlink))
if Manager.get_verbose():
Utils.verbose("[_generate_files_link] [rm_ext=%s] append slink %s"
% (str(remove_ext), slink))
Utils.verbose("[_generate_files_link] [rm_ext=%s] append tlink %s"
% (str(remove_ext), tlink))
# Set self.target for _make_links
self.target = end_target
self._make_links(links=links)
except OSError as err:
Utils.error("[%s] Can't create %s dir: %s (%s)" % (self.bank_name, end_target, str(err),
os.access(end_target, os.W_OK)))
return True
def generate_rss(self, rss_file=None, data=None):
"""
Generate RSS file from news
:param rss_file: Path to file rss.xml
:type rss_file: str
:param data: Data to create RSS from
:type data: dict data['news'] = { ... }
:return: Boolean
:rtype: bool
:raises SystemExit: If 'news' key is not found in 'data' dict
:raises SystemExit: If rss file cannot be opened
"""
if rss_file is not None:
Utils.verbose("[rss] rss_file set to %s" % rss_file)
self.rss_file = rss_file
if data is None:
data = self.get_news()
elif 'news' not in data:
Utils.error("Could not find 'news' key in data")
if len(data['news']) == 0:
Utils.warn("No data to display")
return True
items = []
try:
for new in data['news']:
item = Item(title=new['title'],
description=new['text'],
author=self.config.get('RSS', 'feed.author'),
guid=Guid(self.config.get('RSS', 'feed.news.link') + '#' + str(new['item'])),
pubDate=datetime.strptime(new['date'],
self.config.get('RSS', 'rss.date.format')
.replace('%%', '%'))
)
items.append(item)
feed = Feed(title=self.config.get('RSS', 'feed.title'),
link=self.config.get('RSS', 'feed.link'),
description=self.config.get('RSS', 'feed.description'),
language=self.config.get('RSS', 'feed.language'),
lastBuildDate=datetime.now(),
items=items)
if self.fh is None:
self.fh = open(self.rss_file, 'w')
Utils.uprint(feed.rss(), to=self.fh)
if self.rss_file is not None:
self.fh.close()
except (NoOptionError, NoSectionError) as err:
Utils.error("Option missing in config file: %s" % str(err))
except (OSError, IOError) as err:
Utils.error("Can't open file %s: %s" % (self.rss_file, str(err)))
return True
def _generate_files_link(self, source=None, target=None, remove_ext=False):
"""
Links list of file from 'source' to 'target' directory.
If remove_ext is set to True, then another link is created. This link is the same as the
target link, without the file extension
:param source: Source directory to link
:type source: str
:param target: Destination directory name (relative to config param 'production_dir')
:type target: str
:param remove_ext: Create another link of the file without the file name extension
:type remove_ext: bool (default False)
:return: Number of created link(s)
:rtype: int
"""
if not self._prepare_links(source=source, target=target, get_deepest=True):
return 0
# Get files in the source directory
files = Utils.get_files(path=self.source)
links = []
for ffile in files:
# Source file link
slink = os.path.join(self.source, ffile)
tlink = os.path.join(self.target, ffile)
links.append((slink, tlink))
if Manager.get_verbose():
Utils.verbose("[_generate_files_link] append slink %s" % slink)
Utils.verbose("[_generate_files_link] append tlink %s" % tlink)
# If asked to create another symbolic link without extension name
if remove_ext:
new_file = os.path.splitext(os.path.basename(ffile))[0]
tlink = os.path.join(self.target, new_file)
links.append((slink, tlink))
if Manager.get_verbose():
Utils.verbose("[_generate_files_link] [rm_ext=%s] append slink %s" % (str(remove_ext), slink))
Utils.verbose("[_generate_files_link] [rm_ext=%s] append tlink %s" % (str(remove_ext), tlink))
self._make_links(links=links)
if Manager.get_simulate() and Manager.get_verbose():
Utils.verbose("%s -> %s file link done" % (self.target, self.source))
return self.created_links
def _check_bank_required(*args, **kwargs):
"""Small function to check a bank object is set in BioMAJ Manager instance"""
self = args[0]
if self.bank is None:
Utils.error("A bank name is required")
return func(*args, **kwargs)
def _dep_func():
Utils.error("Call to deprecated function '{}'. Not executed.".format(func.__name__))
def update_bioweb_from_mysql(self):
"""
Get the history from the MySQL database (Biomaj 1.2.x) and transform it to a json document.
:return: Boolean
:rtype: bool
"""
import mysql.connector
from mysql.connector import errorcode
params = {}
for param in ['host', 'database', 'user', 'password']:
if not self.config.has_option(self.get_name(), 'bioweb.mysql.%s' % param):
Utils.error("MySQL %s parameter not set!" % param)
else:
params[param] = self.config.get(self.get_name(), 'bioweb.mysql.%s' % param)
try:
cnx = mysql.connector.connect(**params)
cursor = cnx.cursor(dictionary=True)
query = "SELECT b.name AS bank, ub.updateRelease AS version, pd.remove AS removed, pd.creation AS created, "
query += "pd.path AS path, pd.state AS status, pd.idproductionDirectory AS id FROM productionDirectory pd "
query += "JOIN updateBank ub ON ub.idLastSession = pd.session JOIN bank b on b.idbank = pd.ref_idbank WHERE "
query += "b.name = %(bank)s ORDER BY creation DESC;"
cursor.execute(query, {'bank': self.manager.bank.name})
history = [ ]
packages = self.manager.get_bank_packages()
description = self.manager.bank.config.get('db.fullname').replace('"', '').strip()
bank_type = self.manager.bank.config.get('db.type').split(',')
bank_format = self.manager.bank.config.get('db.formats').split(',')
status = None
for row in cursor.fetchall():
if not row['removed']:
if not status:
status = 'online'
else:
status = 'deprecated'
else:
status = row['status']
history.append({'_id': '@'.join(['bank',
self.manager.bank.name,
row['version'],
row['created'].strftime(Utils.DATE_FMT)]),
'type': 'bank',
'name': self.manager.bank.name,
'version': row['version'],
'publication_date': row['created'], #Utils.local2utc(row['created']),
'removal_date': row['removed'], #Utils.local2utc(row['removed']) if row['removed'] else row['removed'],
'bank_type': bank_type,
'bank_format': bank_format,
'packageVersions': packages,
'description': description,
'status': status,
})
if not self._update_mongodb(data=history):
Utils.error("Can't update bioweb.catalog")
except mysql.connector.ProgrammingError as error:
Utils.error("[Syntax Error] %s" % str(error))
except mysql.connector.Error as error:
if error.errno == errorcode.ER_ACCESS_DENIED_ERROR:
Utils.error("[Access Denied] Wrong username or password: %s" % error.msg)
elif error.errno == errorcode.ER_BAD_DB_ERROR:
Utils.error("[Database] Database does not exist: %s" % error.msg)
else:
Utils.error("Unknown error: %s" % error)
finally:
cnx.close()
return True
def __init__(self, manager=None, name=None):
"""
Create the plugin object
:param manager: Manager instance
:type manager: :class:`biomajmanager.manager.Manager`
:param name: Name of the plugin to load. [DEFAULT: load all plugins]
:type name: String
:raises SystemExit: If 'manager' arg is not given
:raises SystemExit: If 'PLUGINS' section not found in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.list' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' does not exist
"""
self.pm = None
self.name = None
self.config = None
self.manager = None
if not manager:
Utils.error("'manager' is required")
self.manager = manager
self.config = self.manager.config
if not self.config.has_section('PLUGINS'):
Utils.error("Can't load plugins, no section found!")
if not self.config.has_option('MANAGER', 'plugins.dir'):
Utils.error("plugins.dir not defined!")
if not self.config.has_option('PLUGINS', 'plugins.list'):
Utils.error("plugins.list is not defined!")
if not os.path.isdir(self.config.get('MANAGER', 'plugins.dir')):
Utils.error("Can't find plugins.dir")
plugin_manager = PluginManager(directories_list=[self.config.get('MANAGER', 'plugins.dir')],
categories_filter={Plugins.CATEGORY: BMPlugin})
plugin_manager.collectPlugins()
self.pm = plugin_manager
self.name = name
user_plugins = []
# Load user wanted plugin(s)
for plugin in self.config.get('PLUGINS', 'plugins.list').split(','):
plugin.strip()
# We need to lower the plugin name
user_plugins.append(plugin)
# This means that all plugins must inherits from BMPlugin
for pluginInfo in plugin_manager.getPluginsOfCategory(Plugins.CATEGORY):
Utils.verbose("[manager] plugin name => %s" % pluginInfo.name)
if pluginInfo.name in user_plugins:
if not pluginInfo.is_activated:
Utils.verbose("[manager] plugin %s activated" % pluginInfo.name)
plugin_manager.activatePluginByName(pluginInfo.name)
setattr(self, pluginInfo.name, pluginInfo.plugin_object)
pluginInfo.plugin_object.set_config(self.config)
pluginInfo.plugin_object.set_manager(self.manager)
