def _getUserDir(self, dir_type, tool = None, create=False):
base_dir = dict(base='', config=User.CONFIG_DIR, cache=User.CACHE_DIR, output=User.OUTPUT_DIR, \
plots=User.PLOTS_DIR, processes=User.PROCESSES_DIR, \
scheduler_in=config.get(config.SCHEDULER_INPUT_DIR), \
scheduler_out=config.get(config.SCHEDULER_OUTPUT_DIR))
if tool is None:
bd = base_dir[dir_type]
# concatenate relative paths only
if bd and bd[0]=='/':
dir_name = bd
else:
#return the directory where the tool configuration files are stored
dir_name = os.path.join(self._getUserBaseDir(), bd)
else:
#It's too confusing if we create case sensitive directories...
tool = tool.lower()
#return the specific directory for the given tool
dir_name = os.path.join(self._getUserBaseDir(), base_dir[dir_type], tool)
#make sure we have a canonical path
dir_name = os.path.abspath(dir_name)
if create and not os.path.isdir(dir_name):
#we are letting this fail in case of problems.
utils.supermakedirs(dir_name, 0755)
return dir_name
def test_central_directory_Creation(self):
tmp_dir = tempfile.mkdtemp(__name__)
config._config[config.BASE_DIR_LOCATION] = tmp_dir
config._config[
config.
DIRECTORY_STRUCTURE_TYPE] = config.DIRECTORY_STRUCTURE.CENTRAL
testUser = DummyUser(random_home=False, **Test.DUMMY_USER)
dir1 = testUser.getUserBaseDir()
self.assertEquals(
dir1,
os.path.join(config.get(config.BASE_DIR_LOCATION),
config.get(config.BASE_DIR), str(testUser.getName())))
dir2 = testUser.getUserOutputDir('sometool')
self.assertEquals(
dir2,
os.path.join(config.get(config.BASE_DIR_LOCATION),
config.get(config.BASE_DIR), str(testUser.getName()),
User.OUTPUT_DIR, 'sometool'))
print dir2
config.reloadConfiguration()
if os.path.isdir(tmp_dir) and tmp_dir.startswith(
tempfile.gettempdir()):
#make sure the home is a temporary one!!!
print "Cleaning up %s" % tmp_dir
shutil.rmtree(tmp_dir)
def _getUserBaseDir(self):
if self._dir_type == config.DIRECTORY_STRUCTURE.LOCAL:
return os.path.join(self.getUserHome(), config.get(config.BASE_DIR))
elif self._dir_type == config.DIRECTORY_STRUCTURE.CENTRAL:
return os.path.join(config.get(config.BASE_DIR_LOCATION), config.get(config.BASE_DIR), str(self.getName()))
elif self._dir_type == config.DIRECTORY_STRUCTURE.SCRATCH:
return os.path.join(config.get(config.BASE_DIR_LOCATION), str(self.getName()), config.get(config.BASE_DIR))
def _run(self):
root_path = config.get('project_data')
# Setup argument parser
args = self.args
crawl_dir = args.path
t1 = time.time()
sys.stderr.flush()
# For miklip we want to use a prefix
# TODO: Get this directly from config?
project = config.get('project_name')
path_prefix = ''
if project.lower() == 'miklip':
path_prefix = 'user-'
user_root_path = os.path.join(root_path, path_prefix+User().getName())
if crawl_dir:
if root_path not in crawl_dir:
raise Exception('You are only allowed to crawl data in this root path %s' % root_path)
else:
crawl_dir = user_root_path
path = os.path.dirname(os.path.abspath(__file__))
#script_path = path + '/../../../sbin/crawl_data'
script_path = path + '/../../../sbin/solr_server path2ingest'
print 'Please wait while the system is crawling your data'
out = subprocess.Popen('/bin/bash '+script_path + ' ' + crawl_dir, shell=True)
print out.wait()
print 'Finished.\nCrawling took ' + str(time.time() - t1) + ' seconds'
def test_get(self):
base_dir = config.get(config.BASE_DIR)
self.assertEquals(base_dir, 'evaluation_system')
self.failUnlessRaises(config.ConfigurationException, config.get,
'non-existing-key')
self.assertEquals(config.get('non-existing-key', 'default-answer'),
'default-answer')
def test_reload(self):
"""Test we can reload the configuration"""
config._config[config.BASE_DIR_LOCATION] = 'TEST'
c1 = config.get(config.BASE_DIR_LOCATION)
self.assertEquals(c1, 'TEST')
config.reloadConfiguration()
c2 = config.get(config.BASE_DIR_LOCATION)
self.assertNotEquals(c1, c2)
def linkmydata(self,outputdir=None):
"""Link the CMOR Data Structure of any output created by a tool
crawl the directory and ingest the directory with solr::
:param outputdir: cmor outputdir that where created by the tool.
:return: nothing
"""
user = self._user
workpath = os.path.join(user.getUserBaseDir(),'CMOR4LINK')
rootpath = config.get('project_data')
solr_in = config.get('solr.incoming')
solr_bk = config.get('solr.backup')
solr_ps = config.get('solr.processing')
# look for tool in tool
toolintool = re.compile(r'^((?P<tool>[\w%]+)%(\d+|none)%(?P<project>[\w_]+)%(?P<product>[\w_]+)$)')
# Maybe os.walk for multiple projects or products
if len(os.listdir(outputdir)) == 1:
project = os.listdir(outputdir)[0]
# link?
if len(os.listdir(os.path.join(outputdir,project))) == 1:
product = os.listdir(os.path.join(outputdir,project))[0]
new_product = '%s.%s.%s.%s' % (self.__class__.__name__.lower(),self.rowid,project,product)
if re.match(toolintool,product):
nproduct = re.match(toolintool,product).group('product')
nproject = re.match(toolintool,product).group('project')
ntool = '.%s' % re.match(toolintool,product).group('tool')
new_product = '%s.s%s.%s.%s' % (self.__class__.__name__.lower(),ntool,self.rowid,nproject,nproduct)
# Link section
link_path = os.path.join(rootpath,'user-' + user.getName())
if os.path.islink(link_path):
if not os.path.exists(link_path):
os.unlink(link_path)
os.symlink(workpath, os.path.join(link_path))
if not os.path.isdir(workpath): os.makedirs(workpath)
workpath = os.path.join(os.path.dirname(link_path), os.readlink(link_path))
else:
if not os.path.isdir(workpath): os.makedirs(workpath)
os.symlink(workpath, link_path)
os.symlink(os.path.join(outputdir,project,product), os.path.join(workpath,new_product))
# Prepare for solr
crawl_dir=os.path.join(link_path,new_product)
now = datetime.now().strftime('%Y-%m-%d_%H%M%S')
output = os.path.join(solr_in,'solr_crawl_%s.csv.gz' %(now))
# Solr part with move orgy
SolrCore.dump_fs_to_file(crawl_dir, output)
shutil.move(os.path.join(solr_in,output),os.path.join(solr_ps,output))
hallo = SolrCore.load_fs_from_file(dump_file=os.path.join(solr_ps,output))
shutil.move(os.path.join(solr_ps,output),os.path.join(solr_bk,output))
def setUp(self):
os.environ['EVALUATION_SYSTEM_CONFIG_FILE'] = os.path.dirname(
__file__) + '/test.conf'
config.reloadConfiguration()
self.solr_port = config.get('solr.port')
self.solr_host = config.get('solr.host')
# test instances, check they are as expected
self.all_files = SolrCore(core='files',
host=self.solr_host,
port=self.solr_port)
self.latest = SolrCore(core='latest',
host=self.solr_host,
port=self.solr_port)
self.assertEquals(self.all_files.status()['index']['numDocs'], 0)
self.assertEquals(self.latest.status()['index']['numDocs'], 0)
def test_config_file(self):
"""If a config file is provided it should be read"""
import tempfile
fd, name = tempfile.mkstemp(__name__, text=True)
with os.fdopen(fd, 'w') as f:
f.write('[evaluation_system]\n%s=nowhere\n' % config.BASE_DIR)
self.assertEquals(config.get(config.BASE_DIR), 'evaluation_system')
os.environ[config._DEFAULT_ENV_CONFIG_FILE] = name
config.reloadConfiguration()
self.assertEquals(config.get(config.BASE_DIR), 'nowhere')
os.unlink(name)
# check wrong section
fd, name = tempfile.mkstemp(__name__, text=True)
with os.fdopen(fd, 'w') as f:
f.write('[wrong_section]\n%s=nowhere\n' % config.BASE_DIR)
os.environ[config._DEFAULT_ENV_CONFIG_FILE] = name
self.failUnlessRaises(ConfigurationException, reloadConfiguration)
os.unlink(name)
# check directory structure value
fd, name = tempfile.mkstemp(__name__, text=True)
with os.fdopen(fd, 'w') as f:
f.write('[evaluation_system]\n%s=wrong_value\n' %
config.DIRECTORY_STRUCTURE_TYPE)
os.environ[config._DEFAULT_ENV_CONFIG_FILE] = name
self.failUnlessRaises(ConfigurationException, reloadConfiguration)
os.unlink(name)
# check $EVALUATION_SYSTEM_HOME get's resolved properly
fd, name = tempfile.mkstemp(__name__, text=True)
with os.fdopen(fd, 'w') as f:
f.write('[evaluation_system]\n%s=$EVALUATION_SYSTEM_HOME\n' %
config.BASE_DIR)
self.assertEquals(config.get(config.BASE_DIR), 'evaluation_system')
os.environ[config._DEFAULT_ENV_CONFIG_FILE] = name
config.reloadConfiguration()
self.assertEquals(config.get(config.BASE_DIR),
'/'.join(__file__.split('/')[:-4]))
os.unlink(name)
def getVersion(src_file):
retval = __version_cache.get(src_file, None)
if retval is None:
(dir_name, filename) = os.path.split(src_file)
command = 'module load git > /dev/null 2> /dev/null;'
if dir_name:
command += 'cd %s 2> /dev/null;' % dir_name
command += 'git config --get remote.origin.url;'
command += 'git show-ref --heads --hash'
options = config.get(config.GIT_BASH_STARTOPTIONS, '-lc')
bash = ['/bin/bash', options, command]
p = Popen(bash, stdout=PIPE, stderr=PIPE)
(stdout, stderr) = p.communicate()
try:
lines = stdout.split('\n')
repository = lines[-3]
version = lines[-2]
except Exception, e:
if not stderr:
stderr = str(e)
log.warn("Could not read git version")
log.debug("Error while reading git version:\n%s", stderr)
repository = 'unknown'
version = 'unknown'
retval = (repository, version)
__version_cache[src_file] = retval
def test_getters(self):
"""Test the object creation and some basic return functions"""
self.assertEqual(Test.DUMMY_USER['pw_name'], self.user.getName())
self.assertTrue(self.user.getUserHome().startswith(
tempfile.gettempdir()))
self.assertEqual(int(Test.runCmd('id -u')), self.user.getUserID())
db = self.user.getUserDB()
self.assertTrue(db is not None)
baseDir = '/'.join(
[self.user.getUserHome(),
config.get(config.BASE_DIR)])
self.assertEqual(baseDir, self.user.getUserBaseDir())
tool1_cnfDir = os.path.join(baseDir, User.CONFIG_DIR, 'tool1')
tool1_chDir = os.path.join(baseDir, User.CACHE_DIR, 'tool1')
tool1_outDir = os.path.join(baseDir, User.OUTPUT_DIR, 'tool1')
tool1_plotDir = os.path.join(baseDir, User.PLOTS_DIR, 'tool1')
self.assertEqual(self.user.getUserScratch(),
'/tmp/scratch/%s' % self.user.getName())
#check we get the configuration directory of the given tool
self.assertEqual(tool1_cnfDir, self.user.getUserConfigDir('tool1'))
self.assertEqual(tool1_chDir, self.user.getUserCacheDir('tool1'))
self.assertEqual(tool1_outDir, self.user.getUserOutputDir('tool1'))
self.assertEqual(tool1_plotDir, self.user.getUserPlotsDir('tool1'))
#check we get the general directory of the tools (should be the parent of the previous one)
self.assertEqual(os.path.dirname(tool1_cnfDir),
self.user.getUserConfigDir())
self.assertEqual(os.path.dirname(tool1_chDir),
self.user.getUserCacheDir())
self.assertEqual(os.path.dirname(tool1_outDir),
self.user.getUserOutputDir())
self.assertEqual(os.path.dirname(tool1_plotDir),
self.user.getUserPlotsDir())
def _preview_create(plugin_name, result):
"""
This routine creates the preview. And adds the created files
to the result dictionary.
:type plugin_name: str
:param plugin_name: name of the referred plugin.
:type result: meta_dict
:param result: a meta dictionary describing the result files
"""
todo_list = []
result_list = []
for file_name in result:
metadata = result[file_name]
todo = metadata.get('todo', '')
if todo == 'copy':
ext = os.path.splitext(file_name)[-1]
target_name = _preview_unique_file(plugin_name, file_name, ext, metadata)
todo_list.append((_preview_copy, file_name, target_name))
metadata['preview_path'] = target_name
result_list.append(target_name)
elif todo == 'convert':
target_name = _preview_unique_file(plugin_name, file_name, '.png', metadata)
todo_list.append((_preview_convert, file_name, target_name))
metadata['preview_path'] = target_name
result_list.append(target_name)
result[file_name] = metadata
preview_path = config.get(config.PREVIEW_PATH)
if preview_path.strip() and todo_list:
p = Pool(config.NUMBER_OF_PROCESSES)
p.map(utils.mp_wrap_fn, todo_list)
return result_list
def _preview_unique_file(plugin_name, file_name, ext, metadata):
"""
This routine creates a unique filename for the preview
:type plugin_name: str
:param plugin_name: name of the referred plugin.
:type file_name: str
:param file_name: the file to create a preview name for
:type ext: str
:param ext: the extension of the file to be created
:type metadata: dict
:param metadata: the meta-data for the file, to access timestamp
"""
path = config.get(config.PREVIEW_PATH)
subdir = datetime.datetime.now().strftime('%Y%m%d')
name = _preview_generate_name(plugin_name, file_name, metadata)
name += ext
full_path = os.path.join(path, subdir)
full_name = os.path.join(full_path, name)
if path.strip() and not os.path.isdir(full_path):
utils.supermakedirs(full_path, 0777)
if os.path.isfile(full_name):
return _preview_unique_file(plugin_name, file_name, ext, metadata)
return full_name
def setUp(self):
os.environ['EVALUATION_SYSTEM_CONFIG_FILE'] = os.path.dirname(
__file__) + '/test.conf'
config.reloadConfiguration()
self.solr_port = config.get('solr.port')
self.solr_host = config.get('solr.host')
# test instances, check they are as expected
self.all_files = SolrCore(core='files',
host=self.solr_host,
port=self.solr_port)
self.latest = SolrCore(core='latest',
host=self.solr_host,
port=self.solr_port)
self.assertEquals(self.all_files.status()['index']['numDocs'], 0)
self.assertEquals(self.latest.status()['index']['numDocs'], 0)
# add some files to the cores
supermakedirs('/tmp/some_temp_solr_core/', 0777)
self.tmpdir = '/tmp/some_temp_solr_core'
self.orig_dir = DRSFile.DRS_STRUCTURE[CMIP5]['root_dir']
DRSFile.DRS_STRUCTURE[CMIP5]['root_dir'] = self.tmpdir
self.files = [
'cmip5/output1/MOHC/HadCM3/historical/mon/aerosol/aero/r2i1p1/v20110728/wetso2/wetso2_aero_HadCM3_historical_r2i1p1_190912-193411.nc',
'cmip5/output1/MOHC/HadCM3/decadal2008/mon/atmos/Amon/r9i3p1/v20120523/tauu/tauu_Amon_HadCM3_decadal2008_r9i3p1_200811-201812.nc',
'cmip5/output1/MOHC/HadCM3/decadal2009/mon/atmos/Amon/r7i2p1/v20110719/ua/ua_Amon_HadCM3_decadal2009_r7i2p1_200911-201912.nc',
'cmip5/output1/MOHC/HadCM3/decadal2009/mon/atmos/Amon/r7i2p1/v20110819/ua/ua_Amon_HadCM3_decadal2009_r7i2p1_200911-201912.nc',
'cmip5/output1/MOHC/HadCM3/decadal2009/mon/atmos/Amon/r7i2p1/v20110419/ua/ua_Amon_HadCM3_decadal2009_r7i2p1_200911-201912.nc'
]
for f in self.files:
abs_path = os.path.abspath(os.path.join(self.tmpdir, f))
try:
os.makedirs(os.path.dirname(abs_path))
except: # pragma nocover
pass
with open(abs_path, 'w') as f_out:
f_out.write(' ')
dump_file = self.tmpdir + '/dump1.csv'
# add the files to solr
SolrCore.dump_fs_to_file(self.tmpdir + '/cmip5', dump_file)
SolrCore.load_fs_from_file(dump_file,
abort_on_errors=True,
core_all_files=self.all_files,
core_latest=self.latest)
self.fn = os.path.join(self.tmpdir, self.files[0])
self.drs = DRSFile.from_path(self.fn)
def getUserScratch(self):
""":returns: the path to the user's scratch directory.
:rtype: str"""
path = config.get(config.SCRATCH_DIR)
path = path.replace('$USER', self.getName())
return path
def __init__(self,
core=None,
host=None,
port=None,
echo=False,
instance_dir=None,
data_dir=None,
get_status=True):
"""Create the connection pointing to the proper solr url and core.
:param core: The name of the core referred (default: loaded from config file)
:param host: the hostname of the Solr server (default: loaded from config file)
:param port: The port number of the Solr Server (default: loaded from config file)
:param echo: If True, show all urls before issuing them.
:param instance_dir: the core instance directory (if empty but the core exists it will get downloaded from Solr)
:param data_dir: the directory where the data is being kept (if empty but the core exists it will
get downloaded from Solr)"""
if host is None:
host = config.get(config.SOLR_HOST)
if port is None:
port = config.get(config.SOLR_PORT)
if core is None:
core = config.get(config.SOLR_CORE)
self.solr_url = 'http://%s:%s/solr/' % (host, port)
self.core = core
self.core_url = self.solr_url + core + '/'
self.echo = echo
self.instance_dir = instance_dir
self.data_dir = data_dir
if get_status:
st = self.status()
else:
st = {}
if self.instance_dir is None and 'instanceDir' in st:
self.instance_dir = st['instanceDir']
if self.data_dir is None and 'dataDir' in st:
self.data_dir = st['dataDir']
else:
self.data_dir = 'data'
# Other Defaults
import socket
socket.setdefaulttimeout(20)
def _run(self):
doc_path = self.args.docpath
tex_file = self.args.tex_file
tool = self.args.tool.lower()
if not tex_file:
print 'Can\'t find a .tex file in this directory!'
return
file_root = tex_file.split('.')[0]
# copy folder to /tmp for processing
new_path = '/tmp/%s/' % tool
self.copy_and_overwrite(doc_path, new_path)
# change path and run "htlatex" and "bibtex"
os.chdir(new_path)
cfg_file = os.path.dirname(
__file__) + '/../../../../etc/ht5mjlatex.cfg'
os.system('htlatex %s "%s"' % (new_path + tex_file, cfg_file))
os.system('bibtex %s' % file_root)
os.system('htlatex %s "%s"' % (new_path + tex_file, cfg_file))
# open html file and remove <head> and <body> tags
fi = open(os.path.join(new_path, file_root + '.html'))
text = fi.read()
text = re.sub("<head>.*?</head>", "", text, flags=re.DOTALL)
text = text.replace('</html>', '')
text = text.replace('<html>', '')
text = text.replace('</body>', '')
text = text.replace('<body>', '')
figure_prefix = 'figures'
# replace img src
text = text.replace(
'src="%s/' % figure_prefix,
'style="width:80%;" src="/static/preview/doc/' + tool + '/')
# remove too big sigma symbols
text = text.replace('mathsize="big"', '')
flat_page, created = FlatPage.objects.get_or_create(
title=self.args.tool, url='/about/%s/' % tool)
if created:
flat_page.sites = [1]
flat_page.content = text
flat_page.save()
# Copy images to website preview path
preview_path = config.get('preview_path')
dest_dir = os.path.join(preview_path, 'doc/%s/' % tool)
self.copy_and_overwrite('%s/' % figure_prefix, dest_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copyfile('%s.css' % tool, os.path.join(dest_dir,
'%s.css' % tool))
# remove tmp files
shutil.rmtree(new_path)
def __module_interaction(self, command, module_name):
"""
Function to interact with the module interface
"""
module_path = config.get('module_path', None)
if module_path is None:
logging.warning('Module path is not set. Module %s NOT %sed' % (module_name, command))
return None
cmd = os.popen('%s python %s %s' % (module_path, command, module_name))
exec cmd in globals(), locals()
return True
def __init__(self):
self.logcache = config.get('esgf_logcache')
# self.logcache= '/pf/b/b324029/cache'
self.esgftmp = '%s/%s/' % (self.logcache, 'ESGF_CACHE')
self.logpath = '%s/%s/' % (self.logcache, 'ESGF_LOG')
self.esgf_server = config.get('esgf_server').split(',')
self.hostname = socket.gethostname()
self.certs = config.get('private_key')
self.wget = config.get('wget_path')
self.threadLimiter = threading.BoundedSemaphore(
int(config.get('parallel_downloads')))
self.rwlock = threading.Lock()
self.gid = grp.getgrnam("bmx828").gr_gid
try:
os.makedirs(self.logpath, 0775)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
def __init__(self, project, experiment, outpath, p2p=P2P()):
self.project = {'project': project}
self.experiment = experiment
self.outpath = outpath
self.show_facets = 'experiment'
self.facets = {'project': self.project.values()[0], 'type': 'File'}
self.fields = [
'title', 'size', 'project', 'product', 'institute', 'model',
'experiment', 'time_frequency', 'realm', 'variable', 'ensemble',
'timestamp'
]
self.prefix = 'esgf-'
self.prepath = config.get('project_data')
def storeResults(self, rowid, results):
"""
:type rowid: integer
:param rowid: the row id of the history entry where the results belong to
:type results: dict with entries {str : dict}
:param results: meta-dictionary with meta-data dictionaries assigned to the file names.
"""
reg_ex = None
# regex to get the relative path
preview_path = config.get(config.PREVIEW_PATH, None)
expression = '(%s\\/*){1}(.*)' % re.escape(preview_path)
# only try to create previews, when a preview path is given
if preview_path:
reg_ex = re.compile(expression)
for file_name in results:
metadata = results[file_name]
type_name = metadata.get('type', '')
type_number = hist.Result.Filetype.unknown
preview_path = metadata.get('preview_path', '')
preview_file = ''
if preview_path and reg_ex is not None:
# We store the relative path for previews only.
# Which allows us to move the preview files to a different folder.
preview_file = reg_ex.match(preview_path).group(2)
if type_name == 'plot':
type_number = hist.Result.Filetype.plot
elif type_name == 'data':
type_number = hist.Result.Filetype.data
h = hist.Result(history_id_id=rowid,
output_file=file_name,
preview_file=preview_file,
file_type=type_number)
h.save()
result_id = h.pk
self._storeResultTags(result_id, metadata)
def __init__(self, uid = None, email = None):
'''Creates a user object for the provided id. If no id is given, a user object for
the current user, i.e. the one that started the application, is created instead.
:type uid: int
:param uid: user id in the local system, if not provided the current user is used.
:type email: str
:param email: user's email address
'''
self._dir_type = config.get(config.DIRECTORY_STRUCTURE_TYPE)
if uid is None:
uid = os.getuid()
self._userdata = None
if isinstance(uid, basestring):
self._userdata = pwd.getpwnam(uid)
else:
self._userdata = pwd.getpwuid(uid)
if self._userdata is None:
raise Exception("Cannot find user %s" % uid)
if email is None:
self._email = ''
else:
self._email = email
self._userconfig = Config()
#try to load teh configuration from the very first time.
self._userconfig.read([User.EVAL_SYS_DEFAULT_CONFIG, os.path.join(self._userdata.pw_dir, User.EVAL_SYS_CONFIG)])
self._db = UserDB(self)
row_id = self._db.getUserId(self.getName())
if row_id:
try:
self._db.updateUserLogin(row_id, email)
except:
raise
pass
else:
self._db.createUser(self.getName(), email=self._email)
def set_default_options(self, user, cmdstring, outdir=None):
"""
Sets the default options for a given user and a
given command string.
:param user: an user object
:type user: evaluation_system.model.user.User
:param cmdstring: the command
:type cmdstring: string
"""
# read output directory from configuration
if not outdir:
outdir = user.getUserSchedulerOutputDir()
email = user.getEmail()
# we check if the user is external and activate batch mode
django_user = User.objects.get(username=user.getName())
if django_user.groups.filter(name=config.get(
'external_group', 'noexternalgroupset')).exists():
options = config.get_section('scheduler_options_extern')
elif django_user.groups.filter(name='frevastud').exists():
options = config.get_section('scheduler_options_frevastud')
else:
options = config.get_section('scheduler_options')
self.scheduler_options = options
# set the default options
self.add_dash_option("D", outdir)
if email:
self.add_ddash_option("mail-user", email)
self.set_cmdstring(cmdstring)
self.source_file = options.pop('source', False)
module_file = options.pop('module_command')
self.add_module(module_file)
for opt, val in options.iteritems():
if opt.startswith('option_'):
opt = opt.replace('option_', '')
if val == 'None':
self.add_ddash_option(opt, None)
else:
self.add_ddash_option(opt, val)
def test_preview_generation(self):
d = '/tmp/preview.pdf'
s = os.path.dirname(__file__) + '/test_output/vecap_test_output.pdf'
pm._preview_copy(s, d)
self.assertTrue(os.path.isfile(d))
os.remove(d)
d = '/tmp/preview.png'
s = os.path.dirname(__file__) + '/test_output/test_image.png'
pm._preview_copy(s, d)
self.assertTrue(os.path.isfile(d))
os.remove(d)
d = '/tmp/preview.png'
s = os.path.dirname(__file__) + '/test_output/test_image.png'
pm._preview_convert(s, d)
self.assertTrue(os.path.isfile(d))
os.remove(d)
r = pm._preview_generate_name('dummy', 'old_fn', {})
self.assertIn('dummy', r)
self.assertEqual(len(r), 14)
ts = time.time()
r = pm._preview_generate_name('murcss', 'old_fn', {'timestamp': ts})
self.assertIn('murcss', r)
self.assertIn(
datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d_%H%M%S'), r)
u = pm._preview_unique_file('murcss', 'old_fn', 'pdf',
{'timestamp': ts})
self.assertIn(
datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d_%H%M%S'), u)
self.assertIn('murcss', u)
self.assertIn(config.get('preview_path'), u)
r1 = os.path.dirname(__file__) + '/test_output/vecap_test_output.pdf'
r2 = os.path.dirname(__file__) + '/test_output/test_image.png'
result = {r1: {'todo': 'copy'}, r2: {'todo': 'convert'}}
res = pm._preview_create('murcss', result)
for r in res:
self.assertTrue(os.path.isfile(r))
os.remove(r)
def test_plugin_conf(self):
import tempfile
fd, name = tempfile.mkstemp(__name__, text=True)
with os.fdopen(fd, 'w') as f:
f.write("""
[evaluation_system]
base_dir=~
[plugin:pca]
plugin_path=$EVALUATION_SYSTEM_HOME/tool/pca
python_path=$EVALUATION_SYSTEM_HOME/tool/pca/integration
module=pca.api
[plugin:climval]
plugin_path=$EVALUATION_SYSTEM_HOME/tool/climval
python_path=$EVALUATION_SYSTEM_HOME/tool/climval/src
module=climval.tool
""")
os.environ[config._DEFAULT_ENV_CONFIG_FILE] = name
config.reloadConfiguration()
plugins_dict = config.get(config.PLUGINS)
self.assertEquals(set(plugins_dict), set(['pca', 'climval']))
es_home = '/'.join(__file__.split('/')[:-4])
self.assertEquals(config.get_plugin('pca', config.PLUGIN_PATH),
es_home + '/tool/pca')
self.assertEquals(config.get_plugin('pca', config.PLUGIN_PYTHON_PATH),
es_home + '/tool/pca/integration')
self.assertEquals(config.get_plugin('pca', config.PLUGIN_MODULE),
'pca.api')
self.assertEquals(
config.get_plugin('pca', 'not_existing', 'some_default'),
'some_default')
self.assertEquals(config.get_plugin('climval', config.PLUGIN_MODULE),
'climval.tool')
os.unlink(name)
def _run(self):
# defaults
options = self.args
last_args = self.last_args
# check if tool is specified
try:
tool_name = last_args[0]
except IndexError:
return self.list_tools()
# here we handle the tool pull request
if options.pull_request:
self.handle_pull_request(tool_name)
return 0
if options.repos_version:
(repos, version) = pm.getPluginVersion(tool_name)
print 'Repository and version of %s:\n%s\n%s' % (tool_name, repos, version)
return 0
email = None
unique_output = options.unique_output.lower() if options.unique_output else 'true'
unique_output = unique_output not in ['false', '0', 'no']
mode = options.batchmode.lower() if options.batchmode else 'false'
batchmode = mode in ['true', '1', 'yes', 'on', 'web']
if not batchmode and mode not in ['false', '0', 'no', 'off']:
raise ValueError('batchmode should be set to one of those {1,0, true, false, yes, no, on, off}')
# get the plugin
if tool_name:
caption = None
if options.caption:
caption = pm.generateCaption(options.caption, tool_name)
if options.save_config or options.save:
tool_dict = pm.parseArguments(tool_name, self.last_args[1:])
cfg_file_save = options.save_config
save_in = pm.writeSetup(tool_name, tool_dict, config_file=cfg_file_save)
logging.info("Configuration file saved in %s", save_in)
elif options.show_config:
tool_dict = pm.parseArguments(tool_name, self.last_args[1:])
print pm.getPluginInstance(tool_name).getCurrentConfig(config_dict=tool_dict)
elif options.scheduled_id:
scheduled_id = options.scheduled_id
logging.debug('Running %s as scheduled in history with ID %i', tool_name, scheduled_id)
if not options.dry_run:
pm.runTool(tool_name, scheduled_id=scheduled_id,
unique_output=unique_output)
else:
# now run the tool
(error, warning) = pm.getErrorWarning(tool_name)
if warning:
log.warning(warning)
if error:
log.error(error)
tool_dict = pm.parseArguments(tool_name, self.last_args[1:])
logging.debug('Running %s with configuration: %s', tool_name, tool_dict)
if not options.dry_run and (not error or DEBUG):
# we check if the user is external and activate batchmode
django_user = User.objects.get(username=user.User().getName())
if django_user.groups.filter(name=config.get('external_group', 'noexternalgroupset')).exists():
batchmode = True
if batchmode:
[id, file] = pm.scheduleTool(tool_name,
config_dict=tool_dict,
user=user.User(email=email),
caption=caption,
unique_output=unique_output)
print 'Scheduled job with history id', id
print 'You can view the job\'s status with the command squeue'
print 'Your job\'s progress will be shown with the command'
print 'tail -f ', file
else:
pm.runTool(tool_name, config_dict=tool_dict,
caption=caption, unique_output=unique_output)
# repeat the warning at the end of the run
# for readability don't show the warning in debug mode
if warning and not DEBUG:
log.warning(warning)
if self.DEBUG:
logging.debug("Arguments: %s", self.last_args)
import json
logging.debug('Current configuration:\n%s', json.dumps(tool_dict, indent=4))
from django.conf import settings
from evaluation_system.misc import config
import json,time
import django
SETTINGS = dict()
try:
# Application definition
SETTINGS['INSTALLED_APPS'] = (
'django.contrib.flatpages',
'django.contrib.auth', # We need this to access user groups
)
SETTINGS['DATABASES'] = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': config.get(config.DB_DB),
'USER': config.get(config.DB_USER),
'PASSWORD': config.get(config.DB_PASSWD),
'HOST': config.get(config.DB_HOST), # Or an IP Address that your DB is hosted on
'PORT': '3306',
}
}
settings.configure(**SETTINGS)
django.setup()
except:
pass
def is_admin(self):
if self.__is_admin is None:
admins = config.get('admins', '')
# if user.User().getName() in admins.split(','):
self.__is_admin = user.User().getName() in admins.split(',')
return self.__is_admin
def reloadPlugins(user_name=None):
"""Reload all plug-ins. Plug-ins are then loaded first from the :class:`PLUGIN_ENV` environmental
variable and then from the configuration file. This means that the environmental variable has precedence
and can therefore overwrite existing plug-ins (useful for debugging and testing)."""
if not user_name:
user_name = User().getName()
# reset all current plugins
# for item in __plugins_meta.keys():
# __plugins_meta.pop(item)
# for item in __plugin_modules__.keys():
# __plugin_modules__.pop(item)
# for item in __plugins__.keys():
# __plugins__.pop(item)
__plugin_modules__ = py27.OrderedDict() # we use a ordered dict. This allows to override plugins
__plugins__ = {}
__plugins_meta = {}
__plugin_modules_user__[user_name] = py27.OrderedDict()
__plugins_user__[user_name] = py27.OrderedDict()
__plugins_meta_user[user_name] = py27.OrderedDict()
extra_plugins = list()
if PLUGIN_ENV in os.environ:
# now get all modules loaded from the environment
for path, module_name in map(lambda item: tuple([e.strip() for e in item.split(',')]),
os.environ[PLUGIN_ENV].split(':')):
# extend path to be exact by resolving all "user shortcuts" (e.g. '~' or '$HOME')
path = os.path.abspath(os.path.expandvars(os.path.expanduser(path)))
if os.path.isdir(path):
# we have a plugin_imp with defined api
sys.path.append(path)
# TODO this is not working like in the previous loop. Though we might just want to remove it,
# as there seem to be no use for this info...
__plugin_modules__[module_name] = os.path.join(path, module_name)
extra_plugins.append(module_name)
else:
log.warn("Cannot load %s, directory missing: %s", module_name, path)
# the same for user specific env variable
if user_name:
if PLUGIN_ENV+'_'+user_name in os.environ:
# now get all modules loaded from the environment
for path, module_name in map(lambda item: tuple([e.strip() for e in item.split(',')]),
os.environ[PLUGIN_ENV+'_'+user_name].split(':')):
# extend path to be exact by resolving all "user shortcuts" (e.g. '~' or '$HOME')
path = os.path.abspath(os.path.expandvars(os.path.expanduser(path)))
if os.path.isdir(path):
# we have a plugin_imp with defined api
sys.path.append(path)
# TODO this is not working like in the previous loop. Though we might just want to remove it,
# as there seem to be no use for this info...
__plugin_modules__[module_name] = os.path.join(path, module_name)
extra_plugins.append(module_name)
else:
log.warn("Cannot load %s, directory missing: %s", module_name, path)
# get the tools directory from the current one
# get all modules from the tool directory
plugins = list(config.get(config.PLUGINS))
for plugin_name in plugins:
py_dir = config.get_plugin(plugin_name, config.PLUGIN_PYTHON_PATH)
py_mod = config.get_plugin(plugin_name, config.PLUGIN_MODULE)
if os.path.isdir(py_dir):
if py_mod in __plugin_modules__:
file_path = __plugin_modules__[py_mod]+'.py'
log.warn("Module '%s' is test being overwritten by: %s", py_mod, file_path)
else:
log.debug("Loading '%s'", plugin_name)
sys.path.append(py_dir)
__plugin_modules__[plugin_name] = os.path.join(py_dir, py_mod)
else:
log.warn("Cannot load '%s' directory missing: %s", plugin_name, py_dir)
# new way of loading plugins
import re
reg = re.compile(r'__short_description__\s*=(.*)')
r = re.compile(r'\'(.*)\'')
r_2 = re.compile(r'\"(.*)\"')
r_list = re.compile(r'\[(.*)\]')
reg_class_name = re.compile(r'class\s*(.*)')
# reg for categories
cat_reg = re.compile(r'__category__\s*=(.*)')
# reg for tags
tag_reg = re.compile(r'__tags__\s*=(.*)')
for plugin_name, plugin_mod in __plugin_modules__.iteritems():
f = open(plugin_mod+'.py', 'r')
description = None
class_name = None
category = None
tags = None
class_name_str = ''
for line in f:
description = re.search(reg, line)
if description is not None:
description_str = re.search(r, description.groups()[0])
if description_str is None:
description_str = re.search(r_2, description.groups()[0])
if description_str is not None:
description_str = description_str.groups()[0]
# search for category
category_search = re.search(cat_reg, line)
if category_search:
category = re.search(r, category_search.groups()[0])
category = category.groups()[0]
# search for tags
tags_search = re.search(tag_reg, line)
if tags_search:
tags = re.search(r_list, tags_search.groups()[0])
tags = list(eval(tags.groups()[0]))
# search for classname
class_name = re.search(reg_class_name, line)
if class_name is not None:
# TODO: Maybe this is not robust enough.
# What if class inherits from other Base Class?
if 'PluginAbstract' in class_name.groups()[0]:
class_name_str = re.sub(r'\(.*', '', class_name.groups()[0])
if class_name_str != '' and class_name_str.lower() not in __plugins_meta.keys():
__plugins_meta[class_name_str.lower()] = dict(name=class_name_str,
plugin_class=class_name_str,
plugin_module=plugin_mod,
description=description_str,
user_exported=plugin_name in extra_plugins,
category=category,
tags=tags)
__plugins__[class_name_str] = class_name_str
elif class_name_str != '':
log.warn("Default plugin %s is being overwritten by: %s",
class_name_str, __plugins_meta[class_name_str.lower()]['plugin_module']+'.py')
sys.path = [p for p in munge(sys.path)]
__plugin_modules_user__[user_name] = __plugin_modules__
__plugins_user__[user_name] = __plugins__
__plugins_meta_user[user_name] = __plugins_meta
def scheduleTool(plugin_name, slurmoutdir=None, config_dict=None, user=None,
caption=None, unique_output=True):
"""Schedules a tool and stores this "run" in the :class:`evaluation_system.model.db.UserDB`.
:type plugin_name: str
:param plugin_name: name of the referred plugin.
:type slurmoutdir: string
:param slurmoutdir: directory for the output
:type config_dict: dict or metadict
:param config_dict: The configuration used for running the tool. If is None, the default configuration will be stored,
this might be incomplete.
:type user: :class:`evaluation_system.model.user.User`
:param user: The user starting the tool
:type scheduled_id: int
:param scheduled_id: if the process is already scheduled then put the row id here
:type caption: str
:param caption: the caption to set.
"""
plugin_name = plugin_name.lower()
if user is None:
user = User()
p = getPluginInstance(plugin_name, user)
complete_conf = None
# check whether a scheduled id is given
if config_dict is None:
conf_file = user.getUserToolConfig(plugin_name)
if os.path.isfile(conf_file):
log.debug('Loading config file %s', conf_file)
with open(conf_file, 'r') as f:
complete_conf = p.readConfiguration(f)
else:
log.debug('No config file was found in %s', conf_file)
if complete_conf is None:
# at this stage we want to resolve or tokens and perform some kind of sanity check before going further
complete_conf = p.setupConfiguration(config_dict=config_dict, recursion=True)
log.debug('Schedule %s with %s', plugin_name, complete_conf)
slurmindir = os.path.join(user.getUserSchedulerInputDir(), user.getName())
if not os.path.exists(slurmindir):
utils.supermakedirs(slurmindir, 0777)
version_details = getVersion(plugin_name)
rowid = user.getUserDB().storeHistory(p,
complete_conf,
user.getName(),
History.processStatus.not_scheduled,
version_details=version_details,
caption=caption)
# follow the notes
followHistoryTag(rowid, user.getName(), 'Owner')
# set the SLURM output directory
if not slurmoutdir:
slurmoutdir = user.getUserSchedulerOutputDir()
slurmoutdir = os.path.join(slurmoutdir, plugin_name)
if not os.path.exists(slurmoutdir):
utils.supermakedirs(slurmoutdir, 0777)
# write the SLURM file
full_path = os.path.join(slurmindir, p.suggestSlurmFileName())
with open(full_path, 'w') as fp:
p.writeSlurmFile(fp,
scheduled_id=rowid,
user=user,
slurmoutdir=slurmoutdir,
unique_output=unique_output)
# create the batch command
command = ['/bin/bash',
'-c',
'%s %s %s\n' % (config.get('scheduler_command'), # SCHEDULER_COMMAND,
config.SCHEDULER_OPTIONS,
#user.getName(),
full_path)]
# run this
logging.debug("Command: " + str(command))
p = Popen(command, stdout=PIPE, stderr=STDOUT)
(stdout, stderr) = p.communicate()
logging.debug("scheduler call output:\n" + str(stdout))
logging.debug("scheduler call error:\n" + str(stderr))
# get the very first line only
out_first_line = stdout.split('\n')[0]
# read the id from stdout
if out_first_line.split(' ')[0] == 'Submitted':
slurm_id = int(out_first_line.split(' ')[-1])
else:
slurm_id = 0
raise Exception('Unexpected scheduler output:\n%s' % out_first_line)
slurm_out = os.path.join(slurmoutdir,
'slurm-%i.out' % slurm_id)
# create a standard slurm file to view with less
with open(slurm_out, 'w') as the_file:
the_file.write('Certainly, your job is pending with id %i.\n' % slurm_id)
the_file.write('You can get further information using the command squeue.\n')
the_file.write('\nThis file was automatically created by the evaluation system.\n')
the_file.write('It will be overwritten by the output of %s.\n' % plugin_name)
# set the slurm output file
user.getUserDB().scheduleEntry(rowid, user.getName(), slurm_out)
return rowid, slurm_out
