def test_filemanager_populate_no_sta(self):
"""
run filemanager set and populate with sta turned off
"""
"""
############### SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 1
simend = 10
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
mutex = threading.Lock()
"""
############## TEST ###############
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
simlength = simend - simstart + 1
for _type in ['atm', 'lnd', 'ocn', 'ice']:
file_names = [
x.name
for x in DataFile.select().where(DataFile.datatype == _type)
]
if not len(file_names) == (simlength * 12):
print _type + ' does not have ' + str(
simlength * 12) + ' files'
self.assertEqual(len(file_names), (simlength * 12))
for year in range(simstart, simend + 1):
for month in range(1, 13):
name = (file_type_map[_type].replace(
'EXPERIMENT',
experiment).replace('YEAR',
'{:04d}'.format(year)).replace(
'MONTH',
'{:02}'.format(month)))
self.assertTrue(name in file_names)
filemanager.mutex.release()
"""
############## CLEANUP ###############
"""
os.remove(database)
def init_run(self, show_progress=False, chunkcount=1):
self.setlanguages()
self.gatherer = Gatherer(self.monitor, self.autolanguages)
self.downloader = FileManager(self.debug, chunkcount > 1)
self.freshstart = str(datetime_now() - timedelta(days=365))
self.chunkcount = chunkcount
self.currentchunk = 0
if get_kodi_version() >= 18:
populate_musiccentraldir()
if show_progress:
self.create_progress()
def cache_artwork(librarytype='videos'):
fileman = FileManager(False, True)
if not fileman.imagecachebase:
xbmcgui.Dialog().notification("Artwork Beef",
L(M.REMOTE_CONTROL_REQUIRED),
xbmcgui.NOTIFICATION_WARNING)
return
heading = L(M.CACHE_VIDEO_ARTWORK if librarytype ==
'videos' else M.CACHE_MUSIC_ARTWORK)
cached = runon_medialist(lambda mi: fileman.cachefor(mi.art, True),
heading,
librarytype,
fg=True)
xbmcgui.Dialog().ok("Artwork Beef", L(M.CACHED_COUNT).format(cached))
def make_local():
fileman = FileManager()
def downloadforitem(mediaitem):
try:
fileman.downloadfor(mediaitem)
newart = dict((k, v)
for k, v in mediaitem.downloadedart.iteritems()
if not v or not v.startswith('http'))
for arttype in newart:
# remove old URL from texture cache
if mediaitem.art.get(arttype, '').startswith('http'):
quickjson.remove_texture_byurl(mediaitem.art[arttype])
return newart
except FileError as ex:
mediaitem.error = ex.message
log(ex.message, xbmc.LOGERROR)
xbmcgui.Dialog().notification("Artwork Beef", ex.message,
xbmcgui.NOTIFICATION_ERROR)
return {}
downloaded = runon_medialist(downloadforitem, L(M.MAKE_LOCAL), fg=True)
xbmcgui.Dialog().ok(
"Artwork Beef",
L(M.DOWNLOAD_COUNT).format(downloaded) +
' - {0:0.2f}MB'.format(fileman.size / 1000000.00))
def test_filemanager_setup_no_sta(self):
"""
run filemansger setup with no sta
"""
"""
############## SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
"""
############### TEST ################
"""
filemanager = FileManager(mutex=mutex,
event_list=EventList(),
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(database))
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_runmanager_setup(self):
"""
Run the runmanager setup
"""
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
db_path = os.path.join(
self.project_path,
'{}.db'.format(inspect.stack()[0][3]))
if os.path.exists(self.project_path):
shutil.rmtree(self.project_path)
os.makedirs(self.project_path)
filemanager = FileManager(
ui=False,
event_list=EventList(),
database=db_path,
types=['atm'],
sta=False,
mutex=self.mutex,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.project_path,
experiment=self.experiment)
runmanager = RunManager(
short_name='testname',
account='',
resource_path='./resources',
ui=False,
event_list=EventList(),
output_path=self.output_path,
caseID=self.experiment,
scripts_path=self.run_scripts_path,
thread_list=[],
event=threading.Event(),
no_host=True,
url_prefix='',
always_copy=False)
runmanager.setup_job_sets(
set_frequency=[5, 10],
sim_start_year=int(self.config['global']['simulation_start_year']),
sim_end_year=int(self.config['global']['simulation_end_year']),
config=self.config,
filemanager=filemanager)
self.assertEqual(len(runmanager.job_sets), 1)
for job_set in runmanager.job_sets:
if job_set.set_number == 1:
self.assertEqual(job_set.length, 5)
self.assertEqual(job_set.set_start_year, 1)
self.assertEqual(job_set.set_end_year, 5)
job_names = job_set.get_job_names()
self.assertTrue('ncclimo' in job_names)
self.assertTrue('timeseries' in job_names)
self.assertTrue('aprime_diags' in job_names)
self.assertTrue('e3sm_diags' in job_names)
def test_filemanager_update_local(self):
"""
run filemanager set and populate, then create a dummy file in the
input directory and run update_local which should mark it as present
"""
"""
############# SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean']
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
simend = 60
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
"""
############### TEST #################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
df = DataFile.select().limit(1)
filemanager.mutex.release()
name = df[0].name
head, tail = os.path.split(df[0].local_path)
if not os.path.exists(head):
os.makedirs(head)
dummy_file_path = df[0].local_path
print '----- writing out dummy file at {} -----'.format(
dummy_file_path)
with open(dummy_file_path, 'w') as fp:
fp.write('this is a test file')
filemanager.update_local_status()
filemanager.mutex.acquire()
df = DataFile.select().where(DataFile.name == name)[0]
filemanager.mutex.release()
self.assertEqual(df.local_status, 0)
self.assertTrue(df.local_size > 0)
"""
############### CLEANUP #################
"""
os.remove(database)
def test_runmanager_write(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
db_path = os.path.join(
self.project_path,
'{}.db'.format(inspect.stack()[0][3]))
if os.path.exists(db_path):
os.remove(db_path)
local_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
database = '{}.db'.format(inspect.stack()[0][3])
local_path = os.path.join(
self.project_path,
'input')
types = ['atm']
mutex = threading.Lock()
filemanager = FileManager(
mutex=mutex,
event_list=EventList(),
sta=False,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=local_path,
experiment=self.experiment)
self.assertTrue(isinstance(filemanager, FileManager))
runmanager = RunManager(
short_name='testname',
account='',
resource_path='./resources',
ui=False,
event_list=EventList(),
output_path=self.output_path,
caseID=self.experiment,
scripts_path=self.run_scripts_path,
thread_list=[],
event=threading.Event(),
no_host=True,
url_prefix='',
always_copy=False)
runmanager.setup_job_sets(
set_frequency=[5, 10],
sim_start_year=int(self.config['global']['simulation_start_year']),
sim_end_year=int(self.config['global']['simulation_end_year']),
config=self.config,
filemanager=filemanager)
path = os.path.join(self.project_path, 'output', 'job_state.txt')
runmanager.write_job_sets(path)
self.assertTrue(os.path.exists(path))
os.remove(database)
def test_filemanager_update_remote_yes_sta(self):
"""
run filemanager setup and populate, then run update_remote_status on a directory
that has been short term archived
"""
"""
############### SETUP #################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = True
types = [
'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice',
'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport'
]
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
source_path = '/global/cscratch1/sd/golaz/ACME_simulations/20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
simend = 60
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
mutex = threading.Lock()
"""
############### TEST #################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=source_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
filemanager.mutex.acquire()
for datafile in DataFile.select():
if datafile.remote_status != 0:
print datafile.name, datafile.remote_path
self.assertEqual(datafile.remote_status, 0)
if filemanager.mutex.locked():
filemanager.mutex.release()
self.assertTrue(filemanager.all_data_remote())
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_filemanager_update_remote_no_sta(self):
"""
run filemanager setup and populate, then run update_remote_status
with 10 years of atm output, and finally run all_data_remote to show that
all the remote data has been recognized
"""
"""
############# SETUP ##################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
sta = False
types = ['atm']
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
simend = 60
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
"""
################ TEST ##################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=False,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
filemanager.mutex.acquire()
for datafile in DataFile.select():
if datafile.remote_status != 0:
print datafile.name, datafile.remote_path, datafile.remote_status, datafile.datatype
self.assertEqual(datafile.remote_status, 0)
if filemanager.mutex.locked():
filemanager.mutex.release()
self.assertTrue(filemanager.all_data_remote())
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_cmor_valid_completed(self):
"""
tests that a valid config on a completed case will mark itself as
already being run and not start
"""
print_message(
'\n---- Starting Test: {} ----'.format(inspect.stack()[0][3]),
'ok')
config = ConfigObj(self.valid_config_path)
config['post-processing']['cmor']['variable_list'] = [
config['post-processing']['cmor']['variable_list']
]
case_name = '20180129.DECKv1b_piControl.ne30_oEC.edison'
case = config['simulations'][case_name]
messages = verify_config(config)
self.assertEqual(len(messages), 0)
config['global']['resource_path'] = 'resources/'
filemanager = FileManager(config=config, event_list=EventList())
filemanager.populate_file_list()
filemanager.update_local_status()
timeseries = Timeseries(short_name=case['short_name'],
case=case_name,
start=config['simulations']['start_year'],
end=config['simulations']['end_year'],
config=config,
run_type='atm')
timeseries.check_data_ready(filemanager=filemanager)
timeseries.setup_data(config=config,
filemanager=filemanager,
case=case_name)
timeseries.execute(config=config, event_list=EventList())
timeseries.handle_completion(filemanager=filemanager,
config=config,
event_list=EventList())
cmor = Cmor(short_name=case['short_name'],
case=case_name,
start=config['simulations']['start_year'],
end=config['simulations']['end_year'],
config=config)
cmor.check_data_ready(filemanager=filemanager)
cmor.setup_data(config=config, filemanager=filemanager, case=case_name)
self.assertTrue(cmor.postvalidate(config=config))
self.assertTrue(cmor.execute(config=config, event_list=EventList()))
self.assertEquals(cmor.status, JobStatus.COMPLETED)
self.assertTrue(
cmor.handle_completion(filemanager=filemanager,
event_list=EventList(),
config=config))
def remove_art():
# TODO: seasons and episodes and whatever like "add missing artwork" does
listitem = sys.listitem
mediatype = get_mediatype(listitem)
dbid = get_dbid(listitem)
if not (dbid or mediatype):
return
if not xbmcgui.Dialog().yesno("Artwork Beef: " + L(32427), L(750)):
return
remove_localfiles = xbmcgui.Dialog().yesno("Artwork Beef", L(32062))
mediaitem = info.MediaItem(quickjson.get_item_details(dbid, mediatype))
mediaitem.selectedart = cleaner.remove_specific_arttype(mediaitem, '* all')
if remove_localfiles:
FileManager().remove_deselected_files(mediaitem, True)
info.update_art_in_library(mediatype, dbid, mediaitem.selectedart)
info.remove_local_from_texturecache(mediaitem.art.values(), True)
xbmcgui.Dialog().notification("Artwork Beef",
L(32027).format(len(mediaitem.selectedart)))
def test_filemanager_setup_valid_from_scratch(self):
"""
run filemansger setup with no sta
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
db = '{}.db'.format(inspect.stack()[0][3])
config_path = 'tests/test_configs/valid_config_from_scratch.cfg'
config = ConfigObj(config_path)
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
filemanager = FileManager(database=db,
event_list=EventList(),
config=config)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(db))
os.remove(db)
def test_filemanager_setup_valid_with_inplace_data(self):
"""
run the filemanager setup with sta turned on
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/e3sm_diags_complete.cfg'
config = ConfigObj(config_path)
db = '{}.db'.format(inspect.stack()[0][3])
filemanager = FileManager(database=db,
event_list=EventList(),
config=config)
filemanager.populate_file_list()
filemanager.update_local_status()
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(db))
self.assertTrue(filemanager.all_data_local())
os.remove(db)
def test_filemanager_setup_with_sta(self):
"""
run the filemanager setup with sta turned on
"""
"""
############## SETUP ###############
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = True
types = [
'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice',
'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport', 'lnd'
]
database = '{}.db'.format(inspect.stack()[0][3])
mutex = threading.Lock()
"""
############## TEST ###############
"""
filemanager = FileManager(mutex=mutex,
event_list=EventList(),
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(database))
"""
############## CLEANUP ###############
"""
os.remove(database)
import yaml, json
from lib.filemanager import FileManager
from lib.polling import PollingManager
from lib.ip_webcam import IPWebcam
from lib.config import ConfigManager
from utils.surveillance_web_server import set_filemanager, run
## test
if __name__ == "__main__":
cm = ConfigManager()
configs = cm.get_configs()
fm = FileManager(configs["record_path"])
fm.set_max_days_age(1)
fm.start_rotate_thread()
set_filemanager(fm)
for stream in config["streams"]:
host, port = stream["source"].split(":")
cam = IPWebcam(host, port)
pm = PollingManager()
pm.set_device(cam)
pm.set_filemanager(fm)
pm.set_prefix(stream["name"])
pm.start_polling()
run()
def test_filemanager_get_file_paths(self):
"""
run the filemanager setup with sta turned on
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/filemanager_partial_data.cfg'
config = ConfigObj(config_path)
db = '{}.db'.format(inspect.stack()[0][3])
filemanager = FileManager(database=db,
event_list=EventList(),
config=config)
filemanager.populate_file_list()
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(db))
filemanager.update_local_status()
filemanager.write_database()
self.assertFalse(filemanager.all_data_local())
# test that the filemanager returns correct paths
paths = filemanager.get_file_paths_by_year(
datatype='atm',
case='20180129.DECKv1b_piControl.ne30_oEC.edison',
start_year=1,
end_year=2)
for path in paths:
self.assertTrue(os.path.exists(path))
# test that the filemanager returns correct paths with no year
paths = filemanager.get_file_paths_by_year(
datatype='ocn_streams',
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
for path in paths:
self.assertTrue(os.path.exists(path))
# test nothing is returned for incorrect yeras
paths = filemanager.get_file_paths_by_year(
datatype='ocn_streams',
case='20180129.DECKv1b_piControl.ne30_oEC.edison',
start_year=1,
end_year=100)
self.assertTrue(paths is None)
# test the filemanager knows when data is ready
ready = filemanager.check_data_ready(
data_required=['atm'],
case='20180129.DECKv1b_piControl.ne30_oEC.edison',
start_year=1,
end_year=2)
self.assertTrue(ready)
# test the filemanager knows when data is NOT ready
ready = filemanager.check_data_ready(
data_required=['atm'],
case='20180129.DECKv1b_piControl.ne30_oEC.edison',
start_year=1,
end_year=3)
self.assertFalse(ready)
ready = filemanager.check_data_ready(
data_required=['ocn_streams'],
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
self.assertTrue(ready)
os.remove(db)
import json
import yaml
import threading
from lib.surveillance import Surveillance
from lib.filemanager import FileManager
from lib.config import ConfigManager
from utils.cam_web_server import run, set_frame_func
if __name__ == "__main__":
cm = ConfigManager()
configs = cm.get_configs()
source = 0
fm = FileManager("/tmp/cworkspace2")
fm.set_max_days_age(1)
sv = Surveillance(source)
sv.set_filemanager(fm)
sv.set_prefix("test")
sv.set_remote_server(None)
set_frame_func(sv.get_frame_out)
threading.Thread(target=run).start()
sv.start()
class ArtworkProcessor(object):
def __init__(self, monitor=None):
self.monitor = monitor or xbmc.Monitor()
self.language = None
self.autolanguages = None
self.progress = xbmcgui.DialogProgressBG()
self.visible = False
self.freshstart = "0"
self.processed = ProcessedItems()
self.gatherer = None
self.downloader = None
self.chunkcount = 1
self.currentchunk = 0
self.debug = False
self.localmode = False
settings.update_settings()
mediatypes.update_settings()
def create_progress(self):
if not self.visible and settings.progressdisplay == PROGRESS_DISPLAY_FULLPROGRESS:
self.progress.create("Artwork Beef: " + L(ADDING_ARTWORK_MESSAGE),
"")
self.visible = True
def update_progress(self, percent, message, heading=None):
if self.chunkcount > 1:
onechunkp = 100 / (self.chunkcount * 1.0)
percent = int(onechunkp * self.currentchunk + percent /
(self.chunkcount * 1.0))
if self.visible and settings.progressdisplay == PROGRESS_DISPLAY_FULLPROGRESS:
self.progress.update(percent, heading, message)
def finalupdate(self, header, message):
if settings.final_notification:
xbmcgui.Dialog().notification("Artwork Beef: " + header, message,
'-', 8000)
elif settings.progressdisplay == PROGRESS_DISPLAY_FULLPROGRESS:
self.update_progress(100, message, header)
try:
self.monitor.really_waitforabort(8)
except AttributeError:
self.monitor.waitForAbort(8)
def close_progress(self):
if self.visible and settings.progressdisplay == PROGRESS_DISPLAY_FULLPROGRESS:
self.progress.close()
self.visible = False
def notify_warning(self, message, header=None, error=False):
if settings.progressdisplay != PROGRESS_DISPLAY_NONE:
header = "Artwork Beef: " + header if header else "Artwork Beef"
xbmcgui.Dialog().notification(
header, message, xbmcgui.NOTIFICATION_ERROR
if error else xbmcgui.NOTIFICATION_WARNING)
def init_run(self, show_progress=False, chunkcount=1):
self.setlanguages()
self.gatherer = Gatherer(self.monitor, self.autolanguages)
self.downloader = FileManager(self.debug, chunkcount > 1)
self.freshstart = str(datetime_now() - timedelta(days=365))
self.chunkcount = chunkcount
self.currentchunk = 0
if get_kodi_version() >= 18:
populate_musiccentraldir()
if show_progress:
self.create_progress()
def set_debug(self, debug):
if self.downloader:
self.downloader.debug = debug
reporting.debug = debug
self.debug = debug
def finish_run(self):
info.clear_cache()
self.downloader = None
self.set_debug(False)
self.close_progress()
@property
def processor_busy(self):
# DEPRECATED: StringCompare is deprecated in Krypton, gone in Leia
return pykodi.get_conditional(
'![StringCompare(Window(Home).Property(ArtworkBeef.Status),idle) | String.IsEqual(Window(Home).Property(ArtworkBeef.Status),idle)]'
)
def process_item(self, mediatype, dbid, mode):
if self.processor_busy:
return
if mode == MODE_DEBUG:
mode = MODE_AUTO
self.set_debug(True)
if mode == MODE_GUI:
busy = pykodi.get_busydialog()
busy.create()
if mediatype in mediatypes.artinfo and (mediatype
not in mediatypes.audiotypes
or get_kodi_version() >= 18):
mediaitem = info.MediaItem(
quickjson.get_item_details(dbid, mediatype))
log("Processing {0} '{1}' {2}.".format(
mediatype, mediaitem.label,
'automatically' if mode == MODE_AUTO else 'manually'))
else:
if mode == MODE_GUI:
busy.close()
xbmcgui.Dialog().notification(
"Artwork Beef",
L(NOT_SUPPORTED_MESSAGE).format(mediatype), '-', 6500)
return
self.init_run()
if mediatype == mediatypes.EPISODE:
series = quickjson.get_item_details(mediaitem.tvshowid,
mediatypes.TVSHOW)
if not any(uniqueid in settings.autoadd_episodes
for uniqueid in series['uniqueid'].itervalues()):
mediaitem.skip_artwork = ['fanart']
info.add_additional_iteminfo(mediaitem, self.processed, search)
if not mediaitem.uniqueids and not mediatypes.only_filesystem(
mediaitem.mediatype):
if mediatype in mediatypes.require_manualid:
self.manual_id(mediaitem)
if mode == MODE_GUI:
self._manual_item_process(mediaitem, busy)
else:
medialist = [mediaitem]
if mediatype == mediatypes.TVSHOW and not mediatypes.disabled(
mediatypes.EPISODE):
gen_epthumb = mediatypes.generatethumb(mediatypes.EPISODE)
download_ep = mediatypes.downloadanyartwork(mediatypes.EPISODE)
if mediaitem.uniqueids and any(
x in mediaitem.uniqueids.values()
for x in settings.autoadd_episodes):
medialist.extend(
info.MediaItem(ep)
for ep in quickjson.get_episodes(dbid))
elif gen_epthumb or download_ep:
for episode in quickjson.get_episodes(dbid):
if gen_epthumb and not info.has_generated_thumbnail(episode) \
or download_ep and info.has_art_todownload(episode['art'], mediatypes.EPISODE):
episode = info.MediaItem(episode)
episode.skip_artwork = ['fanart']
medialist.append(episode)
elif mediatype == mediatypes.ARTIST and not mediatypes.disabled(
mediatypes.ALBUM):
medialist.extend(
info.MediaItem(album) for album in quickjson.get_albums(
mediaitem.label, mediaitem.dbid))
if mediatype in (mediatypes.ALBUM, mediatypes.ARTIST) and not mediatypes.disabled(mediatypes.ALBUM) \
and not mediatypes.disabled(mediatypes.SONG):
medialist.extend(
info.MediaItem(song) for song in quickjson.get_songs(
mediaitem.mediatype, mediaitem.dbid))
self.process_medialist(medialist, True)
def _manual_item_process(self, mediaitem, busy):
self._process_item(mediaitem, True, False)
busy.close()
if mediaitem.availableart or mediaitem.forcedart:
availableart = dict(mediaitem.availableart)
if mediaitem.mediatype == mediatypes.TVSHOW and 'fanart' in availableart:
# add unseasoned backdrops as manual-only options for each season fanart
unseasoned_backdrops = [
dict(art) for art in availableart['fanart']
if not art.get('hasseason')
]
if unseasoned_backdrops:
for season in mediaitem.seasons.keys():
key = 'season.{0}.fanart'.format(season)
if key in availableart:
availableart[key].extend(unseasoned_backdrops)
else:
availableart[key] = list(unseasoned_backdrops)
if mediaitem.mediatype in (mediatypes.MOVIE, mediatypes.MOVIESET
) and 'poster' in availableart:
# add no-language posters from TMDB as manual-only options for 'keyart'
nolang_posters = [
dict(art) for art in availableart['poster']
if not art['language']
]
for art in nolang_posters:
if art['provider'].sort == 'themoviedb.org':
if 'keyart' not in availableart:
availableart['keyart'] = []
availableart['keyart'].append(art)
tag_forcedandexisting_art(availableart, mediaitem.forcedart,
mediaitem.art)
selectedarttype, selectedart = prompt_for_artwork(
mediaitem.mediatype, mediaitem.label, availableart,
self.monitor)
if selectedarttype and selectedarttype not in availableart:
self.manual_id(mediaitem)
return
if selectedarttype and selectedart:
if mediatypes.get_artinfo(mediaitem.mediatype,
selectedarttype)['multiselect']:
selectedart = info.fill_multiart(mediaitem.art,
selectedarttype,
selectedart)
else:
selectedart = {selectedarttype: selectedart}
selectedart = get_simpledict_updates(mediaitem.art,
selectedart)
mediaitem.selectedart = selectedart
toset = dict(selectedart)
if settings.remove_deselected_files:
self.downloader.remove_deselected_files(mediaitem)
if mediatypes.downloadanyartwork(mediaitem.mediatype):
try:
self.downloader.downloadfor(mediaitem, False)
except FileError as ex:
mediaitem.error = ex.message
log(ex.message, xbmc.LOGERROR)
xbmcgui.Dialog().notification(
"Artwork Beef", ex.message,
xbmcgui.NOTIFICATION_ERROR)
toset.update(mediaitem.downloadedart)
if toset:
mediaitem.updatedart = toset.keys()
add_art_to_library(mediaitem.mediatype, mediaitem.seasons,
mediaitem.dbid, toset)
self.cachelocal(mediaitem, toset)
reporting.report_item(mediaitem, True, True,
self.downloader.size)
if not mediaitem.error:
notifycount(len(toset))
else:
xbmcgui.Dialog().notification(
L(NOT_AVAILABLE_MESSAGE),
L(SOMETHING_MISSING) + ' ' + L(FINAL_MESSAGE), '-', 8000)
self.finish_run()
def process_medialist(self, medialist, alwaysnotify=False):
return self.process_chunkedlist([medialist], 1, alwaysnotify)
def process_chunkedlist(self, chunkedlist, chunkcount, alwaysnotify=False):
self.init_run(True, chunkcount)
aborted = False
artcount = 0
for idx, medialist in enumerate(chunkedlist):
self.currentchunk = idx
if not idx and chunkcount == 1 and len(medialist) > 100:
self.downloader.set_bigcache()
if self.monitor.abortRequested() or medialist is False:
aborted = True
break
reporting.report_start(medialist)
this_aborted, updateditemcount, this_artcount = \
self._process_chunk(medialist, len(medialist) == 1 and not idx, alwaysnotify)
artcount += this_artcount
reporting.report_end(medialist,
updateditemcount if this_aborted else 0,
self.downloader.size)
self.downloader.size = 0
if this_aborted:
aborted = True
break
if artcount or alwaysnotify:
self.finalupdate(*finalmessages(artcount))
self.finish_run()
return not aborted
def _process_chunk(self, medialist, singleitemlist, singleitem):
artcount = 0
currentitem = 0
aborted = False
for mediaitem in medialist:
if is_excluded(mediaitem):
continue
if mediaitem.mediatype in mediatypes.audiotypes and get_kodi_version(
) < 18:
continue
self.update_progress(currentitem * 100 // len(medialist),
mediaitem.label)
info.add_additional_iteminfo(mediaitem, self.processed,
None if self.localmode else search)
currentitem += 1
try:
services_hit = self._process_item(mediaitem, singleitem)
except JSONException as ex:
mediaitem.error = "Kodi threw a non-descript JSON error."
log("Kodi threw a non-descript JSON error.", xbmc.LOGERROR)
log(ex.message, xbmc.LOGERROR)
services_hit = True
except FileError as ex:
services_hit = True
mediaitem.error = ex.message
log(ex.message, xbmc.LOGERROR)
self.notify_warning(ex.message, None, True)
reporting.report_item(mediaitem, singleitemlist or mediaitem.error)
artcount += len(mediaitem.updatedart)
if not services_hit:
if self.monitor.abortRequested():
aborted = True
break
elif self.monitor.waitForAbort(THROTTLE_TIME):
aborted = True
break
return aborted, currentitem, artcount
def _process_item(self, mediaitem, singleitem=False, auto=True):
log("Processing {0} '{1}' automatically.".format(
mediaitem.mediatype, mediaitem.label))
mediatype = mediaitem.mediatype
onlyfs = self.localmode or mediatypes.only_filesystem(
mediaitem.mediatype)
if not mediaitem.uniqueids and not onlyfs:
mediaitem.missingid = True
if singleitem:
header = L(NO_IDS_MESSAGE)
message = "{0} '{1}'".format(mediatype, mediaitem.label)
log(header + ": " + message, xbmc.LOGNOTICE)
xbmcgui.Dialog().notification("Artwork Beef: " + header,
message,
xbmcgui.NOTIFICATION_INFO)
if auto:
cleaned = get_simpledict_updates(mediaitem.art,
cleaner.clean_artwork(mediaitem))
if cleaned:
if not self.debug:
add_art_to_library(mediatype, mediaitem.seasons,
mediaitem.dbid, cleaned)
mediaitem.art.update(cleaned)
mediaitem.art = dict(item
for item in mediaitem.art.iteritems()
if item[1])
mediaitem.missingart = list(
info.iter_missing_arttypes(mediaitem, mediaitem.art))
services_hit, error = self.gatherer.getartwork(mediaitem, onlyfs, auto)
if auto:
existingart = dict(mediaitem.art)
selectedart = dict(
(key, image['url'])
for key, image in mediaitem.forcedart.iteritems())
existingart.update(selectedart)
# Then add the rest of the missing art
selectedart.update(
self.get_top_missing_art(
info.iter_missing_arttypes(mediaitem, existingart),
mediatype, existingart, mediaitem.availableart))
selectedart = get_simpledict_updates(mediaitem.art, selectedart)
mediaitem.selectedart = selectedart
toset = dict(selectedart)
if not self.localmode and mediatypes.downloadanyartwork(
mediaitem.mediatype):
sh, er = self.downloader.downloadfor(mediaitem)
services_hit = services_hit or sh
error = error or er
toset.update(mediaitem.downloadedart)
if toset:
mediaitem.updatedart = list(
set(mediaitem.updatedart + toset.keys()))
if not self.debug:
add_art_to_library(mediatype, mediaitem.seasons,
mediaitem.dbid, toset)
self.cachelocal(mediaitem, toset)
if error:
if isinstance(error, dict):
header = L(PROVIDER_ERROR_MESSAGE).format(
error['providername'])
error = '{0}: {1}'.format(header, error['message'])
mediaitem.error = error
log(error, xbmc.LOGWARNING)
self.notify_warning(error)
elif auto and not self.debug and not self.localmode:
if not (mediatype == mediatypes.EPISODE and 'fanart' in mediaitem.skip_artwork) and \
mediatype != mediatypes.SONG:
self.processed.set_nextdate(
mediaitem.dbid, mediatype, mediaitem.label,
datetime_now() +
timedelta(days=self.get_nextcheckdelay(mediaitem)))
if mediatype == mediatypes.TVSHOW:
self.processed.set_data(mediaitem.dbid, mediatype,
mediaitem.label, mediaitem.season)
if mediaitem.borked_filename:
msg = L(FILENAME_ENCODING_ERROR).format(mediaitem.file)
if not mediaitem.error:
mediaitem.error = msg
log(msg, xbmc.LOGWARNING)
if self.debug:
log(mediaitem, xbmc.LOGNOTICE)
return services_hit
def cachelocal(self, mediaitem, toset):
ismusic = mediaitem.mediatype in mediatypes.audiotypes
if settings.cache_local_video_artwork and not ismusic or \
settings.cache_local_music_artwork and ismusic:
artmap = dict(mediaitem.art)
artmap.update(toset)
self.downloader.cachefor(artmap)
def get_nextcheckdelay(self, mediaitem):
weeks = 4 if mediatypes.only_filesystem(mediaitem.mediatype) \
else 32 if mediaitem.missingid or not mediaitem.missingart \
or mediaitem.mediatype in (mediatypes.MOVIE, mediatypes.TVSHOW) \
and mediaitem.premiered < self.freshstart \
else 16
return plus_some(weeks * 7, weeks)
def manual_id(self, mediaitem):
label = ENTER_COLLECTION_NAME if mediaitem.mediatype == mediatypes.MOVIESET else ENTER_ARTIST_TRACK_NAMES
result = xbmcgui.Dialog().input(L(label), mediaitem.label)
if not result:
return False # Cancelled
options = search[mediaitem.mediatype].search(result,
mediaitem.mediatype)
selected = xbmcgui.Dialog().select(
mediaitem.label, [option['label'] for option in options])
if selected < 0:
return False # Cancelled
uq = options[selected]['uniqueids']
toset = uq.get('tmdb')
if not toset and 'mbtrack' in uq and 'mbgroup' in uq and 'mbartist' in uq:
toset = '{0}/{1}/{2}'.format(uq['mbtrack'], uq['mbgroup'],
uq['mbartist'])
if toset:
self.processed.set_data(mediaitem.dbid, mediaitem.mediatype,
mediaitem.label, toset)
return bool(toset)
def setlanguages(self):
languages = []
if settings.language_override:
languages.append(settings.language_override)
if settings.language_fallback_kodi:
newlang = pykodi.get_language(xbmc.ISO_639_1)
if newlang not in languages:
languages.append(newlang)
if settings.language_fallback_en and 'en' not in languages:
languages.append('en')
self.autolanguages = languages
log("Working language filter: " + str(languages))
def get_top_missing_art(self, missingarts, mediatype, existingart,
availableart):
# TODO: refactor to `get_top_artwork` that works on a single art type
if not availableart:
return {}
newartwork = {}
for missingart in missingarts:
if missingart not in availableart:
continue
itemtype, artkey = mediatypes.hack_mediaarttype(
mediatype, missingart)
artinfo = mediatypes.get_artinfo(itemtype, artkey)
if artinfo['multiselect']:
existingurls = []
existingartnames = []
for art, url in existingart.iteritems():
if info.arttype_matches_base(art, missingart) and url:
existingurls.append(url)
existingartnames.append(art)
newart = [
art
for art in availableart[missingart] if self._auto_filter(
missingart, art, mediatype, availableart[missingart],
existingurls)
]
if not newart:
continue
newartcount = 0
for i in range(0, artinfo['autolimit']):
exacttype = '%s%s' % (artkey, i if i else '')
if exacttype not in existingartnames:
if newartcount >= len(newart):
break
if exacttype not in newartwork:
newartwork[exacttype] = []
newartwork[exacttype] = newart[newartcount]['url']
newartcount += 1
else:
newart = next((art for art in availableart[missingart]
if self._auto_filter(missingart, art, mediatype,
availableart[missingart])),
None)
if newart:
newartwork[missingart] = newart['url']
return newartwork
def _auto_filter(self,
basearttype,
art,
mediatype,
availableart,
ignoreurls=(),
skippreferred=False):
if art['rating'].sort < settings.minimum_rating:
return False
if not skippreferred and mediatypes.haspreferred_source(mediatype) and \
not mediatypes.ispreferred_source(mediatype, art['provider'][0]) and \
any(1 for i in availableart if mediatypes.ispreferred_source(mediatype, i['provider'][0]) and
self._auto_filter(basearttype, i, mediatype, availableart, ignoreurls, True)):
return False
if basearttype.endswith(
'fanart') and art['size'].sort < settings.minimum_size:
return False
if art['provider'].sort == 'theaudiodb.com' or not art['language'] and \
(basearttype.endswith('poster') and settings.titlefree_poster or
basearttype.endswith(('fanart', 'keyart', 'characterart'))):
return skippreferred or art['url'] not in ignoreurls
return art['language'] in self.autolanguages and (
skippreferred or art['url'] not in ignoreurls)
def test_filemanager_all_data_local(self):
"""
Create a dummy project and populate it with empty files
to test that filemanager.all_data_local works correctly"""
"""
############### SETUP ##################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = True
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 1
simend = 10
event_list = EventList()
remote_path = '/dummy/remote/20180215.DECKv1b_1pctCO2.ne30_oEC.edison/run/something'
local_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/dummyproject'
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
types = ['atm', 'ocn', 'lnd', 'ice']
mutex = threading.Lock()
if os.path.exists(local_path):
shutil.rmtree(local_path)
"""
############### TEST ##################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=local_path,
experiment=self.experiment)
self.assertEqual(
filemanager.remote_path,
'/dummy/remote/20180215.DECKv1b_1pctCO2.ne30_oEC.edison')
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.update_local_status()
self.assertFalse(filemanager.all_data_local())
filemanager.mutex.acquire()
for df in DataFile.select():
name = df.name
head, tail = os.path.split(df.local_path)
if not os.path.exists(head):
os.makedirs(head)
with open(df.local_path, 'w') as fp:
fp.write('this is a test file')
size = os.path.getsize(df.local_path)
df.remote_size = size
df.local_size = size
df.save()
if filemanager.mutex.locked():
filemanager.mutex.release()
filemanager.update_local_status()
self.assertTrue(filemanager.all_data_local())
"""
#########################################
"""
os.remove(database)
def setup(argv, display_event, **kwargs):
"""
Parse the commandline arguments, and setup the master config dict
Parameters:
parser (argparse.ArgumentParser): The parser object
display_event (Threadding_event): The event to turn the display on and off
"""
print "entering setup"
# Setup the parser
args = parse_args(argv=argv)
if not args.config:
parse_args(print_help=True)
return False, False, False
event_list = kwargs['event_list']
thread_list = kwargs['thread_list']
mutex = kwargs['mutex']
# check if globus config is valid, else remove it
globus_config = os.path.join(os.path.expanduser('~'), '.globus.cfg')
if os.path.exists(globus_config):
try:
conf = ConfigObj(globus_config)
except:
os.remove(globus_config)
if not os.path.exists(args.config):
print "Invalid config, {} does not exist".format(args.config)
return False, False, False
# Check that there are no white space errors in the config file
line_index = check_config_white_space(args.config)
if line_index != 0:
print '''
ERROR: line {num} does not have a space after the \'=\', white space is required.
Please add a space and run again.'''.format(num=line_index)
return False, False, False
# read the config file and setup the config dict
try:
config = ConfigObj(args.config)
except Exception as e:
print "Error parsing config file {}".format(args.config)
parse_args(print_help=True)
return False, False, False
# run validator for config file
if config.get('global'):
if args.resource_dir:
config['global']['resource_dir'] = args.resource_dir
else:
config['global']['resource_dir'] = os.path.join(
sys.prefix, 'share', 'processflow', 'resources')
else:
return False, False, False
template_path = os.path.join(config['global']['resource_dir'],
'config_template.json')
with open(template_path, 'r') as template_file:
template = json.load(template_file)
valid, messages = verify_config(config, template)
if not valid:
for message in messages:
print message
return False, False, False
config['global']['input_path'] = os.path.join(
config['global']['project_path'], 'input')
config['global']['output_path'] = os.path.join(
config['global']['project_path'], 'output')
# setup output and cache directories
if not os.path.exists(config['global']['input_path']):
os.makedirs(config['global']['input_path'])
if not os.path.exists(config['global']['output_path']):
os.makedirs(config['global']['output_path'])
# Copy the config into the input directory for safe keeping
input_config_path = os.path.join(config['global']['input_path'], 'run.cfg')
try:
copy(args.config, input_config_path)
except:
pass
# setup logging
if args.log:
log_path = args.log
else:
log_path = os.path.join(
config.get('global').get('output_path'), 'workflow.log')
config['global']['log_path'] = log_path
logging.basicConfig(format='%(asctime)s:%(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
filename=log_path,
filemode='w',
level=logging.INFO)
logging.getLogger('globus_sdk').setLevel(logging.WARNING)
# Make sure the set_frequency is a list of ints
set_frequency = config['global']['set_frequency']
if not isinstance(set_frequency, list):
set_frequency = [int(set_frequency)]
else:
# These are sometimes strings which break things later
new_freqs = []
for freq in set_frequency:
new_freqs.append(int(freq))
set_frequency = new_freqs
config['global']['set_frequency'] = set_frequency
# setup config for file type directories
if not isinstance(config['global']['file_types'], list):
config['global']['file_types'] = [config['global']['file_types']]
# setup run_scipts_path
run_script_path = os.path.join(config['global']['output_path'],
'run_scripts')
config['global']['run_scripts_path'] = run_script_path
if not os.path.exists(run_script_path):
os.makedirs(run_script_path)
# setup tmp_path
tmp_path = os.path.join(config['global']['output_path'], 'tmp')
config['global']['tmp_path'] = tmp_path
if os.path.exists(tmp_path):
rmtree(tmp_path)
os.makedirs(tmp_path)
# setup the year_set list
config['global']['simulation_start_year'] = int(
config['global']['simulation_start_year'])
config['global']['simulation_end_year'] = int(
config['global']['simulation_end_year'])
sim_start_year = int(config['global']['simulation_start_year'])
sim_end_year = int(config['global']['simulation_end_year'])
config['global']['short_term_archive'] = int(
config['global']['short_term_archive'])
# initialize the filemanager
event_list.push(message='Initializing file manager')
head, tail = os.path.split(config['global']['source_path'])
if tail == 'run':
config['global']['source_path'] = head
filemanager = FileManager(
database=os.path.join(config['global']['project_path'], 'input',
'workflow.db'),
types=config['global']['file_types'],
sta=config['global']['short_term_archive'],
remote_path=config['global']['source_path'],
remote_endpoint=config['transfer']['source_endpoint'],
local_path=os.path.join(config['global']['project_path'], 'input'),
local_endpoint=config['transfer']['destination_endpoint'],
mutex=mutex)
filemanager.populate_file_list(
simstart=config['global']['simulation_start_year'],
simend=config['global']['simulation_end_year'],
experiment=config['global']['experiment'])
print 'Updating local status'
filemanager.update_local_status()
print 'Local status update complete'
all_data = filemanager.all_data_local()
if all_data:
print 'All data is local'
else:
print 'Additional data needed'
logging.info("FileManager setup complete")
logging.info(str(filemanager))
if all_data or args.no_monitor:
print "skipping globus setup"
else:
endpoints = [endpoint for endpoint in config['transfer'].values()]
if args.no_ui:
print 'Running in no-ui mode'
addr = config.get('global').get('email')
if not addr:
print 'When running in no-ui mode, you must enter an email address.'
return False, False, False
setup_success = setup_globus(endpoints=endpoints,
no_ui=True,
src=config.get('global').get('email'),
dst=config.get('global').get('email'),
event_list=event_list)
else:
output_path = config.get('global').get('output_path')
error_output = os.path.join(output_path, 'workflow.error')
config['global']['error_path'] = error_output
if not os.path.exists(output_path):
os.makedirs(output_path)
sys.stderr = open(error_output, 'w')
msg = 'Activating endpoints {}'.format(' '.join(endpoints))
logging.info(msg)
setup_success = setup_globus(endpoints=endpoints,
display_event=display_event,
no_ui=False)
if not setup_success:
print "Globus setup error"
return False, False, False
else:
print 'Globus authentication complete'
print 'Checking file access on globus transfer nodes'
setup_success, endpoint = check_globus(
source_endpoint=config['transfer']['source_endpoint'],
source_path=config['global']['source_path'],
destination_endpoint=config['transfer']['destination_endpoint'],
destination_path=config['global']['input_path'])
if not setup_success:
print 'ERROR! Unable to access {} globus node'.format(
endpoint['type'])
print 'The node may be down, or you may not have access to the requested directory'
return False, False, False
# setup the runmanager
runmanager = RunManager(event_list=event_list,
output_path=config['global']['output_path'],
caseID=config['global']['experiment'],
scripts_path=run_script_path,
thread_list=kwargs['thread_list'],
event=kwargs['kill_event'])
runmanager.setup_job_sets(set_frequency=config['global']['set_frequency'],
sim_start_year=sim_start_year,
sim_end_year=sim_end_year,
config=config,
filemanager=filemanager)
# Turning off the GUI for the time being
# config['global']['ui'] = False if args.no_ui else True
config['global']['ui'] = False
config['global']['no_cleanup'] = True if args.no_cleanup else False
config['global']['no_monitor'] = True if args.no_monitor else False
config['global']['print_file_list'] = True if args.file_list else False
logging.info('Starting run with config')
logging.info(pformat(config))
return config, filemanager, runmanager