class CmdShell(CmdBase):
"""
shell
Load interactive IPython shell for StarFlow
The following objects are automatically available at the prompt:
All starflow modules are automatically imported in the IPython session
"""
names = ['shell']
def execute(self,args):
DE_MANAGER = de.DataEnvironmentManager()
if args:
local_name = args[0]
reg_info = DE_MANAGER.get_registry_info(local_name = local_name)
os.environ["WORKING_DE_PATH"] = reg_info["root_dir"]
WORKING_DE = DE_MANAGER.working_de
for modname in starflow.__all__:
log.info('Importing module %s' % modname)
fullname = starflow.__name__ + '.' + modname
try:
__import__(fullname)
locals()[modname] = sys.modules[fullname]
except ImportError,e:
log.error("Error loading module %s: %s" % (modname, e))
log.info("Importing data environment manager object as DE_MANAGER")
locals()["DE_MANAGER"] = DE_MANAGER
log.info("Importing working dataenvironment object as WORKING_DE")
locals()["WORKING_DE"] = WORKING_DE
from starflow.system_io_override import io_override
io_override(WORKING_DE)
os.chdir(WORKING_DE.temp_dir)
print '\n... done loading. May the Organization be with you as you journey through the Land of Data.'
try:
import IPython.Shell
ipy_shell = IPython.Shell.IPShellEmbed(argv=[])
ipy_shell()
except ImportError,e:
def ipy_shell():
log.error("Unable to load IPython.")
log.error("Please check that IPython is installed and working.")
log.error("If not, you can install it via: easy_install ipython")
def execute(self, args):
log.info("Initializing data environment ...")
argdict = {}
argdict["root_dir"] = args[0]
argdict["name"] = args[1]
if len(args) > 2:
argdict["gmail_account_name"] = args[2]
argdict["gmail_account_passwd"] = args[3]
de_manager = de.DataEnvironmentManager()
de_manager.make_de(**argdict)
def register(self,path,reg_info):
log.info("Registering %s" %path)
reg_info = reg_info.copy()
if "root_dir" not in reg_info:
reg_info["root_dir"] = path
local_name = reg_info.get("local_name")
if local_name in self.local_names:
other_reg_info = self.get_registry_info(local_name = local_name)
other_path = other_reg_info["root_dir"]
if path != other_path:
raise exception.NameAlreadyInRegistryError(local_name)
self.registry[path] = reg_info
registry_fh = open(static.GLOBAL_REGISTRY_FILE,'w')
pickle.dump(self.registry,registry_fh)
registry_fh.close()
def execute(self, args):
log.info("Initializing data environment ...")
argdict = {}
argdict["root_dir"] = args[0]
argdict["name"] = args[1]
if len(args) > 2:
argdict["gmail_account_name"] = args[2]
argdict["gmail_account_passwd"] = args[3]
de_manager = de.DataEnvironmentManager()
de_manager.make_de(**argdict)
def create_global_config(self):
cfg_file = self._cfg_file
cfg_dir = self._cfg_dir
if not os.path.exists(cfg_dir):
os.makedirs(cfg_dir)
import starflow.templates as templates
cfg_fh = open(cfg_file, 'w')
cfg_fh.write(templates.GLOBAL_CONFIG)
cfg_fh.close()
log.info("Global config file written to %s. Please customize this file." %
cfg_file)
import cPickle as pickle
empty_registry = {}
registry_file = self._registry_file
registry_fh = open(registry_file,'w')
pickle.dump(empty_registry,registry_fh)
registry_fh.close()
log.info("Empty data environment registry written to %s. Please customize this file." %
registry_file)
template_dir = static.GLOBAL_TEMPLATE_DIR
os.makedirs(os.path.join(template_dir, static.LOCAL_CFG_DIR))
local_cfg_template_file = os.path.join(template_dir, static.LOCAL_CFG_FILE)
local_cfg_template_fh = open(local_cfg_template_file, 'w')
local_cfg_template_fh.write(templates.LOCAL_CONFIG)
local_cfg_template_fh.close()
log.info(TEMPLATE_WRITTEN_MESSAGE %
("local config", local_cfg_template_file))
local_lmf_template_file = os.path.join(template_dir,
static.LOCAL_LIVE_MODULE_FILTER_FILE)
local_lmf_template_fh = open(local_lmf_template_file, 'w')
local_lmf_template_fh.write(templates.LOCAL_LIVE_MODULE_FILTER_TEXT)
local_lmf_template_fh.close()
log.info(TEMPLATE_WRITTEN_MESSAGE %
("live module filters", local_lmf_template_file))
local_setup_template_file = os.path.join(template_dir, static.LOCAL_SETUP_FILE)
local_setup_template_fh = open(local_setup_template_file, 'w')
local_setup_template_fh.write(templates.LOCAL_SETUP_TEXT)
local_setup_template_fh.close()
log.info(TEMPLATE_WRITTEN_MESSAGE %
("local setup.py file",local_setup_template_file))
def execute(self,args):
DE_MANAGER = de.DataEnvironmentManager()
if args:
local_name = args[0]
reg_info = DE_MANAGER.get_registry_info(local_name = local_name)
os.environ["WORKING_DE_PATH"] = reg_info["root_dir"]
WORKING_DE = DE_MANAGER.working_de
for modname in starflow.__all__:
log.info('Importing module %s' % modname)
fullname = starflow.__name__ + '.' + modname
try:
__import__(fullname)
locals()[modname] = sys.modules[fullname]
except ImportError,e:
log.error("Error loading module %s: %s" % (modname, e))
def clean_registry(self,path=None,local_name=None,strong=False,force=False):
if path is None and local_name is None:
infolist = [{"path":p} for p in self.registry]
else:
infolist = [{"path":path,"local_name":local_name}]
for info in infolist:
reg_info = self.get_registry_info(**info)
path = reg_info["root_dir"]
if force or not os.path.exists(path):
self.deregister(path)
elif strong:
try:
data_environment = self.verify(path)
except exception.ValidationError:
self.degister(path)
else:
log.info("Data environment %s at %s passes." %
(data_environment.name,data_environment.root_dir))
def init_de(self,**arg_dict):
path = arg_dict["root_dir"]
name = arg_dict["name"]
local_name = arg_dict["local_name"]
if not self.location_format_check(path):
raise exception.DELocationFormatError(name,self.location_format_rules)
if not self.name_format_check(name):
raise exception.DENameFormatError(name,self.name_format_rules)
if local_name is not None and not self.name_format_check(local_name):
raise exception.DENameFormatError(local_name,self.name_format_rules)
try:
self.make_de(path,arg_dict)
self.register(path,arg_dict)
except (exception.ValidationError,exception.RegistryError),e:
log.info("An error occured during creation process (see below). There\
may be detritus of a partially created data enviroment at %s" % path)
raise e
def get_data_environment(self,path=None,local_name=None):
if path is None and local_name is not None:
reg_info = self.get_registry_info(local_name = local_name)
path = reg_info["root_dir"]
data_environment = DataEnvironment(path = path)
path = data_environment.root_dir
if not path in self.registry:
log.info("Unregistered data environment at %s, registering."
% path)
reg_info = {"name":data_environment.name}
if data_environment.name not in self.local_names:
reg_info['local_name'] = data_environment.name
log.info("using local name from de internal name: %s"
% data_environment.name)
else:
log.info("No local name selected. Using set local name command \
if you want.")
self.register(path,reg_info)
return data_environment
def deregister(self,path):
log.info("Deregistering %s" %path)
self.registry.pop(path)
registry_fh = open(static.GLOBAL_REGISTRY_FILE,'w')
pickle.dump(self.registry,registry_fh)
registry_fh.close()
def cancel_command(self, signum, frame):
print
log.info("Exiting...")
sys.exit(1)
if not self.name_format_check(name):
raise exception.DENameFormatError(name,self.name_format_rules)
if local_name is not None and not self.name_format_check(local_name):
raise exception.DENameFormatError(local_name,self.name_format_rules)
try:
self.make_de(path,arg_dict)
self.register(path,arg_dict)
except (exception.ValidationError,exception.RegistryError),e:
log.info("An error occured during creation process (see below). There\
may be detritus of a partially created data enviroment at %s" % path)
raise e
else:
log.info("Data environment %s successfully initialized." % path)
def make_de(self,path,cfg_dict):
if os.path.exists(path):
raise exception.PathAlreadyExistsError(path)
os.makedirs(path)
copy_contents(static.GLOBAL_TEMPLATE_DIR,path)
cfg_dir = os.path.join(path,static.LOCAL_CFG_DIR)
local_cfg_file = os.path.join(path,static.LOCAL_CFG_FILE)
local_cfg = open(local_cfg_file,'r').read()
local_cfg = local_cfg % cfg_dict
new_local_cfg_fh = open(local_cfg_file,'w')
new_local_cfg_fh.write(local_cfg)
new_local_cfg_fh.close()
'''
this is a script that is used by the data Environment to initialize a python interpreter session during automatic updating. Do not modify it unless you know what you're doing.
'''
import os
import sys
import starflow.de as de
from starflow.logger import log
DE_MANAGER = de.DataEnvironmentManager()
WORKING_DE = DE_MANAGER.working_de
#Print a welcome message -- optional :)
log.info('Loading StarFlow for production ...')
#set some environment variables
import os
os.chdir(WORKING_DE.temp_dir)
loc = os.getcwd()
assert loc == WORKING_DE.temp_dir, 'Local Directory Must be Temp'
#print 'Data Environment directory = ', Dir
#modify the pythonpath to ensure that commands from this data environment are used
import sys
sys.path.insert(0,WORKING_DE.root_dir)
sys.path.insert(0,WORKING_DE.config_dir)
WORKING_DE.system_mode = 'PRODUCTION'
def execute(self, args):
log.info("Listing DEs ...")
de_manager = de.DataEnvironmentManager()
print('\n\n'.join(map(repr,de_manager.data_environments)))
def execute(self, args):
log.info("Listing DEs ...")
de_manager = de.DataEnvironmentManager()
print('\n\n'.join(map(repr, de_manager.data_environments)))
'''
this is a script that is used by the data Environment to initialize a python interpreter session during automatic updating. Do not modify it unless you know what you're doing.
'''
import os
import sys
import starflow.de as de
from starflow.logger import log
DE_MANAGER = de.DataEnvironmentManager()
WORKING_DE = DE_MANAGER.working_de
#Print a welcome message -- optional :)
log.info('Loading StarFlow for production ...')
#set some environment variables
import os
os.chdir(WORKING_DE.temp_dir)
loc = os.getcwd()
assert loc == WORKING_DE.temp_dir, 'Local Directory Must be Temp'
#print 'Data Environment directory = ', Dir
#modify the pythonpath to ensure that commands from this data environment are used
import sys
sys.path.insert(0, WORKING_DE.root_dir)
sys.path.insert(0, WORKING_DE.config_dir)
WORKING_DE.system_mode = 'PRODUCTION'
from starflow.system_io_override import io_override
def cancel_command(self, signum, frame):
print
log.info("Exiting...")
sys.exit(1)
