def main():
try:
config = SafeConfigParser()
if config.read('config.ini') != ['config.ini']:
raise OSError('no config.ini file present')
if config.getboolean('Generate CSVs', 'autogen-q'):
if config.get('Generate CSVs', 'type') == 'logarithmic':
minQ = config.getint('Generate CSVs', 'min-q')
maxQ = config.getint('Generate CSVs', 'max-q')
numQ = config.getint('Generate CSVs', 'num-q')
pd.Series(np.logspace(round(math.log(minQ, 10)), round(math.log(maxQ,10)), numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
if config.get('Generate CSVs', 'type') == 'linear':
minQ = config.getint('Generate CSVs', 'min-q')
maxQ = config.getint('Generate CSVs', 'max-q')
numQ = config.getint('Generate CSVs', 'num-q')
pd.Series(np.linspace(minQ, maxQ, numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
if config.get('Generate CSVs', 'type') == 'digikey':
pd.Series([1,5,10,25,50,100,250,500,1000,5000,10000]).to_csv('DataIN/Quantity.csv', index=False)
qs = pd.read_csv('DataIN/Quantity.csv', squeeze=True)
titlesVarCost = ['Name']
titlesDigiKey = ['Name', 'URL', 'Quantity']
for i in range(0, len(qs)):
titlesVarCost.append('Cost @' + str(qs.iloc[i]) + "pcs")
if config.getboolean('Generate CSVs', 'gen-fc'):
pd.DataFrame(columns = ['Name', 'Cost']).to_csv('DataIN/FixedCosts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-dkp'):
pd.DataFrame(columns = titlesDigiKey).to_csv('DataIN/DigiKeyParts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-vc'):
pd.DataFrame(columns = titlesVarCost).to_csv('DataIN/VarCosts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-p'):
pd.DataFrame(columns = ['Price']).to_csv('DataIN/Price.csv', index=False)
except OSError as e:
print(e)
except:
print("something went wrong, most likely your config.ini file is not properly configured")
def run(self):
""" Build the tutorials (iPython notebook files) located in tutorials/*
into static HTML pages.
"""
from IPython.nbconvert.nbconvertapp import NbConvertApp
check_ipython_version()
current_directory = os.getcwd()
html_base = os.path.join(current_directory,"html")
if not os.path.exists(html_base):
os.mkdir(html_base)
tutorials_base = os.path.join(current_directory,'tutorials')
app = NbConvertApp()
app.initialize()
app.export_format = 'html'
template_path = os.path.join(tutorials_base, 'templates')
app.config.Exporter.template_path = ['templates', template_path]
app.config.Exporter.template_file = 'astropy'
# walk through each directory in tutorials/ to find all .ipynb file
index_list = []
for tutorial_name in os.listdir(tutorials_base):
path = os.path.join(tutorials_base, tutorial_name)
if not os.path.isdir(path):
continue
# read metadata from config file
config = SafeConfigParser()
config.read(os.path.join(path,"metadata.cfg"))
is_published = config.getboolean("config", "published")
if not is_published:
continue
for filename in os.listdir(path):
base,ext = os.path.splitext(filename)
if ext.lower() == ".ipynb" and "checkpoint" not in base:
app.output_base = os.path.join(html_base,base)
app.notebooks = [os.path.join(path,filename)]
app.start()
index_listing = dict()
index_listing["link_path"] = "{}.html".format(base)
index_listing["link_name"] = config.get("config", "link_name")
index_list.append(index_listing)
# Make an index of all notes
entries = []
for page in index_list:
entries.append(' <li><a href="{0[link_path]}">{0[link_name]}</a></li>'.format(page))
with open(os.path.join(current_directory,'html','index.html'), 'w') as f:
f.write(INDEX_TEMPLATE.format(entries='\n'.join(entries)))
def test_pulp():
tasker = DockerTasker()
parsed_config = SafeConfigParser()
assert len(parsed_config.read(PULP_CONF_PATH)) > 0
host = parsed_config.get("server", "host")
un = parsed_config.get("server", "username")
pswd = parsed_config.get("server", "password")
verify_ssl = parsed_config.getboolean("server", "verify_ssl")
push_image_to_pulp("busybox-test", "busybox", host, un, pswd, verify_ssl,
tasker, logging.getLogger("dock.tests"))
def init():
ConfigFilePath = os.path.dirname(os.path.realpath(__file__)) + '/config.ini'
print("Looking for config file at: " + ConfigFilePath)
parser = SafeConfigParser()
parser.read(ConfigFilePath)
global ConnectionString
ConnectionString = parser.get('PostgreSQL', 'ConnectionString')
global DebugEnabled
DebugEnabled = parser.getboolean('Etc', 'Debug')
def parse_config(self,config,section=None):
if section is None: section = self.section
if isinstance(config,basestring):
configfile = config
config = SafeConfigParser()
config.read(configfile)
# Set the options first (otherwise optional positional arguments disappear)
if OPTIONS in config._sections[section].keys():
self.parse_args(config.get(section,OPTIONS))
# Don't grab the default keys
dests = [a.dest for a in self.parser._actions]
for key in config._sections[section].keys():
if key == '__name__': continue
if key == OPTIONS: continue
try:
index = dests.index(key)
action = self.parser._actions[index]
except ValueError:
msg = "Unrecognized argument in config: %s\n"%key
msg += self.get_help()
raise ValueError(msg)
if isinstance(action,argparse._StoreTrueAction):
value = config.getboolean(section,key)
if isinstance(action,argparse._StoreFalseAction):
value = config.getboolean(section,key)
if isinstance(action,argparse._CountAction):
try:
value = int(config.getboolean(section,key))
except ValueError:
value = config.getint(section,key)
else:
value = config.get(section,key)
setattr(self.namespace,key,value)
return self.namespace
class Config(object):
def __init__(self):
self._configFileName = "rover.cfg"
self._readConfig()
def _readConfig(self):
self._config = SafeConfigParser({
"db_host": "localhost",
"db_port": "27017",
"db_name": "rover",
"serial_path": "/dev/ttymxc3",
"serial_baud": "115200",
"serial_timeout": "0.5",
"serial_encoding": "windows-1252"})
self._config.read(self._configFileName)
self._validateConfig()
def _writeConfig(self):
self._config.write(open(self._configFileName, "w"))
def _validateConfig(self):
changed = False
if not self._config.has_section("db"):
self._config.add_section("db")
changed = True
if not self._config.has_section("arduino_serial"):
self._config.add_section("arduino_serial")
changed = True
if changed:
self._writeConfig()
def get(self, section, key):
return self._config.get(section, key)
def getInt(self, section, key):
return self._config.getint(section, key)
def getFloat(self, section, key):
return self._config.getfloat(section, key)
def getBoolean(self, section, key):
return self._config.getboolean(section, key)
def set(self, section, key, value):
self._config.set(section, key, value)
self._writeConfig()
def _init(apiurl):
"""Initialize osc library.
apiurl is the apiurl which should be used.
"""
if hasattr(apiurl, 'extend') and len(set(apiurl)) > 1:
msg = ("Different apiurls are not supported at the moment: "
"%s (using %s)" % (', '.join(apiurl), apiurl[0]))
logger().info(msg)
apiurl = apiurl[0]
conf_filename = os.environ.get('OSC_CONFIG', '~/.oscrc')
conf_filename = os.path.expanduser(conf_filename)
cp = SafeConfigParser({'plaintext_password': True, 'aliases': ''})
cp.read(conf_filename)
apiurl = apiurl.strip('/')
if apiurl == 'api':
apiurl = 'https://api.opensuse.org'
for section in cp.sections():
aliases = cp.get(section, 'aliases', raw=True)
aliases = aliases.split(',')
if section.strip('/') == apiurl or apiurl in aliases:
user = cp.get(section, 'user', raw=True)
password = None
if cp.has_option(section, 'pass'):
password = cp.get(section, 'pass', raw=True)
if cp.has_option(section, 'passx'):
password = cp.get(section, 'pass', raw=True)
password = password.decode('base64').decode('bz2')
if (cp.has_option(section, 'keyring')
and cp.getboolean(section, 'keyring')):
try:
import keyring
host = urllib.parse.urlparse(apiurl).hostname
password = keyring.get_password(host, user)
except ImportError:
msg = ("keyring module not available but '%s' "
"stores password there") % conf_filename
raise ValueError(msg)
if password is None:
msg = "No password provided for %s" % section
raise ValueError(msg)
if '://' not in section:
section = 'https://{0}'.format(section)
Osc.init(section, username=user, password=password)
return section
class Configuration(object):
def __init__(self):
self.parser = SafeConfigParser(defaults={
'theme' : 'default',
'theme.mobile' : 'touch',
'hostname' : '0.0.0.0',
'port' : '7000',
'log.file' : '/tmp/rb-serve.log',
'log.level' : 'INFO',
'log.format' : '%(levelname)s %(asctime)s %(name)s: %(message)s',
'debug' : 'False',
})
self.parser.add_section('server')
self.parser.read(path.expanduser('~/.rhythmweb'))
self.configure_logger()
def get_string(self, key):
return self.parser.get('server', key, raw=True)
def get_int(self, key):
return self.parser.getint('server', key)
def get_boolean(self, key):
return self.parser.getboolean('server', key)
def configure_logger(self):
root = logging.getLogger()
root.setLevel(self.get_string('log.level'))
handler = logging.handlers.RotatingFileHandler(
self.get_string('log.file'),
backupCount=5,
maxBytes=1024*1024)
handler.setFormatter(logging.Formatter(fmt=self.get_string('log.format')))
root.addHandler(handler)
if root.isEnabledFor(logging.DEBUG):
root.debug('Logger configured')
root.debug('Showing app configuration:')
for section_name in self.parser.sections():
root.debug('Section: %s' % section_name)
for key in self.parser.options(section_name):
root.debug(' %s = %s' % (key, self.parser.get(section_name, key, raw=True)))
def _load_config(self):
"Load and parse config file, pass options to livestreamer"
config = SafeConfigParser()
config_file = os.path.join(self.config_path, 'settings.ini')
config.read(config_file)
for option, type in list(AVAILABLE_OPTIONS.items()):
if config.has_option('DEFAULT', option):
if type == 'int':
value = config.getint('DEFAULT', option)
if type == 'float':
value = config.getfloat('DEFAULT', option)
if type == 'bool':
value = config.getboolean('DEFAULT', option)
if type == 'str':
value = config.get('DEFAULT', option)
self.livestreamer.set_option(option, value)
class CrabClient:
"""Crab client class, used for interaction with the server."""
def __init__(self, command=None, crabid=None):
"""Constructor for CrabClient.
This causes the client to configure itself,
by looking for the crab.ini file. If the environment
variables CRABHOST or CRABPORT exist, these override
settings from the configuration files.
If the client has been started to report on the status of a
job, then the command must be supplied, and the crabid should
be given if known.
"""
self.command = command
self.crabid = crabid
self.config = SafeConfigParser()
self.config.add_section('server')
self.config.set('server', 'host', 'localhost')
self.config.set('server', 'port', '8000')
self.config.set('server', 'timeout', '30')
self.config.add_section('client')
self.config.set('client', 'use_fqdn', 'false')
env = os.environ
# Read configuration files -- first system and then user.
sysconfdir = env.get('CRABSYSCONFIG', '/etc/crab')
userconfdir = env.get('CRABUSERCONFIG', os.path.expanduser('~/.crab'))
self.configfiles = self.config.read([
os.path.join(sysconfdir, 'crab.ini'),
os.path.join(userconfdir, 'crab.ini')
])
# Override configuration as specified by environment variables.
if 'CRABHOST' in env:
self.config.set('server', 'host', env['CRABHOST'])
if 'CRABPORT' in env:
self.config.set('server', 'port', env['CRABPORT'])
if 'CRABUSERNAME' in env:
self.config.set('client', 'username', env['CRABUSERNAME'])
if 'CRABCLIENTHOSTNAME' in env:
self.config.set('client', 'hostname', env['CRABCLIENTHOSTNAME'])
# Add computed defaults for some values if they have not already
# been determined. This avoids the need to perform these operations
# if the value is already known and would allow the way in which this
# is done to be customized based on other values.
if not self.config.has_option('client', 'hostname'):
if self.config.getboolean('client', 'use_fqdn'):
self.config.set('client', 'hostname', socket.getfqdn())
else:
self.config.set('client', 'hostname',
socket.gethostname().split('.', 1)[0])
if not self.config.has_option('client', 'username'):
self.config.set('client', 'username', pwd.getpwuid(os.getuid())[0])
def start(self):
"""Notify the server that the job is starting.
Return the decoded server response, which may include
an inhibit dictionary item."""
return self._write_json(self._get_url('start'),
{'command': self.command},
read=True)
def finish(self, status=CrabStatus.UNKNOWN, stdoutdata='', stderrdata=''):
"""Notify the server that the job is finishing."""
self._write_json(
self._get_url('finish'), {
'command': self.command,
'status': status,
'stdout': stdoutdata,
'stderr': stderrdata
})
def send_crontab(self, crontab, timezone=None):
"""Takes the crontab as a string, breaks it into lines,
and transmits it to the server.
Returns a list of warnings."""
data = self._write_json(self._get_url('crontab'), {
'crontab': crontab.split('\n'),
'timezone': timezone
},
read=True)
return data['warning']
def fetch_crontab(self, raw=False):
"""Retrieves crontab lines from the server, and returns
them as a single string."""
url = self._get_url('crontab')
if raw:
url = url + '?raw=true'
data = self._read_json(url)
if data['crontab'] is None:
return ''
else:
return '\n'.join(data['crontab'])
def get_info(self):
info = []
info.append('Server: ' + self.config.get('server', 'host') + ':' +
self.config.get('server', 'port'))
info.append('Client: ' + self.config.get('client', 'username') + '@' +
self.config.get('client', 'hostname'))
info.append('Files: ' + ', '.join(self.configfiles))
return '\n'.join(info)
def _get_url(self, action):
"""Creates the URL to be used to perform the given server action."""
url = ('/api/0/' + action + '/' +
urlquote(self.config.get('client', 'hostname'), '') + '/' +
urlquote(self.config.get('client', 'username'), ''))
if self.crabid is not None:
url = url + '/' + urlquote(self.crabid, '')
return url
def _get_conn(self):
"""Opens an HTTP connection to the configured server."""
# Try first to construct the connection with a timeout. However
# this feature was added in Python 2.6, so for older versions of
# Python, we must catch the TypeError and construct the object
# without a timeout.
try:
return HTTPConnection(self.config.get('server', 'host'),
self.config.get('server', 'port'),
timeout=int(
self.config.get('server', 'timeout')))
except TypeError:
return HTTPConnection(self.config.get('server', 'host'),
self.config.get('server', 'port'))
def _read_json(self, url):
"""Performs an HTTP GET on the given URL and interprets the
response as JSON."""
try:
try:
conn = self._get_conn()
conn.request('GET', url)
res = conn.getresponse()
if res.status != 200:
raise CrabError('server error: ' + self._read_error(res))
return json.loads(latin_1_decode(res.read(), 'replace')[0])
# except HTTPException as err:
except HTTPException:
err = sys.exc_info()[1]
raise CrabError('HTTP error: ' + str(err))
# except socket.error as err:
except socket.error:
err = sys.exc_info()[1]
raise CrabError('socket error: ' + str(err))
# except ValueError as err:
except ValueError:
err = sys.exc_info()[1]
raise CrabError('did not understand response: ' + str(err))
finally:
conn.close()
def _write_json(self, url, obj, read=False):
"""Converts the given object to JSON and sends it with an
HTTP PUT to the given URL.
Optionally attempts to read JSON from the response."""
try:
try:
conn = self._get_conn()
conn.request('PUT', url, json.dumps(obj))
res = conn.getresponse()
if res.status != 200:
raise CrabError('server error: ' + self._read_error(res))
if read:
response = latin_1_decode(res.read(), 'replace')[0]
# Check we got a response before attempting to decode
# it as JSON. (Some messages did not have responses
# for previous server versions.)
if response:
return json.loads(response)
else:
return {}
# except HTTPException as err:
except HTTPException:
err = sys.exc_info()[1]
raise CrabError('HTTP error: ' + str(err))
# except socket.error as err:
except socket.error:
err = sys.exc_info()[1]
raise CrabError('socket error: ' + str(err))
# except ValueError as err:
except ValueError:
err = sys.exc_info()[1]
raise CrabError('did not understand response: ' + str(err))
finally:
conn.close()
def _read_error(self, res):
"""Determine the error message to show based on an
unsuccessful HTTP response.
Currently use the HTTP status phrase or the first
paragraph of the body, if found with a regular expression."""
message = res.reason
try:
body = latin_1_decode(res.read(), 'replace')[0]
match = re.search('<p>([^<]*)', body)
if match:
message = match.group(1)
except:
pass
return message
def run(ini_file='TOPKAPI.ini'):
"""Run the model with the set-up defined by `ini_file`.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
print('Read the file ',ini_file)
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getfloat('numerical_options', 'solve_s')
solve_o = config.getfloat('numerical_options', 'solve_o')
solve_c = config.getfloat('numerical_options', 'solve_c')
only_channel_output = config.getboolean('numerical_options',
'only_channel_output')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
print('Read the forcing data')
#~~~~Rainfall
h5file_in = h5.openFile(file_rain,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'rainfall')
ndar_rain = node.read()
h5file_in.close()
#~~~~ETr - Reference crop ET
h5file_in = h5.openFile(file_ET,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'ETr')
ndar_ETr = node.read()
h5file_in.close()
#~~~~ETo - Open water potential evap.
h5file_in = h5.openFile(file_ET,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'ETo')
ndar_ETo = node.read()
h5file_in.close()
#~~~~external_flow flows
if external_flow:
ar_Qexternal_flow = np.loadtxt(file_Qexternal_flow)[:, 5]
##============================##
## Pretreatment of input data ##
##============================##
print('Pretreatment of input data')
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, ar_lambda, \
ar_Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L0, ar_Ks0, \
ar_theta_r, ar_theta_s, \
ar_n_o0, ar_n_c0, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
ar_kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L0*fac_L
ar_Ks = ar_Ks0*fac_Ks
ar_n_o = ar_n_o0*fac_n_o
ar_n_c = ar_n_c0*fac_n_c
print('Max L=', max(ar_L))
print('Max Ks=', max(ar_Ks))
print('Max n_o=', max(ar_n_o))
print('Max n_c=', max(ar_n_c))
#~~~~Computation of model parameters from physical parameters
ar_Vsm, ar_b_s, ar_b_o, \
ar_W, ar_b_c = pm.compute_cell_param(X, ar_Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
ar_lambda, ar_tan_beta,
ar_tan_beta_channel, ar_L,
ar_Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow,
ar_coorx,
ar_coory,
ar_lambda)
print('external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')')
#~~~~Number of simulation time steps
nb_time_step = len(ndar_rain[:,0])
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
print('Initialize from file')
h5file_in = h5py.File(file_out)
ar_Vs0 = h5file_in['/Soil/V_s'][-1, :]
ar_Vc0 = h5file_in['/Channel/V_c'][-1, :]
ar_Vo0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
print('Initialize from parms')
ar_Vs0 = fl.initial_volume_soil(ar_pVs_t0, ar_Vsm)
ar_Vo0 = ar_Vo_t0
ar_Vc0 = fl.initial_volume_channel(ar_Qc_t0, ar_W, X, ar_n_c)
## Computed variables
#Matrix of soil,overland and channel store at the end of the time step
ar_Vs1 = np.ones(nb_cell)*-99.9
ar_Vo1 = np.ones(nb_cell)*-99.9
ar_Vc1 = np.ones(nb_cell)*-99.9
#Matrix of outflows between two time steps
ar_Qs_out = np.ones(nb_cell)*-99.9
ar_Qo_out = np.ones(nb_cell)*-99.9
ar_Qc_out = np.zeros(nb_cell)
## Intermediate variables
ar_a_s = np.ones(nb_cell)*-99.9
ar_a_o = np.ones(nb_cell)*-99.9
ar_a_c = np.ones(nb_cell)*-99.9
ar_Q_to_next_cell = np.ones(nb_cell)*-99.9
ar_Q_to_channel = np.ones(nb_cell)*-99.9
ar_Q_to_channel_sub = np.zeros(nb_cell)
ar_Qc_cell_up = np.zeros(nb_cell)
ar_ETa = np.zeros(nb_cell)
ar_ET_channel = np.zeros(nb_cell)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file = h5.openFile(file_out, mode=fmode, title='TOPKAPI_out')
root = h5file.getNode('/')
root._v_attrs.pytopkapi_version = pytopkapi.__version__
root._v_attrs.pytopkapi_git_revision = pytopkapi.__git_revision__
atom = h5.Float32Atom()
h5filter = h5.Filters(9)# maximum compression
# create file structure as necessary
grp_name = '/Soil'
if grp_name not in h5file:
h5file.createGroup('/', 'Soil', 'Soil arrays')
if grp_name+'/Qs_out' not in h5file:
array_Qs_out = h5file.createEArray(grp_name, 'Qs_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qs_out = h5file.getNode(grp_name+'/Qs_out')
if grp_name+'/V_s' not in h5file:
array_Vs = h5file.createEArray(grp_name, 'V_s',
atom, shape=(0, nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step+1)
else:
array_Vs = h5file.getNode(grp_name+'/V_s')
grp_name = '/Overland'
if grp_name not in h5file:
h5file.createGroup('/', 'Overland', 'Overland arrays')
if grp_name+'/Qo_out' not in h5file:
array_Qo_out = h5file.createEArray(grp_name, 'Qo_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qo_out = h5file.getNode(grp_name+'/Qo_out')
if grp_name+'/V_o' not in h5file:
array_Vo = h5file.createEArray(grp_name, 'V_o',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step+1)
else:
array_Vo = h5file.getNode(grp_name+'/V_o')
grp_name = '/Channel'
if grp_name not in h5file:
h5file.createGroup('/', 'Channel', 'Channel arrays')
if grp_name+'/Qc_out' not in h5file:
array_Qc_out = h5file.createEArray(grp_name, 'Qc_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qc_out = h5file.getNode(grp_name+'/Qc_out')
if grp_name+'/V_c' not in h5file:
array_Vc = h5file.createEArray(grp_name, 'V_c',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Vc = h5file.getNode(grp_name+'/V_c')
if grp_name+'/Ec_out' not in h5file:
array_Ec_out = h5file.createEArray(grp_name, 'Ec_out',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Ec_out = h5file.getNode(grp_name+'/Ec_out')
if '/ET_out' not in h5file:
array_ET_out = h5file.createEArray('/', 'ET_out',
atom, shape=(0,nb_cell),
title='mm', filters=h5filter,
expectedrows=nb_time_step)
else:
array_ET_out = h5file.getNode('/ET_out')
if '/Q_down' not in h5file:
array_Q_down = h5file.createEArray('/', 'Q_down',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Q_down = h5file.getNode('/Q_down')
if append_output is False or first_run is True:
#Write the initial values into the output file
array_Vs.append(ar_Vs0.reshape((1,nb_cell)))
array_Vo.append(ar_Vo0.reshape((1,nb_cell)))
array_Vc.append(ar_Vc0.reshape((1,nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1,nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1,nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1,nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1,nb_cell)))
array_ET_out.append(ar_ETa.reshape((1,nb_cell)))
E_vol = ar_ET_channel*1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1,nb_cell)))
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
print('** NB_CELL=',nb_cell)
print('** NB_TIME_STEP=',nb_time_step)
print('--> SIMULATIONS <--')
## Loop on time
for t in range(nb_time_step):
print(t+1, '/', nb_time_step)
eff_sat = ar_Vs0/ar_Vsm
# estimate soil suction head using Brookes and Corey (1964)
psi = psi_b/np.power(eff_sat, 1.0/lamda)
## Loop on cells
n=-1
for cell1 in ar_label_sort:
cell=np.where(ar_cell_label==cell1)[0][0]
n=n+1
## ======================== ##
## ===== INTERCEPTION ===== ##
## ======================== ##
## No interception for the moment
## ======================== ##
## ===== INFILTRATION ===== ##
## ======================== ##
rain_rate = ndar_rain[t, cell]/Dt
infiltration_depth = green_ampt_cum_infiltration(rain_rate,
psi[cell],
eff_theta[cell],
eff_sat[cell],
ar_Ks[cell], Dt)
## ====================== ##
## ===== SOIL STORE ===== ##
## ====================== ##
#~~~~ Computation of soil input
ar_a_s[cell] = fl.input_soil(infiltration_depth,
Dt, X,
ar_Q_to_next_cell,
li_cell_up[cell])
#~~~~ Resolution of the equation dV/dt=a_s-b_s*V^alpha_s
# Calculate the volume in the soil store at the end of the
# current time-step.
Vs_prim = om.solve_storage_eq(ar_a_s[cell], ar_b_s[cell],
alpha_s, ar_Vs0[cell], Dt, solve_s)
#~~~~ Computation of soil outflow and overland input
ar_Qs_out[cell], ar_Vs1[cell] = fl.output_soil(ar_Vs0[cell],
Vs_prim,
ar_Vsm[cell],
ar_a_s[cell],
ar_b_s[cell],
alpha_s, Dt)
if ar_Qs_out[cell] < 0:
print('Problem Soil:output greater than input....')
print('n=', n, 'label=', cell)
stop
## ========================== ##
## ===== OVERLAND STORE ===== ##
## ========================== ##
#~~~~ Computation of overland input
rain_excess = ndar_rain[t, cell] - infiltration_depth
# convert mm to m^3/s
rain_excess = max(0, (rain_excess*(10**-3)/Dt)*X**2)
ar_a_o[cell] = max(0,
ar_a_s[cell] \
- ((ar_Vs1[cell]-ar_Vs0[cell])/Dt \
+ ar_Qs_out[cell]) \
+ rain_excess)
#~~~~ Resolution of the equation dV/dt=a_o-b_o*V^alpha_o
ar_Vo1[cell] = om.solve_storage_eq(ar_a_o[cell],
ar_b_o[cell], alpha_o,
ar_Vo0[cell], Dt, solve_o)
#~~~~ Computation of overland outflows
ar_Qo_out[cell] = fl.Qout_computing(ar_Vo0[cell], ar_Vo1[cell],
ar_a_o[cell], Dt)
if ar_Qo_out[cell] < 0:
print('Problem Overland:output greater than input....')
print('n=', n, 'label=', cell)
stop
## ============================= ##
## ===== FLOW PARTITIONING ===== ##
## ============================= ##
# ar_Q_to_channel_sub doesn't get used for anything?
ar_Q_to_next_cell[cell], \
ar_Q_to_channel[cell], \
ar_Q_to_channel_sub[cell] = fl.flow_partitioning(ar_lambda[cell],
ar_Qs_out[cell],
ar_Qo_out[cell],
ar_W[cell],
X, ar_Xc[cell])
## ======================== ##
## ===== CHANNEL STORE ==== ##
## ======================== ##
if ar_lambda[cell] == 1:
if ar_cell_down[cell] >= 0 \
and ar_lambda[ar_cell_down[cell]] == 0:
print('Problem: the present cell has a channel but not the cell down...')
Stop
#~~~~ Computation of channel input
ar_a_c[cell], \
ar_Qc_cell_up[cell] = fl.input_channel(ar_Qc_out,
ar_Q_to_channel[cell],
li_cell_up[cell])
if external_flow \
and cell == np.where(ar_cell_label==cell_external_flow)[0][0]:
ar_a_c[cell] = ar_a_c[cell] + ar_Qexternal_flow[t]
#~~~~ Resolution of the equation dV/dt=a_c-b_c*V^alpha_c
ar_Vc1[cell] = om.solve_storage_eq(ar_a_c[cell],
ar_b_c[cell], alpha_c,
ar_Vc0[cell], Dt, solve_c)
#~~~~ Computation of channel outflows
ar_Qc_out[cell] = fl.Qout_computing(ar_Vc0[cell],
ar_Vc1[cell],
ar_a_c[cell], Dt)
if ar_Qc_out[cell] < 0:
print('Problem Channel: output greater than input....')
stop
if str(ar_Qc_out[cell]).count('N') > 0:
print(ar_Qc_out[cell])
print('Problem Channel: Non authorized operand....')
stop
else:
ar_a_c[cell] = 0.
ar_Vc1[cell] = 0.
ar_Qc_out[cell] = 0.
## ============================== ##
## ===== EVAPOTRANSPIRATION ===== ##
## ============================== ##
#~~~~~ From soil
ar_ETa[cell], \
ar_Vs1[cell], \
ar_Vo1[cell] = em.evapot_soil_overland(ar_Vo1[cell],
ar_Vs1[cell],
ar_Vsm[cell],
ar_kc[cell],
ndar_ETr[t, cell], X)
#~~~~~ Evaporation from channel
if ar_lambda[cell] == 1:
ar_ET_channel[cell], \
ar_Vc1[cell] = em.evapor_channel(ar_Vc1[cell],
ndar_ETo[t, cell],
ar_W[cell], ar_Xc[cell])
####===================================####
#### Affectation of new vector values ####
####===================================####
ar_Vs0 = np.array(ar_Vs1)
ar_Vo0 = np.array(ar_Vo1)
ar_Vc0 = np.array(ar_Vc1)
####===================================####
#### Results writing at each time step ####
####===================================####
array_Vs.append(ar_Vs1.reshape((1,nb_cell)))
array_Vo.append(ar_Vo1.reshape((1,nb_cell)))
array_Vc.append(ar_Vc1.reshape((1,nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1,nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1,nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1,nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1,nb_cell)))
array_ET_out.append(ar_ETa.reshape((1,nb_cell)))
E_vol = ar_ET_channel*1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1,nb_cell)))
h5file.close()
print(' ')
print('***** THE END *****')
item_seperator('fill')
#AlarmDistanceActivate
item_options('Distance travelled alarm, status','AlarmDistanceActivate')
option('Active','True');
option('Inactive','False');
item_count+=1
#AlarmDistanceValue
item_input_alphanumeric('Distance travelled alarm, set value','AlarmDistanceValue','6','0','0','',' NM')
#Seperator
item_seperator('fill')
#Get baro unit
barounit_mmhg=config.getboolean(config_category2, 'BaroUnitMmhg')
if barounit_mmhg is True:
barounit='mmHg'
else:
barounit='Mbar'
#AlarmBaroLowActivate
item_options('Barometric pressure (low) alarm, status','AlarmBaroLowActivate')
option('Active','True');
option('Inactive','False');
item_count+=1
#AlarmBaroLowValue
item_input_alphanumeric('Barometric pressure (low) alarm, set lower limit','AlarmBaroLowValue','4','0','0','',' '+barounit)
#AlarmBaroHighActivate
async def on_message(message):
global giveaway_is_running
# we do not want the bot to reply to itself
if message.author == client.user:
return
# Log all recieved commands, even if it came from DM
if message.content.startswith('.'):
# Don't log if second character is ., _ or " "
char_2 = message.content[1]
if char_2 not in [" ", "_", ".", "-"]:
delta = datetime.now()
username = message.author.name
channel = message.channel.name
msg = message.content
log_message = "{} - {}@{}: {}".format(delta, username, channel, message.content) + "\n"
with open("command_log", "a") as log_file:
log_file.write(log_message)
# Check if the message was recieved over DM
if message.channel.type == discord.ChannelType.private:
return
# Check for the next (expected) patch date for GW2
if message.content.startswith('.patch'):
link = 'https://www.thatshaman.com/tools/countdown/'
json_link = link + '?format=json'
with urllib.request.urlopen(json_link) as url:
s = url.read()
response = s.decode('utf-8')
json_obj = json.loads(response)
date = json_obj['date'].split("T")[0]
if json_obj['confirmed']:
status = "Confirmed"
else:
status = "Expected"
output = "Here's what I could find...\n"
output += "**Status**: {}\n".format(status)
output += "**Date**: {}\n".format(date)
output += "Based on info from that_shaman: <{}>".format(link)
await client.send_message(message.channel, output)
return
# Test for voice access
if message.content.startswith('.listchannelacces'):
if message.author.id in devs:
server = next(iter(client.servers))
channels = []
for channel in server.channels:
if not channel.is_private:
if channel.type == discord.ChannelType.text:
typename = "text"
elif channel.type == discord.ChannelType.voice:
typename = "voice"
else:
typename = "unknown"
channels.append("{} ({}),".format(channel.name, typename))
msg = ""
for c in channels:
msg += c
await client.send_message(message.channel, msg)
# Show given permissions
# COMMUNITY GIVEAWAYS!
if message.content.startswith('.giveaway --check-rank'):
if not giveaway_is_running:
if has_admin_rights(message.author) or message.author.id in devs:
# Set config rank restriction to True
config = SafeConfigParser()
config.read('config.ini')
config.set('giveaway', 'rank_restriction', 'True')
with open('config.ini', 'w') as f:
config.write(f)
giveaway_is_running = True
await client.send_message(message.channel, "{} has started a giveaway! This giveaway is restricted to Little Sprouts and higher (lvl 2+)! Type .enter to enter the giveaway!".format(message.author.mention))
return
if message.content.startswith('.giveaway'):
if not giveaway_is_running:
if has_admin_rights(message.author) or message.author.id in devs:
# Set the rank restriction in config to false
config = SafeConfigParser()
config.read('config.ini')
config.set('giveaway', 'rank_restriction', 'False')
with open('config.ini', 'w') as f:
config.write(f)
giveaway_is_running = True
await client.send_message(message.channel, "{} has started a giveaway! This giveaway is open to all! Type .enter to enter the giveaway!".format(message.author.mention))
return
if message.content.startswith('.enter'):
if giveaway_is_running:
discord_id = message.author.id
# Check if user is entered, either manually by admin or by the user himself
resultset = session.query(Giveaway).filter_by(discord_id = discord_id)
if resultset.count() > 0:
sent_msg = await client.send_message(message.channel, "{}, you are already entered in the giveaway!".format(message.author.mention))
await asyncio.sleep(5)
await client.delete_message(sent_msg)
await client.delete_message(message)
return
# Check if the giveaway is rank restricted
config = SafeConfigParser()
config.read('config.ini')
rank_restricted = config.getboolean('giveaway', 'rank_restriction')
# If the rank is restricted, and the user doesn't match the rank
# They get a DM and the function is exited because it doesn't need to check the rest
if rank_restricted:
# Check if the user has the required rank
member = message.author
has_rank = False
for role in member.roles:
if role.name == "Little Sprout":
has_rank = True
if not has_rank:
explain = ""
explain += "The giveaway you tried to enter has a rank restriction.\n\n"
explain += "**Why is there a rank restriction?**\n"
explain += "We have two types of giveaways. There's global giveaways set up by the GM's and WP, and "
explain += "there are giveaways set up by individual members. Global giveaways are open to all. However, when an "
explain += "individual member gives away something, they can choose to put this rank restriction in place. Usually this is "
explain += "to prevent people generally not really engaging with the community snatching away keys from more active members. "
explain += "You should see this type of giveaway as a giveaway where a community member gives away something personal to another (active) community member.\n\n"
explain += "**So how do these ranks work?**\n"
explain += "The ranks are tied in to the Mee6 leveling system. When you post a message on the discord server, Mee6 gives you XP. "
explain += "When you've accumulated enough XP, Mee6 will give you a new level, and at certain levels also a new rank. You need to be "
explain += "at least level 2 (which is the Little Sprout rank on discord) to enter this giveaway.\n\n"
explain += "**Can I still enter the giveaway if I manage to get the required rank before the giveaway ends?**\n"
explain += "Absolutely! Just be aware that it might take several minutes for Mee6 to give you the rank once you hit level 2.\n\n"
explain += "**So I should just spam all day to get my rank up huh?**\n"
explain += "Mee6 only gives you XP once every minute. Spamming will not help to get your rank any faster.\n\n"
explain += "**I still engage with the community through other means. Shouldn't I still be allowed to enter?**\n"
explain += "It depends. If you are not active through chat but active through other means, send a message to the member facilitating "
explain += "this giveaway. Ultimately, since they are giving away the goods, it's their decision. If they agree with you, you can be "
explain += "entered in the giveaway manually!"
await client.send_message(member, explain)
await client.delete_message(message)
return
new_record = Giveaway(discord_id = discord_id)
session.add(new_record)
session.commit()
sent_msg = await client.send_message(message.channel, "{}, you have succesfully entered the giveaway!".format(message.author.mention))
await asyncio.sleep(5)
await client.delete_message(sent_msg)
await client.delete_message(message)
return
else:
await client.send_message(message.author, "You attempted to enter a giveaway, but there isn't one running. If you believe this is in error, please ask in chat or contact the bot owner or a server admin.")
await client.delete_message(message)
return
if message.content.startswith('.roll'):
if has_admin_rights(message.author) or message.author.id in devs:
giveaway_is_running = False
# Check if there are any entries
resultset = session.query(Giveaway)
if resultset.count() <= 0:
await client.send_message(message.channel, "There were no entries (left) so no winner was drawn.")
return
# Fetch all entries as a list
entries = []
server = next(iter(client.servers))
# First, get all the entries from the db
for instance in session.query(Giveaway):
# Lookup the discord member object from the user_id
member = server.get_member(instance.discord_id)
# If the member has left the server, they can no longer be found during lookup and will result in a NoneType
if member is not None:
entries.append(member)
win_chance = 1 / len(entries) * 100
winner = random.choice(entries)
msg = "Congratulations {}! You won the giveaway (win chance {:.2f}%).".format(winner.mention, win_chance)
# Remove the winner from the giveaway table
session.query(Giveaway).filter_by(discord_id = winner.id).delete()
session.commit()
await client.send_message(message.channel, msg)
return
if message.content.startswith('.end'):
if has_admin_rights(message.author) or message.author.id in devs:
session.query(Giveaway).delete()
session.commit()
giveaway_is_running = False
await client.send_message(message.channel, "The current giveaway has ended!")
return
if message.content.startswith('.approve'):
if has_admin_rights(message.author) or message.author.id in devs:
if giveaway_is_running:
member = message.mentions[0]
resultset = session.query(Giveaway).filter_by(discord_id = member.id)
# Don't save a new record if the person already entered
if resultset.count() > 0:
sent_msg = await client.send_message(message.channel, "This user has already entered the giveaway!")
await asyncio.sleep(5)
await client.delete_message(sent_msg)
await client.delete_message(message)
else:
new_record = Giveaway(discord_id = member.id)
session.add(new_record)
session.commit()
sent_msg = await client.send_message(message.channel, "{} has approved {} for the giveaway manually!".format(message.author.mention, member.mention))
await asyncio.sleep(5)
await client.delete_message(message)
return
else:
await client.send_message(message.author, "You attempted to manually approve someone for the giveaway, but I couldn't find a giveaway running at this time.")
return
# POLLS!
if message.content.startswith('.poll'):
if has_admin_rights(message.author) or message.author.id in devs:
split_message = message.content.split()
poll_body = "**POLL TIME BA-BY!** \n"
poll_body += " ".join(split_message[1:])
poll = await client.send_message(message.channel, poll_body)
await client.add_reaction(poll, '\U0001F44D')
await client.add_reaction(poll, '\U0001F44E')
else:
return
# Remove self-assignable role
if message.content.startswith('.iamn'):
author = message.author
split_message = message.content.split()
try:
role_cmd = split_message[1].lower()
if role_cmd in self_roles:
role_to_set = self_roles[role_cmd]
rm = RoleManager(client, message)
role = rm.fetch_role_by_name(role_to_set)
await rm.take_role(role, author)
return
else:
return
except:
return
# Add self-assignable role
if message.content.startswith('.iam'):
author = message.author
split_message = message.content.split()
try:
role_cmd = split_message[1].lower()
if role_cmd in self_roles:
role_to_set = self_roles[role_cmd]
rm = RoleManager(client, message)
role = rm.fetch_role_by_name(role_to_set)
await rm.give_role(role, author)
return
else:
raise
return
except:
return
# Add moderator assignable roles
# i.e. .giverole Role MemberMention MemberMention (as *args)
if message.content.startswith('.giverole'):
author = message.author
rm = RoleManager(client, message)
# Check if message author has access to command
if has_admin_rights(author) or message.author.id in devs:
# Parse the given command arguments.
try:
arguments = message.content.split()
role_txt = arguments[1].lower()
# Find the role, see if it is a mass-assignable role and fetch the role object
if role_txt in mod_roles:
role_name = mod_roles[role_txt]
role = rm.fetch_role_by_name(role_name)
else:
return
# Parse all the members given to the command
member_mentions = arguments[2:]
members = [] # Keep track of the fetched member objects
for mention in member_mentions:
if '!' in mention:
member_id = mention[3:-1]
else:
member_id = mention[2:-1]
member = message.server.get_member(member_id)
members.append(member)
# Hand over all parsed data to the RoleManager
await rm.mass_give_role(role, members)
return
except:
raise
return
else:
return
# Remove moderator assignable role
if message.content.startswith('.takerole'):
author = message.author
rm = RoleManager(client, message)
# Check if message author has access to command
if has_admin_rights(author) or message.author.id in devs:
# Parse the given command arguments.
try:
arguments = message.content.split()
role_txt = arguments[1].lower()
# Find the role, see if it is a mass-assignable role and fetch the role object
if role_txt in mod_roles:
role_name = mod_roles[role_txt]
role = rm.fetch_role_by_name(role_name)
else:
return
# Parse all the members given to the command
member_mentions = arguments[2:]
members = [] # Keep track of the fetched member objects
for mention in member_mentions:
if '!' in mention:
member_id = mention[3:-1]
else:
member_id = mention[2:-1]
member = message.server.get_member(member_id)
members.append(member)
# Hand over all parsed data to the RoleManager
await rm.mass_take_role(role, members)
return
except:
raise
return
else:
return
# Reset the giveaway entries table
if message.content.startswith('.reset'):
pm = PrizeManager(client)
await pm.reset_giveaway_entries(message)
return
# List the entries for the giveaway
if message.content.startswith('.entries'):
pm = PrizeManager(client)
await pm.print_entries(message)
return
# DM the list of giveaway entries
if message.content.startswith('.dmentries'):
pm = PrizeManager(client)
await pm.dm_entries(message)
return
# MAKE SURE THIS STAYS ABOVE .DRAW OR IT WILL NOT FIRE
if message.content.startswith('.draw --preserve'):
pm = PrizeManager(client)
await pm.draw_winner(message, intermediary=True)
return
if message.content.startswith('.draw'):
pm = PrizeManager(client)
await pm.draw_winner(message)
return
if message.content.startswith('.bug'):
report = " ".join(message.content.split()[1:])
br = BugReporter(client)
await br.report_bug(message, report)
return
if message.content.startswith('.denyreporting'):
br = BugReporter(client)
await br.add_abuser(message, abuser_mention=message.content.split()[1])
return
if message.content.startswith('.allowreporting'):
br = BugReporter(client)
await br.remove_abuser(message, abuser_mention=message.content.split()[1])
return
if message.content.startswith('.issue'):
command = message.content[6:]
try:
title, body = command.split("|")
br = BugReporter(client)
await br.create_issue(message, title, body)
except:
return
return
# Send copyright notice
if message.content.startswith('.copyright'):
msg = "All Guildwars-related texts, names and images are © 2015 ArenaNet, "
msg += "LLC. All rights reserved. NCSOFT, the interlocking NC logo, ArenaNet, Guild Wars, "
msg += "Guild Wars Factions, Guild Wars Nightfall, Guild Wars: Eye of the North, Guild Wars 2, "
msg += "Heart of Thorns, and all associated logos and designs are trademarks or registered "
msg += "trademarks of NCSOFT Corporation. All other trademarks are the property of their "
msg += "respective owners."
await client.send_message(message.author, msg)
await client.delete_message(message)
return
if message.content.startswith('.build'):
if has_admin_rights(message.author) or has_build_rights(message.author):
mb = MessageBuilder(client)
await mb.print_stuff_happening()
await client.delete_message(message)
return
# Dm the evoker a list of all token positions in the event modules
if message.content.startswith('.modules'):
mb = MessageBuilder(client)
await mb.dm_modules(message)
return
if message.content.startswith('.delmodule'):
try:
token = message.content.split(" ")[1]
mb = MessageBuilder(client)
await mb.delete_module(message, token)
except:
raise
return
if message.content.startswith('.addmodule'):
if has_admin_rights(message.author):
try:
mb = MessageBuilder(client)
contents = message.content[11:]
commands = contents.split("|")
token = commands[0].strip() # Strip leading and trailing spaces
if not mb.token_in_use(token):
position = commands[1].strip()
image_link = commands[2].strip()
contents = commands[3].strip()
await mb.add_module(message, token, position, image_link, contents)
else:
resp = "Sorry, the token you have given me already exists. "
resp += "Please find your original message below:\n{}".format(message.content)
await client.send_message(message.author, resp)
except:
resp = "I'm sorry, but I could not understand you command. Contact the hoster of this bot if you have any questions. "
resp += "Original message: \n{}".format(message.content)
await client.send_message(message.author, resp)
raise
await client.delete_message(message)
return
if message.content.startswith('.commands'):
file = open("commands.md", "r")
msg = ""
for line in file:
msg += line
await client.send_message(message.author, msg)
await client.delete_message(message)
return
# Sends some bot stats over DM
if message.content.startswith('.stats'):
if message.author.id in settings.REPORT_TO_DEVS or has_admin_rights(message.author):
server = next(iter(client.servers))
msg = "**Bot Version**\n"
msg += "Bot version: {}\n".format(settings.BOT_VERSION)
msg += "Lib version: {}\n\n".format(discord.__version__)
cnt_eu = 0
cnt_na = 0
cnt_no_role = 0
cnt_any = 0
for member in server.members:
valid_roles = 0
for role in member.roles:
if role.name == 'NA':
cnt_na += 1
valid_roles += 1
elif role.name == "EU":
cnt_eu += 1
valid_roles += 1
if valid_roles == 0:
cnt_no_role += 1
else:
cnt_any += valid_roles
total_found = len(server.members)
percent_eu = cnt_eu / total_found * 100 if cnt_eu > 0 else 0
percent_na = cnt_na / total_found * 100 if cnt_na > 0 else 0
percent_none = cnt_no_role / total_found * 100 if cnt_no_role > 0 else 0
msg += "**Region role distribution** (of entire population) \n"
msg += "Total discord members: {}\n".format(total_found)
msg += "EU role percentage: {:.2f}% \n".format(percent_eu)
msg += "NA role percentage: {:.2f}% \n".format(percent_na)
msg += "No role percentage: {:.2f}% \n".format(percent_none)
msg += "\n"
percent_eu = cnt_eu / cnt_any * 100 if cnt_eu > 0 else 0
percent_na = cnt_na / cnt_any * 100 if cnt_na > 0 else 0
msg += "**Region role distribution amongst distributed roles**\n"
msg += "EU role percentage: {:.2f}% ({} members)\n".format(percent_eu, cnt_eu)
msg += "NA role percentage: {:.2f}% ({} members)\n\n".format(percent_na, cnt_na)
msg += "**Assignable roles allowed** \n"
msg += "Self-assignable roles list: \n"
for key in self_roles:
msg += "\t{} => {} \n".format(key, self_roles[key])
msg += "\nGM-assignable roles list: \n"
for key in mod_roles:
msg += "\t{} => {} \n".format(key, mod_roles[key])
msg += "\n"
pm = PrizeManager(client)
sum_tickets = pm.sum_entries()
avg_tickets = pm.avg_entries()
participants = len(pm.get_entries_as_list())
msg += "**Giveaway Stats**\n"
msg += "Giveaway entries distributed this month: {}\n".format(sum_tickets)
msg += "Average giveaway entries/entered member: {:.2f}\n".format(avg_tickets)
msg += "Total giveaway participants: {}\n".format(participants)
await client.send_message(message.author, msg)
await client.delete_message(message)
return
import sys
config = SafeConfigParser()
log = Logger('nn_model.log')
config.read('config.ini')
sys.setrecursionlimit(10000)
hidden_1 = config.getint('NeuralNetConfig', 'HIDDEN_1')
hidden_2 = config.getint('NeuralNetConfig', 'HIDDEN_2')
hidden_3 = config.getint('NeuralNetConfig', 'HIDDEN_3')
hidden_4 = config.getint('NeuralNetConfig', 'HIDDEN_4')
rstate = config.getint('NeuralNetConfig', 'RSTATE')
activationFunc = config.get('NeuralNetConfig', 'ACTIVATION')
m_iter = config.getint('NeuralNetConfig', 'MAX_ITER')
l_rate = config.getfloat('NeuralNetConfig', 'LEARNING_RATE')
ver_bose = config.getboolean('NeuralNetConfig', 'VERBOSE')
t_size = config.getfloat('Misc', 'TEST_SIZE_1')
digit_num = config.getint('Misc', 'DIGIT_NUM')
class NN_model:
def __init__(self):
log.info('Initializing Model.')
self.current_report = None
log.info('Initializing Neural Network.')
self.model = MLPClassifier(solver='adam', \
alpha=config.getfloat('NeuralNetConfig','ALPHA'),\
hidden_layer_sizes=(hidden_1, hidden_2, hidden_3, hidden_4), \
random_state=rstate, activation=activationFunc, max_iter=m_iter, learning_rate_init=l_rate, verbose=ver_bose)
def main():
global business_logic, auth_key
config = SafeConfigParser()
dirs = ('.', '/etc', '/usr/local/etc')
if not config.read([os.path.join(dir, config_file) for dir in dirs]):
sys.exit('Could not find {} in {}'.format(config_file, dirs))
try:
logfile = config.get('logging', 'file')
rotating = config.getboolean('logging', 'rotate', fallback=False)
if rotating:
max_size = config.get('logging', 'max_size', fallback=1048576)
backup_count = config.get('logging', 'backup_count', fallback=5)
handler = logbook.RotatingFileHandler(logfile, max_size=max_size,
backup_count=backup_count)
else:
handler = logbook.FileHandler(logfile)
handler.push_application()
except:
pass
try:
kwargs = dict(config.items('mongodb'))
except NoSectionError:
sys.exit('No "mongodb" section in config file')
args = []
for arg in ('hosts', 'database', 'username', 'password'):
try:
args.append(config.get('mongodb', arg))
except NoOptionError:
sys.exit('No "{}" setting in "mongodb" section of config file'.
format(arg))
kwargs.pop(arg)
store = MongoStore(*args, **kwargs)
try:
email_sender = config.get('email', 'sender')
except NoSectionError:
sys.exit('No "email" section in config file')
except NoOptionError:
sys.exit('No "sender" setting in "email" section of config file')
business_logic = BusinessLogic(store, email_sender)
try:
listen_port = int(config.get('wsgi', 'port'))
log.info('Binding to port {}'.format(listen_port))
except:
listen_port = 80
log.info('Binding to default port {}'.format(listen_port))
try:
auth_key = config.get('wsgi', 'auth_key')
log.info('Server authentication enabled')
except:
log.warning('Server authentication DISABLED')
httpd = make_server('', listen_port, application,
handler_class=LogbookWSGIRequestHandler)
business_logic.schedule_next_deadline()
httpd.serve_forever()
def run(ini_file='TOPKAPI.ini',
verbose=False, quiet=False,
parallel_exec=True, nworkers=int(mp.cpu_count()-1)):
"""Run the model.
Parameters
----------
ini_file : str
The name of the PyTOPKAPI initialization file. This file describes the
locations of the parameter files and model setup options. Default is to
use a file named `TOPKAPI.ini` in the current directory.
verbose : bool
Prints runtime information [default False - don't display runtime
info]. Is independent of the `quiet` keyword argument.
quiet : bool
Toggles whether to display an informational banner at runtime [default
False - display banner]. Is independent of the `verbose` keyword
argument.
nworkers : int
Number of worker processes to spawn for solving each cell's time-series
in parallel. Default is one fewer than CPU count reported by
multiprocessing.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getint('numerical_options', 'solve_s')
solve_o = config.getint('numerical_options', 'solve_o')
solve_c = config.getint('numerical_options', 'solve_c')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
if verbose:
print('Read the forcing data')
#~~~~Rainfall
h5_rain = h5py.File(file_rain)
dset_name = '/{}/rainfall'.format(group_name)
rainfall_forcing = h5_rain[dset_name][...]
h5_rain.close()
#~~~~ETr - Reference crop ET
h5_ET = h5py.File(file_ET)
dset_name = '/{}/ETr'.format(group_name)
ETr_forcing = h5_ET[dset_name][...]
#~~~~ETo - Open water potential evap.
dset_name = '/{}/ETo'.format(group_name)
ET0_forcing = h5_ET[dset_name][...]
h5_ET.close()
#~~~~external_flow flows
if external_flow:
external_flow_records = np.loadtxt(file_Qexternal_flow)[:, 5]
else:
external_flow_records = None
##============================##
## Pretreatment of input data ##
##============================##
if verbose:
print('Pretreatment of input data')
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, channel_flag, \
Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L, Ks, \
ar_theta_r, ar_theta_s, \
ar_n_o, ar_n_c, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
Kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
node_hierarchy = pm.compute_node_hierarchy(ar_cell_label, ar_cell_down)
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L*fac_L
Ks = Ks*fac_Ks
ar_n_o = ar_n_o*fac_n_o
ar_n_c = ar_n_c*fac_n_c
if verbose:
print('Max L=', max(ar_L))
print('Max Ks=', max(Ks))
print('Max n_o=', max(ar_n_o))
print('Max n_c=', max(ar_n_c))
#~~~~Computation of model parameters from physical parameters
Vsm, b_s, b_o, \
W, b_c = pm.compute_cell_param(X, Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
channel_flag, ar_tan_beta,
ar_tan_beta_channel, ar_L,
Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow,
ar_coorx,
ar_coory,
channel_flag)
if verbose:
print('external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')')
else:
cell_external_flow = None
#~~~~Number of simulation time steps
nb_time_step = rainfall_forcing.shape[0]
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
if verbose:
print('Initialize from simulation file')
h5file_in = h5py.File(file_out)
Vs_t0 = h5file_in['/Soil/V_s'][-1, :]
Vc_t0 = h5file_in['/Channel/V_c'][-1, :]
Vo_t0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
if verbose:
print('Initialize from parameters')
Vs_t0 = fl.initial_volume_soil(ar_pVs_t0, Vsm)
Vo_t0 = ar_Vo_t0
Vc_t0 = fl.initial_volume_channel(ar_Qc_t0, W, X, ar_n_c)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file, dset_Vs, dset_Vo, dset_Vc, \
dset_Qs_out, dset_Qo_out, dset_Qc_out, \
dset_Q_down, dset_ET_out, dset_Ec_out \
= ut.open_simulation_file(file_out, fmode,
Vs_t0, Vo_t0, Vc_t0, no_data,
nb_cell, nb_time_step,
append_output, first_run)
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
if not quiet:
ut.show_banner(ini_file, nb_cell, nb_time_step)
progress_desc = 'Simulation'
else:
progress_desc = 'PyTOPKAPI v{}'.format(pytopkapi.__version__)
# prepare parameter dict
exec_params = {'nb_cell': nb_cell,
'nb_time_step': nb_time_step,
'progress_desc': progress_desc,
'Dt': Dt,
'rainfall_forcing': rainfall_forcing,
'ETr_forcing': ETr_forcing,
'ET0_forcing': ET0_forcing,
'psi_b': psi_b,
'lamda': lamda,
'eff_theta': eff_theta,
'Ks': Ks,
'X': X,
'b_s': b_s,
'b_o': b_o,
'b_c': b_c,
'alpha_s': alpha_s,
'alpha_o': alpha_o,
'alpha_c': alpha_c,
'Vs_t0': Vs_t0,
'Vo_t0': Vo_t0,
'Vc_t0': Vc_t0,
'Vsm': Vsm,
'dset_Vs': dset_Vs,
'dset_Vo': dset_Vo,
'dset_Vc': dset_Vc,
'dset_Qs_out': dset_Qs_out,
'dset_Qo_out': dset_Qo_out,
'dset_Qc_out': dset_Qc_out,
'dset_Q_down': dset_Q_down,
'dset_ET_out': dset_ET_out,
'dset_Ec_out': dset_Ec_out,
'solve_s': solve_s,
'solve_o': solve_o,
'solve_c': solve_c,
'channel_flag': channel_flag,
'W': W,
'Xc': Xc,
'Kc': Kc,
'cell_external_flow': cell_external_flow,
'external_flow_records': external_flow_records,
'node_hierarchy': node_hierarchy,
'li_cell_up': li_cell_up,
'nworkers': nworkers}
if not parallel_exec:
# Serial execution. Solve by timestep in a single process.
# Outer loop timesteps - inner loop cells
_serial_execute(exec_params)
else:
# Parallel execution. Solve by cell using multiple processes.
# Outer loop cells - inner loop timesteps
_parallel_execute(exec_params)
h5file.close()
import pygeoutil.util as util
# Parse config file
parser = SafeConfigParser()
parser.read('config_rotations.txt')
# Get directory path (3 levels up is the parent directory)
dir_prj = str(Path(__file__).parents[3])
FAO_START_YR = parser.getint('PARAMETERS', 'FAO_START_YR') # Starting year of FAO data
FAO_END_YR = parser.getint('PARAMETERS', 'FAO_END_YR') # Ending year of FAO data
TAG = parser.get('PROJECT', 'TAG')
FAO_FILE = parser.get('PROJECT', 'fao_data')
FAO_SHEET = parser.get('PROJECT', 'fao_sheet')
PROJ_NAME = parser.get('PROJECT', 'project_name')
DO_PARALLEL = parser.getboolean('PARAMETERS', 'DO_PARALLEL') # Use multiprocessing or not?
NUM_LATS = 180.0
NUM_LONS = 360.0
PLOT_CNTRS = 10
PLOT_CROPS = 10
DPI = 300
CFT_FRAC_YR = parser.getint('PARAMETERS', 'CFT_FRAC_YR')
GLM_STRT_YR = parser.getint('PARAMETERS', 'GLM_STRT_YR')
GLM_END_YR = parser.getint('PARAMETERS', 'GLM_END_YR')
FILL_ZEROS = parser.getboolean('PARAMETERS', 'FILL_ZEROS')
TEST_CFT = parser.getboolean('PROJECT', 'TEST_CFT')
# Directories
data_dir = dir_prj + os.sep + parser.get('PATHS', 'data_dir') + os.sep
out_dir = dir_prj + os.sep + parser.get('PATHS', 'out_dir') + os.sep + PROJ_NAME + os.sep
log_dir = out_dir + os.sep + 'Logs'
# Parse config file
parser = SafeConfigParser()
parser.read('../config_IAM.txt')
# Get directory path (3 levels up is the parent directory)
dir_prj = str(Path(__file__).parents[3])
# Common values
dict_conts = {0: 'Antartica', 1: 'North_America', 2: 'South_America', 3: 'Europe', 4: 'Asia', 5: 'Africa',
6: 'Australia'}
#####################################################################################
# Tags to be modified by user
#####################################################################################
do_email = parser.getboolean('CONTROL', 'do_email')
email_list = ast.literal_eval(parser.get('CONTROL', 'email_list'))
SHFT_MAP = parser.get('CONTROL', 'SHFT_MAP') # Use Andreas or Butler?
MOVIE_SEP = 10
do_LUH1 = parser.getboolean('CONTROL', 'do_LUH1')
PLOT_HYDE = parser.getboolean('CONTROL', 'PLOT_HYDE')
PREPROCESS_GCAM = parser.getboolean('CONTROL', 'PREPROCESS_GCAM')
PREPROCESS_IMAG = parser.getboolean('CONTROL', 'PREPROCESS_IMAG')
CONVERT_WH = parser.getboolean('CONTROL', 'CONVERT_WH') # Convert WH information from AEZ to country level
ending_diag_cols = ast.literal_eval(parser.get('CONTROL', 'ending_diag_cols'))
MATURITY_AGE = parser.getfloat('CONTROL', 'MATURITY_AGE')
# Directories
input_dir = dir_prj + os.sep + parser.get('GLM', 'path_input')
gcam_dir = input_dir + os.sep + parser.get('PATHS', 'gcam_dir') + os.sep
out_dir = dir_prj + os.sep + parser.get('PATHS', 'out_dir') + os.sep + parser.get('PROJECT', 'project_name') + os.sep
'update_startup_delay': '30',
'autocheck_updates': '0',
'update_notify': True,
'update_showicon': True
}
CONFIG = SafeConfigParser(CONF_DEFAULTS)
if not CONFIG.has_section('yumex'):
CONFIG.add_section('yumex')
if os.path.exists(CONF_FILE):
CONFIG.read(CONF_FILE)
TIMER_STARTUP_DELAY = CONFIG.getint('yumex', 'update_startup_delay')
UPDATE_INTERVAL = CONFIG.getint('yumex', 'update_interval')
AUTOCHECK_UPDATE = CONFIG.getboolean('yumex', 'autocheck_updates')
NOTIFY = CONFIG.getboolean('yumex', 'update_notify')
SHOWICON = CONFIG.getboolean('yumex', 'update_showicon')
def check_pid(pid):
""" Check For the existence of a unix pid. """
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
class UpdateTimestamp:
class Buildozer(object):
ERROR = 0
INFO = 1
DEBUG = 2
standard_cmds = ('distclean', 'update', 'debug', 'release', 'deploy',
'run', 'serve')
def __init__(self, filename='buildozer.spec', target=None):
super(Buildozer, self).__init__()
self.log_level = 2
self.environ = {}
self.specfilename = filename
self.state = None
self.build_id = None
self.config_profile = ''
self.config = SafeConfigParser(allow_no_value=True)
self.config.optionxform = lambda value: value
self.config.getlist = self._get_config_list
self.config.getlistvalues = self._get_config_list_values
self.config.getdefault = self._get_config_default
self.config.getbooldefault = self._get_config_bool
self.config.getrawdefault = self._get_config_raw_default
if exists(filename):
try:
self.config.read(filename, "utf-8")
except TypeError: # python 2 has no second arg here
self.config.read(filename)
self.check_configuration_tokens()
# Check all section/tokens for env vars, and replace the
# config value if a suitable env var exists.
set_config_from_envs(self.config)
try:
self.log_level = int(
self.config.getdefault('buildozer', 'log_level', '2'))
except Exception:
pass
self.user_bin_dir = self.config.getdefault('buildozer', 'bin_dir',
None)
if self.user_bin_dir:
self.user_bin_dir = realpath(join(self.root_dir,
self.user_bin_dir))
self.targetname = None
self.target = None
if target:
self.set_target(target)
def set_target(self, target):
'''Set the target to use (one of buildozer.targets, such as "android")
'''
self.targetname = target
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
self.target = m.get_target(self)
self.check_build_layout()
self.check_configuration_tokens()
def prepare_for_build(self):
'''Prepare the build.
'''
assert (self.target is not None)
if hasattr(self.target, '_build_prepared'):
return
self.info('Preparing build')
self.info('Check requirements for {0}'.format(self.targetname))
self.target.check_requirements()
self.info('Install platform')
self.target.install_platform()
self.info('Check application requirements')
self.check_application_requirements()
self.info('Check garden requirements')
self.check_garden_requirements()
self.info('Compile platform')
self.target.compile_platform()
# flag to prevent multiple build
self.target._build_prepared = True
def build(self):
'''Do the build.
The target can set build_mode to 'release' or 'debug' before calling
this method.
(:meth:`prepare_for_build` must have been call before.)
'''
assert (self.target is not None)
assert (hasattr(self.target, '_build_prepared'))
if hasattr(self.target, '_build_done'):
return
# increment the build number
self.build_id = int(self.state.get('cache.build_id', '0')) + 1
self.state['cache.build_id'] = str(self.build_id)
self.info('Build the application #{}'.format(self.build_id))
self.build_application()
self.info('Package the application')
self.target.build_package()
# flag to prevent multiple build
self.target._build_done = True
#
# Log functions
#
def log(self, level, msg):
if level > self.log_level:
return
if USE_COLOR:
color = COLOR_SEQ(LOG_LEVELS_C[level])
print(''.join((RESET_SEQ, color, '# ', msg, RESET_SEQ)))
else:
print('{} {}'.format(LOG_LEVELS_T[level], msg))
def debug(self, msg):
self.log(self.DEBUG, msg)
def log_env(self, level, env):
"""dump env into debug logger in readable format"""
self.log(level, "ENVIRONMENT:")
for k, v in env.items():
self.log(level, " {} = {}".format(k, pformat(v)))
def info(self, msg):
self.log(self.INFO, msg)
def error(self, msg):
self.log(self.ERROR, msg)
#
# Internal check methods
#
def checkbin(self, msg, fn):
self.debug('Search for {0}'.format(msg))
if exists(fn):
return realpath(fn)
for dn in environ['PATH'].split(':'):
rfn = realpath(join(dn, fn))
if exists(rfn):
self.debug(' -> found at {0}'.format(rfn))
return rfn
self.error('{} not found, please install it.'.format(msg))
exit(1)
def cmd(self, command, **kwargs):
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('stdout', PIPE)
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', True)
kwargs.setdefault('show_output', self.log_level > 1)
show_output = kwargs.pop('show_output')
get_stdout = kwargs.pop('get_stdout', False)
get_stderr = kwargs.pop('get_stderr', False)
break_on_error = kwargs.pop('break_on_error', True)
sensible = kwargs.pop('sensible', False)
if not sensible:
self.debug('Run {0!r}'.format(command))
else:
if type(command) in (list, tuple):
self.debug('Run {0!r} ...'.format(command[0]))
else:
self.debug('Run {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
# open the process
if sys.platform == 'win32':
kwargs.pop('close_fds', None)
process = Popen(command, **kwargs)
# prepare fds
fd_stdout = process.stdout.fileno()
fd_stderr = process.stderr.fileno()
if fcntl:
fcntl.fcntl(fd_stdout, fcntl.F_SETFL,
fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL,
fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
ret_stdout = [] if get_stdout else None
ret_stderr = [] if get_stderr else None
while True:
try:
readx = select.select([fd_stdout, fd_stderr], [], [])[0]
except select.error:
break
if fd_stdout in readx:
chunk = process.stdout.read()
if not chunk:
break
if get_stdout:
ret_stdout.append(chunk)
if show_output:
if IS_PY3:
stdout.write(chunk.decode('utf-8', 'replace'))
else:
stdout.write(chunk)
if fd_stderr in readx:
chunk = process.stderr.read()
if not chunk:
break
if get_stderr:
ret_stderr.append(chunk)
if show_output:
if IS_PY3:
stderr.write(chunk.decode('utf-8', 'replace'))
else:
stderr.write(chunk)
stdout.flush()
stderr.flush()
process.communicate()
if process.returncode != 0 and break_on_error:
self.error('Command failed: {0}'.format(command))
self.log_env(self.ERROR, kwargs['env'])
self.error('')
self.error('Buildozer failed to execute the last command')
if self.log_level <= self.INFO:
self.error(
'If the error is not obvious, please raise the log_level to 2'
)
self.error('and retry the latest command.')
else:
self.error(
'The error might be hidden in the log above this error')
self.error(
'Please read the full log, and search for it before')
self.error('raising an issue with buildozer itself.')
self.error(
'In case of a bug report, please add a full log with log_level = 2'
)
raise BuildozerCommandException()
if ret_stdout:
ret_stdout = b''.join(ret_stdout)
if ret_stderr:
ret_stderr = b''.join(ret_stderr)
return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None,
ret_stderr.decode('utf-8') if ret_stderr else None,
process.returncode)
def cmd_expect(self, command, **kwargs):
from pexpect import spawnu
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('show_output', self.log_level > 1)
sensible = kwargs.pop('sensible', False)
show_output = kwargs.pop('show_output')
if show_output:
if IS_PY3:
kwargs['logfile'] = codecs.getwriter('utf8')(stdout.buffer)
else:
kwargs['logfile'] = codecs.getwriter('utf8')(stdout)
if not sensible:
self.debug('Run (expect) {0!r}'.format(command))
else:
self.debug('Run (expect) {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
return spawnu(command, **kwargs)
def check_configuration_tokens(self):
'''Ensure the spec file is 'correct'.
'''
self.info('Check configuration tokens')
self.migrate_configuration_tokens()
get = self.config.getdefault
errors = []
adderror = errors.append
if not get('app', 'title', ''):
adderror('[app] "title" is missing')
if not get('app', 'source.dir', ''):
adderror('[app] "source.dir" is missing')
package_name = get('app', 'package.name', '')
if not package_name:
adderror('[app] "package.name" is missing')
elif package_name[0] in map(str, range(10)):
adderror('[app] "package.name" may not start with a number.')
version = get('app', 'version', '')
version_regex = get('app', 'version.regex', '')
if not version and not version_regex:
adderror('[app] One of "version" or "version.regex" must be set')
if version and version_regex:
adderror('[app] Conflict between "version" and "version.regex"'
', only one can be used.')
if version_regex and not get('app', 'version.filename', ''):
adderror('[app] "version.filename" is missing'
', required by "version.regex"')
orientation = get('app', 'orientation', 'landscape')
if orientation not in ('landscape', 'portrait', 'all',
'sensorLandscape'):
adderror('[app] "orientation" have an invalid value')
if errors:
self.error('{0} error(s) found in the buildozer.spec'.format(
len(errors)))
for error in errors:
print(error)
exit(1)
def migrate_configuration_tokens(self):
config = self.config
if config.has_section("app"):
migration = (("android.p4a_dir", "p4a.source_dir"),
("android.p4a_whitelist", "android.whitelist"),
("android.bootstrap",
"p4a.bootstrap"), ("android.branch", "p4a.branch"),
("android.p4a_whitelist_src",
"android.whitelist_src"),
("android.p4a_blacklist_src",
"android.blacklist_src"))
for entry_old, entry_new in migration:
if not config.has_option("app", entry_old):
continue
value = config.get("app", entry_old)
config.set("app", entry_new, value)
config.remove_option("app", entry_old)
self.error(
"In section [app]: {} is deprecated, rename to {}!".format(
entry_old, entry_new))
def check_build_layout(self):
'''Ensure the build (local and global) directory layout and files are
ready.
'''
self.info('Ensure build layout')
if not exists(self.specfilename):
print('No {0} found in the current directory. Abandon.'.format(
self.specfilename))
exit(1)
# create global dir
self.mkdir(self.global_buildozer_dir)
self.mkdir(self.global_cache_dir)
# create local .buildozer/ dir
self.mkdir(self.buildozer_dir)
# create local bin/ dir
self.mkdir(self.bin_dir)
self.mkdir(self.applibs_dir)
self.state = JsonStore(join(self.buildozer_dir, 'state.db'))
target = self.targetname
if target:
self.mkdir(join(self.global_platform_dir, target, 'platform'))
self.mkdir(join(self.buildozer_dir, target, 'platform'))
self.mkdir(join(self.buildozer_dir, target, 'app'))
def check_application_requirements(self):
'''Ensure the application requirements are all available and ready to be
packaged as well.
'''
requirements = self.config.getlist('app', 'requirements', '')
target_available_packages = self.target.get_available_packages()
if target_available_packages is True:
# target handles all packages!
return
# remove all the requirements that the target can compile
onlyname = lambda x: x.split('==')[0] # noqa: E731
requirements = [
x for x in requirements
if onlyname(x) not in target_available_packages
]
if requirements and hasattr(sys, 'real_prefix'):
e = self.error
e('virtualenv is needed to install pure-Python modules, but')
e('virtualenv does not support nesting, and you are running')
e('buildozer in one. Please run buildozer outside of a')
e('virtualenv instead.')
exit(1)
# did we already installed the libs ?
if (exists(self.applibs_dir)
and self.state.get('cache.applibs', '') == requirements):
self.debug('Application requirements already installed, pass')
return
# recreate applibs
self.rmdir(self.applibs_dir)
self.mkdir(self.applibs_dir)
# ok now check the availability of all requirements
for requirement in requirements:
self._install_application_requirement(requirement)
# everything goes as expected, save this state!
self.state['cache.applibs'] = requirements
def _install_application_requirement(self, module):
self._ensure_virtualenv()
self.debug('Install requirement {} in virtualenv'.format(module))
self.cmd('pip install --target={} {}'.format(self.applibs_dir, module),
env=self.env_venv,
cwd=self.buildozer_dir)
def check_garden_requirements(self):
'''Ensure required garden packages are available to be included.
'''
garden_requirements = self.config.getlist('app', 'garden_requirements',
'')
# have we installed the garden packages?
if exists(self.gardenlibs_dir) and \
self.state.get('cache.gardenlibs', '') == garden_requirements:
self.debug('Garden requirements already installed, pass')
return
# we're going to reinstall all the garden libs.
self.rmdir(self.gardenlibs_dir)
# but if we don't have requirements, or if the user removed everything,
# don't do anything.
if not garden_requirements:
self.state['cache.gardenlibs'] = garden_requirements
return
self._ensure_virtualenv()
self.cmd('pip install Kivy-Garden==0.1.1', env=self.env_venv)
# recreate gardenlibs
self.mkdir(self.gardenlibs_dir)
for requirement in garden_requirements:
self._install_garden_package(requirement)
# save gardenlibs state
self.state['cache.gardenlibs'] = garden_requirements
def _install_garden_package(self, package):
self._ensure_virtualenv()
self.debug(
'Install garden package {} in buildozer_dir'.format(package))
self.cmd('garden install --app {}'.format(package),
env=self.env_venv,
cwd=self.buildozer_dir)
def _ensure_virtualenv(self):
if hasattr(self, 'venv'):
return
self.venv = join(self.buildozer_dir, 'venv')
if not self.file_exists(self.venv):
self.cmd('virtualenv --python=python2.7 ./venv',
cwd=self.buildozer_dir)
# read virtualenv output and parse it
output = self.cmd('bash -c "source venv/bin/activate && env"',
get_stdout=True,
cwd=self.buildozer_dir)
self.env_venv = copy(self.environ)
for line in output[0].splitlines():
args = line.split('=', 1)
if len(args) != 2:
continue
key, value = args
if key in ('VIRTUAL_ENV', 'PATH'):
self.env_venv[key] = value
if 'PYTHONHOME' in self.env_venv:
del self.env_venv['PYTHONHOME']
# ensure any sort of compilation will fail
self.env_venv['CC'] = '/bin/false'
self.env_venv['CXX'] = '/bin/false'
def mkdir(self, dn):
if exists(dn):
return
self.debug('Create directory {0}'.format(dn))
makedirs(dn)
def rmdir(self, dn):
if not exists(dn):
return
self.debug('Remove directory and subdirectory {}'.format(dn))
rmtree(dn)
def file_matches(self, patterns):
from glob import glob
result = []
for pattern in patterns:
matches = glob(expanduser(pattern.strip()))
result.extend(matches)
return result
def file_exists(self, *args):
return exists(join(*args))
def file_rename(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Rename {0} to {1}'.format(source, target))
if not os.path.isdir(os.path.dirname(target)):
self.error(('Rename {0} to {1} fails because {2} is not a '
'directory').format(source, target, target))
move(source, target)
def file_copy(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Copy {0} to {1}'.format(source, target))
copyfile(source, target)
def file_extract(self, archive, cwd=None):
if archive.endswith('.tgz') or archive.endswith('.tar.gz'):
# XXX tarfile doesn't work for NDK-r8c :(
#tf = tarfile.open(archive, 'r:*')
#tf.extractall(path=cwd)
#tf.close()
self.cmd('tar xzf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.tbz2') or archive.endswith('.tar.bz2'):
# XXX same as before
self.cmd('tar xjf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.bin'):
# To process the bin files for linux and darwin systems
self.cmd('chmod a+x {0}'.format(archive), cwd=cwd)
self.cmd('./{0}'.format(archive), cwd=cwd)
return
if archive.endswith('.zip'):
self.cmd('unzip -q {}'.format(join(cwd, archive)), cwd=cwd)
return
raise Exception('Unhandled extraction for type {0}'.format(archive))
def file_copytree(self, src, dest):
print('copy {} to {}'.format(src, dest))
if os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
for f in files:
self.file_copytree(os.path.join(src, f), os.path.join(dest, f))
else:
copyfile(src, dest)
def clean_platform(self):
self.info('Clean the platform build directory')
if not exists(self.platform_dir):
return
rmtree(self.platform_dir)
def download(self, url, filename, cwd=None):
def report_hook(index, blksize, size):
if size <= 0:
progression = '{0} bytes'.format(index * blksize)
else:
progression = '{0:.2f}%'.format(index * blksize * 100. /
float(size))
if "CI" not in environ:
stdout.write('- Download {}\r'.format(progression))
stdout.flush()
url = url + filename
if cwd:
filename = join(cwd, filename)
if self.file_exists(filename):
unlink(filename)
self.debug('Downloading {0}'.format(url))
urlretrieve(url, filename, report_hook)
return filename
def get_version(self):
c = self.config
has_version = c.has_option('app', 'version')
has_regex = c.has_option('app', 'version.regex')
has_filename = c.has_option('app', 'version.filename')
# version number specified
if has_version:
if has_regex or has_filename:
raise Exception(
'version.regex and version.filename conflict with version')
return c.get('app', 'version')
# search by regex
if has_regex or has_filename:
if has_regex and not has_filename:
raise Exception('version.filename is missing')
if has_filename and not has_regex:
raise Exception('version.regex is missing')
fn = c.get('app', 'version.filename')
with open(fn) as fd:
data = fd.read()
regex = c.get('app', 'version.regex')
match = search(regex, data)
if not match:
raise Exception('Unable to find capture version in {0}\n'
' (looking for `{1}`)'.format(fn, regex))
version = match.groups()[0]
self.debug('Captured version: {0}'.format(version))
return version
raise Exception('Missing version or version.regex + version.filename')
def build_application(self):
self._copy_application_sources()
self._copy_application_libs()
self._copy_garden_libs()
self._add_sitecustomize()
def _copy_application_sources(self):
# XXX clean the inclusion/exclusion algo.
source_dir = realpath(self.config.getdefault('app', 'source.dir', '.'))
include_exts = self.config.getlist('app', 'source.include_exts', '')
exclude_exts = self.config.getlist('app', 'source.exclude_exts', '')
exclude_dirs = self.config.getlist('app', 'source.exclude_dirs', '')
exclude_patterns = self.config.getlist('app',
'source.exclude_patterns', '')
include_patterns = self.config.getlist('app',
'source.include_patterns', '')
app_dir = self.app_dir
self.debug('Copy application source from {}'.format(source_dir))
rmtree(self.app_dir)
for root, dirs, files in walk(source_dir, followlinks=True):
# avoid hidden directory
if True in [x.startswith('.') for x in root.split(sep)]:
continue
# need to have sort-of normalization. Let's say you want to exclude
# image directory but not images, the filtered_root must have a / at
# the end, same for the exclude_dir. And then we can safely compare
filtered_root = root[len(source_dir) + 1:].lower()
if filtered_root:
filtered_root += '/'
# manual exclude_dirs approach
is_excluded = False
for exclude_dir in exclude_dirs:
if exclude_dir[-1] != '/':
exclude_dir += '/'
if filtered_root.startswith(exclude_dir.lower()):
is_excluded = True
break
# pattern matching
if not is_excluded:
# match pattern if not ruled out by exclude_dirs
for pattern in exclude_patterns:
if fnmatch(filtered_root, pattern):
is_excluded = True
break
for pattern in include_patterns:
if fnmatch(filtered_root, pattern):
is_excluded = False
break
if is_excluded:
continue
for fn in files:
# avoid hidden files
if fn.startswith('.'):
continue
# pattern matching
is_excluded = False
dfn = fn.lower()
if filtered_root:
dfn = join(filtered_root, fn)
for pattern in exclude_patterns:
if fnmatch(dfn, pattern):
is_excluded = True
break
for pattern in include_patterns:
if fnmatch(dfn, pattern):
is_excluded = False
break
if is_excluded:
continue
# filter based on the extension
# TODO more filters
basename, ext = splitext(fn)
if ext:
ext = ext[1:]
if include_exts and ext not in include_exts:
continue
if exclude_exts and ext in exclude_exts:
continue
sfn = join(root, fn)
rfn = realpath(join(app_dir, root[len(source_dir) + 1:], fn))
# ensure the directory exists
dfn = dirname(rfn)
self.mkdir(dfn)
# copy!
self.debug('Copy {0}'.format(sfn))
copyfile(sfn, rfn)
def _copy_application_libs(self):
# copy also the libs
copytree(self.applibs_dir, join(self.app_dir, '_applibs'))
def _copy_garden_libs(self):
if exists(self.gardenlibs_dir):
copytree(self.gardenlibs_dir, join(self.app_dir, 'libs'))
def _add_sitecustomize(self):
copyfile(join(dirname(__file__), 'sitecustomize.py'),
join(self.app_dir, 'sitecustomize.py'))
main_py = join(self.app_dir, 'service', 'main.py')
if not self.file_exists(main_py):
#self.error('Unable to patch main_py to add applibs directory.')
return
header = (b'import sys, os; '
b'sys.path = [os.path.join(os.getcwd(),'
b'"..", "_applibs")] + sys.path\n')
with open(main_py, 'rb') as fd:
data = fd.read()
data = header + data
with open(main_py, 'wb') as fd:
fd.write(data)
self.info('Patched service/main.py to include applibs')
def namify(self, name):
'''Return a "valid" name from a name with lot of invalid chars
(allowed characters: a-z, A-Z, 0-9, -, _)
'''
return re.sub('[^a-zA-Z0-9_\-]', '_', name)
@property
def root_dir(self):
return realpath(dirname(self.specfilename))
@property
def user_build_dir(self):
"""The user-provided build dir, if any."""
# Check for a user-provided build dir
# Check the (deprecated) builddir token, for backwards compatibility
build_dir = self.config.getdefault('buildozer', 'builddir', None)
if build_dir is not None:
# for backwards compatibility, append .buildozer to builddir
build_dir = join(build_dir, '.buildozer')
build_dir = self.config.getdefault('buildozer', 'build_dir', build_dir)
if build_dir is not None:
build_dir = realpath(join(self.root_dir, build_dir))
return build_dir
@property
def buildozer_dir(self):
'''The directory in which to run the app build.'''
if self.user_build_dir is not None:
return self.user_build_dir
return join(self.root_dir, '.buildozer')
@property
def bin_dir(self):
if self.user_bin_dir:
return self.user_bin_dir
return join(self.root_dir, 'bin')
@property
def platform_dir(self):
return join(self.buildozer_dir, self.targetname, 'platform')
@property
def app_dir(self):
return join(self.buildozer_dir, self.targetname, 'app')
@property
def applibs_dir(self):
return join(self.buildozer_dir, 'applibs')
@property
def gardenlibs_dir(self):
return join(self.buildozer_dir, 'libs')
@property
def global_buildozer_dir(self):
return join(expanduser('~'), '.buildozer')
@property
def global_platform_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'platform')
@property
def global_packages_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'packages')
@property
def global_cache_dir(self):
return join(self.global_buildozer_dir, 'cache')
@property
def package_full_name(self):
package_name = self.config.getdefault('app', 'package.name', '')
package_domain = self.config.getdefault('app', 'package.domain', '')
if package_domain == '':
return package_name
return '{}.{}'.format(package_domain, package_name)
#
# command line invocation
#
def targets(self):
for fn in listdir(join(dirname(__file__), 'targets')):
if fn.startswith('.') or fn.startswith('__'):
continue
if not fn.endswith('.py'):
continue
target = fn[:-3]
try:
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
yield target, m
except NotImplementedError:
pass
except:
raise
pass
def usage(self):
print('Usage:')
print(
' buildozer [--profile <name>] [--verbose] [target] <command>...'
)
print(' buildozer --version')
print('')
print('Available targets:')
targets = list(self.targets())
for target, m in targets:
try:
doc = m.__doc__.strip().splitlines()[0].strip()
except Exception:
doc = '<no description>'
print(' {0:<18} {1}'.format(target, doc))
print('')
print('Global commands (without target):')
cmds = [x for x in dir(self) if x.startswith('cmd_')]
for cmd in cmds:
name = cmd[4:]
meth = getattr(self, cmd)
if not meth.__doc__:
continue
doc = [x for x in meth.__doc__.strip().splitlines()][0].strip()
print(' {0:<18} {1}'.format(name, doc))
print('')
print('Target commands:')
print(' clean Clean the target environment')
print(' update Update the target dependencies')
print(' debug Build the application in debug mode')
print(' release Build the application in release mode')
print(' deploy Deploy the application on the device')
print(' run Run the application on the device')
print(' serve Serve the bin directory via SimpleHTTPServer')
for target, m in targets:
mt = m.get_target(self)
commands = mt.get_custom_commands()
if not commands:
continue
print('')
print('Target "{0}" commands:'.format(target))
for command, doc in commands:
if not doc:
continue
doc = textwrap.fill(textwrap.dedent(doc).strip(),
59,
subsequent_indent=' ' * 21)
print(' {0:<18} {1}'.format(command, doc))
print('')
def run_default(self):
self.check_build_layout()
if 'buildozer:defaultcommand' not in self.state:
print('No default command set.')
print('Use "buildozer setdefault <command args...>"')
print('Use "buildozer help" for a list of all commands"')
exit(1)
cmd = self.state['buildozer:defaultcommand']
self.run_command(cmd)
def run_command(self, args):
while args:
if not args[0].startswith('-'):
break
arg = args.pop(0)
if arg in ('-v', '--verbose'):
self.log_level = 2
elif arg in ('-h', '--help'):
self.usage()
exit(0)
elif arg in ('-p', '--profile'):
self.config_profile = args.pop(0)
elif arg == '--version':
print('Buildozer {0}'.format(__version__))
exit(0)
self._merge_config_profile()
self.check_root()
if not args:
self.run_default()
return
command, args = args[0], args[1:]
cmd = 'cmd_{0}'.format(command)
# internal commands ?
if hasattr(self, cmd):
getattr(self, cmd)(*args)
return
# maybe it's a target?
targets = [x[0] for x in self.targets()]
if command not in targets:
print('Unknown command/target {}'.format(command))
exit(1)
self.set_target(command)
self.target.run_commands(args)
def check_root(self):
'''If effective user id is 0, display a warning and require
user input to continue (or to cancel)'''
if IS_PY3:
input_func = input
else:
input_func = raw_input
warn_on_root = self.config.getdefault('buildozer', 'warn_on_root', '1')
try:
euid = os.geteuid() == 0
except AttributeError:
if sys.platform == 'win32':
import ctypes
euid = ctypes.windll.shell32.IsUserAnAdmin() != 0
if warn_on_root == '1' and euid:
print('\033[91m\033[1mBuildozer is running as root!\033[0m')
print(
'\033[91mThis is \033[1mnot\033[0m \033[91mrecommended, and may lead to problems later.\033[0m'
)
cont = None
while cont not in ('y', 'n'):
cont = input_func('Are you sure you want to continue [y/n]? ')
if cont == 'n':
sys.exit()
def cmd_init(self, *args):
'''Create a initial buildozer.spec in the current directory
'''
if exists('buildozer.spec'):
print('ERROR: You already have a buildozer.spec file.')
exit(1)
copyfile(join(dirname(__file__), 'default.spec'), 'buildozer.spec')
print('File buildozer.spec created, ready to customize!')
def cmd_distclean(self, *args):
'''Clean the whole Buildozer environment.
'''
print("Warning: Your ndk, sdk and all other cached packages will be"
" removed. Continue? (y/n)")
if sys.stdin.readline().lower()[0] == 'y':
self.info('Clean the global build directory')
if not exists(self.global_buildozer_dir):
return
rmtree(self.global_buildozer_dir)
def cmd_appclean(self, *args):
'''Clean the .buildozer folder in the app directory.
This command specifically refuses to delete files in a
user-specified build directory, to avoid accidentally deleting
more than the user intends.
'''
if self.user_build_dir is not None:
self.error((
'Failed: build_dir is specified as {} in the buildozer config. `appclean` will '
'not attempt to delete files in a user-specified build directory.'
).format(self.user_build_dir))
elif exists(self.buildozer_dir):
self.info('Deleting {}'.format(self.buildozer_dir))
rmtree(self.buildozer_dir)
else:
self.error('{} already deleted, skipping.'.format(
self.buildozer_dir))
def cmd_help(self, *args):
'''Show the Buildozer help.
'''
self.usage()
def cmd_setdefault(self, *args):
'''Set the default command to run when no arguments are given
'''
self.check_build_layout()
self.state['buildozer:defaultcommand'] = args
def cmd_version(self, *args):
'''Show the Buildozer version
'''
print('Buildozer {0}'.format(__version__))
def cmd_serve(self, *args):
'''Serve the bin directory via SimpleHTTPServer
'''
try:
from http.server import SimpleHTTPRequestHandler
from socketserver import TCPServer
except ImportError:
from SimpleHTTPServer import SimpleHTTPRequestHandler
from SocketServer import TCPServer
os.chdir(self.bin_dir)
handler = SimpleHTTPRequestHandler
httpd = TCPServer(("", SIMPLE_HTTP_SERVER_PORT), handler)
print("Serving via HTTP at port {}".format(SIMPLE_HTTP_SERVER_PORT))
print("Press Ctrl+c to quit serving.")
httpd.serve_forever()
#
# Private
#
def _merge_config_profile(self):
profile = self.config_profile
if not profile:
return
for section in self.config.sections():
# extract the profile part from the section name
# example: [app@default,hd]
parts = section.split('@', 1)
if len(parts) < 2:
continue
# create a list that contain all the profiles of the current section
# ['default', 'hd']
section_base, section_profiles = parts
section_profiles = section_profiles.split(',')
if profile not in section_profiles:
continue
# the current profile is one available in the section
# merge with the general section, or make it one.
if not self.config.has_section(section_base):
self.config.add_section(section_base)
for name, value in self.config.items(section):
print('merged ({}, {}) into {} (profile is {})'.format(
name, value, section_base, profile))
self.config.set(section_base, name, value)
def _get_config_list_values(self, *args, **kwargs):
kwargs['with_values'] = True
return self._get_config_list(*args, **kwargs)
def _get_config_list(self,
section,
token,
default=None,
with_values=False):
# monkey-patch method for ConfigParser
# get a key as a list of string, separated from the comma
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
# if a section:token is defined, let's use the content as a list.
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
values = self.config.options(l_section)
if with_values:
return [
'{}={}'.format(key, self.config.get(l_section, key))
for key in values
]
else:
return [x.strip() for x in values]
values = self.config.getdefault(section, token, '')
if not values:
return default
values = values.split(',')
if not values:
return default
return [x.strip() for x in values]
def _get_config_default(self, section, token, default=None):
# monkey-patch method for ConfigParser
# get an appropriate env var if it exists, else
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.get(section, token)
def _get_config_bool(self, section, token, default=False):
# monkey-patch method for ConfigParser
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.getboolean(section, token)
def _get_config_raw_default(self,
section,
token,
default=None,
section_sep="=",
split_char=" "):
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
return [
section_sep.join(item) for item in self.config.items(l_section)
]
if not self.config.has_option(section, token):
return default.split(split_char)
return self.config.get(section, token).split(split_char)
def run(ini_file='TOPKAPI.ini',
verbose=False,
quiet=False,
parallel_exec=True,
nworkers=int(mp.cpu_count() - 1)):
"""Run the model.
Parameters
----------
ini_file : str
The name of the PyTOPKAPI initialization file. This file describes the
locations of the parameter files and model setup options. Default is to
use a file named `TOPKAPI.ini` in the current directory.
verbose : bool
Prints runtime information [default False - don't display runtime
info]. Is independent of the `quiet` keyword argument.
quiet : bool
Toggles whether to display an informational banner at runtime [default
False - display banner]. Is independent of the `verbose` keyword
argument.
nworkers : int
Number of worker processes to spawn for solving each cell's time-series
in parallel. Default is one fewer than CPU count reported by
multiprocessing.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getint('numerical_options', 'solve_s')
solve_o = config.getint('numerical_options', 'solve_o')
solve_c = config.getint('numerical_options', 'solve_c')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
if verbose:
print('Read the forcing data')
#~~~~Rainfall
h5_rain = h5py.File(file_rain)
dset_name = '/{}/rainfall'.format(group_name)
rainfall_forcing = h5_rain[dset_name][...]
h5_rain.close()
#~~~~ETr - Reference crop ET
h5_ET = h5py.File(file_ET)
dset_name = '/{}/ETr'.format(group_name)
ETr_forcing = h5_ET[dset_name][...]
#~~~~ETo - Open water potential evap.
dset_name = '/{}/ETo'.format(group_name)
ET0_forcing = h5_ET[dset_name][...]
h5_ET.close()
#~~~~external_flow flows
if external_flow:
external_flow_records = np.loadtxt(file_Qexternal_flow)[:, 5]
else:
external_flow_records = None
##============================##
## Pretreatment of input data ##
##============================##
if verbose:
print('Pretreatment of input data')
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, channel_flag, \
Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L, Ks, \
ar_theta_r, ar_theta_s, \
ar_n_o, ar_n_c, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
Kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
node_hierarchy = pm.compute_node_hierarchy(ar_cell_label, ar_cell_down)
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L * fac_L
Ks = Ks * fac_Ks
ar_n_o = ar_n_o * fac_n_o
ar_n_c = ar_n_c * fac_n_c
if verbose:
print('Max L=', max(ar_L))
print('Max Ks=', max(Ks))
print('Max n_o=', max(ar_n_o))
print('Max n_c=', max(ar_n_c))
#~~~~Computation of model parameters from physical parameters
Vsm, b_s, b_o, \
W, b_c = pm.compute_cell_param(X, Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
channel_flag, ar_tan_beta,
ar_tan_beta_channel, ar_L,
Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow, ar_coorx,
ar_coory, channel_flag)
if verbose:
print('external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')')
else:
cell_external_flow = None
#~~~~Number of simulation time steps
nb_time_step = rainfall_forcing.shape[0]
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
if verbose:
print('Initialize from simulation file')
h5file_in = h5py.File(file_out)
Vs_t0 = h5file_in['/Soil/V_s'][-1, :]
Vc_t0 = h5file_in['/Channel/V_c'][-1, :]
Vo_t0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
if verbose:
print('Initialize from parameters')
Vs_t0 = fl.initial_volume_soil(ar_pVs_t0, Vsm)
Vo_t0 = ar_Vo_t0
Vc_t0 = fl.initial_volume_channel(ar_Qc_t0, W, X, ar_n_c)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file, dset_Vs, dset_Vo, dset_Vc, \
dset_Qs_out, dset_Qo_out, dset_Qc_out, \
dset_Q_down, dset_ET_out, dset_Ec_out \
= ut.open_simulation_file(file_out, fmode,
Vs_t0, Vo_t0, Vc_t0, no_data,
nb_cell, nb_time_step,
append_output, first_run)
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
if not quiet:
ut.show_banner(ini_file, nb_cell, nb_time_step)
progress_desc = 'Simulation'
else:
progress_desc = 'PyTOPKAPI v{}'.format(pytopkapi.__version__)
# prepare parameter dict
exec_params = {
'nb_cell': nb_cell,
'nb_time_step': nb_time_step,
'progress_desc': progress_desc,
'Dt': Dt,
'rainfall_forcing': rainfall_forcing,
'ETr_forcing': ETr_forcing,
'ET0_forcing': ET0_forcing,
'psi_b': psi_b,
'lamda': lamda,
'eff_theta': eff_theta,
'Ks': Ks,
'X': X,
'b_s': b_s,
'b_o': b_o,
'b_c': b_c,
'alpha_s': alpha_s,
'alpha_o': alpha_o,
'alpha_c': alpha_c,
'Vs_t0': Vs_t0,
'Vo_t0': Vo_t0,
'Vc_t0': Vc_t0,
'Vsm': Vsm,
'dset_Vs': dset_Vs,
'dset_Vo': dset_Vo,
'dset_Vc': dset_Vc,
'dset_Qs_out': dset_Qs_out,
'dset_Qo_out': dset_Qo_out,
'dset_Qc_out': dset_Qc_out,
'dset_Q_down': dset_Q_down,
'dset_ET_out': dset_ET_out,
'dset_Ec_out': dset_Ec_out,
'solve_s': solve_s,
'solve_o': solve_o,
'solve_c': solve_c,
'channel_flag': channel_flag,
'W': W,
'Xc': Xc,
'Kc': Kc,
'cell_external_flow': cell_external_flow,
'external_flow_records': external_flow_records,
'node_hierarchy': node_hierarchy,
'li_cell_up': li_cell_up,
'nworkers': nworkers
}
if not parallel_exec:
# Serial execution. Solve by timestep in a single process.
# Outer loop timesteps - inner loop cells
_serial_execute(exec_params)
else:
# Parallel execution. Solve by cell using multiple processes.
# Outer loop cells - inner loop timesteps
_parallel_execute(exec_params)
h5file.close()
target1_freq = config.get('hw_server', 'target1_freq')
mgt_rx = config.get('mgt_parameters', 'mgt_rx')
mgt_tx = config.get('mgt_parameters', 'mgt_tx')
TXDIFFSWING = config.get('mgt_parameters', 'TXDIFFSWING')
TXPOST = config.get('mgt_parameters', 'TXPOST')
TXPRE = config.get('mgt_parameters', 'TXPRE')
RXTERM = config.get('mgt_parameters', 'RXTERM')
tcl_dir = config.get('test', 'tcl_dir')
tcl_transm_name = config.get('test', 'tcl_transm_name')
tcl_rcv_name = config.get('test', 'tcl_rcv_name')
results_dir = config.get('test', 'results_dir')
results_name = config.get('test', 'results_name')
desired_area = config.getint('test', 'desired_area')
BER = config.get('test', 'BER')
err_req = config.getint('test', 'err_req')
include_all_results = config.getboolean('test', 'include_all_results')
mgt_rx = format_to_list(mgt_rx)
mgt_tx = format_to_list(mgt_tx)
TXDIFFSWING = format_to_list(TXDIFFSWING)
TXPOST = format_to_list(TXPOST)
TXPRE = format_to_list(TXPRE)
RXTERM = format_to_list(RXTERM)
# Main script
print("----------------------------------------------------------------------")
print("Creating Instance 0")
rcv = pyIBERT(server0_addr, server0_port, target0_name, target0_freq)
print("Creating Instance 1")
transm = pyIBERT(server1_addr, server1_port, target1_name, target1_freq)
host = '0.0.0.0' # 'localhost'
port = 8080
debug = False
datapath_sub = dict()
search_path = ['/etc', '']
for path in search_path:
candidate = os.path.join(path, 'spider.conf')
if os.path.isfile(candidate):
try:
config = SafeConfigParser()
config.read([candidate])
database = config.get('Spider', 'database')
title = config.get('WebUI', 'title')
host = config.get('WebUI', 'host')
port = config.getint('WebUI', 'port')
debug = config.getboolean('WebUI', 'debug')
#datapathsub = list(filter(None, [x.strip() for x in datapath_regexps.splitlines()]))
except:
pass
datapath_sub = dict()
try:
datapath_sub = config.items('datapath_substitutions')
except:
pass
engine = create_engine(database)
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(engine)
@app.route('/')
args = parser.parse_args()
# === PARAMS ===
np.random.seed(rank)
Nsims = args.Nsims
exp_name = args.Exp
expf_name = args.ExpFilter
# Read config
iniFile = "input/recon.ini"
Config = SafeConfigParser()
Config.optionxform=str
Config.read(iniFile)
pol = Config.getboolean("reconstruction","pol")
output_dir = Config.get("general","output_dir")
lmax,tellmin,tellmax,pellmin,pellmax,kellmin,kellmax = aio.ellbounds_from_config(Config,"reconstruction")
gradCut = None
pol_list = ['TT']
debug = False
out_dir = os.environ['WWW']+"plots/halotest/smallpatch_"
# === SIM AND RECON LOOP ===
kappa_stack = {}
encrypted_text = self.gpg.encrypt(message, recipientEmail, sign=self.signID, passphrase=self.passphrase)
return encrypted_text
def sign(self, message):
signed_text = self.gpg.sign(message, keyid=self.signID, passphrase=self.passphrase)
return signed_text
# read config file in
parser = SafeConfigParser()
parser.read(configFile)
# Check for debug flag
DEBUG = False
if parser.has_option('debug', 'debug'):
DEBUG = parser.getboolean('debug', 'debug')
# Ignore config.cfg file if --debug flag is present in args
if args.debug is True:
DEBUG = True
#################
# Logging Setup #
#################
validLogLevels = ('ERROR', 'DEBUG', 'INFO', 'NOTSET')
# check for logging flag from config file
if parser.has_option('debug', 'loglevel'):
loglevel = parser.get('debug', 'loglevel').upper()
if loglevel not in validLogLevels:
loglevel = 'NOTSET'
else:
def cli():
parser = argparse.ArgumentParser(
description='Check HTTPs rules for validity')
parser.add_argument(
'checker_config', help='an integer for the accumulator')
parser.add_argument('rule_files', nargs="*", default=[],
help="Specific XML rule files")
parser.add_argument('--json_file', default=None,
help='write results in json file')
args = parser.parse_args()
config = SafeConfigParser()
config.read(args.checker_config)
logfile = config.get("log", "logfile")
loglevel = convertLoglevel(config.get("log", "loglevel"))
if logfile == "-":
logging.basicConfig(stream=sys.stderr, level=loglevel,
format="%(levelname)s %(message)s")
else:
logging.basicConfig(filename=logfile, level=loglevel,
format="%(asctime)s %(levelname)s %(message)s [%(pathname)s:%(lineno)d]")
autoDisable = False
if config.has_option("rulesets", "auto_disable"):
autoDisable = config.getboolean("rulesets", "auto_disable")
# Test rules even if they have default_off=...
includeDefaultOff = False
if config.has_option("rulesets", "include_default_off"):
includeDefaultOff = config.getboolean(
"rulesets", "include_default_off")
ruledir = config.get("rulesets", "rulesdir")
checkCoverage = False
if config.has_option("rulesets", "check_coverage"):
checkCoverage = config.getboolean("rulesets", "check_coverage")
checkTargetValidity = False
if config.has_option("rulesets", "check_target_validity"):
checkTargetValidity = config.getboolean(
"rulesets", "check_target_validity")
checkNonmatchGroups = False
if config.has_option("rulesets", "check_nonmatch_groups"):
checkNonmatchGroups = config.getboolean(
"rulesets", "check_nonmatch_groups")
checkTestFormatting = False
if config.has_option("rulesets", "check_test_formatting"):
checkTestFormatting = config.getboolean(
"rulesets", "check_test_formatting")
certdir = config.get("certificates", "basedir")
if config.has_option("rulesets", "skiplist") and config.has_option("rulesets", "skipfield"):
skiplist = config.get("rulesets", "skiplist")
skipfield = config.get("rulesets", "skipfield")
with open(skiplist) as f:
f.readline()
for line in f:
splitLine = line.split(",")
fileHash = splitLine[0]
if splitLine[int(skipfield)] == "1":
skipdict[binascii.unhexlify(fileHash)] = 1
threadCount = config.getint("http", "threads")
httpEnabled = True
if config.has_option("http", "enabled"):
httpEnabled = config.getboolean("http", "enabled")
metricName = config.get("thresholds", "metric")
thresholdDistance = config.getfloat("thresholds", "max_distance")
metricClass = getMetricClass(metricName)
metric = metricClass()
# Debugging options, graphviz dump
dumpGraphvizTrie = False
if config.has_option("debug", "dump_graphviz_trie"):
dumpGraphvizTrie = config.getboolean("debug", "dump_graphviz_trie")
if dumpGraphvizTrie:
graphvizFile = config.get("debug", "graphviz_file")
exitAfterDump = config.getboolean("debug", "exit_after_dump")
if args.rule_files:
xmlFnames = args.rule_files
else:
xmlFnames = glob.glob(os.path.join(ruledir, "*.xml"))
trie = RuleTrie()
rulesets = []
coverageProblemsExist = False
targetValidityProblemExist = False
nonmatchGroupProblemsExist = False
testFormattingProblemsExist = False
for xmlFname in xmlFnames:
logging.debug("Parsing {}".format(xmlFname))
if skipFile(xmlFname):
logging.debug(
"Skipping rule file '{}', matches skiplist.".format(xmlFname))
continue
ruleset = Ruleset(etree.parse(open(xmlFname, "rb")).getroot(), xmlFname)
if ruleset.defaultOff and not includeDefaultOff:
logging.debug("Skipping rule '{}', reason: {}".format(
ruleset.name, ruleset.defaultOff))
continue
# Check whether ruleset coverage by tests was sufficient.
if checkCoverage:
logging.debug("Checking coverage for '{}'.".format(ruleset.name))
problems = ruleset.getCoverageProblems()
for problem in problems:
coverageProblemsExist = True
logging.error(problem)
if checkTargetValidity:
logging.debug("Checking target validity for '{}'.".format(ruleset.name))
problems = ruleset.getTargetValidityProblems()
for problem in problems:
targetValidityProblemExist = True
logging.error(problem)
if checkNonmatchGroups:
logging.debug("Checking non-match groups for '{}'.".format(ruleset.name))
problems = ruleset.getNonmatchGroupProblems()
for problem in problems:
nonmatchGroupProblemsExist = True
logging.error(problem)
if checkTestFormatting:
logging.debug("Checking test formatting for '{}'.".format(ruleset.name))
problems = ruleset.getTestFormattingProblems()
for problem in problems:
testFormattingProblemsExist = True
logging.error(problem)
trie.addRuleset(ruleset)
rulesets.append(ruleset)
# Trie is built now, dump it if it's set in config
if dumpGraphvizTrie:
logging.debug("Dumping graphviz ruleset trie")
graph = trie.generateGraphizGraph()
if graphvizFile == "-":
graph.dot()
else:
with open(graphvizFile, "w") as gvFd:
graph.dot(gvFd)
if exitAfterDump:
sys.exit(0)
fetchOptions = http_client.FetchOptions(config)
fetchers = list()
# Ensure "default" is in the platform dirs
if not os.path.isdir(os.path.join(certdir, "default")):
raise RuntimeError(
"Platform 'default' is missing from certificate directories")
platforms = http_client.CertificatePlatforms(
os.path.join(certdir, "default"))
fetchers.append(http_client.HTTPFetcher(
"default", platforms, fetchOptions, trie))
# fetches pages with unrewritten URLs
fetcherPlain = http_client.HTTPFetcher("default", platforms, fetchOptions)
urlList = []
if config.has_option("http", "url_list"):
with open(config.get("http", "url_list")) as urlFile:
urlList = [line.rstrip() for line in urlFile.readlines()]
if httpEnabled:
taskQueue = queue.Queue(1000)
resQueue = queue.Queue()
startTime = time.time()
testedUrlPairCount = 0
config.getboolean("debug", "exit_after_dump")
for i in range(threadCount):
t = UrlComparisonThread(
taskQueue, metric, thresholdDistance, autoDisable, resQueue)
t.setDaemon(True)
t.start()
# set of main pages to test
mainPages = set(urlList)
# If list of URLs to test/scan was not defined, use the test URL extraction
# methods built into the Ruleset implementation.
if not urlList:
for ruleset in rulesets:
if ruleset.platform != "default" and os.path.isdir(os.path.join(certdir, ruleset.platform)):
theseFetchers = copy.deepcopy(fetchers)
platforms.addPlatform(ruleset.platform, os.path.join(certdir, ruleset.platform))
theseFetchers.append(http_client.HTTPFetcher(
ruleset.platform, platforms, fetchOptions, trie))
else:
theseFetchers = fetchers
testUrls = []
for test in ruleset.tests:
if not ruleset.excludes(test.url):
testedUrlPairCount += 1
testUrls.append(test.url)
else:
# TODO: We should fetch the non-rewritten exclusion URLs to make
# sure they still exist.
logging.debug("Skipping excluded URL {}".format(test.url))
task = ComparisonTask(testUrls, fetcherPlain, theseFetchers, ruleset)
taskQueue.put(task)
taskQueue.join()
logging.info("Finished in {:.2f} seconds. Loaded rulesets: {}, URL pairs: {}.".format(
time.time() - startTime, len(xmlFnames), testedUrlPairCount))
if args.json_file:
json_output(resQueue, args.json_file, problems)
if checkCoverage:
if coverageProblemsExist:
return 1 # exit with error code
if checkTargetValidity:
if targetValidityProblemExist:
return 1 # exit with error code
if checkNonmatchGroups:
if nonmatchGroupProblemsExist:
return 1 # exit with error code
if checkTestFormatting:
if testFormattingProblemsExist:
return 1 # exit with error code
return 0 # exit with success
class Buildozer(object):
standard_cmds = ('distclean', 'update', 'debug', 'release',
'deploy', 'run', 'serve')
def __init__(self, filename='buildozer.spec', target=None):
super(Buildozer, self).__init__()
self.log_level = 1
self.environ = {}
self.specfilename = filename
self.state = None
self.build_id = None
self.config_profile = ''
self.config = SafeConfigParser(allow_no_value=True)
self.config.optionxform = lambda value: value
self.config.getlist = self._get_config_list
self.config.getlistvalues = self._get_config_list_values
self.config.getdefault = self._get_config_default
self.config.getbooldefault = self._get_config_bool
if exists(filename):
self.config.read(filename)
self.check_configuration_tokens()
# Check all section/tokens for env vars, and replace the
# config value if a suitable env var exists.
set_config_from_envs(self.config)
try:
self.log_level = int(self.config.getdefault(
'buildozer', 'log_level', '1'))
except:
pass
self.builddir = self.config.getdefault('buildozer', 'builddir', None)
self.targetname = None
self.target = None
if target:
self.set_target(target)
def set_target(self, target):
'''Set the target to use (one of buildozer.targets, such as "android")
'''
self.targetname = target
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
self.target = m.get_target(self)
self.check_build_layout()
self.check_configuration_tokens()
def prepare_for_build(self):
'''Prepare the build.
'''
assert(self.target is not None)
if hasattr(self.target, '_build_prepared'):
return
self.info('Preparing build')
self.info('Check requirements for {0}'.format(self.targetname))
self.target.check_requirements()
self.info('Install platform')
self.target.install_platform()
self.info('Check application requirements')
self.check_application_requirements()
self.info('Check garden requirements')
self.check_garden_requirements()
self.info('Compile platform')
self.target.compile_platform()
# flag to prevent multiple build
self.target._build_prepared = True
def build(self):
'''Do the build.
The target can set build_mode to 'release' or 'debug' before calling
this method.
(:meth:`prepare_for_build` must have been call before.)
'''
assert(self.target is not None)
assert(hasattr(self.target, '_build_prepared'))
if hasattr(self.target, '_build_done'):
return
# increment the build number
self.build_id = int(self.state.get('cache.build_id', '0')) + 1
self.state['cache.build_id'] = str(self.build_id)
self.info('Build the application #{}'.format(self.build_id))
self.build_application()
self.info('Package the application')
self.target.build_package()
# flag to prevent multiple build
self.target._build_done = True
#
# Log functions
#
def log(self, level, msg):
if level > self.log_level:
return
if USE_COLOR:
color = COLOR_SEQ(LOG_LEVELS_C[level])
print(''.join((RESET_SEQ, color, '# ', msg, RESET_SEQ)))
else:
print('{} {}'.format(LOG_LEVELS_T[level], msg))
def debug(self, msg):
self.log(2, msg)
def info(self, msg):
self.log(1, msg)
def error(self, msg):
self.log(0, msg)
#
# Internal check methods
#
def checkbin(self, msg, fn):
self.debug('Search for {0}'.format(msg))
if exists(fn):
return realpath(fn)
for dn in environ['PATH'].split(':'):
rfn = realpath(join(dn, fn))
if exists(rfn):
self.debug(' -> found at {0}'.format(rfn))
return rfn
self.error('{} not found, please install it.'.format(msg))
exit(1)
def cmd(self, command, **kwargs):
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('stdout', PIPE)
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', True)
kwargs.setdefault('show_output', self.log_level > 1)
show_output = kwargs.pop('show_output')
get_stdout = kwargs.pop('get_stdout', False)
get_stderr = kwargs.pop('get_stderr', False)
break_on_error = kwargs.pop('break_on_error', True)
sensible = kwargs.pop('sensible', False)
if not sensible:
self.debug('Run {0!r}'.format(command))
else:
if type(command) in (list, tuple):
self.debug('Run {0!r} ...'.format(command[0]))
else:
self.debug('Run {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
# open the process
if sys.platform == 'win32':
kwargs.pop('close_fds', None)
process = Popen(command, **kwargs)
# prepare fds
fd_stdout = process.stdout.fileno()
fd_stderr = process.stderr.fileno()
if fcntl:
fcntl.fcntl(
fd_stdout, fcntl.F_SETFL,
fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(
fd_stderr, fcntl.F_SETFL,
fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
ret_stdout = [] if get_stdout else None
ret_stderr = [] if get_stderr else None
while True:
try:
readx = select.select([fd_stdout, fd_stderr], [], [])[0]
except select.error:
break
if fd_stdout in readx:
chunk = process.stdout.read()
if not chunk:
break
if get_stdout:
ret_stdout.append(chunk)
if show_output:
if IS_PY3:
stdout.write(chunk.decode('utf-8'))
else:
stdout.write(chunk)
if fd_stderr in readx:
chunk = process.stderr.read()
if not chunk:
break
if get_stderr:
ret_stderr.append(chunk)
if show_output:
if IS_PY3:
stderr.write(chunk.decode('utf-8'))
else:
stderr.write(chunk)
stdout.flush()
stderr.flush()
process.communicate()
if process.returncode != 0 and break_on_error:
self.error('Command failed: {0}'.format(command))
self.error('')
self.error('Buildozer failed to execute the last command')
if self.log_level <= 1:
self.error('If the error is not obvious, please raise the log_level to 2')
self.error('and retry the latest command.')
else:
self.error('The error might be hidden in the log above this error')
self.error('Please read the full log, and search for it before')
self.error('raising an issue with buildozer itself.')
self.error('In case of a bug report, please add a full log with log_level = 2')
raise BuildozerCommandException()
if ret_stdout:
ret_stdout = b''.join(ret_stdout)
if ret_stderr:
ret_stderr = b''.join(ret_stderr)
return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None,
ret_stderr.decode('utf-8') if ret_stderr else None,
process.returncode)
def cmd_expect(self, command, **kwargs):
from pexpect import spawnu
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('show_output', self.log_level > 1)
sensible = kwargs.pop('sensible', False)
show_output = kwargs.pop('show_output')
if show_output:
kwargs['logfile'] = codecs.getwriter('utf8')(stdout)
if not sensible:
self.debug('Run (expect) {0!r}'.format(command))
else:
self.debug('Run (expect) {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
return spawnu(command, **kwargs)
def check_configuration_tokens(self):
'''Ensure the spec file is 'correct'.
'''
self.info('Check configuration tokens')
get = self.config.getdefault
errors = []
adderror = errors.append
if not get('app', 'title', ''):
adderror('[app] "title" is missing')
if not get('app', 'source.dir', ''):
adderror('[app] "source.dir" is missing')
package_name = get('app', 'package.name', '')
if not package_name:
adderror('[app] "package.name" is missing')
elif package_name[0] in map(str, range(10)):
adderror('[app] "package.name" may not start with a number.')
version = get('app', 'version', '')
version_regex = get('app', 'version.regex', '')
if not version and not version_regex:
adderror('[app] One of "version" or "version.regex" must be set')
if version and version_regex:
adderror('[app] Conflict between "version" and "version.regex"'
', only one can be used.')
if version_regex and not get('app', 'version.filename', ''):
adderror('[app] "version.filename" is missing'
', required by "version.regex"')
orientation = get('app', 'orientation', 'landscape')
if orientation not in ('landscape', 'portrait', 'all'):
adderror('[app] "orientation" have an invalid value')
if errors:
self.error('{0} error(s) found in the buildozer.spec'.format(
len(errors)))
for error in errors:
print(error)
exit(1)
def check_build_layout(self):
'''Ensure the build (local and global) directory layout and files are
ready.
'''
self.info('Ensure build layout')
if not exists(self.specfilename):
print('No {0} found in the current directory. Abandon.'.format(
self.specfilename))
exit(1)
# create global dir
self.mkdir(self.global_buildozer_dir)
self.mkdir(self.global_cache_dir)
# create local dir
specdir = dirname(self.specfilename)
if self.builddir:
specdir = self.builddir
self.mkdir(join(specdir, '.buildozer'))
self.mkdir(join(specdir, 'bin'))
self.mkdir(self.applibs_dir)
self.state = JsonStore(join(self.buildozer_dir, 'state.db'))
if self.targetname:
target = self.targetname
self.mkdir(join(self.global_platform_dir, target, 'platform'))
self.mkdir(join(specdir, '.buildozer', target, 'platform'))
self.mkdir(join(specdir, '.buildozer', target, 'app'))
def check_application_requirements(self):
'''Ensure the application requirements are all available and ready to be
packaged as well.
'''
requirements = self.config.getlist('app', 'requirements', '')
target_available_packages = self.target.get_available_packages()
# remove all the requirements that the target can compile
onlyname = lambda x: x.split('==')[0]
requirements = [x for x in requirements if onlyname(x) not in
target_available_packages]
# did we already installed the libs ?
if exists(self.applibs_dir) and \
self.state.get('cache.applibs', '') == requirements:
self.debug('Application requirements already installed, pass')
return
# recreate applibs
self.rmdir(self.applibs_dir)
self.mkdir(self.applibs_dir)
# ok now check the availability of all requirements
for requirement in requirements:
self._install_application_requirement(requirement)
# everything goes as expected, save this state!
self.state['cache.applibs'] = requirements
def _install_application_requirement(self, module):
self._ensure_virtualenv()
# resetup distribute, just in case
self.debug('Install distribute')
self.cmd('curl http://python-distribute.org/distribute_setup.py | venv/bin/python', get_stdout=True, cwd=self.buildozer_dir)
self.debug('Install requirement {} in virtualenv'.format(module))
self.cmd('pip install --download-cache={} --target={} {}'.format(
self.global_cache_dir, self.applibs_dir, module),
env=self.env_venv,
cwd=self.buildozer_dir)
def check_garden_requirements(self):
'''Ensure required garden packages are available to be included.
'''
garden_requirements = self.config.getlist('app',
'garden_requirements', '')
# have we installed the garden packages?
if exists(self.gardenlibs_dir) and \
self.state.get('cache.gardenlibs', '') == garden_requirements:
self.debug('Garden requirements already installed, pass')
return
# we're going to reinstall all the garden libs.
self.rmdir(self.gardenlibs_dir)
# but if we don't have requirements, or if the user removed everything,
# don't do anything.
if not garden_requirements:
self.state['cache.gardenlibs'] = garden_requirements
return
self._ensure_virtualenv()
self.cmd('pip install Kivy-Garden==0.1.1', env=self.env_venv)
# recreate gardenlibs
self.mkdir(self.gardenlibs_dir)
for requirement in garden_requirements:
self._install_garden_package(requirement)
# save gardenlibs state
self.state['cache.gardenlibs'] = garden_requirements
def _install_garden_package(self, package):
self._ensure_virtualenv()
self.debug('Install garden package {} in buildozer_dir'.format(package))
self.cmd('garden install --app {}'.format(package),
env=self.env_venv,
cwd=self.buildozer_dir)
def _ensure_virtualenv(self):
if hasattr(self, 'venv'):
return
self.venv = join(self.buildozer_dir, 'venv')
if not self.file_exists(self.venv):
self.cmd('virtualenv --python=python2.7 ./venv',
cwd=self.buildozer_dir)
# read virtualenv output and parse it
output = self.cmd('bash -c "source venv/bin/activate && env"',
get_stdout=True,
cwd=self.buildozer_dir)
self.env_venv = copy(self.environ)
for line in output[0].splitlines():
args = line.split('=', 1)
if len(args) != 2:
continue
key, value = args
if key in ('VIRTUAL_ENV', 'PATH'):
self.env_venv[key] = value
if 'PYTHONHOME' in self.env_venv:
del self.env_venv['PYTHONHOME']
# ensure any sort of compilation will fail
self.env_venv['CC'] = '/bin/false'
self.env_venv['CXX'] = '/bin/false'
def mkdir(self, dn):
if exists(dn):
return
self.debug('Create directory {0}'.format(dn))
makedirs(dn)
def rmdir(self, dn):
if not exists(dn):
return
self.debug('Remove directory and subdirectory {}'.format(dn))
rmtree(dn)
def file_matches(self, patterns):
from glob import glob
result = []
for pattern in patterns:
matches = glob(expanduser(pattern.strip()))
if not matches:
return
result.extend(matches)
return result
def file_exists(self, *args):
return exists(join(*args))
def file_rename(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Rename {0} to {1}'.format(source, target))
if not os.path.isdir(os.path.dirname(target)):
self.error('Rename {0} to {1} fails becaues {2} is not a directory'.format(source, target, os.path.directory(target)))
rename(source, target)
def file_copy(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Copy {0} to {1}'.format(source, target))
copyfile(source, target)
def file_extract(self, archive, cwd=None):
if archive.endswith('.tgz') or archive.endswith('.tar.gz'):
# XXX tarfile doesn't work for NDK-r8c :(
#tf = tarfile.open(archive, 'r:*')
#tf.extractall(path=cwd)
#tf.close()
self.cmd('tar xzf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.tbz2') or archive.endswith('.tar.bz2'):
# XXX same as before
self.cmd('tar xjf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.zip'):
archive = join(cwd, archive)
zf = zipfile.ZipFile(archive)
zf.extractall(path=cwd)
zf.close()
return
raise Exception('Unhandled extraction for type {0}'.format(archive))
def file_copytree(self, src, dest):
print('copy {} to {}'.format(src, dest))
if os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
for f in files:
self.file_copytree(
os.path.join(src, f),
os.path.join(dest, f))
else:
copyfile(src, dest)
def clean_platform(self):
self.info('Clean the platform build directory')
if not exists(self.platform_dir):
return
rmtree(self.platform_dir)
def download(self, url, filename, cwd=None):
def report_hook(index, blksize, size):
if size <= 0:
progression = '{0} bytes'.format(index * blksize)
else:
progression = '{0:.2f}%'.format(
index * blksize * 100. / float(size))
stdout.write('- Download {}\r'.format(progression))
stdout.flush()
url = url + filename
if cwd:
filename = join(cwd, filename)
if self.file_exists(filename):
unlink(filename)
self.debug('Downloading {0}'.format(url))
urlretrieve(url, filename, report_hook)
return filename
def get_version(self):
c = self.config
has_version = c.has_option('app', 'version')
has_regex = c.has_option('app', 'version.regex')
has_filename = c.has_option('app', 'version.filename')
# version number specified
if has_version:
if has_regex or has_filename:
raise Exception(
'version.regex and version.filename conflict with version')
return c.get('app', 'version')
# search by regex
if has_regex or has_filename:
if has_regex and not has_filename:
raise Exception('version.filename is missing')
if has_filename and not has_regex:
raise Exception('version.regex is missing')
fn = c.get('app', 'version.filename')
with open(fn) as fd:
data = fd.read()
regex = c.get('app', 'version.regex')
match = search(regex, data)
if not match:
raise Exception(
'Unable to find capture version in {0}\n'
' (looking for `{1}`)'.format(fn, regex))
version = match.groups()[0]
self.debug('Captured version: {0}'.format(version))
return version
raise Exception('Missing version or version.regex + version.filename')
def build_application(self):
self._copy_application_sources()
self._copy_application_libs()
self._copy_garden_libs()
self._add_sitecustomize()
def _copy_application_sources(self):
# XXX clean the inclusion/exclusion algo.
source_dir = realpath(self.config.getdefault('app', 'source.dir', '.'))
include_exts = self.config.getlist('app', 'source.include_exts', '')
exclude_exts = self.config.getlist('app', 'source.exclude_exts', '')
exclude_dirs = self.config.getlist('app', 'source.exclude_dirs', '')
exclude_patterns = self.config.getlist('app', 'source.exclude_patterns', '')
app_dir = self.app_dir
self.debug('Copy application source from {}'.format(source_dir))
rmtree(self.app_dir)
for root, dirs, files in walk(source_dir):
# avoid hidden directory
if True in [x.startswith('.') for x in root.split(sep)]:
continue
# need to have sort-of normalization. Let's say you want to exclude
# image directory but not images, the filtered_root must have a / at
# the end, same for the exclude_dir. And then we can safely compare
filtered_root = root[len(source_dir) + 1:].lower()
if filtered_root:
filtered_root += '/'
# manual exclude_dirs approach
is_excluded = False
for exclude_dir in exclude_dirs:
if exclude_dir[-1] != '/':
exclude_dir += '/'
if filtered_root.startswith(exclude_dir):
is_excluded = True
break
if is_excluded:
continue
# pattern matching
for pattern in exclude_patterns:
if fnmatch(filtered_root, pattern):
is_excluded = True
break
if is_excluded:
continue
for fn in files:
# avoid hidden files
if fn.startswith('.'):
continue
# exclusion by pattern matching
is_excluded = False
dfn = fn.lower()
if filtered_root:
dfn = join(filtered_root, fn)
for pattern in exclude_patterns:
if fnmatch(dfn, pattern):
is_excluded = True
break
if is_excluded:
continue
# filter based on the extension
# TODO more filters
basename, ext = splitext(fn)
if ext:
ext = ext[1:]
if include_exts and ext not in include_exts:
continue
if exclude_exts and ext in exclude_exts:
continue
sfn = join(root, fn)
rfn = realpath(join(app_dir, root[len(source_dir) + 1:], fn))
# ensure the directory exists
dfn = dirname(rfn)
self.mkdir(dfn)
# copy!
self.debug('Copy {0}'.format(sfn))
copyfile(sfn, rfn)
def _copy_application_libs(self):
# copy also the libs
copytree(self.applibs_dir, join(self.app_dir, '_applibs'))
def _copy_garden_libs(self):
if exists(self.gardenlibs_dir):
copytree(self.gardenlibs_dir, join(self.app_dir, 'libs'))
def _add_sitecustomize(self):
copyfile(join(dirname(__file__), 'sitecustomize.py'),
join(self.app_dir, 'sitecustomize.py'))
main_py = join(self.app_dir, 'service', 'main.py')
if not self.file_exists(main_py):
#self.error('Unable to patch main_py to add applibs directory.')
return
header = ('import sys, os; '
'sys.path = [os.path.join(os.getcwd(),'
'"..", "_applibs")] + sys.path\n')
with open(main_py, 'rb') as fd:
data = fd.read()
data = header + data
with open(main_py, 'wb') as fd:
fd.write(data)
self.info('Patched service/main.py to include applibs')
def namify(self, name):
'''Return a "valid" name from a name with lot of invalid chars
(allowed characters: a-z, A-Z, 0-9, -, _)
'''
return re.sub('[^a-zA-Z0-9_\-]', '_', name)
@property
def root_dir(self):
return realpath(join(dirname(self.specfilename)))
@property
def buildozer_dir(self):
if self.builddir:
return join(self.builddir, '.buildozer')
return join(self.root_dir, '.buildozer')
@property
def bin_dir(self):
return join(self.root_dir, 'bin')
@property
def platform_dir(self):
return join(self.buildozer_dir, self.targetname, 'platform')
@property
def app_dir(self):
return join(self.buildozer_dir, self.targetname, 'app')
@property
def applibs_dir(self):
return join(self.buildozer_dir, 'applibs')
@property
def gardenlibs_dir(self):
return join(self.buildozer_dir, 'libs')
@property
def global_buildozer_dir(self):
return join(expanduser('~'), '.buildozer')
@property
def global_platform_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'platform')
@property
def global_packages_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'packages')
@property
def global_cache_dir(self):
return join(self.global_buildozer_dir, 'cache')
@property
def package_full_name(self):
package_name = self.config.getdefault('app', 'package.name', '')
package_domain = self.config.getdefault('app', 'package.domain', '')
if package_domain == '':
return package_name
return '{}.{}'.format(package_domain, package_name)
#
# command line invocation
#
def targets(self):
for fn in listdir(join(dirname(__file__), 'targets')):
if fn.startswith('.') or fn.startswith('__'):
continue
if not fn.endswith('.py'):
continue
target = fn[:-3]
try:
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
yield target, m
except NotImplementedError:
pass
except:
raise
pass
def usage(self):
print('Usage:')
print(' buildozer [--profile <name>] [--verbose] [target] <command>...')
print(' buildozer --version')
print('')
print('Available targets:')
targets = list(self.targets())
for target, m in targets:
doc = m.__doc__.strip().splitlines()[0].strip()
print(' {0:<18} {1}'.format(target, doc))
print('')
print('Global commands (without target):')
cmds = [x for x in dir(self) if x.startswith('cmd_')]
for cmd in cmds:
name = cmd[4:]
meth = getattr(self, cmd)
if not meth.__doc__:
continue
doc = [x for x in
meth.__doc__.strip().splitlines()][0].strip()
print(' {0:<18} {1}'.format(name, doc))
print('')
print('Target commands:')
print(' clean Clean the target environment')
print(' update Update the target dependencies')
print(' debug Build the application in debug mode')
print(' release Build the application in release mode')
print(' deploy Deploy the application on the device')
print(' run Run the application on the device')
print(' serve Serve the bin directory via SimpleHTTPServer')
for target, m in targets:
mt = m.get_target(self)
commands = mt.get_custom_commands()
if not commands:
continue
print('')
print('Target "{0}" commands:'.format(target))
for command, doc in commands:
if not doc:
continue
doc = doc.strip().splitlines()[0].strip()
print(' {0:<18} {1}'.format(command, doc))
print('')
def run_default(self):
self.check_build_layout()
if 'buildozer:defaultcommand' not in self.state:
print('No default command set.')
print('Use "buildozer setdefault <command args...>"')
print('Use "buildozer help" for a list of all commands"')
exit(1)
cmd = self.state['buildozer:defaultcommand']
self.run_command(cmd)
def run_command(self, args):
while args:
if not args[0].startswith('-'):
break
arg = args.pop(0)
if arg in ('-v', '--verbose'):
self.log_level = 2
elif arg in ('-h', '--help'):
self.usage()
exit(0)
elif arg in ('-p', '--profile'):
self.config_profile = args.pop(0)
elif arg == '--version':
print('Buildozer {0}'.format(__version__))
exit(0)
self._merge_config_profile()
self.check_root()
if not args:
self.run_default()
return
command, args = args[0], args[1:]
cmd = 'cmd_{0}'.format(command)
# internal commands ?
if hasattr(self, cmd):
getattr(self, cmd)(*args)
return
# maybe it's a target?
targets = [x[0] for x in self.targets()]
if command not in targets:
print('Unknown command/target {}'.format(command))
exit(1)
self.set_target(command)
self.target.run_commands(args)
def check_root(self):
'''If effective user id is 0, display a warning and require
user input to continue (or to cancel)'''
if IS_PY3:
input_func = input
else:
input_func = raw_input
warn_on_root = self.config.getdefault('buildozer', 'warn_on_root', '1')
try:
euid = os.geteuid() == 0
except AttributeError:
if sys.platform == 'win32':
import ctypes
euid = ctypes.windll.shell32.IsUserAnAdmin() != 0
if warn_on_root == '1' and euid:
print('\033[91m\033[1mBuildozer is running as root!\033[0m')
print('\033[91mThis is \033[1mnot\033[0m \033[91mrecommended, and may lead to problems later.\033[0m')
cont = None
while cont not in ('y', 'n'):
cont = input_func('Are you sure you want to continue [y/n]? ')
if cont == 'n':
sys.exit()
def cmd_init(self, *args):
'''Create a initial buildozer.spec in the current directory
'''
if exists('buildozer.spec'):
print('ERROR: You already have a buildozer.spec file.')
exit(1)
copyfile(join(dirname(__file__), 'default.spec'), 'buildozer.spec')
print('File buildozer.spec created, ready to customize!')
def cmd_distclean(self, *args):
'''Clean the whole Buildozer environment.
'''
print("Warning: Your ndk, sdk and all other cached packages will be"
" removed. Continue? (y/n)")
if sys.stdin.readline().lower()[0] == 'y':
self.info('Clean the global build directory')
if not exists(self.global_buildozer_dir):
return
rmtree(self.global_buildozer_dir)
def cmd_help(self, *args):
'''Show the Buildozer help.
'''
self.usage()
def cmd_setdefault(self, *args):
'''Set the default command to do when to arguments are given
'''
self.check_build_layout()
self.state['buildozer:defaultcommand'] = args
def cmd_version(self, *args):
'''Show the Buildozer version
'''
print('Buildozer {0}'.format(__version__))
def cmd_serve(self, *args):
'''Serve the bin directory via SimpleHTTPServer
'''
try:
from http.server import SimpleHTTPRequestHandler
from socketserver import TCPServer
except ImportError:
from SimpleHTTPServer import SimpleHTTPRequestHandler
from SocketServer import TCPServer
os.chdir(self.bin_dir)
handler = SimpleHTTPRequestHandler
httpd = TCPServer(("", SIMPLE_HTTP_SERVER_PORT), handler)
print("Serving via HTTP at port {}".format(SIMPLE_HTTP_SERVER_PORT))
print("Press Ctrl+c to quit serving.")
httpd.serve_forever()
#
# Private
#
def _merge_config_profile(self):
profile = self.config_profile
if not profile:
return
for section in self.config.sections():
# extract the profile part from the section name
# example: [app@default,hd]
parts = section.split('@', 1)
if len(parts) < 2:
continue
# create a list that contain all the profiles of the current section
# ['default', 'hd']
section_base, section_profiles = parts
section_profiles = section_profiles.split(',')
if profile not in section_profiles:
continue
# the current profile is one available in the section
# merge with the general section, or make it one.
if not self.config.has_section(section_base):
self.config.add_section(section_base)
for name, value in self.config.items(section):
print('merged ({}, {}) into {} (profile is {})'.format(name,
value, section_base, profile))
self.config.set(section_base, name, value)
def _get_config_list_values(self, *args, **kwargs):
kwargs['with_values'] = True
return self._get_config_list(*args, **kwargs)
def _get_config_list(self, section, token, default=None, with_values=False):
# monkey-patch method for ConfigParser
# get a key as a list of string, seperated from the comma
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
# if a section:token is defined, let's use the content as a list.
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
values = self.config.options(l_section)
if with_values:
return ['{}={}'.format(key, self.config.get(l_section, key)) for
key in values]
else:
return [x.strip() for x in values]
values = self.config.getdefault(section, token, '')
if not values:
return default
values = values.split(',')
if not values:
return default
return [x.strip() for x in values]
def _get_config_default(self, section, token, default=None):
# monkey-patch method for ConfigParser
# get an appropriate env var if it exists, else
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.get(section, token)
def _get_config_bool(self, section, token, default=False):
# monkey-patch method for ConfigParser
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.getboolean(section, token)
def __init__(self, config_file = 'explosion.cfg'):
"""
Initialize explosion from config file explosion.cfg.
Actual values need to be in the [explosion] section, the
[DEFAULT] section is a backup. It is also hard-coded, so you
don't need to overwrite it unless it is being changed. The
current defaults are in this class in attribute
'default_config'.
[DEFAULT]
program = ../k
ext = #presn
dump = ../%(base)s%(ext)s
template = explosion.link
logfile = explosion.log
cmdfile = explosion.cmd
logall = True
verbose = True
alpha = 1.0
precision = 2
force = False
run = True
[Here is what you *have* to add:]
link = <link sentinal like "Da">
ekin_or_mni = <value>
base = <run name>
[example]
link = Da
alpha = 1.0
ekin = 1.2e+51
base = s25
[explosion]
link = Pa
alpha = 1.0
ekin = 1.2e+51
base = u25
======
So, with the defaults, the files that you need in the
explosion directory are just
explosion.link
explosion.cmd
explosion.cfg
"""
start_time = datetime.datetime.now()
config = Config(self.default_config)
# config.readfp(io.BytesIO(self.default_config))
config.read(config_file)
section = 'explosion'
run = config.getboolean(section,'run')
force = config.getboolean(section,'force')
kepler = config.get(section,'program')
presn_dump_name = config.get(section,'dump')
self.base_name = config.get(section,'base')
self.generator_tempelete = config.get(section,'template')
generator_start = config.get(section,'link')
logfile = config.get(section,'logfile')
self.exp_cmd_file = config.get(section,'cmdfile')
verbose = config.getboolean(section,'verbose')
logall = config.getboolean(section,'logall')
# set up log output
# maybe this this be replaced by
# deriving class from Logged and
# self.setup_logger(silent = not verbose, logfile=logfile, format='UTC')
# and at the end: self.close_logger()
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(logfile, 'w')
if verbose:
level = logging.DEBUG
else:
level = logging.WARNING
fh.setLevel(level)
formatter = utils.UTCFormatter('%(asctime)s%(msecs)03d %(nameb)-12s %(levelname)s: %(message)s',
datefmt = '%Y%m%d%H%M%S')
fh.setFormatter(formatter)
root_logger = logging.getLogger('')
if logall and len(root_logger.handlers) == 0:
root_logger.addHandler(fh)
root_logger.setLevel(level)
else:
self.logger.addHandler(fh)
# set up state for explosion
state = State(
config = config)
link = Link(generator_start)
self.logger.info("{:s}: first alpha = {:g}".format(link, state.alpha))
while state.best is None:
if run:
finished = True
dump_file_name = self.base_name + link + "#final"
if not os.path.exists(dump_file_name):
finished = False
dump_file_name = self.base_name + link + "#envel"
if not os.path.exists(dump_file_name):
finished = False
# check if run parameters are identical
if finished and not force:
if not state.eq_alpha(self.existing_alpha(link)):
self.logger.warning('Previous run '+link+' had different alpha. RERUNNING.')
finished = False
force = True
os.remove(self.base_name + link + "#final")
os.remove(self.base_name + link + "#envel")
os.remove(self.base_name + link + "z")
os.remove(link + ".link")
if (not finished) or force:
# call_string = "{} {}{} ".format(kepler, self.base_name, link)
# # look if a restart (z) dump is present
# dump_file_name = self.base_name + link + "z"
# if (not os.path.exists(dump_file_name)) or force:
# self.logger.info("RUN")
# self.generator(state.alpha, link)
# self.make_cmd(link);
# call_string += presn_dump_name
# else:
# self.logger.info("CONTINUE")
# call_string += "z"
# call_string += " k </dev/null >/dev/null"
# self.logger.info("CALL: {}".format(call_string))
# os.system(call_string)
args = [kepler, self.base_name + link]
# look if a restart (z) dump is present
dump_file_name = self.base_name + link + "z"
if (not os.path.exists(dump_file_name)) or force:
self.logger.info("RUN")
self.generator(state.alpha, link)
self.make_cmd(link);
args += [presn_dump_name]
else:
self.logger.info("CONTINUE")
args += ['z']
args += ['k']
self.logger.info("CALL: {}".format(' '.join(args)))
with open(os.devnull,'r') as null_in:
with open(os.devnull,'w') as null_out:
subprocess.call(args,
shell = False,
stdin = null_in,
stdout = null_out,
stderr = subprocess.STDOUT)
else:
self.logger.info("FINISHED")
state.update(link)
if state.best is None:
link += 1
self.logger.info("{}: new alpha = {} ({},{})".format(
link,
state.alpha,
state.lo_val(),
state.hi_val()))
end_time = datetime.datetime.now()
load_time = end_time - start_time
self.logger.info('finished in ' + time2human(load_time.total_seconds()))
self.logger.critical(self.out_format.format(
state.flag,
state.goal,
state.val(state.best),
abs(state.goal - state.val(state.best)),
float2str(state.best.alpha),
state.best.link,
state.best.mass,
state.best.zone))
state.save('explosion.res')
# clean up
for filename in glob.iglob("xxx*"):
os.remove(filename)
def main(args=None):
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument(
'--config-file', default='.bumpversion.cfg', metavar='FILE',
help='Config file to read most of the variables from', required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = None
if os.path.exists(known_args.config_file):
config = SafeConfigParser()
config.readfp(io.open(known_args.config_file, 'rt', encoding='utf-8'))
defaults.update(dict(config.items("bumpversion")))
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
elif known_args.config_file != parser1.get_default('config_file'):
raise argparse.ArgumentTypeError("Could not read config file at {}".format(
known_args.config_file))
parser2 = argparse.ArgumentParser(add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated', required=False)
parser2.add_argument('--parse', metavar='REGEX',
help='Regex parsing the version string',
default='(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
parser2.add_argument('--serialize', metavar='FORMAT',
help='How to format what is parsed back to a version',
default='{major}.{minor}.{patch}')
parser2_2 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
add_help=False,
parents=[parser2],
)
parser2_2.add_argument('part', help='Part of the version to be bumped.', nargs='?')
known_args, remaining_argv = parser2_2.parse_known_args(remaining_argv)
if known_args.part:
remaining_argv[0:0] = [known_args.part]
defaults.update(vars(known_args))
v = Version(
known_args.parse,
known_args.serialize,
context=dict(list(prefixed_environ().items()) + list(vcs_info.items()))
)
if not 'new_version' in defaults and known_args.current_version:
v.parse(known_args.current_version)
v.bump(known_args.part)
defaults['new_version'] = v.serialize()
parser3 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run', '-n', action='store_true',
default=False, help="Don't write any files, just pretend.")
parser3.add_argument('--new-version', metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit', action='store_true', dest="commit",
help='Commit to version control', default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit', action='store_false', dest="commit",
help='Do not commit to version control', default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag', action='store_true', dest="tag", default=defaults.get("commit", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag', action='store_false', dest="tag",
help='Do not create a tag in version control', default=argparse.SUPPRESS)
parser3.add_argument('--tag-name', metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument('--message', '-m', metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get('message', 'Bump version: {current_version} → {new_version}'))
files = []
if 'files' in defaults:
assert defaults['files'] != None
files = defaults['files'].split(' ')
parser3.add_argument('part',
help='Part of the version to be bumped.')
parser3.add_argument('files', metavar='file',
nargs='+' if len(files) == 0 else '*',
help='Files to change', default=files)
args = parser3.parse_args(remaining_argv)
if len(args.files) is 0:
warnings.warn("No files specified")
for vcs in VCS:
if vcs.is_usable():
vcs.assert_nondirty()
break
# make sure files exist and contain version string
for path in args.files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
assert args.current_version in before, 'Did not find string {} in file {}'.format(
args.current_version, path)
# change version string in files
for path in args.files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
# assert type(args.current_version) == bytes
# assert type(args.new_version) == bytes
after = before.replace(args.current_version, args.new_version)
if not args.dry_run:
with io.open(path, 'wt', encoding='utf-8') as f:
f.write(after)
commit_files = args.files
if config:
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
if not args.dry_run:
s = StringIO()
try:
config.write(s)
with io.open(known_args.config_file, 'wb') as f:
f.write(s.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`."
)
commit_files.append(known_args.config_file)
if args.commit:
if not args.dry_run:
for path in commit_files:
vcs.add_path(path)
formatargs = {
"current_version": args.current_version,
"new_version": args.new_version,
}
formatargs.update(prefixed_environ())
vcs.commit(message=args.message.format(**formatargs))
if args.tag:
vcs.tag(args.tag_name.format(**formatargs))
from configparser import SafeConfigParser
from bottle import route, run, template, static_file, redirect, error
from pymongo import MongoClient
from random import randint
from cherrypy import wsgiserver
config = SafeConfigParser()
environment = SafeConfigParser()
config.read('config.ini')
environment.read('environment.ini')
development = environment.getboolean('environment', 'development')
db = MongoClient()[config.get('mongodb', 'db_name')]
register_count = 0
@route('/')
def index():
count = db.domains.find({'status': 'inactive'}).limit(-1).count()
domains = []
if count > 0:
# this is inefficient and slow, but oh well
for x in range(0, 20):
random = randint(0, count - 1)
domain = db.domains.find({'status': 'inactive'}).skip(random).limit(1).next()
domains.append(domain['domain'])
return template('index', page=1, domains=domains)
@route('/about')
def about():
def main(): # pragma: no cover
config = SafeConfigParser()
dirs = ('.', '/etc', '/usr/local/etc')
if not config.read([os.path.join(dir, config_file) for dir in dirs]):
sys.exit('Could not find {} in {}'.format(config_file, dirs))
try:
logfile = config.get('logging', 'file')
rotating = config.getboolean('logging', 'rotate', fallback=False)
if rotating:
max_size = config.get('logging', 'max_size', fallback=1048576)
backup_count = config.get('logging', 'backup_count', fallback=5)
handler = logbook.RotatingFileHandler(logfile, max_size=max_size,
backup_count=backup_count)
else:
handler = logbook.FileHandler(logfile)
handler.push_application()
except Exception:
logbook.StderrHandler().push_application()
try:
kwargs = dict(config.items('mongodb'))
except NoSectionError:
sys.exit('No "mongodb" section in config file')
args = []
for arg in ('hosts', 'database', 'username', 'password'):
try:
args.append(config.get('mongodb', arg))
except NoOptionError:
sys.exit('No "{}" setting in "mongodb" section of config file'.
format(arg))
kwargs.pop(arg)
args[0] = [s.strip() for s in args[0].split(',')]
store = MongoStore(*args, **kwargs)
try:
email_sender = config.get('email', 'sender')
except NoSectionError:
sys.exit('No "email" section in config file')
except NoOptionError:
sys.exit('No "sender" setting in "email" section of config file')
business_logic = BusinessLogic(store, email_sender)
try:
listen_port = int(config.get('wsgi', 'port'))
log.info('Binding to port {}'.format(listen_port))
except Exception:
listen_port = 80
log.info('Binding to default port {}'.format(listen_port))
try:
auth_key = config.get('wsgi', 'auth_key')
log.info('Server authentication enabled')
except Exception:
log.warning('Server authentication DISABLED')
auth_key = None
httpd = make_server('', listen_port,
partial(application, business_logic, auth_key),
handler_class=LogbookWSGIRequestHandler)
business_logic.schedule_next_deadline()
httpd.serve_forever()
except ImportError:
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
parser.read("test.cfg")
def do_nothing(self):
pass
# add a bunch of decorator metafunctions like LIB_CORE
# which can be wrapped around individual tests as needed
for section in parser.sections():
for option in parser.options(section):
if parser.getboolean(section, option):
vars()["{}_{}".format(section.upper(), option.upper())] = \
lambda function: function
else:
vars()["{}_{}".format(section.upper(), option.upper())] = \
lambda function: do_nothing
def BLANK_PCM_Reader(length, sample_rate=44100, channels=2,
bits_per_sample=16, channel_mask=None):
from audiotools.decoders import SameSample
if channel_mask is None:
channel_mask = int(audiotools.ChannelMask.from_channels(channels))
return SameSample(sample=1,
def main():
'''
Main function for this module. Parses all command line arguments, reads in data
from stdin, and sends it to the proper BLS algorithm.
'''
# This is a global list of default values that will be used by the argument parser
# and the configuration parser.
defaults = {'min_duration':'0.0416667', 'max_duration':'0.5', 'n_bins':'100',
'direction':'0', 'mode':'vec', 'print_format':'encoded', 'verbose':'0', 'profiling':'0'}
# Set up the parser for command line arguments and read them.
parser = __init_parser(defaults)
args = parser.parse_args()
if not args.config:
# No configuration file specified -- read in command line arguments.
if not args.segment:
parser.error('No trial segment specified and no configuration file given.')
segment = args.segment
mindur = args.mindur
maxdur = args.maxdur
nbins = args.nbins
direction = args.direction
mode = args.mode
fmt = args.fmt
verbose = args.verbose
profile = args.profile
else:
# Configuration file was given; read in that instead.
cp = SafeConfigParser(defaults)
cp.read(args.config)
segment = cp.getfloat('DEFAULT', 'segment')
mindur = cp.getfloat('DEFAULT', 'min_duration')
maxdur = cp.getfloat('DEFAULT', 'max_duration')
nbins = cp.getint('DEFAULT', 'n_bins')
direction = cp.getint('DEFAULT', 'direction')
mode = cp.get('DEFAULT', 'mode')
fmt = cp.get('DEFAULT', 'print_format')
verbose = cp.getboolean('DEFAULT', 'verbose')
profile = cp.getboolean('DEFAULT', 'profiling')
# Perform any sanity-checking on the arguments.
__check_args(segment, mindur, maxdur, nbins, direction)
# Send the data to the algorithm.
for k, q, time, flux, fluxerr in read_mapper_output(sys.stdin):
# Extract the array columns.
time = np.array(time, dtype='float64')
flux = np.array(flux, dtype='float64')
fluxerr = np.array(fluxerr, dtype='float64')
if profile:
# Turn on profiling.
pr = cProfile.Profile()
pr.enable()
if mode == 'python':
raise NotImplementedError
out = bls_pulse_python(time, flux, fluxerr, nbins, segment, mindur, maxdur,
direction=direction)
elif mode == 'vec':
raise NotImplementedError
out = bls_pulse_vec(time, flux, fluxerr, nbins, segment, mindur, maxdur,
direction=direction)
elif mode == 'cython':
out = bls_pulse_cython(time, flux, fluxerr, nbins, segment, mindur, maxdur,
direction=direction)
else:
raise ValueError('Invalid mode: %s' % mode)
if profile:
# Turn off profiling.
pr.disable()
ps = pstats.Stats(pr, stream=sys.stderr).sort_stats('time')
ps.print_stats()
if direction == 2:
srsq_dip = out['srsq_dip']
duration_dip = out['duration_dip']
depth_dip = out['depth_dip']
midtime_dip = out['midtime_dip']
srsq_blip = out['srsq_blip']
duration_blip = out['duration_blip']
depth_blip = out['depth_blip']
midtime_blip = out['midtime_blip']
segstart = out['segstart']
segend = out['segend']
# Print output.
if fmt == 'encoded':
print "\t".join([k, q, encode_array(segstart), encode_array(segend), encode_array(srsq_dip),
encode_array(duration_dip), encode_array(depth_dip), encode_array(midtime_dip),
encode_array(srsq_blip), encode_array(duration_blip),
encode_array(depth_blip), encode_array(midtime_blip)])
elif fmt == 'normal':
print "-" * 120
print "Kepler " + k
print "Quarters: " + q
print "-" * 120
print '{0: <7s} {1: <13s} {2: <13s} {3: <13s} {4: <13s} {5: <13s} {6: <13s} {7: <13s} ' \
'{8: <13s}'.format('Segment', 'Dip SR^2', 'Dip dur.', 'Dip depth', 'Dip mid.',
'Blip SR^2', 'Blip dur.', 'Blip depth', 'Blip mid.')
for i in xrange(len(srsq_dip)):
print '{0: <7d} {1: <13.6f} {2: <13.6f} {3: <13.6f} {4: <13.6f} ' \
'{5: <13.6f} {6: <13.6f} {7: <13.6f} {8: <13.6f}'.format(i,
srsq_dip[i], duration_dip[i], depth_dip[i], midtime_dip[i],
srsq_blip[i], duration_blip[i], depth_blip[i], midtime_blip[i])
print "-" * 120
print
print
else:
srsq = out['srsq']
duration = out['duration']
depth = out['depth']
midtime = out['midtime']
segstart = out['segstart']
segend = out['segend']
# Print output.
if fmt == 'encoded':
print "\t".join([k, q, encode_array(segstart), encode_array(segend), encode_array(srsq),
encode_array(duration), encode_array(depth), encode_array(midtime)])
elif fmt == 'normal':
print "-" * 80
print "Kepler " + k
print "Quarters: " + q
print "-" * 80
print '{0: <7s} {1: <13s} {2: <10s} {3: <9s} {4: <13s}'.format('Segment',
'SR^2', 'Duration', 'Depth', 'Midtime')
for i in xrange(len(srsq)):
print '{0: <7d} {1: <13.6f} {2: <10.6f} {3: <9.6f} {4: <13.6f}'.format(i,
srsq[i], duration[i], depth[i], midtime[i])
print "-" * 80
print
print
class Configuration(object):
"""Class storing configuration details for Modula"""
def __init__(self, rootdir=".", cfg=None):
defaults = {
"@paths.modules": "modules",
"@paths.storage": "storage"
}
if cfg:
self.cfg = cfg
else:
cfgfiles = ["modules.cfg", "modula.cfg"]
cfgfiles = [os.path.join(rootdir, file) for file in cfgfiles]
self.cfg = Parser()
self.cfg.read(cfgfiles)
# Add defaults
for k, v in defaults.items():
if k not in self:
self[k] = v
# Make sure all paths are absolute
for k, v in self.items("@paths"):
self["@paths.%s" % k] = os.path.abspath(os.path.expanduser(v))
# Make sure all input files are absolute
for k, v in self.items("@inputs"):
self["@inputs.%s" % k] = os.path.abspath(os.path.expanduser(v))
def _parse_name(self, name):
if "." in name:
section, option = name.split(".", 1)
else:
section = "@global"
option = name
return section, option
def sections(self):
return self.cfg.sections()
def get(self, name):
return self.cfg.get(*self._parse_name(name))
def getInt(self, name):
return self.cfg.getint(*self._parse_name(name))
def getFloat(self, name):
return self.cfg.getfloat(*self._parse_name(name))
def getBoolean(self, name):
return self.cfg.getboolean(*self._parse_name(name))
def items(self, section):
return self.cfg.items(section)
def __contains__(self, name):
return self.cfg.has_option(*self._parse_name(name))
def __getitem__(self, name):
try:
return self.cfg.get(*self._parse_name(name))
except KeyError:
raise KeyError(name)
def __setitem__(self, name, value):
section, option = self._parse_name(name)
if not self.cfg.has_section(section):
self.cfg.add_section(section)
self.cfg.set(section, option, value)
def __delitem__(self, name):
section, option = self._parse_name(name)
try:
ok = self.cfg.remove_option(section, option)
if not ok:
raise NoOptionError()
except NoOptionError:
raise KeyError(name)
cc = Cosmology(cosmoDict, constDict, lmax)
theory = cc.theory
expX = Config.get('general', 'X')
expY = Config.get('general', 'Y')
beamX,beamFileX,fgFileX,noiseTX,noisePX,tellminX,tellmaxX,pellminX, \
pellmaxX,lxcutTX,lycutTX,lxcutPX,lycutPX,lkneeTX,alphaTX,lkneePX,alphaPX = getLensParams(Config,expX)
beamY,beamFileY,fgFileY,noiseTY,noisePY,tellminY,tellmaxY,pellminY, \
pellmaxY,lxcutTY,lycutTY,lxcutPY,lycutPY,lkneeTY,alphaTY,lkneePY,alphaPY = getLensParams(Config,expY)
cmb_bin_edges = np.arange(10, 9000, 10)
TCMB = Config.getfloat('general', 'TCMB')
gradCut = Config.getint('general', 'gradCut')
halo = Config.getboolean('general', 'halo')
fsky = Config.getfloat('general', 'sqDeg') / 41250.
kmin = 40
deg = 10.
px = 0.5
dell = 10
kellrange = np.arange(80., 2100., 10.)
Nlmvinv = 0.
pl = Plotter(scaleY='log')
for polComb in ['TT', 'TE', 'EE', 'EB']:
kmax = getMax(polComb, tellmaxY, pellmaxY)
bin_edges = np.arange(kmin, kmax, dell) + dell
lmap = lm.makeEmptyCEATemplate(raSizeDeg=deg,
def main(args):
if args.cluster:
cluster(args.cluster, args.config_file)
return
config = ConfigParser({"sentences": "False"})
config.read(args.config_file)
corpus_filename = config.get("main", "corpus_file")
model_path = config.get("main", "path")
if config.getboolean("main", "sentences"):
from vsm.extensions.ldasentences import CorpusSent as Corpus
else:
from vsm.corpus import Corpus
if args.k is None:
try:
if config.get("main", "topics"):
default = ' '.join(map(str, eval(config.get("main", "topics"))))
if args.quiet:
args.k = [int(n) for n in default.split()]
else:
raise NoOptionError
except NoOptionError:
default = ' '.join(map(str, range(20, 100, 20)))
while args.k is None:
ks = input("Number of Topics [Default '{0}']: ".format(default))
try:
if ks:
args.k = [int(n) for n in ks.split()]
elif not ks.strip():
args.k = [int(n) for n in default.split()]
if args.k:
print("\nTIP: number of topics can be specified with argument '-k N N N ...':")
print(" topicexplorer train %s -k %s\n" %\
(args.config_file, ' '.join(map(str, args.k))))
except ValueError:
print("Enter valid integers, separated by spaces!")
if args.processes < 0:
import multiprocessing
args.processes = multiprocessing.cpu_count() + args.processes
print("Loading corpus... ")
corpus = Corpus.load(corpus_filename)
try:
model_pattern = config.get("main", "model_pattern")
except NoOptionError:
model_pattern = None
if (model_pattern is not None and not args.rebuild and (args.quiet or args.cont or
bool_prompt("""Existing topic models found. You can continue training or start a new model.
Do you want to continue training your existing models? """, default=True))):
from vsm.model.lda import LDA
m = LDA.load(model_pattern.format(args.k[0]),
multiprocessing=args.processes > 1,
n_proc=args.processes)
if args.iter is None and not args.quiet: # pragma: no cover
args.iter = int_prompt("Total number of training iterations:",
default=int(m.iteration * 1.5), min=m.iteration)
print("\nTIP: number of training iterations can be specified with argument '--iter N':")
print(" topicexplorer train --iter %d %s\n" % (args.iter, args.config_file))
elif args.iter is None and args.quiet: # pragma: no cover
args.iter = int(m.iteration * 1.5)
del m
# if the set changes, build some new models and continue some old ones
config_topics = eval(config.get("main", "topics"))
if args.k != config_topics:
new_models = set(args.k) - set(config_topics)
continuing_models = set(args.k) & set(config_topics)
build_models(corpus, corpus_filename, model_path,
config.get("main", "context_type"),
new_models, n_iterations=args.iter,
n_proc=args.processes, seed=args.seed,
dry_run=args.dry_run)
model_pattern = continue_training(model_pattern, continuing_models,
args.iter, n_proc=args.processes,
dry_run=args.dry_run)
else:
model_pattern = continue_training(model_pattern, args.k, args.iter,
n_proc=args.processes,
dry_run=args.dry_run)
else:
# build a new model
if args.iter is None and not args.quiet: # pragma: no cover
args.iter = int_prompt("Number of training iterations:", default=200)
print("\nTIP: number of training iterations can be specified with argument '--iter N':")
print(" topicexplorer train --iter %d %s\n" % (args.iter, args.config_file))
elif args.iter is None and args.quiet: # pragma: no cover
args.iter = 200
# TODO: if only one context_type, make it just the one context type.
ctxs = corpus.context_types
if len(ctxs) == 1:
args.context_type = ctxs[0]
else:
ctxs = sorted(ctxs, key=lambda ctx: len(corpus.view_contexts(ctx)))
if args.context_type not in ctxs:
while args.context_type not in ctxs:
contexts = ctxs[:]
contexts[0] = contexts[0].upper()
contexts = '/'.join(contexts)
args.context_type = input("Select a context type [%s] : " % contexts)
if args.context_type.strip() == '':
args.context_type = ctxs[0]
if args.context_type == ctxs[0].upper():
args.context_type = ctxs[0]
print("\nTIP: context type can be specified with argument '--context-type TYPE':")
print(" topicexplorer train --context-type %s %s\n" % (args.context_type, args.config_file))
print("\nTIP: This configuration can be automated as:")
print(" topicexplorer train %s --iter %d --context-type %s -k %s\n" %\
(args.config_file, args.iter, args.context_type,
' '.join(map(str, args.k))))
model_pattern = build_models(corpus, corpus_filename, model_path,
args.context_type, args.k,
n_iterations=args.iter,
n_proc=args.processes, seed=args.seed,
dry_run=args.dry_run)
config.set("main", "model_pattern", model_pattern)
if args.context_type:
# test for presence, since continuing doesn't require context_type
config.set("main", "context_type", args.context_type)
args.k.sort()
config.set("main", "topics", str(args.k))
if not args.dry_run:
if config.has_option("main", "cluster"):
cluster_path = config.get("main", "cluster", fallback=None)
config.remove_option("main", "cluster")
try:
os.remove(cluster_path)
except IOError:
# fail silently on IOError
pass
with open(args.config_file, "w") as configfh:
config.write(configfh)
with open(config_file, 'r') as f:
config.readfp(f, config_file)
except IOError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode",
"zulip_email_suffix", "jabber_server_address",
"jabber_server_port"):
if (getattr(options, option) is None
and config.has_option("jabber_mirror", option)):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls", ):
if getattr(options, option) is None:
if config.has_option("jabber_mirror", option):
setattr(options, option,
config.getboolean("jabber_mirror", option))
else:
setattr(options, option, False)
if options.mode is None:
options.mode = "personal"
if options.zulip_email_suffix is None:
options.zulip_email_suffix = ''
if options.mode not in ('public', 'personal'):
config_error(
"Bad value for --mode: must be one of 'public' or 'personal'")
if None in (options.jid, options.jabber_password):
config_error(
class GlobalSettings(GObject.Object):
"""
Pitivi app settings.
The settings object loads settings from different sources, currently:
- the local configuration file,
- environment variables.
Modules declare which settings they wish to access by calling the
addConfigOption() class method during initialization.
@cvar options: A dictionnary of available settings.
@cvar environment: A list of the controlled environment variables.
"""
options = {}
environment = set()
defaults = {}
__gsignals__ = {}
def __init__(self, **unused_kwargs):
GObject.Object.__init__(self)
self._config = SafeConfigParser()
self._readSettingsFromConfigurationFile()
self._readSettingsFromEnvironmentVariables()
def _readSettingsFromConfigurationFile(self):
"""
Read the configuration from the user configuration file.
"""
try:
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
self._config.read(conf_file_path)
except UnicodeDecodeError:
unicode_error_dialog()
return
except ParsingError:
return
for (section, attrname, typ, key, env, value) in self.iterAllOptions():
if not self._config.has_section(section):
continue
if key and self._config.has_option(section, key):
if typ == int or typ == int:
try:
value = self._config.getint(section, key)
except ValueError:
# In previous configurations we incorrectly stored
# ints using float values.
value = int(self._config.getfloat(section, key))
elif typ == float:
value = self._config.getfloat(section, key)
elif typ == bool:
value = self._config.getboolean(section, key)
else:
value = self._config.get(section, key)
setattr(self, attrname, value)
@classmethod
def readSettingSectionFromFile(self, cls, section):
"""
Force reading a particular section of the settings file.
Use this if you dynamically determine settings sections/keys at runtime
(like in tabsmanager.py). Otherwise, the settings file would be read
only once (at the initialization phase of your module) and your config
sections would never be read, and thus values would be reset to defaults
on every startup because GlobalSettings would think they don't exist.
"""
if cls._config.has_section(section):
for option in cls._config.options(section):
# We don't know the value type in advance, just try them all.
try:
value = cls._config.getfloat(section, option)
except:
try:
value = cls._config.getint(section, option)
except:
try:
value = cls._config.getboolean(section, option)
except:
value = cls._config.get(section, option)
setattr(cls, section + option, value)
def _readSettingsFromEnvironmentVariables(self):
"""
Override options values using their registered environment variables.
"""
for section, attrname, typ, key, env, value in self.iterAllOptions():
if not env:
# This option does not have an environment variable name.
continue
var = get_env_by_type(typ, env)
if var is not None:
setattr(self, attrname, var)
def _writeSettingsToConfigurationFile(self):
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
for (section, attrname, typ, key, env_var, value) in self.iterAllOptions():
if not self._config.has_section(section):
self._config.add_section(section)
if key:
if value is not None:
self._config.set(section, key, str(value))
else:
self._config.remove_option(section, key)
try:
file = open(conf_file_path, 'w')
except IOError as OSError:
return
self._config.write(file)
file.close()
def storeSettings(self):
"""
Write settings to the user's local configuration file. Note that only
those settings which were added with a section and a key value are
stored.
"""
self._writeSettingsToConfigurationFile()
def iterAllOptions(self):
"""
Iterate over all registered options
@return: an iterator which yields a tuple of (attrname, type, key,
environment, value) for each option.
"""
for section, options in list(self.options.items()):
for attrname, (typ, key, environment) in list(options.items()):
yield section, attrname, typ, key, environment, getattr(self, attrname)
def isDefault(self, attrname):
return getattr(self, attrname) == self.defaults[attrname]
def setDefault(self, attrname):
setattr(self, attrname, self.defaults[attrname])
@classmethod
def addConfigOption(cls, attrname, type_=None, section=None, key=None,
environment=None, default=None, notify=False,):
"""
Add a configuration option.
This function should be called during module initialization, before
the config file is actually read. By default, only options registered
beforehand will be loaded.
See mainwindow.py and medialibrary.py for examples of usage.
If you want to add configuration options after initialization,
use the readSettingSectionFromFile method to force reading later on.
See tabsmanager.py for an example of such a scenario.
@param attrname: the attribute of this class which represents the option
@type attrname: C{str}
@param type_: type of the attribute. Unnecessary if default is given.
@type type_: a builtin or class
@param section: The section of the config file under which this option is
saved. This section must have been added with addConfigSection(). Not
necessary if key is not given.
@param key: the key under which this option is to be saved. Can be none if
this option should not be saved.
@type key: C{str}
@param notify: whether or not this attribute should emit notification
signals when modified (default is False).
@type notify: C{boolean}
"""
if section and section not in cls.options:
raise ConfigError(
"You must add the section \"%s\" first." % section)
if key and not section:
raise ConfigError(
"You must specify a section for key \"%s\"" % key)
if section and key in cls.options[section]:
raise ConfigError("Option \"%s\" is already in use.")
if hasattr(cls, attrname):
raise ConfigError("Settings attribute \"%s\" is already in use.")
if environment and environment in cls.environment:
raise ConfigError("Settings environment varaible \"%s\" is"
"already in use.")
if not type_ and default is None:
raise ConfigError("Settings attribute \"%s\" has must have a"
" type or a default." % attrname)
if not type_:
type_ = type(default)
if notify:
notification = Notification(attrname)
setattr(cls, attrname, notification)
setattr(cls, "_" + attrname, default)
GObject.signal_new(notification.signame,
cls,
GObject.SIGNAL_RUN_LAST,
None,
())
else:
setattr(cls, attrname, default)
if section and key:
cls.options[section][attrname] = type_, key, environment
cls.environment.add(environment)
cls.defaults[attrname] = default
@classmethod
def addConfigSection(cls, section):
"""
Add a section to the local config file.
@param section: The section name. This section must not already exist.
@type section: C{str}
"""
if section in cls.options:
raise ConfigError("Duplicate Section \"%s\"." % section)
cls.options[section] = {}
@classmethod
def notifiesConfigOption(cls, attrname):
signal_name = Notification.signalName(attrname)
GObject.signal_lookup(signal_name, cls)
except ImportError:
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
parser.read("test.cfg")
def do_nothing(self):
pass
# add a bunch of decorator metafunctions like LIB_CORE
# which can be wrapped around individual tests as needed
for section in parser.sections():
for option in parser.options(section):
if (parser.getboolean(section, option)):
vars()["%s_%s" % (section.upper(),
option.upper())] = lambda function: function
else:
vars()["%s_%s" % (section.upper(),
option.upper())] = lambda function: do_nothing
def BLANK_PCM_Reader(length, sample_rate=44100, channels=2,
bits_per_sample=16, channel_mask=None):
from audiotools.decoders import SameSample
if (channel_mask is None):
channel_mask = int(audiotools.ChannelMask.from_channels(channels))
return SameSample(sample=1,
def __init__(self):
"""Init"""
cp = SafeConfigParser(defaults={
# True if axial symmetry
'axisym': True,
# "Physical" length of the domain (in meters)
'LENGTH': 3000,
# Number of elements
'NSPEC': 250,
# Degree of the basis functions
'N': 4,
# Degree of basis functions in the first element
'NGLJ': 4,
# Number of time steps
'NTS': 2,
# Courant CFL number
'CFL': 0.45,
# Grid description
'GRID_TYPE': 'homogeneous',
'GRID_FILE': 'grid_homogeneous.txt',
'TICKS_FILE': 'ticks_homogeneous.txt',
# kg/m^3
'DENSITY': 2500,
# Pa
'RIGIDITY': 30000000000,
# Duration of the source in dt
'TSOURCE': 100,
# GLL point number on which the source is situated
'ISOURCE': 0,
# Maximum amplitude
'MAX_AMPL': 1e7,
# Source's type
'SOURCE_TYPE': 'ricker',
# Decay rate for the ricker
'DECAY_RATE': 2.628,
# Time steps between snapshots (0 == disabled)
'SNAPSHOT': 0,
# Plot grid, source, and periodic results
'PLOT': False,
# One image is displayed each DPLOT time step
'DPLOT': 10,
})
with open('Par_file') as f:
try:
# Python 3
cp.read_string('[global]\n' + f.read(), source='Par_file')
except AttributeError:
# Python 2
cp.readfp(FakeGlobalSectionHead(f))
self.axisym = cp.getboolean('global', 'AXISYM')
self.length = cp.getfloat('global', 'LENGTH')
self.nSpec = cp.getint('global', 'NSPEC')
self.N = cp.getint('global', 'N')
self.NGLJ = cp.getint('global', 'NGLJ')
self.nts = cp.getint('global', 'NTS')
self.cfl = cp.getfloat('global', 'CFL')
self.gridType = cp.get('global', 'GRID_TYPE').strip("'\"")
self.gridFile = cp.get('global', 'GRID_FILE').strip("'\"")
self.ticksFile = cp.get('global', 'TICKS_FILE').strip("'\"")
self.meanRho = cp.getfloat('global', 'DENSITY')
self.meanMu = cp.getfloat('global', 'RIGIDITY')
self.tSource = cp.getfloat('global', 'TSOURCE')
self.iSource = cp.getint('global', 'ISOURCE')
self.maxAmpl = cp.getfloat('global', 'MAX_AMPL')
self.sourceType = cp.get('global', 'SOURCE_TYPE').strip("'\"")
self.decayRate = cp.getfloat('global', 'DECAY_RATE')
self.snapshot = cp.getint('global', 'SNAPSHOT')
self.plot = cp.getboolean('global', 'PLOT')
self.dplot = cp.getfloat('global', 'DPLOT')
parser = argparse.ArgumentParser(
description='Spectral element method in a 1D medium')
parser.add_argument('--no-plot', action='store_true',
help='Force disable plotting')
args = parser.parse_args()
self.plot = self.plot and not args.no_plot
# Number of GLL points per elements
self.nGLL = self.N + 1
# Number of GLJ in the first element
self.nGLJ = self.NGLJ + 1
# Number of points in the array
self.nGlob = (self.nSpec - 1) * self.N + self.NGLJ + 1
self.ibool = self.make_global_index()
# Time step (will be updated)
self.dt = 0
# Gauss Lobatto Legendre points and integration weights:
try:
# Position of the GLL points in [-1,1]
self.ksiGLL = gll.GLL_POINTS[self.N]
# Integration weights
self.wGLL = gll.GLL_WEIGHTS[self.N]
except KeyError:
raise ValueError('N = %d is invalid!' % (self.N, ))
try:
# Position of the GLJ points in [-1,1]
self.ksiGLJ = gll.GLJ_POINTS[self.NGLJ]
# Integration weights
self.wGLJ = gll.GLJ_WEIGHTS[self.NGLJ]
except KeyError:
raise ValueError('NGLJ = %d is invalid!' % (self.NGLJ, ))
# Derivatives of the Lagrange polynomials at the GLL points
self.deriv = gll.lagrange_derivative(self.ksiGLL)
self.derivGLJ = gll.glj_derivative(self.ksiGLJ)
for opt, arg in options:
if opt in ('-l', '--list'):
list_option = True
elif opt in ('-c', '--config'):
config_file = arg
else:
showJelp("unknow option")
config = SafeConfigParser()
config.read(config_file)
if debug: eprint("CONFIG FILE PATH: " + config_file)
try:
debug = config.getboolean('shh', 'debug')
except:
pass
try:
SHH_INSTANCE = config.get('shh', 'instance').strip('"').strip()
if debug: eprint("INSTANCE: " + SHH_INSTANCE)
except Exception as e:
sys.exit("ERROR: instance is mandatory - " + str(e))
try:
SHH_USERNAME = config.get('shh', 'username').strip('"').strip()
if debug: eprint("SHH_USERNAME: "******"ERROR: username is mandatory")
