def read_nrcan_data(self):
''' Read links from enercan and save them into bilinguages dataset file or database '''
config = SafeConfigParser()
config.read('nrcan.config')
opener = urllib2.build_opener()
infile = open(os.normpath('/temp/nrcan.links', "r"))
outfile = open(os.normpath('/temp/nrcan.dat', "w"))
for line in infile:
links = str(line).strip("\n").split(', ')
req = urllib2.Request(links[0])
try:
f = opener.open(req,timeout=50)
except socket.timeout:
print "socket timeout"
package_dict = {'extras': {}, 'resources': [], 'tags': []}
response = f.read()
n = json.loads(response)
# create english fields
for ckan, nrcan in config.items('package'):
if nrcan == "SELECT":
print "SELECT"
package_dict[ckan] = schema_description.dataset_field_by_id[ckan]['choices'][1]['key']
elif "$." in nrcan:
print "Use JSON Path"
print nrcan
print jsonpath(n, nrcan)
elif nrcan:
print n[nrcan]
package_dict[ckan] = n[nrcan]
pprint(package_dict)
sys.exit()
pass
def getdata(self):
global disallowed_prefixes
data = self.request.recv(1024).strip()
data = os.normpath(data)
for prefix in disallowed_prefixes:
if data.startswith(prefix):
self.request.sendall("<b>403 Forbidden(Security Breach Detected!)</b>")
break
else:
self.data = data
self.handle_fake()
def main():
if not os.path.exists(DATADIR):
try:
os.mkdir(DATADIR)
except:
print 'Unable to create data directory. Please create manually: %s' % os.normpath(DATADIR)
exit()
plogging.init_logger(name='client', logfile=os.path.join(DATADIR, 'clientlog.log'))
should_exit = False
try:
SINGLE_LOCK = singleton.SingleInstance()
print 'Successfully made single lock'
except Exception, e:
print 'The exception was: ', e
should_exit = True
def main():
if not os.path.exists(DATADIR):
try:
os.mkdir(DATADIR)
except:
print 'Unable to create data directory. Please create manually: %s' % os.normpath(
DATADIR)
exit()
plogging.init_logger(name='client',
logfile=os.path.join(DATADIR, 'clientlog.log'))
should_exit = False
try:
SINGLE_LOCK = singleton.SingleInstance()
print 'Successfully made single lock'
except Exception, e:
print 'The exception was: ', e
should_exit = True
def BTconverter():
"""
Depreciated.
Was used when the PSCF4GUI.py script didn't worked with hysplit format. Now
it's ok.
"""
#Convert the file for the PSCF
with open(os.normpath('parameters/localParamBackTraj.json'),
'r') as dataFile:
param = json.load(dataFile)
dirOutput = param["dirOutput"] + os.sep + param[
"station"] + os.sep + "raw" + os.sep
dirConverted = param["dirOutput"] + os.sep + param[
"station"] + os.sep + "converted" + os.sep
filelist = os.listdir(dirOutput)
for file in filelist:
f = open(dirOutput + file)
lines = f.readlines()
f.close()
##find where the BT data starts (number of meteo file + 4)
l = int(lines[0].replace(' ', '')[0])
##find where the rainfall lives
#lineRainfall= lines[l+3]
#idxRainfall = 12 + lineRainfall.split().index('RAINFALL')
nameout = dirConverted + file + '_converted.txt'
fileout = open(nameout, 'w')
regex = re.compile(" +")
for i in range(l + 4, len(lines)):
lineout = re.sub(regex, ";", lines[i].strip())
lineout += '\n'
fileout.write(lineout)
fileout.close()
sys.stdout.write('jojo_return_value JOB_STATUS=fail\n')
sys.stdout.write('jojo_return_value ERROR_META_MISSING_KEY=' + name +
'\n')
sys.exit(254)
else:
if verbose: print "DEBUG:: " + name + "=" + str(variable)
def display(err, text):
""" Prints some informational messages that are preformatted."""
print "************ >>" + err + "<< " + text
# Get parameters.
var_file = bash_real_escape_string(os.environ.get('ARTIFACT'))
var_file = os.normpath('/' + var_file).lstrip('/')
var_environment = bash_real_escape_string(os.environ.get('ENVIRONMENT'))
var_datacenter = bash_real_escape_string(os.environ.get('DATACENTER'))
var_tier = bash_real_escape_string(os.environ.get('TIER'))
var_signed = bash_real_escape_string(os.environ.get('SIGNED'))
# Don't let script continue without these parameters.
halt_if_value_empty(var_file, 'artifact')
halt_if_value_empty(var_environment, 'environment')
halt_if_value_empty(var_datacenter, 'datacenter')
halt_if_value_empty(var_tier, 'tier')
def return_artifact_servers():
stream = open(configfile, 'r')
try:
def glob_wildcards(pattern, files=None):
"""
Glob the values of the wildcards by matching the given pattern to the
filesystem.
Returns a named tuple with a list of values for each wildcard.
"""
from snakemake.io import _wildcard_regex, namedtuple, regex
import regex as re
pattern = os.path.normpath(pattern)
first_wildcard = re.search("{[^{]", pattern)
dirname = os.path.dirname(pattern[:first_wildcard.start()]
) if first_wildcard else os.path.dirname(pattern)
if not dirname:
dirname = "."
names = [
match.group('name') for match in _wildcard_regex.finditer(pattern)
]
Wildcards = namedtuple("Wildcards", names)
wildcards = Wildcards(*[list() for name in names])
pattern = regex(pattern)
# work around partial matching bug in python regex module
# by replacing matches for "\" with "[/\0]" (0x0 can't occur in filenames)
pattern = re.sub('\\\\/', '[/\0]', pattern)
cpattern = re.compile(pattern)
def walker(dirname, pattern):
"""finds files/dirs matching `pattern` in `dirname`"""
for dirpath, dirnames, filenames in os.walk(dirname):
dirpath = os.path.normpath(dirpath)
for f in filenames:
if dirpath != ".":
f = os.path.join(dirpath, f)
match = pattern.match(f)
if match:
yield match
for i in range(len(dirnames) - 1, -1, -1):
d = dirnames[i]
if dirpath != ".":
d = os.path.join(dirpath, d)
match = pattern.match(os.path.join(d, ""), partial=True)
if not match:
del dirnames[i]
continue
if match.partial:
continue
yield match
print("searching {}".format(pattern))
if files is None:
for match in walker(dirname, cpattern):
for name, value in match.groupdict().items():
getattr(wildcards, name).append(value)
else:
for f in files:
match = re.match(cpattern, os.normpath(f))
if match:
for name, value in match.groupdict().items():
getattr(wildcards, name).append(value)
print("searching {}: done".format(pattern))
return wildcards
def send_static_file(self, filename):
filename = os.normpath('/' + filename).lstrip('/')
return super(SecuredStaticFlask, self).send_static_file(filename)
""" Have this script bail if missing POST values. """
if variable == "" or variable == """''""":
sys.stdout.write('jojo_return_value ERROR_MESSAGE=Undefined key `'+name+'` in JSON POST request (required) \n')
sys.stdout.write('jojo_return_value JOB_STATUS=fail\n')
sys.stdout.write('jojo_return_value ERROR_META_MISSING_KEY='+name+'\n')
sys.exit(254)
else:
if verbose: print "DEBUG:: " + name + "=" + str(variable)
def display(err, text):
""" Prints some informational messages that are preformatted."""
print "************ >>" + err + "<< " + text
# Get parameters.
var_file = bash_real_escape_string(os.environ.get('ARTIFACT'))
var_file = os.normpath('/' + var_file).lstrip('/')
var_environment = bash_real_escape_string(os.environ.get('ENVIRONMENT'))
var_datacenter = bash_real_escape_string(os.environ.get('DATACENTER'))
var_tier = bash_real_escape_string(os.environ.get('TIER'))
var_signed = bash_real_escape_string(os.environ.get('SIGNED'))
# Don't let script continue without these parameters.
halt_if_value_empty(var_file,'artifact')
halt_if_value_empty(var_environment,'environment')
halt_if_value_empty(var_datacenter,'datacenter')
halt_if_value_empty(var_tier,'tier')
def return_artifact_servers():
stream = open(configfile, 'r')
try: