Ejemplo n.º 1
0
 def read_nrcan_data(self):
        ''' Read links from enercan and save them into bilinguages dataset file or database '''
        config = SafeConfigParser()
        config.read('nrcan.config')
        opener = urllib2.build_opener()
        infile = open(os.normpath('/temp/nrcan.links', "r"))
        outfile = open(os.normpath('/temp/nrcan.dat', "w"))
        for line in infile:
            links = str(line).strip("\n").split(', ')
            req = urllib2.Request(links[0])  
            try: 
                f = opener.open(req,timeout=50)
            except socket.timeout:
                print "socket timeout"
            package_dict = {'extras': {}, 'resources': [], 'tags': []}
            response = f.read() 
            n = json.loads(response)
            # create english fields
 
            for ckan, nrcan in config.items('package'):
                if nrcan == "SELECT":
                   print "SELECT"
                   package_dict[ckan] = schema_description.dataset_field_by_id[ckan]['choices'][1]['key']
                elif "$." in nrcan:
                    print "Use JSON Path"
                    print nrcan
                    print jsonpath(n, nrcan)
                elif nrcan:
                    print n[nrcan]
                    package_dict[ckan] = n[nrcan]
 
            pprint(package_dict)
 
            sys.exit()
            pass
Ejemplo n.º 2
0
 def getdata(self):
     global disallowed_prefixes
     data = self.request.recv(1024).strip()
     data = os.normpath(data)
     for prefix in disallowed_prefixes:
         if data.startswith(prefix):
             self.request.sendall("<b>403 Forbidden(Security Breach Detected!)</b>")
             break
     else:
         self.data = data
         self.handle_fake()
Ejemplo n.º 3
0
def main():
    if not os.path.exists(DATADIR):
        try:
            os.mkdir(DATADIR)
        except:
            print 'Unable to create data directory. Please create manually: %s' % os.normpath(DATADIR)
            exit()
    plogging.init_logger(name='client', logfile=os.path.join(DATADIR, 'clientlog.log'))


    should_exit = False
    try:
        SINGLE_LOCK = singleton.SingleInstance()
        print 'Successfully made single lock'
    except Exception, e:
        print 'The exception was: ', e
        should_exit = True
Ejemplo n.º 4
0
def main():
    if not os.path.exists(DATADIR):
        try:
            os.mkdir(DATADIR)
        except:
            print 'Unable to create data directory. Please create manually: %s' % os.normpath(
                DATADIR)
            exit()
    plogging.init_logger(name='client',
                         logfile=os.path.join(DATADIR, 'clientlog.log'))

    should_exit = False
    try:
        SINGLE_LOCK = singleton.SingleInstance()
        print 'Successfully made single lock'
    except Exception, e:
        print 'The exception was: ', e
        should_exit = True
Ejemplo n.º 5
0
def BTconverter():
    """
    Depreciated.
    Was used when the PSCF4GUI.py script didn't worked with hysplit format. Now
    it's ok.
    """
    #Convert the file for the PSCF
    with open(os.normpath('parameters/localParamBackTraj.json'),
              'r') as dataFile:
        param = json.load(dataFile)
    dirOutput = param["dirOutput"] + os.sep + param[
        "station"] + os.sep + "raw" + os.sep
    dirConverted = param["dirOutput"] + os.sep + param[
        "station"] + os.sep + "converted" + os.sep

    filelist = os.listdir(dirOutput)
    for file in filelist:
        f = open(dirOutput + file)
        lines = f.readlines()
        f.close()

        ##find where the BT data starts (number of meteo file + 4)
        l = int(lines[0].replace(' ', '')[0])

        ##find where the rainfall lives
        #lineRainfall= lines[l+3]
        #idxRainfall = 12 + lineRainfall.split().index('RAINFALL')

        nameout = dirConverted + file + '_converted.txt'
        fileout = open(nameout, 'w')

        regex = re.compile(" +")
        for i in range(l + 4, len(lines)):
            lineout = re.sub(regex, ";", lines[i].strip())
            lineout += '\n'
            fileout.write(lineout)
        fileout.close()
Ejemplo n.º 6
0
        sys.stdout.write('jojo_return_value JOB_STATUS=fail\n')
        sys.stdout.write('jojo_return_value ERROR_META_MISSING_KEY=' + name +
                         '\n')
        sys.exit(254)
    else:
        if verbose: print "DEBUG:: " + name + "=" + str(variable)


def display(err, text):
    """ Prints some informational messages that are preformatted."""
    print "************  >>" + err + "<< " + text


# Get parameters.
var_file = bash_real_escape_string(os.environ.get('ARTIFACT'))
var_file = os.normpath('/' + var_file).lstrip('/')
var_environment = bash_real_escape_string(os.environ.get('ENVIRONMENT'))
var_datacenter = bash_real_escape_string(os.environ.get('DATACENTER'))
var_tier = bash_real_escape_string(os.environ.get('TIER'))

var_signed = bash_real_escape_string(os.environ.get('SIGNED'))
# Don't let script continue without these parameters.
halt_if_value_empty(var_file, 'artifact')
halt_if_value_empty(var_environment, 'environment')
halt_if_value_empty(var_datacenter, 'datacenter')
halt_if_value_empty(var_tier, 'tier')


def return_artifact_servers():
    stream = open(configfile, 'r')
    try:
Ejemplo n.º 7
0
def glob_wildcards(pattern, files=None):
    """
    Glob the values of the wildcards by matching the given pattern to the
    filesystem.
    Returns a named tuple with a list of values for each wildcard.
    """
    from snakemake.io import _wildcard_regex, namedtuple, regex
    import regex as re

    pattern = os.path.normpath(pattern)
    first_wildcard = re.search("{[^{]", pattern)
    dirname = os.path.dirname(pattern[:first_wildcard.start()]
                              ) if first_wildcard else os.path.dirname(pattern)
    if not dirname:
        dirname = "."

    names = [
        match.group('name') for match in _wildcard_regex.finditer(pattern)
    ]
    Wildcards = namedtuple("Wildcards", names)
    wildcards = Wildcards(*[list() for name in names])

    pattern = regex(pattern)
    # work around partial matching bug in python regex module
    # by replacing matches for "\" with "[/\0]" (0x0 can't occur in filenames)
    pattern = re.sub('\\\\/', '[/\0]', pattern)
    cpattern = re.compile(pattern)

    def walker(dirname, pattern):
        """finds files/dirs matching `pattern` in `dirname`"""
        for dirpath, dirnames, filenames in os.walk(dirname):
            dirpath = os.path.normpath(dirpath)
            for f in filenames:
                if dirpath != ".":
                    f = os.path.join(dirpath, f)
                match = pattern.match(f)
                if match:
                    yield match
            for i in range(len(dirnames) - 1, -1, -1):
                d = dirnames[i]
                if dirpath != ".":
                    d = os.path.join(dirpath, d)
                match = pattern.match(os.path.join(d, ""), partial=True)
                if not match:
                    del dirnames[i]
                    continue
                if match.partial:
                    continue
                yield match

    print("searching {}".format(pattern))
    if files is None:
        for match in walker(dirname, cpattern):
            for name, value in match.groupdict().items():
                getattr(wildcards, name).append(value)
    else:
        for f in files:
            match = re.match(cpattern, os.normpath(f))
            if match:
                for name, value in match.groupdict().items():
                    getattr(wildcards, name).append(value)
    print("searching {}: done".format(pattern))
    return wildcards
Ejemplo n.º 8
0
 def send_static_file(self, filename):
     filename = os.normpath('/' + filename).lstrip('/')
     return super(SecuredStaticFlask, self).send_static_file(filename)
Ejemplo n.º 9
0
    """ Have this script bail if missing POST values. """
    if variable == "" or variable == """''""":
        sys.stdout.write('jojo_return_value ERROR_MESSAGE=Undefined key `'+name+'` in JSON POST request (required) \n')
        sys.stdout.write('jojo_return_value JOB_STATUS=fail\n')
        sys.stdout.write('jojo_return_value ERROR_META_MISSING_KEY='+name+'\n')
        sys.exit(254)
    else:
        if verbose: print "DEBUG:: " + name + "=" + str(variable)

def display(err, text):
    """ Prints some informational messages that are preformatted."""
    print "************  >>" + err + "<< " + text

# Get parameters.
var_file        = bash_real_escape_string(os.environ.get('ARTIFACT'))
var_file = os.normpath('/' + var_file).lstrip('/')
var_environment = bash_real_escape_string(os.environ.get('ENVIRONMENT'))
var_datacenter  = bash_real_escape_string(os.environ.get('DATACENTER'))
var_tier        = bash_real_escape_string(os.environ.get('TIER'))

var_signed      = bash_real_escape_string(os.environ.get('SIGNED'))
# Don't let script continue without these parameters.
halt_if_value_empty(var_file,'artifact')
halt_if_value_empty(var_environment,'environment')
halt_if_value_empty(var_datacenter,'datacenter')
halt_if_value_empty(var_tier,'tier')


def return_artifact_servers():
    stream = open(configfile, 'r')
    try: