Example #1
0
    def test_parse(self):
        # Create simple-minded event handler
        evthdl = EventHandler()
        evthdl.buf = ''
        evthdl.offset = 0

        # Create temporary accounting file
        fd, path = tempfile.mkstemp()
        w = os.fdopen(fd, 'w')
        r = open(path, 'r')

        # Write the first bit to it
        l = L[:40]
        print "writing " + l
        w.write(l)
        w.flush()
        os.fsync(fd)
        self.assertEqual(list(common.parse(r, evthdl)), [])

        # Write the remainder to it
        l = L[40:]
        print "writing " + l
        w.write(l + '\n')
        w.flush()
        os.fsync(fd)
        self.assertEqual(len(list(common.parse(r, evthdl))), 1)

        # Cleanup
        r.close()
        w.close()
Example #2
0
 def check_previous_installation(self):
     log('Check previous installation')
     env_list = parse(self.ssh.exec_cmd('fuel env list'))
     if env_list:
         self.cleanup_fuel_environments(env_list)
         node_list = parse(self.ssh.exec_cmd('fuel node list'))
         if node_list:
             self.cleanup_fuel_nodes(node_list)
Example #3
0
 def listen(self):
   #Secondary thread, waits for packets, then parses and executes
   while self.connected:
     data = self.conn.recv(common.packetSize)
     if data != "":
       commands = common.parse(data)
       self.execute(commands)
 def env_exists(self, env_name):
     env_list = parse(exec_cmd('fuel env --list'))
     for env in env_list:
         if env[E['name']] == env_name and env[E['status']] == 'new':
             self.env_id = env[E['id']]
             return True
     return False
Example #5
0
def post(resource):
    """ Adds one or more documents to a resource. Each document is validated
    against the domain schema. If validation passes the document is inserted
    and ID_FIELD, LAST_UPDATED and DATE_CREATED along with a link to the
    document are returned. If validation fails, a list of validation issues
    is returned.

    :param resource: name of the resource involved.

    .. versionchanged:: 0.0.3
       JSON links. Superflous ``response`` container removed.
    """

    if len(request.form) == 0:
        abort(400)

    response = dict()
    date_utc = datetime.utcnow()

    schema = app.config['DOMAIN'][resource]['schema']
    validator = app.validator(schema, resource)

    for key, value in request.form.items():

        response_item = dict()
        issues = list()

        try:
            document = parse(value, resource)
            validation = validator.validate(document)
            if validation:
                document[config.LAST_UPDATED] = \
                    document[config.DATE_CREATED] = date_utc
                document[config.ID_FIELD] = app.data.insert(resource, document)

                response_item[config.ID_FIELD] = document[config.ID_FIELD]
                response_item[config.LAST_UPDATED] = \
                    document[config.LAST_UPDATED]
                response_item['_links'] = \
                    {'self': document_link(resource,
                                           response_item[config.ID_FIELD])}

            else:
                issues.extend(validator.errors)
        except ValidationError as e:
            raise e
        except Exception as e:
            issues.append(str(e))

        if len(issues):
            response_item['issues'] = issues
            response_item['status'] = config.STATUS_ERR
        else:
            response_item['status'] = config.STATUS_OK

        response[key] = response_item

    return response, None, None, 200
Example #6
0
def transform_lines(url):
    """ Download an xml file and add line numbering and ctsize it

    :param url: A Perseus Github Raw address
    :type url: str
    :param urn: The urn of the text
    :type urn: str
    :param lang: Iso code for lang
    :type lang: str

    """

    lang, urn, target, parsed = common.parse(url)

    if "grc" not in urn and "lat" not in urn:
        type_text = "translation"
    else:
        type_text = "edition"

    """
        Change div1 to div, moving their @type to @subtype 
    """

    # We find the lines
    lines = parsed.xpath("//l")
    # We set a counter
    i = 1
    # We loop over lines
    for line in lines:
        # We set the @n attribute using str(i) because .set(_,_) accepts only string
        line.set("n", str(i))
        # We increment the counter
        i += 1

    # We find divs called div1 or div2. Mind the |// !
    divs = parsed.xpath("//div1|//div2")
    # We loop over them
    for div in divs:
        # We change it's tag
        div.tag = "div"

    citations = list()

    """
        Add refsDecl information for CTS
    """
    citations.append(
        MyCapytain.resources.texts.tei.Citation(
            name="line", refsDecl="/tei:TEI/tei:text/tei:body/tei:div[@type='" + type_text + "']//tei:l[@n='$1']"
        )
    )

    try:
        common.write_and_clean(urn, lang, parsed, citations, target)
    except:
        print(urn + " failed")
Example #7
0
 def configure_environment(self):
     release_list = parse(exec_cmd('fuel release -l'))
     for release in release_list:
         if release[R['name']] == self.wanted_release:
             break
     config_env = ConfigureEnvironment(self.dea, YAML_CONF_DIR,
                                       release[R['id']],
                                       self.node_roles_dict)
     config_env.configure_environment()
     self.env_id = config_env.env_id
Example #8
0
File: deploy.py Project: rski/fuel
 def configure_environment(self):
     release_list = parse(exec_cmd('fuel release -l'))
     for release in release_list:
         if release[R['name']] == self.wanted_release:
             break
     config_env = ConfigureEnvironment(self.dea, YAML_CONF_DIR,
                                       release[R['id']],
                                       self.node_roles_dict)
     config_env.configure_environment()
     self.env_id = config_env.env_id
Example #9
0
def real(line):
    encrypted, sector, checksum = parse(line)
    letters = Counter(encrypted)
    del letters['-']
    return int(sector) if [
        letter[0]
        for letter in sorted(sorted(letters.items(), key=itemgetter(0)),
                             key=itemgetter(1),
                             reverse=True)[:5]
    ] == list(checksum) else 0
Example #10
0
 def check_supported_release(self):
     log('Check supported release: %s' % self.wanted_release)
     found = False
     release_list = parse(self.ssh.exec_cmd('fuel release -l'))
     for release in release_list:
         if release[R['name']] == self.wanted_release:
             found = True
             break
     if not found:
         err('This Fuel does not contain the following release: %s' %
             self.wanted_release)
Example #11
0
 def check_supported_release(self):
     log('Check supported release: %s' % self.wanted_release)
     found = False
     release_list = parse(self.ssh.exec_cmd('fuel release -l'))
     for release in release_list:
         if release[R['name']] == self.wanted_release:
             found = True
             break
     if not found:
         err('This Fuel does not contain the following release: %s'
             % self.wanted_release)
Example #12
0
    def verify_node_status(self):
        node_list = parse(exec_cmd('fuel --env %s node' % self.env_id))
        failed_nodes = []
        for node in node_list:
            if node[N['status']] != 'ready':
                failed_nodes.append((node[N['id']], node[N['status']]))

        if failed_nodes:
            summary = ''
            for node, status in failed_nodes:
                summary += '[node %s, status %s]\n' % (node, status)
            err('Deployment failed: %s' % summary, self.collect_logs)
Example #13
0
def post(resource):
    """ Adds one or more documents to a resource. Each document is validated
    against the domain schema. If validation passes the document is inserted
    and ID_FIELD, LAST_UPDATED and DATE_CREATED along with a link to the
    document are returned. If validation fails, a list of validation issues
    is returned.

    :param resource: name of the resource involved.
    """

    if len(request.form) == 0:
        abort(400)

    response = dict()
    date_utc = datetime.utcnow()

    schema = app.config['DOMAIN'][resource]['schema']
    validator = app.validator(schema, resource)

    for key, value in request.form.items():

        response_item = dict()
        issues = list()

        try:
            document = parse(value, resource)
            validation = validator.validate(document)
            if validation:
                document[config.LAST_UPDATED] = \
                    document[config.DATE_CREATED] = date_utc
                document[config.ID_FIELD] = app.data.insert(resource, document)

                response_item[config.ID_FIELD] = document[config.ID_FIELD]
                response_item[config.LAST_UPDATED] = \
                    document[config.LAST_UPDATED]
                response_item['link'] = \
                    document_link(resource, response_item[config.ID_FIELD])
            else:
                issues.extend(validator.errors)
        except ValidationError as e:
            raise e
        except Exception as e:
            issues.append(str(e))

        if len(issues):
            response_item['issues'] = issues
            response_item['status'] = config.STATUS_ERR
        else:
            response_item['status'] = config.STATUS_OK

        response[key] = response_item

    return response, None, None, 200
Example #14
0
    def verify_node_status(self):
        node_list = parse(exec_cmd('fuel node list'))
        failed_nodes = []
        for node in node_list:
            if node[N['status']] != 'ready' and node[N['cluster']] != 'None':
                failed_nodes.append((node[N['id']], node[N['status']]))

        if failed_nodes:
            summary = ''
            for node, status in failed_nodes:
                summary += '[node %s, status %s]\n' % (node, status)
            err('Deployment failed: %s' % summary)
Example #15
0
    def verify_node_status(self):
        node_list = parse(exec_cmd('fuel node list'))
        failed_nodes = []
        for node in node_list:
            if node[N['status']] != 'ready' and node[N['cluster']] != 'None':
                failed_nodes.append((node[N['id']], node[N['status']]))

        if failed_nodes:
            summary = ''
            for node, status in failed_nodes:
                summary += '[node %s, status %s]\n' % (node, status)
            err('Deployment failed: %s' % summary)
Example #16
0
def info():
    """Info account
    """
    url=vars.eHost+'/htsp/info'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "jwt" in conf_data):
        print("You need to login first")
        return
    headers={"Authorization":conf_data["jwt"]}
    response=common.sendingGet(url,headers)
    print(json.dumps(response["content"],indent=2))
Example #17
0
def info_key(key_name):
    """Info API key
    """
    url=vars.eHost+'/snippets/info'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not key_name in conf_data["branch"]):
        print("Bad key name")
        return
    message = {"api_key": conf_data["branch"][key_name]}
    response=common.sendingPost(url,message)
    print(json.dumps(response["content"],indent=2))
Example #18
0
 def discovery_waiting_loop(self, discovered_macs):
     WAIT_LOOP = 360
     SLEEP_TIME = 10
     all_discovered = False
     for i in range(WAIT_LOOP):
         node_list = parse(self.ssh.exec_cmd('fuel node list'))
         if node_list:
             self.node_discovery(node_list, discovered_macs)
         if self.all_blades_discovered():
             all_discovered = True
             break
         else:
             time.sleep(SLEEP_TIME)
     return all_discovered
Example #19
0
 def discovery_waiting_loop(self, discovered_macs):
     WAIT_LOOP = 360
     SLEEP_TIME = 10
     all_discovered = False
     for i in range(WAIT_LOOP):
         node_list = parse(self.ssh.exec_cmd('fuel node list'))
         if node_list:
             self.node_discovery(node_list, discovered_macs)
         if self.all_blades_discovered():
             all_discovered = True
             break
         else:
             time.sleep(SLEEP_TIME)
     return all_discovered
Example #20
0
def get(file_sign):
    """Get a cloud id_hjws
    """
    for i in file_sign:
        data = common.parse(i.name)
        if not data:
            print("Bad hjws file")
            return
        if (not "id_hjws" in data):
            print("Bad id_hjws")
            return
        url=vars.eHost+'/htsp/hjws/'+data["id_hjws"]
        response=common.sendingGet(url)
        print('\n'+i.name)
        print(json.dumps(response["content"],indent=2))
Example #21
0
    def reap_environment_info(self):
        network_file = ('%s/network_%s.yaml'
                        % (self.temp_dir, self.env_id))
        network = self.read_yaml(network_file)

        env = {'environment':
                   {'name': self.env[E['name']],
                    'net_segment_type':
                        network['networking_parameters']['segmentation_type']}}
        self.write_yaml(self.dea_file, env)
        wanted_release = None
        rel_list = parse(exec_cmd('fuel release'))
        for rel in rel_list:
            if rel[R['id']] == self.env[E['release_id']]:
                wanted_release = rel[R['name']]
        self.write_yaml(self.dea_file, {'wanted_release': wanted_release})
Example #22
0
def delete(key_name,id_hjws):
    """Delete a cloud hjws with a id_hjws
    """
    url=vars.eHost+'/htsp/hjws'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not key_name in conf_data["branch"]):
        print("Bad key name")
        return
    for i in id_hjws:
        message = {"api_key": conf_data["branch"][key_name], "id_hjws": i}
        response=common.sendingDel(url,message)
        print('\n'+i)
        print(json.dumps(response["content"],indent=2))
Example #23
0
File: reap.py Project: rski/fuel
    def reap_nodes_interfaces_transformations(self):
        node_list = parse(exec_cmd('fuel node'))
        real_node_ids = [node[N['id']] for node in node_list]
        real_node_ids.sort()
        min_node = real_node_ids[0]
        interfaces = {}
        transformations = {}
        dea_nodes = []
        dha_nodes = []

        for real_node_id in real_node_ids:
            node_id = int(real_node_id) - int(min_node) + 1
            self.last_node = node_id
            node = self.get_node_by_id(node_list, real_node_id)
            roles = commafy(node[N['roles']])
            if not roles:
                err('Fuel Node %s has no role' % real_node_id)
            dea_node = {'id': node_id, 'role': roles}
            dha_node = {'id': node_id}
            if_name, mac = self.reap_interface(real_node_id, interfaces)
            log('reap transformation for node %s' % real_node_id)
            tr_name = self.reap_transformation(real_node_id, roles,
                                               transformations)
            dea_node.update({
                'interfaces': if_name,
                'transformations': tr_name
            })

            dha_node.update({
                'pxeMac': mac if mac else None,
                'ipmiIp': None,
                'ipmiUser': None,
                'ipmiPass': None,
                'libvirtName': None,
                'libvirtTemplate': None
            })

            dea_nodes.append(dea_node)
            dha_nodes.append(dha_node)

        self.write_yaml(self.dha_file, {'nodes': dha_nodes}, False)
        self.write_yaml(self.dea_file, {'nodes': dea_nodes})
        self.write_yaml(self.dea_file, interfaces)
        self.write_yaml(self.dea_file, transformations)
        self.reap_fuel_node_info()
        self.write_yaml(self.dha_file, {'disks': DISKS})
Example #24
0
def info_key(key_name):
    """Info API key
    """
    url=vars.eHost+'/htsp/info/apikey'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not key_name in conf_data["branch"]):
        print("Bad key name")
        return
    message = {"api_key": conf_data["branch"][key_name]}
    response1=common.sendingPost(url,message)
    url=vars.eHost+'/htsp/pubkey/'+conf_data["branch"][key_name].split('.')[0]
    response=common.sendingGet(url)
    response1["content"]["pubkey"] = response["content"]
    print(json.dumps(response1["content"],indent=2))
Example #25
0
def partOneMod(instr: str) -> int:
    input_list = common.parse(instr)

    current_direction = "e"
    lat = 0  # north/south, pos/neg
    long = 0  # west/east, pos/neg

    location_history = []

    for instruction in input_list:
        current_direction, lad, lod = partOne.translate_movement(
            current_direction, instruction)
        lat += lad
        long += lod
        location_history.append((lat, long))

    return location_history
Example #26
0
    def reap_nodes_interfaces_transformations(self):
        node_list = parse(exec_cmd('fuel node'))
        real_node_ids = [node[N['id']] for node in node_list]
        real_node_ids.sort()
        min_node = real_node_ids[0]
        interfaces = {}
        transformations = {}
        dea_nodes = []
        dha_nodes = []

        for real_node_id in real_node_ids:
            node_id = int(real_node_id) - int(min_node) + 1
            self.last_node = node_id
            node = self.get_node_by_id(node_list, real_node_id)
            roles = commafy(node[N['roles']])
            if not roles:
                err('Fuel Node %s has no role' % real_node_id)
            dea_node = {'id': node_id,
                        'role': roles}
            dha_node = {'id': node_id}
            if_name, mac = self.reap_interface(real_node_id, interfaces)
            log('reap transformation for node %s' % real_node_id)
            tr_name = self.reap_transformation(real_node_id, roles,
                                               transformations)
            dea_node.update(
                {'interfaces': if_name,
                 'transformations': tr_name})

            dha_node.update(
                {'pxeMac': mac if mac else None,
                 'ipmiIp': None,
                 'ipmiUser': None,
                 'ipmiPass': None,
                 'libvirtName': None,
                 'libvirtTemplate': None})

            dea_nodes.append(dea_node)
            dha_nodes.append(dha_node)

        self.write_yaml(self.dha_file, {'nodes': dha_nodes}, False)
        self.write_yaml(self.dea_file, {'nodes': dea_nodes})
        self.write_yaml(self.dea_file, interfaces)
        self.write_yaml(self.dea_file, transformations)
        self.reap_fuel_node_info()
        self.write_yaml(self.dha_file, {'disks': DISKS})
Example #27
0
 def cleanup_fuel_environments(self, env_list):
     WAIT_LOOP = 60
     SLEEP_TIME = 10
     for env in env_list:
         log('Deleting environment %s' % env[E['id']])
         self.ssh.exec_cmd('fuel env --env %s --delete --force'
                           % env[E['id']])
     all_env_erased = False
     for i in range(WAIT_LOOP):
         env_list = parse(self.ssh.exec_cmd('fuel env list'))
         if env_list:
             time.sleep(SLEEP_TIME)
         else:
             all_env_erased = True
             break
     if not all_env_erased:
         err('Could not erase these environments %s'
             % [(env[E['id']], env[E['status']]) for env in env_list])
Example #28
0
def get(file_sign):
    """Get a snippet
    """
    for i in file_sign:
        print('\n'+i.name)
        message = common.parse(i.name)
        if not message:
            print("Bad json file: ", i.name)
            return
        if (not"id_data" in message):
            print("Bad snpt file: ", i.name)
            return
        url=vars.eHost+'/snippets/data/'+message["id_data"]
        response=common.sendingGet(url)
        if response["status_code"] != 200:
            print(json.dumps(response["content"],indent=2))
            return
        print(json.dumps(response["content"],indent=2))
Example #29
0
    def command_alias(self, line):
        """
        /alias <alias> <command> [args]
        """
        arg = common.shell_split(line)
        if len(arg) < 2:
            self.core.information("Alias: Not enough parameters", "Error")
            return
        alias = arg[0]
        command = arg[1]
        tmp_args = arg[2] if len(arg) > 2 else ""

        if alias in self.core.commands or alias in self.commands:
            self.core.information("Alias: command already exists", "Error")
            return
        self.commands[alias] = lambda arg: self.get_command(command)(parse(arg, tmp_args))
        self.add_command(alias, self.commands[alias], "This command is an alias for /%s %s" % (command, tmp_args))
        self.core.information("Alias /%s successfuly created" % alias, "Info")
Example #30
0
 def cleanup_fuel_environments(self, env_list):
     WAIT_LOOP = 60
     SLEEP_TIME = 10
     for env in env_list:
         log('Deleting environment %s' % env[E['id']])
         self.ssh.exec_cmd('fuel env --env %s --delete --force' %
                           env[E['id']])
     all_env_erased = False
     for i in range(WAIT_LOOP):
         env_list = parse(self.ssh.exec_cmd('fuel env list'))
         if env_list:
             time.sleep(SLEEP_TIME)
         else:
             all_env_erased = True
             break
     if not all_env_erased:
         err('Could not erase these environments %s' %
             [(env[E['id']], env[E['status']]) for env in env_list])
Example #31
0
    def intro(self):
        delete(self.dea_file)
        delete(self.dha_file)
        self.temp_dir = tempfile.mkdtemp()
        date = time.strftime('%c')
        self.write(self.dea_file, DEA_1.format(date=date,
                                               comment=self.comment), False)
        self.write(self.dha_file, DHA_1.format(date=date,
                                               comment=self.comment))
        self.get_env()

        # Need to download deployment with explicit node ids
        node_list = parse(exec_cmd('fuel node'))
        real_node_ids = [node[N['id']] for node in node_list]
        real_node_ids.sort()
        self.download_node_config(','.join(real_node_ids))

        self.download_config('settings')
        self.download_config('network')
Example #32
0
    def intro(self):
        delete(self.dea_file)
        delete(self.dha_file)
        self.temp_dir = tempfile.mkdtemp()
        date = time.strftime('%c')
        self.write(self.dea_file,
                   DEA_1.format(date=date, comment=self.comment), False)
        self.write(self.dha_file,
                   DHA_1.format(date=date, comment=self.comment))
        self.get_env()

        # Need to download deployment with explicit node ids
        node_list = parse(exec_cmd('fuel node'))
        real_node_ids = [node[N['id']] for node in node_list]
        real_node_ids.sort()
        self.download_node_config(','.join(real_node_ids))

        self.download_config('settings')
        self.download_config('network')
Example #33
0
File: reap.py Project: rski/fuel
    def reap_environment_info(self):
        network_file = ('%s/network_%s.yaml' % (self.temp_dir, self.env_id))
        network = self.read_yaml(network_file)

        env = {
            'environment': {
                'name':
                self.env[E['name']],
                'net_segment_type':
                network['networking_parameters']['segmentation_type']
            }
        }
        self.write_yaml(self.dea_file, env)
        wanted_release = None
        rel_list = parse(exec_cmd('fuel release'))
        for rel in rel_list:
            if rel[R['id']] == self.env[E['release_id']]:
                wanted_release = rel[R['name']]
        self.write_yaml(self.dea_file, {'wanted_release': wanted_release})
Example #34
0
def account():
    """Delete a account
    """
    url = vars.eHost + '/htsp/account'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "jwt" in conf_data):
        print("You need to login first")
        return
    message = {}
    headers = {"Authorization": conf_data["jwt"]}
    click.confirm('Do you want to continue?', abort=True)
    response = common.sendingDel(url, message, headers)
    if response["status_code"] != 200:
        print(json.dumps(response["content"], indent=2))
        return
    print(json.dumps(response["content"], indent=2))
    os.remove(vars.fileConf)
Example #35
0
def partTwoMod(instr: str) -> int:
    input_list = common.parse(instr)

    waypoint_delta = (1, 10)
    lat = 0  # north/south, pos/neg
    long = 0  # west/east, pos/neg

    location_history = []
    waypoint_history = []

    for instruction in input_list:
        waypoint_delta, lad, lod = partTwo.translate_movement(
            waypoint_delta, instruction)
        lat += lad
        long += lod
        location_history.append((lat, long))
        waypoint_history.append(
            (lat + waypoint_delta[0], long + waypoint_delta[1]))

    return location_history, waypoint_history
Example #36
0
def branch(email,suite):
    """Create a branch
    """
    url=vars.eHost+'/htsp/branch'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "jwt" in conf_data):
        print("You need to login first")
        return
    if (not email in conf_data["subject"]):
        print("Bad subject")
        return
    if (not "branch" in conf_data):
        print("You need to init first")
        return
    if (not isinstance(conf_data["branch"],dict)):
        print("Bad branches")
        return
    if (not email in conf_data):
        print("You need to verify your subject")
        return
    if (not isinstance(conf_data[email],list)):
        print("You need to init first")
        return
    branch = input("Your branch name: ")
    if (branch in conf_data["branch"]):
        print("Branch already exists")
        return
    message = {"id_sec": conf_data["subject"][email],"branch":branch, "suite": suite}
    headers={"Authorization":conf_data["jwt"]}
    response=common.sendingPost(url,message,headers)
    if response["status_code"] != 200:
        print(json.dumps(response["content"],indent=2))
        return
    conf_data["branch"][branch]= response["content"]["api_key"]
    conf_data[email].append(branch)
    with open(vars.fileConf, 'w') as outfile:
        json.dump(conf_data, outfile,indent=2)
    print("Branch OK!")
Example #37
0
def update(key_name):
    """Update API KEY credentials
    """
    url=vars.eHost+'/htsp/apikey'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not key_name in conf_data["branch"]):
        print("Bad key name")
        return
    if (not "jwt" in conf_data):
        print("You need to login first")
        return
    message = {"api_key": conf_data["branch"][key_name]}
    headers={"Authorization":conf_data["jwt"]}
    response=common.sendingPut(url,message,headers)
    conf_data["branch"][key_name]= response["content"]["api_key"]
    with open(vars.fileConf, 'w') as outfile:
        json.dump(conf_data, outfile,indent=2)
    print("API updated OK!")
Example #38
0
def login():
    """
    3vidence login
    """
    url = vars.eHost + '/auth/login'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "email" in conf_data) or (not "password" in conf_data):
        print("Bad email/password")
        return
    data = {"email": conf_data["email"], "password": conf_data["password"]}
    response = common.sendingPost(url, data)
    if response["status_code"] != 200:
        print(json.dumps(response["content"], indent=2))
        return
    conf_data["jwt"] = response["content"]["jwt"]
    with open(vars.fileConf, 'w') as outfile:
        json.dump(conf_data, outfile, indent=2)
    print("Login OK")
Example #39
0
def update():
    """
    3vidence password update
    """
    url = vars.eHost + '/auth/password'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "jwt" in conf_data):
        print("Bad key name")
        return
    headers = {"Authorization": conf_data["jwt"]}
    message = {"": ""}
    response = common.sendingPut(url, message, headers)
    if response["status_code"] != 200:
        print(json.dumps(response["content"], indent=2))
        return
    conf_data["password"] = response["content"]["password"]
    with open(vars.fileConf, 'w') as outfile:
        json.dump(conf_data, outfile, indent=2)
    print("Password updated")
Example #40
0
def branch(subject, key_name):
    """Delete a key
    """
    url = vars.eHost + '/htsp/branch'
    conf_data = common.parse(vars.fileConf)
    if not conf_data:
        print("Bad config file")
        return
    if (not "jwt" in conf_data):
        print("You need to login first")
        return
    if (not isinstance(conf_data[subject], list)):
        print("Bad file config subject/branch")
        return
    if (conf_data[subject].count(key_name) == 0):
        print("Bad subject")
        return
    if (not "branch" in conf_data):
        print("You need to init first")
        return
    if (not isinstance(conf_data["branch"], dict)):
        print("Bad branches")
        return
    if (not key_name in conf_data["branch"]):
        print("Bad key name")
        return
    message = {"kid": conf_data["branch"][key_name].split('.')[0]}
    headers = {"Authorization": conf_data["jwt"]}
    click.confirm('Do you want to continue?', abort=True)
    response = common.sendingDel(url, message, headers)
    if response["status_code"] != 200:
        print(json.dumps(response["content"], indent=2))
        return
    print(json.dumps(response["content"], indent=2))
    del conf_data["branch"][key_name]
    conf_data[subject].remove(key_name)
    with open(vars.fileConf, 'w') as outfile:
        json.dump(conf_data, outfile, indent=2)
Example #41
0
    def run_deploy(self):
        SLEEP_TIME = 60
        LOG_FILE = 'cloud.log'

        log('Starting deployment of environment %s' % self.env_id)
        p = run_proc('fuel --env %s deploy-changes | strings > %s' %
                     (self.env_id, LOG_FILE))

        ready = False
        for i in range(int(self.deploy_timeout)):
            env = parse(exec_cmd('fuel env --env %s' % self.env_id))
            log('Environment status: %s' % env[0][E['status']])
            r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
            if r:
                log(r)
            if env[0][E['status']] == 'operational':
                ready = True
                break
            elif (env[0][E['status']] == 'error'
                  or env[0][E['status']] == 'stopped'):
                break
            else:
                time.sleep(SLEEP_TIME)

        p.poll()
        if p.returncode == None:
            log('The process deploying the changes has not yet finished.')
            log('''The file %s won't be deleted''' % LOG_FILE)
        else:
            delete(LOG_FILE)

        if ready:
            log('Environment %s successfully deployed' % self.env_id)
        else:
            self.collect_error_logs()
            err('Deployment failed, environment %s is not operational' %
                self.env_id)
Example #42
0
    def run_deploy(self):
        SLEEP_TIME = 60
        LOG_FILE = 'cloud.log'

        log('Starting deployment of environment %s' % self.env_id)
        p = run_proc('fuel --env %s deploy-changes | strings > %s'
                     % (self.env_id, LOG_FILE))

        ready = False
        for i in range(int(self.deploy_timeout)):
            env = parse(exec_cmd('fuel env --env %s' % self.env_id))
            log('Environment status: %s' % env[0][E['status']])
            r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
            if r:
                log(r)
            if env[0][E['status']] == 'operational':
                ready = True
                break
            elif (env[0][E['status']] == 'error'
                  or env[0][E['status']] == 'stopped'):
                break
            else:
                time.sleep(SLEEP_TIME)

        p.poll()
        if p.returncode == None:
            log('The process deploying the changes has not yet finished.')
            log('''The file %s won't be deleted''' % LOG_FILE)
        else:
            delete(LOG_FILE)

        if ready:
            log('Environment %s successfully deployed' % self.env_id)
        else:
            self.collect_error_logs()
            err('Deployment failed, environment %s is not operational'
                % self.env_id)
def transform(url):
    """ Download an xml file and add line numbering and ctsize it

    :param url: A Perseus Github Raw address
    :type url: str
    :param urn: The urn of the text
    :type urn: str
    :param lang: Iso code for lang
    :type lang: str

    """


    lang, urn, target, parsed = common.parse(url)

    if "grc" not in urn and "lat" not in urn:
        type_text = "translation"
    else:
        type_text = "edition"

    # We find divs called div1
    div1_group = parsed.xpath("//div1")
    i = 1
    for div1 in div1_group:
        # We change it's tag
        div1.tag = "div"
        # To deal with different subtype, we get the former attribute value of type and put it to subtype
        div1_subtype = div1.get("type")
        div1.set("subtype", div1_subtype)
        div1.set("type", "textpart")

        if "n" not in dict(div1.attrib):
            div1.set("n", str(i))
        i += 1
                
        
    """
        Change div2 to div, moving their @type to @subtype 
    """    
    # We find divs called div2
    i = 1
    div2_group = parsed.xpath("//div2")
    for div2 in div2_group:
        # We change it's tag
        div2.tag = "div"
        # To deal with different subtype, we get the former attribute value of type and put it to subtype
        div2_subtype = div2.get("type")
        div2.set("subtype", div2_subtype)
        div2.set("type", "textpart")

        if "n" not in dict(div2.attrib):
            div2.set("n", str(i))
        i += 1
        
    """
        Change div3 to div, moving their @type to @subtype 
    """    
    # We find divs called div2
    i = 1
    div3_group = parsed.xpath("//div3")
    for div3 in div3_group:
        # We change it's tag
        div3.tag = "div"
        # To deal with different subtype, we get the former attribute value of type and put it to subtype
        div3_subtype = div3.get("type")
        div3.set("subtype", div3_subtype)
        div3.set("type", "textpart")

        if "n" not in dict(div3.attrib):
            div3.set("n", str(i))
        i += 1

    """
        Add refsDecl information for CTS
    """
    citations = []
    # Used only if div3 > 0
    if len(div3_group) > 0:
        citations.append(
            MyCapytain.resources.texts.tei.Citation(
                name=div3_subtype, 
                refsDecl="/tei:TEI/tei:text/tei:body/tei:div[@type='"+type_text+"']/tei:div[@n='$1']/tei:div[@n='$2']/tei:div[@n='$3']"
            )
        )
    # Used only if div2 > 0
    if len(div2_group) > 0:
        citations.append(
            MyCapytain.resources.texts.tei.Citation(
                name=div2_subtype, 
                refsDecl="/tei:TEI/tei:text/tei:body/tei:div[@type='"+type_text+"']/tei:div[@n='$1']/tei:div[@n='$2']"
            )
        )
    citations.append(
        MyCapytain.resources.texts.tei.Citation(
            name=div1_subtype, 
            refsDecl="/tei:TEI/tei:text/tei:body/tei:div[@type='"+type_text+"']/tei:div[@n='$1']"
        )
    )

    try:
        common.write_and_clean(urn, lang, parsed, citations, target)   
    except Exception as E:
        print(urn + " failed")
        print(E)
Example #44
0
File: patch.py Project: klyr/eve
def patch(resource, **lookup):
    """Perform a document patch/update. Updates are first validated against
    the resource schema. If validation passes, the document is updated and
    an OK status update is returned. If validation fails, a set of validation
    issues is returned.

    :param resource: the name of the resource to which the document belongs.
    :param **lookup: document lookup query.

    .. versionchanged:: 0.0.7
       Support for Rate-Limiting.

    .. versionchanged:: 0.0.6
        ETag is now computed without the need of an additional db lookup

    .. versionchanged:: 0.0.5
        Support for 'aplication/json' Content-Type.

    .. versionchanged:: 0.0.4
       Added the ``requires_auth`` decorator.

    .. versionchanged:: 0.0.3
       JSON links. Superflous ``response`` container removed.
    """
    payload = payload_()
    if len(payload) > 1:
        # only one update-per-document supported
        abort(400)

    original = get_document(resource, **lookup)
    if not original:
        # not found
        abort(404)

    schema = app.config["DOMAIN"][resource]["schema"]
    validator = app.validator(schema, resource)

    object_id = original[config.ID_FIELD]
    last_modified = None
    etag = None

    issues = []

    key = payload.keys()[0]
    value = payload[key]

    response_item = {}

    try:
        updates = parse(value, resource)
        validation = validator.validate_update(updates, object_id)
        if validation:
            # the mongo driver has a different precision than the python
            # datetime. since we don't want to reload the document once it has
            # been updated, and we still have to provide an updated etag,
            # we're going to update the local version of the 'original'
            # document, and we will use it for the etag computation.
            original.update(updates)
            # some datetime precision magic
            updates[config.LAST_UPDATED] = original[config.LAST_UPDATED] = datetime.utcnow().replace(microsecond=0)
            etag = document_etag(original)

            app.data.update(resource, object_id, updates)
            response_item[config.ID_FIELD] = object_id
            last_modified = response_item[config.LAST_UPDATED] = original[config.LAST_UPDATED]

            # metadata
            response_item["etag"] = etag
            response_item["_links"] = {"self": document_link(resource, object_id)}
        else:
            issues.extend(validator.errors)
    except ValidationError, e:
        # TODO should probably log the error and abort 400 instead (when we
        # got logging)
        issues.append(str(e))
Example #45
0
 def listen(self):
   while self.connected:
     data = self.socket.recv(common.packetSize)
     for cmd,params in common.parse(data):
       print(cmd)
       self.event.notify("cmdRecv", cmd, *params)
Example #46
0
    def process_IN_MODIFY(self, event):
        '''
        Handles a file modification event in currently-read files as well as
        in newly-created ones, typically when one or more event records have
        been appended to the accounting file. The new records are sent to the
        database here.
        '''

        # Issue: not sure how to deal with the currently-read file if it gets
        # overwritten. Two options:
        # 1. The file pointer should be reset because it's a new file
        #    altogether.
        # 2. The file pointer shouldn't be reset because it's the same data
        #    which has been overwritten in the process of being added new lines
        #    (e.g. vi).
        # Let's go for the second option, which the following does without even
        # needing any CREATE event.

        try:
            l = latest(self.acctdir)
            if l == self.acctfile:
                # There's no new accounting file

                f = open(self.acctdir + '/' + l)
                recs = common.parse(f, self)

                insertc, errorc, heartbeat = \
                    common.insert(self.logger, common.CETAB, recs,
                                  self.connection, self.insertc, self.errorc,
                                  self.heartbeat, self.heartbeatdelta)
                f.close()

                self.insertc = insertc
                self.errorc = errorc
                self.heartbeat = heartbeat
            else:
                # There's a new accounting file (or we weren't reading any)

                # Finish reading the current one if we were reading one
                if self.acctfile != None:
                    f = open(self.acctdir + '/' + self.acctfile)
                    recs = common.parse(f, self)

                    insertc, errorc, heartbeat = \
                        common.insert(self.logger, common.CETAB, recs,
                                      self.connection, self.insertc,
                                      self.errorc, self.heartbeat,
                                      self.heartbeatdelta)
                    f.close()

                    self.insertc = insertc
                    self.errorc = errorc
                    self.heartbeat = heartbeat

                # Read the new one
                self.acctfile = l
                self.offset = 0
                self.logger.info("Will now be watching %s" % self.acctfile)

                f = open(self.acctdir + '/' + self.acctfile)
                recs = common.parse(f, self)

                insertc, errorc, heartbeat = \
                    common.insert(self.logger, common.CETAB, recs,
                                  self.connection, self.insertc, self.errorc,
                                  self.heartbeat, self.heartbeatdelta)
                f.close()

                self.insertc = insertc
                self.errorc = errorc
                self.heartbeat = heartbeat
        except common.AcctError: # As raised by latest
            # No need to fuss if a file we're not interested in gets changed
            pass
Example #47
0
def post(resource):
    """ Adds one or more documents to a resource. Each document is validated
    against the domain schema. If validation passes the document is inserted
    and ID_FIELD, LAST_UPDATED and DATE_CREATED along with a link to the
    document are returned. If validation fails, a list of validation issues
    is returned.

    :param resource: name of the resource involved.

    .. versionchanged:: 0.0.6
        Support for bulk inserts.
        Please note: validation constraints are checked against the database,
        and not between the payload documents themselves. This causes an
        interesting corner case: in the event of a multiple documents payload
        where two or more documents carry the same value for a field where the
        'unique' constraint is set, the payload will validate successfully, as
        there are no duplicates in the database (yet). If this is an issue, the
        client can always send the documents once at a time for insertion, or
        validate locally before submitting the payload to the API.

    .. versionchanged:: 0.0.5
       Support for 'application/json' Content-Type .
       Support for 'user-restricted resource access'.

    .. versionchanged:: 0.0.4
       Added the ``requires_auth`` decorator.

    .. versionchanged:: 0.0.3
       JSON links. Superflous ``response`` container removed.
    """

    date_utc = datetime.utcnow().replace(microsecond=0)

    schema = app.config['DOMAIN'][resource]['schema']
    validator = app.validator(schema, resource)
    documents = []
    issues = []

    # validation, and additional fields
    payl = payload()
    for key, value in payl.items():
        document = []
        doc_issues = []
        try:
            document = parse(value, resource)
            validation = validator.validate(document)
            if validation:
                # validation is successful
                document[config.LAST_UPDATED] = \
                    document[config.DATE_CREATED] = date_utc

                # if 'user-restricted resource access' is enabled and there's
                # an Auth request active, inject the username into the document
                username_field = \
                    app.config['DOMAIN'][resource]['auth_username_field']
                if username_field and request.authorization:
                    document[username_field] = request.authorization.username

            else:
                # validation errors added to list of document issues
                doc_issues.extend(validator.errors)
        except ValidationError as e:
            raise e
        except Exception as e:
            # most likely a problem with the incoming payload, report back to
            # the client as if it was a validation issue
            doc_issues.append(str(e))

        issues.append(doc_issues)

        if len(doc_issues) == 0:
            documents.append(document)

    # bulk insert
    if len(documents):
        ids = app.data.insert(resource, documents)

    # build response payload
    response = {}
    for key, doc_issues in zip(payl.keys(), issues):
        response_item = {}
        if len(doc_issues):
            response_item['status'] = config.STATUS_ERR
            response_item['issues'] = doc_issues
        else:
            response_item['status'] = config.STATUS_OK
            response_item[config.ID_FIELD] = ids.pop(0)
            document = documents.pop(0)
            response_item[config.LAST_UPDATED] = document[config.LAST_UPDATED]
            response_item['etag'] = document_etag(document)
            response_item['_links'] = \
                {'self': document_link(resource,
                                       response_item[config.ID_FIELD])}

        response[key] = response_item

    return response, None, None, 200
Example #48
0
File: patch.py Project: marchon/eve
def patch(resource, **lookup):
    """Perform a document patch/update. Updates are first validated against
    the resource schema. If validation passes, the document is updated and
    an OK status update is returned. If validation fails, a set of validation
    issues is returned.

    :param resource: the name of the resource to which the document belongs.
    :param **lookup: document lookup query.
    """
    if len(request.form) > 1 or len(request.form) == 0:
        # only one update-per-document supported
        abort(400)

    original = get_document(resource, **lookup)
    if not original:
        # not found
        abort(404)

    schema = app.config['DOMAIN'][resource]['schema']
    validator = app.validator(schema, resource)

    object_id = original[config.ID_FIELD]
    last_modified = None
    etag = None

    issues = list()

    key = request.form.keys()[0]
    value = request.form[key]

    response_item = dict()

    try:
        updates = parse(value, resource)
        validation = validator.validate_update(updates, object_id)
        if validation:
            updates[config.LAST_UPDATED] = datetime.utcnow()
            app.data.update(resource, object_id, updates)

            # TODO computing etag without reloading the document
            # would be ideal. However, for reasons that need fruther
            # investigation, an etag computed on original.update(updates)
            # won't provide the same result as the saved document.
            # this has probably comething to do with a) the different
            # precision between the BSON (milliseconds) python datetime and,
            # b), the string representation of the documents (being dicts)
            # not matching.
            #
            # TL;DR: find a way to compute a reliable etag without reloading
            updated = app.data.find_one(resource,
                                        **{config.ID_FIELD: object_id})
            updated[config.LAST_UPDATED] = \
                updated[config.LAST_UPDATED].replace(tzinfo=None)
            etag = document_etag(updated)

            response_item[config.ID_FIELD] = object_id
            last_modified = response_item[config.LAST_UPDATED] = \
                updated[config.LAST_UPDATED]

            # metadata
            response_item['etag'] = etag
            response_item['link'] = document_link(resource, object_id)
        else:
            issues.extend(validator.errors)
    except ValidationError, e:
        # TODO should probably log the error and abort 400 instead (when we
        # got logging)
        issues.append(str(e))
Example #49
0
 def get_env(self):
     env_list = parse(exec_cmd('fuel env'))
     if len(env_list) > 1:
         err('Not exactly one environment')
     self.env = env_list[0]
     self.env_id = self.env[E['id']]
Example #50
0
File: post.py Project: Tefnet/eve
def post(resource):
    """ Adds one or more documents to a resource. Each document is validated
    against the domain schema. If validation passes the document is inserted
    and ID_FIELD, LAST_UPDATED and DATE_CREATED along with a link to the
    document are returned. If validation fails, a list of validation issues
    is returned.

    :param resource: name of the resource involved.

    .. versionchanged:: 0.0.5
       Support for 'application/json' Content-Type .
       Support for 'user-restricted resource access'.

    .. versionchanged:: 0.0.4
       Added the ``reqiores_auth`` decorator.

    .. versionchanged:: 0.0.3
       JSON links. Superflous ``response`` container removed.
    """

    response = {}
    date_utc = datetime.utcnow()

    schema = app.config['DOMAIN'][resource]['schema']
    validator = app.validator(schema, resource)

    for key, value in payload().items():

        response_item = {}
        issues = []

        try:
            document = parse(value, resource)
            validation = validator.validate(document)
            if validation:
                document[config.LAST_UPDATED] = \
                    document[config.DATE_CREATED] = date_utc

                # if 'user-restricted resource access' is enabled and there's
                # an Auth request active, inject the username into the document
                username_field = \
                    app.config['DOMAIN'][resource]['auth_username_field']
                if username_field and request.authorization:
                    document[username_field] = request.authorization.username

                document[config.ID_FIELD] = app.data.insert(resource, document)

                response_item[config.ID_FIELD] = document[config.ID_FIELD]
                response_item[config.LAST_UPDATED] = \
                    document[config.LAST_UPDATED]
                response_item['_links'] = \
                    {'self': document_link(resource,
                                           response_item[config.ID_FIELD])}

            else:
                issues.extend(validator.errors)
        except ValidationError as e:
            raise e
        except Exception as e:
            issues.append(str(e))

        if len(issues):
            response_item['issues'] = issues
            response_item['status'] = config.STATUS_ERR
        else:
            response_item['status'] = config.STATUS_OK

        response[key] = response_item

    return response, None, None, 200
Example #51
0
        cfgpath = sys.argv[1]
    print("Using config file: " + cfgpath)

    required_fields = {
        "client": [("trustedca", "string")],
        "server": [("interface", "string"), ("port", "int"),
                   ("keypath", "string"), ("certpath", "string")]
    }

    optional_fields = {
        "server": [("logfile", "string", None), ("outputdir", "string", "."),
                   ("maxsize", "int", 50)]
    }

    try:
        config = common.parse(cfgpath, required_fields, optional_fields)
    except Exception as e:
        print(str(e), file=sys.stderr)
        sys.exit(-1)

    config["maxsize"] *= 1024 * 1024

    logger = common.getLogger(logfile=config['logfile'])
    context = common.getContext(protocol=ssl.PROTOCOL_TLS_SERVER,
                                certpath=config['certpath'],
                                trustedca=config['trustedca'],
                                keypath=config['keypath'])

    logger.info("parsed config: ")
    for k, v in config.items():
        logger.info("{}: {}".format(k, v))
Example #52
0
# 	('napi_gro_receive',			1),
# 	('udp_send_skb',				2),
# 	('tcp_transmit_skb',			2),
# 	('br_handle_frame_finish',		0),
# 	('netif_receive_skb_internal',	0),
# 	('ovs_vport_receive',			0),
# 	('ovs_execute_actions',			0),
# 	('e100_xmit_frame',				3),
# 	('ixgbe_xmit_frame',			3),
# 	('ip_rcv_finish',				0),
# 	('ip_forward',					0),
# 	('ip_forward_finish',			0),
# 	('ixgbevf_xmit_frame',			3),
# ]
SKB = 0
SKB_HEAD = 1
DATA = 2
SADDR = 3
DADDR = 4
PROTO = 5
SPORT = 6
DPORT = 7
FUNC_ID = 8
TSTAMP = 9
if __name__ == '__main__':

    filename = sys.argv[1]
    outfile = sys.argv[2]
    func_table, spec_functable = common.parse('config.json')
    run(filename, outfile, func_table)
Example #53
0
import numpy as np
from math import sqrt

from common import parse, is_corner

tiles = parse()
width = int(sqrt(len(tiles)))
top_left = next(tile for tile in tiles.values() if is_corner(tile))


def fliplr(tile):
    tile.data = np.fliplr(tile.data)
    tile.neighbours[2], tile.neighbours[3] = \
        tile.neighbours[3], tile.neighbours[2]


def flipud(tile):
    tile.data = np.flipud(tile.data)
    tile.neighbours[0], tile.neighbours[1] = \
        tile.neighbours[1], tile.neighbours[0]


def rot90l(tile):
    tile.data = np.rot90(tile.data)
    tile.neighbours = {
        0: tile.neighbours[3],
        1: tile.neighbours[2],
        2: tile.neighbours[0],
        3: tile.neighbours[1]
    }
Example #54
0
from common import parse

A = ord('a')
Z = ord('z')


def decrypt(word, iterations):
    return ''.join(
        ' ' if char == '-' else chr(A + (ord(char) - A + iterations) %
                                    (Z - A + 1)) for char in word)


with open('input.txt') as data:
    for line in data:
        encrypted, sector, _ = parse(line)
        if decrypt(encrypted, int(sector)) == 'northpole object storage':
            print(sector)
            break