def extract_content_rtvslo(file_name):
    tree = html.fromstring(parse_file(file_name, "utf-8"))

    author = tree.xpath('//div[@class="author-name"]/text()')
    publishedTime = tree.xpath('//div[@class="publish-meta"]/text()')
    publishedTime = re.sub('\t|\n', "", publishedTime[0])

    title = tree.xpath('//header[@class="article-header"]//h1/text()')
    subtitle = tree.xpath('//div[@class="subtitle"]/text()')
    lead = tree.xpath('//header[@class="article-header"]//p[@class="lead"]/text()')

    # content = tree.xpath(' //div[@class="article-body"]//figure[not[contains(class, \'gallery-bottom-thumb\')]]//figcaption[@itemprop="caption description"]/text()')
    content = tree.xpath(
        ' //div[@class="article-body"]//p/text() | //div[@class="article-body"]//p/strong/text() | //div[@class="article-body"]//figure/figcaption[@itemprop="caption description"]/span/text()')

    content_processed = []
    for c in content:
        content_processed.append(re.sub("\t|\n", "", c))

    # content = re.sub('\t',"", content[0])
    final_dict = {
        'Author': author[0],
        'PublishedTime': publishedTime,
        'Title': title[0],
        'SubTitle': subtitle[0],
        'Lead': lead[0],
        'Content': " ".join(content_processed)
    }

    print(json.dumps(final_dict, indent=4, ensure_ascii=False))
Beispiel #2
0
def update_configlet(id, data, user):
    cfg = Configlet.objects.get(pk=id)
    cfg.path.delete(save=False)
    file_obj = data[FILE]
    cfg.path = file_obj
    file_content = file_obj.read()

    if cfg.type == SCRIPT:
        params = parse_file(file_content, PARAM_EXP_SCRIPT,
                            PARAM_IDENTIFIER_SCRIPT)
    else:
        params = parse_file(file_content, PARAM_EXP_CONFIGLET,
                            PARAM_IDENTIFIER_CONFIGLET)

    cfg.parameters = params
    cfg.updated_by = user
    cfg.save()
    logger.debug("File content: "+str(file_content))
    logger.debug("File named %s saved in db" % (file_obj))
    return cfg
Beispiel #3
0
def update_feature(id, data, user):
    feature_obj = Feature.objects.get(pk=id)
    feature_obj.path.delete(save=False)
    file_obj = data[FILE]
    feature_obj.path = file_obj
    file_content = file_obj.read()
    params = parse_file(file_content)
    feature_obj.parameters = params
    feature_obj.updated_by = user
    feature_obj.save()
    return feature_obj
def extract_content_gsc(file_name):
    tree = html.fromstring(parse_file(file_name))
    title = tree.xpath('//h1[@class="product_title entry-title"]/text()')[0]
    price_from = tree.xpath(
        '//div[@class="summary entry-summary"]/p/span[1]/span/text() | //div[@class="summary entry-summary"]/p/span[1]/text()')
    price_from = "".join(price_from)

    price_to = tree.xpath(
        '//div[@class="summary entry-summary"]/p/span[2]/span/text() | //div[@class="summary entry-summary"]/p/span[2]/text()')
    price_to = "".join(price_to)

    description = tree.xpath('//div[@class="woocommerce-product-details__short-description"]/p/text()')
    description = ("".join(description)).replace("\n", "")

    category = tree.xpath('//span[@class="posted_in"]/a/text()')[0]

    tags = tree.xpath('//span[@class="tagged_as"]/a/text()')

    attributes = [attr.strip() for attr in
                  tree.xpath('//table[@class="table table-hover variations"]/thead/tr/th[not(@*)]/text()')]
    var_attr = {}
    for attr in attributes:
        var_attr[attr] = tree.xpath(
            f'//table[@class="table table-hover variations"]/tbody/tr/td[@data-title="{attr}"]/text()')

    separate_list_prices = tree.xpath(
        '//span[@class="price"]//span[1]/span/text() | //span[@class="price"]//span[1]/text()')

    c = 0
    separate_list_price = []
    while c < len(separate_list_prices) - 1:
        separate_list_price.append(f"{separate_list_prices[c]}{separate_list_prices[c + 1]}")
        c += 2

    separate_discount_prices = pad_list(tree.xpath('//ins/span/span/text() | //ins/span/text()'), var_attr["Model"],
                                        "currency_xpath")
    c = 0
    separate_discount_price = []
    while c < len(separate_discount_prices) - 1:
        separate_discount_price.append(f"{separate_discount_prices[c]}{separate_discount_prices[c + 1]}")
        c += 2

    variations = zip(separate_list_price, separate_discount_price)
    results = generate_json_gsc(title, price_from, price_to, description, category, tags, var_attr, variations)
    print(json.dumps(results, indent=4))
def extract_content_overstock(file_name):
    tree = html.fromstring(parse_file(file_name))
    i = 1
    potential_break = False
    results = []
    while True:
        product = {}
        titles = tree.xpath(f'//table[2]/tbody/tr/td[5]//table//table/tbody/tr[@bgcolor][{i}]/td[2]/a/b/text()')
        if titles:
            contents = tree.xpath(
                f'//table[2]/tbody/tr/td[5]//table//table/tbody/tr[@bgcolor][{i}]//td[2]/span[@class="normal"]/text()')
            content = contents[0] if len(contents) > 0 else "N/A"
            list_price = tree.xpath(f'//table[2]/tbody/tr/td[5]//table//table/tbody/tr[@bgcolor][{i}]//s/text()')
            list_price = list_price[0] if len(list_price) > 0 else "N/A"
            price = tree.xpath(
                f'//table[2]/tbody/tr/td[5]//table//table/tbody/tr[@bgcolor][{i}]//span[@class="bigred"]/b/text()')
            price = price[0] if len(price) > 0 else "N/A"
            saving = tree.xpath(
                f'//table[2]/tbody/tr/td[5]//table//table/tbody/tr[@bgcolor][{i}]//span[@class="littleorange"]/text()')
            saving = saving[0] if len(saving) > 0 else "N/A"
            product = {
                "Title": titles[0],
                "Content": content.replace("\n", " "),
                "ListPrice": list_price,
                "Price": price,
                "Saving": saving,
                "SavingPercent": "0%"
            }
            if saving != "N/A":
                split_savings_matcher = re.compile(r"([$€]\s*[0-9.,]+)\s*(\([0-9.,]+%\))")
                saving = split_savings_matcher.search(saving)
                product["Saving"] = saving.group(1)
                product["SavingPercent"] = saving.group(2)
            results.append(product)
        i += 1
        if len(titles) == 0:
            if potential_break:
                break
            potential_break = True

    print(json.dumps(results, indent=4))
Beispiel #6
0
def build_config_profile(cfg_list, switch):
    buff = ''
    param_value = {}
    construct_list = []

    for cfg in cfg_list:
        if cfg:
            construct_list += list(cfg.construct_list)

    for item in construct_list:

        cfglt = configlet.get_configlet(item[CONFIGLET_ID])
        buff += '\n!\n!%s config\n!\n' % cfglt.name
        mako_buff = ""
        construct_type = cfglt.type

        if construct_type == SCRIPT:
            logger.debug('Processing script %s' % cfglt.name)
        else:
            logger.debug('Processing configlet %s' % cfglt.name)

        mako_param_value = dict()

        for param_detail in item[PARAM_LIST]:
            value = None

            if param_detail[PARAM_TYPE] == VALUE:
                try:
                    value = param_value[param_detail[PARAM_VALUE]]
                except KeyError:
                    err_str = "%s %s" % (ERR_VALUE_NOT_FOUND,
                                         param_detail[PARAM_VALUE])
                    logger.error(err_str)
                    raise IgniteException(err_str)

            if param_detail[PARAM_TYPE] == FIXED:
                value = param_detail[PARAM_VALUE]

            if param_detail[PARAM_TYPE] == INSTANCE:
                value = fabric.build.get_instance_value(
                                            param_detail[PARAM_VALUE],
                                            switch,
                                            switch.name)

            if param_detail[PARAM_TYPE] == POOL:
                value = allocate_pool_entry(param_detail[PARAM_VALUE],
                                            switch.id,
                                            switch)

            if param_detail[PARAM_TYPE] == EVAL:
                try:
                    value = eval(param_detail[PARAM_VALUE])
                except SyntaxError:
                    raise IgniteException("%s = %s" % (ERR_EVAL_SYNTAX,
                                                       param_detail[PARAM_VALUE]))

            if value is None:
                err_str = "%s %s" % (ERR_VALUE_NOT_FOUND,
                                     param_detail[PARAM_NAME])
                logger.error(err_str)
                raise IgniteException(err_str)

            param_value[param_detail[PARAM_NAME]] = value

            if construct_type == SCRIPT:
                mako_param_name = PARAM_IDENTIFIER_SCRIPT +\
                                  param_detail[PARAM_NAME] +\
                                  PARAM_IDENTIFIER_SCRIPT
                mako_param_value[mako_param_name] = value

        for line in cfglt.path.file:
            logger.debug(line)

            if construct_type == SCRIPT:
                mako_buff += line
            else:
                param_list = parse_file(line, PARAM_EXP_CONFIGLET,
                                        PARAM_IDENTIFIER_CONFIGLET)

                for param in param_list:
                    logger.debug("Param to replace %s by value %s"
                                 % (param, param_value[param]))

                    line = line.replace(PARAM_IDENTIFIER_CONFIGLET + param +
                                        PARAM_IDENTIFIER_CONFIGLET,
                                        param_value[param])

                buff += line

        if mako_buff:
            buff += Template(mako_buff).render(**mako_param_value)

    return buff
Beispiel #7
0
def update_configlet(cfgindex_id,
                     fileobject,
                     cfg_id=0,
                     new_version=None,
                     user=''):
    cfg = None
    current_version = get_current_config_version(cfgindex_id)
    logger.debug("current version " + str(current_version))
    cfg = Configlet.objects.get(configletindex_id=cfgindex_id,
                                version=current_version)
    if cfg_id:
        logger.debug(str(cfg.id))
        logger.debug(str(cfg_id))
        if cfg_id != cfg.id:
            err = "Requested configlet is not latest version(%s)" % (
                str(cfg_id))
            logger.error(err)
            raise IgniteException(err)

        cfg = Configlet.objects.get(pk=cfg_id, configletindex_id=cfgindex_id)
    file_obj = fileobject[FILE]
    file_content = file_obj.read()
    if cfg.path and cfg_id:
        logger.debug(str(cfg.path))
        try:
            old = hashlib.md5(
                open(os.path.join(MEDIA_ROOT, str(cfg.path)),
                     'rb').read()).hexdigest()
            logger.debug("existed configlet md5= " + old)
            logger.debug(str(old))
            new = hashlib.md5(file_content).hexdigest()
            logger.debug("User given  configlet md5= " + new)
            change = False
            params = None
            logger.debug(str(new))
            if old == new:
                return cfg
            if cfg.type == SCRIPT:
                params = parse_file(file_content, PARAM_EXP_SCRIPT,
                                    PARAM_IDENTIFIER_SCRIPT)
            else:
                params = parse_file(file_content, PARAM_EXP_CONFIGLET,
                                    PARAM_IDENTIFIER_CONFIGLET)
            logger.debug("new parameters " + str(params))
            logger.debug("old parameters " + str(cfg.parameters))
            if params != cfg.parameters:
                change = True
            if change:
                logger.error(ERR_CHANGE_IN_PARAMS)
                raise IgniteException(ERR_CHANGE_IN_PARAMS)
            else:
                logger.debug("new version boolean is " + new_version)
                if str(new_version) in ['true', 'True']:
                    logger.debug("User requested new version of configlet")
                    new_cfg = Configlet()
                    new_cfg.name = cfg.name
                    new_cfg.updated_by = user
                    new_cfg.group = cfg.group
                    new_cfg.type = cfg.type
                    new_cfg.parameters = params
                    new_cfg.configletindex = cfg.configletindex
                    new_cfg.save()
                    new_cfg.version = current_version + 1
                    new_cfg.path.delete(save=False)
                    new_cfg.path = file_obj
                    new_cfg.save()
                    logger.debug("File content: " + str(file_content))
                    logger.debug("File named %s saved in db" % (file_obj))
                    return new_cfg
                elif str(new_version) in ['false', 'False']:
                    cfg.path.delete(save=False)
                    cfg.path = file_obj
                    cfg.updated_by = user
                    cfg.save()
                    logger.debug("File content: " + str(file_content))
                    logger.debug("File named %s saved in db" % (file_obj))
                    return cfg
                else:
                    err = "Unknown version type-" + str(new_version)
                    logger.error(err)
                    raise IgniteException(err)

        except Exception as e:
            logger.error(e)
            raise IgniteException(str(e))
    if cfg.path:
        err = "Update/Newversion is not possible on this request"
        logger.error(err)
        raise IgniteException(err)
    cfg.path.delete(save=False)
    cfg.path = file_obj

    logger.debug("adding a new configlet")
    if cfg.type == SCRIPT:
        params = parse_file(file_content, PARAM_EXP_SCRIPT,
                            PARAM_IDENTIFIER_SCRIPT)
    else:
        params = parse_file(file_content, PARAM_EXP_CONFIGLET,
                            PARAM_IDENTIFIER_CONFIGLET)

    cfg.parameters = params
    cfg.updated_by = user
    cfg.save()
    logger.debug("File content: " + str(file_content))
    logger.debug("File named %s saved in db" % (file_obj))
    return cfg
Beispiel #8
0
def update_configlet(cfgindex_id, fileobject, cfg_id=0, new_version=None,  user=''):
    cfg = None
    current_version = get_current_config_version(cfgindex_id)
    logger.debug("current version " + str(current_version))
    cfg = Configlet.objects.get(configletindex_id=cfgindex_id, version=current_version)
    if cfg_id:
        logger.debug(str(cfg.id))
        logger.debug(str(cfg_id))
        if cfg_id != cfg.id:
            err = "Requested configlet is not latest version(%s)" % (str(cfg_id))
            logger.error(err)
            raise IgniteException(err)

        cfg = Configlet.objects.get(pk=cfg_id, configletindex_id=cfgindex_id)
    file_obj = fileobject[FILE]
    file_content = file_obj.read()
    if cfg.path and cfg_id:
        logger.debug(str(cfg.path))
        try:
            old = hashlib.md5(open(os.path.join(MEDIA_ROOT, str(cfg.path)), 'rb').read()).hexdigest()
            logger.debug("existed configlet md5= " + old)
            logger.debug(str(old))
            new = hashlib.md5(file_content).hexdigest()
            logger.debug("User given  configlet md5= " + new)
            change = False
            params = None
            logger.debug(str(new))
            if old == new:
                return cfg
            if cfg.type == SCRIPT:
                params = parse_file(file_content, PARAM_EXP_SCRIPT,
                                    PARAM_IDENTIFIER_SCRIPT)
            else:
                params = parse_file(file_content, PARAM_EXP_CONFIGLET,
                                    PARAM_IDENTIFIER_CONFIGLET)
            logger.debug("new parameters " + str(params))
            logger.debug("old parameters " + str(cfg.parameters))
            if params != cfg.parameters:
                change = True
            if change:
                logger.error(ERR_CHANGE_IN_PARAMS)
                raise IgniteException(ERR_CHANGE_IN_PARAMS)
            else:
                logger.debug("new version boolean is " + new_version)
                if str(new_version) in ['true', 'True']:
                    logger.debug("User requested new version of configlet")
                    new_cfg = Configlet()
                    new_cfg.name = cfg.name
                    new_cfg.updated_by = user
                    new_cfg.group = cfg.group
                    new_cfg.type = cfg.type
                    new_cfg.parameters = params
                    new_cfg.configletindex = cfg.configletindex
                    new_cfg.save()
                    new_cfg.version = current_version + 1
                    new_cfg.path.delete(save=False)
                    new_cfg.path = file_obj
                    new_cfg.save()
                    logger.debug("File content: "+str(file_content))
                    logger.debug("File named %s saved in db" % (file_obj))
                    return new_cfg
                elif str(new_version) in ['false', 'False']:
                    cfg.path.delete(save=False)
                    cfg.path = file_obj
                    cfg.updated_by = user
                    cfg.save()
                    logger.debug("File content: "+str(file_content))
                    logger.debug("File named %s saved in db" % (file_obj))
                    return cfg
                else:
                    err = "Unknown version type-" + str(new_version)
                    logger.error(err)
                    raise IgniteException(err)

        except Exception as e:
            logger.error(e)
            raise IgniteException(str(e))
    if cfg.path:
        err = "Update/Newversion is not possible on this request"
        logger.error(err)
        raise IgniteException(err)
    cfg.path.delete(save=False)
    cfg.path = file_obj

    logger.debug("adding a new configlet")
    if cfg.type == SCRIPT:
        params = parse_file(file_content, PARAM_EXP_SCRIPT,
                            PARAM_IDENTIFIER_SCRIPT)
    else:
        params = parse_file(file_content, PARAM_EXP_CONFIGLET,
                            PARAM_IDENTIFIER_CONFIGLET)

    cfg.parameters = params
    cfg.updated_by = user
    cfg.save()
    logger.debug("File content: "+str(file_content))
    logger.debug("File named %s saved in db" % (file_obj))
    return cfg
Beispiel #9
0
def build_config_profile(cfg_list, switch):
    buff = ''
    param_value = {}
    construct_list = []

    for cfg in cfg_list:
        if cfg:
            construct_list += list(cfg.construct_list)

    for item in construct_list:

        cfglt = configlet.get_configlet_byversion(item[CONFIGLETINDEX_ID], item[VERSION])
        buff += '\n!\n!%s config\n!\n' % cfglt.name
        mako_buff = ""
        construct_type = cfglt.type

        if construct_type == SCRIPT:
            logger.debug('Processing script %s' % cfglt.name)
        else:
            logger.debug('Processing configlet %s' % cfglt.name)

        mako_param_value = dict()

        for param_detail in item[PARAM_LIST]:
            value = None

            if param_detail[PARAM_TYPE] == VALUE:
                try:
                    value = param_value[param_detail[PARAM_VALUE]]
                except KeyError:
                    err_str = "%s %s" % (ERR_VALUE_NOT_FOUND,
                                         param_detail[PARAM_VALUE])
                    logger.error(err_str)
                    raise IgniteException(err_str)

            if param_detail[PARAM_TYPE] == FIXED:
                value = param_detail[PARAM_VALUE]

            if param_detail[PARAM_TYPE] == INSTANCE:
                value = fabric.build.get_instance_value(
                                            param_detail[PARAM_VALUE],
                                            switch,
                                            switch.name)

            if param_detail[PARAM_TYPE] == POOL:
                value = allocate_pool_entry(param_detail[PARAM_VALUE],
                                            switch.id,
                                            switch)

            if param_detail[PARAM_TYPE] == EVAL:
                try:
                    value = eval(param_detail[PARAM_VALUE])
                except SyntaxError:
                    raise IgniteException("%s = %s" % (ERR_EVAL_SYNTAX,
                                                       param_detail[PARAM_VALUE]))

            if value is None:
                err_str = "%s %s" % (ERR_VALUE_NOT_FOUND,
                                     param_detail[PARAM_NAME])
                logger.error(err_str)
                raise IgniteException(err_str)

            param_value[param_detail[PARAM_NAME]] = value

            if construct_type == SCRIPT:
                mako_param_name = PARAM_IDENTIFIER_SCRIPT +\
                                  param_detail[PARAM_NAME] +\
                                  PARAM_IDENTIFIER_SCRIPT
                mako_param_value[mako_param_name] = value

        for line in cfglt.path.file:
            logger.debug(line)

            if construct_type == SCRIPT:
                mako_buff += line
            else:
                param_list = parse_file(line, PARAM_EXP_CONFIGLET,
                                        PARAM_IDENTIFIER_CONFIGLET)

                for param in param_list:
                    logger.debug("Param to replace %s by value %s"
                                 % (param, param_value[param]))

                    line = line.replace(PARAM_IDENTIFIER_CONFIGLET + param +
                                        PARAM_IDENTIFIER_CONFIGLET,
                                        param_value[param])

                buff += line

        if mako_buff:
            buff += Template(mako_buff).render(**mako_param_value)

    return buff