def reduce_process(self):
workers_folder = [os.path.join(self.main_folder, folder) for folder in os.listdir(self.main_folder)]
sc.message("Processing files")
print(workers_folder)
reduced_folder = sc.check_folder(os.path.join(self.output_folder, "ner_encoded"))
train_folder = sc.check_folder(os.path.join(reduced_folder, "train"))
test_folder = sc.check_folder(os.path.join(reduced_folder, "test"))
total_ads = 0
# Count lines
for dataset_folder in workers_folder:
for line in open(os.path.join(dataset_folder, "dataset.jsonl"), "r", encoding="utf-8"):
total_ads += 1
test_size = int(total_ads * self.test_perc)
train_size = total_ads - test_size
total_ads = 0
# Save train/test set
for dataset_folder in workers_folder:
for line in open(os.path.join(dataset_folder, "dataset.jsonl"), "r", encoding="utf-8"):
save_folder = train_folder if total_ads < train_size else test_folder
with open(os.path.join(save_folder, "dataset.jsonl"), "a", encoding="utf-8") as js:
js.write(line)
total_ads += 1
# Copy maps
maps_path = [os.path.join(workers_folder[0], file_name) for file_name in os.listdir(workers_folder[0]) if "dataset" not in file_name]
for map in maps_path:
copy(map, reduced_folder)
sc.message("DONE! Save @{}".format(reduced_folder))
def base(files: List[str], encoder: "pipelines.encoder.BaseEncoder" = None):
"""
Base pipeline method to be used @pararell_processing
:param files: file paths to be processed
:param encoder: encoder class defined @config.gin
"""
try:
if not encoder:
raise ValueError(
"Encoder cannot be None. PLz Specificy a encoder @gin.config!")
# Initialization
generator = ds.build_advertise_generator(files)
parser = Parser()
enc_client = encoder()
# Source generator, clean and encode advertise
for line in generator:
try:
ad = dc.clean_raw_advertise(line, parser)
if ad:
enc_client.encode_advertise(ad)
except Exception as err:
sc.message(err)
enc_client.save_maps()
except Exception as erro:
sc.message(erro)
def save_maps(self, *maps):
sc.message("Saving Maps...")
if self.model_folder:
for k, map in self.maps.items():
# TODO: may get too big, write as txt
sc.save_dict_2json(os.path.join(self.model_folder, "{}.json".format(k)), map)
else:
# TODO: implement saving in DataStorage
raise NotImplementedError("Saving outside a local folder path is not implemented yet!")
def cutoff_schema(self, schema_counter: Dict[str, int],
ctoff: int) -> List[str]:
tmp_vec = []
for k, v in schema_counter.items():
if v > ctoff:
tmp_vec.append(k)
elif self.debug:
sc.message("{} field dropped !".format(k))
return tmp_vec
def preload_maps(self, folder: str = None):
if folder:
sc.message("Schema loaded !")
tmp_schema = sc.load_json(folder)
if "NUMBER_OF_PROPERTIES" in tmp_schema.keys():
tmp_schema.pop("NUMBER_OF_PROPERTIES")
return tmp_schema
else:
raise ValueError(
"Parsed properties path cannot be None! Run schema pipe to build parsed props!"
)
def get_field_from_ad(self,
advertise: Dict[str, Any],
field_name: str,
alias: List[str],
default_value: Optional[str] = None):
for key in advertise.keys():
if any([key in field for field in alias]):
return advertise[key]
if self.debug:
sc.message("No {0} column found @ advertise {1}".format(
field_name, advertise))
return default_value
def load_model(self, model_path: str):
model_json = os.path.join(model_path, "model_arc.json")
model_weights = [
os.path.join(model_path, x) for x in os.listdir(model_path)
if 'weights' in x
]
with open(model_json, 'r') as dt:
model_arc = json.load(dt)
model = model_from_json(json.dumps(model_arc))
model.load_weights(model_weights[0])
sc.message("Category model loaded!")
return model
def check_measure_fields(self, values: Set[str], tol: float = 0.7) -> bool:
"""
Check if field is a 'measure field' - i.e. 64 gb, 5500 mah
:param values: set of values
:param tol: tolerance of minimum percentage of fit
:return: True if is a measure field
"""
counter: int = 0
if type(values) is list:
if len(values) == 0:
sc.message("WARNING: NER MAPPER HAS A FIELD WITH NO VALUES.")
return False
for value in values:
tmp = value.split()
if len(tmp) == 2 and tmp[0].isnumeric():
counter += 1
return (counter / len(values)) > tol
else:
return False
def build_advertise_generator(files: List[str]):
qeue = deque(files)
total_files = len(qeue)
process_counter = 0
while len(qeue) > 0:
try:
file_name = qeue.pop()
if "jsonl" in file_name:
for line in open(file_name, "r", encoding="utf-8"):
ad = json.loads(line)
print(ad)
yield ad
else:
ad = sc.load_json(file_name)
for advertise in ad:
yield advertise
process_counter += 1
sc.message("{} PROCESSED!".format(file_name))
sc.message("Worker {}% complete!".format(
round(process_counter / total_files, 2) * 100))
except Exception as err:
sc.message(err)
return None
def valid_advertise(advertise: Dict[str, Any],
minimum_cols: Tuple[str] = ('price', 'url'),
category: str = None,
market: str = None,
silent: bool = True) -> bool:
"""
Checks if there are any missing crucial fields.
In case of missing, adds default value (if there is one) or raises error.
:param adv_dict: (dict) scrapped item
:return: (bool) valid or not
"""
try:
# Checks if data has all necessary columns
for col in minimum_cols:
if col not in advertise.keys() or not advertise[col]:
raise Exception(
'{0}:{1} - Missing necessary column: {2}'.format(
advertise["id"], "Missing Value", str(col)))
elif col == "price" and not validate_prices(
advertise[col], advertise["category"]):
raise Exception(
'{0}:{1} - Invalid price for category: {2} of {3}'.format(
advertise["id"], "Invalid Price",
advertise["category"], advertise["price"]))
elif category is not None and advertise["category"] != category:
raise Exception('{0}:{1} - Invalid category: {2}'.format(
advertise["id"], "Invalid Category",
advertise["category"]))
elif market is not None and advertise["market"] != market:
raise Exception('{0}:{1} - Invalid market: {2}'.format(
advertise["id"], "Invalid Market", advertise["market"]))
return True
except Exception as err:
if not silent:
sc.message(str(err))
return False
def reduce_process(self):
workers_folder = [
os.path.join(self.main_folder, folder)
for folder in os.listdir(self.main_folder)
]
sc.message("Processing files")
print(workers_folder)
reduced_folder = sc.check_folder(
os.path.join(self.output_folder, "sequence"))
with open(os.path.join(reduced_folder, "dataset.jsonl"),
"a",
encoding="utf-8") as js:
for folderzin in workers_folder:
for line in open(os.path.join(folderzin,
"sequence_enriched.jsonl"),
"r",
encoding="utf-8"):
js.write(line)
sc.message("DONE !")
def clean(files: List[str], encoder: "pipelines.encoder.BaseEncoder" = None):
"""
Data pipeline method to be used @pararell_processing when data is already clean
So, no parser is needed.
:param files: file paths to be processed
:param encoder: encoder class defined @config.gin
"""
try:
if not encoder:
raise ValueError(
"Encoder cannot be None. PLz Specificy a encoder @gin.config!")
generator = ds.build_advertise_generator(files)
enc_client = encoder()
for ad in generator:
try:
enc_client.encode_advertise(ad)
except Exception as err:
sc.message(err)
enc_client.save_maps()
except Exception as erro:
sc.message(erro)
def reduce_process(self):
workers_folder = [
os.path.join(self.main_folder, folder)
for folder in os.listdir(self.main_folder)
]
if self.debug:
print("Paths being aggregated...")
print(workers_folder)
char2idx = set()
word2idx = set()
tag2idx = set()
sc.message("Processing files...")
for folderzin in workers_folder:
tmp_chars = sc.load_json(os.path.join(folderzin, "char2idx.json"))
tmp_words = sc.load_json(os.path.join(folderzin, "word2idx.json"))
tmp_tags = sc.load_json(os.path.join(folderzin, "tag2idx.json"))
for c in tmp_chars.keys():
char2idx.add(c)
for w in tmp_words.keys():
word2idx.add(w)
for t in tmp_tags.keys():
tag2idx.add(t)
reduced_folder = sc.check_folder(
os.path.join(self.output_folder, "ner_mapping"))
basec2i = {"__PAD__": 0, "UNK": 1}
basew2i = {"__PAD__": 0, "UNK": 1}
baset2i = {"__PAD__": 0}
char2idx.remove("__PAD__")
char2idx.remove("UNK")
word2idx.remove("__PAD__")
word2idx.remove("UNK")
tag2idx.remove("__PAD__")
if "UNK" in tag2idx:
tag2idx.remove("UNK")
for i, c in enumerate(char2idx):
basec2i[c] = i + 2
for i, c in enumerate(word2idx):
basew2i[c] = i + 2
for i, c in enumerate(tag2idx):
baset2i[c] = i + 1
sc.message("Saving chars")
sc.save_dict_2json(os.path.join(reduced_folder, "char2idx.json"),
basec2i)
sc.message("Saving words")
sc.save_dict_2json(os.path.join(reduced_folder, "word2idx.json"),
basew2i)
sc.message("Saving tags")
sc.save_dict_2json(os.path.join(reduced_folder, "tag2idx.json"),
baset2i)