def main(debug_mode=DEBUG_MODE):
# Imports are at this level, in case modules are reloaded.
from tools.helpers.interface import MainTk
# import functions here:
from scripts import FUNCTIONS
# import debug functions here:
DEBUG_FUNCTIONS = []
parts = OrderedDict([("sample functions", FUNCTIONS),
]) # add functions here
if debug_mode:
parts['Debugging (developer only)'] = DEBUG_FUNCTIONS # add debug functions here
logger_level = logging.DEBUG if debug_mode else logging.INFO
root = MainTk(parts, logger_level, help_msg=__help__, title=TITLE)
if debug_mode:
reload_but = ttk.Button(master=root.info_frame, text="Reload interface and scripts",
command=lambda: reload_main(root, debug_mode=debug_mode))
reload_but.grid(row=2, column=2, sticky='se')
ttk.Button(master=root.info_frame, text="logger INFO",
command=lambda: change_logger_to_info(root)).grid(row=2, column=0, sticky='e')
ttk.Button(master=root.info_frame, text="logger DEBUG",
command=lambda: change_logger_to_debug(root)).grid(row=2, column=1, sticky='e')
ttk.Button(master=root.info_frame, text="Standard mode",
command=lambda: reload_main(root, debug_mode=False)).grid(row=3, column=0, sticky='w')
ttk.Button(master=root.info_frame, text="Open debugger",
command=debugger).grid(row=3, column=1, sticky='w')
change_logger_to_debug(root)
else:
ttk.Button(master=root.info_frame, text="Advanced mode",
command=lambda: reload_main(root, debug_mode=True)).grid(row=3, column=0, sticky='w')
change_logger_to_info(root)
root.mainloop()
logger.debug('Main window closed.')
def load(self, path=None, force_load=None, auto_cast=None, load_empty=None, merge_how='right'):
"""Load configuration from file.
If force-load, reload_default values on error.."""
path = self._path if path is None else Path(path)
auto_cast = self._auto_cast if auto_cast is None else auto_cast
force_load = self._force_load if force_load is None else force_load
load_empty = self._load_empty if load_empty is None else load_empty
if not path.isfile:
if force_load:
self.reload_default()
return None
config_dict = self.read_config(path, auto_cast=auto_cast)
if config_dict is None:
if force_load:
self.reload_default()
return None
elif path.isfile: # Change path
self._path = path
logger.debug("Path changed to '{}' with 'load' method.".format(self.path))
elif config_dict.isempty:
if not load_empty:
logger.info("Configuration loaded is empty! It won't replace the existing one.")
logger.debug("Use 'clear' method to empty the ConfigDict")
return None
else:
raise UnknownError("bad case in 'load'")
# config_dict.section = self.section
# config_dict.default_section = self.default_section
# config_dict._conversion_dict = self._conversion_dict
self.merge(config_dict, how=merge_how, inplace=True)
logger.info("Configuration loaded!")
def _handle_duplicates(dico, key, value, flag='first', inplace=False):
"""Handle duplicates in dico.
:param dico: dico to update
:param key: key to check
:param value: value to set
:param flag: 'first', 'last', 'rename' or 'error' (or whatever, which means 'error')
:param inplace: modification of dico inplace if True
:return: None if inplace is True, else dico updated
"""
n_dico = type(dico)()
if key in dico:
logger.debug("Key '{}' is duplicated.".format(key))
if flag == 'first':
pass
elif flag == 'last':
n_dico[key] = value
elif flag == 'rename':
i = 0
exists = True
while exists:
i += 1
n_key = "{}_{}".format(key, i)
exists = True if n_key in dico else False
n_dico[n_key] = value
else:
err_msg = "Duplicate keys '{}' found! Conversion process aborting."
logger.error(err_msg)
raise ValueError(err_msg)
else:
n_dico[key] = value
if inplace:
dico.update(n_dico)
return
return n_dico
def __init__(self,
master: 'MainTk',
text,
answers,
labelanchor='n',
geometry_manager='pack',
geometry_options=None):
"""Main part of the MainTk window. Should not be instantiated outside a MainTk window.
:param answers: list of functions or list of tuples withe the format [(function_name, function, tooltip), ...]
"""
super().__init__(master, text=text, labelanchor=labelanchor)
self.master = master
answers = self._format_answers(answers)
logger.debug("Possible functions: {}".format(answers))
self.question_frame = CustomQuestionFrame(
master=self,
choices=answers,
auto_quit=False,
geometry_manager=geometry_manager,
geometry_options=geometry_options)
self.question_frame.pack(fill=tk.BOTH)
self.question_frame.change_res.trace(
mode='w', callback=self.exec_func) # exec func on click
self.queue = None
def reload_functions(module, functions: Union[list, tuple, Callable]):
"""Reloads a function or a list/tuple of functions."""
if not isinstance(functions, (list, tuple)):
functions = [functions]
for func in functions:
if isinstance(
func,
(list, tuple)): # format [(_, func1, _), (_, func2, _), ...]
fn = func[1] if len(func) > 1 else None
else: # format [func1, func2, ...]
fn = func
if '__name__' in dir(fn): # case of functions and classes
name = fn.__name__
if name == '<lambda>':
continue
elif isinstance(
fn,
str) and fn in dir(module): # particular case of __all__ list
name = fn
else:
logger.error("object '{}' not reloaded, wrong type or name.")
return
getattr(module,
name) # access to function seems sufficient to reload it.
logger.debug("object '{}' reloaded".format(name))
def reload_recursive_ex(module):
try:
importlib.reload(module)
except ImportError as err:
logger.exception(err)
logger.error("module '{}' could not be reloaded.".format(module))
return
if reload_func:
for func_names in ls_func_names:
if func_names in dir(module):
ls_functions = getattr(module, func_names)
reload_functions(module, ls_functions)
logger.debug("module '{}' reloaded!".format(module.__name__))
for module_child in vars(
module).values(): # search subpackages in vars(module)
if isinstance(module_child, types.ModuleType): # if it is a module
fn_child = getattr(module_child, "__file__", None)
if (fn_child is not None
) and fn_child.startswith(fn_dir): # if it is a subpackage
if fn_child not in module_visit: # if module has not benn reloaded yet
# print("reloading:", fn_child, "from", module)
module_visit.add(fn_child)
reload_recursive_ex(
module_child) # reload subpackages of this module
def handle_file_error(err, func, path, args=None, kwargs=None,
pos_path=0, key_path=None, change_path_func=save_file,
title='', msg='', return_if_ignore=None):
"""If PermissionError when opening/saving file, propose to retry, change file path or cancel
:param err: exception
:param func: function to execute if the user wants to retry
:param path: file path
:param args: args to pass to func
:param kwargs: kwargs to pass to func
:param pos_path: position of the positional argument path in func (only if key_path is None)
:param key_path: name of the keyword argument path in func (if None, positional argument is used)
:param change_path_func: function to get a new path, with no positional argument and 'initialdir' keyword argument
:param title: title of the error
:param msg: message of the error
:param return_if_ignore: return if Ignore option is selected
:return:
"""
logger.debug(err)
args = args or []
kwargs = kwargs or {}
title = title or 'File error!'
msg = msg or "Unknown error with file '{}'. \nOriginal error: {}".format(path, err)
logger.warning('User action needed!')
res = messagebox.askcustomquestion(title=title, message=msg,
choices=["Retry", "Rename automatically", "Change file path",
"Ignore", "Debug (developer only)", "Cancel"])
if res == "Retry":
if key_path is not None:
kwargs[key_path] = path
else:
args.insert(pos_path, path)
return func(*args, **kwargs)
if res == "Rename automatically":
n_path = _handle_existing_file_conflict(path=path, overwrite='rename')
if key_path is not None:
kwargs[key_path] = n_path
else:
args.insert(pos_path, n_path)
return func(*args, **kwargs)
elif res == "Change file path":
initialdir = Path(path).dirname if Path(path).dirname.exists else None
if key_path is not None:
kwargs[key_path] = change_path_func(initialdir=initialdir)
else:
args.insert(pos_path, change_path_func(initialdir=initialdir))
return func(*args, **kwargs)
elif res == "Ignore":
logger.warning("Function ignored!")
logger.debug("Function '{}' with path '{}' ignored!")
return return_if_ignore
elif res == "Debug (developer only)":
pdb.set_trace()
elif res in [None, "Cancel"]:
err = UnknownError if not isinstance(err, BaseException) else err
logger.exception(err)
raise err.__class__(err)
else:
raise TypeError("Bad return of function 'messagebox.askcustomquestion': '{}'".format(res))
def path(self, path):
if self._path.isnone:
title = "Select a configuration file to load."
else:
title = "Bad configuration file set. Please select a valid configuration file."
title += " [Cancel for default configuration]"
self._path = open_file(path, title=title, ask_path=self._ask_path, behavior_on_cancellation='ignore')
logger.debug("Config path set to '{}'".format(self._path))
def to_list(v, sep=','): # deprecated, replaced by ast.literal_eval
logger.debug(
"DeprecationWarning: to_list function is deprecated. Use ast.literal_eval instead."
)
assert isinstance(sep, str)
_s = str(v).strip().lstrip('[').rstrip(']').strip()
_l = _s.split(sep)
_l = [ele.strip() for ele in _l if ele]
return _l
def users_list(self, usernames, only_new=True):
usernames = [u.lower() for u in usernames]
if only_new:
for username in usernames.copy(): # Todo: verify copy() is necessary
if get_a_profile(username):
logger.debug(f'Username {username} already exists')
usernames.remove(username)
self.usersnames_df = pd.DataFrame(usernames, columns=['username'])
return self
def q_save_a_tweet(tweet):
collection = get_collection()
try:
result = collection.insert_one(tweet)
except DuplicateKeyError as e:
logger.debug(f"Duplicate: {tweet['tweet_id']} - {tweet['date']} - {tweet['name']}")
except:
logger.error(f'Unknown error: {sys.exc_info()[0]}')
raise
def _set_writable(path: str):
"""Make a file writable if it exists and is read-only."""
if Path(path).isfile and not os.access(path, os.W_OK):
try:
os.chmod(path, stat.S_IWUSR | stat.S_IREAD) # if read-only existing file, make it writable
except PermissionError:
logger.debug("Failed to change file properties of '{}'".format(path))
return
logger.debug("File '{}' is now writable.")
def q_update_a_tweet(tweet):
collection = get_collection()
f = {'tweet_id': tweet['tweet_id']}
u = {'$set': tweet}
try:
result = collection.update_one(f, u, upsert=True)
logger.debug(f"Updated: {result.raw_result} - {tweet['tweet_id']} - {tweet['date']} - {tweet['name']}")
except DuplicateKeyError as e:
logger.debug(f"Duplicate: {tweet['tweet_id']} - {tweet['date']} - {tweet['name']}")
def on_closing(self):
res = messagebox.askyesnocancel(
message="Do you want to select this date? "
"Click 'Cancel' to end the program, "
"click 'No' to come back to the selection window.")
if res is None:
logger.debug("Program terminated by the user ('Cancel' button).")
self.close()
raise KeyboardInterrupt(
"Program terminated by the user ('Cancel' button).")
if res:
self.close()
def _check_section(self, section: Union[str, list], search_in_default_config: bool = None):
"""If the section doesn't exist and search_in_default_config,
append the section from default config to current configuration"""
section = None if section is None else ConfigDict.TO_KEY_FUNC(section)
search_in_default_config = self._search_in_default_config if search_in_default_config is None \
else search_in_default_config
if section not in self.sections() and section is not None and search_in_default_config:
# if the section doesn't exist, append the default configuration to the configuration
self.add_default_config_sections(sections=section)
# self.reload_default(write=False, how='append') # old method
logger.debug("Section(s) '{}' of default configuration appended to config.".format(section))
return section
def q_update_a_tweet(tweet): # Todo: replace tweet with its fields
collection = get_collection()
f = {'tweet_id': tweet['tweet_id']}
tweet.update({'timestamp': datetime.now()})
u = {'$set': tweet}
try:
result = collection.update_one(f, u, upsert=True)
logger.debug(
f"Updated: {result.raw_result} | {tweet['date']} {tweet['name']}")
except DuplicateKeyError as e:
logger.error(
f"Duplicate: {tweet['tweet_id']} - {tweet['date']} - {tweet['name']}"
)
def __getitem__(self, item):
if item in self._temp_config: # 1st, try to find the key in temp config
logger.debug("temporary config used for key '{}'".format(item))
return self._temp_config[item]
try:
return self._cfg[item] # 2nd, try to find the key in current config
except KeyError as _err_msg:
try: # 3rd, try to find the key in default config TODO: use search in default config
res = self._default_config[item]
logger.debug("Item '{}' found in default configuration instead of current configuration.".format(item))
self._cfg[item] = res # set item to current configuration
return res
except KeyError as err_msg:
raise KeyError(err_msg)
def __init__(self,
parts: OrderedDict,
logger_level=logging.INFO,
nb_columns=3,
help_msg="No help_msg available.",
title="main",
**options):
super().__init__(title=title, **options)
self.help_text = help_msg
self.help_win = None
self.load_help()
self.bind('<F1>', func=self.open_help)
self.protocol("WM_DELETE_WINDOW", self.on_closing)
self.parts = OrderedDict()
for i, (title, answers) in enumerate(parts.items()):
q_f = _MainPart(master=self,
text=title,
answers=answers,
geometry_manager='grid',
geometry_options=dict(sticky='ew',
nb_columns=nb_columns))
q_f.pack(fill=tk.BOTH)
self.parts[title] = q_f
# Info frame
self.info_frame = ttk.Frame(master=self)
self.info_frame.pack(fill=tk.BOTH, expand=True)
self.func_text = tk.StringVar()
self.func_label = ttk.Label(master=self.info_frame,
textvariable=self.func_text)
self.func_label.grid(row=0, column=0, sticky=tk.W)
self.progressbar = ttk.Progressbar(master=self.info_frame,
mode='indeterminate',
length=200,
value=0)
self.help_button = ttk.Button(master=self.info_frame,
text="Help",
command=self.open_help)
self.help_button.grid(row=0, column=2, sticky='e')
self.logger_frame = LoggingHandlerFrame(master=self.info_frame,
level=logger_level)
self.logger_frame.grid(row=1, column=0, columnspan=3, sticky='nsew')
self.info_frame.grid_rowconfigure(
1, weight=1) # make logger frame expandable vertically
self.info_frame.grid_columnconfigure(0, weight=1) # idem horizontally
self.active_thread = None
logger.addHandler(self.logger_frame.logging_handler)
logger.debug("Window loaded.")
logger.info("Main window loaded. Please select an option or quit.")
def _no_flag_handling(key, no_flag='ignore'):
if no_flag == 'ignore':
return []
elif no_flag == 'drop':
logger.debug(
"Key '{}' will be dropped because it has no flag.".format(key))
return None
elif no_flag == 'auto-conversion':
logger.debug(
"Auto conversion will be performed for key '{}' because it has no flag."
.format(key))
return [AUTO_FLAG]
else:
err_msg = "No flag for the key '{}'".format(key)
logger.error(err_msg)
raise ValueError(err_msg)
def download_xml(self, download_url, item_id):
response = requests.get(download_url,
auth=HTTPBasicAuth(
os.environ['PLANET_API_KEY'], ''))
status = check_response(response)
output_file = os.path.join(self.indir,
item_id.rstrip() + '_metadata.xml')
logger.debug(output_file)
if not "<?xml" in response.text:
status = False
with open(output_file, 'w') as f:
f.write(response.text)
return status
def run(self):
logger.debug("start task with func '{}'".format(self.func_name))
try:
self.func(*self.args, **self.kwargs)
msg = "'{}' successfully ended.".format(self.func_name)
logger.info(
"Function '{}' successfully ended.\n***************\n".format(
self.func_name))
except Exception as err:
logger.exception(err)
msg = "'{}' ended with error.".format(self.func_name)
logger.error(
"Function '{}' ended with error.\n***************\n".format(
self.func_name))
self.queue.put(
msg
) # add func name to queue to indicate that the task is finished
logger.debug("end task with func '{}'".format(self.func_name))
def download_udm(self, download_url, item_id):
vsicurl_url = '/vsicurl/' + download_url
logger.debug(vsicurl_url)
output_file = os.path.join(self.indir,
item_id.rstrip() + '_subarea_udm.tif')
logger.debug(output_file)
status = True
try:
# GDAL Warp crops the image by our AOI, and saves it
gdal.Warp(output_file,
vsicurl_url,
dstSRS='EPSG:4326',
cutlineDSName=self.aoi_geojson,
cropToCutline=True)
except Exception as exc:
logger.error(exc)
status = False
return status
def discover(self, base_directory, only=None):
# Check if there are tests
if not os.path.isdir(base_directory):
logger.critical(
'The {} directory is absent!'.format(base_directory))
exit(1)
if only is not None and isinstance(only, list):
only = set(only)
# Discover the tests
nb_tests_before = len(self.tests)
for test_directory in ['.'] + list(sorted(os.listdir(base_directory))):
logger.debug('Looking for tests in:', test_directory)
if os.path.isdir(os.path.join(base_directory, test_directory)):
for test_file in sorted(
os.listdir(os.path.join(base_directory,
test_directory))):
if only is None or test_file in only:
if only is not None:
only.remove(test_file)
test_path = os.path.join(base_directory,
test_directory, test_file)
if os.path.isfile(test_path):
self.add_test(test_file, test_path,
test_directory == 'invalid')
# Consider remaining test files as relative paths
if only is not None:
for test_path in only:
if not os.path.isfile(test_path):
logger.fatal('Could not find test: {}'.format(test_path))
exit(1)
self.add_test(test_path.split('/')[-1], test_path, False)
logger.info(
'Discovered',
colored('{} tests.'.format(len(self.tests) - nb_tests_before),
color='yellow'))
def try_it():
"""Try something! (this message is shown by default as tooltip)"""
# Test logger
logger.debug('Test debug message')
logger.info('Test info message')
logger.error('Test error message')
# Print directories
logger.info("Working directory: {}".format(os.getcwd()))
logger.info("File path: {}".format(__file__))
# Test message box
res = messagebox.askyesno(title="This is a message box!",
message="Do you want to raise an error?")
if res:
err_msg = "This is the error you asked!"
logger.error(err_msg)
raise ValueError(err_msg)
logger.info("No error was raised!")
# Test get_user_date
message = "This is the end of this sample script ending at {}".format(
get_user_date(bypass_dialog=True))
messagebox.showinfo(title="This is already the end...", message=message)
def __init__(self, *args, **kwargs):
"""Initialisation of a _Config instance.
:param args: see 'init' method
:param kwargs: see 'init' method
"""
super().__init__()
self._cfg = ConfigDict() # current configuration
self._default_config = ConfigDict() # default configuration
self._temp_config = OrderedDict() # temporary configuration
self._path = Path() # current configuration path
self._default_path = Path() # default configuration path
self._conversion_dict = None
self._auto_cast = None
self._write_flags = None
self._force_load = None
self._load_empty = None
self._ask_path = None
self._search_in_default_config = None
self._init_count = 0
self._policies = defaultdict(bool) # by default every modification is forbidden # WIP
if args or kwargs:
self.init(*args, **kwargs)
logger.debug("Config object created.")
def on_closing(self):
# Case of an active thread
if self.active_thread and self.active_thread.is_alive():
logger.warning("Process running in background!")
logger.debug("Thread '{}: {}' still alive.".format(
self.active_thread.name, self.active_thread))
logger.debug(
"number of active threads: {}, current thread: {}, main thread: {}."
.format(threading.active_count(), threading.current_thread(),
threading.main_thread()))
self.active_thread.join(timeout=1) # wait 1 second
if self.active_thread.is_alive(
): # if the thread is still alive after 1 second
if not self.ask_exit(self.active_thread.func_name):
logger.debug("Exit cancelled")
return False
# Remove logger handler attached to tkinter frame
self.logger_frame.quit()
# Destroy the window
self.destroy()
logger.debug("Main window destroyed.")
self.quit()
logger.debug("Main window quited.")
return True
def _handle_existing_file_conflict(path: Path, overwrite='ask', backup=False, **kwargs) -> Union[Path, None]:
"""Handle conflict if a file already exist by opening adapted dialog."""
# overwrite 'ask': ask user to modify overwrite arg into 'overwrite' (Yes) or 'rename' (No) or return None (Cancel)
if overwrite == 'ask':
logger.warning('User action needed!')
res = messagebox.askyesnocancel(title="File existing",
message="File {} already exists.\nDo you want to overwrite it?"
"\n\nIf you select 'No', the file will be "
"renamed automatically.".format(path))
if res is None:
logger.info("'Save file' operation cancelled by the user.".format(path))
logger.debug("The path 'None' will be returned.")
return None
if res:
overwrite = 'overwrite'
else:
overwrite = 'rename'
# overwrite 'rename' or False: add '-i' at the end of the path to make it unique, where 'i' is an integer.
if overwrite == 'rename' or overwrite is False:
r_path, r_ext = path.splitext
# def rename_method1(r_path, sep='-'):#todo
ls_end = re.findall(r'-(\d+)$', r_path)
if ls_end: # if the path already ends by '-i', change end to 'i+1'
end = ls_end[0]
r_path = r_path[:-(len(end) + 1)]
added_ending = "-{}".format(int(end) + 1)
else:
added_ending = "-1"
n_path = r_path + added_ending + r_ext
logger.debug("Path {} changed to {} (renaming)".format(path, n_path))
return save_file(n_path, overwrite=overwrite, backup=backup, **kwargs)
# backup True: backup the old file
if backup and path.isfile:
suffix = datetime.datetime.now().strftime("-%Y-%m-%d_%H-%M_") + uuid.uuid4().hex[:5]
try:
shutil.copyfile(path, path.radix + suffix + path.ext)
except (PermissionError, FileNotFoundError) as err:
logger.exception(err)
logger.error("Failed to backup previous configuration file.")
# overwrite 'overwrite' or True: do not modify the path and make the old file writable to allow overwriting
if overwrite == 'overwrite' or overwrite is True:
logger.debug("File {} will be overwritten".format(path))
_set_writable(path)
# overwrite 'ignore': do nothing
elif overwrite == 'ignore':
pass
# other case of overwrite: do nothing (same as 'ignore')
else:
logger.warning("Unexpected argument 'overwrite'! File {} will be overwritten".format(path))
return path
def edit(self, **kwargs):
"""Edit attributes of the configuration."""
for attr in self.EDITABLE_ATTR:
kwarg = kwargs.pop(attr, self._WILDCARD)
if kwarg is not self._WILDCARD:
setattr(self, attr, kwarg)
logger.debug("Attribute '{}' changed to '{}'.".format(attr, kwarg))
for p_attr in self.EDITABLE_PRIVATE_ATTR:
kwarg = kwargs.pop(p_attr, self._WILDCARD)
if kwarg is not self._WILDCARD:
setattr(self, "_" + p_attr, kwarg)
logger.debug("Private attribute '{}' changed to '{}'.".format(p_attr, kwarg))
logger.debug("Configuration edited.")
def _write_excel(path, dataframes, sheet_names, index=False, **kwargs):
"""Save dataframes to an Excel workbook.
:param path: output path
:param dataframes: list of dataframes
:param sheet_names: list of sheet names (same index as dataframes)
:param index: if True, index names are exported
:param kwargs: keyword arguments for pd.DataFrame.to_excel function
:return: final output path
"""
if path is None or path.isnone:
logger.warning('No path set to write data to Excel! No file has been created.')
return None
try:
logger.debug("Trying to open file '{}'".format(path))
with pd.ExcelWriter(path) as writer:
logger.debug("File {} opened.".format(path))
for df, sheet_name in zip(dataframes, sheet_names):
if isinstance(df.columns, pd.MultiIndex):
index = True # index must be True if MultiIndex columns, otherwise NotImplementedError is raised
df.to_excel(writer, sheet_name=sheet_name, index=index, **kwargs)
logger.debug("Sheet {} written.".format(sheet_name))
logger.debug("File {} closed.".format(path))
except PermissionError as err:
logger.exception(err)
path = handle_permission_error(err, func=_write_excel, path=path, args=[dataframes, sheet_names, index],
kwargs=kwargs, change_path_func=save_file, handle_read_only_error=True)
except ValueError as err:
if str(err).startswith("No engine for filetype"):
path = handle_bad_extension_error(err, func=_write_excel, path=path, args=[dataframes, sheet_names, index],
kwargs=kwargs, change_path_func=save_file, extension=".xlsx")
else:
raise err
except FileNotFoundError as err:
logger.exception(err)
path = handle_file_not_found_error(err, func=_write_excel, path=path, args=[dataframes, sheet_names, index],
kwargs=kwargs, change_path_func=save_file)
return path
async def parse_item_page(self, content=None, detail=None, rate=None):
if content:
sku_map = re.search('skuMap.*?(\{.*)', content)
shop_id = store_trans(self.fromStore, 'code_2_id')
doc = PyQuery(content)
items = doc("li[data-value]").items()
logger.debug(items)
attr_map = {}
if items:
for item in items:
attr_map[item.attr('data-value')] = item.find(
'span').text().replace("（", "(").replace("）", ")")
if sku_map:
sku_dict = json.loads(sku_map.group(1))
for k, v in sku_dict.items():
for price_tb_item in self.price_tb_items:
if price_tb_item.skuId == v.get('skuId'):
price_tb_item.price_tb = v.get('price')
price_tb_item.shop_id = shop_id
price_tb_item.attribute_map = k
price_tb_item.attribute = "-".join([
attr_map.get(r)
for r in re.sub('^;|;$', "", k).split(";")
])
else:
self.price_tb_items[0].shop_id = shop_id
self.price_tb_items[0].price_tb = doc(
'input[name="current_price"]').val()
self.completed = 2
if detail:
while 1:
if self.completed == 2:
break
await asyncio.sleep(1)
logger.debug(detail)
detail = re.sub(r'span class=\"wl-yen\"',
r'span class=\\"wl-yen\\"', detail)
data = re.search('uccess\((.*?)\);', detail)
if data:
x = json.loads(data.group(1))
else:
await self.login.slider(self.item_page)
return
promo_data = jsonpath(x, '$..promoData')
for price_tb_item in self.price_tb_items:
price_tb_item.sales = jsonpath(x, '$..soldTotalCount')[0]
price_tb_item.typeabbrev = self.fromStore
if promo_data and promo_data[0]:
if price_tb_item.attribute_map:
for k, v in promo_data[0].items():
if k == price_tb_item.attribute_map:
price_tb_item.promotionprice = jsonpath(
v, '$..price')[0]
else:
price_tb_item.promotionprice = jsonpath(
x, '$..promoData..price')[0]
self.completed = 3
if rate:
while 1:
if self.completed == 3:
break
await asyncio.sleep(1)
logger.debug(rate)
ms = MySql()
for price_tb_item in self.price_tb_items:
count = re.search('count.*?(\d+)', rate)
if count:
price_tb_item.rates = count.group(1)
price_tb_item.need_to_update = 0
price_tb_item.save(ms)
# print(price_tb_item)
self.price_tb_items[0].delete(ms)
del ms
self.completed = 4
