def workerProc(arg):
client = Marionette('localhost', port=2828)
client.start_session()
client.navigate('https://www.mercari.com/sell/')
time.sleep(5)
#first open the excel file to grab information
#photos will be stored in photos file
num_rows = arg
items = load_excel("Mercari.xlsx", num_rows)
for item in items:
title = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[3]/div[1]/div[2]/input")
title.send_keys(item[0])
textarea = client.find_element(By.TAG_NAME, 'textarea')
textarea.send_keys(item[1])
zipcode = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[6]/div[1]/div[2]/input")
zipcode.send_keys(item[8])
price = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[7]/div/div[2]/input")
price.send_keys(item[10])
brand = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[7]/div/div[2]/input")
brand.send_keys(item[6])
category(client, item[2], item[3], item[4])
if item[6] >= 0:
submit_size(client, item[6])
time.sleep(1)
#determineSize(client, 1)
selectCondition(client, item[7])
ship_selfpaid(client)
for i in range(item[10]):
path = os.path.dirname(os.path.realpath("photos\img" + str(i) + ".jpg"))
fileUpload(client, getPID(), path + "\img" + str(i) + ".jpg", i)
submit_button(client)
time.sleep(2)
client.navigate('https://www.mercari.com/sell/')
class Puppet:
MIME_TYPES = [
"application/epub+zip",
"application/gzip",
"application/java-archive",
"application/json",
"application/ld+json",
"application/msword",
"application/octet-stream",
"application/ogg",
"application/pdf",
"application/rtf",
"application/vnd.amazon.ebook",
"application/vnd.apple.installer+xml",
"application/vnd.mozilla.xul+xml",
"application/vnd.ms-excel",
"application/vnd.ms-fontobject",
"application/vnd.ms-powerpoint",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.text",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.visio",
"application/x-7z-compressed",
"application/x-abiword",
"application/x-bzip",
"application/x-bzip2",
"application/x-csh",
"application/x-freearc",
"application/xhtml+xml",
"application/xml",
"application/x-rar-compressed",
"application/x-sh",
"application/x-shockwave-flash",
"application/x-tar",
"application/zip",
"appliction/php",
"audio/aac",
"audio/midi audio/x-midi",
"audio/mpeg",
"audio/ogg",
"audio/wav",
"audio/webm",
"font/otf",
"font/ttf",
"font/woff",
"font/woff2",
"image/bmp",
"image/gif",
"image/jpeg",
"image/png",
"image/svg+xml",
"image/tiff",
"image/vnd.microsoft.icon",
"image/webp",
"text/calendar",
"text/css",
"text/csv",
"text/html",
"text/javascript",
"text/javascript",
"text/plain",
"text/xml",
"video/3gpp",
"video/3gpp2",
"video/mp2t",
"video/mpeg",
"video/ogg",
"video/webm",
"video/x-msvideo"
]
METHOD_CSS_SELECTOR = "css selector"
NO_LOG = "-"
DELETE_TARGET_FILES = ["mimeTypes.rdf", "handlers.json"]
USER_DEFINED = 2
GECKO_LOG = Path(__file__).parent.resolve()
def __init__(self, binary: str, profile: str):
self.__has_session = False
self.__auto_download = False
self.__download_dir = ""
if not Path(binary).is_file():
print(f"Binary {binary} Not Found")
return
if not Path(profile).is_dir():
print(f"Profile {profile} Not Found")
return
# geckodriver の log ファイル出力を抑止する
self.marionette = Marionette(
bin=binary, gecko_log=self.NO_LOG, profile=profile)
# start_session 前にファイルを消しておかないと
# 後で自動ダウンロードできない
self.__delete_download_profile()
# start_session しないと quit もできない
self.marionette.start_session()
self.__has_session = True
def __enter__(self):
return self
def __exit__(self, ex_type, ex_value, trace):
if self.has_session:
self.quit()
@property
def has_session(self):
return self.__has_session
@property
def auto_download(self):
return self.__auto_download
def __delete_download_profile(self):
# mimeTypes.rdf と handlers.json に
# ファイル読み込み時の動作設定が保存されている（text/plain はファイルを保存、など）
# 自動ダウンロードするため既存の設定は削除する
for name in self.DELETE_TARGET_FILES:
p = Path(self.marionette.profile_path).joinpath(name)
if p.is_file():
p.unlink()
def __activate_auto_download(self):
# 一度有効にすると同セッション内では無効にできない
self.marionette.set_pref("browser.download.useDownloadDir", True)
self.marionette.set_pref("browser.helperApps.neverAsk.saveToDisk",
",".join(self.MIME_TYPES))
self.marionette.set_pref(
"browser.download.folderList", self.USER_DEFINED)
self.marionette.set_pref("browser.download.lastDir", None)
self.__auto_download = True
@property
def download_dir(self):
if self.__auto_download == False:
raise Exception("auto download not activated")
return self.__download_dir
@download_dir.setter
def download_dir(self, dir: str):
p = Path(dir)
if not p.is_dir():
print(f"Download Dir {dir} Not Found")
return
full_path = str(p.resolve())
if self.__auto_download == False:
self.__activate_auto_download()
# self.__auto_download = True
self.marionette.set_pref("browser.download.dir", full_path)
self.marionette.set_pref("browser.download.downloadDir", full_path)
self.__download_dir = full_path
def set_download(self, dir: str):
self.download_dir = dir
def query_selector(self, selectors: str) -> HTMLElement:
return self.marionette.find_element(self.METHOD_CSS_SELECTOR, selectors)
def query_selectors(self, selectors: str) -> List[HTMLElement]:
return self.marionette.find_elements(self.METHOD_CSS_SELECTOR, selectors)
def wait(self, seconds: int):
actions = Actions(self.marionette)
actions.wait(seconds).perform()
def quit(self):
profile = Path(self.marionette.profile_path)
self.marionette.quit(clean=True)
# self.__forced_rmdir(profile)
# Path(self.GECKO_LOG).unlink()
self.__has_session = False
def exec(self, script: str) -> Optional[str]:
# script 内での記述簡略化のため
mrnt = self.marionette
set_download = self.set_download
wait = self.wait
quit = self.quit
query_selector = self.query_selector
query_selectors = self.query_selectors
try:
exec(script)
return None
except Exception as err:
return str(err.args[0])
@classmethod
def __forced_rmdir(self, p: Path):
if p.is_dir():
for f in p.iterdir():
if f.is_file():
f.unlink()
elif f.is_dir():
self.__forced_rmdir(f)
p.rmdir()
class MarionetteHelper:
""" Helper for starting firefox and for remote browsing
"""
def __init__(self, firefox_path, logger=None):
self.logger = logger # type: logging.Logger
self.client = None # type: Marionette
self.ffpopen = None # type: subprocess.Popen
self.ffpath = firefox_path # type: str
if logger is None:
self.logger = logging.getLogger("MarionetteHelper")
self.logger.debug("Marionette helper init done")
def _open_session(self, host='localhost', port=2828):
""" Opens the session for marionette"""
caps = {u'acceptInsecureCerts': True, }
self.client = Marionette(host, port=port)
self.client.start_session(capabilities=caps)
def run_firefox(self):
""" Start the firefox process"""
self.logger.debug("Starting firefox process")
self.ffpopen = subprocess.Popen([self.ffpath, "-marionette"])
self.logger.debug("Opening marionette session")
self._open_session()
def quit_firefox(self):
""" Close the firefox process and close marionette session"""
#self.logger.debug("Closing firefox")
#self.client._send_message("Marionette:Quit")
self.client._request_in_app_shutdown("eForceQuit")
self.client.delete_session(False)
self.client.cleanup()
self.client = None # reset client state
#try:
# self.client.close() # try to close the window anyway
#except InvalidSessionIdException:
# pass
#except socket.error:
# pass
#finally:
# try:
# self.logger.debug("Closing marionette session")
# self.client.delete_session(False) # close client session
# except InvalidSessionIdException:
# pass
# except socket.error:
# pass
# self.client = None # reset client state
#self.logger.debug("Waiting for firefox to close")
#for _ in range(3): # give the process 3 seconds to terminate
# time.sleep(1)
# self.ffpopen.poll()
# if self.ffpopen.returncode is not None:
# break
#self.ffpopen.poll()
#if self.ffpopen.returncode is None:
# self.logger.warning("Firefox not closed in time, killing it!")
# self.ffpopen.kill() # process was not quit in time, kill it
#self.ffpopen = None # reset popen state
#self.logger.debug("Firefox is closed")
def ___get_client(self):
""" Returns the internal marionette client object"""
return self.client
def navigate_to_url(self, url):
""" Open an url in format of http[s]://example.com"""
try:
if "http" not in url:
url = "http://" + url
self.client.navigate(url)
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def back(self):
""" Go a page backward"""
try:
self.client.go_back()
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def forward(self):
""" Go a page forward"""
try:
self.client.go_forward()
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def follow_link(self, index=0):
""" Click on a link"""
try:
links = self.client.find_elements(By.TAG_NAME, "a")
links[index].click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except IndexError:
self.logger.warning("Error: Out of bound")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_class_name(self, html_class_name):
""" Click on first element via class name"""
try:
e = self.client.find_element(By.CLASS_NAME, html_class_name)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_css_selector(self, css_selector):
""" Click on first element via css selector"""
try:
e = self.client.find_element(By.CSS_SELECTOR, css_selector)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_id(self, html_id):
""" Click on first element via element id"""
try:
e = self.client.find_element(By.ID, html_id)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_name(self, html_name):
""" Click on first element via element name"""
try:
e = self.client.find_element(By.NAME, html_name)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_tag_name(self, html_tag_name):
""" Click on first element via tag name"""
try:
e = self.client.find_element(By.TAG_NAME, html_tag_name)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_xpath(self, xpath):
""" Click on first element via xpath"""
try:
e = self.client.find_element(By.XPATH, xpath)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def click_element_by_link_text(self, html_link_text):
""" Click on first element via link text"""
try:
e = self.client.find_element(By.LINK_TEXT, html_link_text)
e.click()
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
except InsecureCertificateException:
self.logger.warning("Warning: Insecure Certificate")
return True
except UnknownException as e:
if "Reached error page" in str(e):
self.logger.warning("Reached error page, ignoring...")
else: # reraise, something very unexpected happened here
t, v, tb = sys.exc_info()
raise t, v, tb
return False
return True
def send_keys_to_element_by_name(self, name, text):
""" Sends text to an element via name"""
try:
e = self.client.find_element(By.NAME, name)
e.send_keys(text)
except ElementNotVisibleException:
self.logger.warning("Error: Element not visible")
return False
except ElementNotSelectableException:
self.logger.warning("Error: Element not selectable")
return False
except ElementNotAccessibleException:
self.logger.warning("Error: Element not accessible")
return False
except ElementNotInteractableException:
self.logger.warning("Error: Element not interactable")
return False
except NoSuchElementException:
self.logger.warning("Error: Element does not exist")
return False
except TimeoutException:
self.logger.warning("Error: Timeout")
return False
return True
def select_window(self, index):
""" Switch window via index"""
try:
self.client.switch_to_window(self.client.window_handles[index])
except NoSuchWindowException:
self.logger.warning("Error: Window does not exist")
return False
return True
def close_window(self):
""" Close the current window"""
self.client.close() # this won't close the last window, call quit_firefox instead
def get_current_window_index(self):
""" Get current windows index"""
return self.client.window_handles.index(self.client.current_window_handle)
def new_tab(self):
""" Open a new empty tab"""
with self.client.using_context("chrome"):
self.client.find_element(By.ID, "menu_newNavigatorTab").click()
def new_window(self):
""" Open a new empty window"""
with self.client.using_context("chrome"):
self.client.execute_script("window.open();")
def close_tab(self):
""" Close the current tab"""
self.close_window() # basically the same as close windows
class MarionetteHelper:
def __init__(self, logger, success_symbol, failure_symbol):
"""
Initialise the helper class.
"""
self.client = None
self.logger = logger
self.success_symbol = success_symbol
self.failure_symbol = failure_symbol
def init_ff(self):
"""
Initialises the connection to Firefox and starts a session.
@return: -
"""
if not check_socket(MARIONETTE_HOST, MARIONETTE_PORT):
self.logger.error(
u" > [ERROR] Please check if you started Firefox with the '-marionette' "
"option or set 'marionette.enabled' to 'true' in 'about:config'. {}"
.format(self.failure_symbol))
sys.exit(1)
self.client = Marionette(host=MARIONETTE_HOST, port=MARIONETTE_PORT)
self.client.start_session()
def get_existing_bookmarks(self):
"""
Get the existing bookmarks from the Google Bookmarks API.
We need to do this in Firefox to have the cookie set which authorities us with the API.
@return: -
"""
self.client.navigate(
"https://www.google.com/bookmarks/?output=xml&num=10000")
# Initialise XML object
root = XML(self.client.page_source.encode("utf-8"))
# Return set of bookmarks
return set([bookmark[1].text for bookmark in root[0]])
def save_button_contains_correct_text_save(self, *args):
"""
Helper method for Marionette, here: check if fav button contains text "SAVE"
@return: Whether or not the fav button contains the text "SAVE"
"""
save_button = self.client.find_element(
By.CLASS_NAME, "section-entity-action-save-button")
return save_button.text == "SAVE"
def save_button_contains_correct_text_saved(self, *args):
"""
Helper method for Marionette, here: check if fav button contains text "SAVED"
@return: Whether or not the fav button contains the text "SAVED"
"""
save_button = self.client.find_element(
By.CLASS_NAME, "section-entity-action-save-button")
return save_button.text == "SAVED"
def interactive_add_feature(self, coordinates):
"""
Navigates to the Google Maps URL for the provided coordinates and waits for input.
@return: -
"""
url = "https://www.google.com/maps/search/?api=1&query={},{}"
# This navigates Firefox to the passed URL
self.client.navigate(url.format(coordinates[0], coordinates[1]))
# Wait for input
if sys.version_info[0] < 3:
raw_input("Press Enter to continue...")
else:
input("Press Enter to continue...")
def add_feature_2(self, url, list_add):
"""
Tries to add a feature (bookmark / place) to your Google Maps fav list.
@return: -
"""
self.client.navigate(url)
try:
saved_button = Wait(self.client, timeout=1).until(
expected.element_present(By.CSS_SELECTOR,
"[data-value='Saved']"))
self.logger.info(" > Feature was already saved")
return utils.constants.ADD_FEATURE_ALREADY_ADDED
except TimeoutException:
pass
try:
save_button = Wait(self.client, timeout=5).until(
expected.element_present(By.CSS_SELECTOR,
"[data-value='Save']"))
Wait(self.client,
timeout=5).until(expected.element_displayed(save_button))
except TimeoutException:
self.logger.error(" > Unable to find save button")
return utils.constants.ADD_FEATURE_UNKNOWN_ERROR
save_button.click()
if list_add == utils.constants.LIST_STARRED_PLACES:
data_index = 2
elif list_add == utils.constants.LIST_WANT_TO_GO:
data_index = 1
else:
data_index = -1
css_selector = "#action-menu [data-index='{}']".format(data_index)
sub_save_item = Wait(self.client, timeout=5).until(
expected.element_present(By.CSS_SELECTOR, css_selector))
Wait(self.client,
timeout=5).until(expected.element_displayed(sub_save_item))
sub_save_item.click()
def add_feature(self, url):
"""
Tries to add a feature (bookmark / place) to your Google Maps fav list.
@return:
- ADD_FEATURE_FAILURE if adding resulted in a known failure
- ADD_FEATURE_SUCCESS if everything went fine
- ADD_FEATURE_UNKNOWN_ERROR if we don't know what happened
"""
# This navigates Firefox to the passed URL
self.client.navigate(url)
# We wait for the fav button to be present...
save_button = Wait(self.client, timeout=10).until(
expected.element_present(By.CLASS_NAME,
"section-entity-action-save-button"))
# ... and to be displayed
displayed = Wait(self.client, timeout=10).until(
expected.element_displayed(save_button))
try:
# Now we look for the correct text, it should say "SAVE"
Wait(self.client,
timeout=6).until(self.save_button_contains_correct_text_save)
try:
# Click it to add the feature (bookmark / place) to the Google Maps fav list
save_button.click()
except NoSuchElementException:
pass
try:
# Now the text should be "SAVED" and this indicates it was saved
Wait(self.client, timeout=6).until(
self.save_button_contains_correct_text_saved)
except TimeoutException:
# We clicked but the fav button text didn't change, i.e. the click went wrong or timed out
self.logger.error(" > [ERROR] Feature: '{}'".format(url))
save_button = self.client.find_element(
By.CLASS_NAME, "section-entity-action-save-button")
self.logger.error(
" > [ERROR] Save button didn't switch to 'SAVED', it contains '{}'"
.format(save_button.text))
return ADD_FEATURE_FAILURE
return ADD_FEATURE_SUCCESS
except TimeoutException:
# This is the case if the fave button didn't contain the text "SAVE".
# This can happen if it contains "SAVED", but this shouldn't happen in the
# first place because we don't try to add features if we know that they're
# already added.
# So most likely something truly went wrong here.
self.logger.error(" > [ERROR] Feature: '{}'".format(url))
save_button = self.client.find_element(
By.CLASS_NAME, "section-entity-action-save-button")
self.logger.error(
" > [ERROR] Save button contained unknown text '{}'".format(
save_button.text))
return ADD_FEATURE_UNKNOWN_ERROR
from marionette_driver.marionette import Marionette
from marionette_driver import By
client = Marionette('localhost', port=2828)
client.start_session()
client.get_url()
client.find_element(By.TAG_NAME, 'body')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from marionette_driver.marionette import Marionette
import pdb
client = Marionette("localhost", port=2828)
client.start_session()
client.navigate("http://www.tianyancha.com/company/75507246")
element = client.find_element("class name", 'company_info')
print element.text
class Puppet:
def __init__(self, binary: str, profile: str):
self.__has_marionette = False
self.__auto_download = False
self.__download_dir = ""
if not Path(binary).is_file():
return
if not Path(profile).is_dir():
return
# geckodriver の log ファイル出力を抑止する
NO_LOG = "-"
self.marionette = Marionette(bin=binary,
gecko_log=NO_LOG,
profile=profile)
# start_session しないと quit もできない
self.marionette.start_session()
self.__has_marionette = True
@property
def has_marionette(self):
return self.__has_marionette
@property
def auto_download(self):
return self.__auto_download
def __activate_auto_download(self):
# 一度有効にすると同セッション内では無効にできない
# firefox52 では MIME_TYPES.rdf, firefox60 では handlers.json に
# ファイルダウンロード時の動作設定が記述されている（text/plain はプログラムで開く、など）
# 自動ダウンロードするため既存の設定は削除する
MIME_TYPES_HANDLERS = ["MIME_TYPES.rdf", "handlers.json"]
for name in MIME_TYPES_HANDLERS:
p = Path(self.marionette.profile_path).joinpath(name)
if p.is_file():
p.unlink()
self.marionette.set_pref("browser.download.useDownloadDir", True)
self.marionette.set_pref("browser.helperApps.neverAsk.saveToDisk",
",".join(MIME_TYPES))
USER_DEFINED = 2
self.marionette.set_pref("browser.download.folderList", USER_DEFINED)
@property
def download_dir(self):
if self.__auto_download == False:
raise Exception("auto download has not been activated")
return self.__download_dir
@download_dir.setter
def download_dir(self, dir: str):
p = Path(dir)
if not p.is_dir():
return
full_path = str(p.resolve())
if self.__auto_download == False:
self.__activate_auto_download()
self.__auto_download = True
self.marionette.set_pref("browser.download.dir", full_path)
self.__download_dir = full_path
def set_download(self, dir: str):
self.download_dir = dir
def query_selector(self, selectors: str) -> HTMLElement:
METHOD_CSS_SELECTOR = "css selector"
return self.marionette.find_element(METHOD_CSS_SELECTOR, selectors)
def query_selectors(self, selectors: str) -> List[HTMLElement]:
METHOD_CSS_SELECTOR = "css selector"
return self.marionette.find_elements(METHOD_CSS_SELECTOR, selectors)
def wait(self, seconds: int):
actions = Actions(self.marionette)
actions.wait(seconds).perform()
def quit(self):
self.marionette.quit()
def exec(self, script: str):
# script 内での記述簡略化のため
mrnt = self.marionette
set_download = self.set_download
wait = self.wait
quit = self.quit
query_selector = self.query_selector
query_selectors = self.query_selectors
exec(script)
class FirefoxMarionetteBase(object):
"""
Wrap Marionette/Firefox into convenient interface.
- https://marionette-client.readthedocs.io/
- https://marionette-client.readthedocs.io/en/master/reference.html
- https://marionette-client.readthedocs.io/en/master/interactive.html
"""
def __init__(self):
logger.info('Starting Marionette Gecko wrapper')
# Configuration
self.firefox_bin = self.find_firefox()
self.firefox_host = 'localhost'
self.firefox_port = 2828
# TODO: Make configurable
self.firefox_verbosity = 1
#self.firefox_verbosity = 2
# Timeout configuration
self.startup_timeout = 20.0
self.socket_timeout = 32.0
self.page_timeout = 30.0
self.script_timeout = 20.0
self.shutdown_timeout = 10.0
# Instance state defaults
self.marionette = None
self.firefox_run_headless = True
self.firefox_do_shutdown = False
self.firefox_already_started = False
def enable_headless(self, run_headless=True):
self.firefox_run_headless = run_headless
def enable_shutdown(self, do_shutdown=True):
self.firefox_do_shutdown = do_shutdown
def boot_firefox(self, headless=True):
# Indicate whether to run in headless mode
self.enable_headless(headless)
# Optionally shut down Marionette/Firefox after performing work
# This will just be called if Python exits normally
atexit.register(self.shutdown)
# Check whether Firefox is already running
logger.info(
'Check for running instance of Marionette/Firefox at {}:{}'.format(
self.firefox_host, self.firefox_port))
if check_socket(self.firefox_host, self.firefox_port):
logger.info('Will reuse running Marionette/Firefox')
self.firefox_bin = None
self.firefox_already_started = True
else:
logger.info('Will launch new Marionette/Firefox instance')
# Connect to / start Marionette Gecko engine
self.marionette = Marionette(host=self.firefox_host,
port=self.firefox_port,
bin=self.firefox_bin,
socket_timeout=self.socket_timeout,
startup_timeout=self.startup_timeout,
headless=self.firefox_run_headless,
verbose=self.firefox_verbosity)
self.marionette.DEFAULT_SHUTDOWN_TIMEOUT = self.shutdown_timeout
# Start a session with Marionette Gecko engine
self.marionette.start_session()
# Configure Marionette
self.configure_marionette()
def configure_marionette(self):
# This specifies the time to wait for the page loading to complete.
self.marionette.timeout.page_load = self.page_timeout
# This specifies the time to wait for injected scripts to finish
# before interrupting them.
self.marionette.timeout.script = self.script_timeout
# Configure a HTTP proxy server
self.marionette.set_pref('network.proxy.type', 0, default_branch=True)
@classmethod
def find_firefox(cls):
candidates = where.where('firefox')
candidates += [
'/Applications/Firefox.app/Contents/MacOS/firefox-bin',
]
firefox = find_program_candidate(candidates)
logger.info('Found "firefox" program at {}'.format(firefox))
return firefox
def get_status(self):
attributes = ['session', 'session_id']
data = OrderedDict()
for attribute in attributes:
data[attribute] = getattr(self.marionette, attribute)
return data
def log_status(self):
logger.info('Marionette report: {}'.format(
json.dumps(self.get_status(), indent=4)))
def has_active_session(self):
is_initialized = self.marionette is not None and self.marionette.session_id is not None
return is_initialized
def ensure_session(self):
#self.log_status()
if not self.has_active_session():
self.boot_firefox()
logger.info(
'No session with Marionette, started new session {}'.format(
self.marionette.session_id))
def shutdown(self):
if self.firefox_do_shutdown:
logger.info('Aiming at shutdown')
if self.firefox_already_started:
logger.warning(
'Can not shutdown Firefox as it was already running before starting this program'
)
return False
logger.info('Shutting down Marionette/Firefox')
if self.marionette is not None:
self.marionette.quit()
return True
def find_tag(self, tagname):
try:
element = self.marionette.find_element("tag name", tagname)
return element
except NoSuchElementException:
pass
def wait_for_element_tag(self, tagname):
"""
Wait for element to appear.
"""
waiter = Wait(self.marionette, timeout=20.0, interval=0.1)
element = waiter.until(lambda m: self.find_tag(tagname))
return element
def render_image(self, element=None):
"""
Return screenshot from element.
"""
image = self.marionette.screenshot(element=element, format='binary')
return image
def set_window_size(self, width, height):
self.marionette.set_window_rect(width=width, height=height)
def get_window_rect(self):
return self.marionette.window_rect
parser = argparse.ArgumentParser(description='Bulk Reject translations on GlotPress with Firefox')
parser.add_argument('--search', dest='search', help='The term with problems', required=True, type=str)
parser.add_argument('--remove', dest='remove', help='The wrong translation to remove', required=True, type=str)
parser.add_argument('--replace', dest='replace', help='The new translation to submit', required=False, type=str)
parser.add_argument('--lang', dest='lang', help='The locale, eg: it', default="it")
args = parser.parse_args()
# Load configuration
config = ConfigParser.RawConfigParser()
config.readfp(open('config.ini'))
print "Connection in progress to Firefox"
client = Marionette(host='127.0.0.1', port=28288)
client.start_session()
print "Connection to Firefox Done"
# Detect if already logged
try:
client.find_element(By.CSS_SELECTOR, 'body.logged-in')
client.navigate("https://translate.wordpress.org/wp-login.php?action=logout&redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F&_wpnonce=583839252e")
except:
pass
# Login
client.navigate("https://login.wordpress.org/?redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F")
try:
#Log In Form
usernameLogin = client.find_element(By.ID, 'user_login')
usernameLogin.click()
usernameLogin.send_keys(config.get('Login', 'user'))
passwordLogin = client.find_element(By.ID, 'user_pass')
passwordLogin.click()
passwordLogin.send_keys(config.get('Login', 'pass'))
# Click on the Log in button to connect
client.find_element(By.ID, 'wp-submit').click()
parser = argparse.ArgumentParser(description='Bulk Reject translations on GlotPress with Firefox')
parser.add_argument('--search', dest='search', help='The term with problems', required=True, type=str)
parser.add_argument('--remove', dest='remove', help='The wrong translation to remove', required=True, type=str)
parser.add_argument('--replace', dest='replace', help='The new translation to submit', required=False, type=str)
parser.add_argument('--lang', dest='lang', help='The locale, eg: it', default="it")
args = parser.parse_args()
# Load configuration
config = ConfigParser.RawConfigParser()
config.readfp(open('config.ini'))
print "Connection in progress to Firefox"
client = Marionette(host='127.0.0.1', port=28288)
client.start_session()
print "Connection to Firefox Done"
# Detect if already logged
try:
client.find_element(By.CSS_SELECTOR, 'body.logged-in')
client.navigate("https://translate.wordpress.org/wp-login.php?action=logout&redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F&_wpnonce=583839252e")
except:
pass
# Login
client.navigate("https://login.wordpress.org/?redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F")
try:
#Log In Form
usernameLogin = client.find_element(By.ID, 'user_login')
usernameLogin.click()
usernameLogin.send_keys(config.get('Login', 'user'))
passwordLogin = client.find_element(By.ID, 'user_pass')
passwordLogin.click()
passwordLogin.send_keys(config.get('Login', 'pass'))
# Click on the Log in button to connect
client.find_element(By.ID, 'wp-submit').click()