def get(self):
"""
Given a long URL, returns a short URL.
"""
long_url = self.get_argument('longUrl', None) # decoded by Tornado.
android_url = self.get_argument('androidUrl', None) # decoded by Tornado.
android_fallback_url = self.get_argument('androidFallbackUrl', None) # decoded by Tornado.
ios_url = self.get_argument('iosUrl', None) # decoded by Tornado.
ios_fallback_url = self.get_argument('iosFallbackUrl', None) # decoded by Tornado.
domain = self.get_argument('domain', self.settings['default_domain'])
# Normalize and validate long_url.
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) is True
if android_url:
# TODO: Validate and normalize!
pass
# android_url = utils.normalize_url(android_url)
# assert utils.validate_url(android_url) is True
if android_fallback_url:
android_fallback_url = utils.normalize_url(android_fallback_url)
assert utils.validate_url(android_fallback_url) is True
if ios_url:
# TODO: Validate and normalize!
pass
# ios_url = utils.normalize_url(ios_url)
# assert utils.validate_url(ios_url) is True
if ios_fallback_url:
ios_fallback_url = utils.normalize_url(ios_fallback_url)
assert utils.validate_url(ios_fallback_url) is True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []})
# Validate domain.
if not utils.validate_url('http://' + domain):
return self.finish(
{'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': []})
# Generate a unique hash, assemble short url and store result in Redis.
url_hash = utils.generate_hash(self.application.redis,
self.settings['redis_namespace'],
self.settings['hash_salt'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url, android_url,
android_fallback_url, ios_url, ios_fallback_url)
# Return success response.
data = {
'long_url': long_url,
'android_url': android_url,
'android_fallback_url': android_fallback_url,
'ios_url': ios_url,
'ios_fallback_url': ios_fallback_url,
'url': short_url,
'hash': url_hash,
'global_hash': url_hash
}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def update_artifactory_config(host_url, auth, config_dict):
""" take a configuraiton dict and upload it to artifactory
Parameters
----------
host_url : string
A url of the form http(s)://domainname:port[/context] or http(s)://ip:port[/context]
auth : tuple
A tuple of (user, password), as used by requests
config_dict : OrderedDict
a dict representation that will be returned to xml
Returns:
--------
success : boolean
true if we succeeded
"""
headers = {'Content-type': 'application/xml'}
config_url = "{}/artifactory/api/system/configuration".format(
normalize_url(host_url)
)
xml_config = xmltodict.unparse(config_dict)
r = requests.post(config_url, auth=auth, headers=headers, data=xml_config)
if r.ok:
return True
else:
return False
def _parse_title_url(html_chunk):
"""
Parse title/name of the book and URL of the book.
Args:
html_chunk (obj): HTMLElement containing slice of the page with details.
Returns:
tuple: (title, url), both as strings.
"""
url = None
title_tags = html_chunk.match(
["div", {"class": "polozka_nazev"}],
["a", None, has_param("href")]
)
if not title_tags:
return _parse_alt_title(html_chunk), _parse_alt_url(html_chunk)
title = title_tags[0]
url = normalize_url(BASE_URL, title.params["href"])
title = title.getContent()
if not title:
title = _parse_alt_title(html_chunk)
return title, url
def _parse_authors(html_chunk):
"""
Parse authors of the book.
Args:
html_chunk (obj): HTMLElement containing slice of the page with details.
Returns:
list: List of :class:`structures.Author` objects. Blank if no author \
found.
"""
authors_tags = html_chunk.match(
["div", {"class": "polozka_autor"}],
"a"
)
authors = []
for author_tag in authors_tags:
# get name
name = author_tag.getContent().strip()
# skip tags without name
if not name:
continue
# get url - if not found, set it to None
url = author_tag.params.get("href", None)
if url:
url = normalize_url(BASE_URL, url)
authors.append(
Author(name, url)
)
return authors
def get(self):
"""
Given a long URL, returns a short URL.
"""
long_url = self.get_argument('longUrl', None) # Is decoded by Tornado.
domain = self.get_argument('domain', self.settings['default_domain'])
# Normalize and validate long_url.
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) == True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []})
# Validate domain.
if not utils.validate_url('http://' + domain):
return self.finish({'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': []})
# Generate a unique hash, assemble short url and store result in Redis.
url_hash = utils.get_hash_from_map(long_url)
#check exist hash if do some customize
is_exist = self.load_url(url_hash)
if is_exist:
url_hash = utils.generate_hash(self.application.redis,
self.settings['redis_namespace'],
self.settings['hash_salt'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url)
# Return success response.
data = {'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def _parse_title_url(html_chunk):
"""
Parse title/name of the book and URL of the book.
Args:
html_chunk (obj): HTMLElement containing slice of the page with details.
Returns:
tuple: (title, url), both as strings.
"""
title = html_chunk.find("div", {"class": "comment"})
if not title:
return _parse_alt_title(html_chunk), None
title = title[0].find("h2")
if not title:
return _parse_alt_title(html_chunk), None
# look for the url of the book if present
url = None
url_tag = title[0].find("a")
if url_tag:
url = url_tag[0].params.get("href", None)
title = url_tag
return title[0].getContent(), normalize_url(BASE_URL, url)
def _parse_authors(html_chunk):
"""
Parse authors of the book.
Args:
html_chunk (obj): HTMLElement containing slice of the page with details.
Returns:
list: List of :class:`structures.Author` objects. Blank if no author \
found.
"""
authors = html_chunk.match(
["div", {"class": "comment"}],
"h3",
"a",
)
if not authors:
return []
authors = map(
lambda x: Author( # create Author objects
x.getContent().strip(),
normalize_url(BASE_URL, x.params.get("href", None))
),
authors
)
return filter(lambda x: x.name.strip(), authors)
def get(self):
long_url = self.get_argument('longUrl', None)
url_hash = self.get_argument('hash', None)
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) == True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({
'status_code': 500,
'status_txt': 'INVALID_URI',
'data': []
})
domain = self.get_argument('domain', self.settings['default_domain'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url)
print 'xxxxx'
data = {
'long_url': long_url,
'url': short_url,
'hash': url_hash,
'global_hash': url_hash
}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def verify_distant():
body_dict = json.load(request.body)
url, cert_fingerprint = body_dict['url'], body_dict['cert_fingerprint']
url = utils.normalize_url(url)
cert_fingerprint = utils.normalize_cert_fingerprint(cert_fingerprint)
is_valid = dv.verify(url, cert_fingerprint)
response.status = 200
return str(json.dumps({'is_valid': is_valid}))
def get_avatarurl(self):
if self.avatar:
url = self.avatar.url
elif self.gender == 'M':
url = '/images/user_male.png'
elif self.gender == 'F':
url = '/images/user_female.png'
else:
url = '/images/user_unisex.png'
return normalize_url(url)
def cr_repository(host_url, repo_dict, auth=None, session=None):
""" take a configuration dict and post it host_url
Should use
https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON
for the inputs.
Does not error checking; will fail if the json is malformed.
Parameters
----------
host_url : string
A url of the form
http(s)://domainname:port[/context] or http(s)://ip:port[/context]
repo_dict : OrderedDict
a dictionary of the inputs required by artifactroy. see above.
auth : tuple, optional
A tuple of (user, password), as used by requests
session : requests Session object, optional
A session object (that has any necessary cookies / headers defined)
Either auth or session must be defined. Session overrides auth.
Returns
-------
success : boolean
true if succeeded
"""
ses = _get_artifactory_session(auth, session)
if 'key' not in repo_dict:
raise InvalidAPICallError("The repo_dict must include a repo key (repo_dict['key'])")
repo_url = '{}/artifactory/api/repositories/{}'.format(
normalize_url(host_url),
repo_dict['key']
)
headers = {'Content-type': 'application/json'}
exists_resp = ses.get(repo_url)
if exists_resp.ok:
resp = ses.post(repo_url, json=repo_dict, headers=headers)
else:
resp = ses.put(repo_url, json=repo_dict, headers=headers)
# YELLOW need to add more logic to make this aware of if the configuration
# is changing
if resp.ok:
return True
else:
return False
def verify_central():
body_dict = json.load(request.body)
url, cert_fingerprint = body_dict['url'], body_dict['cert_fingerprint']
url = utils.normalize_url(url)
cert_fingerprint = utils.normalize_cert_fingerprint(cert_fingerprint)
try:
is_valid = cv.verify(url, cert_fingerprint)
except Exception as e:
return str(json.dumps({'Result': 'Internal error \n {}'.format(e)}))
response.status = 200
if is_valid:
return str(json.dumps({'Result': 'OK'}))
else:
return str(json.dumps({'Result': 'Cert Mismatch'}))
def get(self):
"""
Given a long URL, returns a short URL.
"""
long_url = self.get_argument('longUrl', None) # Is decoded by Tornado.
domain = self.get_argument('domain', self.settings['default_domain'])
# Normalize and validate long_url.
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) == True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({
'status_code': 500,
'status_txt': 'INVALID_URI',
'data': []
})
# Validate domain.
if not utils.validate_url('http://' + domain):
return self.finish({
'status_code': 500,
'status_txt': 'INVALID_ARG_DOMAIN',
'data': []
})
# Generate a unique hash, assemble short url and store result in Redis.
url_hash = utils.get_hash_from_map(long_url)
#check exist hash if do some customize
is_exist = self.load_url(url_hash)
if is_exist:
url_hash = utils.generate_hash(self.application.redis,
self.settings['redis_namespace'],
self.settings['hash_salt'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url)
# Return success response.
data = {
'long_url': long_url,
'url': short_url,
'hash': url_hash,
'global_hash': url_hash
}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def get_repo_configs(host_url, repo_list, username=None, passwd=None,
auth=None, session=None):
""" return repository configuration dictionaries for specified set of repos
Parameters
----------
host_url : string
An artifactory url of the form
http(s)://domainname:port[/context] or http(s)://ip:port[/context]
repo_list : list of strings
A list of repo keys that you want to get configs for. repo
keys should match the url in the artifactory rest call
username : string, optional
username to create auth tuple from
passwd : string, optional
password for auth tuple
auth : tuple, optional
A tuple of (user, password), as used by requests
session : requests.Session
A requests.Session object, with auth
Either session, auth, or user/pass must be defined.
Session overrides auth overides username/password
See _get_artifactory_session for details
"""
ses = _get_artifactory_session(
username=username,
passwd=passwd,
auth=auth,
session=session
)
repo_configs_list = []
for repo in repo_list:
repo_url = '{}/artifactory/api/repositories/{}'.format(
normalize_url(host_url),
repo
)
resp = ses.get(repo_url)
if not resp.ok:
msg = "Failed to fetch config for {}".format(repo)
raise UnknownArtifactoryRestError(msg, resp)
repo_dict = resp.json()
repo_configs_list.append(repo_dict)
return repo_configs_list
def get(self):
long_url = self.get_argument('longUrl', None)
url_hash = self.get_argument('hash', None)
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) == True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []})
domain = self.get_argument('domain', self.settings['default_domain'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url)
print 'xxxxx'
data = {'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def _parse_alt_url(html_chunk):
"""
Parse URL from alternative location if not found where it should be.
Args:
html_chunk (obj): HTMLElement containing slice of the page with details.
Returns:
str: Book's URL.
"""
url_list = html_chunk.find("a", fn=has_param("href"))
url_list = map(lambda x: x.params["href"], url_list)
url_list = filter(lambda x: not x.startswith("autori/"), url_list)
if not url_list:
return None
return normalize_url(BASE_URL, url_list[0])
def get_artifactory_config_from_url(host_url, auth):
"""retrieve the artifactory configuration xml doc
Parameters
----------
host_url: string
A url of the form http(s)://domainname:port/context or
http(s)://ip:port/context
auth: tuple
a tuple a la requests auth of the form (user, password)
"""
headers = {'Accept': 'application/xml'}
config_url = "{}/artifactory/api/system/configuration".format(
normalize_url(host_url)
)
r = requests.get(config_url, auth=auth, headers=headers)
if r.ok:
return(xmltodict.parse(r.text))
else:
raise ConfigFetchError("Something went wrong getting the config", r)
def get(self):
"""
Given a long URL, returns a short URL.
"""
long_url = self.get_argument('longUrl', None) # decoded by Tornado.
android_url = self.get_argument('androidUrl',
None) # decoded by Tornado.
android_fallback_url = self.get_argument('androidFallbackUrl',
None) # decoded by Tornado.
ios_url = self.get_argument('iosUrl', None) # decoded by Tornado.
ios_fallback_url = self.get_argument('iosFallbackUrl',
None) # decoded by Tornado.
domain = self.get_argument('domain', self.settings['default_domain'])
# Normalize and validate long_url.
try:
long_url = utils.normalize_url(long_url)
assert utils.validate_url(long_url) is True
if android_url:
# TODO: Validate and normalize!
pass
# android_url = utils.normalize_url(android_url)
# assert utils.validate_url(android_url) is True
if android_fallback_url:
android_fallback_url = utils.normalize_url(
android_fallback_url)
assert utils.validate_url(android_fallback_url) is True
if ios_url:
# TODO: Validate and normalize!
pass
# ios_url = utils.normalize_url(ios_url)
# assert utils.validate_url(ios_url) is True
if ios_fallback_url:
ios_fallback_url = utils.normalize_url(ios_fallback_url)
assert utils.validate_url(ios_fallback_url) is True
except:
logging.info('Wrong URL', exc_info=1)
return self.finish({
'status_code': 500,
'status_txt': 'INVALID_URI',
'data': []
})
# Validate domain.
if not utils.validate_url('http://' + domain):
return self.finish({
'status_code': 500,
'status_txt': 'INVALID_ARG_DOMAIN',
'data': []
})
# Generate a unique hash, assemble short url and store result in Redis.
url_hash = utils.generate_hash(self.application.redis,
self.settings['redis_namespace'],
self.settings['hash_salt'])
short_url = 'http://' + domain + '/' + url_hash
self.store_url(url_hash, long_url, android_url, android_fallback_url,
ios_url, ios_fallback_url)
# Return success response.
data = {
'long_url': long_url,
'android_url': android_url,
'android_fallback_url': android_fallback_url,
'ios_url': ios_url,
'ios_fallback_url': ios_fallback_url,
'url': short_url,
'hash': url_hash,
'global_hash': url_hash
}
self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def update_wikidata(project: Project, edit_group_hash: str):
""" Update wikidata entry with data from github """
# Wikidata boilerplate
wikidata = Settings.wikidata_repo
q_value = project.project.replace("http://www.wikidata.org/entity/", "")
item = ItemPage(wikidata, title=q_value)
item.get()
url_raw = project.repo
url_normalized = str(normalize_url(url_raw))
if Settings.normalize_repo_url:
normalize_repo_url(item, url_normalized, url_raw, q_value,
edit_group_hash)
set_website(item, project, url_normalized, edit_group_hash)
set_license(item, project, url_normalized, edit_group_hash)
# Add all stable releases
stable_releases = project.stable_release
stable_releases.sort(
key=lambda x: LooseVersion(re.sub(r"[^0-9.]", "", x.version)))
if len(stable_releases) == 0:
logger.info("No stable releases")
return
versions = [i.version for i in stable_releases]
if len(versions) != len(set(versions)):
duplicates = [
release for release in stable_releases
if versions.count(release.version) > 1
]
logger.warning("There are duplicate releases in {}: {}".format(
q_value, duplicates))
return
latest_version: Optional[str] = stable_releases[-1].version
existing_versions = item.claims.get(Properties.software_version, [])
github_version_names = [i.version for i in stable_releases]
for i in existing_versions:
if i.getRank() == "preferred" and i.getTarget(
) not in github_version_names:
logger.warning(
"There's a preferred rank for {} for a version which is not in the github page: {}"
.format(q_value, i.getTarget()))
latest_version = None
if len(stable_releases) > 100:
logger.warning("Limiting {} to 100 of {} stable releases".format(
q_value, len(stable_releases)))
stable_releases = stable_releases[-100:]
else:
logger.info("There are {} stable releases".format(
len(stable_releases)))
for release in stable_releases:
claim, created = get_or_create_claim(item, Properties.software_version,
release.version, edit_group_hash)
if created:
logger.info("Added '{}'".format(release.version))
# Assumption: A preexisting publication date is more reliable than the one from github
date_p = Properties.publication_date
if date_p not in claim.qualifiers:
get_or_create_qualifiers(claim, date_p, release.date,
edit_group_hash)
title = "Release %s" % release.version
get_or_create_sources(claim, release.page, project.retrieved,
edit_group_hash, title, release.date)
# Give the latest release the preferred rank
# And work around a bug in pywikibot
try:
set_claim_rank(claim, latest_version, release, edit_group_hash)
except AssertionError:
logger.warning(
f"Using the fallback for setting the preferred rank of {q_value}"
)
item.get(force=True)
claim, created = get_or_create_claim(item,
Properties.software_version,
release.version,
edit_group_hash)
assert not created
set_claim_rank(claim, latest_version, release, edit_group_hash)
def test_repo_normalization():
url = "git://github.com/certbot/certbot.git"
actual = str(normalize_url(url))
expected = "https://github.com/certbot/certbot"
assert actual == expected
def update_password(host_url, username, orig_pass, target_pass):
""" set the password for the user to the target_pass
Parameters
----------
host_url : string
A url of the form http(s)://domainname:port/context or
http(s)://ip:port/context
username : string
username of the password to change
orig_pass : string
original password to use for the update
target_pass : string
the desired new password
Returns
-------
changed : boolean
True if changes were made
Raises
------
InvalidCredentialsError :
If neither the original or target credentials work to update the password
UnknownArtifactoryRestError :
If we get a response we haven't encountered and don't kow what to do with
"""
orig_auth = (username, orig_pass)
target_auth = (username, target_pass)
get_pass_url = '{}/artifactory/api/security/encryptedPassword'.format(
normalize_url(host_url)
)
orig_resp = requests.get(get_pass_url, auth=orig_auth)
if orig_resp.status_code == 401:
resp = requests.get(get_pass_url, auth=target_auth)
auth = target_auth
elif orig_resp.status_code == 200:
resp = orig_resp
auth = orig_auth
else:
raise UnknownArtifactoryRestError(
"Unexpected response when verifying credentials",
orig_resp
)
if resp.status_code != 200:
raise InvalidCrentialsError
if auth == target_auth:
return False
user_json_url = '{}/artifactory/api/security/users/{}'.format(
normalize_url(host_url),
username
)
headers = {'Content-type': 'application/json'}
user_dict_resp = requests.get(user_json_url, auth=auth)
if not user_dict_resp.ok:
if user_dict_resp.status == 401:
msg = "Received an unauthorized message after authorization "
msg += "has been checked. Wtf?"
raise UnknownArtifactoryRestError(msg, user_dict_resp)
else:
raise UnknownArtifactoryRestError(
"Couldn't get user information",
user_dict_resp
)
admin_dict = user_dict_resp.json()
admin_dict.pop('lastLoggedIn')
admin_dict.pop('realm')
admin_dict['password'] = target_pass
update_resp = requests.post(
user_json_url,
auth=auth,
json=admin_dict,
headers=headers
)
if not update_resp.ok:
if update_resp.status == 401:
msg = "Received an unauthorized message after authorization "
msg += "has been checked. Wtf?"
raise UnknownArtifactoryRestError(msg, update_resp)
else:
raise UnknownArtifactoryRestError(
"Couldn't post user password update",
update_resp
)
final_check_resp = requests.get(get_pass_url, auth=target_auth)
if not final_check_resp.ok:
raise UnknownArtifactoryRestError(
"Final password check failed. Could not use new credentials",
final_check_resp
)
else:
return True