def testSimpleRequirements(self):
assert (
list(parse_requirements('Twis-Ted>=1.2-1'))
==
[Requirement('Twis-Ted>=1.2-1')]
)
assert (
list(parse_requirements('Twisted >=1.2, \\ # more\n<2.0'))
==
[Requirement('Twisted>=1.2,<2.0')]
)
assert (
Requirement.parse("FooBar==1.99a3")
==
Requirement("FooBar==1.99a3")
)
with pytest.raises(ValueError):
Requirement.parse(">=2.3")
with pytest.raises(ValueError):
Requirement.parse("x\\")
with pytest.raises(ValueError):
Requirement.parse("x==2 q")
with pytest.raises(ValueError):
Requirement.parse("X==1\nY==2")
with pytest.raises(ValueError):
Requirement.parse("#")
def testOrdering(self):
r1 = Requirement("Twisted==1.2c1,>=1.2")
r2 = Requirement("Twisted>=1.2,==1.2c1")
assert r1 == r2
assert str(r1) == str(r2)
assert str(r2) == "Twisted==1.2c1,>=1.2"
assert (Requirement("Twisted") !=
Requirement("Twisted @ https://localhost/twisted.zip"))
def config_section_data():
"""sample config data for use in app.config"""
section_config_fn = resource_filename(Requirement("rc-query-rest"), "query_runner/data/app.config.rest")
query_dir = resource_filename(Requirement("rc-query-rest"), "query_runner/data/queries_rest")
with open(section_config_fn, 'r') as section_config_file:
section_config = Template(section_config_file.read())
return section_config.safe_substitute(directory=query_dir)
def config_section_data():
"""sample config data for use in app.config"""
section_config_fn = resource_filename(Requirement("{{ cookiecutter.package_name }}"), "{{ cookiecutter.module_name }}/data/app.config.{{ cookiecutter.package_name }}")
query_dir = resource_filename(Requirement("{{ cookiecutter.package_name }}"), "{{ cookiecutter.module_name }}/data/queries_data.json")
with open(section_config_fn, 'r') as section_config_file:
section_config = Template(section_config_file.read())
return section_config.safe_substitute(directory=query_dir)
def testBasics(self):
r = Requirement.parse("Twisted>=1.2")
self.assertEqual(str(r),"Twisted>=1.2")
self.assertEqual(repr(r),"Requirement.parse('Twisted>=1.2')")
self.assertEqual(r, Requirement("Twisted", [('>=','1.2')], ()))
self.assertEqual(r, Requirement("twisTed", [('>=','1.2')], ()))
self.assertNotEqual(r, Requirement("Twisted", [('>=','2.0')], ()))
self.assertNotEqual(r, Requirement("Zope", [('>=','1.2')], ()))
self.assertNotEqual(r, Requirement("Zope", [('>=','3.0')], ()))
self.assertNotEqual(r, Requirement.parse("Twisted[extras]>=1.2"))
def testBasics(self):
r = Requirement.parse("Twisted>=1.2")
assert str(r) == "Twisted>=1.2"
assert repr(r) == "Requirement.parse('Twisted>=1.2')"
assert r == Requirement("Twisted", [('>=', '1.2')], ())
assert r == Requirement("twisTed", [('>=', '1.2')], ())
assert r != Requirement("Twisted", [('>=', '2.0')], ())
assert r != Requirement("Zope", [('>=', '1.2')], ())
assert r != Requirement("Zope", [('>=', '3.0')], ())
assert r != Requirement.parse("Twisted[extras]>=1.2")
def testSimpleRequirements(self):
self.assertEqual(list(parse_requirements('Twis-Ted>=1.2-1')),
[Requirement('Twis-Ted', [('>=', '1.2-1')], ())])
self.assertEqual(
list(parse_requirements('Twisted >=1.2, \ # more\n<2.0')),
[Requirement('Twisted', [('>=', '1.2'), ('<', '2.0')], ())])
self.assertEqual(Requirement.parse("FooBar==1.99a3"),
Requirement("FooBar", [('==', '1.99a3')], ()))
self.assertRaises(ValueError, Requirement.parse, ">=2.3")
self.assertRaises(ValueError, Requirement.parse, "x\\")
self.assertRaises(ValueError, Requirement.parse, "x==2 q")
self.assertRaises(ValueError, Requirement.parse, "X==1\nY==2")
self.assertRaises(ValueError, Requirement.parse, "#")
def parse_requirements(strs):
for req in split_requirements(strs):
try:
url = furl.furl(req)
except Exception:
pass
else:
if url.scheme:
req = url.fragment.args['egg'].split('==')[0]
yield Requirement('{}@{}'.format(req, url))
continue
yield Requirement(req)
def add_pip_package(self, pip_package: str):
# str must be a valid pip requirement specifier
# https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers
from pkg_resources import Requirement
package_req = Requirement(pip_package)
self._add_pip_package_requirement(package_req)
def resolve_url_list(settings, package):
"""Build a url list for packages matching the specifications."""
requires = Requirement(package)
package_addr = _GENERIC_ADDRESS.format(package=requires.name)
package_request = requests.get(package_addr)
package_request.raise_for_status() # raise if response is 4xx/5xx
package_data = package_request.json()
all_versions = package_data["releases"].keys()
wanted_versions = [v for v in all_versions if (requires.__contains__(v))]
url_list = []
accepted_types = settings['packagetypes']
for version in wanted_versions:
packages_for_version = package_data['releases'].get(version)
for package in packages_for_version:
if (package_is_valid(settings, package)):
file_name = str(package.get('filename')) # file name
url = str(package.get('url')) # url
project_name = str(requires.name) # sub-folder in cache
url_list.append((project_name, file_name, url))
else:
continue
return url_list
def _add_pip_package_requirement(self, pkg_req: Requirement):
if pkg_req.name in self._pip_packages:
if (pkg_req.specs and
pkg_req.specs != self._pip_packages[pkg_req.name].specs):
logger.warning(
f"Overwriting existing pip package requirement "
f"'{self._pip_packages[pkg_req.name]}' to '{pkg_req}'")
else:
logger.warning(
f"pip package requirement {pkg_req} already exist")
return
verification_result = verify_pkg(pkg_req)
if verification_result == EPP_PKG_NOT_EXIST:
logger.warning(
f"pip package requirement `{str(pkg_req)}` not found in current "
f"python environment")
elif verification_result == EPP_PKG_VERSION_MISMATCH:
logger.warning(
f"pip package requirement `{str(pkg_req)}` does not match the "
f"version installed in current python environment")
if verification_result == EPP_NO_ERROR and not pkg_req.specs:
# pin the current version when there's no version spec found
pkg_version = get_pkg_version(pkg_req.name)
if pkg_version:
pkg_req = Requirement(f"{pkg_req.name}=={pkg_version}")
self._pip_packages[pkg_req.name] = pkg_req
def load_template(filename, override_template_file=None):
""" Load contents of template file return as json.
:param filename: Template file.
:param override_template_file: Options user defined template path.
:return : Return contents of file as json.
"""
file_contents = None
template_file = None
override_file_exists = False
if override_template_file:
# Check user defined template file exists.
if os.path.exists(override_template_file):
override_file_exists = True
template_file = override_template_file
LOG.debug("Using user defined template file %s",
override_template_file)
if not override_file_exists:
# Use the default template file.
template_file = resource_filename(Requirement(const.PACKAGE),
const.PATH_TO_TEMPLATES + filename)
if not os.path.exists(template_file):
raise Exception(
"Template file '{}' not found".format(template_file))
LOG.debug("Using package defined template file %s", template_file)
LOG.debug("Attempting to use template file %s", template_file)
with io.open(template_file, mode="rt", encoding="utf-8") as load_file:
file_contents = json.load(load_file)
return file_contents
def get_dependencies(self):
for req in self.metadata.get_all("Requires-Dist", []):
r = Requirement(req)
if not r.marker:
yield r
elif r.marker.evaluate(
{"extra": None}): # We don't consider markers for this demo
yield r
def _close_incident(self, incident, ticket):
"""
Close a Resilient incident by rendering a jinja2 template
:param ticket: Secureworks CTP ticket (json object)
:return: Resilient incident
"""
try:
# Close a Resilient incident from this ticket
# using a JSON (JINJA2) template file
template_file_path = self.options.get('template_file_close')
if template_file_path and not os.path.exists(template_file_path):
LOG.warning(u"Template file '%s' not found.",
template_file_path)
template_file_path = None
if not template_file_path:
# Use the template file installed by this package
template_file_path = resource_filename(
Requirement("fn-secureworks-ctp"),
"fn_secureworks_ctp/data/scwx_ctp_template_close.jinja")
if not os.path.exists(template_file_path):
raise Exception(
u"Template file for close'{0}' not found".format(
template_file_path))
LOG.info(
u"Secureworks CTP jinja template file for closing incident: %s",
template_file_path)
with open(template_file_path, "r") as definition:
close_template = definition.read()
incident_payload = render_json(close_template, ticket)
# Set the scwx_ctp_status incident field to the ticket status (Closed or Resolved) so that the
# automatic rule to close the Securework ticket is not triggered as the ticket is already closed in SCWX.
incident_payload['properties']['scwx_ctp_status'] = ticket.get(
"status")
# Render the template.
incident_id = incident.get('id')
result = self._update_incident(incident_id, incident_payload)
LOG.debug(incident_payload)
ticket_id = ticket.get('ticketId')
if result and result.get('success'):
message = u"Closed incident {0} for Secureworks CTP ticket {1}".format(
incident_id, ticket_id)
LOG.info(message)
else:
message = u"Unable to update incident {0} for closing. Secureworks CTP ticket {1}".format(
incident_id, ticket_id)
LOG.error(message)
return result
except Exception as err:
raise IntegrationError(err)
def _process_approval_request(self, event):
# Process one approval request
log = self.log
request = event.request
request_id = request["id"]
# special "test the process by escalating a single request" mode
test_single_request = self.options.get("test_single_request")
if test_single_request:
if str(request_id) not in str(test_single_request).split(","):
log.info(u"Skipping request %s, test", request_id)
return
# Find the Resilient incident corresponding to this CbProtect approval request (if available)
resilient_incident = self._find_resilient_incident_for_req(request_id)
if resilient_incident:
log.info(u"Skipping request %s, already escalated", request_id)
return
log.info(u"Processing request %s", request_id)
try:
# Create a new Resilient incident from this approval request
# using a JSON (JINJA2) template file
template_file_path = self.options.get("template_file")
if template_file_path and not os.path.exists(template_file_path):
log.warn(u"Template file '%s' not found.", template_file_path)
template_file_path = None
if not template_file_path:
# Use the template file installed by this package
template_file_path = resource_filename(
Requirement("fn-cb-protection"),
"fn_cb_protection/data/template.jinja")
if not os.path.exists(template_file_path):
raise Exception(u"Template file '{}' not found".format(
template_file_path))
log.info(u"Template file: %s", template_file_path)
with open(template_file_path, "r") as definition:
escalate_template = definition.read()
# Render the template. Be sure to set the CbProtect ID in the result!
new_resilient_inc = render_json(escalate_template, request)
new_resilient_inc["properties"][REQUEST_ID_FIELDNAME] = request_id
log.debug(new_resilient_inc)
inc = self.rest_client().post("/incidents", new_resilient_inc)
rs_inc_id = inc["id"]
message = u"Created incident {} for CbProtect {}".format(
rs_inc_id, request_id)
log.info(message)
except Exception as exc:
log.exception(exc)
raise
def update_req(req):
info = get_package_info(req.project_name)
newest_version = info['version']
current_spec = req.specs[0] if req.specs else ('==', 'unspecified')
new_spec = ('==', newest_version)
if current_spec != new_spec:
newreq = Requirement(req.unsafe_name, [new_spec], req.extras)
print('Updated {} from {} -> {}'.format(req.project_name,
current_spec[1],
newest_version))
return newreq
return req
def testBasicContains(self):
r = Requirement("Twisted", [('>=','1.2')], ())
foo_dist = Distribution.from_filename("FooPkg-1.3_1.egg")
twist11 = Distribution.from_filename("Twisted-1.1.egg")
twist12 = Distribution.from_filename("Twisted-1.2.egg")
self.assertTrue(parse_version('1.2') in r)
self.assertTrue(parse_version('1.1') not in r)
self.assertTrue('1.2' in r)
self.assertTrue('1.1' not in r)
self.assertTrue(foo_dist not in r)
self.assertTrue(twist11 not in r)
self.assertTrue(twist12 in r)
def testBasicContains(self):
r = Requirement("Twisted>=1.2")
foo_dist = Distribution.from_filename("FooPkg-1.3_1.egg")
twist11 = Distribution.from_filename("Twisted-1.1.egg")
twist12 = Distribution.from_filename("Twisted-1.2.egg")
assert parse_version('1.2') in r
assert parse_version('1.1') not in r
assert '1.2' in r
assert '1.1' not in r
assert foo_dist not in r
assert twist11 not in r
assert twist12 in r
def get_audit_policy():
"""read the"""
if getattr(sys, 'frozen', False):
path = os.path.join(
sys._MEIPASS, # pylint: disable=no-member, protected-access
'provision/userdata/manifests/audit-policy.yml')
else:
path = resource_filename(
Requirement('koris'),
os.path.join(BOOTSTRAP_SCRIPTS_DIR, 'manifests',
'audit-policy.yml'))
with open(path) as policy:
return policy.read()
def _load_installed_versions(pip):
pip_freeze = check_output([pip, 'freeze']).decode('utf8')
versions = {}
for pkg_ver in pip_freeze.splitlines():
try:
req = Requirement(pkg_ver)
except ValueError:
continue
versions.update({
req.name: LooseVersion(_.version)
for _ in req.specifier if _.operator == '=='
})
return versions
def _update_custom_fields(self, incident, ticket):
"""
Update a Resilient incident by rendering a jinja2 template
:param ticket: Secureworks CTP ticket (json object)
:return: Resilient incident
"""
try:
# Update Resilient custom incident fields from this ticket
# using a JSON (JINJA2) template file
template_file_path = self.options.get('template_file_update')
if template_file_path and not os.path.exists(template_file_path):
LOG.warning(u"Template file '%s' not found.",
template_file_path)
template_file_path = None
if not template_file_path:
# Use the template file installed by this package
template_file_path = resource_filename(
Requirement("fn-secureworks-ctp"),
"fn_secureworks_ctp/data/scwx_ctp_template_update.jinja")
if not os.path.exists(template_file_path):
raise Exception(
u"Template file for updating incident'{0}' not found".
format(template_file_path))
LOG.info(
u"Secureworks CTP jinja template file for updating incident: %s",
template_file_path)
with open(template_file_path, "r") as definition:
update_template = definition.read()
incident_payload = render_json(update_template, ticket)
# Render the template.
incident_id = incident.get('id')
result = self._update_incident(incident_id, incident_payload)
LOG.debug(incident_payload)
ticket_id = ticket.get('ticketId')
if result and result.get('success'):
message = u"Updated incident {0} for Secureworks CTP ticket {1}".format(
incident_id, ticket_id)
LOG.info(message)
else:
message = u"Unable to update incident {0} for Secureworks CTP ticket {1}".format(
incident_id, ticket_id)
LOG.error(message)
return result
except Exception as err:
raise IntegrationError(err)
def _get_requirements(file_path):
with open(file_path, 'r') as file:
for line in file:
line = _COMMENT_RE.sub('', line)
line = line.strip()
if line.startswith('-r '):
yield from _get_requirements(
join(dirname(abspath(file_path)), line[3:]))
elif line:
req = Requirement(line)
req_str = f'{req.name}{req.specifier}'
if req.marker:
req_str += f'; {req.marker}'
yield req_str
def open_resource(module_name, fname):
"""
Open a resource file
This lookup for 'fname' in list of resources and open it if it exists.
In the case where the resource doesn't exist, this raises OSError.
:param fname: Name of the file to open. It could be a pattern.
:return: A file object for the first resource matching with fname
"""
pkg = Requirement(module_name.split('.')[0])
files = get_resource_list(module_name, '/', '.*' + fname)
if not files:
raise OSError
return resource_stream(pkg, files[0])
def main(requirements):
reqs = [Requirement(r) for r in requirements]
reporter = BaseReporter()
provider = Provider()
resolver = Resolver(provider, reporter)
state = resolver.resolve(reqs)
target_path = Path("__pypackages__") / PYTHON_VERSION[:3] / "lib"
print("resolution result:")
for k, c in state.mapping.items():
print(c.name, c.version)
for c in state.mapping.values():
c.install(target_path)
def process(requirement_sets, extras, environment):
for requirements in requirement_sets:
if 'extra' in requirements:
if safe_extra(requirements['extra']) not in extras:
continue
if 'environment' in requirements:
marker = Marker(requirements['environment'])
if not marker.evaluate(environment):
continue
for req in requirements['requires']:
req = utils.parse_requirement(req)
req.extras = extras
yield Requirement(str(req))
def testBasics(self):
r = Requirement.parse("Twisted>=1.2")
assert str(r) == "Twisted>=1.2"
assert repr(r) == "Requirement.parse('Twisted>=1.2')"
assert r == Requirement("Twisted>=1.2")
assert r == Requirement("twisTed>=1.2")
assert r != Requirement("Twisted>=2.0")
assert r != Requirement("Zope>=1.2")
assert r != Requirement("Zope>=3.0")
assert r != Requirement("Twisted[extras]>=1.2")
def version_information(sdk_develop=False):
"""Return an HTML table with the contents of requirements.txt"""
def escaped_operator(operator):
"""Return the HTML-escaped operator."""
if operator == '==':
return ''
else:
return escape(operator)
def requirement_as_row(requirement: Requirement):
"""Return a Requirement as an HTML table row."""
tds = [requirement.name, '']
tds[1] = ', '.join([
'{} {}'.format(escaped_operator(operator), target)
for operator, target in sorted(requirement.specs, reverse=True)
])
# If the version of QISKIT is not specified, show development branch.
if sdk_develop and requirement.name == 'QISKit':
tds[1] = '(git master branch)'
return '<tr>{}</tr>'.format(''.join(
['<td>{}</td>'.format(td) for td in tds]))
filename = 'requirements.txt'
# If the notebook is run from another one, the path will be the parent's.
if not path.exists(filename):
filename = '../../requirements.txt'
with open(filename, 'r') as requirements_file:
str_requirements = requirements_file.readlines()
requirements = [Requirement(i) for i in str_requirements]
qiskit_branch = '<b>stable</b>'
if sdk_develop:
qiskit_branch = '<b>development</b>'
output = [
'<h2>Version information</h2>',
'<p>Please note that this tutorial is targeted to the %s '
'version of the QISKit. The following versions of the '
'packages are recommended:</p>' % qiskit_branch, '<table>',
'<tr><th>Package</th><th colspan="2">Version</th></tr>'
]
output.extend([requirement_as_row(req) for req in requirements])
output.extend(['</table>'])
return display(HTML('\n'.join(output)))
def _get_bootstrap_script(self):
name = "bootstrap-k8s-%s-%s-%s.sh" % (self.role, self.os_type,
self.os_version)
if getattr(sys, 'frozen', False):
path = os.path.join(
sys._MEIPASS, # pylint: disable=no-member, protected-access
'provision/userdata',
name)
else:
path = resource_filename(Requirement('koris'),
os.path.join(BOOTSTRAP_SCRIPTS_DIR, name))
with open(path) as fh:
script = fh.read()
return name, script
def _sanitize_requirements(requirements):
"""Remove duplicate keys such as setuptools or pex which may be injected multiple times into the
resulting ipex when first executed."""
project_names = []
new_requirements = {}
for r in requirements:
r = Requirement(r)
if r.marker and not r.marker.evaluate():
continue
if r.name not in new_requirements:
project_names.append(r.name)
new_requirements[r.name] = str(r)
sanitized_requirements = [new_requirements[n] for n in project_names]
return sanitized_requirements
def process_body(self, body):
urls = []
reqs = body.decode('utf-8').splitlines()
for req in utils.split_requirements(reqs):
try:
req = Requirement(req)
except RequirementParseError:
if req.startswith('https://') or req.startswith('http://'):
urls.append(self._resolve_url(req))
else:
urls.append(req)
else:
urls.append(self._resolve_package(req))
return HttpResponse(u'\n'.join(urls) + u'\n',
content_type='text/plain')
